From 26254246a3529d468e420c488a87760ffda10175 Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 4 Oct 2026 13:02:20 +0800 Subject: [PATCH 01/22] Fix E2B template chunk upload --- .github/workflows/e2b-template.yml | 86 ++++++++++++++++++++++++++++-- deploy/e2b/README.md | 8 ++- 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2b-template.yml b/.github/workflows/e2b-template.yml index 44fce7615..dbf15c0d2 100644 --- a/.github/workflows/e2b-template.yml +++ b/.github/workflows/e2b-template.yml @@ -14,6 +14,11 @@ on: type: string required: false + resume_build: + description: Existing unsubmitted templateID:build_UUID after an upload failure; requires ref + type: string + required: false + permissions: contents: read @@ -63,6 +68,7 @@ jobs: env: SOURCE_BRANCH: ${{ inputs.source_branch }} REQUESTED_REF: ${{ inputs.ref }} + RESUME_BUILD: ${{ inputs.resume_build }} run: | set -euo pipefail case "$SOURCE_BRANCH" in main|beta) ;; *) exit 1 ;; esac @@ -70,6 +76,10 @@ jobs: echo '::error::ref must be a full lowercase commit SHA.' exit 1 fi + if [ -n "$RESUME_BUILD" ]; then + [ -n "$REQUESTED_REF" ] || { echo '::error::Recovery requires the original source SHA.'; exit 1; } + [[ "$RESUME_BUILD" =~ ^[a-zA-Z0-9_-]+:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] || exit 1 + fi git fetch --no-tags origin "refs/heads/$SOURCE_BRANCH:refs/remotes/origin/$SOURCE_BRANCH" revision="$(git rev-parse "${REQUESTED_REF:-origin/$SOURCE_BRANCH}^{commit}")" git merge-base --is-ancestor "$revision" "origin/$SOURCE_BRANCH" || { @@ -147,6 +157,8 @@ jobs: printf '%s\n' "$image" > "$output_root/runtime-image.id" - name: Build one immutable E2B template env: + RESUME_BUILD: ${{ inputs.resume_build }} + SOURCE_COMMIT: ${{ steps.source.outputs.revision }} E2B_API_KEY: ${{ secrets.E2B_API_KEY }} E2B_API_URL: ${{ vars.E2B_API_URL }} run: | @@ -157,10 +169,75 @@ jobs: printf '%s' "$E2B_API_KEY" > "$key" unset E2B_API_KEY mkdir -p "$RUNNER_TEMP/e2b-result" - "$RUNNER_TEMP/e2b-sdk/bin/python" source/services/core/deploy/e2b/build-template.py \ + "$RUNNER_TEMP/e2b-sdk/bin/python" - source/services/core/deploy/e2b/build-template.py \ --image "$(cat "$HOME/.oac/build/e2b-runtime/runtime-image.id")" \ --name "sandbase-oac-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ - --api-key-file "$key" --output "$RUNNER_TEMP/e2b-result/template.json" + --api-key-file "$key" --output "$RUNNER_TEMP/e2b-result/template.json" <<'PY' + import hashlib, json, os, pathlib, runpy, sys + from types import SimpleNamespace + from e2b.template.main import TemplateBuilder + import e2b.template_sync.main as sdk + + output = pathlib.Path(os.environ['RUNNER_TEMP']) / 'e2b-result' + original_copy = TemplateBuilder.copy + def copy(self, src, dest, *args, **kwargs): + if str(src) != 'runtime.tar.gz': + return original_copy(self, src, dest, *args, **kwargs) + if dest != '/root/runtime.tar.gz' or args or kwargs != {'user': 'root'}: + raise RuntimeError('Runtime archive copy contract changed') + context = pathlib.Path(self._template._file_context_path) + digest = hashlib.sha256() + parts = [] + with (context / src).open('rb') as stream: + for index in range(256): + block = stream.read(32 * 1024 * 1024) + if not block: + break + digest.update(block) + name = f'runtime.part{index:04d}' + (context / name).write_bytes(block) + self = original_copy(self, name, '/root/' + name, user='root') + parts.append('/root/' + name) + if stream.read(1): + raise RuntimeError('Runtime archive exceeds 8 GiB') + if not parts: + raise RuntimeError('Runtime archive is empty') + (output / 'upload.json').write_text(json.dumps({ + 'chunks': len(parts), 'chunk_bytes': 32 * 1024 * 1024, + 'runtime_sha256': digest.hexdigest()}, indent=2) + '\n') + command = ('cat ' + ' '.join(parts) + ' > /root/runtime.tar.gz && ' + 'echo "' + digest.hexdigest() + ' /root/runtime.tar.gz" | sha256sum -c - && ' + 'rm ' + ' '.join(parts)) + return self.run_cmd(command, user='root') + TemplateBuilder.copy = copy + + original_request = sdk.request_build + def request(client, **kwargs): + selector = os.environ.get('RESUME_BUILD', '') + if selector: + template_id, build_id = selector.split(':') + # Inspect the raw response before the SDK's typed parser. + response = client.get_httpx_client().get( + f'/templates/{template_id}/builds/{build_id}/status') + response.raise_for_status() + state = response.json() + if (state.get('templateID') != template_id or state.get('buildID') != build_id + or state.get('status') != 'waiting' or state.get('logEntries') + or state.get('logs')): + raise RuntimeError('Recovery requires an unsubmitted waiting build with no logs') + result = SimpleNamespace(template_id=template_id, build_id=build_id, tags=[]) + else: + result = original_request(client, **kwargs) + receipt = {'template': result.template_id + ':' + result.build_id, + 'source_commit': os.environ['SOURCE_COMMIT'], + 'api_url': os.environ['E2B_API_URL'], 'resumed': bool(selector)} + (output / 'build-request.json').write_text(json.dumps(receipt, indent=2) + '\n') + return result + sdk.request_build = request + builder = sys.argv[1] + sys.argv = sys.argv[1:] + runpy.run_path(builder, run_name='__main__') + PY - name: Record the build identity env: SOURCE_COMMIT: ${{ steps.source.outputs.revision }} @@ -180,8 +257,9 @@ jobs: out.write('One combined Runtime template built. No Core configuration or production deployment was changed.\n') PY - uses: actions/upload-artifact@v6 + if: always() with: name: e2b-template-${{ steps.source.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/e2b-result/template.json - if-no-files-found: error + path: ${{ runner.temp }}/e2b-result/*.json + if-no-files-found: warn retention-days: 90 diff --git a/deploy/e2b/README.md b/deploy/e2b/README.md index 0da9f2e1f..8e675dc1e 100644 --- a/deploy/e2b/README.md +++ b/deploy/e2b/README.md @@ -17,8 +17,14 @@ Both are required. For the SandBase endpoint, set `E2B_API_URL` to `https://sand The E2B key is available only to the settings check and template-build steps. It is temporarily written to a private file, removed on exit, excluded from Runtime images and build reports, and never sent to a model provider. No model credentials are needed to build the template. A run creates a cloud template build and may incur the provider's build charges; do not trigger a run merely to validate workflow syntax. +## Bounded archive uploads + +The workflow splits the Runtime archive into files of at most 32 MiB and uploads each through the SDK's standard template file-copy API. These are independent files, not a multipart upload extension. The template concatenates them in order, verifies the complete archive's SHA-256, then removes the parts before the maintained extraction step. It still produces one template with unchanged Runtime contents. Archives are bounded to 8 GiB. + +For an upload failure before build submission, inspect the original run's logs and use `resume_build` with its `templateID:build_UUID` and the same explicit source SHA in `ref`. Recovery checks that the authenticated build is still `waiting` with no logs before reusing its identifiers. Do not use recovery after build submission or an ambiguous request outcome. All other runs create a new template build. + ## Output and qualification -The run summary and the `e2b-template-*` artifact contain `template.json`: the immutable `templateID:build_UUID`, source commit and branch, image ID, packaged Runtime checksum, base image, endpoint and advertised Harnesses. Template names include the run ID and attempt so another run does not replace this build's name. There is no automatic retry of an uncertain cloud build; inspect the provider before starting another run after a failure. +The successful run summary and the `e2b-template-*` artifact contain `template.json`: the immutable `templateID:build_UUID`, source commit and branch, image ID, packaged Runtime checksum, base image, endpoint and advertised Harnesses. Template names include the run ID and attempt so another run does not replace this build's name. The artifact also retains `upload.json` (chunk count and archive checksum) and `build-request.json` (cloud identifiers and source revision) when those stages are reached, including on failure. There is no automatic retry of an uncertain cloud build; inspect the provider before starting another run after a failure. The image checks verify committed daemon/adapter identity and native package loading/version checks. Template build completion establishes packaging readiness, not Core enrollment, real model execution or pause/resume qualification. The workflow changes no active Core selection, Kubernetes resource or production Session. [Sandbox deployment](../../contracts/agents-api/sandbox-deployment.md) owns later template selection and generation behavior. From 7ac004f0cad983d78d3648a6eeff174792687aec Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 4 Oct 2026 13:15:55 +0800 Subject: [PATCH 02/22] Fix E2B builder sibling imports --- .github/workflows/e2b-template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2b-template.yml b/.github/workflows/e2b-template.yml index dbf15c0d2..64dd9475b 100644 --- a/.github/workflows/e2b-template.yml +++ b/.github/workflows/e2b-template.yml @@ -235,6 +235,7 @@ jobs: return result sdk.request_build = request builder = sys.argv[1] + sys.path.insert(0, str(pathlib.Path(builder).resolve().parent)) sys.argv = sys.argv[1:] runpy.run_path(builder, run_name='__main__') PY From af2e840e01c04b0ae97d8e9bdb2c7c7e32453835 Mon Sep 17 00:00:00 2001 From: Jelly Su Date: Sat, 10 Oct 2026 22:16:25 +0800 Subject: [PATCH 03/22] docs: add Japanese README (#639) --- README.ja.md | 78 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 +- README.zh-CN.md | 2 +- 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 README.ja.md diff --git a/README.ja.md b/README.ja.md new file mode 100644 index 000000000..b5c2e3cf6 --- /dev/null +++ b/README.ja.md @@ -0,0 +1,78 @@ +
+ +![OpenAgentCore — One core. Many agents.](docs/assets/openagentcore-banner.jpeg) + +# OpenAgentCore + +複数のネイティブ実行エンジンに対応し、自分のインフラにデプロイできる OpenAI Agents API のオープンソース実装です。 + +[公式サイト](https://openagentcore.dev/) · [インストール](#インストール) · [API を使う](https://openagentcore.dev/docs/getting-started/quickstart) · [ドキュメント](https://openagentcore.dev/docs/getting-started/) · [コントリビューション](CONTRIBUTING.md) + +[English](README.md) · [简体中文](README.zh-CN.md) · **日本語** + +
+ +## コンポーネントの関係 + +![OpenAgentCore のアーキテクチャ](docs/assets/architecture.png) + +アプリケーションと管理者は、次の Core API を使用します。 + +| API | パス | 利用者 | +| --- | --- | --- | +| **[Agents API](https://openagentcore.dev/docs/api/public-agent-api)** | `/v1` | アプリケーション。[OpenAI の Agents API](https://developers.openai.com/api/docs/guides/agents-api/overview) と同じプロトコルを使用 | +| **[Core API](https://openagentcore.dev/contracts/agents-api/admin-api)** | `/core/v1` | 管理者。Web 経由で利用 | + +Core は実行状態を永続化し、Runtime は Environment 内で選択した Harness を実行します。各コンポーネントは定義されたプロトコルで接続されているため、個別に置き換えられます。詳しくは[アーキテクチャガイド](https://openagentcore.dev/docs/architecture)をご覧ください。 + +## OpenAgentCore とは + +OpenAgentCore は、自分のインフラ上で AI エージェントを実行し、[OpenAI Agents API](https://developers.openai.com/api/docs/guides/agents-api/overview) を提供します。 + +- **OpenAI と同じ API。** [公式 OpenAI SDK](https://developers.openai.com/api/docs/guides/agents/sdk) または直接の HTTP リクエストで、接続先を変更するだけで利用できます。新しいクライアントを覚える必要はありません。 +- **エージェントを選択可能。** 各 [Session](https://openagentcore.dev/docs/api/public-agent-api) は、[Codex](https://github.com/openai/codex)、[Claude Code](https://code.claude.com/docs/en/overview)、[MiniMax Code](https://github.com/MiniMax-AI/minimax-code) のいずれかの[ネイティブ Harness](https://openagentcore.dev/contracts/agents-api/harness-onboarding) を実行し、[設定したモデルプロバイダー](https://openagentcore.dev/contracts/agents-api/model-execution)を使用します。 +- **実行するマシンを選択可能。** エージェントは、[マネージドサンドボックス](https://openagentcore.dev/contracts/agents-api/sandbox-deployment)（[Docker](https://www.docker.com/)、[microsandbox](https://github.com/zerocore-ai/microsandbox)、[E2B](https://e2b.dev/)）でも、自分の Linux、macOS、Windows マシンでも動作します。 +- **すべてのコンポーネントを置き換え可能。** [サンドボックス](https://openagentcore.dev/docs/sandbox-provider)、[Harness](https://openagentcore.dev/contracts/agents-api/harness-onboarding)、[モデルプロバイダー](https://openagentcore.dev/contracts/agents-api/model-execution)は、[定義されたプロトコル](AGENTS.md#protocols-at-every-boundary)を通じて接続されます。 + +## スクリーンショット + +| 概要 | エージェントの監視 | +| --- | --- | +| ![デプロイの概要](docs/assets/console-overview-en.webp) | ![エージェントの監視](docs/assets/console-agent-metrics-en.webp) | + +## インストール + +[前提条件](https://openagentcore.dev/docs/getting-started/install#prerequisites)に従って Docker をセットアップした Linux または macOS で、次のコマンドを実行します。 + +```sh +curl -fsSL https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.sh | bash +``` + +Windows PowerShell の場合： + +```powershell +irm https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/install.ps1 | iex +``` + +続いて、以下の手順で設定します。 + +1. インストーラーが生成した Core key で **Web（管理コンソール）にサインイン**し、**ドメインと HTTPS を設定**します。 +2. **デフォルトモデルを設定**し、**Project API key を発行**します。 +3. ノード、E2B、または自分のマシンを**実行リソースとして追加**します。 +4. OpenAI SDK で **[最初の Session を実行](https://openagentcore.dev/docs/getting-started/quickstart)** します。 + +[インストールガイド](https://openagentcore.dev/docs/getting-started/install)では、各手順に加え、HTTPS の設定とローカルで手軽に試す方法を説明しています。リッスンアドレスやポートなどの設定は、[インストールオプション](https://openagentcore.dev/docs/getting-started/install-options)をご覧ください。 + +## ドキュメント + +| 目的 | 参照先 | +| --- | --- | +| インストールと運用 | [インストールガイド](https://openagentcore.dev/docs/getting-started/install)、続いて[運用ガイド](https://openagentcore.dev/docs/getting-started/operations) | +| API を使ったアプリケーション開発 | [クイックスタート](https://openagentcore.dev/docs/getting-started/quickstart)、続いて [Agents API ガイド](https://openagentcore.dev/docs/api/public-agent-api) | +| 完成したアプリケーションの確認 | [サンプル](https://openagentcore.dev/docs/examples) | +| 自分のマシンでエージェントを実行 | [セルフホスト実行](https://openagentcore.dev/docs/getting-started/self-hosted) | +| Harness の機能と制限の確認 | [Harness の機能](https://openagentcore.dev/contracts/agents-api/harness-capabilities) | +| 設計の理解 | [アーキテクチャガイド](https://openagentcore.dev/docs/architecture) | +| 新しいサンドボックス、Harness などのコンポーネントの追加 | [開発ガイド](https://openagentcore.dev/docs/development) | + +すべてのドキュメントは[ドキュメント一覧](https://openagentcore.dev/docs/getting-started/)から参照できます。コードを変更する前に、[コントリビューションガイド](CONTRIBUTING.md)をお読みください。 diff --git a/README.md b/README.md index d8056f965..bf0a5187f 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ An open-source, self-hosted implementation of the OpenAI Agents API with multipl [Website](https://openagentcore.dev/) · [Install](#install) · [Call the API](https://openagentcore.dev/docs/getting-started/quickstart) · [Documentation](https://openagentcore.dev/docs/getting-started/) · [Contributing](CONTRIBUTING.md) -**English** · [简体中文](README.zh-CN.md) +**English** · [简体中文](README.zh-CN.md) · [日本語](README.ja.md) diff --git a/README.zh-CN.md b/README.zh-CN.md index fb55a62a5..8d632d49a 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -8,7 +8,7 @@ OpenAI Agents API 的开源实现，支持多种原生执行引擎，可部署 [官网](https://openagentcore.dev/zh/) · [安装](#安装) · [调用 API](https://openagentcore.dev/zh/docs/getting-started/quickstart) · [文档](https://openagentcore.dev/zh/docs/getting-started/) · [参与贡献](CONTRIBUTING.md) -[English](README.md) · **简体中文** +[English](README.md) · **简体中文** · [日本語](README.ja.md) From 4d9f26308ab8efd845279ca557742fcc7c23963d Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 10 Oct 2026 23:14:49 +0800 Subject: [PATCH 04/22] perf(runtime): reduce managed Session startup waits --- apps/daemon/internal/cli/connect.go | 42 ++- apps/daemon/internal/cli/connect_bootstrap.go | 3 +- .../internal/cli/connect_bootstrap_test.go | 12 +- .../internal/cli/connect_environment.go | 7 +- apps/daemon/internal/cli/connect_startup.go | 50 ++++ .../internal/cli/connect_startup_test.go | 266 ++++++++++++++++++ .../internal/cli/native_discovery_test.go | 20 ++ .../agents-api/node-generation-protocol.md | 4 +- .../agents-api/zh/node-generation-protocol.md | 6 +- docs/getting-started/operations.md | 7 + docs/runtime-bootstrap.md | 3 + docs/sandbox-provider.md | 4 + docs/zh/getting-started/operations.md | 9 +- docs/zh/runtime-bootstrap.md | 5 +- docs/zh/sandbox-provider.md | 6 +- internal/runtimebootstrap/bootstrap.go | 22 +- internal/runtimebootstrap/bootstrap_test.go | 11 +- .../cmd/server/managed_generations_test.go | 4 +- .../server/managed_setup_preflight_test.go | 10 +- .../deploy/e2b/helper_contract_generated.py | 2 +- services/core/deploy/e2b/managed_init_test.py | 2 +- .../execution/environment_admission.go | 2 +- .../execution/runtime_fresh_hint_test.go | 208 ++++++++++++++ .../internal/execution/runtime_lifecycle.go | 6 +- .../internal/execution/runtime_wake_hint.go | 33 ++- .../execution/runtime_workspace_test.go | 4 + services/core/internal/execution/worker.go | 10 +- .../execution/worker_capability_test.go | 63 +++++ .../internal/execution/worker_schedule.go | 5 +- .../runtimegateway/capability_hint_test.go | 138 +++++++++ .../core/internal/runtimegateway/registry.go | 38 ++- .../core/internal/runtimegateway/session.go | 25 +- .../internal/sandbox/docker/bootstrap_test.go | 2 +- .../internal/sandbox/docker/contract_test.go | 2 +- .../sandbox/docker/deployment_test.go | 2 +- .../internal/sandbox/docker/provider_test.go | 2 +- .../internal/sandbox/docker/recovery_test.go | 2 +- .../internal/sandbox/e2b/contract_test.go | 4 +- .../internal/sandbox/e2b/helper_contract.go | 2 +- .../sandbox/e2b/helper_contract_test.go | 1 + .../sandbox/e2b/installed_paths_test.go | 2 +- .../sandbox/e2b/internal/contractgen/main.go | 7 +- .../core/internal/sandbox/e2b/process_test.go | 8 +- .../internal/sandbox/e2b/provider_test.go | 4 +- .../sandbox/microsandbox/contract_test.go | 23 +- .../microsandbox/creation_settlement_test.go | 2 +- .../internal/sandbox/microsandbox/types.go | 2 +- .../sandbox/microsandbox/workspace_test.go | 4 +- .../internal/sandbox/node/connection_test.go | 2 +- .../sandbox/node/creation_settlement_test.go | 2 +- .../internal/sandbox/node/docker_live_test.go | 2 +- .../node/generation_connection_test.go | 2 +- .../internal/sandbox/node/generation_json.go | 8 +- .../sandbox/node/generation_json_test.go | 20 ++ .../core/internal/sandbox/node/node_test.go | 5 +- .../internal/sandbox/node/recovery_test.go | 2 +- .../internal/sandbox/node/timeout_test.go | 5 +- services/core/internal/sandbox/node/wire.go | 5 +- .../internal/sandbox/node/workspace_test.go | 23 +- .../internal/sandbox/runtime_bootstrap.go | 2 +- .../core/internal/sandbox/sandbox_provider.go | 6 +- .../integration/runtime_lifecycle_test.go | 31 ++ services/core/tools/e2b-provider/README.md | 6 +- .../e2b-provider/helper_contract_generated.py | 2 +- .../tools/e2b-provider/observation_test.py | 2 +- services/core/tools/e2b-provider/provider.py | 57 +++- .../core/tools/e2b-provider/provider_test.py | 152 +++++++++- .../core/tools/e2b-provider/state_test.py | 2 +- .../tools/e2b-provider/testdata/contract.json | 124 ++++---- .../tools/microsandbox-provider/README.md | 4 +- 70 files changed, 1355 insertions(+), 205 deletions(-) create mode 100644 apps/daemon/internal/cli/connect_startup.go create mode 100644 apps/daemon/internal/cli/connect_startup_test.go create mode 100644 services/core/internal/execution/runtime_fresh_hint_test.go create mode 100644 services/core/internal/execution/worker_capability_test.go create mode 100644 services/core/internal/runtimegateway/capability_hint_test.go diff --git a/apps/daemon/internal/cli/connect.go b/apps/daemon/internal/cli/connect.go index dc87bb4a9..0ec595e66 100644 --- a/apps/daemon/internal/cli/connect.go +++ b/apps/daemon/internal/cli/connect.go @@ -70,7 +70,7 @@ func runConnect(ctx *runContext, args []string) error { if *remote != "" || *environment != "" || *credentialFile != "" || fs.NArg() != 0 { return errors.New("connect: bootstrap input cannot be combined with enrollment options") } - bootstrapped, err = bootstrapProfile(*bootstrapFile) + bootstrapped, err = bootstrapProfile(*bootstrapFile, ctx) if err != nil { return err } @@ -98,13 +98,6 @@ func runConnect(ctx *runContext, args []string) error { return spawnBackground(context.Background(), ctx, *profile, os.Args) } - // Self-check before loading credentials so a machine with no - // supported agent CLI fails fast. - agentCLIs, err := preflightAgentCLIs(context.Background(), ctx, *profile) - if err != nil { - return err - } - var prof auth.Profile if bootstrapped != nil { prof = *bootstrapped @@ -115,7 +108,7 @@ func runConnect(ctx *runContext, args []string) error { } } - return mainLoop(ctx, *profile, prof, agentCLIs) + return mainLoop(ctx, *profile, prof) } // spawnBackground forks the daemon into the background. Parent @@ -186,11 +179,16 @@ func spawnBackground(ctx context.Context, rc *runContext, profile string, argv [ // background process. SIGINT / SIGTERM cancels the root context, which // unblocks the read pump and any in-flight Send so the daemon exits // without orphaning agent subprocesses. -func mainLoop(rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery) error { - return mainLoopRemote(context.Background(), rc, profile, prof, agentCLIs, "") +func mainLoop(rc *runContext, profile string, prof auth.Profile) error { + return mainLoopRemote(context.Background(), rc, profile, prof, "") } -func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery, remote string) error { +func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof auth.Profile, remote string) error { + return mainLoopRemoteWithDiscovery(parent, rc, profile, prof, remote, func(ctx context.Context) (agentCLIDiscovery, error) { + return preflightAgentCLIs(ctx, rc, profile) + }) +} +func mainLoopRemoteWithDiscovery(parent context.Context, rc *runContext, profile string, prof auth.Profile, remote string, discover func(context.Context) (agentCLIDiscovery, error)) error { // Route through obs/log so daemon log lines pick up the same // trace_id / span_id auto-injection as the server side — when the // daemon adopts an envelope's trace, every log call under that ctx @@ -205,18 +203,18 @@ func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof rootCtx, cancel := daemonize.NotifyContext(parent) defer cancel() - bootCtx, bootCancel := context.WithTimeout(rootCtx, bootstrapTimeout) - var boot *transport.BootstrapResponse - var err error - if remote == "" { - boot, err = transport.Bootstrap(bootCtx, prof.ServerURL, prof.RuntimeID, prof.RunnerCredential, Version) - } else { - boot, err = environmentBootstrap(bootCtx, prof, remote) - } - bootCancel() + boot, agentCLIs, err := prepareConnection(rootCtx, discover, func(ctx context.Context) (*transport.BootstrapResponse, error) { + bootCtx, stop := context.WithTimeout(ctx, bootstrapTimeout) + defer stop() + if remote != "" { + return environmentBootstrap(bootCtx, prof, remote) + } + return transport.Bootstrap(bootCtx, prof.ServerURL, prof.RuntimeID, prof.RunnerCredential, Version) + }) if err != nil { - return fmt.Errorf("connect: bootstrap: %w", err) + return err } + wsURL, err := transport.DeriveWSURL(*boot, prof.ServerURL) if err != nil { return fmt.Errorf("connect: derive ws url: %w", err) diff --git a/apps/daemon/internal/cli/connect_bootstrap.go b/apps/daemon/internal/cli/connect_bootstrap.go index 1e7848808..27705a1c8 100644 --- a/apps/daemon/internal/cli/connect_bootstrap.go +++ b/apps/daemon/internal/cli/connect_bootstrap.go @@ -9,7 +9,7 @@ import ( // The launch file is the sole credential source for this connection. Reopening // it on process restart never reads or overwrites an auth profile. -func bootstrapProfile(path string) (*auth.Profile, error) { +func bootstrapProfile(path string, rc *runContext) (*auth.Profile, error) { raw, err := runtimefs.ReadPrivatePath(path, runtimebootstrap.MaxBytes) if err != nil { return nil, errors.New("connect: Runtime bootstrap file unavailable") @@ -18,5 +18,6 @@ func bootstrapProfile(path string) (*auth.Profile, error) { if err != nil { return nil, err } + rc.installedKinds = map[string]bool{input.Harness: true} return &auth.Profile{ServerURL: input.CoreURL, RuntimeID: input.DeviceID, RunnerCredential: input.Credential}, nil } diff --git a/apps/daemon/internal/cli/connect_bootstrap_test.go b/apps/daemon/internal/cli/connect_bootstrap_test.go index 7f510d699..fbfb704ab 100644 --- a/apps/daemon/internal/cli/connect_bootstrap_test.go +++ b/apps/daemon/internal/cli/connect_bootstrap_test.go @@ -31,7 +31,7 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { if err = os.WriteFile(filepath.Join(profileDir, "auth.json"), priorRaw, 0600); err != nil { t.Fatal(err) } - input := runtimebootstrap.Connection{Version: runtimebootstrap.Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "bootstrap-secret"} + input := runtimebootstrap.Connection{Version: runtimebootstrap.Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "bootstrap-secret", Harness: "codex"} raw, err := input.Marshal() if err != nil { t.Fatal(err) @@ -41,7 +41,11 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { t.Fatal(err) } for range 2 { - p, err := bootstrapProfile(path) + rc := &runContext{} + p, err := bootstrapProfile(path, rc) + if !rc.installedKinds["codex"] || len(rc.installedKinds) != 1 { + t.Fatal("bootstrap did not scope discovery") + } if err != nil || p.ServerURL != input.CoreURL || p.RuntimeID != input.DeviceID || p.RunnerCredential != input.Credential { t.Fatal("failed bootstrap/restart", err) } @@ -53,13 +57,13 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { if err = os.WriteFile(path, []byte("bootstrap-secret"), 0600); err != nil { t.Fatal(err) } - if _, err = bootstrapProfile(path); err == nil || strings.Contains(err.Error(), input.Credential) { + if _, err = bootstrapProfile(path, &runContext{}); err == nil || strings.Contains(err.Error(), input.Credential) { t.Fatal("invalid input fell back or leaked") } if err = os.Remove(path); err != nil { t.Fatal(err) } - if _, err = bootstrapProfile(path); err == nil { + if _, err = bootstrapProfile(path, &runContext{}); err == nil { t.Fatal("missing input fell back to private auth") } } diff --git a/apps/daemon/internal/cli/connect_environment.go b/apps/daemon/internal/cli/connect_environment.go index 0b1c3b4b1..4e9628e9e 100644 --- a/apps/daemon/internal/cli/connect_environment.go +++ b/apps/daemon/internal/cli/connect_environment.go @@ -163,13 +163,8 @@ func runEnvironmentConnect(parent context.Context, rc *runContext, profile strin if background && !daemonize.IsBackgroundChild() { return spawnBackground(parent, rc, profile, os.Args) } - // Discovery consumes the immutable Runtime binding; it must follow enrollment. - discovery, err := preflightAgentCLIs(parent, rc, profile) - if err != nil { - return err - } prof := auth.Profile{ServerURL: base, RuntimeID: bound.DeviceID, RunnerCredential: credential} - return rejected(mainLoopRemote(parent, rc, profile, prof, discovery, remote)) + return rejected(mainLoopRemote(parent, rc, profile, prof, remote)) } var ( diff --git a/apps/daemon/internal/cli/connect_startup.go b/apps/daemon/internal/cli/connect_startup.go new file mode 100644 index 000000000..68258f297 --- /dev/null +++ b/apps/daemon/internal/cli/connect_startup.go @@ -0,0 +1,50 @@ +package cli + +import ( + "context" + "errors" + "fmt" + + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/transport" +) + +// prepareConnection overlaps independent preflight work after local credentials +// and enrollment are resolved. Both workers are joined before returning, including +// on failure. Their temporary context never owns the live connection or executor. +func prepareConnection(parent context.Context, discover func(context.Context) (agentCLIDiscovery, error), bootstrap func(context.Context) (*transport.BootstrapResponse, error)) (*transport.BootstrapResponse, agentCLIDiscovery, error) { + if err := parent.Err(); err != nil { + return nil, nil, err + } + ctx, cancel := context.WithCancel(parent) + defer cancel() + var agents agentCLIDiscovery + var boot *transport.BootstrapResponse + done := make(chan error, 2) + go func() { + var err error + agents, err = discover(ctx) + done <- err + }() + go func() { + var err error + boot, err = bootstrap(ctx) + if err != nil { + err = fmt.Errorf("connect: bootstrap: %w", err) + } + done <- err + }() + var result error + for range 2 { + if err := <-done; err != nil { + result = errors.Join(result, err) + cancel() + } + } + if result != nil { + return nil, nil, result + } + if err := parent.Err(); err != nil { + return nil, nil, err + } + return boot, agents, nil +} diff --git a/apps/daemon/internal/cli/connect_startup_test.go b/apps/daemon/internal/cli/connect_startup_test.go new file mode 100644 index 000000000..6b5bca1b6 --- /dev/null +++ b/apps/daemon/internal/cli/connect_startup_test.go @@ -0,0 +1,266 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent" + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/auth" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/transport" +) + +func awaitStartup(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatal("startup worker did not reach barrier") + } +} + +func TestConnectionPreflightOverlapsAndJoins(t *testing.T) { + for _, first := range []string{"bootstrap", "discovery"} { + t.Run(first, func(t *testing.T) { + discoveryStarted, bootstrapStarted := make(chan struct{}), make(chan struct{}) + releaseDiscovery, releaseBootstrap := make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + boot := &transport.BootstrapResponse{DeviceID: "device"} + go func() { + result, _, err := prepareConnection(t.Context(), func(context.Context) (agentCLIDiscovery, error) { + close(discoveryStarted) + <-releaseDiscovery + return agentCLIDiscovery{}, nil + }, func(context.Context) (*transport.BootstrapResponse, error) { + close(bootstrapStarted) + <-releaseBootstrap + return boot, nil + }) + if err == nil && result != boot { + err = errors.New("bootstrap result lost") + } + done <- err + }() + awaitStartup(t, discoveryStarted) + awaitStartup(t, bootstrapStarted) + if first == "bootstrap" { + close(releaseBootstrap) + } else { + close(releaseDiscovery) + } + select { + case <-done: + t.Fatal("returned before both prerequisites completed") + default: + } + if first == "bootstrap" { + close(releaseDiscovery) + } else { + close(releaseBootstrap) + } + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("join blocked") + } + }) + } +} + +func TestConnectionPreflightFailureCancelsAndJoinsSibling(t *testing.T) { + for _, failing := range []string{"bootstrap", "discovery"} { + t.Run(failing, func(t *testing.T) { + cause := errors.New("rejected") + started, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + wait := func(ctx context.Context) error { + close(started) + <-ctx.Done() + close(canceled) + <-release + return ctx.Err() + } + fail := func() error { <-started; return cause } + go func() { + boot, agents, err := prepareConnection(t.Context(), func(ctx context.Context) (agentCLIDiscovery, error) { + if failing == "discovery" { + return nil, fail() + } + return nil, wait(ctx) + }, func(ctx context.Context) (*transport.BootstrapResponse, error) { + if failing == "bootstrap" { + return nil, fail() + } + return nil, wait(ctx) + }) + if boot != nil || agents != nil { + err = errors.New("partial successful result escaped") + } + done <- err + }() + awaitStartup(t, canceled) + select { + case <-done: + t.Fatal("returned before sibling cleanup") + default: + } + close(release) + select { + case err := <-done: + if !errors.Is(err, cause) { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("join blocked") + } + }) + } +} + +func TestConnectionPreflightParentCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + started := make(chan struct{}, 2) + done := make(chan error, 1) + go func() { + _, _, err := prepareConnection(ctx, func(ctx context.Context) (agentCLIDiscovery, error) { + started <- struct{}{} + <-ctx.Done() + return nil, ctx.Err() + }, func(ctx context.Context) (*transport.BootstrapResponse, error) { + started <- struct{}{} + <-ctx.Done() + return nil, ctx.Err() + }) + done <- err + }() + awaitStartup(t, started) + awaitStartup(t, started) + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("cancellation blocked") + } + _, _, err := prepareConnection(ctx, func(context.Context) (agentCLIDiscovery, error) { + t.Error("discovery started after cancellation") + return nil, nil + }, func(context.Context) (*transport.BootstrapResponse, error) { + t.Error("bootstrap started after cancellation") + return nil, nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +// Controlled waits model independent work, not production latency. Compare the +// same two operations to demonstrate the removed dependency edge. +func BenchmarkConnectionPreflight(b *testing.B) { + for _, parallel := range []bool{false, true} { + name := "serial" + if parallel { + name = "overlap" + } + b.Run(name, func(b *testing.B) { + discover := func(context.Context) (agentCLIDiscovery, error) { + time.Sleep(5 * time.Millisecond) + return agentCLIDiscovery{}, nil + } + bootstrap := func(context.Context) (*transport.BootstrapResponse, error) { + time.Sleep(5 * time.Millisecond) + return &transport.BootstrapResponse{}, nil + } + for b.Loop() { + if parallel { + _, _, _ = prepareConnection(b.Context(), discover, bootstrap) + } else { + _, _ = discover(b.Context()) + _, _ = bootstrap(b.Context()) + } + } + }) + } +} + +func TestMainLoopOverlapsPreflightWithoutEarlyRegistration(t *testing.T) { + for _, failure := range []string{"", "discovery", "bootstrap"} { + t.Run(failure, func(t *testing.T) { + t.Setenv("OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE", "") + original := harnessDeclarations + defer func() { harnessDeclarations = original }() + started, release, completed := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var probes, boots, dials atomic.Int32 + declaration := original[0] + declaration.Discover = func(ctx context.Context, _ agent.DiscoveryOptions, info proto.SupportedAgentKind) *agent.Runtime { + probes.Add(1) + close(started) + select { + case <-release: + case <-ctx.Done(): + } + defer close(completed) + info.Available = failure != "discovery" && ctx.Err() == nil + return &agent.Runtime{Info: info} + } + harnessDeclarations = []agent.Declaration{declaration} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/agent-daemon/bootstrap": + boots.Add(1) + select { + case <-started: + case <-r.Context().Done(): + return + } + if dials.Load() != 0 { + t.Error("dial before discovery completed") + } + close(release) + if failure == "bootstrap" { + http.Error(w, "rejected", http.StatusUnauthorized) + return + } + _ = json.NewEncoder(w).Encode(transport.BootstrapResponse{DeviceID: "device"}) + case "/agent-daemon/ws": + select { + case <-completed: + default: + t.Error("registration before discovery joined") + } + dials.Add(1) + http.Error(w, "end test", http.StatusUnauthorized) + default: + t.Errorf("unexpected route %s", r.URL.Path) + http.NotFound(w, r) + } + })) + defer server.Close() + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + err := mainLoopRemote(ctx, &runContext{stdout: io.Discard, stderr: io.Discard}, "default", auth.Profile{ServerURL: server.URL, RuntimeID: "device", RunnerCredential: "fixture"}, "") + if err == nil || ctx.Err() != nil { + t.Fatalf("unexpected completion: %v, context %v", err, ctx.Err()) + } + expected := int32(0) + if failure == "" { + expected = 1 + } + if probes.Load() != 1 || boots.Load() != 1 || dials.Load() != expected { + t.Fatalf("calls discovery=%d bootstrap=%d dial=%d", probes.Load(), boots.Load(), dials.Load()) + } + }) + } +} diff --git a/apps/daemon/internal/cli/native_discovery_test.go b/apps/daemon/internal/cli/native_discovery_test.go index 5f3592c9d..29776bcde 100644 --- a/apps/daemon/internal/cli/native_discovery_test.go +++ b/apps/daemon/internal/cli/native_discovery_test.go @@ -73,3 +73,23 @@ func TestDiscoveryUnavailableAndCancelled(t *testing.T) { t.Fatal("unconfigured adapter retained") } } + +func TestSelectedHarnessUnavailableDoesNotProbeOthers(t *testing.T) { + declarations := append([]agent.Declaration(nil), harnessDeclarations...) + for i := range declarations { + declarations[i].Discover = func(_ context.Context, _ agent.DiscoveryOptions, info proto.SupportedAgentKind) *agent.Runtime { + if info.Kind != "codex" { + t.Fatalf("unselected Harness was probed: %s", info.Kind) + } + return &agent.Runtime{Info: info} + } + } + rc := &runContext{stdout: io.Discard, stderr: io.Discard, installedKinds: map[string]bool{"codex": true}} + if _, err := discoverAgentCLIs(t.Context(), rc, "default", declarations); err == nil { + t.Fatal("unavailable selection was accepted") + } + rc.installedKinds = map[string]bool{"unknown": true} + if _, err := discoverAgentCLIs(t.Context(), rc, "default", declarations); err == nil { + t.Fatal("unknown selection fell back") + } +} diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index de65b9069..5b7892cac 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -123,4 +123,6 @@ The readiness classes, one exported error and one code each, are authored in `se Preparation diagnostics keep fixed typed causes. Only artifact transfer, checksum or release-provenance failures report `runtime_download_failed`; the private preparer signals that class through its exit category, without Core or the node parsing stderr. Provider, ownership, cancellation and unclassified failures keep their typed code or `provider_unavailable`. No raw provider text crosses the protocol. -The current wire version is 5. Create bootstrap and Resume requests may carry an optional workspace filesystem binding. The node validates its tenant and Environment against the allocation reference, its immutable ObjectID, and its attachment configuration ID against the supplied configuration before forwarding it. The filesystem resolver validates adapter-native ownership. A missing binding selects owned storage; a present binding cannot fall back. Error responses may retain a Resume target only when its nonempty native ID, name, generation and snapshot provenance match the requested target; this is cleanup evidence, never successful restore. +The current wire version is 6. Create bootstrap and Resume requests may carry an optional workspace filesystem binding. The node validates its tenant and Environment against the allocation reference, its immutable ObjectID, and its attachment configuration ID against the supplied configuration before forwarding it. The filesystem resolver validates adapter-native ownership. A missing binding selects owned storage; a present binding cannot fall back. Error responses may retain a Resume target only when its nonempty native ID, name, generation and snapshot provenance match the requested target; this is cleanup evidence, never successful restore. + +Creation carries the Session-selected `Bootstrap.Harness` into Runtime bootstrap version 2. Core and nodes use protocol version 6; upgrade matching nodes before publishing this Core release. diff --git a/contracts/agents-api/zh/node-generation-protocol.md b/contracts/agents-api/zh/node-generation-protocol.md index 481dbc4af..b24278ee8 100644 --- a/contracts/agents-api/zh/node-generation-protocol.md +++ b/contracts/agents-api/zh/node-generation-protocol.md @@ -1,7 +1,7 @@ --- title: "沙箱节点协议" source: contracts/agents-api/node-generation-protocol.md -source_hash: 3fa1d9257acefa2d144f6e76ac5c2d3ff1d074ebb2b21fb48a2f79ef58ad6073 +source_hash: 961c7993fa9c75ddadb6604fece87fb556bbcc40d4f190d03d2e90927c5be7bb --- 沙箱节点在其主机上运行 Docker 或 microsandbox Provider，并通过一个 WebSocket 与 Core 相连。Core 通过该连接发送 Provider 操作；节点针对本地 Provider 执行这些操作，并报告就绪状态、主机测量值及其持有的部署代次。Core 始终是唯一的生命周期所有者：节点绝不重试变更操作或调度工作。帧和校验器位于 [`services/core/internal/sandbox/node`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/sandbox/node)（`wire.go`、`generation_wire.go`）；节点用于注册和读取配置的 HTTP 路由位于[机器连接 API](machine-api.md#node-routes)。 @@ -125,4 +125,6 @@ Runtime 字节缺失时，绝不将固定的放置实例迁移到当前 Runtime 准备诊断使用固定的类型化原因。只有制品传输、校验和或版本来源验证失败才会报告 `runtime_download_failed`；私有准备器通过退出类别指示这一类失败，Core 和节点都不解析 stderr。Provider 故障、所有权故障、取消和未分类故障保留其类型化代码，或使用 `provider_unavailable`。协议中不会传输任何 Provider 原始文本。 -当前线协议版本为 5。Create 引导和 Resume 请求可携带可选的工作区文件系统绑定。节点在转发前校验其租户及 Environment 与分配引用一致、ObjectID 不可变且有效，以及挂载配置 ID 与所传配置一致。文件系统解析器负责适配器原生所有权校验。未提供绑定时选择自有存储；已提供绑定时不得回退。错误响应仅可保留原生 ID 非空且名称、代次、快照来源均匹配请求的 Resume 目标；这仅为清理证据，不代表恢复成功。 +当前线协议版本为 6。Create 引导和 Resume 请求可携带可选的工作区文件系统绑定。节点在转发前校验其租户及 Environment 与分配引用一致、ObjectID 不可变且有效，以及挂载配置 ID 与所传配置一致。文件系统解析器负责适配器原生所有权校验。未提供绑定时选择自有存储；已提供绑定时不得回退。错误响应仅可保留原生 ID 非空且名称、代次、快照来源均匹配请求的 Resume 目标；这仅为清理证据，不代表恢复成功。 + +创建操作通过 `Bootstrap.Harness` 将会话选择传递到 Runtime 启动协议版本 2。Core 和节点使用协议版本 6；发布此 Core 版本前需升级配套节点。 diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index 55760e76b..0ea13768f 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -24,6 +24,9 @@ docker compose -f ~/.oac/core/compose.yaml ps The examples use the default installation directory. On Windows, invoke the management command with `& "$HOME/.oac/core/oac.exe"` followed by the same arguments. For a custom installation directory, replace the path in each command. +After local credentials and enrollment are resolved, Runtime Harness discovery and the authenticated bootstrap HTTP request run concurrently. Both must succeed before the Runtime opens its connection or publishes capabilities. Failure cancels the sibling operation and waits for cleanup. The executor remains owned by the connection lifetime. Runtime startup changes require a rebuilt, qualified Runtime template. + + ## Service health Use these observations for different questions: @@ -187,3 +190,7 @@ Installation and mutating `oac` commands share the [installation lock](../config Web signs administrators in with the Core key, checks the origin of every request, and forwards signed-in `/core/v1` requests to Core with the Core key, which stays on the server. It forwards `/v1` and `/api/v1` to Core unchanged, with the caller's own credential, serves only the non-secret node payload at `/node-install/`, and has no Docker or KVM access. Machine routes under `/api/v1` use their own enrollment and connection credentials. No service receives a Docker socket. Sandboxes are the isolation boundary ([Runtime and outer isolation](../concepts.md#runtime-and-outer-isolation)). Docker sandboxes share the node's kernel, and a Docker node is [root-equivalent](./nodes.md#what-the-installer-sets-up) on its host; microsandbox gives each sandbox a microVM with an explicit [network policy](./nodes.md#what-the-installer-sets-up). Core itself has no Docker socket or KVM access. + +## Runtime availability probes + +Codex CLI availability emits `runtime version probe` records for `process_spawn` and `process_wait`, with duration and outcome only. The wait interval includes executable loading and the version command; neither interval is model execution. diff --git a/docs/runtime-bootstrap.md b/docs/runtime-bootstrap.md index 7397e07fa..8284d9f51 100644 --- a/docs/runtime-bootstrap.md +++ b/docs/runtime-bootstrap.md @@ -17,6 +17,7 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json | `version` | The exact bootstrap version, `runtimebootstrap.Version` | | `core_url` | HTTP(S) machine API base ending in `/api/v1`, without credentials, query or fragment | | `device_id` | Canonical nonzero UUID of the daemon identity Core issued | +| `harness` | The immutable Session Harness identifier; only this Harness is probed and registered by a managed Runtime | | `credential` | Nonempty daemon credential Core issued, without whitespace or NUL | The decoder rejects unknown, duplicate, missing and case-aliased fields, other versions and documents larger than `runtimebootstrap.MaxBytes` (16 KiB). Errors never include submitted values. A missing or malformed file fails before the daemon connects. @@ -27,6 +28,8 @@ The file is the only authentication input for this launch: the daemon refuses to The provider creates the account, mounts and workspace, delivers this file, sets the Runtime's resource and Environment binding settings, and starts the daemon as the unprivileged Runtime account. Docker writes the file into the Runtime's owned home volume; microsandbox and E2B deliver it before launching the same command. +Core derives `harness` from the Session that owns the allocation. A combined image can contain several Harnesses, but its managed Runtime discovers only this selection; an unknown, missing or unavailable selection fails without probing another Harness. The bootstrap version is 2. Upgrade Core, its provider helpers and newly launched Runtime images together; retained allocations keep their existing bootstrap and Runtime. Self-hosted installations continue to discover their installed Harness set. + The Runtime validates the input and owns authentication and connection. A successful launch proves only the handoff: an authenticated connection, prepared capabilities and execution readiness are separate observations under the [Core–Runtime protocol](./runtime-protocol.md), and the [Sandbox Provider guide](./sandbox-provider.md#four-distinct-readiness-facts) lists what each one proves. Self-hosted executors and operator-provisioned devices get their daemon identity in other ways; the [machine connection API](../contracts/agents-api/machine-api.md#credentials) lists every credential source. All of them enter the same Runtime execution loop. diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 0eaa49ff4..149b7c352 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -160,6 +160,8 @@ Node readiness binds to the exact generation, the current connection and the own ### Allocation lifecycle +Core includes the owning Session’s immutable Harness in `Bootstrap.Harness`; every provider projects it into the [Runtime bootstrap](./runtime-bootstrap.md) without choosing an implementation. + The allocation, its dedicated daemon credential digest and the exact Session binding commit atomically before `Create`, under the execution lease and the Session lock. Only a fresh allocation receipt permits `Create`; retries and a Core restart observe the same reference without replaying it or rotating the credential. An allocation is private compute ownership, separate from public Environment connection and native readiness; adapters qualify bootstrap completion, and Core never infers it from an engine or provider name. With a configured provider, the Worker scans committed pending hosted Environments that have no allocation, which covers idle Session creation and recovery after an interruption between commit and bootstrap; an existing allocation never re-enters that path. The scan is bounded and serialized by the lifecycle owner and needs no caller action. An initial reservation without a Turn leaves its Session idle, and a daemon connection is never treated as native readiness. The same scan publishes authenticated connection observations with durable generations, after verifying the exact Session and device binding and a settled bootstrap. @@ -172,6 +174,8 @@ Terminal cleanup atomically revokes the device's authority, records the Environm Each registered node has one serial lifecycle worker that owns its gate, allocation and pending cursors, connections and wake hints; E2B allocations share one serial lifecycle without a node. A thin coordinator discovers nodes and shuts workers down, and never holds its map mutex during database, provider or wait operations. Workers advance independently, so a stuck provider on one online node never stalls another: lifecycle concurrency is one operation per node and grows with the node count. Offline workers stay, so their retained resources remain observable after reconnection. +After a hosted Environment commits, Core sends a bounded hint to its placement’s lifecycle worker. A committed pending input also sends a hint, recovering a missed creation notification. Hints use the existing serial gate, lease, capacity checks and one-shot allocation receipts; they do not provision inline in the admission handler. Each normal maintenance period permits at most one extra hinted scan, and periodic scans recover missed or coalesced hints. Existing allocation observations still precede pending provisioning, so a slow observation on the same node can delay a fresh Environment. + Allocation scans filter by node before their 32-row page limit, and pending scans join the unreleased committed placement. Each node advances its own cursor, including past failed observations, and wraps once at the end. Direct provisioning resolves the tenant-scoped placement before entering that node's gate, and an existing allocation must agree with it; Core never chooses another node. Before releasing the execution lease, the coordinator stops accepting work and cancels and drains every node worker and direct caller. Lease loss affects everything; ordinary provider failures stay within their node. A planned deployment drain or node retirement cancels lifecycle contexts synchronously between leased operations, through the lease gate with its five-second bound, including an active manual reconcile, and never cancels an in-flight leased query just to change configuration. A failed cancellation fence closes manager admission and reports owner failure. A failed retirement keeps the original lifecycle identity and gate until owner shutdown, and the drain barrier stays closed. Session locks, deployment capacity transactions and revision-checked receipts stay authoritative, and no external operation holds a database lock. diff --git a/docs/zh/getting-started/operations.md b/docs/zh/getting-started/operations.md index e1b4d1112..cfe626664 100644 --- a/docs/zh/getting-started/operations.md +++ b/docs/zh/getting-started/operations.md @@ -1,7 +1,7 @@ --- title: "运维" source: docs/getting-started/operations.md -source_hash: 97f5d49b9b39441a78026433d662120d46471ef5f9a0b6b4508998a3b4491d09 +source_hash: f6ce9d76fdd074d9e7e7da4a69da6a4583a67848b86aa641ad09f127fcae346b --- 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。 @@ -26,6 +26,9 @@ docker compose -f ~/.oac/core/compose.yaml ps 示例使用默认安装目录。Windows 上使用 `& "$HOME/.oac/core/oac.exe"` 调用管理命令，后接相同参数。使用自定义安装目录时，替换各命令中的路径。 +本地凭据和注册解析完成后，Runtime 的 Harness 发现与已认证的引导 HTTP 请求并发执行。两者都成功后才建立连接并发布能力。失败会取消另一项操作并等待其清理。执行器仍由连接生命周期持有。Runtime 启动变更需要重新构建并验证 Runtime 模板。 + + ## 服务健康状态 {#service-health} 根据不同问题使用这些观察： @@ -190,3 +193,7 @@ cd && rm -rf ~/.oac/core Web 使用 Core 密钥让管理员登录，检查每个请求来源，并用保留在服务器上的 Core 密钥将已登录的 `/core/v1` 请求转发到 Core。它把 `/v1` 和 `/api/v1` 原样转发给 Core，使用调用方自己的凭据；Web 仅在 `/node-install/` 提供不含密钥的节点文件，没有 Docker 或 KVM 访问权限。`/api/v1` 机器路由使用独立注册和连接凭据。没有服务持有 Docker 套接字。 沙箱是隔离边界（[Runtime 与外层隔离](../concepts.md#runtime-and-outer-isolation)）。Docker 沙箱共享节点内核，Docker 节点在主机上[等同于 root 权限](nodes.md#what-the-installer-sets-up)；microsandbox 为每个沙箱提供具有显式[网络策略](nodes.md#what-the-installer-sets-up)的 microVM。Core 自身无 Docker 套接字或 KVM 访问权限。 + +## Runtime 可用性探测 {#runtime-availability-probes} + +Codex CLI 可用性探测为 `process_spawn` 和 `process_wait` 输出 `runtime version probe` 记录，只包含耗时和结果。等待区间包含可执行文件加载和版本命令，两者都不是模型执行。 diff --git a/docs/zh/runtime-bootstrap.md b/docs/zh/runtime-bootstrap.md index d3338fa61..5bd813902 100644 --- a/docs/zh/runtime-bootstrap.md +++ b/docs/zh/runtime-bootstrap.md @@ -1,7 +1,7 @@ --- title: "Runtime 引导" source: docs/runtime-bootstrap.md -source_hash: f7e9275feee79ac1fb1299227e148b85c26efc8b8af5ca0e8f498126c563d182 +source_hash: 73fb9968007f83d35f1e6fe185d9e376fa81fc7683fe822426b290632ca8de24 --- Sandbox Provider 通过交付一个引导文件来启动托管 Runtime。本文负责 Provider 到 Runtime 的启动输入。类型与验证器位于 [`internal/runtimebootstrap`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/runtimebootstrap/bootstrap.go)；Go provider 使用 [`runtime_bootstrap.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/runtime_bootstrap.go) 中的 `sandbox.Bootstrap.RuntimeConnection()` 构造输入，SDK helper 原样转发序列化对象。provider 不读取或写入 Runtime 的私有认证存储。 @@ -19,6 +19,7 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json | `version` | 精确的引导版本 `runtimebootstrap.Version` | | `core_url` | 以 `/api/v1` 结尾的 HTTP(S) 机器 API 基址，不含凭据、查询或片段 | | `device_id` | Core 签发的 daemon 身份的规范非零 UUID | +| `harness` | 会话固定选择的 Harness 标识；托管 Runtime 只探测并注册此 Harness | | `credential` | Core 签发的非空 daemon 凭据，不含空白或 NUL | 解码器拒绝未知、重复、缺失和大小写别名字段，拒绝其他版本及超过 `runtimebootstrap.MaxBytes`（16 KiB）的文档。错误不包含提交的值。文件缺失或格式错误时，daemon 在连接前失败。 @@ -29,6 +30,8 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json provider 创建账户、挂载和工作区，交付该文件，设置 Runtime 的资源与 Environment 绑定配置，然后以无特权 Runtime 账户启动 daemon。Docker 将文件写入 Runtime 拥有的 home volume；microsandbox 和 E2B 在启动同一命令之前交付文件。 +Core 从分配所属会话派生 `harness`。组合镜像可以包含多个 Harness，但托管 Runtime 只探测所选 Harness；未知、缺失或不可用的选择会失败，不探测其他 Harness。启动协议版本为 2，Core、提供商辅助程序和新启动的 Runtime 镜像需配套升级；已有分配保留原启动文件和 Runtime。自托管安装仍探测已安装的 Harness 集合。 + Runtime 验证输入，并负责认证与连接。启动成功仅证明交付完成：经过认证的连接、已准备的能力和执行就绪是 [Core–Runtime 协议](runtime-protocol.md) 下的独立观测；[Sandbox Provider 指南](sandbox-provider.md#four-distinct-readiness-facts) 列出各自证明的事实。 自托管 executor 和运维人员供应的设备通过其他方式获取 daemon 身份；[机器连接 API](../../contracts/agents-api/zh/machine-api.md#credentials) 列出所有凭据来源。它们都进入同一 Runtime 执行循环。 diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 67e1d5f25..90b262e43 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: ec6c807e262a37d2ecd456101dadf7ae4818f1bd711b4af5f08e27c9d1690d83 +source_hash: a69ba63ef2b5bd883e0a235807df9d270c23dab9b6df11cf6019534281ced8e9 --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -162,6 +162,8 @@ Node readiness 绑定到精确 generation、当前连接和 owner epoch。持久 ### Allocation 生命周期 {#allocation-lifecycle} +Core 在 `Bootstrap.Harness` 中传递所属会话固定选择的 Harness；所有提供商都将其投影到 [Runtime 启动输入](./runtime-bootstrap.md)，不自行选择实现。 + allocation、专用 daemon credential digest 和精确 Session binding 在 `Create` 前、execution lease 与 Session lock 下原子提交。只有新 allocation receipt 允许 `Create`；重试和 Core 重启观察同一 reference，不重放或轮换凭据。allocation 是私有计算资源所有权，与公开 Environment connection 和原生 readiness 独立；adapter 验证 bootstrap completion，Core 不从 engine 或 provider name 推断。 配置 provider 后，Worker 扫描已提交且没有 allocation 的 pending hosted Environment，涵盖空闲 Session 创建以及 commit 与 bootstrap 之间中断后的恢复；已有 allocation 不重新进入此路径。scan 有界，由 lifecycle owner 串行化，不需要调用方操作。没有 Turn 的初始预约让 Session 保持空闲，daemon 连接不被当作原生 readiness。同一 scan 在验证精确 Session、device binding 和已结算 bootstrap 后，发布带持久 generation 的认证连接观测。 @@ -174,6 +176,8 @@ Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 每个注册 node 有一个串行 lifecycle worker，负责 gate、allocation 与 pending cursor、connection 和 wake hint；E2B allocation 共享一个没有 node 的串行 lifecycle。薄 coordinator 发现 node 并关闭 worker，数据库、provider 或等待操作期间不持有 map mutex。worker 独立推进，因此一个在线 node 的 provider 卡住不会阻塞其他 node：lifecycle 并发为每 node 一项操作，随 node 数量增长。离线 worker 保留，因此其保留资源在重连后仍可观察。 +托管 Environment 提交后，Core 向其 placement 的生命周期 worker 发送有界提示。已提交的待处理输入也会发送提示，以恢复遗漏的创建通知。提示沿用串行 gate、租约、容量检查和一次性 allocation 收据，不在准入处理器内直接创建资源。每个正常维护周期最多允许一次额外提示扫描；周期扫描恢复遗漏或合并的提示。现有 allocation 观察仍先于待创建资源扫描，因此同一节点上缓慢的观察可能延迟新 Environment。 + allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan join 尚未释放的已提交 placement。每个 node 推进自己的 cursor，包括越过失败观测，并在末尾回绕一次。direct provisioning 在进入该 node gate 前解析 tenant 范围内 placement，已有 allocation 必须与其一致；Core 不选择另一 node。 释放 execution lease 前，coordinator 停止接受工作，取消并排空每个 node worker 和 direct caller。lease 丢失影响全部；普通 provider failure 限于所属 node。计划 deployment drain 或 node retirement 通过五秒有界 lease gate，在 leased operation 之间同步取消 lifecycle context，包括活动 manual reconcile，不仅为了改变配置就取消进行中的 leased query。失败的 cancellation fence 关闭 manager admission 并报告 owner failure。失败的 retirement 保留原 lifecycle identity 和 gate，直到 owner shutdown，drain barrier 保持关闭。Session lock、deployment capacity transaction 和 revision-checked receipt 仍是权威依据，外部操作不持有数据库锁。 diff --git a/internal/runtimebootstrap/bootstrap.go b/internal/runtimebootstrap/bootstrap.go index bab5e7b16..1beabb40f 100644 --- a/internal/runtimebootstrap/bootstrap.go +++ b/internal/runtimebootstrap/bootstrap.go @@ -14,7 +14,7 @@ import ( "github.com/google/uuid" ) -const Version = 1 +const Version = 2 const MaxBytes = 16 * 1024 var ErrInvalid = errors.New("invalid Runtime bootstrap input") @@ -27,12 +27,13 @@ type Connection struct { CoreURL string `json:"core_url"` DeviceID string `json:"device_id"` Credential string `json:"credential"` + Harness string `json:"harness"` } func (c Connection) Validate() error { u, err := url.Parse(c.CoreURL) id, idErr := uuid.Parse(c.DeviceID) - if c.Version != Version || err != nil || (u.Scheme != "https" && u.Scheme != "http") || + if c.Version != Version || !ValidHarness(c.Harness) || err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.RawPath != "" || u.Path != "/api/v1" || strings.ContainsAny(c.CoreURL, "?#") || strings.ContainsFunc(c.CoreURL, unicode.IsSpace) || @@ -83,6 +84,8 @@ func Decode(raw []byte) (Connection, error) { err = d.Decode(&c.DeviceID) case "credential": err = d.Decode(&c.Credential) + case "harness": + err = d.Decode(&c.Harness) default: return Connection{}, ErrInvalid } @@ -93,8 +96,21 @@ func Decode(raw []byte) (Connection, error) { if token, err = d.Token(); err != nil || token != json.Delim('}') { return Connection{}, ErrInvalid } - if len(seen) != 4 || d.Decode(new(any)) != io.EOF || c.Validate() != nil { + if len(seen) != 5 || d.Decode(new(any)) != io.EOF || c.Validate() != nil { return Connection{}, ErrInvalid } return c, nil } + +// ValidHarness checks the identifier syntax shared by startup projections. +func ValidHarness(kind string) bool { + if len(kind) == 0 || len(kind) > 64 || kind[0] < 'a' || kind[0] > 'z' { + return false + } + for _, c := range kind { + if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '_') { + return false + } + } + return true +} diff --git a/internal/runtimebootstrap/bootstrap_test.go b/internal/runtimebootstrap/bootstrap_test.go index 132e5659a..7fb194495 100644 --- a/internal/runtimebootstrap/bootstrap_test.go +++ b/internal/runtimebootstrap/bootstrap_test.go @@ -7,7 +7,7 @@ import ( ) func TestConnectionRoundTrip(t *testing.T) { - input := Connection{Version: Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-credential"} + input := Connection{Version: Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-credential", Harness: "codex"} raw, err := input.Marshal() if err != nil { t.Fatal(err) @@ -19,11 +19,11 @@ func TestConnectionRoundTrip(t *testing.T) { } func TestConnectionRejectsAmbiguousOrForeignInput(t *testing.T) { - good := `{"version":1,"core_url":"https://core.example/api/v1","device_id":"da912024-1543-4242-a2c1-5f4f7ebbc6c7","credential":"test-secret"}` + good := `{"version":2,"harness":"codex","core_url":"https://core.example/api/v1","device_id":"da912024-1543-4242-a2c1-5f4f7ebbc6c7","credential":"test-secret"}` for name, raw := range map[string]string{ "null": "null", "array": "[]", "trailing": good + "{}", - "duplicate": strings.Replace(good, `"version":1`, `"version":1,"version":1`, 1), - "unknown": strings.Replace(good, `"version":1`, `"extra":1,"version":1`, 1), + "duplicate": strings.Replace(good, `"version":2`, `"version":2,"version":2`, 1), + "unknown": strings.Replace(good, `"version":2`, `"extra":1,"version":2`, 1), "case": strings.Replace(good, "credential", "Credential", 1), "old auth": `{"server_url":"https://core.example","runtime_id":"old","runner_credential":"test-secret"}`, "oversized": strings.Repeat(" ", MaxBytes) + good, @@ -35,9 +35,10 @@ func TestConnectionRejectsAmbiguousOrForeignInput(t *testing.T) { }) } for field, values := range map[string][]any{ - "version": {nil, 0, 2, "1"}, + "version": {nil, 0, 1, 3, "2"}, "core_url": {"http://user:pass@core.example/api/v1", "https://core.example/api/v1?", "https://core.example/api/v1#", "https://core.example", "https://core.example/api/v1/", "https://core.example/api/v1?q=1", ""}, "device_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, + "harness": {nil, "", "Codex", "unknown-kind", "../codex", strings.Repeat("x", 65)}, "credential": {nil, "", "test\nsecret", "test\x00secret"}, } { for _, value := range values { diff --git a/services/core/cmd/server/managed_generations_test.go b/services/core/cmd/server/managed_generations_test.go index 7e5855f6b..4b1c57aac 100644 --- a/services/core/cmd/server/managed_generations_test.go +++ b/services/core/cmd/server/managed_generations_test.go @@ -29,7 +29,7 @@ q=json.load(sys.stdin) with (pathlib.Path(q['Config']['StateDir'])/'requests').open('a') as f: f.write(json.dumps(q)+'\n') info=dict(q['Reference'],State='running',ProviderID='owned',CreateSettled=True) if q['Operation']=='kill': info['State']='absent' -print(json.dumps({'Version':1,'Info':info})) +print(json.dumps({'Version':2,'Info':info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -132,7 +132,7 @@ code = '' if k == 'revoked': code = 'unauthorized' elif k == 'unconfirmed': code = 'unconfirmed' elif not (k.startswith('team-a') and template in ('owned-a', 'new-a') or k == 'team-b' and template == 'public-b'): code = 'team_mismatch' -result = {'Version': 1, 'ErrorCode': code} +result = {'Version': 2, 'ErrorCode': code} if not code: result['DeploymentValid'] = True if q['Operation'] == 'validate_deployment': diff --git a/services/core/cmd/server/managed_setup_preflight_test.go b/services/core/cmd/server/managed_setup_preflight_test.go index dc977e41a..884b89385 100644 --- a/services/core/cmd/server/managed_setup_preflight_test.go +++ b/services/core/cmd/server/managed_setup_preflight_test.go @@ -19,7 +19,7 @@ import ( func TestE2BRejectedSpecificationHasSafeActionableDiagnostic(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") @@ -49,12 +49,12 @@ op = q['Operation'] with (pathlib.Path(q['Config']['StateDir']) / 'operations').open('a') as log: log.write(op + '\n') if op == 'validate_deployment': - print(json.dumps({'Version': 1, 'ErrorCode': 'invalid'})) + print(json.dumps({'Version': 2, 'ErrorCode': 'invalid'})) else: info = dict(q['Reference'], ProviderID='owned-compute', State='stopped', CreateSettled=True) if op == 'kill': info.update(ProviderID='', State='absent') - print(json.dumps({'Version': 1, 'Info': info})) + print(json.dumps({'Version': 2, 'Info': info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -101,7 +101,7 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { } helper := filepath.Join(t.TempDir(), "provider") requests := filepath.Join(state, "requests") - script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" + script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -133,7 +133,7 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index 77d1869fd..0b7635c78 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -11,7 +11,7 @@ MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 1 +PROTOCOL_VERSION = 2 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] diff --git a/services/core/deploy/e2b/managed_init_test.py b/services/core/deploy/e2b/managed_init_test.py index d9017ba82..f38a92ffd 100644 --- a/services/core/deploy/e2b/managed_init_test.py +++ b/services/core/deploy/e2b/managed_init_test.py @@ -16,7 +16,7 @@ def payload(): value = {key: str(uuid4()) for key in ['InstallationID', 'TenantID', 'EnvironmentID', 'AllocationID', 'SessionID', 'DeviceID']} - return dict(value, RuntimeBootstrap={'version': 1, 'core_url': 'https://core.example/api/v1', + return dict(value, RuntimeBootstrap={'version': 2, 'harness': 'codex', 'core_url': 'https://core.example/api/v1', 'device_id': value['DeviceID'], 'credential': 'private-managed-token'}, NetworkAccess='restricted', AllowedDomains=['example.com']) diff --git a/services/core/internal/execution/environment_admission.go b/services/core/internal/execution/environment_admission.go index 0f34d9f47..18ec95ac5 100644 --- a/services/core/internal/execution/environment_admission.go +++ b/services/core/internal/execution/environment_admission.go @@ -99,7 +99,7 @@ func (w *Worker) submitEnvironmentInputs(ctx context.Context, session sessions.S return nil, err } w.wakeScheduler() - if reservation.State == sessions.EnvironmentInputPending && !reservation.IsInitial { + if reservation.State == sessions.EnvironmentInputPending { w.hintRuntimeWake(ctx, session) } ticker := time.NewTicker(250 * time.Millisecond) diff --git a/services/core/internal/execution/runtime_fresh_hint_test.go b/services/core/internal/execution/runtime_fresh_hint_test.go new file mode 100644 index 000000000..334052b4e --- /dev/null +++ b/services/core/internal/execution/runtime_fresh_hint_test.go @@ -0,0 +1,208 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + projectpg "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/projectpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/projects" + "sync" + "sync/atomic" + "testing" + "time" + + v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" + "github.com/google/uuid" +) + +type freshHintProvider struct { + sandbox.SandboxProvider + creates atomic.Int32 + created chan struct{} +} + +type hintSessionReader struct { + sessions.Reader + environment sessions.Environment +} + +func (r hintSessionReader) GetSessionEnvironment(ctx context.Context, _, _ string) (sessions.Environment, error) { + return r.environment, ctx.Err() +} + +func TestFreshHintRoutesAndPreservesLifecycleGuards(t *testing.T) { + for _, scenario := range []string{"direct", "node", "closed", "switching", "cancelled", "self_hosted", "released_placement", "lookup_failed", "suspended"} { + t.Run(scenario, func(t *testing.T) { + m := testRuntimeManager(t) + m.setupGate = make(chan struct{}, 1) + m.config = RuntimeProvider{InstallationID: uuid.NewString(), Mode: "direct", Provider: &freshHintProvider{}} + nodeID := "" + if scenario == "node" || scenario == "released_placement" { + m.config.ProviderKind = "docker" + m.config.Mode = "nodes" + nodeID = uuid.NewString() + } + node, err := m.node(nodeID) + if err != nil { + t.Fatal(err) + } + environment := sessions.Environment{ID: uuid.NewString(), Configuration: json.RawMessage(`{"type":"openai_hosted"}`)} + if scenario == "self_hosted" { + environment.Configuration = json.RawMessage(`{"type":"self_hosted","workspace_directory":"/workspace"}`) + } + if scenario == "suspended" { + environment.Initialization = "complete" + m.config.Suspension = &RuntimeSuspensionPolicy{} + } + reader := &strictDeploymentReader{t: t, + environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + if scenario == "lookup_failed" { + return deployment.Allocation{}, errors.New("database unavailable") + } + if scenario == "suspended" { + return deployment.Allocation{ProviderKey: m.config.InstallationID, State: "running", CreateSettled: true, ComputePhase: "suspended"}, nil + } + return deployment.Allocation{}, deployment.ErrNotFound + }, + lifecyclePlacement: func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) { + return deployment.LifecyclePlacement{Provider: m.config.ProviderKind, PlacementNodeID: nodeID, PlacementReleased: scenario == "released_placement"}, nil + }, + } + m.deploymentService, _ = deploymentOperations(t, &strictDeploymentStorage{t: t}, reader, &strictExecutionStorage{t: t}) + worker := &Worker{runtimes: m, dispatcher: &Dispatcher{SessionsReader: hintSessionReader{environment: environment}, DeploymentReader: reader}} + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + switch scenario { + case "closed": + m.closed = true + case "switching": + m.switching = true + case "cancelled": + cancel() + } + worker.hintRuntimeWake(ctx, sessions.Session{TenantID: uuid.NewString(), ID: uuid.NewString()}) + want := 0 + if scenario == "direct" || scenario == "node" || scenario == "suspended" { + want = 1 + } + if got := len(node.lifecycle.wakeHints); got != want { + t.Fatalf("hints = %d, want %d", got, want) + } + }) + } +} + +func (p *freshHintProvider) Create(_ context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { + p.creates.Add(1) + select { + case p.created <- struct{}{}: + default: + } + return sandbox.Info{Reference: b.Reference, ProviderID: b.AllocationID, State: "running", BootstrapComplete: true, CreateSettled: true}, nil +} + +func TestFreshEnvironmentHintProvisionsWithoutMaintenanceTick(t *testing.T) { + for _, mode := range []string{"create", "recovered_input", "recovered_initial"} { + t.Run(mode, func(t *testing.T) { + owner, deployments, reader, pool := resetManagerDB(t, nil) + installation := initializeE2BDeployment(t, owner) + sessionReader, s := testSessions(t, pool, pgtest.CredentialKey(t)) + provider := &freshHintProvider{created: make(chan struct{}, 1)} + m := testRuntimeManager(t) + m.setupGate = make(chan struct{}, 1) + m.sessions, m.sessionExecution = sessionReader, owner.Sessions + m.deployment, m.deploymentService, m.deploymentReader = owner.Deployment, deployments, reader + m.lease, m.registry = owner.Lease, runtimegateway.NewRegistry() + m.config = RuntimeProvider{Mode: "direct", InstallationID: installation, CoreURL: fixturePublicURL + "/api/v1", Provider: provider} + worker := &Worker{lease: owner.Lease, runtimes: m, dispatcher: &Dispatcher{Sessions: s, sessionExecution: owner.Sessions, SessionsReader: sessionReader, DeploymentReader: reader, notifications: &executionNotifications{}}, scheduleWake: make(chan struct{}, 1)} + node, err := m.node("") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + done, scans := make(chan error, 1), make(chan struct{}, 4) + // No value is ever sent to ticks: provisioning must come from a hint. + ticks := make(chan time.Time) + go func() { + done <- runRuntimeMaintenance(ctx, ticks, node.lifecycle.wakeHints, func(ctx context.Context) error { + err := node.lifecycle.reconcile(ctx) + scans <- struct{}{} + return err + }) + }() + t.Cleanup(func() { cancel(); <-done }) + <-scans // Startup scan finishes before any Session is committed. + projectID := uuid.NewString() + audit := adminaudit.WithSource(ctx, adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) + management, err := projects.NewService(projectpg.New(pgunit.NewPool(pool))) + if err != nil { + t.Fatal(err) + } + project, err := management.CreateProject(audit, projects.CreateProject{ID: projectID, Name: "Fresh hint"}) + if err != nil { + t.Fatal(err) + } + tenant := project.TenantID + input := sessions.CreateSession{Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted"}}`), ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ExecutionSourceSession} + messages := []sessions.Input{{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"fixture"}]}]}`)}} + if mode == "recovered_initial" { + input.InitialInputs = messages + } + var creation sessions.Creation + switch mode { + case "create": + creation, err = worker.CreateSession(ctx, tenant, input) + case "recovered_input", "recovered_initial": + creation, err = s.CreateSession(ctx, tenant, input) + } + if err != nil { + t.Fatal(err) + } + session := creation.Session + if mode == "recovered_input" || mode == "recovered_initial" { + key := uuid.NewString() + if mode == "recovered_initial" { + if err := pool.QueryRow(ctx, "SELECT idempotency_key FROM environment_input_reservations WHERE session_id=$1 AND is_initial", session.ID).Scan(&key); err != nil { + t.Fatal(err) + } + } + inputDone := make(chan error, 20) + inputCtx, stopInput := context.WithCancel(ctx) + for range 20 { + go func() { + _, inputErr := worker.submitEnvironmentInputs(inputCtx, session, key, messages) + inputDone <- inputErr + }() + } + t.Cleanup(func() { + stopInput() + for range 20 { + <-inputDone + } + }) + } + select { + case <-provider.created: + case <-time.After(2 * time.Second): + t.Fatal("committed Environment waited for a maintenance tick") + } + var requests sync.WaitGroup + for range 20 { + requests.Go(func() { worker.hintRuntimeWake(ctx, session) }) + } + requests.Wait() + <-scans // Wait for the hinted scan to settle its durable allocation. + if got := provider.creates.Load(); got != 1 { + t.Fatalf("concurrent hints issued %d Creates", got) + } + }) + } +} diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 49e23afd4..8b0eea006 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -202,6 +202,10 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p } } key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} + session, err := r.sessions.GetSession(ctx, tenant, environmentValue.SessionID) + if err != nil { + return deployment.Allocation{}, err + } secret := make([]byte, 32) if _, err := rand.Read(secret); err != nil { return deployment.Allocation{}, err @@ -222,7 +226,7 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p } info, err := provider.Create(ctx, sandbox.Bootstrap{ Reference: runtimeReference(owner), SessionID: owner.SessionID, DeviceID: owner.DeviceID, - Workspace: workspace, CoreURL: r.config.CoreURL, Credential: token, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, + Workspace: workspace, CoreURL: r.config.CoreURL, Credential: token, Harness: session.Engine, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, }) if info.Reference == runtimeReference(owner) && info.CreateSettled && info.State == "absent" { record, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) diff --git a/services/core/internal/execution/runtime_wake_hint.go b/services/core/internal/execution/runtime_wake_hint.go index ed7c9f1d6..b25fb4d03 100644 --- a/services/core/internal/execution/runtime_wake_hint.go +++ b/services/core/internal/execution/runtime_wake_hint.go @@ -2,14 +2,15 @@ package execution import ( "context" + "errors" "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -// A hint only accelerates observation of an already committed input. Lookup or -// delivery failure leaves that input for the normal maintenance scan. +// Hints only accelerate lifecycle work for committed Environments and inputs. +// Lookup or delivery failure leaves that work for the normal maintenance scan. func (w *Worker) hintRuntimeWake(ctx context.Context, session sessions.Session) { r := w.runtimes if r == nil { @@ -19,16 +20,40 @@ func (w *Worker) hintRuntimeWake(ctx context.Context, session sessions.Session) config := r.config available := !r.closed && !r.switching r.mu.Unlock() - if !available || config.Suspension == nil || r.ctx.Err() != nil { + if !available || r.ctx.Err() != nil { return } lookup, cancel := context.WithTimeout(ctx, time.Second) defer cancel() environment, err := w.dispatcher.SessionsReader.GetSessionEnvironment(lookup, session.TenantID, session.ID) - if err != nil || environment.Initialization != "complete" { + if err != nil { + return + } + placement, err := parseEnvironmentPlacement(environment.Configuration) + if err != nil || placement.Type != "openai_hosted" { return } owner, err := w.dispatcher.DeploymentReader.EnvironmentAllocation(lookup, deployment.AllocationKey{TenantID: session.TenantID, EnvironmentID: environment.ID}) + if errors.Is(err, deployment.ErrNotFound) { + // Placement exists before allocation. Route through its lifecycle owner; + // the scan still enforces lease, capacity and one-shot Create receipts. + nodeID, err := r.deploymentService.LifecycleNode(lookup, session.TenantID, environment.ID) + if err != nil || lookup.Err() != nil { + return + } + node, err := r.node(nodeID) + if err != nil { + return + } + select { + case node.lifecycle.wakeHints <- struct{}{}: + default: + } + return + } + if config.Suspension == nil || environment.Initialization != "complete" { + return + } if err != nil || owner.ProviderKey != config.InstallationID || owner.State != "running" || !owner.CreateSettled || owner.SessionDeleted || owner.Expired { return diff --git a/services/core/internal/execution/runtime_workspace_test.go b/services/core/internal/execution/runtime_workspace_test.go index 54ca2141c..8c52d590f 100644 --- a/services/core/internal/execution/runtime_workspace_test.go +++ b/services/core/internal/execution/runtime_workspace_test.go @@ -16,6 +16,10 @@ import ( type workspaceEnvironmentReader struct{ sessions.Reader } +func (workspaceEnvironmentReader) GetSession(context.Context, string, string) (sessions.Session, error) { + return sessions.Session{Engine: "codex"}, nil +} + func (workspaceEnvironmentReader) GetEnvironment(context.Context, string, string) (sessions.Environment, error) { return sessions.Environment{Configuration: []byte(`{"type":"openai_hosted"}`)}, nil } diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 92fabc153..6d67ff3cb 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -134,6 +134,9 @@ func (w *Worker) CreateSession(ctx context.Context, tenant string, input session return sessions.Creation{}, err } creation, err := w.dispatcher.Sessions.CreateSession(ctx, tenant, input) + if err == nil { + w.hintRuntimeWake(ctx, creation.Session) + } if err == nil && len(input.InitialInputs) > 0 { w.wakeScheduler() } @@ -264,6 +267,8 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { rescanOnCompletion = false case <-w.scheduleWake: rescanOnCompletion = true + case <-w.dispatcher.Registry.CapabilityHints(): + rescanOnCompletion = true case <-ticker.C: maintenance = true } @@ -293,10 +298,7 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { w.observeSchedulerPoll(0, nil) continue } - if !maintenance { - schedule.nextEnvironmentScan = time.Time{} - } - work, err := schedule.selectWork(ctx, w, devices, active) + work, err := schedule.selectWork(ctx, w, devices, active, !maintenance) w.observeSlots(len(active)) if err != nil { w.observeSchedulerPoll(0, err) diff --git a/services/core/internal/execution/worker_capability_test.go b/services/core/internal/execution/worker_capability_test.go new file mode 100644 index 000000000..d4d76c5bd --- /dev/null +++ b/services/core/internal/execution/worker_capability_test.go @@ -0,0 +1,63 @@ +package execution + +import ( + "context" + "encoding/json" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" +) + +type candidateQueryObserver struct { + sessions.Reader + scans atomic.Int32 +} + +func (o *candidateQueryObserver) ListExecutionWork(context.Context, string, []string, []string) ([]sessions.ExecutionWork, error) { + return nil, nil +} +func (o *candidateQueryObserver) ListEnvironmentInputWork(context.Context, string, []string) ([]sessions.EnvironmentInputWork, error) { + o.scans.Add(1) + return nil, nil +} + +func TestSchedulingHintScansBeforeDeadlineAndRestoresPollingDelay(t *testing.T) { + observer := &candidateQueryObserver{} + worker := &Worker{dispatcher: &Dispatcher{SessionsReader: observer}, concurrency: 1} + schedule := workerSchedule{nextEnvironmentScan: time.Now().Add(time.Hour)} + for _, hinted := range []bool{false, true, false} { + if _, err := schedule.selectWork(t.Context(), worker, nil, map[string]bool{}, hinted); err != nil { + t.Fatal(err) + } + } + if observer.scans.Load() != 1 { + t.Fatalf("candidate scans = %d; hint must bypass the deadline once", observer.scans.Load()) + } +} + +type phaseSessionReader struct{ sessions.Reader } + +func (phaseSessionReader) GetSessionEnvironment(context.Context, string, string) (sessions.Environment, error) { + return sessions.Environment{ID: "environment", Configuration: json.RawMessage(`{"type":"openai_hosted"}`)}, nil +} +func (phaseSessionReader) GetSessionDevice(context.Context, string, string) (sessions.ExecutionDevice, error) { + panic("non-running compute must not reach device eligibility") +} +func TestCapabilityHintDoesNotBypassComputePhase(t *testing.T) { + for _, phase := range []string{"quiescing", "suspending", "suspended", "restoring", "waking"} { + t.Run(phase, func(t *testing.T) { + reader := &strictDeploymentReader{t: t, environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + return deployment.Allocation{ComputePhase: phase}, nil + }} + worker := &Worker{dispatcher: &Dispatcher{SessionsReader: phaseSessionReader{}, DeploymentReader: reader}} + session := sessions.Session{Configuration: json.RawMessage(`{"environment":{"type":"openai_hosted"}}`)} + ready, err := worker.bindSessionDevice(t.Context(), session, func(string) bool { t.Fatal("phase bypassed"); return true }) + if err != nil || ready { + t.Fatal("non-running compute selected", ready, err) + } + }) + } +} diff --git a/services/core/internal/execution/worker_schedule.go b/services/core/internal/execution/worker_schedule.go index bf3a821cc..c38c07077 100644 --- a/services/core/internal/execution/worker_schedule.go +++ b/services/core/internal/execution/worker_schedule.go @@ -20,7 +20,10 @@ type scheduledWork struct { reservationID string } -func (s *workerSchedule) selectWork(ctx context.Context, w *Worker, devices []string, active map[string]bool) ([]scheduledWork, error) { +func (s *workerSchedule) selectWork(ctx context.Context, w *Worker, devices []string, active map[string]bool, hinted bool) ([]scheduledWork, error) { + if hinted { + s.nextEnvironmentScan = time.Time{} + } turns, err := w.dispatcher.SessionsReader.ListExecutionWork(ctx, s.turnCursor, []string{sessions.TurnQueued}, devices) if err != nil { return nil, err diff --git a/services/core/internal/runtimegateway/capability_hint_test.go b/services/core/internal/runtimegateway/capability_hint_test.go new file mode 100644 index 000000000..ba4fbd9e3 --- /dev/null +++ b/services/core/internal/runtimegateway/capability_hint_test.go @@ -0,0 +1,138 @@ +package runtimegateway + +import ( + "context" + "encoding/json" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "testing" + "time" +) + +func expectCapabilityHint(t *testing.T, reg *Registry, want bool) { + t.Helper() + select { + case <-reg.CapabilityHints(): + if !want { + t.Fatal("unexpected capability hint") + } + default: + if want { + t.Fatal("missing capability hint") + } + } +} + +func TestCapabilityHintsRequireCurrentAvailableSnapshot(t *testing.T) { + reg := NewRegistry() + old := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(old) + expectCapabilityHint(t, reg, false) // Transport alone is insufficient. + publishCapabilityHeartbeat(t, old, []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: false}}) + expectCapabilityHint(t, reg, false) + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: true}} + publishCapabilityHeartbeat(t, old, kinds) + expectCapabilityHint(t, reg, true) + publishCapabilityHeartbeat(t, old, kinds) + expectCapabilityHint(t, reg, false) // Ordinary heartbeats do not rescan. + newer := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(newer) + kinds[0].Version = "changed" + publishCapabilityHeartbeat(t, old, kinds) + expectCapabilityHint(t, reg, false) // Superseded socket cannot accelerate work. + publishCapabilityHeartbeat(t, newer, kinds) + expectCapabilityHint(t, reg, true) +} + +func TestCapabilityHintsCoalesceAndIgnoreDisconnectedPeers(t *testing.T) { + reg := NewRegistry() + s := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(s) + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: true}} + publishCapabilityHeartbeat(t, s, kinds) + kinds[0].Capabilities.Streaming = true + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, true) + expectCapabilityHint(t, reg, false) + reg.Deregister(s) + kinds[0].Capabilities.Steering = true + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, false) +} + +func TestInvalidHeartbeatCannotWakeScheduler(t *testing.T) { + reg := NewRegistry() + s := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(s) + s.handleHeartbeat(proto.Envelope{Type: proto.TypeHeartbeat, Payload: json.RawMessage(`{"ts":1,"active_requests":0,"supported_agent_kinds":[{"kind":"fake","available":true,"capabilities":{"streaming":"invented"}}]}`)}) + expectCapabilityHint(t, reg, false) + if !s.IsClosed() { + t.Fatal("malformed capabilities did not close transport") + } +} + +func publishCapabilityHeartbeat(t *testing.T, s *Session, kinds []runtimedevice.SupportedAgentKind) { + t.Helper() + advertised := make([]proto.SupportedAgentKind, len(kinds)) + for i, k := range kinds { + advertised[i] = proto.SupportedAgentKind{Kind: k.Kind, Available: k.Available, Version: k.Version, + Capabilities: prototest.Capabilities(proto.AgentKindCapabilities{ + Streaming: proto.CapabilityFromBool(k.Capabilities.Streaming), + Steering: proto.CapabilityFromBool(k.Capabilities.Steering), + })} + } + env, err := proto.NewEnvelope(proto.TypeHeartbeat, "", proto.HeartbeatPayload{SupportedAgentKinds: advertised}) + if err != nil { + t.Fatal(err) + } + s.handleHeartbeat(env) +} + +type capabilityAuthorityStore struct { + HeartbeatTouch + deleted bool + err error +} + +func (a *capabilityAuthorityStore) TouchAgentDaemonHeartbeat(context.Context, runtimedevice.Heartbeat) (runtimedevice.HeartbeatStatus, error) { + return runtimedevice.HeartbeatStatus{Deleted: a.deleted}, a.err +} +func TestCapabilityConfirmationRetriesWithoutObservingRejectedAuthority(t *testing.T) { + for _, mode := range []string{"error", "deleted", "draining"} { + t.Run(mode, func(t *testing.T) { + reg := NewRegistry() + s := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(s) + t.Cleanup(func() { s.Close("test finished") }) + authority := &capabilityAuthorityStore{deleted: mode != "error"} + if mode == "error" { + authority.err = errors.New("private credential must not be observed") + } + s.heartbeat = authority + if mode == "draining" { + s.credentialHash = "original-hash" + s.archivedCancellations = &receiptStore{receipt: runtimedevice.ArchivedCancellationReceipt{RunID: "run", Deadline: time.Now().Add(time.Minute)}} + release, err := s.TrackExecutionDelivery("run") + if err != nil { + t.Fatal(err) + } + t.Cleanup(release) + } + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: true}} + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, false) + if mode == "draining" && s.IsClosed() { + t.Fatal("existing receipt drain was interrupted") + } + if mode == "error" { + authority.err = nil + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, true) + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, false) + } + }) + } +} diff --git a/services/core/internal/runtimegateway/registry.go b/services/core/internal/runtimegateway/registry.go index b5984821e..c30084280 100644 --- a/services/core/internal/runtimegateway/registry.go +++ b/services/core/internal/runtimegateway/registry.go @@ -12,6 +12,8 @@ import ( "errors" "sync" "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" ) // ErrDeviceNotRegistered is returned by Registry lookups when a caller @@ -36,16 +38,18 @@ type Registry struct { // is inserted into byDevice. Buffered(1) so a Register that // happens between WaitForDevice registering and selecting on the // chan still wakes the waiter. - waiters map[string][]chan *Session + waiters map[string][]chan *Session + capabilityHints chan struct{} } // NewRegistry returns an empty registry. The zero value would also // work but the constructor avoids accidental nil-map panics. func NewRegistry() *Registry { return &Registry{ - byDevice: map[string]*Session{}, - byRun: map[string]*Session{}, - waiters: map[string][]chan *Session{}, + byDevice: map[string]*Session{}, + byRun: map[string]*Session{}, + waiters: map[string][]chan *Session{}, + capabilityHints: make(chan struct{}, 1), } } @@ -237,3 +241,29 @@ func (r *Registry) removeWaiter(deviceID string, ch chan *Session) { r.waiters[deviceID] = filtered } } + +// CapabilityHints coalesces capability changes for the single execution Worker. +// A hint grants no execution authority; the Worker rechecks its normal gates. +func (r *Registry) CapabilityHints() <-chan struct{} { return r.capabilityHints } + +func (r *Registry) observeCapabilitySnapshot(sess *Session, kinds []runtimedevice.SupportedAgentKind) bool { + r.mu.RLock() + defer r.mu.RUnlock() + if r.byDevice[sess.DeviceID] != sess || sess.IsClosed() { + return false + } + available := 0 + for _, kind := range kinds { + if kind.Available { + available++ + } + } + if available == 0 { + return true + } + select { + case r.capabilityHints <- struct{}{}: + default: + } + return true +} diff --git a/services/core/internal/runtimegateway/session.go b/services/core/internal/runtimegateway/session.go index 9032c2f48..ebcf0da07 100644 --- a/services/core/internal/runtimegateway/session.go +++ b/services/core/internal/runtimegateway/session.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "strings" "sync" "time" @@ -100,9 +101,10 @@ type Session struct { // supportedKinds is the latest daemon-advertised agent_kind snapshot, // updated from heartbeat frames and read by the connector before // sending execution_prepare so unsupported engines fail on the server. - kindsMu sync.RWMutex - kindsSeen bool - supportedKinds []runtimedevice.SupportedAgentKind + kindsMu sync.RWMutex + kindsSeen bool + supportedKinds []runtimedevice.SupportedAgentKind + capabilityObservationPending bool // Subscribers keyed by runID. The read loop only sends on these // channels; Unsubscribe is the only place that closes them. @@ -222,11 +224,25 @@ func (s *Session) setSupportedAgentKinds(kinds []runtimedevice.SupportedAgentKin copyKinds := make([]runtimedevice.SupportedAgentKind, len(kinds)) copy(copyKinds, kinds) s.kindsMu.Lock() + changed := !s.kindsSeen || !reflect.DeepEqual(s.supportedKinds, copyKinds) s.kindsSeen = true s.supportedKinds = copyKinds + if changed { + s.capabilityObservationPending = true + } s.kindsMu.Unlock() } +// A failed authorization or persistence check retains the pending hint +// for the next confirmed heartbeat without changing capability publication. +func (s *Session) observeConfirmedCapabilities(kinds []runtimedevice.SupportedAgentKind) { + s.kindsMu.Lock() + defer s.kindsMu.Unlock() + if s.capabilityObservationPending && s.reg.observeCapabilitySnapshot(s, kinds) { + s.capabilityObservationPending = false + } +} + // Close closes the transport and subscriptions with ErrSessionClosed, then // releases connection ownership. It establishes no execution outcome. Idempotent. func (s *Session) Close(reason string) { @@ -453,6 +469,7 @@ func (s *Session) handleHeartbeat(env proto.Envelope) { kinds := deviceKindsFromHeartbeat(p) s.setSupportedAgentKinds(kinds) if s.heartbeat == nil { + s.observeConfirmedCapabilities(kinds) return } @@ -481,7 +498,9 @@ func (s *Session) handleHeartbeat(env proto.Envelope) { // actual admin action; the daemon only sees it's no longer // the current owner. s.CloseWithCode(CloseRuntimeDeleted, "runtime retired") + return } + s.observeConfirmedCapabilities(kinds) } func deviceKindsFromHeartbeat(p proto.HeartbeatPayload) []runtimedevice.SupportedAgentKind { diff --git a/services/core/internal/sandbox/docker/bootstrap_test.go b/services/core/internal/sandbox/docker/bootstrap_test.go index 31d2845ff..5d0b0f088 100644 --- a/services/core/internal/sandbox/docker/bootstrap_test.go +++ b/services/core/internal/sandbox/docker/bootstrap_test.go @@ -14,7 +14,7 @@ import ( ) func TestBootstrapDeliversOnlyPublicConnectionInput(t *testing.T) { - b := sandbox.Bootstrap{CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-secret"} + b := sandbox.Bootstrap{CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-secret", Harness: "codex"} found := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != "PUT" || !strings.HasSuffix(r.URL.Path, "/containers/test/archive") { diff --git a/services/core/internal/sandbox/docker/contract_test.go b/services/core/internal/sandbox/docker/contract_test.go index 09db3fef8..8105d6d40 100644 --- a/services/core/internal/sandbox/docker/contract_test.go +++ b/services/core/internal/sandbox/docker/contract_test.go @@ -26,7 +26,7 @@ type contractStep struct { func dockerContractFixture(t *testing.T, cancel context.CancelFunc, script func(*Provider, sandbox.Reference) []contractStep) contracttest.Fixture { t.Helper() - b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} var mu sync.Mutex var calls []string var steps []contractStep diff --git a/services/core/internal/sandbox/docker/deployment_test.go b/services/core/internal/sandbox/docker/deployment_test.go index 9df0727d2..7dfa1283c 100644 --- a/services/core/internal/sandbox/docker/deployment_test.go +++ b/services/core/internal/sandbox/docker/deployment_test.go @@ -87,7 +87,7 @@ func TestManagedDriftRejectsBeforeBootstrapAndRetainsCleanup(t *testing.T) { t.Fatal(err) } resources.CPUs = 9 - b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "secret", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "secret", Harness: "codex", NetworkAccess: "enabled"} info, err := p.Create(t.Context(), b) if !errors.Is(err, sandbox.ErrInvalid) { t.Fatal("drift accepted", err) diff --git a/services/core/internal/sandbox/docker/provider_test.go b/services/core/internal/sandbox/docker/provider_test.go index a7f71289b..0df733084 100644 --- a/services/core/internal/sandbox/docker/provider_test.go +++ b/services/core/internal/sandbox/docker/provider_test.go @@ -76,7 +76,7 @@ func TestDockerProviderLifecycle(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() bootstrap := func() sandbox.Bootstrap { - return sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-test-credential", NetworkAccess: "enabled"} + return sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-test-credential", Harness: "codex", NetworkAccess: "enabled"} } b := bootstrap() b.NetworkAccess, b.AllowedDomains = "restricted", []string{"Example.com", "api.example.com"} diff --git a/services/core/internal/sandbox/docker/recovery_test.go b/services/core/internal/sandbox/docker/recovery_test.go index 06ce2e8ae..9cb813598 100644 --- a/services/core/internal/sandbox/docker/recovery_test.go +++ b/services/core/internal/sandbox/docker/recovery_test.go @@ -87,7 +87,7 @@ func TestDockerProviderRecoveryObservations(t *testing.T) { if e != nil { t.Fatal(e) } - b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-recovery-token", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-recovery-token", Harness: "codex", NetworkAccess: "enabled"} direct, e := client.New(client.WithHost("unix:///var/run/docker.sock")) if e != nil { t.Fatal(e) diff --git a/services/core/internal/sandbox/e2b/contract_test.go b/services/core/internal/sandbox/e2b/contract_test.go index b4d64227b..0216cfc23 100644 --- a/services/core/internal/sandbox/e2b/contract_test.go +++ b/services/core/internal/sandbox/e2b/contract_test.go @@ -16,7 +16,7 @@ func (f contractCaller) Call(ctx context.Context, q Request) (Response, error) { func TestProviderContract(t *testing.T) { contracttest.RunFailures(t, func(t *testing.T, s contracttest.Scenario, cancel context.CancelFunc) contracttest.Fixture { p, _, r := fixture(t) - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} var calls []string p.caller = contractCaller(func(ctx context.Context, q Request) (Response, error) { calls = append(calls, q.Operation) @@ -47,7 +47,7 @@ func TestProviderContract(t *testing.T) { func TestProviderContractObservation(t *testing.T) { p, f, r := fixture(t) f.response.Info = &sandbox.Info{Reference: r, ProviderID: "native-owned", State: "running", CreateSettled: true, BootstrapComplete: true} - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} got, err := p.Create(bounded(t), b) contracttest.AssertObservation(t, got, err, r, "native-owned", "running") got, err = p.GetInfo(bounded(t), r) diff --git a/services/core/internal/sandbox/e2b/helper_contract.go b/services/core/internal/sandbox/e2b/helper_contract.go index f4cf30ffb..f6fe533b4 100644 --- a/services/core/internal/sandbox/e2b/helper_contract.go +++ b/services/core/internal/sandbox/e2b/helper_contract.go @@ -11,7 +11,7 @@ import ( //go:generate go run ./internal/contractgen // This adapter-private boundary is documented in tools/e2b-provider/README.md. -const ProtocolVersion = 1 +const ProtocolVersion = 2 const MaxOutputBytes = 1024 * 1024 const MaxRequestBytes = 72 * 1024 * 1024 const MaxResponseBytes = 16 * 1024 * 1024 diff --git a/services/core/internal/sandbox/e2b/helper_contract_test.go b/services/core/internal/sandbox/e2b/helper_contract_test.go index 686321cb1..de319aa09 100644 --- a/services/core/internal/sandbox/e2b/helper_contract_test.go +++ b/services/core/internal/sandbox/e2b/helper_contract_test.go @@ -67,6 +67,7 @@ func validManagedExchange(data []byte) bool { _ = json.Unmarshal(bootstrapBytes, &expected) delete(expected, "CoreURL") delete(expected, "Credential") + delete(expected, "Harness") expected["InstallationID"] = nil expected["RuntimeBootstrap"] = nil if workspace, present := fields["Workspace"]; present { diff --git a/services/core/internal/sandbox/e2b/installed_paths_test.go b/services/core/internal/sandbox/e2b/installed_paths_test.go index 5012e0d97..60d5b467b 100644 --- a/services/core/internal/sandbox/e2b/installed_paths_test.go +++ b/services/core/internal/sandbox/e2b/installed_paths_test.go @@ -31,7 +31,7 @@ func TestConfigurationDiscoveryUsesSuppliedProcessPaths(t *testing.T) { if err := os.MkdirAll(filepath.Dir(binary), 0700); err != nil { t.Fatal(err) } - if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"Templates\":[]}'\n"), 0700); err != nil { + if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"Templates\":[]}'\n"), 0700); err != nil { t.Fatal(err) } input := sandbox.ConfigurationDiscoveryInput{Credential: json.RawMessage(`{"api_key":"synthetic-key"}`)} diff --git a/services/core/internal/sandbox/e2b/internal/contractgen/main.go b/services/core/internal/sandbox/e2b/internal/contractgen/main.go index 118fe1c3f..a7a9c788f 100644 --- a/services/core/internal/sandbox/e2b/internal/contractgen/main.go +++ b/services/core/internal/sandbox/e2b/internal/contractgen/main.go @@ -71,9 +71,9 @@ func main() { write("services/core/internal/sandbox/e2b/helper_sdk_generated.go", goCode) bootstrap := fields(reflect.TypeFor[sandbox.Bootstrap]()) requiredBootstrap := fieldNames(reflect.TypeFor[sandbox.Bootstrap](), true) - requiredBootstrap = slices.DeleteFunc(requiredBootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" }) + requiredBootstrap = slices.DeleteFunc(requiredBootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" || s == "Harness" }) requiredBootstrap = append(requiredBootstrap, "InstallationID", "RuntimeBootstrap") - bootstrap = slices.DeleteFunc(bootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" }) + bootstrap = slices.DeleteFunc(bootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" || s == "Harness" }) bootstrap = append(bootstrap, "InstallationID", "RuntimeBootstrap") identity := fields(reflect.TypeFor[sandbox.Reference]()) for _, name := range []string{"SessionID", "DeviceID"} { @@ -115,7 +115,7 @@ func main() { // envelopes and managed bootstrap inputs in both implementations. func fixtures() []byte { r := sandbox.Reference{TenantID: "11111111-1111-4111-8111-111111111111", EnvironmentID: "22222222-2222-4222-8222-222222222222", AllocationID: "33333333-3333-4333-8333-333333333333"} - b := sandbox.Bootstrap{Reference: r, SessionID: "44444444-4444-4444-8444-444444444444", DeviceID: "55555555-5555-4555-8555-555555555555", CoreURL: "https://core.example/api/v1", Credential: "fixture-only", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: "44444444-4444-4444-8444-444444444444", DeviceID: "55555555-5555-4555-8555-555555555555", CoreURL: "https://core.example/api/v1", Credential: "fixture-only", Harness: "codex", NetworkAccess: "enabled"} installation := "66666666-6666-4666-8666-666666666666" connection := b.RuntimeConnection() q := e2b.Request{Version: e2b.ProtocolVersion, Operation: "create", Config: e2b.Config{InstallationID: installation}, Reference: r, Bootstrap: &b, RuntimeBootstrap: &connection, Deadline: time.Date(2099, 1, 1, 0, 0, 0, 0, time.UTC)} @@ -187,6 +187,7 @@ func fixtures() []byte { managed := object(b) delete(managed, "CoreURL") delete(managed, "Credential") + delete(managed, "Harness") managed["InstallationID"] = installation managed["RuntimeBootstrap"] = runtimebootstrap.Connection(connection) for _, policy := range []agentnetwork.Policy{{Access: "enabled"}, {Access: "disabled"}, {Access: "restricted", AllowedDomains: []string{"example.com"}}} { diff --git a/services/core/internal/sandbox/e2b/process_test.go b/services/core/internal/sandbox/e2b/process_test.go index 2d15997f4..a2e90a5b4 100644 --- a/services/core/internal/sandbox/e2b/process_test.go +++ b/services/core/internal/sandbox/e2b/process_test.go @@ -12,7 +12,7 @@ import ( ) func TestProcessWireRejectsUnknownAndTrailingOutput(t *testing.T) { - for _, output := range []string{`{"Version":1,"Secret":"private"}`, `{"Version":1} {"Version":1}`} { + for _, output := range []string{`{"Version":2,"Secret":"private"}`, `{"Version":2} {"Version":2}`} { root := t.TempDir() binary := filepath.Join(root, "helper") if err := os.WriteFile(binary, []byte("#!/bin/sh\nprintf '%s' '"+output+"'\n"), 0700); err != nil { @@ -29,7 +29,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { marker := filepath.Join(root, "settled") // The private test path contains no shell syntax; the real adapter never puts // credentials or request contents in argv or inherited environment. - script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":2}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -50,7 +50,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { func TestEnvironmentDropsProviderSelectorsAndCredentials(t *testing.T) { root := t.TempDir() binary := filepath.Join(root, "helper") - script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":2}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -67,7 +67,7 @@ func TestCredentialFenceWaitsForActualHelperExitAfterCancellation(t *testing.T) binary := filepath.Join(root, "helper") marker := filepath.Join(root, "started") finish := filepath.Join(root, "finish") - script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":2}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } diff --git a/services/core/internal/sandbox/e2b/provider_test.go b/services/core/internal/sandbox/e2b/provider_test.go index 406160671..6263ede3e 100644 --- a/services/core/internal/sandbox/e2b/provider_test.go +++ b/services/core/internal/sandbox/e2b/provider_test.go @@ -103,7 +103,7 @@ func TestKillRequiresTerminalProof(t *testing.T) { } func TestCreateAndCommandUseOnlyPrivateRequest(t *testing.T) { p, f, r := fixture(t) - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private-bootstrap", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private-bootstrap", Harness: "codex", NetworkAccess: "enabled"} f.response.Info = &sandbox.Info{Reference: r, State: "running", ProviderID: "native-id", CreateSettled: true, BootstrapComplete: true} if _, err := p.Create(bounded(t), b); err != nil { t.Fatal(err) @@ -143,7 +143,7 @@ func TestDeadlineAndCommandAdmission(t *testing.T) { func TestDefinitePreHelperCreateFailureCarriesAbsenceProof(t *testing.T) { p, f, r := fixture(t) - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private", Harness: "codex", NetworkAccess: "enabled"} for _, err := range []error{errHelperNotStarted, context.DeadlineExceeded} { f.err = err info, gotErr := p.Create(bounded(t), b) diff --git a/services/core/internal/sandbox/microsandbox/contract_test.go b/services/core/internal/sandbox/microsandbox/contract_test.go index 0733824a9..7ecfc5da8 100644 --- a/services/core/internal/sandbox/microsandbox/contract_test.go +++ b/services/core/internal/sandbox/microsandbox/contract_test.go @@ -7,12 +7,13 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/contracttest" "github.com/google/uuid" "testing" + "time" ) func TestProviderContract(t *testing.T) { contracttest.RunFailures(t, func(t *testing.T, s contracttest.Scenario, cancel context.CancelFunc) contracttest.Fixture { c, r := testConfig(), testRef() - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} var calls []string p, err := NewWithCaller(c, callerFunc(func(ctx context.Context, q Request) (Response, error) { calls = append(calls, q.Operation) @@ -62,7 +63,7 @@ func TestProviderContractObservation(t *testing.T) { if err != nil { t.Fatal(err) } - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} got, err := p.Create(deadline(t), b) contracttest.AssertObservation(t, got, err, r, "native-owned", "running") got, err = p.GetInfo(deadline(t), r) @@ -70,3 +71,21 @@ func TestProviderContractObservation(t *testing.T) { got, err = p.Renew(deadline(t), r) contracttest.AssertObservation(t, got, err, r, "native-owned", "stopped") } + +func TestBootstrapWireRejectsSupersededVersion(t *testing.T) { + c, r := testConfig(), testRef() + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "enabled"} + q := Request{Version: ProtocolVersion, Operation: "create", Config: c, Reference: r, Bootstrap: &b, Deadline: time.Now().Add(time.Minute)} + if err := ValidateRequest(q); err != nil { + t.Fatal(err) + } + q.Version = 2 + if err := ValidateRequest(q); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("superseded bootstrap wire accepted", err) + } + q.Version = ProtocolVersion + b.Harness = "" + if err := ValidateRequest(q); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("missing selected Harness accepted", err) + } +} diff --git a/services/core/internal/sandbox/microsandbox/creation_settlement_test.go b/services/core/internal/sandbox/microsandbox/creation_settlement_test.go index db52bcfae..6fab743b3 100644 --- a/services/core/internal/sandbox/microsandbox/creation_settlement_test.go +++ b/services/core/internal/sandbox/microsandbox/creation_settlement_test.go @@ -11,7 +11,7 @@ import ( func TestCreateConfigurationRejectionSettlement(t *testing.T) { config, ref := testConfig(), testRef() bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, - CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled"} + CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "disabled"} for _, test := range []struct { name string change func(*Response) diff --git a/services/core/internal/sandbox/microsandbox/types.go b/services/core/internal/sandbox/microsandbox/types.go index bb8d9246d..0a04f9e53 100644 --- a/services/core/internal/sandbox/microsandbox/types.go +++ b/services/core/internal/sandbox/microsandbox/types.go @@ -9,7 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -const ProtocolVersion = 4 +const ProtocolVersion = 5 const SDKVersion = "v0.7.2" const MaxOutputBytes = 1024 * 1024 const MaxRequestBytes = 72 * 1024 * 1024 diff --git a/services/core/internal/sandbox/microsandbox/workspace_test.go b/services/core/internal/sandbox/microsandbox/workspace_test.go index 64ba57717..f345df118 100644 --- a/services/core/internal/sandbox/microsandbox/workspace_test.go +++ b/services/core/internal/sandbox/microsandbox/workspace_test.go @@ -62,7 +62,7 @@ func TestWorkspaceDoesNotFallbackWithoutResolver(t *testing.T) { config := testConfig() config.ExternalWorkspace = true p, _ := NewWithCaller(config, callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) - _, err := p.Create(deadline(t), sandbox.Bootstrap{Reference: testRef(), SessionID: testRef().TenantID, DeviceID: testRef().EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()}) + _, err := p.Create(deadline(t), sandbox.Bootstrap{Harness: "codex", Reference: testRef(), SessionID: testRef().TenantID, DeviceID: testRef().EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()}) if !errors.Is(err, workspacefs.ErrUnsupported) || calls != 0 { t.Fatal(err, calls) } @@ -81,7 +81,7 @@ func TestPreNativeWorkspaceFailureSettlesAbsentCreation(t *testing.T) { provider.workspace = workspaceResolverFunc(func(context.Context, workspacefs.Binding) (workspacefs.Directory, error) { return workspacefs.Directory{}, workspacefs.ErrUnavailable }) - bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()} + bootstrap := sandbox.Bootstrap{Harness: "codex", Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()} ctx := deadline(t) switch failure { case "mode": diff --git a/services/core/internal/sandbox/node/connection_test.go b/services/core/internal/sandbox/node/connection_test.go index abee501b3..931d1e78e 100644 --- a/services/core/internal/sandbox/node/connection_test.go +++ b/services/core/internal/sandbox/node/connection_test.go @@ -179,7 +179,7 @@ func TestCopiedIdentityCannotReplaceNodeWithInflightCreate(t *testing.T) { proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) r := reference() created := make(chan error, 1) - go func() { _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: r}); created <- err }() + go func() { _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: r, Harness: "codex"}); created <- err }() <-original.started stopDuplicate, duplicateDone := start(duplicateDir, duplicate) defer func() { diff --git a/services/core/internal/sandbox/node/creation_settlement_test.go b/services/core/internal/sandbox/node/creation_settlement_test.go index 09d8daec5..cc0048506 100644 --- a/services/core/internal/sandbox/node/creation_settlement_test.go +++ b/services/core/internal/sandbox/node/creation_settlement_test.go @@ -76,7 +76,7 @@ func TestNodeCarriesCreationSettlementWithoutConvertingFailureToSuccess(t *testi proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) for _, operation := range []func(context.Context) (sandbox.Info, error){ func(ctx context.Context) (sandbox.Info, error) { - return proxy.Create(ctx, sandbox.Bootstrap{Reference: ref}) + return proxy.Create(ctx, sandbox.Bootstrap{Reference: ref, Harness: "codex"}) }, func(ctx context.Context) (sandbox.Info, error) { return proxy.GetInfo(ctx, ref) }, } { diff --git a/services/core/internal/sandbox/node/docker_live_test.go b/services/core/internal/sandbox/node/docker_live_test.go index 48695bfff..3c59d3bc7 100644 --- a/services/core/internal/sandbox/node/docker_live_test.go +++ b/services/core/internal/sandbox/node/docker_live_test.go @@ -87,7 +87,7 @@ func TestDockerNodeTransportLifecycle(t *testing.T) { }() ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-node-transport-credential", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-node-transport-credential", Harness: "codex", NetworkAccess: "enabled"} callCtx, callCancel := context.WithTimeout(ctx, 25*time.Second) info, err := proxy.Create(callCtx, b) callCancel() diff --git a/services/core/internal/sandbox/node/generation_connection_test.go b/services/core/internal/sandbox/node/generation_connection_test.go index 1c3e59d6c..efc41798d 100644 --- a/services/core/internal/sandbox/node/generation_connection_test.go +++ b/services/core/internal/sandbox/node/generation_connection_test.go @@ -69,7 +69,7 @@ func TestGenerationWireRoutesOldOwnershipAndCurrentTargetSeparately(t *testing.T t.Fatal("retained generation info failed", generation, err) } if generation != 9 { - if _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: ref}); err != nil { + if _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: ref, Harness: "codex"}); err != nil { t.Fatal("exact ready provider Create failed", generation, err) } } diff --git a/services/core/internal/sandbox/node/generation_json.go b/services/core/internal/sandbox/node/generation_json.go index 5336a320f..661d177b2 100644 --- a/services/core/internal/sandbox/node/generation_json.go +++ b/services/core/internal/sandbox/node/generation_json.go @@ -130,9 +130,15 @@ func validateGenerationJSON(raw []byte, kind string) error { } } if value := values["request"]; value != nil { - if _, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume snapshot observation", ""); err != nil { + request, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume snapshot observation", "") + if err != nil { return err } + if bootstrap := request["bootstrap"]; bootstrap != nil { + if _, err := generationObject(bootstrap, "TenantID EnvironmentID AllocationID SessionID DeviceID CoreURL Credential Harness NetworkAccess AllowedDomains", "Workspace", "AllowedDomains"); err != nil { + return err + } + } } if value := values["response"]; value != nil { if _, err := generationObject(value, "id connection_id", "error_code unsupported info compute state command sample", ""); err != nil { diff --git a/services/core/internal/sandbox/node/generation_json_test.go b/services/core/internal/sandbox/node/generation_json_test.go index b6fef3a10..e9f52cbf9 100644 --- a/services/core/internal/sandbox/node/generation_json_test.go +++ b/services/core/internal/sandbox/node/generation_json_test.go @@ -90,3 +90,23 @@ func TestNodeGenerationManagementIsAnExplicitCurrentCapability(t *testing.T) { } } } + +func TestNodeBootstrapRejectsAmbiguousHarness(t *testing.T) { + r := sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()} + b := sandbox.Bootstrap{Reference: r, Harness: "codex"} + f := frame{Version: ProtocolVersion, Type: "request", Request: &request{DeploymentGeneration: 1, ID: uuid.NewString(), Sequence: 1, ConnectionID: uuid.NewString(), OwnerEpoch: 1, Operation: "create", TimeoutMillis: 1000, Reference: r, Bootstrap: &b}} + raw, _ := json.Marshal(f) + good := string(raw) + if _, err := decodeFrame(raw); err != nil { + t.Fatal("valid selected bootstrap", err) + } + for _, replacement := range []string{`"Harness":null`, `"Harness":""`, `"harness":"codex"`, `"Harness":"codex","Harness":"mcode"`, `"Harness":"codex","harness":"mcode"`, `"Harness":"Codex"`, `"Harness":42`} { + bad := strings.Replace(good, `"Harness":"codex"`, replacement, 1) + if decoded, err := decodeFrame([]byte(bad)); err == nil && decoded.Request.validate() == nil { + t.Fatal("ambiguous selection reached dispatch", replacement) + } + } + if _, err := decodeFrame([]byte(strings.Replace(good, `,"Harness":"codex"`, "", 1))); err == nil { + t.Fatal("missing Harness accepted") + } +} diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index 946470389..a8a21fbce 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -129,7 +129,10 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing createCtx, stopCreate := context.WithTimeout(ctx, 150*time.Millisecond) defer stopCreate() createDone := make(chan error, 1) - go func() { _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r}); createDone <- err }() + go func() { + _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r, Harness: "codex"}) + createDone <- err + }() <-p.started if err := <-createDone; !errors.Is(err, sandbox.ErrComputeUnconfirmed) { t.Fatalf("lost reply = %v", err) diff --git a/services/core/internal/sandbox/node/recovery_test.go b/services/core/internal/sandbox/node/recovery_test.go index 78d106da3..c5039ea9d 100644 --- a/services/core/internal/sandbox/node/recovery_test.go +++ b/services/core/internal/sandbox/node/recovery_test.go @@ -236,7 +236,7 @@ func TestDegradedNodeRetainsObservationAndCleanup(t *testing.T) { } proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) r := reference() - if _, err = proxy.Create(ctx, sandbox.Bootstrap{Reference: r}); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { + if _, err = proxy.Create(ctx, sandbox.Bootstrap{Reference: r, Harness: "codex"}); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { t.Fatalf("create = %v", err) } if _, err = proxy.GetInfo(ctx, r); err != nil { diff --git a/services/core/internal/sandbox/node/timeout_test.go b/services/core/internal/sandbox/node/timeout_test.go index 15f8da3c5..aa6184be5 100644 --- a/services/core/internal/sandbox/node/timeout_test.go +++ b/services/core/internal/sandbox/node/timeout_test.go @@ -108,7 +108,10 @@ func TestQueuedMutationExpiresWithoutExecution(t *testing.T) { createCtx, stopCreate := context.WithTimeout(ctx, 3*time.Second) defer stopCreate() created := make(chan error, 1) - go func() { _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r}); created <- err }() + go func() { + _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r, Harness: "codex"}) + created <- err + }() <-p.started killCtx, stopKill := context.WithTimeout(ctx, 80*time.Millisecond) defer stopKill() diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index d4ab402ab..e7a9d80f2 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -11,6 +11,7 @@ import ( "io" "time" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" @@ -18,7 +19,7 @@ import ( "github.com/gorilla/websocket" ) -const ProtocolVersion = 5 +const ProtocolVersion = 6 const MaxControlFrameBytes = 32 * 1024 const MaxFrameBytes = 72 * 1024 * 1024 const maxPending = 32 @@ -254,7 +255,7 @@ func (q request) validate() error { return nil } case "create": - if count == 1 && q.Bootstrap != nil && q.Bootstrap.Reference == q.Reference && sandbox.ValidateWorkspaceBinding(q.Reference, q.Bootstrap.Workspace) == nil { + if count == 1 && q.Bootstrap != nil && q.Bootstrap.Reference == q.Reference && runtimebootstrap.ValidHarness(q.Bootstrap.Harness) && sandbox.ValidateWorkspaceBinding(q.Reference, q.Bootstrap.Workspace) == nil { return nil } case "info", "renew", "kill", "initial": diff --git a/services/core/internal/sandbox/node/workspace_test.go b/services/core/internal/sandbox/node/workspace_test.go index 35ab50569..787447247 100644 --- a/services/core/internal/sandbox/node/workspace_test.go +++ b/services/core/internal/sandbox/node/workspace_test.go @@ -30,10 +30,29 @@ func TestWorkspaceWireBindsConfigurationAndEnvironment(t *testing.T) { case "object": b.Attachment.Reference.ObjectID = "invalid" } - q := request{ID: ref.AllocationID, Reference: ref, Operation: "create", TimeoutMillis: 1000, Bootstrap: &sandbox.Bootstrap{Reference: ref, Workspace: &b}} + q := request{ID: ref.AllocationID, Reference: ref, Operation: "create", TimeoutMillis: 1000, Bootstrap: &sandbox.Bootstrap{Harness: "codex", Reference: ref, Workspace: &b}} if (q.validate() == nil) != (fault == "match") { t.Fatal("invalid create binding forwarding", fault) } + for _, external := range []bool{false, true} { + wireRequest := q + bootstrap := *q.Bootstrap + wireRequest.Bootstrap = &bootstrap + if !external { + bootstrap.Workspace = nil + } + wireRequest.DeploymentGeneration, wireRequest.Sequence, wireRequest.OwnerEpoch = 1, 1, 1 + wireRequest.ConnectionID = ref.TenantID + raw, err := json.Marshal(frame{Version: ProtocolVersion, Type: "request", Request: &wireRequest}) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeFrame(raw) + want := !external || fault == "match" + if (err == nil && decoded.Request.validate() == nil) != want { + t.Fatalf("workspace wire external=%v fault=%s: %v", external, fault, err) + } + } q.Operation = "resume" q.Bootstrap = nil q.Resume = &sandbox.ResumeRequest{Reference: ref, Workspace: &b} @@ -75,7 +94,7 @@ func TestActualWorkspaceRefusalKeepsSettledAbsenceOnWire(t *testing.T) { } fsConfig := workspacefs.Configuration{ID: "44444444-4444-4444-8444-444444444444", Adapter: "fixture", Parameters: json.RawMessage(`{}`)} binding := &workspacefs.Binding{Configuration: fsConfig, Attachment: workspacefs.Attachment{Reference: workspacefs.Reference{TenantID: ref.TenantID, EnvironmentID: ref.EnvironmentID, ObjectID: "55555555-5555-4555-8555-555555555555"}, ConfigurationID: fsConfig.ID, Kind: workspacefs.AttachmentHostDirectory, Native: json.RawMessage(`{}`)}} - bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: binding} + bootstrap := sandbox.Bootstrap{Harness: "codex", Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: binding} ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() out := execute(ctx, provider, request{ID: ref.AllocationID, ConnectionID: ref.TenantID, Reference: ref, Operation: "create", TimeoutMillis: 1000, Bootstrap: &bootstrap}) diff --git a/services/core/internal/sandbox/runtime_bootstrap.go b/services/core/internal/sandbox/runtime_bootstrap.go index a4a05ecad..66c83ee86 100644 --- a/services/core/internal/sandbox/runtime_bootstrap.go +++ b/services/core/internal/sandbox/runtime_bootstrap.go @@ -7,6 +7,6 @@ import "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" func (b Bootstrap) RuntimeConnection() runtimebootstrap.Connection { return runtimebootstrap.Connection{ Version: runtimebootstrap.Version, CoreURL: b.CoreURL, - DeviceID: b.DeviceID, Credential: b.Credential, + DeviceID: b.DeviceID, Credential: b.Credential, Harness: b.Harness, } } diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index 8d311697f..e5770790e 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -53,9 +53,9 @@ type Reference struct{ TenantID, EnvironmentID, AllocationID string } type Bootstrap struct { Workspace *workspacefs.Binding `json:",omitempty"` Reference - SessionID, DeviceID, CoreURL, Credential string - NetworkAccess string - AllowedDomains []string + SessionID, DeviceID, CoreURL, Credential, Harness string + NetworkAccess string + AllowedDomains []string } // Info describes compute only. Running does not establish daemon authentication, diff --git a/services/core/tests/integration/runtime_lifecycle_test.go b/services/core/tests/integration/runtime_lifecycle_test.go index 4f28fc6f0..829f4fef7 100644 --- a/services/core/tests/integration/runtime_lifecycle_test.go +++ b/services/core/tests/integration/runtime_lifecycle_test.go @@ -28,6 +28,7 @@ type lifecycleProvider struct { loseCreate, absent, unavailable bool credentialHash string credential string + harness string } func (p *lifecycleProvider) Create(_ context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { @@ -36,6 +37,7 @@ func (p *lifecycleProvider) Create(_ context.Context, b sandbox.Bootstrap) (sand p.creates++ p.credentialHash = runtimedevice.HashCredential(b.Credential) p.credential = b.Credential + p.harness = b.Harness i := sandbox.Info{Reference: b.Reference, ProviderID: b.AllocationID, State: "running", BootstrapComplete: true} if !p.absent { p.resources[b.AllocationID] = i @@ -245,3 +247,32 @@ func TestManagedRuntimeStoppedComputeDoesNotRequestCleanup(t *testing.T) { } } } + +func TestManagedRuntimeBootstrapUsesSessionHarness(t *testing.T) { + for _, kind := range []string{"codex", "mcode", "claude_sdk"} { + t.Run(kind, func(t *testing.T) { + s, key := configuredStore(t) + tenant := uuid.NewString() + session, err := s.CreateSession(t.Context(), tenant, WithFixtureModelProvider(sessions.CreateSession{Creator: FixtureCreator(), Engine: kind, IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted","network":{"access":"enabled"}}}`)})) + if err != nil { + t.Fatal(err) + } + env, err := sessionAdapter(s).GetSessionEnvironment(t.Context(), tenant, session.ID) + if err != nil { + t.Fatal(err) + } + p := &lifecycleProvider{resources: map[string]sandbox.Info{}} + w, _ := managedWorker(t, s, key, p) + for range 2 { + if _, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key); err != nil { + t.Fatal(err) + } + } + p.mu.Lock() + defer p.mu.Unlock() + if p.harness != kind || p.creates != 1 { + t.Fatalf("bootstrap selection=%s creates=%d", p.harness, p.creates) + } + }) + } +} diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index 1e7588c76..46f1e9f15 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -30,7 +30,7 @@ Compatible endpoints must return the SDK 2.51.0 template-list and template-build Run `go generate ./services/core/internal/sandbox/e2b` from the repository root after changing these declarations. `make check-e2b-provider` and the Go adapter tests reject stale projections; both languages consume generated valid and invalid exchanges covering wire types, extra fields, operation/reference bounds and managed-bootstrap fields. The helper build copies those fixtures with its source before running the pinned-SDK suite. -The boundary has version 1. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. +The boundary has version 2. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. ## Receipts and state directory @@ -42,9 +42,9 @@ A helper holds its allocation's lock until the SDK operation returns, even after 1. Record `create_pending` with the bootstrap identity, then call `Sandbox.create` with the template, the configured timeout, the ownership metadata, `on_timeout=kill` and auto-resume disabled. A definite rejection records a settled `rejected` receipt with no sandbox IDs. 2. Record the sandbox ID and connection material, check the sandbox domain, then read the sandbox by ID and check its ownership metadata, template and resources before writing any credential. A mismatch records a settled rejection and returns `CreateSettled` with the error. -3. Check that `/opt/oac-e2b/managed_init.py` is readable, write the managed bootstrap input to `/root/.oac/e2b/managed-bootstrap.json` and run `managed_init.py` as root. +3. Check that `/opt/oac-e2b/managed_init.py` has regular-file type through SDK file information, then open it through the SDK streaming file API as root and close the response without downloading its contents. This verifies readability without a remote probe process. Then write the managed bootstrap input to `/root/.oac/e2b/managed-bootstrap.json` and run `managed_init.py` as root. -`managed_init.py` prepares the image as the [application-managed startup](../../deploy/e2b/README.md#startup-and-security-boundary) does, writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json` (mode 0600, owned by UID 1000), sets the Environment, Session and network variables and starts `oac-daemon connect --profile default --bootstrap-file /home/runtime/runtime-bootstrap.json` as UID/GID 1000. It records process handoff in `/root/.oac/e2b/managed-ready.json` and refuses to run again once any launch record exists. `BootstrapComplete` becomes true when a later inspection reads that record with the expected identity; it does not prove enrollment or native readiness. +`managed_init.py` prepares the image as the [application-managed startup](../../deploy/e2b/README.md#startup-and-security-boundary) does, writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json` (mode 0600, owned by UID 1000), sets the Environment, Session and network variables and starts `oac-daemon connect --profile default --bootstrap-file /home/runtime/runtime-bootstrap.json` as UID/GID 1000. It records process handoff in `/root/.oac/e2b/managed-ready.json` and refuses to run again once any launch record exists. `BootstrapComplete` becomes true when Core validates that record with the expected identity. The initialization command returns the bounded durable receipt on its existing output stream, avoiding a separate file read on successful Create; Core still rereads cloud ownership and configuration before returning. If the command response is lost or its receipt is unreadable, inspection can recover the record without replaying startup; it does not prove enrollment or native readiness. An unknown Create is never repeated. A Create whose connection material was lost can be discovered and destroyed but cannot resume bootstrap, and an unconfirmed startup requires reclaiming the whole allocation. diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index 77d1869fd..0b7635c78 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -11,7 +11,7 @@ MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 1 +PROTOCOL_VERSION = 2 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] diff --git a/services/core/tools/e2b-provider/observation_test.py b/services/core/tools/e2b-provider/observation_test.py index ef204d6d7..f72554e8d 100644 --- a/services/core/tools/e2b-provider/observation_test.py +++ b/services/core/tools/e2b-provider/observation_test.py @@ -15,7 +15,7 @@ def observe(self, reference=None): try: return Provider(request).execute() except Failure as error: - return {'Version': 1, 'ErrorCode': error.code} + return {'Version': 2, 'ErrorCode': error.code} def listing(self, *pages): pages = [list(page) for page in pages] or [[]] diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index 76557b500..264b39489 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -7,7 +7,7 @@ from datetime import datetime, timezone from uuid import UUID -from e2b import Sandbox, SandboxQuery, SandboxState +from e2b import FileType, Sandbox, SandboxQuery, SandboxState from e2b.api.client.models.sandbox_metric import SandboxMetric from e2b.exceptions import AuthenticationException, FileNotFoundException, SandboxNotFoundException @@ -19,6 +19,20 @@ PREFIX = 'oac_' FIELDS = ('InstallationID', *REFERENCE_FIELDS) +# Execute the existing protected entry point and return its durable receipt on +# the same command stream. A read failure leaves successful startup recoverable +# through Inspect, without replaying initialization. +BOOTSTRAP_SCRIPT = """import runpy,sys +runpy.run_path('/opt/oac-e2b/managed_init.py', run_name='__main__') +try: + with open('/root/.oac/e2b/managed-ready.json', 'rb') as source: + receipt = source.read(4097) + if len(receipt) <= 4096: + sys.stdout.buffer.write(receipt) +except OSError: + pass +""" + def valid_id(value): try: @@ -196,13 +210,17 @@ def inspect(self): except FileNotFoundException: receipt = None if receipt is not None: - expected = record.get('bootstrap_identity') - if (receipt.get('identity') != expected or receipt.get('status') != 'daemon_started' or - type(receipt.get('daemon_pid')) is not int or receipt['daemon_pid'] <= 0): - raise Failure('ownership') - self.receipt.save(settled=True, bootstrap_complete=True) + self.accept_bootstrap_receipt(receipt) return cloud + def accept_bootstrap_receipt(self, receipt): + expected = (self.receipt.data or {}).get('bootstrap_identity') + if (not isinstance(receipt, dict) or receipt.get('identity') != expected or + receipt.get('status') != 'daemon_started' or + type(receipt.get('daemon_pid')) is not int or receipt['daemon_pid'] <= 0): + raise Failure('ownership') + self.receipt.save(settled=True, bootstrap_complete=True) + def create(self): if self.receipt.data is not None: raise Failure('exists') @@ -233,27 +251,40 @@ def create(self): self.receipt.save(status='configuration_rejected', settled=True) raise # Validate the current template entry point before writing any credential. - check = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', - "import os,sys; sys.exit(78 if not os.path.isfile('/opt/oac-e2b/managed_init.py') or not os.access('/opt/oac-e2b/managed_init.py', os.R_OK) else 0)"]}, - self.remaining, user='root') - if check['ExitCode'] != 0: + try: + entry = cloud.files.get_info('/opt/oac-e2b/managed_init.py', + user='root', request_timeout=self.remaining()) + if entry.type != FileType.FILE: + self.receipt.save(status='bootstrap_failed', settled=True) + raise Failure('template_invalid') + # A successful download can also refer to a special file. Check + # the type first, then verify readability without a probe process. + with cloud.files.read('/opt/oac-e2b/managed_init.py', format='stream', + user='root', request_timeout=self.remaining()): + pass + except FileNotFoundException: self.receipt.save(status='bootstrap_failed', settled=True) - raise Failure('template_invalid' if check['ExitCode'] == 78 else 'unconfirmed') + raise Failure('template_invalid') from None payload = dict(bootstrap, InstallationID=self.config['InstallationID'], RuntimeBootstrap=self.q['RuntimeBootstrap']) - del payload['CoreURL'], payload['Credential'] + del payload['CoreURL'], payload['Credential'], payload['Harness'] if (set(payload) - set(MANAGED_BOOTSTRAP_FIELDS) or not set(MANAGED_BOOTSTRAP_REQUIRED_FIELDS) <= set(payload)): raise Failure('invalid') cloud.files.write('/root/.oac/e2b/managed-bootstrap.json', json.dumps(payload), user='root', request_timeout=self.remaining()) self.receipt.save(status='bootstrap_pending') - result = run(cloud, {'Args': ['/usr/bin/python3', '-I', '/opt/oac-e2b/managed_init.py']}, + result = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', BOOTSTRAP_SCRIPT]}, self.remaining, user='root') if result['ExitCode'] != 0: self.receipt.save(status='bootstrap_failed', settled=True) raise Failure('unconfirmed') self.receipt.save(status='bootstrap_exited', settled=True) + try: + receipt = json.loads(result['Stdout']) + except (ValueError, KeyError): + raise Failure('unconfirmed') from None + self.accept_bootstrap_receipt(receipt) return self.inspect() def renew(self): diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 74d708547..240e05eab 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -1,22 +1,63 @@ """Controlled SDK boundary failures; live qualification remains separate.""" import copy +import io +import httpx from datetime import datetime, timedelta, timezone import json from pathlib import Path import tempfile +import subprocess +import sys from types import SimpleNamespace import unittest -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch from uuid import uuid4 -from e2b import SandboxState -from e2b.exceptions import AuthenticationException, SandboxNotFoundException +from e2b import FileType, SandboxState +from e2b.connection_config import ConnectionConfig +from e2b.sandbox_sync.filesystem.filesystem import Filesystem +from packaging.version import Version +from e2b.exceptions import AuthenticationException, FileNotFoundException, SandboxNotFoundException -from provider import Provider +from provider import BOOTSTRAP_SCRIPT, Provider +from helper_contract_generated import PROTOCOL_VERSION from sdk import restore, run from state import Failure, Receipt +class BootstrapScriptTest(unittest.TestCase): + def execute(self, startup, receipt=None): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + entry = root / 'managed_init.py' + ready = root / 'managed-ready.json' + entry.write_text(startup.replace('READY_PATH', repr(str(ready)))) + if receipt is not None: + ready.write_bytes(receipt) + script = BOOTSTRAP_SCRIPT.replace('/opt/oac-e2b/managed_init.py', str(entry)).replace( + '/root/.oac/e2b/managed-ready.json', str(ready)) + return subprocess.run([sys.executable, '-I', '-c', script], + capture_output=True, timeout=5) + + def test_success_reads_receipt_after_initialization(self): + result = self.execute("from pathlib import Path; Path(READY_PATH).write_bytes(b'new receipt')", + receipt=b'stale receipt') + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, b'new receipt') + + def test_failed_initialization_cannot_return_old_receipt(self): + result = self.execute('raise SystemExit(7)', receipt=b'stale receipt') + self.assertEqual(result.returncode, 7) + self.assertEqual(result.stdout, b'') + + def test_missing_and_oversized_receipts_leave_startup_recoverable(self): + for receipt in (None, b'x' * 4097): + with self.subTest(size=len(receipt) if receipt else None): + result = self.execute('pass', receipt=receipt) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, b'') + + class ProviderTest(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() @@ -24,13 +65,13 @@ def setUp(self): self.reference = {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']} self.config = {'StateDir': self.temporary.name, 'InstallationID': str(uuid4()), 'APIKey': 'private-account-secret', 'Template': 'test:' + str(uuid4()), 'TimeoutSeconds': 120} - self.request = {'Version': 1, 'Operation': 'create', 'Config': self.config, + self.request = {'Version': 2, 'Operation': 'create', 'Config': self.config, 'Reference': self.reference, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=30)).isoformat(), 'Bootstrap': dict(self.reference, SessionID=str(uuid4()), DeviceID=str(uuid4()), CoreURL='https://core.example/api/v1', Credential='private-runtime-secret', - NetworkAccess='enabled', AllowedDomains=[])} + Harness='codex', NetworkAccess='enabled', AllowedDomains=[])} self.request['RuntimeBootstrap'] = { - 'version': 1, 'core_url': self.request['Bootstrap']['CoreURL'], + 'version': 2, 'harness': 'codex', 'core_url': self.request['Bootstrap']['CoreURL'], 'device_id': self.request['Bootstrap']['DeviceID'], 'credential': self.request['Bootstrap']['Credential']} self.cloud = Mock(sandbox_id='owned-id', sandbox_domain='e2b.app', _envd_version='0.5.0', @@ -40,7 +81,12 @@ def setUp(self): self.identity = dict(self.reference, InstallationID=self.config['InstallationID'], SessionID=self.request['Bootstrap']['SessionID'], DeviceID=self.request['Bootstrap']['DeviceID']) self.ready = json.dumps({'identity': self.identity, 'status': 'daemon_started', 'daemon_pid': 123}) + self.cloud.files.get_info.return_value = SimpleNamespace(type=FileType.FILE) + self.template_stream = MagicMock() self.cloud.files.read.return_value = self.ready + self.cloud.files.read.side_effect = lambda path, **kwargs: ( + self.template_stream if path == '/opt/oac-e2b/managed_init.py' + else self.cloud.files.read.return_value) self.api = Mock() self.api.create.return_value = self.cloud self.api.get_info.return_value = self.cloud @@ -51,7 +97,8 @@ def setUp(self): self.runtime = patch('provider.restore', return_value=self.cloud) self.runtime.start() self.addCleanup(self.runtime.stop) - self.command = patch('provider.run', return_value={'Stdout': '', 'Stderr': '', 'ExitCode': 0}) + self.command = patch('provider.run', side_effect=lambda *a, **k: { + 'Stdout': self.cloud.files.read.return_value, 'Stderr': '', 'ExitCode': 0}) self.command.start() self.addCleanup(self.command.stop) @@ -108,6 +155,7 @@ def test_create_refuses_other_owner_before_credentials(self): self.assertEqual(self.call('create')['ErrorCode'], 'ownership') self.api.get_info.assert_called_once() self.cloud.files.write.assert_not_called() + self.cloud.files.read.assert_not_called() def test_qualified_gateway_template_id_accepts_only_selected_build(self): self.config['Resources'] = {'cpus': 2, 'memory_mib': 2048} @@ -145,7 +193,7 @@ def test_create_refuses_foreign_data_plane_before_envd(self): self.cloud.commands.run.assert_not_called() def test_template_invalid_refuses_before_credentials_and_retains_owned_cleanup(self): - with patch('provider.run', return_value={'ExitCode': 78, 'Stdout': '', 'Stderr': ''}): + with patch.object(self.cloud.files, 'read', side_effect=FileNotFoundException('missing')): result = self.call('create') self.assertEqual(result['ErrorCode'], 'template_invalid') self.assertTrue(result['Info']['CreateSettled']) @@ -155,6 +203,75 @@ def test_template_invalid_refuses_before_credentials_and_retains_owned_cleanup(s self.api.kill.assert_not_called() self.assertEqual(self.call('create')['ErrorCode'], 'exists') + def test_template_non_regular_files_are_rejected_before_open_or_credentials(self): + for kind in (FileType.DIR, None): + with self.subTest(kind=kind): + self.reference['AllocationID'] = str(uuid4()) + self.request['Bootstrap']['AllocationID'] = self.reference['AllocationID'] + self.cloud.metadata = Provider(self.request).metadata + self.cloud.files.get_info.return_value = SimpleNamespace(type=kind) + self.assertEqual(self.call('create')['ErrorCode'], 'template_invalid') + self.cloud.files.read.assert_not_called() + self.cloud.files.write.assert_not_called() + + def test_template_read_is_closed_and_does_not_launch_a_probe(self): + with patch('provider.run', return_value={'ExitCode': 0, 'Stdout': self.ready}) as command: + self.assertEqual(self.call('create')['ErrorCode'], '') + self.template_stream.__enter__.assert_called_once() + self.template_stream.__exit__.assert_called_once() + self.template_stream.__iter__.assert_not_called() + command.assert_called_once() + self.assertEqual(command.call_args.args[1]['Args'], + ['/usr/bin/python3', '-I', '-c', BOOTSTRAP_SCRIPT]) + args, options = self.cloud.files.read.call_args_list[0] + self.assertEqual(args, ('/opt/oac-e2b/managed_init.py',)) + self.assertEqual(options['format'], 'stream') + self.assertEqual(options['user'], 'root') + self.assertGreater(options['request_timeout'], 0) + + def test_template_probe_uses_sdk_stream_and_closes_without_downloading(self): + class Body(httpx.SyncByteStream): + closed = False + + def __iter__(self): + raise AssertionError('template contents must not be downloaded') + yield b'' + + def close(self): + self.closed = True + + body = Body() + requests = [] + + def respond(request): + requests.append(request) + return httpx.Response(200, stream=body) + + with httpx.Client(base_url='https://fixture.invalid', + transport=httpx.MockTransport(respond)) as client: + with patch('e2b.sandbox_sync.filesystem.filesystem.get_envd_api', return_value=client), \ + patch('e2b.sandbox_sync.filesystem.filesystem.create_rpc_client'): + files = Filesystem('https://fixture.invalid', Version('0.5.0'), + ConnectionConfig(api_key='fixture'), client) + original_read = self.cloud.files.read.side_effect + self.cloud.files.read.side_effect = lambda path, **kwargs: ( + files.read(path, **kwargs) if path == '/opt/oac-e2b/managed_init.py' + else original_read(path, **kwargs)) + self.assertEqual(self.call('create')['ErrorCode'], '') + self.assertTrue(body.closed) + self.assertEqual(len(requests), 1) + self.assertEqual(requests[0].method, 'GET') + self.assertEqual(requests[0].url.params['path'], '/opt/oac-e2b/managed_init.py') + self.assertEqual(requests[0].url.params['username'], 'root') + + def test_uncertain_template_read_never_writes_credentials_or_replays_create(self): + with patch.object(self.cloud.files, 'read', side_effect=TimeoutError('private secret')): + self.assertEqual(self.call('create')['ErrorCode'], 'unconfirmed') + self.cloud.files.write.assert_not_called() + self.assertFalse(self.record().get('bootstrap_complete', False)) + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + self.api.create.assert_called_once() + def test_unknown_create_empty_lookup_never_proves_cleanup(self): self.api.create.side_effect = TimeoutError('confidential SDK diagnostic') self.assertEqual(self.call('create')['ErrorCode'], 'unconfirmed') @@ -239,6 +356,23 @@ def test_mismatched_receipt_cannot_prove_bootstrap_complete(self): self.assertTrue(self.record()['settled']) self.assertFalse(self.record()['bootstrap_complete']) + def test_successful_create_returns_receipt_without_remote_receipt_read(self): + self.assertTrue(self.call('create')['Info']['BootstrapComplete']) + self.assertEqual([call.args[0] for call in self.cloud.files.read.call_args_list], + ['/opt/oac-e2b/managed_init.py']) + self.assertEqual(self.api.get_info.call_count, 2) + + def test_lost_command_receipt_recovers_from_file_without_replaying_startup(self): + with patch('provider.run', return_value={'ExitCode': 0, 'Stdout': ''}) as command: + result = self.call('create') + self.assertEqual(result['ErrorCode'], 'unconfirmed') + self.assertTrue(result['Info']['CreateSettled']) + self.assertFalse(result['Info']['BootstrapComplete']) + self.assertTrue(self.call('inspect')['Info']['BootstrapComplete']) + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + command.assert_called_once() + self.api.create.assert_called_once() + def test_foreign_owner_prevents_renew_and_delete(self): self.call('create') self.cloud.metadata = {} diff --git a/services/core/tools/e2b-provider/state_test.py b/services/core/tools/e2b-provider/state_test.py index ae573370a..9ff5b89b4 100644 --- a/services/core/tools/e2b-provider/state_test.py +++ b/services/core/tools/e2b-provider/state_test.py @@ -18,7 +18,7 @@ class StateTest(unittest.TestCase): def setUp(self): self.root = tempfile.TemporaryDirectory() self.addCleanup(self.root.cleanup) - self.request = {'Version': 1, 'Operation': 'kill', + self.request = {'Version': 2, 'Operation': 'kill', 'Config': {'StateDir': self.root.name, 'InstallationID': str(uuid4()), 'APIKey': 'test'}, 'Reference': {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']}, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=2)).isoformat()} diff --git a/services/core/tools/e2b-provider/testdata/contract.json b/services/core/tools/e2b-provider/testdata/contract.json index 80d09bf84..0efaa314d 100644 --- a/services/core/tools/e2b-provider/testdata/contract.json +++ b/services/core/tools/e2b-provider/testdata/contract.json @@ -1,66 +1,66 @@ [ -{"kind":"managed","name":"disabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"disabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"enabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":"example.com","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":[1],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"unknown","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":true,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-AllocationID","payload":{"AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"disabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"disabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"enabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":"example.com","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":[1],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"unknown","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":true,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-AllocationID","payload":{"AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, {"kind":"managed","name":"missing-RuntimeBootstrap","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444"},"valid":false}, -{"kind":"managed","name":"restricted","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":["example.com"],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"restricted","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, -{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, -{"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"create","payload":{"Version":1,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"inspect","payload":{"Version":1,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"kill","payload":{"Version":1,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_builds","payload":{"Version":1,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_templates","payload":{"Version":1,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"renew","payload":{"Version":1,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"validate_deployment","payload":{"Version":1,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-0","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-32","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-33","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":true},"valid":false}, -{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":null},"valid":false}, -{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":"1"},"valid":false}, -{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, -{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":true}, -{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"response","name":"error-","payload":{"Version":1,"ErrorCode":""},"valid":true}, -{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":1,"ErrorCode":"command_unconfirmed"},"valid":true}, -{"kind":"response","name":"error-exists","payload":{"Version":1,"ErrorCode":"exists"},"valid":true}, -{"kind":"response","name":"error-invalid","payload":{"Version":1,"ErrorCode":"invalid"},"valid":true}, -{"kind":"response","name":"error-not_found","payload":{"Version":1,"ErrorCode":"not_found"},"valid":true}, -{"kind":"response","name":"error-ownership","payload":{"Version":1,"ErrorCode":"ownership"},"valid":true}, -{"kind":"response","name":"error-team_mismatch","payload":{"Version":1,"ErrorCode":"team_mismatch"},"valid":true}, -{"kind":"response","name":"error-template_invalid","payload":{"Version":1,"ErrorCode":"template_invalid"},"valid":true}, -{"kind":"response","name":"error-unauthorized","payload":{"Version":1,"ErrorCode":"unauthorized"},"valid":true}, -{"kind":"response","name":"error-unconfirmed","payload":{"Version":1,"ErrorCode":"unconfirmed"},"valid":true}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":1},"valid":false}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":1},"valid":false}, -{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":1},"valid":false}, +{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444"},"valid":false}, +{"kind":"managed","name":"restricted","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":["example.com"],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"restricted","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, +{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, +{"kind":"request","name":"command","payload":{"Version":2,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"create","payload":{"Version":2,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"inspect","payload":{"Version":2,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill","payload":{"Version":2,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_builds","payload":{"Version":2,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_templates","payload":{"Version":2,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"observe","payload":{"Version":2,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"renew","payload":{"Version":2,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"validate_deployment","payload":{"Version":2,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-0","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-32","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-33","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":true},"valid":false}, +{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":null},"valid":false}, +{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":"1"},"valid":false}, +{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":3},"valid":false}, +{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":true}, +{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"response","name":"error-","payload":{"Version":2,"ErrorCode":""},"valid":true}, +{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":2,"ErrorCode":"command_unconfirmed"},"valid":true}, +{"kind":"response","name":"error-exists","payload":{"Version":2,"ErrorCode":"exists"},"valid":true}, +{"kind":"response","name":"error-invalid","payload":{"Version":2,"ErrorCode":"invalid"},"valid":true}, +{"kind":"response","name":"error-not_found","payload":{"Version":2,"ErrorCode":"not_found"},"valid":true}, +{"kind":"response","name":"error-ownership","payload":{"Version":2,"ErrorCode":"ownership"},"valid":true}, +{"kind":"response","name":"error-team_mismatch","payload":{"Version":2,"ErrorCode":"team_mismatch"},"valid":true}, +{"kind":"response","name":"error-template_invalid","payload":{"Version":2,"ErrorCode":"template_invalid"},"valid":true}, +{"kind":"response","name":"error-unauthorized","payload":{"Version":2,"ErrorCode":"unauthorized"},"valid":true}, +{"kind":"response","name":"error-unconfirmed","payload":{"Version":2,"ErrorCode":"unconfirmed"},"valid":true}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":2},"valid":false}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":2},"valid":false}, +{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":2},"valid":false}, {"kind":"response","name":"invalid-Version","payload":{"ErrorCode":"","Version":true},"valid":false} ] diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index ac7e22c2c..58e190c59 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -1,6 +1,6 @@ # microsandbox Sandbox Provider helper -microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 4; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. +microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 5; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. [Add a Sandbox Provider](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources, Runtime release and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy. @@ -64,3 +64,5 @@ The helper returns only the native observation time, exact uptime, cumulative vC ## Tests `make check-microsandbox-provider`, part of `make check`, runs the pure-Go adapter tests everywhere and this module's tests on Linux; other hosts print an explicit skip for the Linux-only module. + +The native helper wire uses version 5 and carries the Session-selected Harness into Runtime bootstrap version 2. Upgrade the helper with its node and Core. From 96bcbcd2534a2083b1bbf869da8b3d8e7ddf606f Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 10 Oct 2026 23:25:15 +0800 Subject: [PATCH 05/22] test(core): account for capability-driven scheduler wakeups --- .../tests/integration/environment_worker_scan_test.go | 8 ++++---- services/core/tests/integration/worker_wakeup_test.go | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/services/core/tests/integration/environment_worker_scan_test.go b/services/core/tests/integration/environment_worker_scan_test.go index 5149190ff..779ff5cbb 100644 --- a/services/core/tests/integration/environment_worker_scan_test.go +++ b/services/core/tests/integration/environment_worker_scan_test.go @@ -8,7 +8,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -func TestWorkerEnvironmentRetriesNewlyReadyAtNextScan(t *testing.T) { +func TestWorkerEnvironmentRetriesNewlyReadyOnCapabilityHint(t *testing.T) { h := newDispatchHarness(t) enableWorkerEnvironment(t, h) pending := workerEnvironmentReservation(t, h) @@ -20,7 +20,6 @@ func TestWorkerEnvironmentRetriesNewlyReadyAtNextScan(t *testing.T) { h.session = publicSession(t, h, "scan-barrier") receipt := h.message("barrier", "ordinary work") - scanned := time.Now() _, stop := startEnvironmentExpiryWorker(t, h.s, h.d) // Dispatch starts only after selectWork has examined the pending input on // the same pass, while its exact Runtime is still incapable of preparation. @@ -28,13 +27,14 @@ func TestWorkerEnvironmentRetriesNewlyReadyAtNextScan(t *testing.T) { if barrier.ID != receipt.TurnID { t.Fatal("unexpected scan barrier") } + readyAt := time.Now() awaitFixtureCapabilities(t, runtime, workerEnvironmentCapabilities()) h.write(barrier.ID, proto.TypeDone, proto.DonePayload{Content: "complete"}) waitTurn(t, h, barrier.ID, sessions.TurnCompleted) prepare := nextWorkerFrame(t, frames, proto.TypeExecutionPrepare) - if elapsed := time.Since(scanned); elapsed < 750*time.Millisecond || elapsed > 3*time.Second { - t.Fatal("readiness retry must use the next scan, without an empty scan interval", elapsed) + if elapsed := time.Since(readyAt); elapsed >= 750*time.Millisecond { + t.Fatal("capability hint waited for the periodic scan", elapsed) } if workerRuntimeForPreparation(t, h, prepare) != runtime { t.Fatal("readiness retry moved Runtime ownership") diff --git a/services/core/tests/integration/worker_wakeup_test.go b/services/core/tests/integration/worker_wakeup_test.go index 5ab16df8d..cea2990fd 100644 --- a/services/core/tests/integration/worker_wakeup_test.go +++ b/services/core/tests/integration/worker_wakeup_test.go @@ -76,6 +76,12 @@ func TestWorkerSchedulerCommittedAdmissionWakesBeforeMaintenance(t *testing.T) { if err != nil { t.Fatal(err) } + // Fixture Runtime enrollment also emits a capability hint. Isolate + // the admission signal under test from that already committed work. + select { + case <-h.d.Registry.CapabilityHints(): + default: + } trace.armed.Store(true) started = true go func() { done <- worker.Run(ctx) }() From 582fc689134baf00d8b91aa640eb1b19c3c74767 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 10 Oct 2026 23:25:49 +0800 Subject: [PATCH 06/22] feat(observability): trace managed Session startup boundaries --- .../internal/agent/codex/model_verbosity.go | 27 +- .../internal/agent/codex/preparation.go | 7 +- .../agent/codex/preparation_observation.go | 19 ++ .../codex/preparation_observation_test.go | 54 ++++ apps/daemon/internal/agent/codex/rpc.go | 18 +- .../internal/agent/codex/rpc_request.go | 30 +- .../agent/codex/startup_observation_test.go | 260 ++++++++++++++++++ apps/daemon/internal/agent/codex/version.go | 50 +++- apps/daemon/internal/cli/agent_discovery.go | 7 + apps/daemon/internal/cli/connect.go | 7 +- .../internal/cli/connect_environment.go | 1 + .../internal/cli/startup_observation.go | 30 ++ apps/daemon/internal/dispatch/executor.go | 2 + docs/getting-started/operations.md | 22 +- docs/zh/getting-started/operations.md | 26 +- services/core/deploy/e2b/init.py | 70 +++-- services/core/deploy/e2b/managed_init.py | 64 +++-- services/core/deploy/e2b/managed_init_test.py | 30 +- services/core/internal/execution/delivery.go | 8 +- .../execution/directory_preparation.go | 2 +- .../execution/environment_admission.go | 23 +- .../execution/execution_observation.go | 46 ++++ .../execution/execution_observation_test.go | 178 ++++++++++++ .../execution/executor_observation.go | 8 +- .../execution/executor_preparation.go | 2 +- .../core/internal/execution/preparation.go | 5 +- .../internal/execution/prepared_dispatch.go | 19 +- .../internal/execution/runtime_compute.go | 2 + .../internal/execution/runtime_connections.go | 4 + .../execution/runtime_initialization.go | 2 + .../internal/execution/runtime_lifecycle.go | 9 +- .../internal/execution/runtime_pending.go | 2 + services/core/internal/execution/worker.go | 1 + .../runtimegateway/capability_hint_test.go | 40 +++ .../core/internal/runtimegateway/handler.go | 3 + .../core/internal/runtimegateway/registry.go | 2 + .../sandbox/e2b/create_observations.go | 58 ++++ .../sandbox/e2b/create_observations_test.go | 104 +++++++ services/core/internal/sandbox/e2b/process.go | 9 + services/core/tools/e2b-provider/README.md | 4 + services/core/tools/e2b-provider/provider.py | 133 ++++++--- .../core/tools/e2b-provider/provider_test.py | 96 ++++++- services/core/tools/e2b-provider/sdk.py | 15 +- 43 files changed, 1366 insertions(+), 133 deletions(-) create mode 100644 apps/daemon/internal/agent/codex/preparation_observation.go create mode 100644 apps/daemon/internal/agent/codex/preparation_observation_test.go create mode 100644 apps/daemon/internal/agent/codex/startup_observation_test.go create mode 100644 apps/daemon/internal/cli/startup_observation.go create mode 100644 services/core/internal/execution/execution_observation.go create mode 100644 services/core/internal/execution/execution_observation_test.go create mode 100644 services/core/internal/sandbox/e2b/create_observations.go create mode 100644 services/core/internal/sandbox/e2b/create_observations_test.go diff --git a/apps/daemon/internal/agent/codex/model_verbosity.go b/apps/daemon/internal/agent/codex/model_verbosity.go index 6c9d4f28e..5bab427be 100644 --- a/apps/daemon/internal/agent/codex/model_verbosity.go +++ b/apps/daemon/internal/agent/codex/model_verbosity.go @@ -8,11 +8,14 @@ import ( "path/filepath" "slices" "strings" + "time" ) // Validate against the binary's active catalog and use that same snapshot for // execution. A CLI override alone is silently ignored for unsupported models. -func prepareModelVerbosity(ctx context.Context, binary string, plan *SessionPlan) error { +func prepareModelVerbosity(ctx context.Context, binary string, plan *SessionPlan) (resultErr error) { + started := time.Now() + defer func() { observePreparationStage(ctx, "model_catalog", started, resultErr) }() args := []string{} for _, kv := range plan.ExtraConfig { args = append(args, "-c", kv[0]+"="+kv[1]) @@ -26,14 +29,30 @@ func prepareModelVerbosity(ctx context.Context, binary string, plan *SessionPlan } cmd.Dir = plan.Cwd cmd.Env = append(os.Environ(), plan.Env...) + commandStarted := time.Now() catalog, err := cmd.Output() + commandEnded := time.Now() // The launcher can exit before its children, ending the context watcher. + cleanupStarted := time.Now() + var cleanupErr error if cmd.Process != nil { - _ = cmd.Cancel() + cleanupErr = cmd.Cancel() + } + cleanupEnded := time.Now() + observePreparationInterval(ctx, "model_catalog_command", commandStarted, commandEnded, err) + // This is the existing best-effort process-group signal, not a second Wait. + if cmd.Process != nil { + observePreparationInterval(ctx, "model_catalog_cleanup", cleanupStarted, cleanupEnded, cleanupErr) } if err != nil { return fmt.Errorf("codex: cannot verify model verbosity support: %w", err) } + validationStarted := time.Now() + defer func() { + if !validationStarted.IsZero() { + observePreparationStage(ctx, "model_catalog_validation", validationStarted, resultErr) + } + }() supported, err := catalogSupportsVerbosity(catalog, plan.Model) if err != nil { return fmt.Errorf("codex: cannot read model verbosity support: %w", err) @@ -49,6 +68,10 @@ func prepareModelVerbosity(ctx context.Context, binary string, plan *SessionPlan if !filepath.IsAbs(codexHome) { return fmt.Errorf("codex: missing managed home for model catalog") } + observePreparationStage(ctx, "model_catalog_validation", validationStarted, nil) + validationStarted = time.Time{} + snapshotStarted := time.Now() + defer func() { observePreparationStage(ctx, "model_catalog_snapshot", snapshotStarted, resultErr) }() file, err := os.CreateTemp(codexHome, "model-catalog-*.json") if err != nil { return err diff --git a/apps/daemon/internal/agent/codex/preparation.go b/apps/daemon/internal/agent/codex/preparation.go index 0098920c0..468e08576 100644 --- a/apps/daemon/internal/agent/codex/preparation.go +++ b/apps/daemon/internal/agent/codex/preparation.go @@ -6,12 +6,13 @@ import ( "fmt" "os" "sync" + "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" ) -func newPreparation(parent context.Context, req proto.PromptRequestPayload, cfg sessionConfig) (*Prepared, error) { +func newPreparation(parent context.Context, req proto.PromptRequestPayload, cfg sessionConfig) (_ *Prepared, resultErr error) { if req.ExecutionControls != nil && req.ExecutionControls.OutputFormat != nil { return nil, errors.New("codex: structured output is not qualified") } @@ -34,7 +35,9 @@ func newPreparation(parent context.Context, req proto.PromptRequestPayload, cfg if err != nil { return nil, err } + planStarted := time.Now() plan, skillRoots, err := prepareSessionPlan(parent, req, cfg) + observePreparationStage(parent, "session_plan", planStarted, err) if err != nil { return nil, err } @@ -84,6 +87,8 @@ func newPreparation(parent context.Context, req proto.PromptRequestPayload, cfg if _, err := rpc.Start(cancelCtx, initParams); err != nil { return p.preparationFailed(fmt.Errorf("codex: rpc start: %w", err)) } + verificationStarted := time.Now() + defer func() { observePreparationStage(parent, "verification", verificationStarted, resultErr) }() if req.ExecutionControls != nil && req.ExecutionControls.DisableProgrammaticToolCalling { if err := verifyProgrammaticToolsDisabled(cancelCtx, rpc); err != nil { return p.preparationFailed(err) diff --git a/apps/daemon/internal/agent/codex/preparation_observation.go b/apps/daemon/internal/agent/codex/preparation_observation.go new file mode 100644 index 000000000..d006d39e5 --- /dev/null +++ b/apps/daemon/internal/agent/codex/preparation_observation.go @@ -0,0 +1,19 @@ +package codex + +import ( + "context" + "time" + + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" +) + +// Callers supply fixed stage names. Never include native error text, environment, +// catalog contents or command output; the owner context carries trace correlation. +func observePreparationStage(ctx context.Context, stage string, started time.Time, err error) { + observePreparationInterval(ctx, stage, started, time.Now(), err) +} + +func observePreparationInterval(ctx context.Context, stage string, started, ended time.Time, err error) { + obslog.Ctx(ctx).Info("codex preparation stage", "stage", stage, + "duration_ms", float64(ended.Sub(started))/float64(time.Millisecond), "success", err == nil) +} diff --git a/apps/daemon/internal/agent/codex/preparation_observation_test.go b/apps/daemon/internal/agent/codex/preparation_observation_test.go new file mode 100644 index 000000000..36b9e7063 --- /dev/null +++ b/apps/daemon/internal/agent/codex/preparation_observation_test.go @@ -0,0 +1,54 @@ +package codex + +import ( + "bytes" + "encoding/json" + "errors" + "log/slog" + "strings" + "testing" + "time" + + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" +) + +func TestPreparationObservationIsCorrelatedAndSecretSafe(t *testing.T) { + var output bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(obslog.NewContextHandler(slog.NewJSONHandler(&output, nil)))) + defer slog.SetDefault(previous) + carrier, err := obslog.ParseTraceparent("00-12345678901234567890123456789012-1234567890123456-01") + if err != nil { + t.Fatal(err) + } + ctx := obslog.WithTrace(t.Context(), carrier) + for _, err := range []error{nil, errors.New("fixture-secret-command-output")} { + observePreparationStage(ctx, "model_catalog", time.Now(), err) + } + lines := strings.Split(strings.TrimSpace(output.String()), "\n") + if len(lines) != 2 { + t.Fatal(output.String()) + } + for i, line := range lines { + var entry map[string]any + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatal(err) + } + if entry["trace_id"] != "12345678901234567890123456789012" || entry["stage"] != "model_catalog" || entry["success"] != (i == 0) { + t.Fatal(entry) + } + if entry["duration_ms"].(float64) < 0 { + t.Fatal(entry) + } + for key := range entry { + switch key { + case "time", "level", "msg", "trace_id", "span_id", "stage", "duration_ms", "success": + default: + t.Fatalf("unexpected field %s", key) + } + } + } + if strings.Contains(output.String(), "fixture-secret") { + t.Fatal("native error leaked") + } +} diff --git a/apps/daemon/internal/agent/codex/rpc.go b/apps/daemon/internal/agent/codex/rpc.go index 78607a12c..61cbae743 100644 --- a/apps/daemon/internal/agent/codex/rpc.go +++ b/apps/daemon/internal/agent/codex/rpc.go @@ -127,10 +127,11 @@ type deferReplySentinel struct{} var DeferReply = deferReplySentinel{} type pendingRequest struct { - method string - resp chan rpcResponse - timer *time.Timer - onResult func(json.RawMessage) error + method string + resp chan rpcResponse + timer *time.Timer + onResult func(json.RawMessage) error + onResponse func(bool) } type rpcResponse struct { @@ -170,7 +171,7 @@ func NewJSONRPCClient(cfg JSONRPCConfig) *JSONRPCClient { // - exec.LookPath / Start failure → returns the spawn error verbatim // - initialize timeout → kills the child, returns context.DeadlineExceeded // - JSON-RPC error on initialize → kills the child, returns the error -func (c *JSONRPCClient) Start(ctx context.Context, init InitializeParams) (InitializeResult, error) { +func (c *JSONRPCClient) Start(ctx context.Context, init InitializeParams) (_ InitializeResult, resultErr error) { args := append([]string{}, c.cfg.ExtraArgs...) args = append(args, "app-server", "--stdio") for _, f := range c.cfg.EnableFeatures { @@ -180,10 +181,12 @@ func (c *JSONRPCClient) Start(ctx context.Context, init InitializeParams) (Initi args = append(args, "--disable", f) } + spawnStarted := time.Now() process, err := clirunner.Start(clirunner.StartOptions{ Parent: ctx, Binary: c.cfg.Binary, Args: args, Dir: c.cfg.Cwd, Env: c.cfg.Env, NeedStdin: true, KillTimeout: 250 * time.Millisecond, }) + observePreparationStage(ctx, "process_spawn", spawnStarted, err) if err != nil { return InitializeResult{}, fmt.Errorf("codex rpc: spawn %q: %w", c.cfg.Binary, err) } @@ -201,6 +204,8 @@ func (c *JSONRPCClient) Start(ctx context.Context, init InitializeParams) (Initi initCtx, cancel := context.WithTimeout(ctx, rpcInitTimeout) defer cancel() + initializeStarted := time.Now() + defer func() { observePreparationStage(ctx, "rpc_initialize", initializeStarted, resultErr) }() rawResult, err := c.Request(initCtx, "initialize", init) if err != nil { _ = c.Close() @@ -370,6 +375,9 @@ func (c *JSONRPCClient) handleResponse(rawID, rawResult, rawError json.RawMessag if p.timer != nil { p.timer.Stop() } + if p.onResponse != nil { + p.onResponse(len(rawError) == 0 || string(rawError) == "null") + } if len(rawError) > 0 && string(rawError) != "null" { var errBody JsonRpcError if err := json.Unmarshal(rawError, &errBody); err != nil { diff --git a/apps/daemon/internal/agent/codex/rpc_request.go b/apps/daemon/internal/agent/codex/rpc_request.go index 6a0b7be86..80c0dd5f0 100644 --- a/apps/daemon/internal/agent/codex/rpc_request.go +++ b/apps/daemon/internal/agent/codex/rpc_request.go @@ -25,6 +25,7 @@ func (c *JSONRPCClient) requestWithResult(ctx context.Context, method string, pa } func (c *JSONRPCClient) requestWithTimeout(ctx context.Context, method string, params any, write func(any) error, timeout time.Duration, onResult func(json.RawMessage) error) (json.RawMessage, error) { + started := time.Now() if !c.Alive() { return nil, errors.New("codex rpc: client not alive") } @@ -37,12 +38,39 @@ func (c *JSONRPCClient) requestWithTimeout(ctx context.Context, method string, p resp: make(chan rpcResponse, 1), onResult: onResult, } + if method == "initialize" { + type responseObservation struct { + at time.Time + accepted bool + } + observed := make(chan responseObservation, 1) + pending.onResponse = func(success bool) { + // Only capture on the reader: logging must not delay response delivery. + observed <- responseObservation{at: time.Now(), accepted: success} + } + defer func() { + select { + case response := <-observed: + var responseErr error + if !response.accepted { + responseErr = errors.New("native initialize rejection") + } + observePreparationInterval(ctx, "rpc_initialize_response", started, response.at, responseErr) + default: + } + }() + } c.pendingMu.Lock() c.pending[id] = pending c.pendingMu.Unlock() frame := JsonRpcRequest{JsonRpc: JsonRpcVersion, ID: id, Method: method, Params: params} - if err := write(frame); err != nil { + err = write(frame) + if method == "initialize" { + ended, writeErr := time.Now(), err + defer func() { observePreparationInterval(ctx, "rpc_initialize_write", started, ended, writeErr) }() + } + if err != nil { c.pendingMu.Lock() delete(c.pending, id) c.pendingMu.Unlock() diff --git a/apps/daemon/internal/agent/codex/startup_observation_test.go b/apps/daemon/internal/agent/codex/startup_observation_test.go new file mode 100644 index 000000000..15d758233 --- /dev/null +++ b/apps/daemon/internal/agent/codex/startup_observation_test.go @@ -0,0 +1,260 @@ +package codex + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log/slog" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func captureStartupLogs(t *testing.T) *bytes.Buffer { + t.Helper() + var output bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&output, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + return &output +} + +func startupStages(t *testing.T, output *bytes.Buffer) map[string]map[string]any { + t.Helper() + stages := map[string]map[string]any{} + for _, line := range strings.Split(strings.TrimSpace(output.String()), "\n") { + if line == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(line), &row); err != nil { + t.Fatal(err) + } + if stage, ok := row["stage"].(string); ok { + if _, exists := stages[stage]; exists { + t.Fatalf("duplicate stage %s", stage) + } + stages[stage] = row + } + } + return stages +} + +func TestVersionObservationWaitsForExit(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell fixtures") + } + for _, test := range []struct { + name, script string + wantOutput, wantError bool + }{ + {"success", "printf 'codex-cli 0.153.4\\n'; sleep 0.04", true, false}, + {"late_failure", "printf 'codex-cli 0.153.4\\n'; sleep 0.04; echo private-secret >&2; exit 7", true, true}, + {"empty", "echo private-secret >&2", false, true}, + {"cancel_after_output", "printf 'codex-cli 0.153.4\\n'; exec sleep 10", true, true}, + {"spawn_failure", "", false, true}, + } { + t.Run(test.name, func(t *testing.T) { + output := captureStartupLogs(t) + binary := filepath.Join(t.TempDir(), "probe") + marker := filepath.Join(t.TempDir(), "output-written") + content := "#!/bin/sh\n" + test.script + "\n" + if test.name == "cancel_after_output" { + content = "#!/bin/sh\nprintf 'codex-cli 0.153.4\\n'; touch '" + marker + "'; exec sleep 10\n" + } + if test.name == "spawn_failure" { + content = "#!/missing-interpreter\n" + } + if err := os.WriteFile(binary, []byte(content), 0700); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + cancellationDone := make(chan struct{}) + if test.name == "cancel_after_output" { + go func() { + defer close(cancellationDone) + for ctx.Err() == nil { + if _, err := os.Stat(marker); err == nil { + time.Sleep(100 * time.Millisecond) + cancel() + return + } + time.Sleep(5 * time.Millisecond) + } + }() + } else { + close(cancellationDone) + } + version, err := CheckCLIAvailable(ctx, binary) + cancel() + <-cancellationDone + if (err != nil) != test.wantError { + t.Fatalf("version=%q err=%v", version, err) + } + if err != nil && version != "" { + t.Fatal("accepted output before failed exit") + } + stages := startupStages(t, output) + if _, present := stages["first_stdout"]; present != test.wantOutput { + t.Fatal(stages) + } + if test.wantOutput { + if stages["stdout_to_completion"]["duration_ms"].(float64) < 20 { + t.Fatal("returned before process completion", stages) + } + } + if strings.Contains(output.String(), "private-secret") || strings.Contains(output.String(), binary) { + t.Fatal("probe contents leaked") + } + }) + } +} + +func TestFirstOutputIgnoresEmptyWrites(t *testing.T) { + var b firstOutputBuffer + _, _ = b.Write(nil) + if !b.first.IsZero() { + t.Fatal("empty output acquired a timestamp") + } + _, _ = b.Write([]byte("one")) + first := b.first + _, _ = b.Write([]byte("two")) + if first.IsZero() || b.first != first || b.String() != "onetwo" { + t.Fatal("first output or bytes changed") + } +} + +func TestCatalogObservationsPreserveFailureBoundaries(t *testing.T) { + if !SupportsTextVerbosity { + t.Skip("catalog probe requires Unix") + } + for _, test := range []struct { + name, script, failedStage string + }{ + {"success", `printf '%s' '{"models":[{"slug":"known","support_verbosity":true}]}'`, ""}, + {"nonzero", `printf '%s' '{"models":[]}'; exit 7`, "model_catalog_command"}, + {"invalid", `printf '%s' '{"models":'`, "model_catalog_validation"}, + {"unsupported", `printf '%s' '{"models":[]}'`, "model_catalog_validation"}, + {"snapshot_failure", `printf '%s' '{"models":[{"slug":"known","support_verbosity":true}]}'`, "model_catalog_snapshot"}, + {"spawn_failure", "", "model_catalog_command"}, + } { + t.Run(test.name, func(t *testing.T) { + output := captureStartupLogs(t) + home := t.TempDir() + if test.name == "snapshot_failure" { + home = filepath.Join(home, "missing") + } + plan := SessionPlan{Model: "known", Cwd: t.TempDir(), Env: []string{"CODEX_HOME=" + home}, ExtraConfig: [][2]string{{"model_verbosity", `"high"`}}, Cleanup: func() {}} + defer func() { plan.Cleanup() }() + binary := filepath.Join(t.TempDir(), "probe") + content := "#!/bin/sh\n" + test.script + "\n" + if test.name == "spawn_failure" { + content = "#!/missing-interpreter\n" + } + if err := os.WriteFile(binary, []byte(content), 0700); err != nil { + t.Fatal(err) + } + err := prepareModelVerbosity(t.Context(), binary, &plan) + if (err != nil) != (test.failedStage != "") { + t.Fatal(err) + } + stages := startupStages(t, output) + if stages["model_catalog"]["success"] != (err == nil) { + t.Fatal(stages) + } + if test.failedStage != "" && stages[test.failedStage]["success"] != false { + t.Fatal(stages) + } + if test.failedStage == "model_catalog_command" && stages["model_catalog_validation"] != nil { + t.Fatal("validated failed command") + } + if test.name == "spawn_failure" && stages["model_catalog_cleanup"] != nil { + t.Fatal("unstarted process reported cleanup") + } + if test.name == "success" && stages["model_catalog_cleanup"]["success"] != false { + t.Fatal("already-exited group signal did not report its error", stages) + } + if test.failedStage == "model_catalog_validation" && stages["model_catalog_snapshot"] != nil { + t.Fatal("persisted invalid catalog") + } + if strings.Contains(output.String(), "models") || strings.Contains(output.String(), home) { + t.Fatal("catalog or path leaked") + } + }) + } +} + +func TestInitializeObservationDoesNotBlockResponseDelivery(t *testing.T) { + reader, writer := io.Pipe() + defer reader.Close() + defer writer.Close() + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(writer, nil))) + defer slog.SetDefault(previous) + c := NewJSONRPCClient(JSONRPCConfig{}) + c.alive = true + frames := make(chan JsonRpcRequest, 1) + result := make(chan error, 1) + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + go func() { + _, err := c.request(ctx, "initialize", nil, func(frame any) error { frames <- frame.(JsonRpcRequest); return nil }) + result <- err + }() + var frame JsonRpcRequest + select { + case frame = <-frames: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + id, _ := json.Marshal(frame.ID) + delivered := make(chan struct{}) + go func() { c.handleResponse(id, json.RawMessage(`{}`), nil); close(delivered) }() + select { + case <-delivered: + case <-ctx.Done(): + t.Fatal("response reader blocked on log sink") + } + _ = reader.Close() + if err := <-result; err != nil { + t.Fatal(err) + } +} + +func TestInitializeObservationRequiresMatchingResponse(t *testing.T) { + for _, rejected := range []bool{false, true} { + t.Run(map[bool]string{false: "success", true: "rejected"}[rejected], func(t *testing.T) { + output := captureStartupLogs(t) + c := NewJSONRPCClient(JSONRPCConfig{Logger: slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil))}) + c.alive = true + _, err := c.request(t.Context(), "initialize", nil, func(frame any) error { + c.handleResponse(json.RawMessage(`"unmatched"`), json.RawMessage(`{}`), nil) + if strings.Contains(output.String(), "rpc_initialize_response") { + t.Fatal("orphan acquired timing") + } + id, _ := json.Marshal(frame.(JsonRpcRequest).ID) + var nativeError json.RawMessage + if rejected { + nativeError = json.RawMessage(`{"code":1,"message":"private-native-error"}`) + } + c.handleResponse(id, json.RawMessage(`{}`), nativeError) + return nil + }) + if (err != nil) != rejected { + t.Fatal(err) + } + stages := startupStages(t, output) + if len(stages) != 2 || stages["rpc_initialize_response"]["success"] != !rejected || stages["rpc_initialize_write"]["success"] != true { + t.Fatal(stages) + } + if strings.Contains(output.String(), "private-native-error") { + t.Fatal("native response leaked") + } + }) + } +} diff --git a/apps/daemon/internal/agent/codex/version.go b/apps/daemon/internal/agent/codex/version.go index 6a93cf979..432c8399f 100644 --- a/apps/daemon/internal/agent/codex/version.go +++ b/apps/daemon/internal/agent/codex/version.go @@ -7,8 +7,10 @@ import ( "fmt" "os/exec" "strings" + "time" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent/binpath" + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" ) // InstallURL points operators at the Codex install instructions when @@ -38,11 +40,39 @@ func CheckCLIAvailable(ctx context.Context, binary string) (string, error) { return "", fmt.Errorf("%w: %s", ErrCLINotFound, binary) } - var stdout, stderr bytes.Buffer + var stdout firstOutputBuffer + var stderr bytes.Buffer cmd := exec.CommandContext(ctx, binary, "--version") cmd.Stdout = &stdout cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { + spawnAt := time.Now() + err := cmd.Start() + status := "ok" + if err != nil { + status = "error" + } + obslog.Ctx(ctx).Info("runtime version probe", "harness_kind", "codex", "stage", "process_spawn", "duration_ms", float64(time.Since(spawnAt))/float64(time.Millisecond), "status", status) + if err == nil { + waitAt := time.Now() + err = cmd.Wait() + completedAt := time.Now() + if err != nil { + status = "error" + } + obslog.Ctx(ctx).Info("runtime version probe", "harness_kind", "codex", "stage", "process_wait", "duration_ms", float64(completedAt.Sub(waitAt))/float64(time.Millisecond), "status", status) + // Wait joins the stdout copier. Inspect its timestamp only after it returns; + // receiving bytes never replaces process exit or output validation. + if !stdout.first.IsZero() { + obslog.Ctx(ctx).Info("runtime version probe", "harness_kind", "codex", "stage", "first_stdout", "duration_ms", float64(stdout.first.Sub(spawnAt))/float64(time.Millisecond)) + obslog.Ctx(ctx).Info("runtime version probe", "harness_kind", "codex", "stage", "stdout_to_completion", "duration_ms", float64(completedAt.Sub(stdout.first))/float64(time.Millisecond), "status", status) + } + if cmd.ProcessState != nil { + obslog.Ctx(ctx).Info("runtime version probe resources", "harness_kind", "codex", + "user_cpu_ms", float64(cmd.ProcessState.UserTime())/float64(time.Millisecond), + "system_cpu_ms", float64(cmd.ProcessState.SystemTime())/float64(time.Millisecond)) + } + } + if err != nil { msg := strings.TrimSpace(stderr.String()) if msg == "" { msg = err.Error() @@ -58,3 +88,19 @@ func CheckCLIAvailable(ctx context.Context, binary string) (string, error) { } return out, nil } + +// exec writes through a single stdout copier and Wait joins it. Do not embed +// bytes.Buffer: its ReaderFrom would bypass Write and lose the first-byte event. +type firstOutputBuffer struct { + buffer bytes.Buffer + first time.Time +} + +func (b *firstOutputBuffer) Write(p []byte) (int, error) { + if len(p) > 0 && b.first.IsZero() { + b.first = time.Now() + } + return b.buffer.Write(p) +} + +func (b *firstOutputBuffer) String() string { return b.buffer.String() } diff --git a/apps/daemon/internal/cli/agent_discovery.go b/apps/daemon/internal/cli/agent_discovery.go index d29b85acc..0b96f6c34 100644 --- a/apps/daemon/internal/cli/agent_discovery.go +++ b/apps/daemon/internal/cli/agent_discovery.go @@ -3,10 +3,13 @@ package cli import ( "context" "fmt" + "time" + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent/claudesdk" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent/codex" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent/mcode" + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" ) var harnessDeclarations = []agent.Declaration{codex.Declaration, mcode.Declaration, claudesdk.Declaration} @@ -27,7 +30,11 @@ func discoverAgentCLIs(parent context.Context, rc *runContext, profile string, d if rc.installedKinds != nil && !rc.installedKinds[declaration.Info.Kind] { continue } + started := time.Now() runtime := declaration.Discover(parent, agent.DiscoveryOptions{Profile: profile, Stdout: rc.stdout, Stderr: rc.stderr}, declaration.Info) + obslog.Ctx(parent).Info("runtime startup stage", "stage", "harness_discovery", + "harness_kind", declaration.Info.Kind, "duration_ms", float64(time.Since(started))/float64(time.Millisecond), + "available", runtime != nil && runtime.Info.Available) if runtime == nil { continue } diff --git a/apps/daemon/internal/cli/connect.go b/apps/daemon/internal/cli/connect.go index 0ec595e66..b6ab804b2 100644 --- a/apps/daemon/internal/cli/connect.go +++ b/apps/daemon/internal/cli/connect.go @@ -98,6 +98,7 @@ func runConnect(ctx *runContext, args []string) error { return spawnBackground(context.Background(), ctx, *profile, os.Args) } + initializeRuntimeObservations(ctx) var prof auth.Profile if bootstrapped != nil { prof = *bootstrapped @@ -203,7 +204,9 @@ func mainLoopRemoteWithDiscovery(parent context.Context, rc *runContext, profile rootCtx, cancel := daemonize.NotifyContext(parent) defer cancel() - boot, agentCLIs, err := prepareConnection(rootCtx, discover, func(ctx context.Context) (*transport.BootstrapResponse, error) { + boot, agentCLIs, err := prepareConnection(rootCtx, discover, func(ctx context.Context) (boot *transport.BootstrapResponse, err error) { + started := time.Now() + defer func() { observeRuntimeStartup(ctx, "bootstrap", started, err) }() bootCtx, stop := context.WithTimeout(ctx, bootstrapTimeout) defer stop() if remote != "" { @@ -232,6 +235,7 @@ func mainLoopRemoteWithDiscovery(parent context.Context, rc *runContext, profile defer control.Close() } dial := func(ctx context.Context) (*transport.Conn, error) { + dialStarted := time.Now() conn, err := transport.Dial(ctx, transport.DialOptions{ WSURL: wsURL, DeviceID: boot.DeviceID, @@ -242,6 +246,7 @@ func mainLoopRemoteWithDiscovery(parent context.Context, rc *runContext, profile // in heartbeat's DaemonVersion field. DaemonVersion: proto.Version, }) + observeRuntimeStartup(ctx, "transport_dial", dialStarted, err) if remote != "" && err != nil { if errors.Is(err, transport.ErrIncompatibleVersion) { return nil, fmt.Errorf("Environment connection rejected: %w: %w", transport.ErrPermanent, transport.ErrIncompatibleVersion) diff --git a/apps/daemon/internal/cli/connect_environment.go b/apps/daemon/internal/cli/connect_environment.go index 4e9628e9e..5e5f1c592 100644 --- a/apps/daemon/internal/cli/connect_environment.go +++ b/apps/daemon/internal/cli/connect_environment.go @@ -125,6 +125,7 @@ func environmentBootstrap(ctx context.Context, prof auth.Profile, remote string) } func runEnvironmentConnect(parent context.Context, rc *runContext, profile string, background bool, remote, environment, credentialFile string) error { + initializeRuntimeObservations(rc) base, err := environmentBase(remote) if err != nil { return err diff --git a/apps/daemon/internal/cli/startup_observation.go b/apps/daemon/internal/cli/startup_observation.go new file mode 100644 index 000000000..70dc54330 --- /dev/null +++ b/apps/daemon/internal/cli/startup_observation.go @@ -0,0 +1,30 @@ +package cli + +import ( + "context" + "errors" + "log/slog" + "time" + + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" +) + +func observeRuntimeStartup(ctx context.Context, stage string, started time.Time, err error) { + status := "ok" + if err != nil { + status = "error" + if errors.Is(err, context.Canceled) { + status = "cancelled" + } + if errors.Is(err, context.DeadlineExceeded) { + status = "timeout" + } + } + obslog.Ctx(ctx).Info("runtime startup stage", "stage", stage, + "duration_ms", float64(time.Since(started))/float64(time.Millisecond), "status", status) +} + +func initializeRuntimeObservations(rc *runContext) { + obslog.Init(obslog.Config{Format: "text", Level: slog.LevelInfo, Out: rc.stderr}) + obslog.Bg().Info("runtime process starting", "daemon_version", Version) +} diff --git a/apps/daemon/internal/dispatch/executor.go b/apps/daemon/internal/dispatch/executor.go index 0e47e8c20..9c18e9ee2 100644 --- a/apps/daemon/internal/dispatch/executor.go +++ b/apps/daemon/internal/dispatch/executor.go @@ -180,7 +180,9 @@ func (r *Router) prepareExecutor(p *preparationState, req proto.PromptRequestPay var native agent.Executor var err error if owner.ctx.Err() == nil { + workspaceStarted := time.Now() req, err = r.localWorkspace.Prepare(owner.ctx, req) + r.log.InfoContext(owner.ctx, "executor preparation stage", "stage", "workspace", "executor_id", owner.id, "session_id", owner.sessionID, "duration_ms", float64(time.Since(workspaceStarted))/float64(time.Millisecond), "success", err == nil) if err == nil && owner.ctx.Err() == nil { native, err = factory(owner.ctx, req) } diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index 0ea13768f..8eac6f815 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -27,6 +27,16 @@ The examples use the default installation directory. On Windows, invoke the mana After local credentials and enrollment are resolved, Runtime Harness discovery and the authenticated bootstrap HTTP request run concurrently. Both must succeed before the Runtime opens its connection or publishes capabilities. Failure cancels the sibling operation and waits for cleanup. The executor remains owned by the connection lifetime. Runtime startup changes require a rebuilt, qualified Runtime template. +## Runtime startup latency + +The daemon logs `executor preparation stage` with `stage=workspace`, the executor and Session IDs, duration in milliseconds and `success`. Codex logs `codex preparation stage` for `session_plan`, `model_catalog`, `process_spawn`, `rpc_initialize` and `verification`, with the owner trace, duration and `success`. These records contain no native error text, credentials, configuration, catalog contents or command output. An omitted conditional stage is unobserved, not zero. `session_plan` contains `model_catalog`; the executor readiness interval contains workspace preparation, the adapter stages and transport overhead. Do not add nested intervals together. A failed `rpc_initialize` includes its required child cleanup. + +Within `model_catalog`, `model_catalog_command` includes the original native command, exit and pipe draining; `model_catalog_cleanup` measures the existing best-effort process-group signal only when a process was started. Its `success` reports the signal return value, so an already-exited group can report false; it does not prove descendants were reaped and does not change the command outcome. `model_catalog_validation` covers JSON/model/verbosity and managed-home validation; `model_catalog_snapshot` covers snapshot creation, write, close and attachment to the Session plan. Failed stages retain the original rejection and cleanup behavior. + +`rpc_initialize_write` and `rpc_initialize_response` are cumulative milestones from the same request start, not additive intervals or Core network RTT. The response timestamp is captured only for a matching response ID before native result decoding; its `success` means the frame has no native error, not that the handshake is valid. The existing `rpc_initialize` parent reports the complete decoded handshake and any required failure cleanup. Missing replies have no response milestone. The reader captures the timestamp without logging; records are emitted after request resolution, so log timestamps are export time. The 10-second initialize deadline and Session-owned process lifetime are unchanged. + +The model catalog remains validated and pinned before app-server initialization. These timings distinguish catalog preparation from native process initialization; they do not establish a latency improvement. Compare fresh and reused Sessions on the same Runtime template, model and provider, and verify persisted replies and usage in addition to first-text latency. A Runtime startup change needs a rebuilt, qualified Runtime template; replacing Core alone does not update existing sandboxes. + ## Service health Use these observations for different questions: @@ -191,6 +201,14 @@ Web signs administrators in with the Core key, checks the origin of every reques Sandboxes are the isolation boundary ([Runtime and outer isolation](../concepts.md#runtime-and-outer-isolation)). Docker sandboxes share the node's kernel, and a Docker node is [root-equivalent](./nodes.md#what-the-installer-sets-up) on its host; microsandbox gives each sandbox a microVM with an explicit [network policy](./nodes.md#what-the-installer-sets-up). Core itself has no Docker socket or KVM access. -## Runtime availability probes +## Execution latency + +Use the `environment input reserved` log to connect the submitting HTTP trace to `reservation_id` and `execution_trace_id`. The worker derives its diagnostic execution trace from the durable reservation identity, so observer disconnection and owner restart do not require a process-local trace map. This is an explicit link between the HTTP and execution traces, not continuation of the original HTTP span. Follow `environment input selected`, `execution preparation requested` and `environment input admitted` to connect the reservation to the Session, Environment, device, preparation request, Executor and Turn. `preparation_request_id` identifies the control request and is distinct from an HTTP `request_id`. For inputs included in Session creation, join `session initial input origin` to the worker reservation with the same `session_id` and `is_initial=true`; this covers both regular and streaming creation without an extra store query. + +`execution stage` records `stage`, `duration_ms` and `status` (`ok`, `error`, `cancelled` or `timeout`). Stages cover input validation/ownership, reservation and admission wait, execution configuration and admission promotion, lifecycle gate wait, Runtime observation/provisioning, provider create/resume and Environment initialization. Runtime connection logs record confirmed observed connection transitions, not the exact socket connection instant. Use Environment, allocation and node IDs to join background Runtime records to an input; a background record without the input trace is not an unbroken request span. `runtime transport registered` records authenticated WebSocket registration. `runtime capability snapshot observed` records validated declaration changes with device ID and kind counts; transport registration alone does not establish execution readiness. Capability changes with an available Harness send a coalesced scheduling hint, and the Worker rechecks its ordinary ownership, capacity, lifecycle and capability requirements; polling recovers missed hints. Stage durations use a local monotonic clock. `queue_age_ms` instead compares selection time to the database reservation creation time and may reflect clock skew; it is not a measured scheduler-only interval. + +`control_ready_ms` starts after scheduling, Runtime readiness and configuration assembly. `start_control_ms` describes the control acknowledgement. `input_to_first_text_ms` starts immediately before execution delivery, includes start-control time and is not model-only TTFT. The admission, control and first-text intervals overlap; do not sum them as independent durations. A successful provider call describes that operation, not completed daemon connection or a completed Turn. Model request start, response headers, first model text and retries remain unavailable unless the selected native adapter provides those observations. Logs contain correlation IDs and finite status categories, not input content, credentials or raw provider errors. + +Daemon startup logs record `runtime process starting` with its build version and `runtime startup stage` for each Harness discovery, bootstrap and transport dial attempt. Durations use a local monotonic clock and omit credentials and raw errors. Discovery runs before transport registration. Compare daemon timestamps with Core registration and capability observations to separate startup from periodic observation and scheduling. These daemon records require a Runtime built with this instrumentation; updating Core alone does not update an existing Runtime template. A managed startup receipt confirms process spawn, not connection or capability readiness. -Codex CLI availability emits `runtime version probe` records for `process_spawn` and `process_wait`, with duration and outcome only. The wait interval includes executable loading and the version command; neither interval is model execution. +Codex CLI availability emits `runtime version probe` records for `process_spawn` and `process_wait`. `first_stdout` measures from the spawn attempt to the first nonempty stdout chunk observed by the Go copier; `stdout_to_completion` measures from that observation to `Wait` returning, including remaining execution and pipe draining. Both are omitted when stdout is absent. They are emitted after completion and never authorize early success: process exit and output checks still decide availability. `runtime version probe resources` reports the OS process accounting returned by `ProcessState` as `user_cpu_ms` and `system_cpu_ms`; this may include reaped descendants according to the platform, and is omitted when process accounting is unavailable. Fault and storage-byte counters are not collected; their absence is unavailable, not zero or evidence of a storage/network cause. None of these intervals is model execution. diff --git a/docs/zh/getting-started/operations.md b/docs/zh/getting-started/operations.md index cfe626664..78c2503d0 100644 --- a/docs/zh/getting-started/operations.md +++ b/docs/zh/getting-started/operations.md @@ -1,7 +1,7 @@ --- title: "运维" source: docs/getting-started/operations.md -source_hash: f6ce9d76fdd074d9e7e7da4a69da6a4583a67848b86aa641ad09f127fcae346b +source_hash: 516b0768cbe6eeb000b982b29ea46849173a8f319f00b42c2af917ab56ebbda5 --- 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。 @@ -29,6 +29,16 @@ docker compose -f ~/.oac/core/compose.yaml ps 本地凭据和注册解析完成后，Runtime 的 Harness 发现与已认证的引导 HTTP 请求并发执行。两者都成功后才建立连接并发布能力。失败会取消另一项操作并等待其清理。执行器仍由连接生命周期持有。Runtime 启动变更需要重新构建并验证 Runtime 模板。 +## Runtime 启动延迟 {#runtime-startup-latency} + +daemon 通过 `executor preparation stage` 记录 `stage=workspace`、executor 和 Session ID、毫秒耗时及 `success`。Codex 通过 `codex preparation stage` 记录 `session_plan`、`model_catalog`、`process_spawn`、`rpc_initialize` 和 `verification`，携带所属请求的 trace、耗时及 `success`。这些记录不包含原生错误文本、凭据、配置、模型目录内容或命令输出。未执行的条件阶段表示未观测，不能按零计算。`session_plan` 包含 `model_catalog`；executor 就绪耗时包含工作区准备、adapter 各阶段和传输开销，不应重复相加。失败的 `rpc_initialize` 包含必要的子进程清理。 + +`model_catalog` 内，`model_catalog_command` 包含原生命令执行、退出和管道排空；只有进程已启动才输出 `model_catalog_cleanup`，它仅测量既有尽力发送的进程组信号。其 `success` 是信号调用的返回结果，进程组已退出也可能为 false；不证明后代已回收，也不改变命令结果。`model_catalog_validation` 包含 JSON、模型、verbosity 及托管 home 校验；`model_catalog_snapshot` 包含快照创建、写入、关闭及绑定 Session plan。失败保持原有拒绝和清理行为。 + +`rpc_initialize_write`、`rpc_initialize_response` 从同一请求起点累计计时，不能相加，也不是 Core 网络 RTT。仅在匹配响应 ID 时、原生结果解码前记录响应时刻；其 `success` 只表示帧中无原生错误，不代表握手有效。原有 `rpc_initialize` 父阶段记录完整解码握手及失败所需清理。没有匹配响应时不输出响应项。读取协程只捕获时间，不写日志；请求结束后才导出记录，因此日志时间是导出时刻。initialize 的 10 秒期限及 Session 持有的进程生命周期保持不变。 + +app-server 初始化前仍会校验并固定模型目录。阶段计时用于区分目录准备与原生进程初始化，本身不证明提速。应在相同 Runtime 模板、模型和 Provider 下对比全新及复用 Session，并同时验证持久化回复、用量和首字延迟。Runtime 启动逻辑变更需要重新构建和验收 Runtime 模板；仅替换 Core 不会更新既有沙箱。 + ## 服务健康状态 {#service-health} 根据不同问题使用这些观察： @@ -194,6 +204,16 @@ Web 使用 Core 密钥让管理员登录，检查每个请求来源，并用保 沙箱是隔离边界（[Runtime 与外层隔离](../concepts.md#runtime-and-outer-isolation)）。Docker 沙箱共享节点内核，Docker 节点在主机上[等同于 root 权限](nodes.md#what-the-installer-sets-up)；microsandbox 为每个沙箱提供具有显式[网络策略](nodes.md#what-the-installer-sets-up)的 microVM。Core 自身无 Docker 套接字或 KVM 访问权限。 -## Runtime 可用性探测 {#runtime-availability-probes} +## 执行延迟 {#execution-latency} + +通过 `environment input reserved` 日志将提交输入的 HTTP Trace 与 `reservation_id`、`execution_trace_id` 关联。worker 从持久化预留标识派生执行诊断 Trace，观察连接断开或 owner 重启都不依赖进程内 Trace 映射。这是 HTTP Trace 与执行 Trace 之间的明确关联，不是原 HTTP span 的延续。沿 `environment input selected`、`execution preparation requested`、`environment input admitted` 将预留与 Session、Environment、device、准备请求、Executor、Turn 关联。`preparation_request_id` 标识控制准备请求，与 HTTP `request_id` 不同。创建 Session 时附带输入的情况，通过相同 `session_id` 和 `is_initial=true` 将 `session initial input origin` 与 worker 的预留关联；普通和流式创建都支持，不增加 store 查询。 + +`execution stage` 记录 `stage`、`duration_ms` 与 `status`（`ok`、`error`、`cancelled`、`timeout`）。阶段覆盖输入验证/ownership、预留和等待 admission、执行配置与 admission 提升、生命周期 gate 等待、Runtime 观察和供给、provider create/resume、Environment 初始化。Runtime 连接日志记录已确认观察到的连接变化，不是 socket 建立的精确时间。通过 Environment、allocation、node ID 将后台 Runtime 记录与输入关联；缺少输入 Trace 的后台记录不能视为连续的请求 span。阶段耗时使用本进程单调时钟。`queue_age_ms` 则比较选择时间与数据库预留创建时间，可能受到时钟偏差影响，不能当作单独测量的调度等待。 + +`control_ready_ms` 从调度、Runtime 就绪和配置组装完成后开始。`start_control_ms` 描述控制确认。`input_to_first_text_ms` 从执行投递前开始，包含 start-control 时间，不是纯模型 TTFT。admission、control、首字区间存在重叠，不要当作独立耗时相加。provider 调用成功表示该操作成功，不等于 daemon 连接完成或 Turn 完成。除非原生适配器提供相应观测，模型请求开始、响应 headers、模型首字、重试仍为不可观测。日志仅记录关联 ID 与有限状态分类，不记录输入内容、凭据或 provider 原始错误。 + +`runtime transport registered` 记录认证后的 WebSocket 注册，`runtime capability snapshot observed` 记录合法能力声明变化及设备 ID、能力数量。连接注册本身不代表执行就绪。包含可用 Harness 的能力变化会发出合并后的调度提示；Worker 仍检查所有权、容量、生命周期和能力要求，轮询负责兜底。 + +Daemon 启动日志记录 `runtime process starting` 和构建版本，`runtime startup stage` 记录各 Harness 探测、bootstrap 和每次连接尝试的本地单调时钟耗时，不输出凭据或原始错误。探测发生在连接注册之前。结合这些边界与已观察到的持久化连接时间，区分启动、周期观察和调度等待。Daemon 观测要求 Runtime 使用带有这些埋点的构建；仅升级 Core 不会升级现有 Runtime 模板。托管启动回执只确认进程已启动，不确认连接或能力就绪。 -Codex CLI 可用性探测为 `process_spawn` 和 `process_wait` 输出 `runtime version probe` 记录，只包含耗时和结果。等待区间包含可执行文件加载和版本命令，两者都不是模型执行。 +Codex CLI 可用性探测输出 `process_spawn`、`process_wait` 的 `runtime version probe` 记录。`first_stdout` 从启动尝试开始，计时到 Go 复制协程观察到第一块非空 stdout；`stdout_to_completion` 从该时刻计时到 `Wait` 返回，包含剩余执行和管道排空。无 stdout 时不输出这两项。记录在完成后导出，不允许提前成功：仍以进程退出和输出校验决定可用性。`runtime version probe resources` 使用 `ProcessState` 返回的 OS 进程统计，输出 `user_cpu_ms`、`system_cpu_ms`；平台可能包含已回收后代，无法取得进程统计时省略。缺页和存储字节计数未采集，缺失表示不可观测，不能按零计算，也不能据此认定存储或网络根因。这些区间均不是模型执行。 diff --git a/services/core/deploy/e2b/init.py b/services/core/deploy/e2b/init.py index 75d8e1f07..ede349cc1 100644 --- a/services/core/deploy/e2b/init.py +++ b/services/core/deploy/e2b/init.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 """One-shot user-owned Runtime startup; never an enrollment or execution service.""" import json +from contextlib import contextmanager import os from pathlib import Path import subprocess +import sys +import time from uuid import UUID from urllib.parse import urlsplit @@ -12,6 +15,23 @@ IMAGE_ENV = Path('/etc/oac-runtime-env.json') +@contextmanager +def bootstrap_stage(stage): + """Fixed numeric guest timings; receipts and command stdout stay unchanged.""" + started = time.monotonic_ns() + completed = False + try: + yield + completed = True + finally: + try: + sys.stderr.write(json.dumps({'event': 'e2b_bootstrap_stage', 'stage': stage, + 'duration_us': (time.monotonic_ns() - started) // 1000, + 'completed': completed}) + '\n') + except Exception: + pass + + def launch_identity(payload): """Reject malformed startup input without including confidential values in errors.""" if not isinstance(payload, dict) or set(payload) != { @@ -65,30 +85,32 @@ def write_private(path, value, owner=None): def prepare_runtime(): """Restore the protected image and shared Runtime layout before any daemon starts.""" - # E2B finalization makes /usr/local world-writable after template commands. - subprocess.run(['chown', '-R', 'root:root', '/usr/local'], check=True) - subprocess.run(['chmod', '-R', 'go-w', '/usr/local'], check=True) - for protected in ['/usr/bin/envd', '/etc/inittab', '/etc/init.d/rcS']: - if protected == '/etc/init.d/rcS' and not Path(protected).exists(): - continue - os.chown(protected, 0, 0) - os.chmod(protected, 0o755) - # Disable E2B's unused passwordless sudo account before unprivileged startup. - subprocess.run(['usermod', '--lock', '--shell', '/usr/sbin/nologin', 'user'], check=True) - environment = json.loads(IMAGE_ENV.read_text()) - if (environment.get('OAC_RUNTIME_HOME') != '/home/runtime/.oac' - or environment.get('OAC_RUNTIME_WORKSPACE') != '/environment/workspace' - or any(key in environment for key in ['OAC_RUNTIME_ENVIRONMENT_ID', - 'OAC_RUNTIME_SESSION_ID'])): - raise ValueError('Image must contain an unbound packaged Runtime profile') - environment['PATH'] = '/usr/local/bin:/usr/bin:/bin' - subprocess.run(['mount', '--bind', '/environment/workspace', '/workspace'], check=True) - PROFILE.mkdir(mode=0o700, parents=True, exist_ok=True) - for directory in [Path('/home/runtime'), Path('/home/runtime/.oac'), PROFILE.parent, PROFILE, - Path('/environment/workspace'), Path('/environment/staging'), - Path('/environment/initialization'), Path('/environment/packages')]: - os.chown(directory, 1000, 1000) - directory.chmod(0o700) + with bootstrap_stage('bootstrap_protection'): + # E2B finalization makes /usr/local world-writable after template commands. + subprocess.run(['chown', '-R', 'root:root', '/usr/local'], check=True) + subprocess.run(['chmod', '-R', 'go-w', '/usr/local'], check=True) + for protected in ['/usr/bin/envd', '/etc/inittab', '/etc/init.d/rcS']: + if protected == '/etc/init.d/rcS' and not Path(protected).exists(): + continue + os.chown(protected, 0, 0) + os.chmod(protected, 0o755) + # Disable E2B's unused passwordless sudo account before unprivileged startup. + subprocess.run(['usermod', '--lock', '--shell', '/usr/sbin/nologin', 'user'], check=True) + with bootstrap_stage('bootstrap_layout'): + environment = json.loads(IMAGE_ENV.read_text()) + if (environment.get('OAC_RUNTIME_HOME') != '/home/runtime/.oac' + or environment.get('OAC_RUNTIME_WORKSPACE') != '/environment/workspace' + or any(key in environment for key in ['OAC_RUNTIME_ENVIRONMENT_ID', + 'OAC_RUNTIME_SESSION_ID'])): + raise ValueError('Image must contain an unbound packaged Runtime profile') + environment['PATH'] = '/usr/local/bin:/usr/bin:/bin' + subprocess.run(['mount', '--bind', '/environment/workspace', '/workspace'], check=True) + PROFILE.mkdir(mode=0o700, parents=True, exist_ok=True) + for directory in [Path('/home/runtime'), Path('/home/runtime/.oac'), PROFILE.parent, PROFILE, + Path('/environment/workspace'), Path('/environment/staging'), + Path('/environment/initialization'), Path('/environment/packages')]: + os.chown(directory, 1000, 1000) + directory.chmod(0o700) return environment diff --git a/services/core/deploy/e2b/managed_init.py b/services/core/deploy/e2b/managed_init.py index 588f69d80..30b91e65e 100644 --- a/services/core/deploy/e2b/managed_init.py +++ b/services/core/deploy/e2b/managed_init.py @@ -37,37 +37,41 @@ def identity(payload): def initialize(): - root = shared.ROOT - root.mkdir(mode=0o700, parents=True, exist_ok=True) - root.chmod(0o700) - source = root / 'managed-bootstrap.json' - if any((root / name).exists() for name in ['launch.json', 'ready.json', 'managed-launch.json', 'managed-ready.json']): - raise RuntimeError('Runtime bootstrap cannot be replayed') - if source.stat().st_size > 65536: - raise ValueError('Managed bootstrap input too large') - payload = json.loads(source.read_text()) - binding = identity(payload) - shared.write_private(root / 'managed-launch.json', binding) + with shared.bootstrap_stage('bootstrap_claim'): + root = shared.ROOT + root.mkdir(mode=0o700, parents=True, exist_ok=True) + root.chmod(0o700) + source = root / 'managed-bootstrap.json' + if any((root / name).exists() for name in ['launch.json', 'ready.json', 'managed-launch.json', 'managed-ready.json']): + raise RuntimeError('Runtime bootstrap cannot be replayed') + if source.stat().st_size > 65536: + raise ValueError('Managed bootstrap input too large') + payload = json.loads(source.read_text()) + binding = identity(payload) + shared.write_private(root / 'managed-launch.json', binding) environment = shared.prepare_runtime() - environment.update(OAC_RUNTIME_ENVIRONMENT_ID=payload['EnvironmentID'], - OAC_RUNTIME_SESSION_ID=payload['SessionID'], - OAC_RUNTIME_NETWORK_ACCESS=payload['NetworkAccess'], - OAC_RUNTIME_ALLOWED_DOMAINS=json.dumps(payload['AllowedDomains'] or [])) - connection = Path(environment['OAC_RUNTIME_HOME']).parent / 'runtime-bootstrap.json' - shared.write_private(connection, payload['RuntimeBootstrap'], owner=1000) - source.unlink() - with (shared.PROFILE / 'daemon.log').open('xb') as stream: - os.fchmod(stream.fileno(), 0o600) - os.fchown(stream.fileno(), 1000, 1000) - child = subprocess.Popen(['/usr/local/bin/oac-daemon', 'connect', '--profile', 'default', - '--bootstrap-file', str(connection)], - cwd='/environment/workspace', env=environment, user=1000, group=1000, - extra_groups=[], start_new_session=True, stdin=subprocess.DEVNULL, - stdout=stream, stderr=subprocess.STDOUT, umask=0o077) - shared.write_private(root / 'managed-ready.tmp', - {'identity': binding, 'status': 'daemon_started', 'daemon_pid': child.pid}) - os.replace(root / 'managed-ready.tmp', root / 'managed-ready.json') - shared.sync_directory(root) + with shared.bootstrap_stage('bootstrap_credentials'): + environment.update(OAC_RUNTIME_ENVIRONMENT_ID=payload['EnvironmentID'], + OAC_RUNTIME_SESSION_ID=payload['SessionID'], + OAC_RUNTIME_NETWORK_ACCESS=payload['NetworkAccess'], + OAC_RUNTIME_ALLOWED_DOMAINS=json.dumps(payload['AllowedDomains'] or [])) + connection = Path(environment['OAC_RUNTIME_HOME']).parent / 'runtime-bootstrap.json' + shared.write_private(connection, payload['RuntimeBootstrap'], owner=1000) + source.unlink() + with shared.bootstrap_stage('bootstrap_spawn'): + with (shared.PROFILE / 'daemon.log').open('xb') as stream: + os.fchmod(stream.fileno(), 0o600) + os.fchown(stream.fileno(), 1000, 1000) + child = subprocess.Popen(['/usr/local/bin/oac-daemon', 'connect', '--profile', 'default', + '--bootstrap-file', str(connection)], + cwd='/environment/workspace', env=environment, user=1000, group=1000, + extra_groups=[], start_new_session=True, stdin=subprocess.DEVNULL, + stdout=stream, stderr=subprocess.STDOUT, umask=0o077) + with shared.bootstrap_stage('bootstrap_receipt'): + shared.write_private(root / 'managed-ready.tmp', + {'identity': binding, 'status': 'daemon_started', 'daemon_pid': child.pid}) + os.replace(root / 'managed-ready.tmp', root / 'managed-ready.json') + shared.sync_directory(root) if __name__ == '__main__': diff --git a/services/core/deploy/e2b/managed_init_test.py b/services/core/deploy/e2b/managed_init_test.py index f38a92ffd..f71f102a0 100644 --- a/services/core/deploy/e2b/managed_init_test.py +++ b/services/core/deploy/e2b/managed_init_test.py @@ -1,5 +1,6 @@ """Managed bootstrap reuses image protection without changing self-hosted enrollment.""" import copy +import io import json from pathlib import Path import tempfile @@ -64,7 +65,9 @@ def exercise(self, failed=False): process.side_effect = RuntimeError('private process diagnostic') image_env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'OAC_RUNTIME_HOME': str(Path(temporary) / '.oac'), 'OAC_RUNTIME_WORKSPACE': '/environment/workspace'} - with patch.object(managed_init.shared, 'ROOT', root), patch.object(managed_init.shared, 'PROFILE', profile), \ + diagnostics = io.StringIO() + with patch.object(managed_init.shared.sys, 'stderr', diagnostics), \ + patch.object(managed_init.shared, 'ROOT', root), patch.object(managed_init.shared, 'PROFILE', profile), \ patch.object(managed_init.shared, 'prepare_runtime', return_value=image_env), \ patch.object(managed_init.os, 'fchown'), patch.object(managed_init.subprocess, 'Popen', process): if failed: @@ -72,6 +75,16 @@ def exercise(self, failed=False): managed_init.initialize() else: managed_init.initialize() + rows = [json.loads(line) for line in diagnostics.getvalue().splitlines()] + self.assertEqual([row['stage'] for row in rows], [ + 'bootstrap_claim', 'bootstrap_credentials', 'bootstrap_spawn'] + + ([] if failed else ['bootstrap_receipt'])) + self.assertEqual(rows[-1]['completed'], not failed) + for row in rows: + self.assertEqual(set(row), {'event', 'stage', 'duration_us', 'completed'}) + self.assertGreaterEqual(row['duration_us'], 0) + self.assertNotIn('private', diagnostics.getvalue()) + self.assertNotIn(str(root), diagnostics.getvalue()) self.assertTrue((root / 'managed-launch.json').exists()) connection_file = Path(temporary) / 'runtime-bootstrap.json' self.assertEqual(json.loads(connection_file.read_text()), data['RuntimeBootstrap']) @@ -94,6 +107,21 @@ def exercise(self, failed=False): managed_init.initialize() process.assert_called_once() + def test_observation_sink_failure_preserves_business_error(self): + with patch.object(managed_init.shared.sys, 'stderr') as stream: + stream.write.side_effect = OSError('private sink failure') + with self.assertRaisesRegex(RuntimeError, 'business failure'): + with managed_init.shared.bootstrap_stage('bootstrap_claim'): + raise RuntimeError('business failure') + + def test_observation_uses_monotonic_elapsed_time(self): + stream = io.StringIO() + with patch.object(managed_init.shared.sys, 'stderr', stream), \ + patch.object(managed_init.shared.time, 'monotonic_ns', side_effect=[1000, 2501000]): + with managed_init.shared.bootstrap_stage('bootstrap_claim'): + pass + self.assertEqual(json.loads(stream.getvalue())['duration_us'], 2500) + def test_managed_credentials_and_environment(self): self.exercise() diff --git a/services/core/internal/execution/delivery.go b/services/core/internal/execution/delivery.go index 35b4c828b..682523341 100644 --- a/services/core/internal/execution/delivery.go +++ b/services/core/internal/execution/delivery.go @@ -7,6 +7,8 @@ import ( "strconv" "time" + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -37,7 +39,11 @@ func requestCancellation(ctx context.Context, peer *runtimegateway.Session, runI } func send(ctx context.Context, peer *runtimegateway.Session, kind, runID string, payload any) error { - env, err := proto.NewEnvelope(kind, runID, payload) + trace := "" + if carrier, ok := obslog.TraceFromContext(ctx); ok { + trace = carrier.String() + } + env, err := proto.NewEnvelopeWithTrace(kind, runID, payload, trace) if err != nil { return err } diff --git a/services/core/internal/execution/directory_preparation.go b/services/core/internal/execution/directory_preparation.go index e159eace3..c96ff5e44 100644 --- a/services/core/internal/execution/directory_preparation.go +++ b/services/core/internal/execution/directory_preparation.go @@ -29,7 +29,7 @@ func (d *Dispatcher) withPreparedWorkspace(owner context.Context, peer *runtimeg if err := d.configurePreparedEnvironment(session, environment, bound, &req); err != nil { return ErrExecutionUnavailable } - prepared, err := newPreparedStart(peer) + prepared, err := newPreparedStart(owner, peer) if err != nil { return ErrExecutionUnavailable } diff --git a/services/core/internal/execution/environment_admission.go b/services/core/internal/execution/environment_admission.go index 18ec95ac5..c1ee9c902 100644 --- a/services/core/internal/execution/environment_admission.go +++ b/services/core/internal/execution/environment_admission.go @@ -7,6 +7,8 @@ import ( "slices" "time" + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" ) @@ -75,11 +77,17 @@ func (w *Worker) validateCreation(ctx context.Context, input sessions.CreateSess return nil } -func (w *Worker) submitEnvironmentInputs(ctx context.Context, session sessions.Session, key string, inputs []sessions.Input) ([]sessions.InputReceipt, error) { - if err := w.validateEnvironmentAdmission(ctx, session.Engine, session.Configuration); err != nil { +func (w *Worker) submitEnvironmentInputs(ctx context.Context, session sessions.Session, key string, inputs []sessions.Input) (receipts []sessions.InputReceipt, err error) { + checkAt := time.Now() + err = w.validateEnvironmentAdmission(ctx, session.Engine, session.Configuration) + observeExecutionStage(ctx, "input_validate", checkAt, err, "session_id", session.ID) + if err != nil { return nil, err } - if err := w.checkAdmissionOwnership(ctx); err != nil { + checkAt = time.Now() + err = w.checkAdmissionOwnership(ctx) + observeExecutionStage(ctx, "input_ownership", checkAt, err, "session_id", session.ID) + if err != nil { return nil, err } kind := "" @@ -92,12 +100,21 @@ func (w *Worker) submitEnvironmentInputs(ctx context.Context, session sessions.S } changed, unsubscribe := w.dispatcher.notifications.subscribe(session.TenantID, session.ID) defer unsubscribe() + reservedAt := time.Now() reserve, cancel := context.WithTimeout(ctx, 5*time.Second) reservation, err := w.dispatcher.Sessions.ReserveEnvironmentInput(reserve, session.TenantID, session.ID, key, inputs) cancel() + observeExecutionStage(ctx, "input_reserve", reservedAt, err, "session_id", session.ID) if err != nil { return nil, err } + obslog.Ctx(ctx).Info("environment input reserved", "session_id", session.ID, + "reservation_id", reservation.ID, "execution_trace_id", reservationTraceID(reservation.ID).String(), "state", reservation.State) + admittedAt := time.Now() + defer func() { + observeExecutionStage(ctx, "input_admission_wait", admittedAt, err, + "session_id", session.ID, "reservation_id", reservation.ID, "state", reservation.State) + }() w.wakeScheduler() if reservation.State == sessions.EnvironmentInputPending { w.hintRuntimeWake(ctx, session) diff --git a/services/core/internal/execution/execution_observation.go b/services/core/internal/execution/execution_observation.go new file mode 100644 index 000000000..077e451c6 --- /dev/null +++ b/services/core/internal/execution/execution_observation.go @@ -0,0 +1,46 @@ +package execution + +import ( + "context" + "crypto/sha256" + "errors" + "time" + + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" +) + +// A reservation survives observer disconnects and owner restarts. Its opaque +// identity anchors the execution trace; the submitting HTTP trace is linked in +// the reservation log rather than retained in a process-local map. +func reservationTraceID(reservation string) obslog.TraceID { + digest := sha256.Sum256([]byte("oac.environment-input:" + reservation)) + var id obslog.TraceID + copy(id[:], digest[:len(id)]) + return id +} + +func reservationTrace(ctx context.Context, reservation string) context.Context { + return obslog.WithTrace(ctx, obslog.Carrier{Trace: reservationTraceID(reservation), Span: obslog.NewSpanID()}) +} + +// Stage intervals use a local monotonic clock. Error prose and request content +// are deliberately excluded: provider errors may contain private configuration. +func observeExecutionStage(ctx context.Context, stage string, started time.Time, err error, attrs ...any) { + status := "ok" + switch { + case errors.Is(err, context.Canceled): + status = "cancelled" + case errors.Is(err, context.DeadlineExceeded): + status = "timeout" + case err != nil: + status = "error" + } + fields := []any{"stage", stage, "duration_ms", float64(time.Since(started).Microseconds()) / 1000, "status", status} + obslog.Ctx(ctx).Info("execution stage", append(fields, attrs...)...) +} + +// Initial input is committed by Session creation. Joining its origin to the +// worker's is_initial reservation avoids an additional diagnostic store query. +func recordInitialInputOrigin(ctx context.Context, session string) { + obslog.Ctx(ctx).Info("session initial input origin", "session_id", session) +} diff --git a/services/core/internal/execution/execution_observation_test.go b/services/core/internal/execution/execution_observation_test.go new file mode 100644 index 000000000..45eae02e3 --- /dev/null +++ b/services/core/internal/execution/execution_observation_test.go @@ -0,0 +1,178 @@ +package execution + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "strings" + "sync" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" +) + +func captureExecutionLogs(t *testing.T) *bytes.Buffer { + t.Helper() + var out bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(obslog.NewContextHandler(slog.NewJSONHandler(&out, nil)))) + t.Cleanup(func() { slog.SetDefault(previous) }) + return &out +} + +func TestReservationTraceSurvivesRecoveryWithoutRetainingObserver(t *testing.T) { + parent, cancel := context.WithCancel(t.Context()) + first := reservationTrace(parent, "reservation-one") + recovered := reservationTrace(t.Context(), "reservation-one") + a, _ := obslog.TraceFromContext(first) + b, _ := obslog.TraceFromContext(recovered) + if a.Trace != b.Trace || a.Span == b.Span || a.Trace == reservationTraceID("reservation-two") { + t.Fatal("reservation recovery did not retain identity with a distinct attempt span") + } + if _, err := obslog.ParseTraceparent(a.String()); err != nil { + t.Fatal(err) + } + cancel() + if !errors.Is(first.Err(), context.Canceled) || recovered.Err() != nil { + t.Fatal("diagnostic context changed cancellation ownership") + } +} + +func TestExecutionStageStatusAndCredentialRedaction(t *testing.T) { + out := captureExecutionLogs(t) + ctx := reservationTrace(t.Context(), "reservation-one") + for _, sample := range []struct { + err error + status string + }{ + {nil, "ok"}, {context.Canceled, "cancelled"}, {context.DeadlineExceeded, "timeout"}, + {errors.New("private provider URL and credential must never appear"), "error"}, + } { + out.Reset() + observeExecutionStage(ctx, "provider_create", time.Now().Add(-time.Millisecond), sample.err, "allocation_id", "allocation-one") + var entry map[string]any + if err := json.Unmarshal(out.Bytes(), &entry); err != nil { + t.Fatal(err) + } + if entry["status"] != sample.status || entry["stage"] != "provider_create" || entry["trace_id"] != reservationTraceID("reservation-one").String() || entry["duration_ms"].(float64) < 1 { + t.Fatal("stage identity/status/duration lost", entry) + } + if strings.Contains(out.String(), "credential") || strings.Contains(out.String(), "private provider") { + t.Fatal("raw provider error leaked") + } + } +} + +func TestExecutorControlLogsDistinguishPreparationFromHTTP(t *testing.T) { + out := captureExecutionLogs(t) + ctx := obslog.WithRequestID(reservationTrace(t.Context(), "reservation-one"), "http-one") + prepared := &preparedStart{ctx: ctx, requestID: "prepare-one", executorID: "executor-one", createdAt: time.Now(), startSentAt: time.Now()} + recordExecutorReadiness(prepared, proto.PreparationStatusPayload{ExecutorID: "executor-one", Reused: true}) + recordExecutorStart(prepared, "turn-one") + for _, line := range bytes.Split(bytes.TrimSpace(out.Bytes()), []byte("\n")) { + var entry map[string]any + if err := json.Unmarshal(line, &entry); err != nil { + t.Fatal(err) + } + if entry["request_id"] != "http-one" || entry["preparation_request_id"] != "prepare-one" || entry["trace_id"] != reservationTraceID("reservation-one").String() { + t.Fatal("control log confused HTTP and preparation identity", entry) + } + } +} + +func TestLifecycleGateObservationPreservesCancellation(t *testing.T) { + out := captureExecutionLogs(t) + r := &runtimeLifecycle{ctx: t.Context(), gate: make(chan struct{}, 1), nodeID: "node-one"} + r.gate <- struct{}{} + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := r.lock(ctx); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if len(r.gate) != 1 || !strings.Contains(out.String(), `"status":"cancelled"`) { + t.Fatal("cancelled waiter changed gate or log status") + } +} + +type executionTraceConn struct { + writes chan []byte + closed chan struct{} + once sync.Once +} + +func (c *executionTraceConn) ReadMessage() (int, []byte, error) { <-c.closed; return 0, nil, io.EOF } +func (c *executionTraceConn) WriteMessage(_ int, data []byte) error { + c.writes <- append([]byte(nil), data...) + return nil +} +func (c *executionTraceConn) WriteControl(int, []byte, time.Time) error { return nil } +func (c *executionTraceConn) SetReadLimit(int64) {} +func (c *executionTraceConn) SetReadDeadline(time.Time) error { return nil } +func (c *executionTraceConn) SetWriteDeadline(time.Time) error { return nil } +func (c *executionTraceConn) Close() error { c.once.Do(func() { close(c.closed) }); return nil } + +func TestExecutionSendPreservesPayloadAndCarriesTrace(t *testing.T) { + for _, traced := range []bool{false, true} { + conn := &executionTraceConn{writes: make(chan []byte, 2), closed: make(chan struct{})} + peer := runtimegateway.NewSessionWithOwner(conn, "device-one", "workspace-one", "test", runtimegateway.NewRegistry(), func(string, ...any) {}, nil) + peer.Start() + ctx := t.Context() + if traced { + ctx = reservationTrace(ctx, "reservation-one") + } + payload := proto.ExecutionStartPayload{Handle: "handle-one", ExecutorID: "executor-one", RunID: "turn-one", Input: proto.TextInput("private input preserved on wire")} + if err := send(ctx, peer, proto.TypeExecutionStart, "prepare-one", payload); err != nil { + peer.Close("test done") + t.Fatal(err) + } + select { + case data := <-conn.writes: + var env proto.Envelope + if err := json.Unmarshal(data, &env); err != nil { + t.Fatal(err) + } + var got proto.ExecutionStartPayload + if err := env.DecodePayload(&got); err != nil { + t.Fatal(err) + } + expected, _ := json.Marshal(payload) + actual, _ := json.Marshal(got) + if env.ID != "prepare-one" || env.Type != proto.TypeExecutionStart || !bytes.Equal(expected, actual) { + t.Fatal("trace changed execution identity or payload") + } + if traced { + carrier, _ := obslog.TraceFromContext(ctx) + if env.Trace != carrier.String() { + t.Fatal("execution trace did not reach actual gateway wire") + } + } else if env.Trace != "" { + t.Fatal("missing trace changed into a fabricated caller trace") + } + case <-time.After(time.Second): + t.Fatal("gateway did not send") + } + peer.Close("test done") + } +} + +func TestInitialInputOriginKeepsHTTPTraceAndSessionIdentity(t *testing.T) { + out := captureExecutionLogs(t) + ctx, origin := obslog.StartBackgroundTrace(t.Context()) + recordInitialInputOrigin(ctx, "session-one") + var entry map[string]any + if err := json.Unmarshal(out.Bytes(), &entry); err != nil { + t.Fatal(err) + } + if entry["session_id"] != "session-one" || entry["trace_id"] != origin.Trace.String() { + t.Fatal("initial input origin lost its HTTP/session bridge", entry) + } + if _, present := entry["reservation_id"]; present { + t.Fatal("creation fabricated a reservation identity") + } +} diff --git a/services/core/internal/execution/executor_observation.go b/services/core/internal/execution/executor_observation.go index 9d1fd8132..8deec1fd0 100644 --- a/services/core/internal/execution/executor_observation.go +++ b/services/core/internal/execution/executor_observation.go @@ -11,20 +11,20 @@ import ( // These durations use one Core process monotonic clock. They describe control // readiness and input-to-first-text latency, not model-only or native tool time. func recordExecutorReadiness(p *preparedStart, status proto.PreparationStatusPayload) { - obslog.Bg().Info("executor ready", "request_id", p.requestID, + obslog.Ctx(p.ctx).Info("executor ready", "preparation_request_id", p.requestID, "executor_id", status.ExecutorID, "reused", status.Reused, "control_ready_ms", time.Since(p.createdAt).Milliseconds()) } // The started receipt confirms adapter ownership, not model consumption. func recordExecutorStart(p *preparedStart, turn string) { - obslog.Bg().Info("executor start acknowledged", "turn_id", turn, - "request_id", p.requestID, "executor_id", p.executorID, + obslog.Ctx(p.ctx).Info("executor start acknowledged", "turn_id", turn, + "preparation_request_id", p.requestID, "executor_id", p.executorID, "start_control_ms", time.Since(p.startSentAt).Milliseconds()) } func recordFirstText(ctx context.Context, session, turn string, submitted time.Time) { - obslog.Bg().InfoContext(ctx, "executor first text", "session_id", session, + obslog.Ctx(ctx).Info("executor first text", "session_id", session, "turn_id", turn, "input_to_first_text_ms", time.Since(submitted).Milliseconds()) } diff --git a/services/core/internal/execution/executor_preparation.go b/services/core/internal/execution/executor_preparation.go index 1e3854e32..c7476546c 100644 --- a/services/core/internal/execution/executor_preparation.go +++ b/services/core/internal/execution/executor_preparation.go @@ -14,7 +14,7 @@ import ( // Core does not cache native ownership. A replacement requires the Runtime to // confirm cleanup and recover the exact Session history before returning ready. func (d *Dispatcher) prepareTurnExecutor(ctx context.Context, peer *runtimegateway.Session, tenant, session, turn string, request proto.PromptRequestPayload, expectedStatus string) (*preparedStart, error) { - prepared, err := newPreparedStart(peer) + prepared, err := newPreparedStart(ctx, peer) if err != nil { return nil, err } diff --git a/services/core/internal/execution/preparation.go b/services/core/internal/execution/preparation.go index 089a04eff..8685e575a 100644 --- a/services/core/internal/execution/preparation.go +++ b/services/core/internal/execution/preparation.go @@ -21,6 +21,7 @@ type preparationRejection struct { func (e *preparationRejection) Error() string { return "preparation control rejected: " + e.code } type preparedStart struct { + ctx context.Context createdAt time.Time startSentAt time.Time startObserved bool @@ -32,13 +33,13 @@ type preparedStart struct { sub *runtimegateway.Subscription } -func newPreparedStart(peer *runtimegateway.Session) (*preparedStart, error) { +func newPreparedStart(ctx context.Context, peer *runtimegateway.Session) (*preparedStart, error) { id := uuid.NewString() sub, err := peer.SubscribePreparation(id) if err != nil { return nil, err } - return &preparedStart{peer: peer, requestID: id, sub: sub, createdAt: time.Now()}, nil + return &preparedStart{ctx: ctx, peer: peer, requestID: id, sub: sub, createdAt: time.Now()}, nil } func (p *preparedStart) close() { diff --git a/services/core/internal/execution/prepared_dispatch.go b/services/core/internal/execution/prepared_dispatch.go index fa2b11b0b..a3b075694 100644 --- a/services/core/internal/execution/prepared_dispatch.go +++ b/services/core/internal/execution/prepared_dispatch.go @@ -7,6 +7,8 @@ import ( "strings" "time" + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -20,6 +22,9 @@ type EnvironmentRun struct { // checks lease, the lease the Dispatcher's execution operations hold, before // any Runtime preparation. func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, lease Ownership, tenantID, sessionID, reservationID string) (run EnvironmentRun, err error) { + ctx = reservationTrace(ctx, reservationID) + selectedAt := time.Now() + obslog.Ctx(ctx).Info("environment input selected", "session_id", sessionID, "reservation_id", reservationID) if err = lease.CheckOwnership(ctx); err != nil { return run, err } @@ -29,6 +34,10 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, lease Ownership, t if err != nil || run.Reservation.State != sessions.EnvironmentInputPending { return run, err } + if !run.Reservation.CreatedAt.IsZero() { + obslog.Ctx(ctx).Info("environment input queue age", "session_id", sessionID, "reservation_id", reservationID, + "queue_age_ms", time.Since(run.Reservation.CreatedAt).Milliseconds(), "is_initial", run.Reservation.IsInitial) + } session, err := d.SessionsReader.GetSession(ctx, tenantID, sessionID) if err != nil { return run, err @@ -76,11 +85,15 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, lease Ownership, t if err := d.configurePreparedEnvironment(session, environment, bound.Device, &req); err != nil { return run, err } - prepared, err := newPreparedStart(peer) + observeExecutionStage(owner, "execution_configuration", selectedAt, nil, + "session_id", sessionID, "reservation_id", reservationID, "environment_id", environment.ID, "device_id", bound.Device.ID) + prepared, err := newPreparedStart(owner, peer) if err != nil { return run, err } defer prepared.close() + obslog.Ctx(owner).Info("execution preparation requested", "session_id", sessionID, "reservation_id", reservationID, + "environment_id", environment.ID, "device_id", bound.Device.ID, "preparation_request_id", prepared.requestID) if err = send(owner, peer, proto.TypeExecutionPrepare, prepared.requestID, proto.ExecutionPreparePayload{SessionID: sessionID, Configuration: req}); err != nil { return run, err } @@ -91,7 +104,9 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, lease Ownership, t if err := d.messageInputSupport(peer, session.Engine, snapshot, messages); err != nil { return run, err } + promoteAt := time.Now() promoted, err := d.sessionExecution.PromoteEnvironmentInput(owner, tenantID, sessionID, reservationID) + observeExecutionStage(owner, "input_promote", promoteAt, err, "session_id", sessionID, "reservation_id", reservationID) if errors.Is(err, sessions.ErrTurnConflict) { // A rejected claim leaves the reservation pending for a later attempt. return run, err @@ -107,6 +122,8 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, lease Ownership, t return run, nil } turnID := run.Reservation.Receipts[0].TurnID + obslog.Ctx(owner).Info("environment input admitted", "session_id", sessionID, "reservation_id", reservationID, + "turn_id", turnID, "preparation_request_id", prepared.requestID, "executor_id", prepared.executorID) through := run.Reservation.Receipts[len(run.Reservation.Receipts)-1].Sequence releaseDelivery, err := peer.TrackExecutionDelivery(turnID) if err != nil { diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 20d00602b..0b337eca8 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -245,7 +245,9 @@ func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.Sandbox return err } } + resumeAt := time.Now() result, err := p.Resume(ctx, sandbox.ResumeRequest{Workspace: workspace, Reference: runtimeReference(owner), OperationID: state.RestoreID, Snapshot: *state.Snapshot, Target: *state.Target, ObserveOnly: observeOnly}) + observeExecutionStage(ctx, "provider_resume", resumeAt, err, "allocation_id", owner.ID, "session_id", owner.SessionID, "environment_id", owner.EnvironmentID, "node_id", r.nodeID, "reconcile_only", observeOnly) if err != nil { return err } diff --git a/services/core/internal/execution/runtime_connections.go b/services/core/internal/execution/runtime_connections.go index 10c4bfd1a..b2f7f6d61 100644 --- a/services/core/internal/execution/runtime_connections.go +++ b/services/core/internal/execution/runtime_connections.go @@ -4,6 +4,8 @@ import ( "context" "errors" + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/google/uuid" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" @@ -62,6 +64,8 @@ func observeRuntimeConnection(ctx context.Context, operations *sessions.Executio return err } current.connected = connected + obslog.Ctx(ctx).Info("runtime connection observed", "environment_id", environment, "connected", connected, + "connection_generation", current.generation, "connection_revision", current.revision) return nil } diff --git a/services/core/internal/execution/runtime_initialization.go b/services/core/internal/execution/runtime_initialization.go index 4f94d4d52..b9a7ab23e 100644 --- a/services/core/internal/execution/runtime_initialization.go +++ b/services/core/internal/execution/runtime_initialization.go @@ -94,10 +94,12 @@ func (w *Worker) initializeEnvironment(ctx context.Context, owner sessions.Envir operation, cancel := context.WithTimeout(ctx, 30*time.Minute) defer cancel() failure := sessions.ProvisioningFailure{} + initializeAt := time.Now() err := w.prepareEnvironment(operation, owner, &failure) if err == nil { err = w.dispatcher.sessionExecution.CompleteEnvironmentInitialization(operation, owner) } + observeExecutionStage(ctx, "environment_initialize", initializeAt, err, "environment_id", owner.EnvironmentID, "session_id", owner.SessionID, "device_id", owner.DeviceID) if err != nil { log.Warn(ctx, "Environment preparation failed", "environment_id", owner.EnvironmentID, "session_id", owner.SessionID) // A later scan settles an unrecorded failure; it never retries the setup. diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 8b0eea006..f85a0bc95 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -114,7 +114,9 @@ func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway. return copied, nil } -func (r *runtimeLifecycle) lock(ctx context.Context) error { +func (r *runtimeLifecycle) lock(ctx context.Context) (err error) { + started := time.Now() + defer func() { observeExecutionStage(ctx, "runtime_gate_wait", started, err, "node_id", r.nodeID) }() select { case r.gate <- struct{}{}: if err := r.ctx.Err(); err != nil { @@ -224,10 +226,13 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if err := r.lease.CheckOwnership(ctx); err != nil { return owner, err } + createAt := time.Now() info, err := provider.Create(ctx, sandbox.Bootstrap{ Reference: runtimeReference(owner), SessionID: owner.SessionID, DeviceID: owner.DeviceID, Workspace: workspace, CoreURL: r.config.CoreURL, Credential: token, Harness: session.Engine, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, }) + observeExecutionStage(ctx, "provider_create", createAt, err, "allocation_id", owner.ID, + "session_id", owner.SessionID, "environment_id", environment, "device_id", owner.DeviceID, "node_id", r.nodeID) if info.Reference == runtimeReference(owner) && info.CreateSettled && info.State == "absent" { record, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) released, releaseErr := r.deployment.ReleaseAbsentCreation(record, owner) @@ -296,7 +301,9 @@ func (r *runtimeLifecycle) reconcile(ctx context.Context) error { for _, owner := range rows { r.cursor = owner.ID operation, stop := context.WithTimeout(ctx, 30*time.Second) + observeAt := time.Now() err := r.observe(operation, owner) + observeExecutionStage(ctx, "runtime_observe", observeAt, err, "allocation_id", owner.ID, "session_id", owner.SessionID, "node_id", r.nodeID) r.recordObservation(ctx, owner, err) stop() if err != nil { diff --git a/services/core/internal/execution/runtime_pending.go b/services/core/internal/execution/runtime_pending.go index cee0c94a3..79bcc3a81 100644 --- a/services/core/internal/execution/runtime_pending.go +++ b/services/core/internal/execution/runtime_pending.go @@ -22,7 +22,9 @@ func (r *runtimeLifecycle) provisionPending(ctx context.Context) error { r.pendingCursor = environment.ID provider := r.config.InstallationID operation, cancel := context.WithTimeout(ctx, 30*time.Second) + provisionAt := time.Now() _, err := r.provision(operation, environment.TenantID, environment.ID, provider) + observeExecutionStage(ctx, "runtime_provision", provisionAt, err, "environment_id", environment.ID, "node_id", r.nodeID) cancel() if err != nil { if ownership := r.lease.CheckOwnership(ctx); ownership != nil { diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 6d67ff3cb..000e572bf 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -138,6 +138,7 @@ func (w *Worker) CreateSession(ctx context.Context, tenant string, input session w.hintRuntimeWake(ctx, creation.Session) } if err == nil && len(input.InitialInputs) > 0 { + recordInitialInputOrigin(ctx, creation.Session.ID) w.wakeScheduler() } return creation, err diff --git a/services/core/internal/runtimegateway/capability_hint_test.go b/services/core/internal/runtimegateway/capability_hint_test.go index ba4fbd9e3..759f6a1b4 100644 --- a/services/core/internal/runtimegateway/capability_hint_test.go +++ b/services/core/internal/runtimegateway/capability_hint_test.go @@ -1,12 +1,15 @@ package runtimegateway import ( + "bytes" "context" "encoding/json" "errors" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "log/slog" + "strings" "testing" "time" ) @@ -136,3 +139,40 @@ func TestCapabilityConfirmationRetriesWithoutObservingRejectedAuthority(t *testi }) } } + +func TestCapabilityObservationsExcludeRejectedAndStalePeers(t *testing.T) { + var logs bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + reg := NewRegistry() + old := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(old) + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: true}} + publishCapabilityHeartbeat(t, old, kinds) + if !strings.Contains(logs.String(), "runtime capability snapshot observed") { + t.Fatal("valid declaration was not observed") + } + logs.Reset() + old.handleHeartbeat(proto.Envelope{Type: proto.TypeHeartbeat, Payload: json.RawMessage(`{"supported_agent_kinds":[{"kind":"fake","available":true,"capabilities":{"streaming":"invented"}}]}`)}) + if logs.Len() != 0 { + t.Fatal("invalid declaration logged as an accepted snapshot") + } + newer := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(newer) + publishCapabilityHeartbeat(t, old, kinds) + if logs.Len() != 0 { + t.Fatal("superseded connection logged as an accepted snapshot") + } + publishCapabilityHeartbeat(t, newer, kinds) + if !strings.Contains(logs.String(), "runtime capability snapshot observed") { + t.Fatal("replacement declaration was not observed") + } + logs.Reset() + reg.Deregister(newer) + kinds[0].Version = "changed" + publishCapabilityHeartbeat(t, newer, kinds) + if logs.Len() != 0 { + t.Fatal("disconnected connection logged as an accepted snapshot") + } +} diff --git a/services/core/internal/runtimegateway/handler.go b/services/core/internal/runtimegateway/handler.go index a22d6df2d..a45de0fe8 100644 --- a/services/core/internal/runtimegateway/handler.go +++ b/services/core/internal/runtimegateway/handler.go @@ -11,6 +11,7 @@ import ( "github.com/gorilla/websocket" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" ) @@ -176,6 +177,8 @@ func (h *Handler) WS(w http.ResponseWriter, r *http.Request) { prev.Close("preempted by newer connection from same device_id") } h.cfg.Log("agentdaemon gateway: device_id=%s registered in registry, starting session", auth.DeviceID) + obslog.Ctx(r.Context()).Info("runtime transport registered", "device_id", auth.DeviceID, "protocol_version", version, + "allocation_id", auth.RuntimeAllocationID, "node_id", auth.RuntimeNodeID) sess.Start() } diff --git a/services/core/internal/runtimegateway/registry.go b/services/core/internal/runtimegateway/registry.go index c30084280..99ebe9964 100644 --- a/services/core/internal/runtimegateway/registry.go +++ b/services/core/internal/runtimegateway/registry.go @@ -10,6 +10,7 @@ package runtimegateway import ( "context" "errors" + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" "sync" "time" @@ -258,6 +259,7 @@ func (r *Registry) observeCapabilitySnapshot(sess *Session, kinds []runtimedevic available++ } } + obslog.Ctx(context.Background()).Info("runtime capability snapshot observed", "device_id", sess.DeviceID, "kind_count", len(kinds), "available_kind_count", available) if available == 0 { return true } diff --git a/services/core/internal/sandbox/e2b/create_observations.go b/services/core/internal/sandbox/e2b/create_observations.go new file mode 100644 index 000000000..7efbd39db --- /dev/null +++ b/services/core/internal/sandbox/e2b/create_observations.go @@ -0,0 +1,58 @@ +package e2b + +import ( + "bytes" + "context" + "encoding/json" + "io" + + obslog "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" +) + +type createObservation struct { + Event string `json:"event"` + Stage string `json:"stage"` + DurationUS *int64 `json:"duration_us"` + Completed *bool `json:"completed"` +} + +// Helper stderr is untrusted. Only fixed stage names and bounded numeric fields +// reach the logger; arbitrary SDK diagnostics are never forwarded. +func parseCreateObservations(raw []byte) []createObservation { + var out []createObservation + seen := make(map[string]bool) + for _, line := range bytes.Split(raw, []byte{'\n'}) { + if len(line) > 256 { + continue + } + var value createObservation + decoder := json.NewDecoder(bytes.NewReader(line)) + decoder.DisallowUnknownFields() + if decoder.Decode(&value) != nil || value.Event != "e2b_create_stage" || value.DurationUS == nil || value.Completed == nil { + continue + } + var extra any + if decoder.Decode(&extra) != io.EOF || *value.DurationUS < 0 || *value.DurationUS > 1_800_000_000 || seen[value.Stage] { + continue + } + switch value.Stage { + case "sandbox_create", "ownership_check", "template_check", "bootstrap_write", "bootstrap_run", "ready_inspect", + "bootstrap_stream_open", "bootstrap_stream_completion", + "bootstrap_claim", "bootstrap_protection", "bootstrap_layout", "bootstrap_credentials", "bootstrap_spawn", "bootstrap_receipt": + seen[value.Stage] = true + out = append(out, value) + } + } + return out +} + +func observeHelperCreate(ctx context.Context, q Request, raw []byte) { + if q.Operation != "create" { + return + } + for _, value := range parseCreateObservations(raw) { + obslog.Ctx(ctx).Info("e2b create stage", "stage", value.Stage, + "duration_ms", float64(*value.DurationUS)/1000, "completed", *value.Completed, + "environment_id", q.Reference.EnvironmentID, "allocation_id", q.Reference.AllocationID) + } +} diff --git a/services/core/internal/sandbox/e2b/create_observations_test.go b/services/core/internal/sandbox/e2b/create_observations_test.go new file mode 100644 index 000000000..36d0baec5 --- /dev/null +++ b/services/core/internal/sandbox/e2b/create_observations_test.go @@ -0,0 +1,104 @@ +package e2b + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" +) + +func TestCreateObservationsRejectUnsafeDiagnostics(t *testing.T) { + valid := `{"event":"e2b_create_stage","stage":"sandbox_create","duration_us":1250,"completed":true}` + for _, bad := range []string{ + `private SDK secret`, + strings.Replace(valid, `"sandbox_create"`, `"private SDK secret"`, 1), + strings.Replace(valid, `1250`, `-1`, 1), + strings.Replace(valid, `1250`, `1800000001`, 1), + strings.Replace(valid, `1250`, `1.25`, 1), + strings.Replace(valid, `1250`, `null`, 1), + strings.Replace(valid, `true`, `null`, 1), + strings.Replace(valid, `true`, `"private SDK secret"`, 1), + strings.Replace(valid, `"completed":true`, `"credential":"private"`, 1), + strings.TrimSuffix(valid, "}") + `,"credential":"private"}`, + valid + `{}`, + strings.Repeat(" ", 256) + valid, + } { + if got := parseCreateObservations([]byte(bad)); len(got) != 0 { + t.Fatalf("unsafe diagnostic accepted: %q", bad) + } + } + got := parseCreateObservations([]byte("private SDK diagnostic\n" + valid + "\n" + valid)) + if len(got) != 1 || *got[0].DurationUS != 1250 || !*got[0].Completed { + t.Fatalf("valid timing lost or duplicated: %+v", got) + } +} + +func TestCreateObservationsRetainFailedStage(t *testing.T) { + got := parseCreateObservations([]byte(`{"event":"e2b_create_stage","stage":"bootstrap_run","duration_us":0,"completed":false}`)) + if len(got) != 1 || *got[0].Completed { + t.Fatal("failed stage discarded") + } +} + +func TestCreateObservationsAcceptBoundedGuestStages(t *testing.T) { + for _, stage := range []string{"bootstrap_claim", "bootstrap_protection", "bootstrap_layout", "bootstrap_credentials", "bootstrap_spawn", "bootstrap_receipt"} { + raw := `{"event":"e2b_create_stage","stage":"` + stage + `","duration_us":42,"completed":false}` + got := parseCreateObservations([]byte(raw + "\n" + raw)) + if len(got) != 1 || got[0].Stage != stage || *got[0].Completed || *got[0].DurationUS != 42 { + t.Fatal(got) + } + } +} + +func TestProcessCreateObservationsReachLogger(t *testing.T) { + for _, overflow := range []bool{false, true} { + t.Run(strconv.FormatBool(overflow), func(t *testing.T) { + p, caller, ref := fixture(t) + _, _ = p.Create(bounded(t), sandbox.Bootstrap{Reference: ref, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private-bootstrap", Harness: "codex", NetworkAccess: "enabled"}) + q := caller.requests[0] + q.Deadline = time.Now().Add(time.Second) + raw := `{"event":"e2b_create_stage","stage":"sandbox_create","duration_us":1250,"completed":true}` + "\nprivate SDK secret\n" + if overflow { + raw += strings.Repeat("x", MaxOutputBytes) + } + // Keep arbitrary diagnostics in a fixture file, never shell source. + diagnostics := q.Config.StateDir + "/diagnostics" + if err := os.WriteFile(diagnostics, []byte(raw), 0600); err != nil { + t.Fatal(err) + } + script := "#!/bin/sh\ncat '" + diagnostics + "' >&2\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" + if err := os.WriteFile(q.Config.Binary, []byte(script), 0700); err != nil { + t.Fatal(err) + } + var logs bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&logs, nil))) + defer slog.SetDefault(previous) + _, err := (&ProcessCaller{}).Call(context.Background(), q) + if overflow { + if err == nil || logs.Len() != 0 { + t.Fatal("stderr overflow changed failure semantics or exported truncated diagnostics") + } + return + } + if err != nil { + t.Fatal(err) + } + var record map[string]any + if json.Unmarshal(logs.Bytes(), &record) != nil || record["allocation_id"] != ref.AllocationID || record["environment_id"] != ref.EnvironmentID || record["duration_ms"] != 1.25 || record["msg"] != "e2b create stage" { + t.Fatalf("stage did not reach correlated logger: %s", logs.Bytes()) + } + if strings.Contains(logs.String(), "private") { + t.Fatal("SDK diagnostic leaked") + } + }) + } +} diff --git a/services/core/internal/sandbox/e2b/process.go b/services/core/internal/sandbox/e2b/process.go index 1752c71ba..3e67cbded 100644 --- a/services/core/internal/sandbox/e2b/process.go +++ b/services/core/internal/sandbox/e2b/process.go @@ -54,6 +54,9 @@ func (p *ProcessCaller) Call(ctx context.Context, q Request) (Response, error) { case <-ctx.Done(): return Response{}, ctx.Err() case err = <-done: + if !stderr.exceeded { + observeHelperCreate(ctx, q, stderr.Bytes()) + } if err != nil || stdout.exceeded || stderr.exceeded { return Response{}, errors.New("helper result unconfirmed") } @@ -77,6 +80,12 @@ type limitBuffer struct { exceeded bool } +// Do not let the embedded bytes.Buffer's ReaderFrom bypass Write's limit when +// os/exec copies a child's output with io.Copy. +func (b *limitBuffer) ReadFrom(r io.Reader) (int64, error) { + return io.Copy(struct{ io.Writer }{b}, r) +} + func (b *limitBuffer) Write(data []byte) (int, error) { n := len(data) left := b.limit - b.Len() diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index 46f1e9f15..7de77f83f 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -56,6 +56,10 @@ Inspection uses SDK metadata and ID reads only. SDK `connect` is never used beca ## Observations +Create emits six sequential timing records on helper stderr: `sandbox_create`, `ownership_check`, `template_check`, `bootstrap_write`, `bootstrap_run` and `ready_inspect`. Each record contains only `event=e2b_create_stage`, `stage`, integer `duration_us` and boolean `completed`. The Core adapter accepts only the stages listed here, durations between zero and 1800 seconds, and one record per stage, then logs `e2b create stage` with the allocation and Environment IDs from its validated request. Unknown fields, oversized records and arbitrary SDK diagnostics are discarded. Timing uses a monotonic clock; `completed` describes the local step, not Runtime readiness. Records are exported when the helper exits before caller cancellation, including failed operations; the Core log timestamp is export time, not the time the step ran. Missing records are unobserved, not zero. Helper startup, lock waits and receipt work outside these steps remain part of the outer Provider duration. No request payload, credential, SDK error text or command output is logged. + +`bootstrap_run` also contains helper-side `bootstrap_stream_open` (the SDK background start call) and `bootstrap_stream_completion` (the existing handle wait, including command output draining). Guest timings on the same command stderr are filtered to `bootstrap_claim`, `bootstrap_protection`, `bootstrap_layout`, `bootstrap_credentials`, `bootstrap_spawn` and `bootstrap_receipt`, then exported through the same bounded Core log path. These cover the one-shot claim, image protection/account locking, workspace/environment layout, private credential persistence, daemon handoff and durable ready receipt. The helper accepts at most 8192 characters of guest diagnostics and one fixed four-field record per stage, at most 256 characters each; arbitrary stderr is discarded. Receipt JSON and command stdout are unchanged. Guest timings require a template containing the instrumented initializer; they remain unobserved on other templates. A stream failure can prevent guest timings from returning, even when initialization ran. Guest and SDK intervals overlap within `bootstrap_run`; do not sum them, or add daemon startup to provider completion. `bootstrap_spawn` confirms handoff only, and the daemon can start before the provider returns. All existing filesystem protection, fsync, no-replay and recovery behavior remains in force. + `observe` reads the allocation's sandbox ID from its receipt without taking the allocation lock. It then runs one `GET /sandboxes/metrics` request for that sandbox and one listing of running sandboxes with the allocation's labels concurrently, within the caller's deadline; the listing stops once the sandbox has appeared. Only a sandbox that the listing confirms for exactly that allocation is reported, with the listing's start time. A malformed metrics point is unavailable. Observation never connects to, renews or changes a sandbox and writes no receipt. [Runtime observability](../../../../contracts/agents-api/runtime-observability.md) owns the field mapping. ## Credential verification diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index 264b39489..1cfc5d3a1 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -1,8 +1,10 @@ """Five bounded SDK operations for an already authorized Core allocation, plus read-only deployment validation and observation.""" from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager import json import math +import sys import time from datetime import datetime, timezone from uuid import UUID @@ -34,6 +36,50 @@ """ +@contextmanager +def create_stage(stage): + """Emit bounded numeric diagnostics, never SDK output or request contents.""" + started = time.monotonic_ns() + completed = False + try: + yield + completed = True + finally: + try: + sys.stderr.write(json.dumps({'event': 'e2b_create_stage', 'stage': stage, + 'duration_us': (time.monotonic_ns() - started) // 1000, + 'completed': completed}) + '\n') + except Exception: + # Observation must not change allocation or bootstrap outcomes. + pass + + +def observe_bootstrap(raw): + """Forward only bounded guest timings; never expose guest stderr itself.""" + if not isinstance(raw, str) or len(raw) > 8192: + return + seen = set() + for line in raw.splitlines(): + if len(line) > 256: + continue + try: + value = json.loads(line) + if (not isinstance(value, dict) or + set(value) != {'event', 'stage', 'duration_us', 'completed'} or + value['event'] != 'e2b_bootstrap_stage' or + value['stage'] not in ('bootstrap_claim', 'bootstrap_protection', 'bootstrap_layout', + 'bootstrap_credentials', 'bootstrap_spawn', 'bootstrap_receipt') or + value['stage'] in seen or type(value['duration_us']) is not int or + not 0 <= value['duration_us'] <= 1_800_000_000 or type(value['completed']) is not bool): + continue + seen.add(value['stage']) + value['event'] = 'e2b_create_stage' + sys.stderr.write(json.dumps(value) + '\n') + except Exception: + # A malformed record or unavailable sink never changes startup. + pass + + def valid_id(value): try: return isinstance(value, str) and str(UUID(value)) == value and UUID(value).int != 0 @@ -230,62 +276,69 @@ def create(self): identity = dict(self.reference, InstallationID=self.config['InstallationID'], SessionID=bootstrap['SessionID'], DeviceID=bootstrap['DeviceID']) self.receipt.save(status='create_pending', bootstrap_identity=identity) - try: - cloud = Sandbox.create(template=self.config['Template'], timeout=self.config['TimeoutSeconds'], - metadata=self.metadata, lifecycle={'on_timeout': 'kill', 'auto_resume': False}, - **self.options()) - except Exception as error: - if definitely_rejected(error): - self.receipt.save(status='rejected', settled=True) - raise Failure('unconfirmed') from None + with create_stage('sandbox_create'): + try: + cloud = Sandbox.create(template=self.config['Template'], timeout=self.config['TimeoutSeconds'], + metadata=self.metadata, lifecycle={'on_timeout': 'kill', 'auto_resume': False}, + **self.options()) + except Exception as error: + if definitely_rejected(error): + self.receipt.save(status='rejected', settled=True) + raise Failure('unconfirmed') from None self.receipt.save(status='created', ids=[cloud.sandbox_id], connection=connection_material(cloud)) # A create response must not steer envd traffic to an unrelated host. self.check_domain(cloud) # SDK Create returns connection material, but no metadata or resources. # Read its exact ID before writing credentials, even when Core adopts the # template's resources and does not supply explicit limits. - try: - detail = self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options())) - self.qualified(detail) - except Failure: - self.receipt.save(status='configuration_rejected', settled=True) - raise + with create_stage('ownership_check'): + try: + detail = self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options())) + self.qualified(detail) + except Failure: + self.receipt.save(status='configuration_rejected', settled=True) + raise # Validate the current template entry point before writing any credential. - try: - entry = cloud.files.get_info('/opt/oac-e2b/managed_init.py', - user='root', request_timeout=self.remaining()) - if entry.type != FileType.FILE: + with create_stage('template_check'): + try: + entry = cloud.files.get_info('/opt/oac-e2b/managed_init.py', + user='root', request_timeout=self.remaining()) + if entry.type != FileType.FILE: + self.receipt.save(status='bootstrap_failed', settled=True) + raise Failure('template_invalid') + # A successful download can also refer to a special file. Check + # the type first, then verify readability without a probe process. + with cloud.files.read('/opt/oac-e2b/managed_init.py', format='stream', + user='root', request_timeout=self.remaining()): + pass + except FileNotFoundException: self.receipt.save(status='bootstrap_failed', settled=True) - raise Failure('template_invalid') - # A successful download can also refer to a special file. Check - # the type first, then verify readability without a probe process. - with cloud.files.read('/opt/oac-e2b/managed_init.py', format='stream', - user='root', request_timeout=self.remaining()): - pass - except FileNotFoundException: - self.receipt.save(status='bootstrap_failed', settled=True) - raise Failure('template_invalid') from None + raise Failure('template_invalid') from None payload = dict(bootstrap, InstallationID=self.config['InstallationID'], RuntimeBootstrap=self.q['RuntimeBootstrap']) del payload['CoreURL'], payload['Credential'], payload['Harness'] if (set(payload) - set(MANAGED_BOOTSTRAP_FIELDS) or not set(MANAGED_BOOTSTRAP_REQUIRED_FIELDS) <= set(payload)): raise Failure('invalid') - cloud.files.write('/root/.oac/e2b/managed-bootstrap.json', json.dumps(payload), - user='root', request_timeout=self.remaining()) + with create_stage('bootstrap_write'): + cloud.files.write('/root/.oac/e2b/managed-bootstrap.json', json.dumps(payload), + user='root', request_timeout=self.remaining()) self.receipt.save(status='bootstrap_pending') - result = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', BOOTSTRAP_SCRIPT]}, - self.remaining, user='root') - if result['ExitCode'] != 0: - self.receipt.save(status='bootstrap_failed', settled=True) - raise Failure('unconfirmed') + with create_stage('bootstrap_run'): + result = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', BOOTSTRAP_SCRIPT]}, + self.remaining, user='root', observe_stage=create_stage) + observe_bootstrap(result.get('Stderr')) + if result['ExitCode'] != 0: + self.receipt.save(status='bootstrap_failed', settled=True) + raise Failure('unconfirmed') self.receipt.save(status='bootstrap_exited', settled=True) - try: - receipt = json.loads(result['Stdout']) - except (ValueError, KeyError): - raise Failure('unconfirmed') from None - self.accept_bootstrap_receipt(receipt) - return self.inspect() + with create_stage('ready_inspect'): + try: + receipt = json.loads(result['Stdout']) + except (ValueError, KeyError): + raise Failure('unconfirmed') from None + self.accept_bootstrap_receipt(receipt) + return self.inspect() def renew(self): cloud = self.inspect() diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 240e05eab..07ff34387 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -19,13 +19,55 @@ from packaging.version import Version from e2b.exceptions import AuthenticationException, FileNotFoundException, SandboxNotFoundException -from provider import BOOTSTRAP_SCRIPT, Provider +from provider import BOOTSTRAP_SCRIPT, Provider, create_stage, observe_bootstrap from helper_contract_generated import PROTOCOL_VERSION from sdk import restore, run from state import Failure, Receipt class BootstrapScriptTest(unittest.TestCase): + def test_sdk_timings_preserve_single_start_and_wait(self): + for failure in (None, 'start', 'wait'): + with self.subTest(failure=failure): + sandbox = Mock() + process = sandbox.commands.run.return_value + process.wait.return_value = SimpleNamespace(stdout='receipt', stderr='', exit_code=0) + if failure == 'start': + sandbox.commands.run.side_effect = TimeoutError('private error') + if failure == 'wait': + process.wait.side_effect = TimeoutError('private error') + stream = io.StringIO() + with patch('provider.sys.stderr', stream): + if failure: + with self.assertRaises(Failure): + run(sandbox, {'Args': ['python3']}, lambda: 30, observe_stage=create_stage) + else: + self.assertEqual(run(sandbox, {'Args': ['python3']}, lambda: 30, observe_stage=create_stage)['Stdout'], 'receipt') + rows = [json.loads(line) for line in stream.getvalue().splitlines()] + self.assertEqual([row['stage'] for row in rows], ['bootstrap_stream_open'] + ([] if failure == 'start' else ['bootstrap_stream_completion'])) + self.assertEqual(rows[-1]['completed'], failure is None) + sandbox.commands.run.assert_called_once() + self.assertEqual(process.wait.call_count, 0 if failure == 'start' else 1) + self.assertNotIn('private', stream.getvalue()) + + def test_guest_observations_are_bounded_and_secret_safe(self): + valid = dict(event='e2b_bootstrap_stage', stage='bootstrap_claim', duration_us=42, completed=True) + bad = [dict(valid, stage='private-secret'), dict(valid, credential='private-secret'), + dict(valid, duration_us=True), dict(valid, duration_us=-1), dict(valid, duration_us=1.5), + dict(valid, duration_us=1_800_000_001), dict(valid, completed='true'), + dict(valid, stage=[]), None, []] + raw = '\n'.join(['private stderr'] + [json.dumps(row) for row in bad] + [json.dumps(valid)] * 2) + stream = io.StringIO() + with patch('provider.sys.stderr', stream): + observe_bootstrap(raw) + observe_bootstrap('x' * 8193) + observe_bootstrap(None) + self.assertEqual([json.loads(line) for line in stream.getvalue().splitlines()], + [dict(valid, event='e2b_create_stage')]) + with patch('provider.sys.stderr') as stream: + stream.write.side_effect = RuntimeError('sink failure') + observe_bootstrap(json.dumps(valid)) + def execute(self, startup, receipt=None): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -108,6 +150,58 @@ def call(self, operation): def record(self): return json.loads(next(Path(self.temporary.name).glob('*.json')).read_text()) + def test_create_stages_preserve_order_and_exclude_secrets(self): + stream = io.StringIO() + with patch('provider.sys.stderr', stream): + result = self.call('create') + self.assertEqual(result['ErrorCode'], '') + rows = [json.loads(line) for line in stream.getvalue().splitlines()] + self.assertEqual([row['stage'] for row in rows], [ + 'sandbox_create', 'ownership_check', 'template_check', + 'bootstrap_write', 'bootstrap_run', 'ready_inspect']) + for row in rows: + self.assertEqual(set(row), {'event', 'stage', 'duration_us', 'completed'}) + self.assertTrue(row['completed']) + self.assertGreaterEqual(row['duration_us'], 0) + self.assertNotIn('private', stream.getvalue()) + + def test_failed_create_stage_does_not_expose_exception_or_replay(self): + stream = io.StringIO() + self.api.create.side_effect = TimeoutError('private SDK secret') + with patch('provider.sys.stderr', stream): + self.assertEqual(self.call('create')['ErrorCode'], 'unconfirmed') + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + rows = [json.loads(line) for line in stream.getvalue().splitlines()] + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]['stage'], 'sandbox_create') + self.assertFalse(rows[0]['completed']) + self.assertNotIn('private', stream.getvalue()) + self.api.create.assert_called_once() + + def test_failed_guest_stage_is_forwarded_without_receipt_replay(self): + stream = io.StringIO() + stage = dict(event='e2b_bootstrap_stage', stage='bootstrap_spawn', duration_us=123, completed=False) + with patch('provider.run', return_value={'Stdout': '', 'Stderr': json.dumps(stage) + '\nprivate error', 'ExitCode': 1}) as command, \ + patch('provider.sys.stderr', stream): + self.assertEqual(self.call('create')['ErrorCode'], 'unconfirmed') + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + rows = [json.loads(line) for line in stream.getvalue().splitlines()] + self.assertIn(dict(stage, event='e2b_create_stage'), rows) + self.assertNotIn('private', stream.getvalue()) + command.assert_called_once() + + def test_broken_diagnostic_sink_does_not_change_create(self): + with patch('provider.sys.stderr') as stream: + stream.write.side_effect = OSError('sink closed') + self.assertEqual(self.call('create')['ErrorCode'], '') + + def test_stage_uses_monotonic_duration(self): + stream = io.StringIO() + with patch('provider.sys.stderr', stream), patch('provider.time.monotonic_ns', side_effect=[1000, 2501000]): + with create_stage('sandbox_create'): + pass + self.assertEqual(json.loads(stream.getvalue())['duration_us'], 2500) + def test_create_recover_and_never_replay(self): result = self.call('create') self.assertEqual(result['ErrorCode'], '') diff --git a/services/core/tools/e2b-provider/sdk.py b/services/core/tools/e2b-provider/sdk.py index 4b2d3afb7..002fbaf12 100644 --- a/services/core/tools/e2b-provider/sdk.py +++ b/services/core/tools/e2b-provider/sdk.py @@ -1,5 +1,6 @@ """The version-pinned SDK boundary. No handwritten provider HTTP or envd RPC.""" import base64 +from contextlib import nullcontext import shlex from e2b import Sandbox @@ -155,7 +156,7 @@ def definitely_rejected(error): isinstance(error, SandboxException) and error.status_code in (400, 401, 403, 404, 422, 429)) -def run(sandbox, command, remaining, user='runtime'): +def run(sandbox, command, remaining, user='runtime', observe_stage=None): raw = command.get('Stdin') data = base64.b64decode(raw, validate=True) if raw is not None else None if data is not None and len(data) > MAX_COMMAND_INPUT: @@ -174,9 +175,10 @@ def bounded(index, text): raise Failure('command_unconfirmed') try: - process = sandbox.commands.run(shlex.join(args), user=user, cwd=directory, - background=True, stdin=data is not None, - timeout=remaining(), request_timeout=remaining()) + with observe_stage('bootstrap_stream_open') if observe_stage else nullcontext(): + process = sandbox.commands.run(shlex.join(args), user=user, cwd=directory, + background=True, stdin=data is not None, + timeout=remaining(), request_timeout=remaining()) if data is not None: # Avoid an oversized unary SDK message; every chunk is submitted once. for offset in range(0, len(data), 64 * 1024): @@ -184,8 +186,9 @@ def bounded(index, text): request_timeout=remaining()) sandbox.commands.close_stdin(process.pid, request_timeout=remaining()) try: - result = process.wait(on_stdout=lambda text: bounded(0, text), - on_stderr=lambda text: bounded(1, text)) + with observe_stage('bootstrap_stream_completion') if observe_stage else nullcontext(): + result = process.wait(on_stdout=lambda text: bounded(0, text), + on_stderr=lambda text: bounded(1, text)) except CommandExitException as error: result = error return {'Stdout': result.stdout, 'Stderr': result.stderr, 'ExitCode': result.exit_code} From 25c503eae0e72b82d0646f0073152d499fc389eb Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sat, 10 Oct 2026 23:36:27 +0800 Subject: [PATCH 07/22] Preserve native sessions across compute replacement (#641) * Preserve native sessions across compute replacement * Use allocation storage contract during checkpoint restore * Normalize cancellation while waiting for compute * Attach administrator audit source to deployment updates --- .../internal/agent/claudesdk/declaration.go | 22 +- .../agent/claudesdk/declaration_test.go | 10 +- apps/daemon/internal/agent/claudesdk/local.go | 7 +- .../internal/agent/claudesdk/local_test.go | 23 + .../internal/agent/claudesdk/options.go | 10 +- .../internal/agent/claudesdk/session_test.go | 2 +- .../internal/agent/codex/declaration.go | 4 +- .../agent/codex/error_classification_test.go | 2 +- .../agent/codex/execution_controls_test.go | 4 +- apps/daemon/internal/agent/codex/executor.go | 10 +- .../agent/codex/executor_native_test.go | 2 +- .../agent/codex/harness_config_test.go | 4 +- .../agent/codex/mcp_http_bearer_test.go | 6 +- .../agent/codex/mcp_http_preflight_test.go | 2 +- .../internal/agent/codex/mcp_http_test.go | 10 +- .../internal/agent/codex/model_route_test.go | 4 +- .../agent/codex/model_verbosity_test.go | 4 +- apps/daemon/internal/agent/codex/options.go | 19 +- .../internal/agent/codex/options_test.go | 8 +- .../agent/codex/permission_profile_test.go | 6 +- .../agent/codex/preparation_helpers_test.go | 2 +- .../internal/agent/codex/provider_config.go | 10 +- .../agent/codex/provider_config_test.go | 71 ++- .../internal/agent/codex/recovery_test.go | 2 +- .../internal/agent/codex/rpc_close_test.go | 2 +- apps/daemon/internal/agent/codex/session.go | 4 +- .../agent/codex/session_cancel_test.go | 2 +- .../agent/codex/session_cancel_write_test.go | 2 +- .../codex/session_command_output_test.go | 2 +- .../agent/codex/session_messages_test.go | 2 +- .../codex/session_notifications_rpc_test.go | 4 +- .../agent/codex/session_notifications_test.go | 6 +- .../internal/agent/codex/session_plan.go | 2 +- .../agent/codex/session_policy_test.go | 2 +- .../agent/codex/session_tools_test.go | 2 +- .../agent/codex/session_usage_live_test.go | 6 +- .../agent/codex/session_usage_test.go | 8 +- .../internal/agent/codex/state_root_test.go | 16 + .../agent/codex/subagent_observations_test.go | 2 +- apps/daemon/internal/agent/harness.go | 6 +- .../internal/agent/mcode/declaration.go | 3 +- .../agent/mcode/discovery_workspace.go | 8 +- .../agent/mcode/environment_mcp_test.go | 14 +- .../internal/agent/mcode/execution_test.go | 4 +- apps/daemon/internal/agent/mcode/executor.go | 6 +- .../agent/mcode/executor_native_test.go | 4 +- .../internal/agent/mcode/executor_test.go | 6 +- .../agent/mcode/harness_config_test.go | 4 +- .../agent/mcode/model_provider_test.go | 8 +- apps/daemon/internal/agent/mcode/options.go | 4 +- .../internal/agent/mcode/options_test.go | 6 +- .../internal/agent/mcode/session_test.go | 2 +- .../internal/agent/mcode/state_root_test.go | 16 + .../agent/mcode/tool_environment_test.go | 4 +- apps/daemon/internal/agent/mcode/workspace.go | 4 +- .../agent/mcode/workspace_network_test.go | 2 +- .../agent/mcode/workspace_skills_test.go | 2 +- apps/daemon/internal/agent/runtime_paths.go | 9 +- .../internal/agent/runtime_paths_test.go | 6 +- apps/daemon/internal/cli/agent_discovery.go | 11 +- apps/daemon/internal/cli/connect_suspend.go | 2 +- .../internal/cli/connect_suspend_test.go | 132 ++++++ .../internal/cli/native_discovery_test.go | 8 +- .../dispatch/functions_native_test.go | 2 +- apps/daemon/internal/dispatch/suspend.go | 28 +- apps/daemon/internal/dispatch/suspend_test.go | 164 ++++++- .../daemon/internal/localworkspace/binding.go | 13 +- .../localworkspace/snapshot_marker.go | 8 +- .../localworkspace/snapshot_marker_test.go | 33 +- apps/daemon/internal/paths/paths.go | 12 + apps/daemon/internal/paths/paths_test.go | 22 + contracts/agents-api/environments.md | 4 +- contracts/agents-api/harness-capabilities.md | 2 + contracts/agents-api/harness-onboarding.md | 4 +- contracts/agents-api/index.md | 1 + contracts/agents-api/zh/environments.md | 6 +- .../agents-api/zh/harness-capabilities.md | 4 +- contracts/agents-api/zh/harness-onboarding.md | 6 +- contracts/agents-api/zh/index.md | 3 +- docs/architecture.md | 4 +- docs/configuration.md | 14 + docs/runtime-protocol.md | 5 + docs/sandbox-provider.md | 22 +- docs/workspace-provider.md | 8 +- docs/zh/architecture.md | 6 +- docs/zh/configuration.md | 16 +- docs/zh/runtime-protocol.md | 8 +- docs/zh/sandbox-provider.md | 24 +- docs/zh/workspace-provider.md | 10 +- internal/agentdaemon/proto/inbound.go | 4 + .../proto/prototest/capabilities.go | 1 + internal/agentdaemon/proto/suspend.go | 3 + internal/agentdaemon/proto/version.go | 2 +- scripts/name-allowlist.json | 5 + services/core/deploy/claude/Dockerfile | 3 +- services/core/deploy/claude/README.md | 4 +- services/core/deploy/codex/Dockerfile | 3 +- services/core/deploy/codex/README.md | 4 +- services/core/deploy/mcode/Dockerfile | 3 +- services/core/deploy/mcode/README.md | 4 +- .../internal/api/sandbox_deployment_setup.go | 1 + .../db/queries/admin_session_archive.sql | 2 +- .../core/internal/db/queries/environments.sql | 6 +- .../db/queries/local_environment_devices.sql | 24 +- .../db/queries/runtime_allocations.sql | 30 +- .../db/queries/runtime_deployment.sql | 9 +- .../db/queries/runtime_enrollment.sql | 4 +- .../db/queries/runtime_lifecycle_nodes.sql | 23 +- .../internal/db/queries/runtime_nodes.sql | 14 +- .../db/queries/runtime_suspension.sql | 9 +- .../internal/db/queries/sandbox_reset.sql | 12 +- .../db/sqlc/admin_session_archive.sql.go | 2 +- .../core/internal/db/sqlc/environments.sql.go | 22 +- .../db/sqlc/local_environment_devices.sql.go | 38 +- services/core/internal/db/sqlc/models.go | 8 + .../db/sqlc/runtime_allocations.sql.go | 54 ++- .../db/sqlc/runtime_deployment.sql.go | 9 +- .../db/sqlc/runtime_enrollment.sql.go | 15 +- .../db/sqlc/runtime_lifecycle_nodes.sql.go | 51 ++- .../internal/db/sqlc/runtime_nodes.sql.go | 29 +- .../db/sqlc/runtime_suspension.sql.go | 11 +- .../internal/db/sqlc/sandbox_reset.sql.go | 12 +- .../core/internal/deployment/activity_test.go | 27 +- .../core/internal/deployment/allocation.go | 8 +- .../core/internal/deployment/allocations.go | 25 +- .../internal/deployment/allocations_test.go | 61 +++ .../core/internal/deployment/fakes_test.go | 7 + .../core/internal/deployment/replacement.go | 112 +++++ .../internal/deployment/session_archive.go | 2 +- .../deployment/session_archive_test.go | 2 +- services/core/internal/deployment/storage.go | 17 +- services/core/internal/engine/claude.go | 1 + services/core/internal/engine/codex.go | 1 + .../internal/engine/enginetest/profile.go | 1 + services/core/internal/engine/mcode.go | 1 + services/core/internal/engine/profile.go | 1 + .../execution/deployment_fixture_test.go | 15 + .../internal/execution/engine_profile_test.go | 28 ++ .../internal/execution/prepared_dispatch.go | 2 +- .../internal/execution/runtime_compute.go | 12 +- .../execution/runtime_compute_wake.go | 97 +++- .../execution/runtime_compute_wake_test.go | 56 +++ .../execution/runtime_initialization.go | 11 + .../internal/execution/runtime_lifecycle.go | 68 ++- .../internal/execution/runtime_manager.go | 4 + .../internal/execution/runtime_pending.go | 35 ++ .../execution/runtime_replacement_test.go | 428 ++++++++++++++++++ .../runtime_workspace_settlement_test.go | 23 +- .../execution/runtime_workspace_test.go | 8 +- services/core/internal/execution/support.go | 30 ++ .../internal/execution/tool_search_test.go | 1 + services/core/internal/execution/worker.go | 5 + .../core/internal/execution/worker_device.go | 6 +- .../postgres/deploymentpg/allocations.go | 84 +++- .../postgres/deploymentpg/replacement_test.go | 411 +++++++++++++++++ .../postgres/deploymentpg/store.go | 2 +- .../postgres/sessionpg/artifacts.go | 2 +- .../persistence/postgres/sessionpg/binding.go | 7 +- .../postgres/sessionpg/creation.go | 5 + .../postgres/sessionpg/creation_test.go | 62 +++ .../sessionpg/device_capabilities_test.go | 46 ++ .../persistence/postgres/sessionpg/devices.go | 12 +- .../postgres/sessionpg/environment.go | 4 +- .../persistence/postgres/sessionpg/rows.go | 4 +- .../postgres/sessionpg/session_reads.go | 2 +- services/core/internal/runtimedevice/state.go | 1 + .../core/internal/runtimegateway/session.go | 1 + .../core/internal/sessions/artifacts_test.go | 3 +- services/core/internal/sessions/creation.go | 17 + .../core/internal/sessions/creation_test.go | 66 ++- services/core/internal/sessions/devices.go | 7 +- .../core/internal/sessions/devices_test.go | 21 +- .../core/internal/sessions/environment.go | 24 +- services/core/internal/sessions/session.go | 3 + .../000098_compute_incarnations.sql | 40 ++ .../integration/runtime_allocations_test.go | 10 +- .../runtime_compute_lifecycle_test.go | 8 +- .../integration/runtime_idle_clock_test.go | 4 +- .../runtime_lifecycle_nodes_test.go | 2 +- .../runtime_node_lifecycle_test.go | 2 +- .../integration/runtime_suspension_test.go | 45 +- .../sandbox_deployment_audit_http_test.go | 83 ++++ .../session_execution_configuration_test.go | 2 +- 183 files changed, 3063 insertions(+), 439 deletions(-) create mode 100644 apps/daemon/internal/agent/codex/state_root_test.go create mode 100644 apps/daemon/internal/agent/mcode/state_root_test.go create mode 100644 services/core/internal/deployment/replacement.go create mode 100644 services/core/internal/execution/runtime_compute_wake_test.go create mode 100644 services/core/internal/execution/runtime_replacement_test.go create mode 100644 services/core/internal/persistence/postgres/deploymentpg/replacement_test.go create mode 100644 services/core/internal/persistence/postgres/sessionpg/device_capabilities_test.go create mode 100644 services/core/migrations/000098_compute_incarnations.sql create mode 100644 services/core/tests/integration/sandbox_deployment_audit_http_test.go diff --git a/apps/daemon/internal/agent/claudesdk/declaration.go b/apps/daemon/internal/agent/claudesdk/declaration.go index d422248fa..c1d903d98 100644 --- a/apps/daemon/internal/agent/claudesdk/declaration.go +++ b/apps/daemon/internal/agent/claudesdk/declaration.go @@ -24,6 +24,7 @@ var Declaration = agent.Declaration{Info: proto.SupportedAgentKind{Kind: "claude Streaming: proto.CapabilitySupported, Usage: proto.CapabilitySupported, Resume: proto.CapabilitySupported, + RetainedNativeHistory: proto.CapabilityUnsupported, NativeSessionRecovery: proto.CapabilityUnsupported, Steering: proto.CapabilitySupported, MessageItems: proto.CapabilitySupported, @@ -68,18 +69,17 @@ func discoverWithCheck(parent context.Context, options agent.DiscoveryOptions, d if !filepath.IsAbs(entrypoint) { return fail(fmt.Errorf("%s must be absolute", claudeSDKEntrypointEnv)) } - profileDir, err := paths.ProfileDir(options.Profile) - if err != nil { + if err := paths.ValidateProfile(options.Profile); err != nil { return fail(err) } - if !filepath.IsAbs(profileDir) { - return fail(fmt.Errorf("Claude SDK state requires an absolute OAC_RUNTIME_HOME")) + if !filepath.IsAbs(options.RuntimeRoot) || filepath.Clean(options.RuntimeRoot) != options.RuntimeRoot { + return fail(fmt.Errorf("Claude SDK requires an explicit Runtime root")) } node := os.Getenv(claudeSDKNodeEnv) if node == "" { node = "node" } - node, err = exec.LookPath(node) + node, err := exec.LookPath(node) if err != nil { return fail(fmt.Errorf("Claude SDK Node executable is unavailable")) } @@ -87,21 +87,20 @@ func discoverWithCheck(parent context.Context, options agent.DiscoveryOptions, d if err != nil { return fail(err) } - config = Config{Node: node, Entrypoint: entrypoint, StateDir: filepath.Join(profileDir, "runtime", "claude-sdk")} + if !filepath.IsAbs(options.StateRoot) || filepath.Clean(options.StateRoot) != options.StateRoot { + return fail(fmt.Errorf("Claude SDK requires an explicit state root")) + } + config = Config{Node: node, Entrypoint: entrypoint, StateDir: filepath.Join(options.StateRoot, "daemon", options.Profile, "runtime", "claude-sdk")} binding, err := localworkspace.Load() if err != nil { return fail(err) } if binding != nil { - root, err := paths.Root() - if err != nil { - return fail(err) - } config.Node, err = filepath.EvalSymlinks(node) if err != nil { return fail(err) } - config, err = ConfigureLocal(config, root, os.Getenv("OAC_RUNTIME_WORKSPACE"), binding.NetworkPolicy()) + config, err = ConfigureLocal(config, options.StateRoot, options.RuntimeRoot, os.Getenv("OAC_RUNTIME_WORKSPACE"), binding.NetworkPolicy()) if err != nil { return fail(err) } @@ -118,6 +117,7 @@ func discoverWithCheck(parent context.Context, options agent.DiscoveryOptions, d caps.EnvironmentNone, caps.FunctionTools = proto.CapabilityUnsupported, proto.CapabilityFromBool(info.SupportsWorkspaceFunctions()) caps.LocalEnvironment, caps.WorkspaceReadPreparation = proto.CapabilitySupported, proto.CapabilitySupported caps.NativeSessionRecovery = proto.CapabilitySupported + caps.RetainedNativeHistory = proto.CapabilityFromBool(info.SDK == "0.3.269" && info.Native == "2.1.269 (Claude Code)") } out.Info.Available, out.Info.Version = true, info.SDK out.Info.Capabilities.MessageImages = proto.CapabilityFromBool(info.SupportsMessageImages()) diff --git a/apps/daemon/internal/agent/claudesdk/declaration_test.go b/apps/daemon/internal/agent/claudesdk/declaration_test.go index 9181b5454..de5271b7b 100644 --- a/apps/daemon/internal/agent/claudesdk/declaration_test.go +++ b/apps/daemon/internal/agent/claudesdk/declaration_test.go @@ -24,9 +24,9 @@ func TestClaudeSDKInvalidPathsFailBeforeProbe(t *testing.T) { if relative == "entrypoint" { t.Setenv(claudeSDKEntrypointEnv, "main.js") } else { - t.Setenv("OAC_RUNTIME_HOME", "relative-home") + root = "relative-home" } - out := discoverWithCheck(t.Context(), agent.DiscoveryOptions{Profile: "default", Stdout: &strings.Builder{}, Stderr: &strings.Builder{}}, Declaration.Info, func(context.Context, Config) (RuntimeInfo, error) { + out := discoverWithCheck(t.Context(), agent.DiscoveryOptions{RuntimeRoot: root, StateRoot: t.TempDir(), Profile: "default", Stdout: &strings.Builder{}, Stderr: &strings.Builder{}}, Declaration.Info, func(context.Context, Config) (RuntimeInfo, error) { t.Fatal("invalid paths reached runtime probe") return RuntimeInfo{}, nil }) @@ -47,7 +47,7 @@ func TestClaudeSDKFeatureDiscovery(t *testing.T) { } t.Setenv(claudeSDKNodeEnv, node) for _, features := range [][]string{nil, {"mcp_http_tools"}, {"mcp_http_bearer_auth"}, {"mcp_http_tools", "mcp_http_bearer_auth"}, {"mcp_http_required"}, {"mcp_http_tools", "mcp_http_required"}, {"subagent_resources"}, {"structured_output"}} { - out := discoverWithCheck(t.Context(), agent.DiscoveryOptions{Profile: "default", Stdout: &strings.Builder{}, Stderr: &strings.Builder{}}, Declaration.Info, func(context.Context, Config) (RuntimeInfo, error) { + out := discoverWithCheck(t.Context(), agent.DiscoveryOptions{RuntimeRoot: root, StateRoot: t.TempDir(), Profile: "default", Stdout: &strings.Builder{}, Stderr: &strings.Builder{}}, Declaration.Info, func(context.Context, Config) (RuntimeInfo, error) { info := RuntimeInfo{SDK: "0.3.269", Native: "2.1.269 (Claude Code)", Features: features} return info, nil }) @@ -92,7 +92,7 @@ func TestRuntimeDiscoveryConfigurationAndRegistration(t *testing.T) { } t.Setenv(claudeSDKNodeEnv, node) entrypoint := filepath.Join(root, "bundle", "main.js") - options := agent.DiscoveryOptions{Profile: "test", Stdout: io.Discard, Stderr: io.Discard} + options := agent.DiscoveryOptions{RuntimeRoot: root, StateRoot: t.TempDir(), Profile: "test", Stdout: io.Discard, Stderr: io.Discard} for _, configured := range []bool{false, true} { for _, ready := range []bool{false, true} { t.Setenv(claudeSDKEntrypointEnv, "") @@ -102,7 +102,7 @@ func TestRuntimeDiscoveryConfigurationAndRegistration(t *testing.T) { calls := 0 runtime := discoverWithCheck(t.Context(), options, Declaration.Info, func(_ context.Context, c Config) (RuntimeInfo, error) { calls++ - if c.Node != node || c.Entrypoint != entrypoint || c.StateDir != filepath.Join(root, "daemon", "test", "runtime", "claude-sdk") || c.Env != nil { + if c.Node != node || c.Entrypoint != entrypoint || c.StateDir != filepath.Join(options.StateRoot, "daemon", "test", "runtime", "claude-sdk") || c.Env != nil { t.Fatalf("configuration: %+v", c) } if !ready { diff --git a/apps/daemon/internal/agent/claudesdk/local.go b/apps/daemon/internal/agent/claudesdk/local.go index d3340f3f9..d8bf94c47 100644 --- a/apps/daemon/internal/agent/claudesdk/local.go +++ b/apps/daemon/internal/agent/claudesdk/local.go @@ -11,8 +11,11 @@ import ( // ConfigureLocal selects the qualified, dedicated Runtime layout. The shared // localworkspace binding still authorizes every request against its Session. -func ConfigureLocal(config Config, root, workspace string, network agentnetwork.Policy) (Config, error) { - config.StateDir = filepath.Join(root, "runtime", "claude-sdk", "history") +func ConfigureLocal(config Config, stateRoot, root, workspace string, network agentnetwork.Policy) (Config, error) { + if !filepath.IsAbs(stateRoot) || filepath.Clean(stateRoot) != stateRoot { + return Config{}, fmt.Errorf("claudesdk: state root must be a clean absolute directory") + } + config.StateDir = filepath.Join(stateRoot, "runtime", "claude-sdk", "history") config.Workspace = &WorkspaceConfig{ Directory: workspace, PublicDirectory: workspace, NetworkAccess: network.Access, AllowedDomains: network.Hosts(), HomeDir: filepath.Join(root, "runtime", "claude-sdk", "home"), diff --git a/apps/daemon/internal/agent/claudesdk/local_test.go b/apps/daemon/internal/agent/claudesdk/local_test.go index 2d6408645..797562ebc 100644 --- a/apps/daemon/internal/agent/claudesdk/local_test.go +++ b/apps/daemon/internal/agent/claudesdk/local_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentnetwork" ) func TestLocalWorkspaceBindingNetworkAndRequiredHistory(t *testing.T) { @@ -82,3 +83,25 @@ func TestWorkspaceProviderCredentialsReplaceAmbientSelection(t *testing.T) { } } } + +func TestLocalHistorySurvivesChangingInstanceRoot(t *testing.T) { + config := workspaceFixture(t) + retained, instance := t.TempDir(), t.TempDir() + first, err := ConfigureLocal(config, retained, instance, config.Workspace.Directory, agentnetwork.Policy{Access: "enabled"}) + if err != nil { + t.Fatal(err) + } + second, err := ConfigureLocal(config, retained, t.TempDir(), config.Workspace.Directory, agentnetwork.Policy{Access: "enabled"}) + if err != nil { + t.Fatal(err) + } + if first.StateDir != second.StateDir || first.StateDir != filepath.Join(retained, "runtime", "claude-sdk", "history") { + t.Fatal("history moved with compute") + } + if first.Workspace.HomeDir == second.Workspace.HomeDir || first.Workspace.ScratchDir == second.Workspace.ScratchDir { + t.Fatal("instance HOME or scratch retained across compute") + } + if _, err := ConfigureLocal(config, "relative", instance, config.Workspace.Directory, agentnetwork.Policy{Access: "enabled"}); err == nil { + t.Fatal("invalid state root accepted") + } +} diff --git a/apps/daemon/internal/agent/claudesdk/options.go b/apps/daemon/internal/agent/claudesdk/options.go index 46ffde096..81f968eff 100644 --- a/apps/daemon/internal/agent/claudesdk/options.go +++ b/apps/daemon/internal/agent/claudesdk/options.go @@ -6,7 +6,6 @@ import ( "path/filepath" "strings" - "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/paths" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" harnessconfiguration "github.com/MiniMax-AI/OpenAgentCore/internal/harnessconfig/claudesdk" ) @@ -152,13 +151,8 @@ func prepareConfiguration(config Config, req proto.PromptRequestPayload) (startR if req.LocalEnvironment != nil || req.RequireExistingNativeSession { return fail("local execution and history recovery require a dedicated workspace") } - root, err := paths.Root() - if err != nil { - return startRequest{}, nil, err - } - relative, err := filepath.Rel(root, config.StateDir) - if err != nil || !filepath.IsAbs(root) || !filepath.IsAbs(config.StateDir) || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { - return fail("SDK state must be in a managed runtime subdirectory") + if !filepath.IsAbs(config.StateDir) || filepath.Clean(config.StateDir) != config.StateDir || filepath.Dir(config.StateDir) == config.StateDir { + return fail("SDK state must be an explicit private absolute directory") } start.Cwd = filepath.Join(config.StateDir, "work") for _, dir := range []string{config.StateDir, filepath.Join(config.StateDir, "tmp"), start.Cwd} { diff --git a/apps/daemon/internal/agent/claudesdk/session_test.go b/apps/daemon/internal/agent/claudesdk/session_test.go index 30899cfc7..7c0f094c2 100644 --- a/apps/daemon/internal/agent/claudesdk/session_test.go +++ b/apps/daemon/internal/agent/claudesdk/session_test.go @@ -86,7 +86,7 @@ func TestTextFactoryRejectsUnsupportedInput(t *testing.T) { case "tool": request.FunctionTools = []proto.FunctionTool{{}} case "outside": - config.StateDir = filepath.Dir(root) + config.StateDir = "relative-state" } _, err := startSingleTurn(context.Background(), config, request, make(chan proto.Envelope, 1)) if err == nil || !strings.HasPrefix(err.Error(), "claudesdk:") { diff --git a/apps/daemon/internal/agent/codex/declaration.go b/apps/daemon/internal/agent/codex/declaration.go index 14d3277a0..6d02affce 100644 --- a/apps/daemon/internal/agent/codex/declaration.go +++ b/apps/daemon/internal/agent/codex/declaration.go @@ -18,6 +18,7 @@ var Declaration = agent.Declaration{Info: proto.SupportedAgentKind{ Streaming: proto.CapabilitySupported, Usage: proto.CapabilitySupported, Resume: proto.CapabilitySupported, + RetainedNativeHistory: proto.CapabilityUnsupported, NativeSessionRecovery: proto.CapabilityUnsupported, Steering: proto.CapabilitySupported, MessageItems: proto.CapabilitySupported, @@ -62,8 +63,9 @@ func discoverWithCheck(parent context.Context, options agent.DiscoveryOptions, i caps.NativeSessionRecovery = proto.CapabilityFromBool(SupportsNativeSessionRecovery(version)) caps.LocalEnvironment = proto.CapabilityFromBool(SupportsLocalEnvironment(version)) caps.WorkspaceReadPreparation = caps.LocalEnvironment + caps.RetainedNativeHistory = caps.LocalEnvironment caps.MCPHTTPRequired = proto.CapabilityFromBool(SupportsNativeSessionRecovery(version)) - runtime.Executor = NewExecutorFactory() + runtime.Executor = NewExecutorFactory(options.StateRoot) fmt.Fprintf(options.Stdout, "Codex preflight ok (%s)\n", version) return runtime } diff --git a/apps/daemon/internal/agent/codex/error_classification_test.go b/apps/daemon/internal/agent/codex/error_classification_test.go index cc6e21340..6a5a04014 100644 --- a/apps/daemon/internal/agent/codex/error_classification_test.go +++ b/apps/daemon/internal/agent/codex/error_classification_test.go @@ -50,7 +50,7 @@ func TestClassificationOnlyFromRootTerminalError(t *testing.T) { for _, mode := range []string{"failed", "recovered", "notification-only"} { t.Run(mode, func(t *testing.T) { out := make(chan proto.Envelope, 16) - s := &Session{runID: "run", out: out, cancelCtx: context.Background(), cfg: defaultSessionConfig(), rpc: NewJSONRPCClient(JSONRPCConfig{})} + s := &Session{runID: "run", out: out, cancelCtx: context.Background(), cfg: defaultSessionConfig(testStateRoot(t)), rpc: NewJSONRPCClient(JSONRPCConfig{})} s.registerHandlers() s.setThreadID("root") scopeNotification(t, s, "turn/started", `{"threadId":"root","turn":{"id":"turn"}}`) diff --git a/apps/daemon/internal/agent/codex/execution_controls_test.go b/apps/daemon/internal/agent/codex/execution_controls_test.go index e9d6f5451..374e0e1a0 100644 --- a/apps/daemon/internal/agent/codex/execution_controls_test.go +++ b/apps/daemon/internal/agent/codex/execution_controls_test.go @@ -11,7 +11,7 @@ func TestExecutionControlsSelectNativeSettings(t *testing.T) { t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) for _, search := range []string{"disabled", "cached", "live"} { for _, verbosity := range []string{"low", "medium", "high"} { - plan, err := BuildSessionPlan("state", map[string]any{"model": "test-model"}, &proto.ExecutionControls{WebSearch: search, TextVerbosity: verbosity}) + plan, err := BuildSessionPlan(testStateRoot(t), "state", map[string]any{"model": "test-model"}, &proto.ExecutionControls{WebSearch: search, TextVerbosity: verbosity}) if err != nil { t.Fatal(err) } @@ -30,7 +30,7 @@ func TestExecutionControlsRejectIncompleteOrInvalidValues(t *testing.T) { {}, {WebSearch: "disabled"}, {TextVerbosity: "medium"}, {WebSearch: "invalid", TextVerbosity: "medium"}, {WebSearch: "disabled", TextVerbosity: "invalid"}, } { - if plan, err := BuildSessionPlan("state", nil, &controls); err == nil { + if plan, err := BuildSessionPlan(testStateRoot(t), "state", nil, &controls); err == nil { plan.Cleanup() t.Fatal("invalid controls accepted", controls) } diff --git a/apps/daemon/internal/agent/codex/executor.go b/apps/daemon/internal/agent/codex/executor.go index d8997e525..81a545a7c 100644 --- a/apps/daemon/internal/agent/codex/executor.go +++ b/apps/daemon/internal/agent/codex/executor.go @@ -21,8 +21,8 @@ type Executor struct { closeMu sync.Mutex } -func PrepareExecutor(ctx context.Context, req proto.PromptRequestPayload) (agent.Executor, error) { - return newExecutor(ctx, req, defaultSessionConfig()) +func PrepareExecutor(ctx context.Context, stateRoot string, req proto.PromptRequestPayload) (agent.Executor, error) { + return newExecutor(ctx, req, defaultSessionConfig(stateRoot)) } func newExecutor(ctx context.Context, req proto.PromptRequestPayload, cfg sessionConfig) (*Executor, error) { @@ -172,4 +172,8 @@ func (e *Executor) Close(ctx context.Context) error { var _ agent.Executor = (*Executor)(nil) -func NewExecutorFactory() agent.ExecutorFactory { return PrepareExecutor } +func NewExecutorFactory(stateRoot string) agent.ExecutorFactory { + return func(ctx context.Context, req proto.PromptRequestPayload) (agent.Executor, error) { + return PrepareExecutor(ctx, stateRoot, req) + } +} diff --git a/apps/daemon/internal/agent/codex/executor_native_test.go b/apps/daemon/internal/agent/codex/executor_native_test.go index f14f1340e..faee82904 100644 --- a/apps/daemon/internal/agent/codex/executor_native_test.go +++ b/apps/daemon/internal/agent/codex/executor_native_test.go @@ -45,7 +45,7 @@ func TestExecutorNativeReuse(t *testing.T) { for _, name := range []string{"CODEX_EXEC_SERVER_URL", "CODEX_EXEC_SERVER_NOISE_REGISTRY_URL", "CODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID", "CODEX_EXEC_SERVER_NOISE_AUTH_TOKEN"} { t.Setenv(name, "") } - cfg := defaultSessionConfig() + cfg := defaultSessionConfig(testStateRoot(t)) cfg.codexBinary = binary cfg.logger = slog.New(slog.DiscardHandler) req := proto.PromptRequestPayload{ diff --git a/apps/daemon/internal/agent/codex/harness_config_test.go b/apps/daemon/internal/agent/codex/harness_config_test.go index cd3577d3c..7ca086aa9 100644 --- a/apps/daemon/internal/agent/codex/harness_config_test.go +++ b/apps/daemon/internal/agent/codex/harness_config_test.go @@ -13,7 +13,7 @@ import ( func TestHarnessConfigAppliedWithoutChangingProvider(t *testing.T) { t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) - plan, err := BuildSessionPlan("native-config", map[string]any{ + plan, err := BuildSessionPlan(testStateRoot(t), "native-config", map[string]any{ "model": "fixture", "harness_config": map[string]any{"model_reasoning_effort": "high"}, "model_provider": map[string]any{"base_url": "https://provider.invalid/v1", "protocol": "responses", "api_key": "test-key"}, }, nil) @@ -27,7 +27,7 @@ func TestHarnessConfigAppliedWithoutChangingProvider(t *testing.T) { } func TestHarnessConfigConflictFailsBeforePreparation(t *testing.T) { - _, err := BuildSessionPlan("", map[string]any{"harness_config": map[string]any{"model_provider": "bypass"}}, nil) + _, err := BuildSessionPlan(testStateRoot(t), "", map[string]any{"harness_config": map[string]any{"model_provider": "bypass"}}, nil) if err != harnessconfig.ErrHarnessConfig { t.Fatalf("configuration must fail before filesystem preparation: %v", err) } diff --git a/apps/daemon/internal/agent/codex/mcp_http_bearer_test.go b/apps/daemon/internal/agent/codex/mcp_http_bearer_test.go index 0223e6716..5a68ce270 100644 --- a/apps/daemon/internal/agent/codex/mcp_http_bearer_test.go +++ b/apps/daemon/internal/agent/codex/mcp_http_bearer_test.go @@ -22,7 +22,7 @@ func TestMCPHTTPBearerPlanSeparatesServersAndProcesses(t *testing.T) { req := proto.PromptRequestPayload{AgentStateKey: "retained-mcp", DisableExecutionEnvironment: true, MCPHTTPServers: &servers} seen := map[string]bool{} for range 2 { - plan, _, err := prepareSessionPlan(t.Context(), req, defaultSessionConfig()) + plan, _, err := prepareSessionPlan(t.Context(), req, defaultSessionConfig(testStateRoot(t))) if err != nil { t.Fatal(err) } @@ -58,7 +58,7 @@ func TestMCPHTTPBearerRejectsInvalidTokensWithoutPersistence(t *testing.T) { for _, token := range []string{"", "=", " has-space", "has-space ", "has space", "line\r\ninjection", "nul\x00byte", "opaque中文", "middle=padding", "punctuation:invalid"} { servers := []proto.MCPHTTPServer{{ConnectionOrigin: "service", ServerLabel: "tools", ServerURL: "https://tools.example/mcp", BearerToken: &token}} req := proto.PromptRequestPayload{AgentStateKey: "invalid-bearer", DisableExecutionEnvironment: true, MCPHTTPServers: &servers} - if _, _, err := prepareSessionPlan(t.Context(), req, defaultSessionConfig()); err == nil || err.Error() != "invalid HTTPS MCP bearer credential" { + if _, _, err := prepareSessionPlan(t.Context(), req, defaultSessionConfig(testStateRoot(t))); err == nil || err.Error() != "invalid HTTPS MCP bearer credential" { t.Fatal("invalid bearer value accepted or unsafe error returned") } } @@ -82,7 +82,7 @@ func TestMCPHTTPBearerDoesNotReachModelCatalogProbe(t *testing.T) { servers := []proto.MCPHTTPServer{{ConnectionOrigin: "service", ServerLabel: "tools", ServerURL: "https://tools.example/mcp", BearerToken: &token}} req := proto.PromptRequestPayload{AgentStateKey: "catalog", DisableExecutionEnvironment: true, MCPHTTPServers: &servers, AgentOptions: map[string]any{"model": "fixture-model"}, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}} - cfg := defaultSessionConfig() + cfg := defaultSessionConfig(testStateRoot(t)) cfg.codexBinary = binary plan, _, err := prepareSessionPlan(t.Context(), req, cfg) if err != nil { diff --git a/apps/daemon/internal/agent/codex/mcp_http_preflight_test.go b/apps/daemon/internal/agent/codex/mcp_http_preflight_test.go index c57e3297c..138a97182 100644 --- a/apps/daemon/internal/agent/codex/mcp_http_preflight_test.go +++ b/apps/daemon/internal/agent/codex/mcp_http_preflight_test.go @@ -166,7 +166,7 @@ func TestPublicMCPHTTPPreparationChecksBeforeNewAndResumedThread(t *testing.T) { t.Fatal(err) } assertPreparationOnly(t, root) - home, err := allocCodexHome(req.AgentStateKey) + home, err := allocCodexHome(testStateRoot(t), req.AgentStateKey) if err != nil { t.Fatal(err) } diff --git a/apps/daemon/internal/agent/codex/mcp_http_test.go b/apps/daemon/internal/agent/codex/mcp_http_test.go index 20ac1767f..015c0e939 100644 --- a/apps/daemon/internal/agent/codex/mcp_http_test.go +++ b/apps/daemon/internal/agent/codex/mcp_http_test.go @@ -21,7 +21,7 @@ func TestPublicMCPHTTPPlanOwnsConfigurationAndPreservesHistory(t *testing.T) { {ConnectionOrigin: "service", ServerLabel: "blocked", ServerURL: "http://127.0.0.1:12345/mcp", AllowedTools: &denyAll}, } req := proto.PromptRequestPayload{AgentStateKey: "public-mcp", DisableExecutionEnvironment: true, MCPHTTPServers: &servers} - plan, _, err := prepareSessionPlan(t.Context(), req, defaultSessionConfig()) + plan, _, err := prepareSessionPlan(t.Context(), req, defaultSessionConfig(testStateRoot(t))) if err != nil { t.Fatal(err) } @@ -29,7 +29,7 @@ func TestPublicMCPHTTPPlanOwnsConfigurationAndPreservesHistory(t *testing.T) { if !slices.Contains(plan.DisableFeatures, "apps") || !slices.Contains(plan.DisableFeatures, "plugins") || !slices.Contains(plan.ExtraConfig, [2]string{"mcp_oauth_credentials_store", `"file"`}) { t.Fatal("native profile was not pinned") } - home, err := allocCodexHome(req.AgentStateKey) + home, err := allocCodexHome(testStateRoot(t), req.AgentStateKey) if err != nil { t.Fatal(err) } @@ -54,7 +54,7 @@ func TestPublicMCPHTTPPlanOwnsConfigurationAndPreservesHistory(t *testing.T) { t.Fatal(err) } servers = []proto.MCPHTTPServer{{ConnectionOrigin: "service", ServerLabel: "replacement", ServerURL: "https://new.example/mcp"}} - second, _, err := prepareSessionPlan(t.Context(), req, defaultSessionConfig()) + second, _, err := prepareSessionPlan(t.Context(), req, defaultSessionConfig(testStateRoot(t))) if err != nil { t.Fatal(err) } @@ -90,7 +90,7 @@ func TestPublicMCPHTTPRejectsInvalidProfileAndStoredCredentials(t *testing.T) { } } t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) - home, err := allocCodexHome("credentials") + home, err := allocCodexHome(testStateRoot(t), "credentials") if err != nil { t.Fatal(err) } @@ -100,7 +100,7 @@ func TestPublicMCPHTTPRejectsInvalidProfileAndStoredCredentials(t *testing.T) { t.Fatal(err) } req := proto.PromptRequestPayload{AgentStateKey: "credentials", DisableExecutionEnvironment: true, MCPHTTPServers: &valid} - if _, _, err := prepareSessionPlan(t.Context(), req, defaultSessionConfig()); err == nil { + if _, _, err := prepareSessionPlan(t.Context(), req, defaultSessionConfig(testStateRoot(t))); err == nil { t.Fatal("existing MCP credentials accepted") } if after, err := os.ReadFile(path); err != nil || !reflect.DeepEqual(after, stored) { diff --git a/apps/daemon/internal/agent/codex/model_route_test.go b/apps/daemon/internal/agent/codex/model_route_test.go index 32aeb4680..c18fecfe9 100644 --- a/apps/daemon/internal/agent/codex/model_route_test.go +++ b/apps/daemon/internal/agent/codex/model_route_test.go @@ -11,7 +11,7 @@ func TestPlanRejectsNonNativeFrozenProvider(t *testing.T) { for _, protocol := range []string{"anthropic", "chat_completions"} { t.Run(protocol, func(t *testing.T) { provider := map[string]any{"protocol": protocol, "base_url": "https://model.invalid/v1", "api_key": "private-sentinel"} - plan, err := BuildSessionPlan("frozen-state", map[string]any{"model": "frozen-model", "model_provider": provider}, nil) + plan, err := BuildSessionPlan(testStateRoot(t), "frozen-state", map[string]any{"model": "frozen-model", "model_provider": provider}, nil) if plan.Cleanup != nil { plan.Cleanup() } @@ -30,7 +30,7 @@ func TestPlanRejectsIncompleteExplicitProvider(t *testing.T) { {"model": "chosen", "model_provider": nil}, {"model_provider": map[string]any{"protocol": "responses", "base_url": "https://model.example/v1", "api_key": "fixture"}}, } { - if _, err := BuildSessionPlan("state", options, nil); err == nil { + if _, err := BuildSessionPlan(testStateRoot(t), "state", options, nil); err == nil { t.Fatal("explicit provider fell back to native defaults") } } diff --git a/apps/daemon/internal/agent/codex/model_verbosity_test.go b/apps/daemon/internal/agent/codex/model_verbosity_test.go index e1b9ecd45..83a65e502 100644 --- a/apps/daemon/internal/agent/codex/model_verbosity_test.go +++ b/apps/daemon/internal/agent/codex/model_verbosity_test.go @@ -37,7 +37,7 @@ func TestPrepareModelVerbosity(t *testing.T) { if err := os.WriteFile(binary, []byte("#!/bin/sh\nprintf '%s' '"+catalog+"'\n"), 0700); err != nil { t.Fatal(err) } - plan, err := BuildSessionPlan("state", map[string]any{"model": "known-model"}, &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "high"}) + plan, err := BuildSessionPlan(testStateRoot(t), "state", map[string]any{"model": "known-model"}, &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "high"}) if err != nil { t.Fatal(err) } @@ -80,7 +80,7 @@ func TestPrepareDefaultModelVerbosity(t *testing.T) { for _, model := range []string{"supported", "unsupported", "unknown-provider-model"} { for _, level := range []string{"low", "medium", "high"} { t.Run(model+"/"+level, func(t *testing.T) { - plan, err := BuildSessionPlan("state", map[string]any{"model": model}, &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: level}) + plan, err := BuildSessionPlan(testStateRoot(t), "state", map[string]any{"model": model}, &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: level}) if err != nil { t.Fatal(err) } diff --git a/apps/daemon/internal/agent/codex/options.go b/apps/daemon/internal/agent/codex/options.go index 1f53a8d24..02c1e194e 100644 --- a/apps/daemon/internal/agent/codex/options.go +++ b/apps/daemon/internal/agent/codex/options.go @@ -9,7 +9,6 @@ import ( "slices" "strings" - "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/paths" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" harnessconfiguration "github.com/MiniMax-AI/OpenAgentCore/internal/harnessconfig/codex" ) @@ -71,7 +70,7 @@ type SessionPlan struct { // // Execution controls select the native web_search and model_verbosity settings. // The codex binary itself is resolved via PATH only. -func BuildSessionPlan(agentStateKey string, opts map[string]any, controls *proto.ExecutionControls) (SessionPlan, error) { +func BuildSessionPlan(stateRoot, agentStateKey string, opts map[string]any, controls *proto.ExecutionControls) (SessionPlan, error) { plan := SessionPlan{ // Harnesses run unattended: Codex never offers its ask-the-user tool. ExtraConfig: [][2]string{{"tools.experimental_request_user_input.enabled", "false"}}, @@ -99,7 +98,7 @@ func BuildSessionPlan(agentStateKey string, opts map[string]any, controls *proto plan.Model = stringOpt(opts, "model") plan.SystemPrompt = stringOpt(opts, "system_prompt") - codexHome, err := allocCodexHome(agentStateKey) + codexHome, err := allocCodexHome(stateRoot, agentStateKey) if err != nil { return plan, err } @@ -117,6 +116,11 @@ func BuildSessionPlan(agentStateKey string, opts map[string]any, controls *proto return plan, err } plan.ModelProvider = oacProviderSlug + // The pinned native shell snapshot captures the loop environment before + // shell_environment_policy is applied. Do not persist provider credentials. + plan.DisableFeatures = append(plan.DisableFeatures, "shell_snapshot") + env = append(env, providerAPIKeyEnv+"="+provider.BearerToken) + plan.ExtraConfig = append(plan.ExtraConfig, [2]string{"shell_environment_policy.exclude", "[" + strconv(providerAPIKeyEnv) + "]"}) } plan.Env = env @@ -142,18 +146,17 @@ func BuildSessionPlan(agentStateKey string, opts map[string]any, controls *proto // helpers // --------------------------------------------------------------------------- -func allocCodexHome(agentStateKey string) (string, error) { +func allocCodexHome(root, agentStateKey string) (string, error) { if strings.TrimSpace(agentStateKey) == "" { return "", fmt.Errorf("codex: agentStateKey required for CODEX_HOME allocation") } - root, err := paths.Root() - if err != nil { - return "", err + if !filepath.IsAbs(root) || filepath.Clean(root) != root { + return "", fmt.Errorf("codex: state root must be a clean absolute directory") } parts := strings.Split(agentStateKey, "/") safeParts := make([]string, 0, len(parts)) for _, part := range parts { - if safe := safePathPartCodex(part); safe != "" { + if safe := safePathPartCodex(part); safe != "" && safe != "." && safe != ".." { safeParts = append(safeParts, safe) } } diff --git a/apps/daemon/internal/agent/codex/options_test.go b/apps/daemon/internal/agent/codex/options_test.go index cb8910aaa..eb64d84de 100644 --- a/apps/daemon/internal/agent/codex/options_test.go +++ b/apps/daemon/internal/agent/codex/options_test.go @@ -8,7 +8,7 @@ import ( ) func TestBuildSessionPlan_DefaultsToBypass(t *testing.T) { - plan, err := BuildSessionPlan("conv-1/agent-1/codex", nil, nil) + plan, err := BuildSessionPlan(testStateRoot(t), "conv-1/agent-1/codex", nil, nil) if err != nil { t.Fatalf("BuildSessionPlan: %v", err) } @@ -22,7 +22,7 @@ func TestBuildSessionPlan_DefaultsToBypass(t *testing.T) { } func TestBuildSessionPlan_AllocsCodexHomeAndEnv(t *testing.T) { - plan, err := BuildSessionPlan("conv-1/agent-1/codex", nil, nil) + plan, err := BuildSessionPlan(testStateRoot(t), "conv-1/agent-1/codex", nil, nil) if err != nil { t.Fatalf("BuildSessionPlan: %v", err) } @@ -47,11 +47,11 @@ func TestBuildSessionPlan_AllocsCodexHomeAndEnv(t *testing.T) { func TestBuildSessionPlan_StableCodexHomeByStateKey(t *testing.T) { stateKey := "conv-stable/agent-stable/codex" - planA, err := BuildSessionPlan(stateKey, nil, nil) + planA, err := BuildSessionPlan(testStateRoot(t), stateKey, nil, nil) if err != nil { t.Fatalf("BuildSessionPlan A: %v", err) } - planB, err := BuildSessionPlan(stateKey, nil, nil) + planB, err := BuildSessionPlan(testStateRoot(t), stateKey, nil, nil) if err != nil { t.Fatalf("BuildSessionPlan B: %v", err) } diff --git a/apps/daemon/internal/agent/codex/permission_profile_test.go b/apps/daemon/internal/agent/codex/permission_profile_test.go index e69720d6e..c8bd52e71 100644 --- a/apps/daemon/internal/agent/codex/permission_profile_test.go +++ b/apps/daemon/internal/agent/codex/permission_profile_test.go @@ -12,7 +12,7 @@ func TestRuntimeUsesHostPermissions(t *testing.T) { t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) { req := proto.PromptRequestPayload{AgentStateKey: "session", DisableSubagents: true, LocalEnvironment: &proto.LocalEnvironment{NetworkAccess: "enabled", WorkspaceRoot: t.TempDir()}} - plan, _, err := prepareSessionPlan(t.Context(), req, sessionConfig{}) + plan, _, err := prepareSessionPlan(t.Context(), req, sessionConfig{stateRoot: testStateRoot(t)}) if err != nil { t.Fatal(err) } @@ -27,7 +27,7 @@ func TestRuntimeUsesHostPermissions(t *testing.T) { plan.Cleanup() for _, network := range []string{"disabled", "restricted"} { req.LocalEnvironment.NetworkAccess = network - if _, _, err := prepareSessionPlan(t.Context(), req, sessionConfig{}); err == nil { + if _, _, err := prepareSessionPlan(t.Context(), req, sessionConfig{stateRoot: testStateRoot(t)}); err == nil { t.Fatal("unsupported network admitted") } } @@ -49,7 +49,7 @@ func TestSelfHostedToolEnvironmentCannotRedirectNativeHistory(t *testing.T) { t.Setenv(key, value) } req := proto.PromptRequestPayload{AgentStateKey: "session", DisableSubagents: true, LocalEnvironment: &proto.LocalEnvironment{NetworkAccess: "enabled"}} - plan, _, err := prepareSessionPlan(t.Context(), req, sessionConfig{}) + plan, _, err := prepareSessionPlan(t.Context(), req, sessionConfig{stateRoot: testStateRoot(t)}) if err != nil { t.Fatal(err) } diff --git a/apps/daemon/internal/agent/codex/preparation_helpers_test.go b/apps/daemon/internal/agent/codex/preparation_helpers_test.go index 2749dc255..1472c3a5d 100644 --- a/apps/daemon/internal/agent/codex/preparation_helpers_test.go +++ b/apps/daemon/internal/agent/codex/preparation_helpers_test.go @@ -37,7 +37,7 @@ func preparationFixture(t *testing.T) (proto.PromptRequestPayload, sessionConfig if err := os.WriteFile(binary, []byte(body), 0o700); err != nil { t.Fatal(err) } - cfg := defaultSessionConfig() + cfg := defaultSessionConfig(testStateRoot(t)) cfg.codexBinary = binary req := proto.PromptRequestPayload{ AgentKind: "codex", AgentStateKey: "prepared-session", diff --git a/apps/daemon/internal/agent/codex/provider_config.go b/apps/daemon/internal/agent/codex/provider_config.go index b3c571e0d..9eca33486 100644 --- a/apps/daemon/internal/agent/codex/provider_config.go +++ b/apps/daemon/internal/agent/codex/provider_config.go @@ -16,6 +16,7 @@ import ( // the config.toml deterministic and removes a footgun where two prompts // in the same CODEX_HOME could disagree on which provider to use. const oacProviderSlug = "oac" +const providerAPIKeyEnv = "OAC_CODEX_PROVIDER_API_KEY" // providerConfig is the daemon-internal view of the model provider the // Runtime prepared for this Session. Flattened from the common @@ -30,8 +31,7 @@ type providerConfig struct { // BaseURL is the model provider's HTTPS endpoint, including any // path prefix (e.g. /v1). Required. BaseURL string - // BearerToken is the literal API key. Codex's `experimental_bearer_token` - // field accepts a string; we emit it verbatim. Required. + // BearerToken is supplied only through the native env_key mechanism. BearerToken string // HTTPHeaders is forwarded as `[model_providers.oac.http_headers]`. // Keys / values rendered as TOML basic strings. @@ -90,8 +90,8 @@ func writeCodexProviderConfig(codexHome string, cfg providerConfig) error { b.WriteString(tomlQuoteString(cfg.BaseURL)) b.WriteByte('\n') - b.WriteString("experimental_bearer_token = ") - b.WriteString(tomlQuoteString(cfg.BearerToken)) + b.WriteString("env_key = ") + b.WriteString(tomlQuoteString(providerAPIKeyEnv)) b.WriteByte('\n') wireAPI := strings.TrimSpace(cfg.WireAPI) @@ -159,7 +159,7 @@ func writeCodexProviderConfig(codexHome string, cfg providerConfig) error { // // File is opened O_APPEND so concurrent writers in the same prompt // (today: at most one of each) don't race. 0o600 perms because the -// file carries the API bearer token in plaintext. +// file contains private per-Session configuration. func appendConfigTOML(path string, body string) error { f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) if err != nil { diff --git a/apps/daemon/internal/agent/codex/provider_config_test.go b/apps/daemon/internal/agent/codex/provider_config_test.go index 80eb8d48b..9ca848612 100644 --- a/apps/daemon/internal/agent/codex/provider_config_test.go +++ b/apps/daemon/internal/agent/codex/provider_config_test.go @@ -3,6 +3,7 @@ package codex import ( "os" "path/filepath" + "slices" "strings" "testing" ) @@ -21,7 +22,7 @@ func TestWriteCodexProviderConfig_Minimal(t *testing.T) { `[model_providers.oac]`, `name = "OpenAgentCore"`, `base_url = "https://platform-api.example.com/v1"`, - `experimental_bearer_token = "sk-test"`, + `env_key = "OAC_CODEX_PROVIDER_API_KEY"`, `wire_api = "responses"`, } { if !strings.Contains(body, want) { @@ -74,7 +75,7 @@ func TestWriteCodexProviderConfig_FullProvider(t *testing.T) { for _, want := range []string{ `name = "mygw"`, `base_url = "https://platform-api.example.com/v1"`, - `experimental_bearer_token = "sk-test-fixture"`, + `env_key = "OAC_CODEX_PROVIDER_API_KEY"`, `request_max_retries = 4`, `stream_max_retries = 3`, `[model_providers.oac.http_headers]`, @@ -156,7 +157,7 @@ func TestNormaliseProviderConfig_Nil(t *testing.T) { } func TestBuildSessionPlan_PinsModelProviderWhenProviderSet(t *testing.T) { - plan, err := BuildSessionPlan("conv-1/agent-1/codex", map[string]any{ + plan, err := BuildSessionPlan(testStateRoot(t), "conv-1/agent-1/codex", map[string]any{ "model": "fixture-model", "model_provider": map[string]any{"protocol": "responses", "base_url": "https://x/v1", @@ -194,7 +195,7 @@ func TestBuildSessionPlan_PinsModelProviderWhenProviderSet(t *testing.T) { } func TestBuildSessionPlan_NoProviderLeavesBuiltinDefault(t *testing.T) { - plan, err := BuildSessionPlan("conv-1/agent-1/codex", nil, nil) + plan, err := BuildSessionPlan(testStateRoot(t), "conv-1/agent-1/codex", nil, nil) if err != nil { t.Fatalf("BuildSessionPlan: %v", err) } @@ -217,3 +218,65 @@ func mustReadFile(t *testing.T, path string) string { } return string(b) } + +func TestProviderCredentialOnlyInNativeLoopEnvironment(t *testing.T) { + root := t.TempDir() + const token = "synthetic-private-provider-token" + plan, err := BuildSessionPlan(root, "stable-session", map[string]any{"model": "fixture-model", "model_provider": map[string]any{"protocol": "responses", "base_url": "https://model.example/v1", "api_key": token}}, nil) + if err != nil { + t.Fatal(err) + } + defer plan.Cleanup() + body := mustReadFile(t, filepath.Join(nativeHomeFromPlan(plan), "config.toml")) + if strings.Contains(body, token) || strings.Contains(body, "experimental_bearer_token") { + t.Fatal("provider credential persisted") + } + foundEnv, foundExclude := false, false + for _, entry := range plan.Env { + if entry == providerAPIKeyEnv+"="+token { + foundEnv = true + } + } + for _, entry := range plan.ExtraConfig { + if entry[0] == "shell_environment_policy.exclude" && entry[1] == `["OAC_CODEX_PROVIDER_API_KEY"]` { + foundExclude = true + } + if strings.Contains(entry[1], token) { + t.Fatal("credential exposed in native arguments") + } + } + if !foundEnv || !foundExclude || !slices.Contains(plan.DisableFeatures, "shell_snapshot") { + t.Fatal("native auth or explicit tool-environment exclusion absent") + } + if _, exists := os.LookupEnv(providerAPIKeyEnv); exists { + t.Fatal("provider credential leaked to Runtime process environment") + } +} + +func TestExplicitStateRootSurvivesRuntimeReplacement(t *testing.T) { + state := t.TempDir() + t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) + first, err := allocCodexHome(state, "stable-session") + if err != nil { + t.Fatal(err) + } + history := filepath.Join(first, "history-sentinel") + if err = os.WriteFile(history, []byte("retained"), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) + second, err := allocCodexHome(state, "stable-session") + if err != nil || second != first { + t.Fatal("instance identity changed native history", err) + } + if body, err := os.ReadFile(history); err != nil || string(body) != "retained" { + t.Fatal("history lost", err) + } + unavailable := filepath.Join(t.TempDir(), "file") + if err = os.WriteFile(unavailable, nil, 0600); err != nil { + t.Fatal(err) + } + if _, err = allocCodexHome(unavailable, "stable-session"); err == nil { + t.Fatal("unavailable explicit root fell back") + } +} diff --git a/apps/daemon/internal/agent/codex/recovery_test.go b/apps/daemon/internal/agent/codex/recovery_test.go index ebabf06b4..c4ea41f01 100644 --- a/apps/daemon/internal/agent/codex/recovery_test.go +++ b/apps/daemon/internal/agent/codex/recovery_test.go @@ -148,7 +148,7 @@ func TestPreparedRecoveryCannotStartWithoutExistingHistory(t *testing.T) { t.Run(environment, func(t *testing.T) { req, cfg, root := preparationFixture(t) req.RequireExistingNativeSession = true - cwd, err := allocCodexHome(req.AgentStateKey) + cwd, err := allocCodexHome(testStateRoot(t), req.AgentStateKey) if err != nil { t.Fatal(err) } diff --git a/apps/daemon/internal/agent/codex/rpc_close_test.go b/apps/daemon/internal/agent/codex/rpc_close_test.go index 7b387e23a..7d4c2c29b 100644 --- a/apps/daemon/internal/agent/codex/rpc_close_test.go +++ b/apps/daemon/internal/agent/codex/rpc_close_test.go @@ -80,7 +80,7 @@ func TestJSONRPCClientCloseCanRetryUnreapedChild(t *testing.T) { closeConcurrently(true) ownerCtx, cancelOwner := context.WithCancel(t.Context()) s := &Session{rpc: client, cancelCtx: ownerCtx, cancelFn: cancelOwner, - cfg: defaultSessionConfig(), bufs: NewItemBuffers()} + cfg: defaultSessionConfig(testStateRoot(t)), bufs: NewItemBuffers()} ctx, cancelWait := context.WithTimeout(t.Context(), 20*time.Millisecond) defer cancelWait() if err := s.Cancel(ctx); !errors.Is(err, context.DeadlineExceeded) { diff --git a/apps/daemon/internal/agent/codex/session.go b/apps/daemon/internal/agent/codex/session.go index 9d3386609..044c66ae0 100644 --- a/apps/daemon/internal/agent/codex/session.go +++ b/apps/daemon/internal/agent/codex/session.go @@ -23,13 +23,15 @@ const terminalSendTimeout = 2 * time.Second // sessionConfig is the cross-cutting knob bag; production callers use // defaultSessionConfig. type sessionConfig struct { + stateRoot string codexBinary string logger *slog.Logger killTimeout time.Duration } -func defaultSessionConfig() sessionConfig { +func defaultSessionConfig(stateRoot string) sessionConfig { return sessionConfig{ + stateRoot: stateRoot, codexBinary: defaultBinary(), logger: obslog.Bg(), killTimeout: rpcKillTimeout, diff --git a/apps/daemon/internal/agent/codex/session_cancel_test.go b/apps/daemon/internal/agent/codex/session_cancel_test.go index 32e794b6a..e4307ae1e 100644 --- a/apps/daemon/internal/agent/codex/session_cancel_test.go +++ b/apps/daemon/internal/agent/codex/session_cancel_test.go @@ -116,7 +116,7 @@ func cancellationTestSession(t *testing.T) (*Session, *TestClient, ServerSide) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) s := &Session{rpc: client.JSONRPCClient, cancelCtx: ctx, cancelFn: cancel, - cfg: defaultSessionConfig(), out: make(chan proto.Envelope, 8), bufs: NewItemBuffers()} + cfg: defaultSessionConfig(testStateRoot(t)), out: make(chan proto.Envelope, 8), bufs: NewItemBuffers()} return s, client, server } diff --git a/apps/daemon/internal/agent/codex/session_cancel_write_test.go b/apps/daemon/internal/agent/codex/session_cancel_write_test.go index 087b8e78f..e9adb4970 100644 --- a/apps/daemon/internal/agent/codex/session_cancel_write_test.go +++ b/apps/daemon/internal/agent/codex/session_cancel_write_test.go @@ -146,7 +146,7 @@ func TestCancelReleasesBlockedNativeProcess(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() s := &Session{rpc: client, cancelCtx: ctx, cancelFn: cancel, - cfg: defaultSessionConfig(), bufs: NewItemBuffers()} + cfg: defaultSessionConfig(testStateRoot(t)), bufs: NewItemBuffers()} s.setThreadID("native-thread") s.onTurnStarted(json.RawMessage(`{"threadId":"native-thread","turn":{"id":"native-turn"}}`)) cancelDone := make(chan struct{}) diff --git a/apps/daemon/internal/agent/codex/session_command_output_test.go b/apps/daemon/internal/agent/codex/session_command_output_test.go index a51fb6694..f14b3eab3 100644 --- a/apps/daemon/internal/agent/codex/session_command_output_test.go +++ b/apps/daemon/internal/agent/codex/session_command_output_test.go @@ -10,7 +10,7 @@ import ( func TestCommandOutputRequiresRootTurn(t *testing.T) { out := make(chan proto.Envelope, 10) - s := &Session{runID: "run", out: out, cancelCtx: context.Background(), cfg: defaultSessionConfig(), rpc: NewJSONRPCClient(JSONRPCConfig{})} + s := &Session{runID: "run", out: out, cancelCtx: context.Background(), cfg: defaultSessionConfig(testStateRoot(t)), rpc: NewJSONRPCClient(JSONRPCConfig{})} s.registerHandlers() s.setThreadID("root") s.beginRootTurn("root", "turn") diff --git a/apps/daemon/internal/agent/codex/session_messages_test.go b/apps/daemon/internal/agent/codex/session_messages_test.go index b17ed114e..1e09975fb 100644 --- a/apps/daemon/internal/agent/codex/session_messages_test.go +++ b/apps/daemon/internal/agent/codex/session_messages_test.go @@ -12,7 +12,7 @@ func TestMessageObservationIsOptInAndKeepsNativeBoundaries(t *testing.T) { for _, enabled := range []bool{false, true} { t.Run(map[bool]string{false: "legacy", true: "observed"}[enabled], func(t *testing.T) { out := make(chan proto.Envelope, 16) - s := &Session{runID: "run", observeMessages: enabled, out: out, cancelCtx: context.Background(), bufs: NewItemBuffers(), cfg: defaultSessionConfig()} + s := &Session{runID: "run", observeMessages: enabled, out: out, cancelCtx: context.Background(), bufs: NewItemBuffers(), cfg: defaultSessionConfig(testStateRoot(t))} s.setThreadID("thread") s.onTurnStarted(json.RawMessage(`{"threadId":"thread","turn":{"id":"turn"}}`)) s.onItemStarted(json.RawMessage(`{"threadId":"thread","turnId":"turn","item":{"type":"agentMessage","id":"a","phase":"commentary"}}`)) diff --git a/apps/daemon/internal/agent/codex/session_notifications_rpc_test.go b/apps/daemon/internal/agent/codex/session_notifications_rpc_test.go index f7b4d0f26..516f9bbb6 100644 --- a/apps/daemon/internal/agent/codex/session_notifications_rpc_test.go +++ b/apps/daemon/internal/agent/codex/session_notifications_rpc_test.go @@ -19,7 +19,7 @@ func TestRootRPCNotificationOrdering(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() out := make(chan proto.Envelope, 16) - s := &Session{runID: "run", out: out, rpc: client.JSONRPCClient, cancelCtx: ctx, bufs: NewItemBuffers(), cfg: defaultSessionConfig()} + s := &Session{runID: "run", out: out, rpc: client.JSONRPCClient, cancelCtx: ctx, bufs: NewItemBuffers(), cfg: defaultSessionConfig(testStateRoot(t))} s.registerHandlers() barrier := make(chan struct{}, 1) client.OnNotification("test/barrier", func(json.RawMessage) { barrier <- struct{}{} }) @@ -132,7 +132,7 @@ func TestThreadRPCResponseRequiresRootIdentity(t *testing.T) { defer cleanup() ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - s := &Session{rpc: client.JSONRPCClient, cancelCtx: ctx, cfg: defaultSessionConfig()} + s := &Session{rpc: client.JSONRPCClient, cancelCtx: ctx, cfg: defaultSessionConfig(testStateRoot(t))} s.registerHandlers() result := make(chan error, 1) go func() { result <- s.resumeThread("root", SessionPlan{}) }() diff --git a/apps/daemon/internal/agent/codex/session_notifications_test.go b/apps/daemon/internal/agent/codex/session_notifications_test.go index 0ee2a9b94..2c1f1092b 100644 --- a/apps/daemon/internal/agent/codex/session_notifications_test.go +++ b/apps/daemon/internal/agent/codex/session_notifications_test.go @@ -12,7 +12,7 @@ func TestRootNotificationIsolation(t *testing.T) { for _, childStarted := range []bool{false, true} { t.Run(map[bool]string{false: "child without thread started", true: "child thread started"}[childStarted], func(t *testing.T) { out := make(chan proto.Envelope, 64) - s := &Session{runID: "run", out: out, cancelCtx: context.Background(), cfg: defaultSessionConfig(), + s := &Session{runID: "run", out: out, cancelCtx: context.Background(), cfg: defaultSessionConfig(testStateRoot(t)), rpc: NewJSONRPCClient(JSONRPCConfig{}), bufs: NewItemBuffers(), observeMessages: true} s.registerHandlers() s.setThreadID("root") @@ -96,7 +96,7 @@ func TestRootNotificationIsolation(t *testing.T) { } func TestNotificationsCannotEstablishRootIdentity(t *testing.T) { - s := &Session{cfg: defaultSessionConfig(), rpc: NewJSONRPCClient(JSONRPCConfig{}), bufs: NewItemBuffers()} + s := &Session{cfg: defaultSessionConfig(testStateRoot(t)), rpc: NewJSONRPCClient(JSONRPCConfig{}), bufs: NewItemBuffers()} s.registerHandlers() scopeNotification(t, s, "thread/started", `{"thread":{"id":"unrelated"}}`) scopeNotification(t, s, "turn/started", `{"threadId":"unrelated","turn":{"id":"unrelated-turn"}}`) @@ -116,7 +116,7 @@ func TestRootNativeErrorNotification(t *testing.T) { for _, location := range []string{"error notification", "completion"} { t.Run(name+"/"+location, func(t *testing.T) { out := make(chan proto.Envelope, 4) - s := &Session{runID: "run", out: out, cancelCtx: context.Background(), cfg: defaultSessionConfig(), rpc: NewJSONRPCClient(JSONRPCConfig{})} + s := &Session{runID: "run", out: out, cancelCtx: context.Background(), cfg: defaultSessionConfig(testStateRoot(t)), rpc: NewJSONRPCClient(JSONRPCConfig{})} s.registerHandlers() s.setThreadID("root") scopeNotification(t, s, "turn/started", `{"threadId":"root","turn":{"id":"turn"}}`) diff --git a/apps/daemon/internal/agent/codex/session_plan.go b/apps/daemon/internal/agent/codex/session_plan.go index d8e7aede8..b917805f4 100644 --- a/apps/daemon/internal/agent/codex/session_plan.go +++ b/apps/daemon/internal/agent/codex/session_plan.go @@ -20,7 +20,7 @@ func prepareSessionPlan(ctx context.Context, req proto.PromptRequestPayload, cfg if err != nil { return SessionPlan{}, nil, err } - plan, err := BuildSessionPlan(req.AgentStateKey, req.AgentOptions, req.ExecutionControls) + plan, err := BuildSessionPlan(cfg.stateRoot, req.AgentStateKey, req.AgentOptions, req.ExecutionControls) if err != nil { return SessionPlan{}, nil, fmt.Errorf("codex: build session plan: %w", err) } diff --git a/apps/daemon/internal/agent/codex/session_policy_test.go b/apps/daemon/internal/agent/codex/session_policy_test.go index 86c5b3c74..f534f4cf4 100644 --- a/apps/daemon/internal/agent/codex/session_policy_test.go +++ b/apps/daemon/internal/agent/codex/session_policy_test.go @@ -16,7 +16,7 @@ func TestThreadRequestsApplyDeploymentPolicy(t *testing.T) { for _, method := range []string{"thread/start", "thread/resume"} { t.Run(method, func(t *testing.T) { t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) - plan, _, err := prepareSessionPlan(context.Background(), proto.PromptRequestPayload{AgentStateKey: "conv/agent/codex", DisableSubagents: true, DisableExecutionEnvironment: true}, sessionConfig{}) + plan, _, err := prepareSessionPlan(context.Background(), proto.PromptRequestPayload{AgentStateKey: "conv/agent/codex", DisableSubagents: true, DisableExecutionEnvironment: true}, sessionConfig{stateRoot: testStateRoot(t)}) if err != nil { t.Fatal(err) } diff --git a/apps/daemon/internal/agent/codex/session_tools_test.go b/apps/daemon/internal/agent/codex/session_tools_test.go index 8d4e06c8d..460a5fe4f 100644 --- a/apps/daemon/internal/agent/codex/session_tools_test.go +++ b/apps/daemon/internal/agent/codex/session_tools_test.go @@ -29,7 +29,7 @@ func TestToolObservations(t *testing.T) { } t.Run(source.ID, func(t *testing.T) { out := make(chan proto.Envelope, 4) - s := &Session{runID: "run", out: out, cancelCtx: context.Background(), bufs: NewItemBuffers(), cfg: defaultSessionConfig()} + s := &Session{runID: "run", out: out, cancelCtx: context.Background(), bufs: NewItemBuffers(), cfg: defaultSessionConfig(testStateRoot(t))} s.setThreadID("private-thread") s.onTurnStarted(json.RawMessage(`{"threadId":"private-thread","turn":{"id":"private-turn"}}`)) raw := json.RawMessage(`{"threadId":"private-thread","turnId":"private-turn","item":` + item + `}`) diff --git a/apps/daemon/internal/agent/codex/session_usage_live_test.go b/apps/daemon/internal/agent/codex/session_usage_live_test.go index 58ccc1d44..ba9fe754f 100644 --- a/apps/daemon/internal/agent/codex/session_usage_live_test.go +++ b/apps/daemon/internal/agent/codex/session_usage_live_test.go @@ -15,7 +15,7 @@ const activeUsageNotification = `{"method":"thread/tokenUsage/updated","params": func TestUsagePublishedBeforeCompletion(t *testing.T) { out := make(chan proto.Envelope, 8) - s := &Session{runID: "run", out: out, cancelCtx: t.Context(), cfg: defaultSessionConfig()} + s := &Session{runID: "run", out: out, cancelCtx: t.Context(), cfg: defaultSessionConfig(testStateRoot(t))} s.setThreadID("thread") s.onUsageUpdated(json.RawMessage(`{"threadId":"thread","turnId":"old","tokenUsage":{"total":{"inputTokens":10,"cachedInputTokens":1,"outputTokens":3,"reasoningOutputTokens":1,"totalTokens":13}}}`)) if len(s.out) != 0 { @@ -159,7 +159,7 @@ func TestLiveUsageRequiresConsistentCompleteBreakdown(t *testing.T) { } { t.Run(name, func(t *testing.T) { out := make(chan proto.Envelope, 1) - s := &Session{out: out, cancelCtx: t.Context(), cfg: defaultSessionConfig()} + s := &Session{out: out, cancelCtx: t.Context(), cfg: defaultSessionConfig(testStateRoot(t))} s.setThreadID("thread") s.onTurnStarted(json.RawMessage(`{"threadId":"thread","turn":{"id":"turn"}}`)) s.onUsageUpdated(json.RawMessage(`{"threadId":"thread","turnId":"turn","tokenUsage":{"total":` + counters + `}}`)) @@ -177,7 +177,7 @@ func TestLiveUsageRequiresConsistentCompleteBreakdown(t *testing.T) { func TestLiveUsageRejectsInconsistentResumeDelta(t *testing.T) { out := make(chan proto.Envelope, 2) - s := &Session{out: out, cancelCtx: t.Context(), cfg: defaultSessionConfig()} + s := &Session{out: out, cancelCtx: t.Context(), cfg: defaultSessionConfig(testStateRoot(t))} s.setThreadID("thread") s.onUsageUpdated(json.RawMessage(`{"threadId":"thread","turnId":"old","tokenUsage":{"total":{"inputTokens":100,"cachedInputTokens":90,"outputTokens":50,"reasoningOutputTokens":10,"totalTokens":150}}}`)) s.onTurnStarted(json.RawMessage(`{"threadId":"thread","turn":{"id":"turn"}}`)) diff --git a/apps/daemon/internal/agent/codex/session_usage_test.go b/apps/daemon/internal/agent/codex/session_usage_test.go index bdaeec6b0..b75faf625 100644 --- a/apps/daemon/internal/agent/codex/session_usage_test.go +++ b/apps/daemon/internal/agent/codex/session_usage_test.go @@ -62,7 +62,7 @@ func TestNativeTokenUsageAcrossRuns(t *testing.T) { } func TestNativeTokenUsageFreshThreadAndIgnoredPayloads(t *testing.T) { - s := &Session{out: make(chan proto.Envelope, 8), cancelCtx: t.Context(), cfg: defaultSessionConfig()} + s := &Session{out: make(chan proto.Envelope, 8), cancelCtx: t.Context(), cfg: defaultSessionConfig(testStateRoot(t))} s.setThreadID("thread") s.onTurnStarted(json.RawMessage(`{"threadId":"thread","turn":{"id":"current"}}`)) s.onUsageUpdated(json.RawMessage(`{"threadId":"thread","turnId":"current","tokenUsage":{"total":{"inputTokens":321,"outputTokens":45}}}`)) @@ -81,7 +81,7 @@ func TestNativeTokenUsageFreshThreadAndIgnoredPayloads(t *testing.T) { } func TestLegacyTurnUsagePayload(t *testing.T) { - s := &Session{out: make(chan proto.Envelope, 8), cancelCtx: t.Context(), cfg: defaultSessionConfig()} + s := &Session{out: make(chan proto.Envelope, 8), cancelCtx: t.Context(), cfg: defaultSessionConfig(testStateRoot(t))} s.setThreadID("thread") s.onTurnStarted(json.RawMessage(`{"threadId":"thread","turn":{"id":"current"}}`)) s.onUsageUpdated(json.RawMessage(`{"threadId":"thread","usage":{"inputTokens":123,"outputTokens":12}}`)) @@ -95,7 +95,7 @@ func TestLegacyTurnUsagePayload(t *testing.T) { } func TestCompleteTokenBreakdownAndCancellation(t *testing.T) { - s := &Session{out: make(chan proto.Envelope, 8), cancelCtx: t.Context(), cfg: defaultSessionConfig()} + s := &Session{out: make(chan proto.Envelope, 8), cancelCtx: t.Context(), cfg: defaultSessionConfig(testStateRoot(t))} s.setThreadID("thread") s.onUsageUpdated(json.RawMessage(`{"threadId":"thread","turnId":"old","tokenUsage":{"total":{"inputTokens":100,"cachedInputTokens":20,"outputTokens":50,"reasoningOutputTokens":10,"totalTokens":150}}}`)) s.onTurnStarted(json.RawMessage(`{"threadId":"thread","turn":{"id":"new"}}`)) @@ -164,7 +164,7 @@ func TestUnadvancedThreadTotalIsNotTurnUsage(t *testing.T) { } { t.Run(name, func(t *testing.T) { out := make(chan proto.Envelope, 8) - s := &Session{runID: "run", out: out, cancelCtx: t.Context(), cfg: defaultSessionConfig()} + s := &Session{runID: "run", out: out, cancelCtx: t.Context(), cfg: defaultSessionConfig(testStateRoot(t))} s.setThreadID("thread") total := `{"inputTokens":0,"cachedInputTokens":0,"outputTokens":0,"reasoningOutputTokens":0,"totalTokens":0}` if replay != "" { diff --git a/apps/daemon/internal/agent/codex/state_root_test.go b/apps/daemon/internal/agent/codex/state_root_test.go new file mode 100644 index 000000000..c1b456c44 --- /dev/null +++ b/apps/daemon/internal/agent/codex/state_root_test.go @@ -0,0 +1,16 @@ +package codex + +import ( + "os" + "testing" +) + +func testStateRoot(t *testing.T) string { + t.Helper() + if root := os.Getenv("OAC_RUNTIME_HOME"); root != "" { + return root + } + root := t.TempDir() + t.Setenv("OAC_RUNTIME_HOME", root) + return root +} diff --git a/apps/daemon/internal/agent/codex/subagent_observations_test.go b/apps/daemon/internal/agent/codex/subagent_observations_test.go index 3e8f7241a..14589a919 100644 --- a/apps/daemon/internal/agent/codex/subagent_observations_test.go +++ b/apps/daemon/internal/agent/codex/subagent_observations_test.go @@ -74,7 +74,7 @@ func observationSession(t *testing.T, status string) (*Session, *subagentFixture t.Fatal(err) } f.persist(t) - s := &Session{runID: "run", nativeHome: f.home, rpc: client.JSONRPCClient, out: out, cancelCtx: ctx, cancelFn: cancel, cfg: defaultSessionConfig(), bufs: NewItemBuffers()} + s := &Session{runID: "run", nativeHome: f.home, rpc: client.JSONRPCClient, out: out, cancelCtx: ctx, cancelFn: cancel, cfg: defaultSessionConfig(testStateRoot(t)), bufs: NewItemBuffers()} s.setThreadID("root") s.beginRootTurn("root", "root-turn") s.startSubagentObservations() diff --git a/apps/daemon/internal/agent/harness.go b/apps/daemon/internal/agent/harness.go index 23f8d37e7..4804282de 100644 --- a/apps/daemon/internal/agent/harness.go +++ b/apps/daemon/internal/agent/harness.go @@ -45,7 +45,11 @@ type Declaration struct { // DiscoveryOptions provides process context without naming an implementation. type DiscoveryOptions struct { - Profile string + Profile string + // StateRoot holds private history retained with the Environment filesystem. + StateRoot string + // RuntimeRoot holds instance-private configuration and process files. + RuntimeRoot string Stdout, Stderr io.Writer } diff --git a/apps/daemon/internal/agent/mcode/declaration.go b/apps/daemon/internal/agent/mcode/declaration.go index fa22848f1..03bbef288 100644 --- a/apps/daemon/internal/agent/mcode/declaration.go +++ b/apps/daemon/internal/agent/mcode/declaration.go @@ -16,6 +16,7 @@ var Declaration = agent.Declaration{Info: proto.SupportedAgentKind{Kind: "mcode" Streaming: proto.CapabilitySupported, Usage: proto.CapabilityUnsupported, Resume: proto.CapabilitySupported, + RetainedNativeHistory: proto.CapabilityUnsupported, NativeSessionRecovery: proto.CapabilityUnsupported, Steering: proto.CapabilityUnsupported, MessageItems: proto.CapabilityUnsupported, @@ -71,7 +72,7 @@ func discoverWithCheck(parent context.Context, options agent.DiscoveryOptions, r runtime.Info = result workspace := discoverWorkspace(parent, options, runtime) if runtime.Info.Available { - runtime.Executor = NewExecutorFactory(workspace) + runtime.Executor = NewExecutorFactory(options.RuntimeRoot, workspace) } fmt.Fprintf(options.Stdout, "mcode preflight ok (%s)\n", version) return runtime diff --git a/apps/daemon/internal/agent/mcode/discovery_workspace.go b/apps/daemon/internal/agent/mcode/discovery_workspace.go index da4f768d8..aa3461192 100644 --- a/apps/daemon/internal/agent/mcode/discovery_workspace.go +++ b/apps/daemon/internal/agent/mcode/discovery_workspace.go @@ -10,7 +10,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent/binpath" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/localworkspace" - "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/paths" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" ) @@ -27,11 +26,6 @@ func discoverWorkspace(parent context.Context, options agent.DiscoveryOptions, r if binding == nil { return nil } - root, err := paths.Root() - if err != nil { - fail(err) - return nil - } node := os.Getenv("OAC_RUNTIME_MCODE_NODE") if node == "" { node = "node" @@ -56,7 +50,7 @@ func discoverWorkspace(parent context.Context, options agent.DiscoveryOptions, r fail(err) return nil } - c, err := ConfigureLocal(binary, node, os.Getenv("OAC_RUNTIME_MCODE_WORKSPACE_BRIDGE"), root, os.Getenv("OAC_RUNTIME_WORKSPACE"), binding.NetworkPolicy()) + c, err := ConfigureLocal(binary, node, os.Getenv("OAC_RUNTIME_MCODE_WORKSPACE_BRIDGE"), options.RuntimeRoot, os.Getenv("OAC_RUNTIME_WORKSPACE"), binding.NetworkPolicy()) if err == nil { err = CheckWorkspace(parent, c) } diff --git a/apps/daemon/internal/agent/mcode/environment_mcp_test.go b/apps/daemon/internal/agent/mcode/environment_mcp_test.go index 5ca8565f9..dae9ac41f 100644 --- a/apps/daemon/internal/agent/mcode/environment_mcp_test.go +++ b/apps/daemon/internal/agent/mcode/environment_mcp_test.go @@ -32,7 +32,7 @@ func TestEnvironmentMCPUsesFixedLauncherForNewAndLoadedSessions(t *testing.T) { } t.Setenv("USER_SELECTED", "must-not-resolve-from-daemon") t.Setenv("MODEL_SECRET", "must-not-forward") - resource, err := NewExecutorFactory(&c)(t.Context(), req) + resource, err := NewExecutorFactory(testStateRoot(t), &c)(t.Context(), req) if err != nil { t.Fatal(err) } @@ -98,7 +98,7 @@ func TestEnvironmentMCPRejectsUnqualifiedAuthorityBeforePreparation(t *testing.T case "reserved": req.LocalEnvironment.MCP[0].Server.Name = "oac_workspace" } - if _, err := prepareWorkspaceOptions(c, req); err == nil || strings.Contains(err.Error(), "confidential-http-token") { + if _, err := prepareWorkspaceOptions(testStateRoot(t), c, req); err == nil || strings.Contains(err.Error(), "confidential-http-token") { t.Fatal("unqualified declaration accepted or credential exposed") } }) @@ -116,7 +116,7 @@ func TestEnvironmentMCPCancelSettlesPendingObservationBeforeDone(t *testing.T) { if err := os.WriteFile(c.Binary, []byte(strings.Replace(string(script), "HELPER=prepared", "HELPER=prepared-mcp-cancel", 1)), 0700); err != nil { t.Fatal(err) } - resource, err := NewExecutorFactory(&c)(t.Context(), req) + resource, err := NewExecutorFactory(testStateRoot(t), &c)(t.Context(), req) if err != nil { t.Fatal(err) } @@ -190,7 +190,7 @@ func TestEnvironmentHTTPMCPUsesEphemeralACPConfiguration(t *testing.T) { item.BearerToken = &value } req.LocalEnvironment.MCP = []proto.EnvironmentMCP{item} - resource, err := NewExecutorFactory(&c)(t.Context(), req) + resource, err := NewExecutorFactory(testStateRoot(t), &c)(t.Context(), req) if err != nil { t.Fatal(err) } @@ -237,7 +237,7 @@ func TestPublicEnvironmentHTTPMCPKeepsCredentialTransient(t *testing.T) { c.Network, req.LocalEnvironment.NetworkAccess = "enabled", "enabled" token := "selected-public-vault-canary" req.MCPHTTPServers = &[]proto.MCPHTTPServer{{ConnectionOrigin: "environment", ServerLabel: "remote", ServerURL: "https://example.test/mcp", BearerToken: &token}} - opts, err := prepareWorkspaceOptions(c, req) + opts, err := prepareWorkspaceOptions(testStateRoot(t), c, req) if err != nil { t.Fatal(err) } @@ -266,12 +266,12 @@ func TestPublicEnvironmentHTTPMCPKeepsCredentialTransient(t *testing.T) { } empty := []string{} (*req.MCPHTTPServers)[0].AllowedTools = &empty - if _, err := prepareWorkspaceOptions(c, req); err == nil { + if _, err := prepareWorkspaceOptions(testStateRoot(t), c, req); err == nil { t.Fatal("empty allowlist silently treated as all") } (*req.MCPHTTPServers)[0].AllowedTools = nil (*req.MCPHTTPServers)[0].Required = true - if _, err := prepareWorkspaceOptions(c, req); err == nil { + if _, err := prepareWorkspaceOptions(testStateRoot(t), c, req); err == nil { t.Fatal("required initialization silently ignored") } } diff --git a/apps/daemon/internal/agent/mcode/execution_test.go b/apps/daemon/internal/agent/mcode/execution_test.go index 9f188f97f..afa6b0ad1 100644 --- a/apps/daemon/internal/agent/mcode/execution_test.go +++ b/apps/daemon/internal/agent/mcode/execution_test.go @@ -14,7 +14,7 @@ func TestExecutionOptionsInheritUserEnvironment(t *testing.T) { r := testRequest(t) t.Setenv("OAC_TEST_SECRET_CANARY", "secret") t.Setenv("NODE_OPTIONS", "--import=untrusted") - opts, err := prepareOptions(r) + opts, err := prepareOptions(testStateRoot(t), r) if err != nil { t.Fatal(err) } @@ -48,7 +48,7 @@ func TestExecutionRejectsUnqualifiedAuthority(t *testing.T) { } { r := testRequest(t) change(&r) - if _, err := prepareOptions(r); err == nil { + if _, err := prepareOptions(testStateRoot(t), r); err == nil { t.Fatal("unsupported execution accepted") } } diff --git a/apps/daemon/internal/agent/mcode/executor.go b/apps/daemon/internal/agent/mcode/executor.go index 33ea18ed9..0ea08bf3b 100644 --- a/apps/daemon/internal/agent/mcode/executor.go +++ b/apps/daemon/internal/agent/mcode/executor.go @@ -28,7 +28,7 @@ var _ agent.Executor = (*executor)(nil) var _ agent.Turn = (*Session)(nil) // NewExecutorFactory fixes the deployment workspace once; nil selects none. -func NewExecutorFactory(config *WorkspaceConfig) agent.ExecutorFactory { +func NewExecutorFactory(runtimeRoot string, config *WorkspaceConfig) agent.ExecutorFactory { var frozen *WorkspaceConfig if config != nil { value := *config @@ -46,10 +46,10 @@ func NewExecutorFactory(config *WorkspaceConfig) agent.ExecutorFactory { var opts launchOptions binary := defaultBinary() if frozen == nil { - opts, err = prepareOptions(req) + opts, err = prepareOptions(runtimeRoot, req) } else { binary = frozen.Binary - opts, err = prepareWorkspaceOptions(*frozen, req) + opts, err = prepareWorkspaceOptions(runtimeRoot, *frozen, req) } if err != nil { return nil, err diff --git a/apps/daemon/internal/agent/mcode/executor_native_test.go b/apps/daemon/internal/agent/mcode/executor_native_test.go index d2c40d507..7f5b6afc5 100644 --- a/apps/daemon/internal/agent/mcode/executor_native_test.go +++ b/apps/daemon/internal/agent/mcode/executor_native_test.go @@ -33,7 +33,7 @@ func TestNativeMCodeExecutorReuse(t *testing.T) { if json.Unmarshal(raw, &req.AgentOptions) != nil { t.Fatal("invalid private provider options") } - value, err := NewExecutorFactory(nil)(ctx, req) + value, err := NewExecutorFactory(testStateRoot(t), nil)(ctx, req) if err != nil { t.Fatal("native Executor preparation failed") } @@ -153,7 +153,7 @@ func TestNativeMCodeExecutorReuse(t *testing.T) { } cleanupStop() req.AgentSessionID = nativeID - recovered, recoverErr := NewExecutorFactory(nil)(ctx, req) + recovered, recoverErr := NewExecutorFactory(testStateRoot(t), nil)(ctx, req) if recoverErr != nil || recovered == nil { t.Fatal("exact native history recovery failed") } diff --git a/apps/daemon/internal/agent/mcode/executor_test.go b/apps/daemon/internal/agent/mcode/executor_test.go index 59a1f5d60..d9936a712 100644 --- a/apps/daemon/internal/agent/mcode/executor_test.go +++ b/apps/daemon/internal/agent/mcode/executor_test.go @@ -45,12 +45,12 @@ func executorFixture(t *testing.T, scenario string, workspace bool) (*executor, } var factory agent.ExecutorFactory if workspace { - factory = NewExecutorFactory(&config) + factory = NewExecutorFactory(testStateRoot(t), &config) } else { req = testRequest(t) req.RunID, req.Input = "", nil t.Setenv("OAC_RUNTIME_MCODE_BIN", config.Binary) - factory = NewExecutorFactory(nil) + factory = NewExecutorFactory(testStateRoot(t), nil) } value, err := factory(t.Context(), req) if err != nil { @@ -293,7 +293,7 @@ func TestExecutorFactoryPreparationFailureHasNoTypedNilOwner(t *testing.T) { if err := os.WriteFile(config.Binary, []byte("#!/bin/sh\nexit 1\n"), 0700); err != nil { t.Fatal(err) } - value, err := NewExecutorFactory(&config)(t.Context(), req) + value, err := NewExecutorFactory(testStateRoot(t), &config)(t.Context(), req) if err == nil || value != nil { t.Fatalf("settled preparation failure returned owner: nil=%t error=%v", value == nil, err) } diff --git a/apps/daemon/internal/agent/mcode/harness_config_test.go b/apps/daemon/internal/agent/mcode/harness_config_test.go index a704912f9..1de262c5d 100644 --- a/apps/daemon/internal/agent/mcode/harness_config_test.go +++ b/apps/daemon/internal/agent/mcode/harness_config_test.go @@ -8,11 +8,11 @@ import ( func TestNativeConfigDoesNotPretendToSupportParameters(t *testing.T) { req := testRequest(t) req.AgentOptions["harness_config"] = map[string]any{} - if _, err := prepareOptions(req); err != nil { + if _, err := prepareOptions(testStateRoot(t), req); err != nil { t.Fatal(err) } req.AgentOptions["harness_config"] = map[string]any{"temperature": 0.5} - if _, err := prepareOptions(req); err != harnessconfig.ErrHarnessConfig { + if _, err := prepareOptions(testStateRoot(t), req); err != harnessconfig.ErrHarnessConfig { t.Fatalf("unsupported parameter accepted: %v", err) } } diff --git a/apps/daemon/internal/agent/mcode/model_provider_test.go b/apps/daemon/internal/agent/mcode/model_provider_test.go index f97b50508..bdef5fa6c 100644 --- a/apps/daemon/internal/agent/mcode/model_provider_test.go +++ b/apps/daemon/internal/agent/mcode/model_provider_test.go @@ -16,7 +16,7 @@ func TestOptionsModelProviderProtocols(t *testing.T) { req := testRequest(t) req.AgentOptions["model_provider"].(map[string]any)["protocol"] = tc.protocol req.AgentOptions["model"] = "chosen-model" - opts, err := prepareOptions(req) + opts, err := prepareOptions(testStateRoot(t), req) if err != nil { t.Fatal(err) } @@ -60,7 +60,7 @@ func TestOptionsRejectInvalidProvider(t *testing.T) { t.Run(tc.name, func(t *testing.T) { req := testRequest(t) req.AgentOptions["model_provider"].(map[string]any)[tc.field] = tc.value - if _, err := prepareOptions(req); err == nil { + if _, err := prepareOptions(testStateRoot(t), req); err == nil { t.Fatal("invalid model provider accepted") } }) @@ -69,7 +69,7 @@ func TestOptionsRejectInvalidProvider(t *testing.T) { req := testRequest(t) req.AgentOptions["mcode_provider"] = req.AgentOptions["model_provider"] delete(req.AgentOptions, "model_provider") - if _, err := prepareOptions(req); err == nil { + if _, err := prepareOptions(testStateRoot(t), req); err == nil { t.Fatal("retired native provider option accepted") } }) @@ -78,7 +78,7 @@ func TestOptionsRejectInvalidProvider(t *testing.T) { func TestOptionsAllowLoopbackProviderFixture(t *testing.T) { req := testRequest(t) req.AgentOptions["model_provider"].(map[string]any)["base_url"] = "http://127.0.0.1:4321/v1" - if _, err := prepareOptions(req); err != nil { + if _, err := prepareOptions(testStateRoot(t), req); err != nil { t.Fatal(err) } } diff --git a/apps/daemon/internal/agent/mcode/options.go b/apps/daemon/internal/agent/mcode/options.go index 6fc7e3492..f34f57b53 100644 --- a/apps/daemon/internal/agent/mcode/options.go +++ b/apps/daemon/internal/agent/mcode/options.go @@ -18,7 +18,7 @@ type launchOptions struct { MCP []map[string]any } -func prepareOptions(req proto.PromptRequestPayload) (launchOptions, error) { +func prepareOptions(runtimeRoot string, req proto.PromptRequestPayload) (launchOptions, error) { var result launchOptions if _, err := harnessconfiguration.Configuration().PrepareHarnessConfig(req.AgentOptions); err != nil { return result, err @@ -26,7 +26,7 @@ func prepareOptions(req proto.PromptRequestPayload) (launchOptions, error) { if err := validateExecutionRequest(req); err != nil { return result, err } - dataDir, err := agent.StateDir("mcode", req.AgentStateKey) + dataDir, err := agent.StateDir(runtimeRoot, "mcode", req.AgentStateKey) if err != nil { return result, err } diff --git a/apps/daemon/internal/agent/mcode/options_test.go b/apps/daemon/internal/agent/mcode/options_test.go index 70d736c86..5018a743f 100644 --- a/apps/daemon/internal/agent/mcode/options_test.go +++ b/apps/daemon/internal/agent/mcode/options_test.go @@ -13,7 +13,7 @@ import ( func TestOptionsRefreshManagedState(t *testing.T) { t.Setenv("MINIMAX_DATA_DIR", "/wrong") req := testRequest(t) - opts, err := prepareOptions(req) + opts, err := prepareOptions(testStateRoot(t), req) if err != nil { t.Fatal(err) } @@ -31,7 +31,7 @@ func TestOptionsRefreshManagedState(t *testing.T) { } req.AgentOptions["system_prompt"] = "" req.AgentSessionID = "native-1" - refreshed, err := prepareOptions(req) + refreshed, err := prepareOptions(testStateRoot(t), req) if err != nil { t.Fatal(err) } @@ -75,7 +75,7 @@ func TestOptionsRejectDroppedContext(t *testing.T) { t.Run(tt.name, func(t *testing.T) { req := testRequest(t) tt.edit(&req) - if _, err := prepareOptions(req); err == nil { + if _, err := prepareOptions(testStateRoot(t), req); err == nil { t.Fatal("expected validation failure") } }) diff --git a/apps/daemon/internal/agent/mcode/session_test.go b/apps/daemon/internal/agent/mcode/session_test.go index 0dfc9e070..044a1bb35 100644 --- a/apps/daemon/internal/agent/mcode/session_test.go +++ b/apps/daemon/internal/agent/mcode/session_test.go @@ -47,7 +47,7 @@ func helperRequest(t *testing.T, scenario string, resume bool) proto.PromptReque func prepareExecutor(t *testing.T, ctx context.Context, req proto.PromptRequestPayload) (*executor, error) { t.Helper() req.RunID, req.Input = "", nil - value, err := NewExecutorFactory(nil)(ctx, req) + value, err := NewExecutorFactory(testStateRoot(t), nil)(ctx, req) if value == nil { return nil, err } diff --git a/apps/daemon/internal/agent/mcode/state_root_test.go b/apps/daemon/internal/agent/mcode/state_root_test.go new file mode 100644 index 000000000..6c7919cc5 --- /dev/null +++ b/apps/daemon/internal/agent/mcode/state_root_test.go @@ -0,0 +1,16 @@ +package mcode + +import ( + "os" + "testing" +) + +func testStateRoot(t *testing.T) string { + t.Helper() + if root := os.Getenv("OAC_RUNTIME_HOME"); root != "" { + return root + } + root := t.TempDir() + t.Setenv("OAC_RUNTIME_HOME", root) + return root +} diff --git a/apps/daemon/internal/agent/mcode/tool_environment_test.go b/apps/daemon/internal/agent/mcode/tool_environment_test.go index ab6accd5a..fd047583a 100644 --- a/apps/daemon/internal/agent/mcode/tool_environment_test.go +++ b/apps/daemon/internal/agent/mcode/tool_environment_test.go @@ -60,7 +60,7 @@ func TestWorkspaceCredentialsRemainInRuntimeSnapshotAcrossReconnect(t *testing.T if err != nil { t.Fatal(err) } - opts, err := prepareWorkspaceOptions(config, prepared) + opts, err := prepareWorkspaceOptions(testStateRoot(t), config, prepared) if err != nil { t.Fatal(err) } @@ -125,7 +125,7 @@ func TestWorkspaceRejectsInvalidToolEnvironment(t *testing.T) { } t.Setenv("OAC_RUNTIME_TOOL_ENV_FILE", file) t.Setenv("OAC_RUNTIME_INITIALIZATION_DIRECTORY", t.TempDir()) - if _, err := prepareWorkspaceOptions(config, req); err == nil { + if _, err := prepareWorkspaceOptions(testStateRoot(t), config, req); err == nil { t.Fatal("invalid explicit tool configuration was ignored") } } diff --git a/apps/daemon/internal/agent/mcode/workspace.go b/apps/daemon/internal/agent/mcode/workspace.go index 5b913e4ac..b5bbd2899 100644 --- a/apps/daemon/internal/agent/mcode/workspace.go +++ b/apps/daemon/internal/agent/mcode/workspace.go @@ -49,7 +49,7 @@ func ConfigureLocal(binary, node, bridge, root, workspace string, network agentn return c, nil } -func prepareWorkspaceOptions(c WorkspaceConfig, req proto.PromptRequestPayload) (launchOptions, error) { +func prepareWorkspaceOptions(runtimeRoot string, c WorkspaceConfig, req proto.PromptRequestPayload) (launchOptions, error) { if c.Network != "enabled" || len(c.AllowedDomains) != 0 { return launchOptions{}, fmt.Errorf("mcode: Runtime does not implement network isolation") } @@ -66,7 +66,7 @@ func prepareWorkspaceOptions(c WorkspaceConfig, req proto.PromptRequestPayload) private.LocalEnvironment, private.DisableExecutionEnvironment = nil, true // Public declarations have already been resolved into the transient ACP map. private.MCPHTTPServers = nil - opts, err := prepareOptions(private) + opts, err := prepareOptions(runtimeRoot, private) if err != nil { return opts, err } diff --git a/apps/daemon/internal/agent/mcode/workspace_network_test.go b/apps/daemon/internal/agent/mcode/workspace_network_test.go index 83b3156ee..57aed9044 100644 --- a/apps/daemon/internal/agent/mcode/workspace_network_test.go +++ b/apps/daemon/internal/agent/mcode/workspace_network_test.go @@ -7,7 +7,7 @@ func TestWorkspaceRejectsInnerNetworkIsolation(t *testing.T) { c, req, _ := workspaceFixture(t) c.Network = access req.LocalEnvironment.NetworkAccess = access - if _, err := prepareWorkspaceOptions(c, req); err == nil { + if _, err := prepareWorkspaceOptions(testStateRoot(t), c, req); err == nil { t.Fatal("unsupported network isolation accepted") } } diff --git a/apps/daemon/internal/agent/mcode/workspace_skills_test.go b/apps/daemon/internal/agent/mcode/workspace_skills_test.go index 0d7c9470d..1126eaac9 100644 --- a/apps/daemon/internal/agent/mcode/workspace_skills_test.go +++ b/apps/daemon/internal/agent/mcode/workspace_skills_test.go @@ -28,7 +28,7 @@ func TestWorkspaceSkillsUseSelectedSnapshotAndNativeLoader(t *testing.T) { Metadata: agentskill.Metadata{Name: "proof", Description: "Read a marker"}, }} for range 2 { - opts, err := prepareWorkspaceOptions(c, req) + opts, err := prepareWorkspaceOptions(testStateRoot(t), c, req) if err != nil { t.Fatal(err) } diff --git a/apps/daemon/internal/agent/runtime_paths.go b/apps/daemon/internal/agent/runtime_paths.go index 456231282..ce61e9a89 100644 --- a/apps/daemon/internal/agent/runtime_paths.go +++ b/apps/daemon/internal/agent/runtime_paths.go @@ -4,16 +4,13 @@ import ( "fmt" "path/filepath" "strings" - - "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/paths" ) // StateDir returns an adapter-owned state directory scoped to one agent // state. It never derives runtime state from the subprocess cwd. -func StateDir(agentKind, agentStateKey string) (string, error) { - root, err := paths.Root() - if err != nil { - return "", fmt.Errorf("agent: resolve state directory: %w", err) +func StateDir(root, agentKind, agentStateKey string) (string, error) { + if !filepath.IsAbs(root) || filepath.Clean(root) != root { + return "", fmt.Errorf("agent: state root must be a clean absolute directory") } kind := safeRuntimePathPart(agentKind) if kind == "" { diff --git a/apps/daemon/internal/agent/runtime_paths_test.go b/apps/daemon/internal/agent/runtime_paths_test.go index 3e98c1792..7abd9ebd8 100644 --- a/apps/daemon/internal/agent/runtime_paths_test.go +++ b/apps/daemon/internal/agent/runtime_paths_test.go @@ -8,7 +8,7 @@ import ( func TestStateDirUsesStableAgentState(t *testing.T) { home := t.TempDir() t.Setenv("OAC_RUNTIME_HOME", home) - got, err := StateDir("codex", "conv-1/agent-1/codex") + got, err := StateDir(home, "codex", "conv-1/agent-1/codex") if err != nil { t.Fatalf("StateDir: %v", err) } @@ -22,11 +22,11 @@ func TestStateDirRequiresAgentState(t *testing.T) { home := t.TempDir() t.Setenv("OAC_RUNTIME_HOME", home) for _, key := range []string{"", " ", "../.."} { - if _, err := StateDir("codex", key); err == nil { + if _, err := StateDir(home, "codex", key); err == nil { t.Fatalf("state key %q accepted", key) } } - got, err := StateDir("codex", "../session-1/a b") + got, err := StateDir(home, "codex", "../session-1/a b") if want := filepath.Join(home, "runtime", "codex", "state", "session-1", "a_b"); err != nil || got != want { t.Fatalf("sanitized state = %q, %v; want %q", got, err, want) } diff --git a/apps/daemon/internal/cli/agent_discovery.go b/apps/daemon/internal/cli/agent_discovery.go index d29b85acc..208437601 100644 --- a/apps/daemon/internal/cli/agent_discovery.go +++ b/apps/daemon/internal/cli/agent_discovery.go @@ -7,6 +7,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent/claudesdk" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent/codex" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent/mcode" + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/paths" ) var harnessDeclarations = []agent.Declaration{codex.Declaration, mcode.Declaration, claudesdk.Declaration} @@ -21,13 +22,21 @@ func preflightAgentCLIs(parent context.Context, rc *runContext, profile string) return discoverAgentCLIs(parent, rc, profile, harnessDeclarations) } func discoverAgentCLIs(parent context.Context, rc *runContext, profile string, declarations []agent.Declaration) (agentCLIDiscovery, error) { + runtimeRoot, err := paths.Root() + if err != nil { + return nil, err + } + stateRoot, err := paths.StateDirectory() + if err != nil { + return nil, err + } var out agentCLIDiscovery available := false for _, declaration := range declarations { if rc.installedKinds != nil && !rc.installedKinds[declaration.Info.Kind] { continue } - runtime := declaration.Discover(parent, agent.DiscoveryOptions{Profile: profile, Stdout: rc.stdout, Stderr: rc.stderr}, declaration.Info) + runtime := declaration.Discover(parent, agent.DiscoveryOptions{Profile: profile, StateRoot: stateRoot, RuntimeRoot: runtimeRoot, Stdout: rc.stdout, Stderr: rc.stderr}, declaration.Info) if runtime == nil { continue } diff --git a/apps/daemon/internal/cli/connect_suspend.go b/apps/daemon/internal/cli/connect_suspend.go index b1146b94d..3d3228ff1 100644 --- a/apps/daemon/internal/cli/connect_suspend.go +++ b/apps/daemon/internal/cli/connect_suspend.go @@ -201,7 +201,7 @@ func (s *suspendedRouter) pump(ctx context.Context, conn *transport.Conn, boot * if sendErr := sendSuspendResult(ctx, conn, env, proto.TypeEnvironmentQuiesced, request, "resource_busy"); sendErr != nil { return nil, sendErr } - if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + if errors.Is(err, dispatch.ErrRouterQuiesced) || errors.Is(err, dispatch.ErrRouterClosed) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { return nil, err } continue diff --git a/apps/daemon/internal/cli/connect_suspend_test.go b/apps/daemon/internal/cli/connect_suspend_test.go index 4c7177746..52dd2caea 100644 --- a/apps/daemon/internal/cli/connect_suspend_test.go +++ b/apps/daemon/internal/cli/connect_suspend_test.go @@ -17,7 +17,11 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/transport" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentcapabilities" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" + "github.com/MiniMax-AI/OpenAgentCore/internal/harnessconfig" + "github.com/google/uuid" "github.com/gorilla/websocket" ) @@ -334,3 +338,131 @@ func TestSuspensionReconnectBeforeConfirmation(t *testing.T) { }) } } + +func TestQuiesceCloseFailureDisconnectsAndSettlesBeforeReconnect(t *testing.T) { + environment, session := uuid.NewString(), uuid.NewString() + t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) + t.Setenv("OAC_RUNTIME_STATE_DIRECTORY", t.TempDir()) + t.Setenv("OAC_RUNTIME_WORKSPACE", t.TempDir()) + t.Setenv("OAC_RUNTIME_CAPABILITY_DIRECTORY", t.TempDir()) + t.Setenv("OAC_RUNTIME_ENVIRONMENT_ID", environment) + t.Setenv("OAC_RUNTIME_SESSION_ID", session) + t.Setenv("OAC_RUNTIME_NETWORK_ACCESS", "disabled") + t.Setenv("OAC_RUNTIME_ALLOWED_DOMAINS", "") + peers := make(chan *websocket.Conn, 2) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + peer, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err == nil { + peers <- peer + } + })) + defer server.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + dial := func(ctx context.Context) (*transport.Conn, error) { + return transport.Dial(ctx, transport.DialOptions{WSURL: "ws" + strings.TrimPrefix(server.URL, "http"), DeviceID: "device", Credential: "fixture", DaemonVersion: proto.Version}) + } + native := &cleanupExecutor{retry: make(chan struct{}), confirm: make(chan struct{})} + defer func() { + select { + case <-native.confirm: + default: + close(native.confirm) + } + }() + registry := agent.NewRegistry() + registry.RegisterKind(proto.SupportedAgentKind{Kind: "cleanup", Available: true, Capabilities: prototest.Capabilities(proto.AgentKindCapabilities{LocalEnvironment: proto.CapabilitySupported})}, harnessconfig.Configuration{}) + registry.RegisterExecutor("cleanup", func(context.Context, proto.PromptRequestPayload) (agent.Executor, error) { return native, nil }) + control := &suspendControl{path: filepath.Join(t.TempDir(), "control.json"), identity: suspendIdentity{EnvironmentID: environment}, signal: make(chan os.Signal, 1)} + finished := make(chan error, 1) + go func() { + finished <- runSuspendLoop(ctx, dial, registry, &transport.BootstrapResponse{HeartbeatSeconds: 60}, agentCLIDiscovery{}, control) + }() + var peer *websocket.Conn + select { + case peer = <-peers: + case <-time.After(3 * time.Second): + t.Fatal("no connection") + } + defer peer.Close() + preparation, err := proto.NewEnvelope(proto.TypeExecutionPrepare, "prepare", proto.ExecutionPreparePayload{SessionID: session, Configuration: proto.PromptRequestPayload{AgentKind: "cleanup", AgentStateKey: "agents-api-" + session, LocalEnvironment: &proto.LocalEnvironment{ID: environment, NetworkAccess: "disabled", WorkspaceDirectory: "/workspace", CapabilitySources: &agentcapabilities.Input{}}}}) + if err != nil { + t.Fatal(err) + } + if err = peer.WriteJSON(preparation); err != nil { + t.Fatal(err) + } + readStatus := func(want string) proto.PreparationStatusPayload { + t.Helper() + _ = peer.SetReadDeadline(time.Now().Add(3 * time.Second)) + for { + var frame proto.Envelope + if err := peer.ReadJSON(&frame); err != nil { + t.Fatal(err) + } + if frame.Type != proto.TypePreparationStatus { + continue + } + var status proto.PreparationStatusPayload + if err := frame.DecodePayload(&status); err != nil { + t.Fatal(err) + } + if status.State == want { + return status + } + if status.State == "failed" || status.State == "rejected" { + t.Fatalf("preparation failed: %+v", status) + } + } + } + ready := readStatus("ready") + request := proto.EnvironmentSuspendPayload{EnvironmentID: environment, SuspendID: "attempt"} + // Admission is still held: ordinary busy must leave the socket usable. + sendLifecycleFrame(t, peer, proto.TypeEnvironmentQuiesce, request) + if got := readLifecycleResult(t, peer, proto.TypeEnvironmentQuiesced); got.Accepted || got.ErrorCode != "resource_busy" { + t.Fatalf("busy result: %+v", got) + } + release, err := proto.NewEnvelope(proto.TypeExecutionRelease, "prepare", proto.ExecutionReleasePayload{Handle: ready.Handle}) + if err != nil { + t.Fatal(err) + } + if err = peer.WriteJSON(release); err != nil { + t.Fatal("busy rejection disconnected", err) + } + readStatus("released") + sendLifecycleFrame(t, peer, proto.TypeEnvironmentQuiesce, request) + if got := readLifecycleResult(t, peer, proto.TypeEnvironmentQuiesced); got.Accepted || got.ErrorCode != "resource_busy" { + t.Fatalf("failed Close acknowledged: %+v", got) + } + select { + case <-native.retry: + case <-time.After(3 * time.Second): + t.Fatal("fenced close failure did not enter Shutdown retry") + } + _ = peer.SetReadDeadline(time.Now().Add(time.Second)) + for { + var frame proto.Envelope + if peer.ReadJSON(&frame) != nil { + break + } + } + select { + case other := <-peers: + _ = other.Close() + t.Fatal("reconnected before native cleanup settled") + default: + } + cancel() + close(native.confirm) + select { + case <-finished: + case <-time.After(3 * time.Second): + t.Fatal("confirmed cleanup failed to settle loop") + } + if native.closes.Load() != 2 { + t.Fatalf("Close calls=%d", native.closes.Load()) + } + if _, err := os.Stat(control.path); !os.IsNotExist(err) { + t.Fatal("failed quiesce armed snapshot control") + } +} diff --git a/apps/daemon/internal/cli/native_discovery_test.go b/apps/daemon/internal/cli/native_discovery_test.go index 5f3592c9d..fb230b5c3 100644 --- a/apps/daemon/internal/cli/native_discovery_test.go +++ b/apps/daemon/internal/cli/native_discovery_test.go @@ -14,10 +14,16 @@ import ( func TestDiscoveryAndRegistration(t *testing.T) { for _, selected := range []string{"codex", "mcode", "claude_sdk", ""} { t.Run(selected, func(t *testing.T) { + runtimeRoot, stateRoot := t.TempDir(), t.TempDir() + t.Setenv("OAC_RUNTIME_HOME", runtimeRoot) + t.Setenv("OAC_RUNTIME_STATE_DIRECTORY", stateRoot) called := []string{} declarations := append([]agent.Declaration(nil), harnessDeclarations...) for i := range declarations { - declarations[i].Discover = func(_ context.Context, _ agent.DiscoveryOptions, info proto.SupportedAgentKind) *agent.Runtime { + declarations[i].Discover = func(_ context.Context, options agent.DiscoveryOptions, info proto.SupportedAgentKind) *agent.Runtime { + if options.RuntimeRoot != runtimeRoot || options.StateRoot != stateRoot { + t.Fatal("discovery lost explicit directory context") + } called = append(called, info.Kind) info.Available = true info.Version = "test" diff --git a/apps/daemon/internal/dispatch/functions_native_test.go b/apps/daemon/internal/dispatch/functions_native_test.go index f8fe3879b..b022d94dd 100644 --- a/apps/daemon/internal/dispatch/functions_native_test.go +++ b/apps/daemon/internal/dispatch/functions_native_test.go @@ -108,7 +108,7 @@ func TestNativeFunctionBridge(t *testing.T) { })) defer model.Close() reg := agent.NewRegistry() - registerExecutorKind(reg, proto.SupportedAgentKind{Kind: "codex", Available: true, Capabilities: prototest.Capabilities(proto.AgentKindCapabilities{FunctionTools: proto.CapabilitySupported, EnvironmentNone: proto.CapabilitySupported})}, codex.NewExecutorFactory()) + registerExecutorKind(reg, proto.SupportedAgentKind{Kind: "codex", Available: true, Capabilities: prototest.Capabilities(proto.AgentKindCapabilities{FunctionTools: proto.CapabilitySupported, EnvironmentNone: proto.CapabilitySupported})}, codex.NewExecutorFactory(t.TempDir())) sender := make(nativeFunctionSender, 256) ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) defer cancel() diff --git a/apps/daemon/internal/dispatch/suspend.go b/apps/daemon/internal/dispatch/suspend.go index 97a8cc6fc..5945c7cdc 100644 --- a/apps/daemon/internal/dispatch/suspend.go +++ b/apps/daemon/internal/dispatch/suspend.go @@ -3,17 +3,20 @@ package dispatch import ( "context" "errors" + "fmt" "strings" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" ) +// ErrRouterQuiesced also wraps failed quiesce attempts that have fenced admission. var ErrRouterQuiesced = errors.New("dispatch: router quiesced") var ErrRouterBusy = errors.New("dispatch: router has unsettled work") // Quiesce serializes against admission, then drains every admitted output and -// receipt before acknowledging suspension. Busy rejection leaves admission open; -// a drain timeout keeps it closed until the caller shuts the connection down. +// receipt and closes idle Executors before acknowledging suspension. Busy +// rejection leaves admission open; a failed close or drain timeout keeps it +// closed until the caller shuts the connection down. func (r *Router) Quiesce(ctx context.Context, request proto.EnvironmentSuspendPayload) error { if strings.TrimSpace(request.EnvironmentID) == "" || strings.TrimSpace(request.SuspendID) == "" || len(request.SuspendID) > 128 { return errors.New("dispatch: invalid suspension identity") @@ -53,21 +56,32 @@ func (r *Router) Quiesce(ctx context.Context, request proto.EnvironmentSuspendPa p.timer.Stop() } } - for _, owner := range r.executors { + owners := r.closeIdleExecutorsLocked() + for _, owner := range owners { owner.idleLease++ - if owner.timer != nil { - owner.timer.Stop() - } + owner.closeReason = "suspend" } r.mu.Unlock() + r.closeIdleExecutors(owners) err := r.shutdownWG.waitContext(ctx) r.mu.Lock() if r.closed { err = ErrRouterClosed + } else if err == nil { + for _, owner := range r.executors { + cause := owner.closeErr + if cause == nil { + cause = errors.New("cleanup has not settled") + } + err = errors.Join(err, fmt.Errorf("dispatch: executor %s: %w", owner.id, cause)) + } } r.mu.Unlock() - return err + if err != nil { + return errors.Join(ErrRouterQuiesced, err) + } + return nil } // Resume opens admission only after the caller authenticated a new connection diff --git a/apps/daemon/internal/dispatch/suspend_test.go b/apps/daemon/internal/dispatch/suspend_test.go index 054a031ed..ddb48f030 100644 --- a/apps/daemon/internal/dispatch/suspend_test.go +++ b/apps/daemon/internal/dispatch/suspend_test.go @@ -15,13 +15,22 @@ type suspendSender func(context.Context, proto.Envelope) error func (s suspendSender) Send(ctx context.Context, env proto.Envelope) error { return s(ctx, env) } -type suspendedExecutor struct{ closed atomic.Int32 } +type suspendedExecutor struct { + closed atomic.Int32 + close func(context.Context) error +} func (e *suspendedExecutor) StartTurn(context.Context, string, proto.MessageInput, chan<- proto.Envelope) (agent.Turn, error) { return nil, errors.New("suspended executor must not start a Turn") } -func (e *suspendedExecutor) Close(context.Context) error { e.closed.Add(1); return nil } +func (e *suspendedExecutor) Close(ctx context.Context) error { + e.closed.Add(1) + if e.close != nil { + return e.close(ctx) + } + return nil +} func suspensionRouter(t *testing.T, sender Sender) *Router { t.Helper() @@ -39,12 +48,19 @@ func suspensionRouter(t *testing.T, sender Sender) *Router { func TestQuiesceRejectsEveryUnsettledResource(t *testing.T) { cases := map[string]func(*Router){ - "active": func(r *Router) { r.sessions["run"] = &sessionState{} }, - "preparing": func(r *Router) { r.preparations["p"] = &preparationState{owns: true} }, - "receipt": func(r *Router) { r.preparations["p"] = &preparationState{busy: true} }, - "read": func(r *Router) { r.workspaceReads = map[string]struct{}{"read": {}} }, - "write": func(r *Router) { r.workspaceWrite = &workspaceUpload{} }, - "export": func(r *Router) { r.workspaceExport = &workspaceExport{} }, + "active": func(r *Router) { r.sessions["run"] = &sessionState{} }, + "preparing": func(r *Router) { r.preparations["p"] = &preparationState{owns: true} }, + "receipt": func(r *Router) { r.preparations["p"] = &preparationState{busy: true} }, + "read": func(r *Router) { r.workspaceReads = map[string]struct{}{"read": {}} }, + "write": func(r *Router) { r.workspaceWrite = &workspaceUpload{} }, + "export": func(r *Router) { r.workspaceExport = &workspaceExport{} }, + "closing executor": func(r *Router) { r.executors["s"] = &executorState{invalid: true, environmentID: "env"} }, + "preparing executor": func(r *Router) { r.executors["s"] = &executorState{preparing: true, environmentID: "env"} }, + "active executor": func(r *Router) { r.executors["s"] = &executorState{run: &sessionState{}, environmentID: "env"} }, + "admitted executor": func(r *Router) { + r.executors["s"] = &executorState{admission: &preparationState{}, environmentID: "env"} + }, + "foreign executor": func(r *Router) { r.executors["s"] = &executorState{environmentID: "other"} }, } for name, setup := range cases { t.Run(name, func(t *testing.T) { @@ -60,6 +76,7 @@ func TestQuiesceRejectsEveryUnsettledResource(t *testing.T) { r.workspaceReads = nil r.workspaceWrite = nil r.workspaceExport = nil + r.executors = map[string]*executorState{} }) } } @@ -106,7 +123,7 @@ func TestQuiesceDrainsPendingReceiptAndFencesConcurrentAdmission(t *testing.T) { } } -func TestQuiescePreservesIdleExecutorAgainstExpiredTimerAndRequiresExactResume(t *testing.T) { +func TestQuiesceClosesIdleExecutorOnceAndRequiresExactResume(t *testing.T) { sender := suspendSender(func(context.Context, proto.Envelope) error { return nil }) r := suspensionRouter(t, sender) native := &suspendedExecutor{} @@ -121,8 +138,8 @@ func TestQuiescePreservesIdleExecutorAgainstExpiredTimerAndRequiresExactResume(t t.Fatal(err) } r.expireIdleExecutor(owner, oldLease) - if native.closed.Load() != 0 { - t.Fatal("pre-snapshot timer closed retained owner") + if native.closed.Load() != 1 { + t.Fatal("quiesce and stale timer did not close the owner exactly once") } wrong := request wrong.SuspendID = "obsolete" @@ -133,11 +150,10 @@ func TestQuiescePreservesIdleExecutorAgainstExpiredTimerAndRequiresExactResume(t t.Fatal(err) } r.mu.Lock() - newLease := owner.idleLease + remaining := len(r.executors) r.mu.Unlock() - r.expireIdleExecutor(owner, newLease) - if native.closed.Load() != 1 { - t.Fatal("normal idle expiration was not restored") + if remaining != 0 || native.closed.Load() != 1 { + t.Fatal("resume retained or reclosed a retired native owner") } } @@ -174,3 +190,121 @@ func TestQuiesceDrainDeadlineCannotReopenAdmission(t *testing.T) { t.Fatal(err) } } + +func TestQuiesceWaitsForIdleExecutorClose(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + native := &suspendedExecutor{close: func(context.Context) error { close(entered); <-release; return nil }} + r := suspensionRouter(t, suspendSender(func(context.Context, proto.Envelope) error { return nil })) + t.Cleanup(func() { + select { + case <-release: + default: + close(release) + } + }) + owner := &executorState{id: "executor", sessionID: "session", environmentID: "env", native: native, cancel: func() {}} + r.executors[owner.sessionID] = owner + quiet := make(chan error, 1) + go func() { + quiet <- r.Quiesce(t.Context(), proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}) + }() + <-entered + select { + case err := <-quiet: + t.Fatalf("acknowledged before native close: %v", err) + case <-time.After(20 * time.Millisecond): + } + close(release) + if err := <-quiet; err != nil { + t.Fatal(err) + } + if native.closed.Load() != 1 { + t.Fatal("native Close was duplicated") + } +} + +func TestQuiesceCloseFailureRetainsOwnerAndFencesAdmission(t *testing.T) { + failure := errors.New("native history flush failed") + native := &suspendedExecutor{close: func(context.Context) error { return failure }} + r := suspensionRouter(t, suspendSender(func(context.Context, proto.Envelope) error { return nil })) + owner := &executorState{id: "executor", sessionID: "session", environmentID: "env", native: native, cancel: func() {}} + r.executors[owner.sessionID] = owner + request := proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"} + if err := r.Quiesce(t.Context(), request); !errors.Is(err, failure) || !errors.Is(err, ErrRouterQuiesced) { + t.Fatalf("quiesce lost native close failure: %v", err) + } + r.mu.Lock() + retained := r.executors[owner.sessionID] == owner && owner.invalid && owner.closeErr == failure + r.mu.Unlock() + if !retained { + t.Fatal("failed close abandoned ownership") + } + if err := r.Handle(t.Context(), proto.Envelope{Type: proto.TypeExecutionPrepare, ID: "late"}); !errors.Is(err, ErrRouterQuiesced) { + t.Fatalf("failed close reopened admission: %v", err) + } + native.close = nil + if err := r.Shutdown(t.Context()); err != nil { + t.Fatal(err) + } + if native.closed.Load() != 2 { + t.Fatal("shutdown did not retry exactly the settled failure") + } +} + +func TestQuiesceDeadlineJoinsNativeCloseDuringShutdown(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + native := &suspendedExecutor{close: func(context.Context) error { close(entered); <-release; return nil }} + r := suspensionRouter(t, suspendSender(func(context.Context, proto.Envelope) error { return nil })) + t.Cleanup(func() { + select { + case <-release: + default: + close(release) + } + }) + owner := &executorState{id: "executor", sessionID: "session", environmentID: "env", native: native, cancel: func() {}} + r.executors[owner.sessionID] = owner + ctx, cancel := context.WithCancel(t.Context()) + quiet := make(chan error, 1) + go func() { + quiet <- r.Quiesce(ctx, proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}) + }() + <-entered + cancel() + if err := <-quiet; !errors.Is(err, context.Canceled) { + t.Fatalf("quiesce=%v", err) + } + if err := r.Handle(t.Context(), proto.Envelope{Type: proto.TypeExecutionPrepare, ID: "late"}); !errors.Is(err, ErrRouterQuiesced) { + t.Fatalf("deadline reopened admission: %v", err) + } + stopped := make(chan error, 1) + go func() { stopped <- r.Shutdown(t.Context()) }() + select { + case err := <-stopped: + t.Fatalf("shutdown abandoned in-flight close: %v", err) + case <-time.After(20 * time.Millisecond): + } + close(release) + if err := <-stopped; err != nil { + t.Fatal(err) + } + if native.closed.Load() != 1 { + t.Fatal("shutdown duplicated in-flight Close") + } +} + +func TestQuiesceRejectsAdmissionInProgress(t *testing.T) { + r := suspensionRouter(t, suspendSender(func(context.Context, proto.Envelope) error { return nil })) + r.admission.Lock() + err := r.Quiesce(t.Context(), proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}) + r.admission.Unlock() + if !errors.Is(err, ErrRouterBusy) { + t.Fatalf("quiesce=%v", err) + } + r.mu.Lock() + fenced := r.suspension != nil + r.mu.Unlock() + if fenced { + t.Fatal("busy rejection changed suspension") + } +} diff --git a/apps/daemon/internal/localworkspace/binding.go b/apps/daemon/internal/localworkspace/binding.go index 0ead8710a..9d24e3555 100644 --- a/apps/daemon/internal/localworkspace/binding.go +++ b/apps/daemon/internal/localworkspace/binding.go @@ -2,6 +2,7 @@ package localworkspace import ( "errors" + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/paths" "os" "strings" "sync" @@ -22,11 +23,21 @@ type Binding struct { writer *fileWriter capabilityMu sync.Mutex capabilityRoot string + stateRoot string } // NewWithCapabilityDirectory freezes paths selected by the Runtime operator. func NewWithCapabilityDirectory(environment, session, workspace, directory string) (*Binding, error) { - return newNativeBinding(environment, session, workspace, directory) + stateRoot, err := paths.StateDirectory() + if err != nil { + return nil, err + } + b, err := newNativeBinding(environment, session, workspace, directory) + if err != nil { + return nil, err + } + b.stateRoot = stateRoot + return b, nil } func Load() (*Binding, error) { diff --git a/apps/daemon/internal/localworkspace/snapshot_marker.go b/apps/daemon/internal/localworkspace/snapshot_marker.go index fb95e9de1..48ea5e49e 100644 --- a/apps/daemon/internal/localworkspace/snapshot_marker.go +++ b/apps/daemon/internal/localworkspace/snapshot_marker.go @@ -6,7 +6,6 @@ import ( "io" "os" - "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/paths" "github.com/MiniMax-AI/OpenAgentCore/internal/agentcapabilities" "github.com/MiniMax-AI/OpenAgentCore/internal/runtimefs" "github.com/google/uuid" @@ -30,8 +29,11 @@ func (b *Binding) openSnapshotMarker() (*snapshotMarker, error) { return nil, agentcapabilities.ErrInvalid } } - private, err := paths.Root() - if err != nil || agentcapabilities.ValidateLocalDirectories([]string{private}) != nil { + private := b.stateRoot + if agentcapabilities.ValidateLocalDirectories([]string{private}) != nil { + return nil, agentcapabilities.ErrInvalid + } + if err := os.MkdirAll(private, 0700); err != nil { return nil, agentcapabilities.ErrInvalid } current, err := os.OpenRoot(private) diff --git a/apps/daemon/internal/localworkspace/snapshot_marker_test.go b/apps/daemon/internal/localworkspace/snapshot_marker_test.go index b57484af7..7f610e721 100644 --- a/apps/daemon/internal/localworkspace/snapshot_marker_test.go +++ b/apps/daemon/internal/localworkspace/snapshot_marker_test.go @@ -28,7 +28,7 @@ func markerBinding(t *testing.T) (*Binding, proto.PromptRequestPayload) { } func markerPath(b *Binding) string { identity := b.capabilityIdentity() - return filepath.Join(os.Getenv("OAC_RUNTIME_HOME"), "daemon", "capability-installations", identity.EnvironmentID+"-"+identity.SessionID+".json") + return filepath.Join(b.stateRoot, "daemon", "capability-installations", identity.EnvironmentID+"-"+identity.SessionID+".json") } func TestSnapshotMarkerRoundTrip(t *testing.T) { b, _ := markerBinding(t) @@ -220,3 +220,34 @@ func TestSnapshotMarkerCancellationDoesNotPublishCompletion(t *testing.T) { t.Fatal("cancelled preparation published marker", err) } } + +func TestCapabilityMarkerSurvivesComputeReplacement(t *testing.T) { + retained := t.TempDir() + if err := os.Chmod(retained, 0700); err != nil { + t.Fatal(err) + } + t.Setenv("OAC_RUNTIME_STATE_DIRECTORY", retained) + b, _ := markerBinding(t) + marker, err := b.openSnapshotMarker() + if err != nil { + t.Fatal(err) + } + if err = marker.complete(); err != nil { + t.Fatal(err) + } + marker.close() + t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) + identity := b.capabilityIdentity() + replacement, err := NewWithCapabilityDirectory(identity.EnvironmentID, identity.SessionID, b.workspace, b.capabilityRoot) + if err != nil { + t.Fatal(err) + } + marker, err = replacement.openSnapshotMarker() + if err != nil { + t.Fatal(err) + } + defer marker.close() + if !marker.completed { + t.Fatal("compute replacement forgot completed capability installation") + } +} diff --git a/apps/daemon/internal/paths/paths.go b/apps/daemon/internal/paths/paths.go index accdd85c6..d6af78594 100644 --- a/apps/daemon/internal/paths/paths.go +++ b/apps/daemon/internal/paths/paths.go @@ -50,6 +50,18 @@ func Root() (string, error) { return filepath.Join(home, ".oac"), nil } +// StateDirectory is the single process setting for private durable Runtime state. +// An explicit invalid value is an error, never a reason to use the default. +func StateDirectory() (string, error) { + if value, present := os.LookupEnv("OAC_RUNTIME_STATE_DIRECTORY"); present { + if !filepath.IsAbs(value) || filepath.Clean(value) != value { + return "", fmt.Errorf("OAC_RUNTIME_STATE_DIRECTORY must be a clean absolute directory") + } + return value, nil + } + return Root() +} + // ProfileDir returns ~/.oac/daemon/. It is not created here. func ProfileDir(profile string) (string, error) { if err := ValidateProfile(profile); err != nil { diff --git a/apps/daemon/internal/paths/paths_test.go b/apps/daemon/internal/paths/paths_test.go index cd6fc605f..4b355a6c5 100644 --- a/apps/daemon/internal/paths/paths_test.go +++ b/apps/daemon/internal/paths/paths_test.go @@ -107,3 +107,25 @@ func TestRootRejectsRelativeOverride(t *testing.T) { t.Fatal("relative private home accepted") } } + +func TestStateDirectorySingleSetting(t *testing.T) { + home := withTempHome(t) + t.Setenv("OAC_RUNTIME_STATE_DIRECTORY", "") + if err := os.Unsetenv("OAC_RUNTIME_STATE_DIRECTORY"); err != nil { + t.Fatal(err) + } + if got, err := paths.StateDirectory(); err != nil || got != home { + t.Fatal("default state directory", got, err) + } + selected := t.TempDir() + t.Setenv("OAC_RUNTIME_STATE_DIRECTORY", selected) + if got, err := paths.StateDirectory(); err != nil || got != selected { + t.Fatal("explicit state directory", got, err) + } + for _, invalid := range []string{"", "relative", selected + string(filepath.Separator) + ".."} { + t.Setenv("OAC_RUNTIME_STATE_DIRECTORY", invalid) + if _, err := paths.StateDirectory(); err == nil { + t.Fatal("invalid explicit setting fell back to default") + } + } +} diff --git a/contracts/agents-api/environments.md b/contracts/agents-api/environments.md index 4ac2eece0..195c5e3ff 100644 --- a/contracts/agents-api/environments.md +++ b/contracts/agents-api/environments.md @@ -211,10 +211,12 @@ The daemon runs as its launching account and never uses sudo or raises its permi - Setup commands run with Bash; on Windows, Git Bash is required and no other shell substitutes. The default working directory is `/workspace`. - On Windows, npm installation and stdio MCP commands named `npm` or `npx` (including their `.cmd` shims) run through npm's JavaScript entry point with Node, without an extra shell. -Initialization and package directories default to `initialization` and `packages` under the Runtime home (`OAC_RUNTIME_HOME`) and can be set with `OAC_RUNTIME_INITIALIZATION_DIRECTORY` and `OAC_RUNTIME_PACKAGE_DIRECTORY`; packaged Linux images use `/environment/initialization` and `/environment/packages`. These are resource paths, never Environment-source or operating-system switches in Core. +[Runtime resource directories](../../docs/configuration.md#runtime-resource-directories) define the initialization, package and private state locations. These are resource paths, never Environment-source or operating-system switches in Core. Every command uses the launching user's permissions and the host network. Process ownership waits for exit and I/O settlement. Command output is discarded; a confirmed failure keeps only a bounded integer exit status. +When hosted compute uses external workspace storage and is replaced, only the retained Environment filesystem and the qualified Harness's private native history persist. Setup commands are not replayed. Setup and tools can write other locations permitted to the launching user, but changes on the temporary system disk, process memory and background processes are not preserved by cold continuation. Store durable outputs in the workspace and use the declared package installation paths; the [Sandbox Provider lifecycle](../../docs/sandbox-provider.md#suspension) defines when compute can be replaced. + ### Explicit local tool environment The installer's `--tool-env-file` (`OAC_RUNTIME_TOOL_ENV_FILE`) supplies the Runtime operator's base tool variables. Preparation copies these values into its private initialization snapshot, and the Session's `env` keys override them. The Runtime never rewrites the source file or inherits unrelated ambient credentials. Setup, capability resolution and Harness execution read the same prepared snapshot. Reconnecting keeps that snapshot even if the operator edits the file; a new Session reads the current file. A Harness profile may reference the Runtime-owned file but must not persist copies of its values. A missing or invalid configured file fails preparation. diff --git a/contracts/agents-api/harness-capabilities.md b/contracts/agents-api/harness-capabilities.md index c1ad61d8f..b5130ae51 100644 --- a/contracts/agents-api/harness-capabilities.md +++ b/contracts/agents-api/harness-capabilities.md @@ -59,3 +59,5 @@ These operations need a workspace, so they apply to hosted and self-hosted place | Network `disabled` or `restricted` | Rejected | Rejected | Rejected | | [Plugin MCP over stdio](./environments.md#plugin-mcp) | Verified: hosted, self-hosted | Verified: self-hosted; admitted: hosted | Verified: self-hosted; admitted: hosted | | Plugin MCP over HTTP | Admitted, with literal headers or HTTPS bearer | Admitted, anonymous or HTTPS bearer | Verified: self-hosted; admitted: hosted; anonymous or HTTPS bearer | + +External workspace storage additionally requires the selected Harness profile and Runtime to support `retained_native_history`; owned workspace storage does not. The [Workspace Provider guide](../../docs/workspace-provider.md) owns storage combination admission. diff --git a/contracts/agents-api/harness-onboarding.md b/contracts/agents-api/harness-onboarding.md index 2bae3fb5f..6032efbe2 100644 --- a/contracts/agents-api/harness-onboarding.md +++ b/contracts/agents-api/harness-onboarding.md @@ -147,7 +147,9 @@ A Harness that supports the Subagent reads implements the [neutral observation c ## Register the adapter -Registration is static and requires a build. Export one `agent.Declaration` from `apps/daemon/internal/agent//declaration.go`, then add it to `harnessDeclarations` in [`cli/agent_discovery.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/apps/daemon/internal/cli/agent_discovery.go). The declaration contains the kind and complete capability descriptor, the shared model `Configuration` and a `Discover` function. Discovery receives the profile and diagnostic writers, owns native configuration and availability checks, and returns the installed `agent.Runtime` with its descriptor and Executor factory. Return nil when the adapter is not configured; return an unavailable descriptor without a factory when configured prerequisites fail. Keep version gates and factory-selection conditions inside the adapter. +Registration is static and requires a build. Export one `agent.Declaration` from `apps/daemon/internal/agent//declaration.go`, then add it to `harnessDeclarations` in [`cli/agent_discovery.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/apps/daemon/internal/cli/agent_discovery.go). The declaration contains the kind and complete capability descriptor, the shared model `Configuration` and a `Discover` function. Discovery receives the profile, diagnostic writers and the Runtime-resolved `DiscoveryOptions.StateRoot` and instance-private `DiscoveryOptions.RuntimeRoot`, owns native configuration and availability checks, and returns the installed `agent.Runtime` with its descriptor and Executor factory. Return nil when the adapter is not configured; return an unavailable descriptor without a factory when configured prerequisites fail. Keep version gates and factory-selection conditions inside the adapter. + +The Runtime resolves the [state directory](../../docs/configuration.md#runtime-resource-directories) once and passes it explicitly to every adapter. Adapters that support retained native history place it beneath that root with their own namespaced layout; they do not choose a fallback root from ambient environment or put history in the workspace. An adapter without this capability keeps instance-private native state and must not declare history portable merely because a state root is supplied. Private launch homes, scratch data and instance-private native state use the supplied `DiscoveryOptions.RuntimeRoot`. Device credentials and connection identity stay outside retained state. A state path alone does not qualify native continuation after compute replacement: the adapter must preserve native ownership, reject missing or foreign history before model input, and complete native shutdown before another writer attaches. [`cli/agent_registration.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/apps/daemon/internal/cli/agent_registration.go) iterates the discovered runtimes and calls `Registry.Register` from `agent/harness.go`. It verifies that discovery retained the declared kind and registers the Runtime in this order: diff --git a/contracts/agents-api/index.md b/contracts/agents-api/index.md index 9e9f6bb7f..f5eed41ba 100644 --- a/contracts/agents-api/index.md +++ b/contracts/agents-api/index.md @@ -127,6 +127,7 @@ Each item is Core's deliberate or native behavior where the official service beh - Runtimes do not enforce `disabled` or `restricted` networks, so Sessions that need them are rejected ([restricted network policy](./environments.md#restricted-network)). - `packages.system` is rejected; system packages must be preinstalled. +- Cold continuation does not restore process memory, background processes or temporary system-disk changes; completed setup is not replayed. See [Environment persistence](./environments.md#runtime-capability-preparation). Cross-node memory restoration and takeover of an unconfirmed old writer are not qualified. **Files and Environment files** diff --git a/contracts/agents-api/zh/environments.md b/contracts/agents-api/zh/environments.md index 1891bd481..35b847a49 100644 --- a/contracts/agents-api/zh/environments.md +++ b/contracts/agents-api/zh/environments.md @@ -1,7 +1,7 @@ --- title: "环境与模板" source: contracts/agents-api/environments.md -source_hash: 8fb6cbd0b8daed4686d1b6829a49e799f03bb78c6bdb1f93cdb9a4518fec4f8f +source_hash: ac67b8c7a651bae18588cd5c845b40ab4d9c89382ae9f2599a039a27f300c1c2 --- Environment 是 Session 的执行资源，包括 Harness 运行所在的机器、工作区以及已完成准备的能力。Session 通过其 `environment` 配置创建 Environment；不存在独立的 create 调用。Environment Template 是 Session 创建时解析的可复用准备配置。本契约涵盖这两类资源、两种放置方式、输入接纳、能力准备、Skills、Plugins 和 MCP 连接来源。 @@ -213,10 +213,12 @@ daemon 以启动它的账户身份运行，绝不使用 sudo 或提升权限。 - 设置命令使用 Bash 运行；在 Windows 上必须使用 Git Bash，且不能由其他 shell 替代。默认工作目录为 `/workspace`。 - 在 Windows 上，npm 安装以及名为 `npm` 或 `npx` 的 stdio MCP 命令（包括其 `.cmd` shim）会通过 Node 调用 npm 的 JavaScript 入口点运行，而不经过额外的 shell。 -初始化目录和软件包目录默认分别是 Runtime 主目录（`OAC_RUNTIME_HOME`）下的 `initialization` 和 `packages`，也可通过 `OAC_RUNTIME_INITIALIZATION_DIRECTORY` 和 `OAC_RUNTIME_PACKAGE_DIRECTORY` 设置；打包的 Linux 镜像使用 `/environment/initialization` 和 `/environment/packages`。这些是资源路径，在 Core 中绝不是 Environment 源或操作系统开关。 +[Runtime 资源目录](../../../docs/zh/configuration.md#runtime-resource-directories)定义初始化、软件包和私有状态的位置。这些是资源路径，在 Core 中绝不是 Environment 源或操作系统开关。 每条命令都使用启动用户的权限和主机网络。进程所有权会等待退出及 I/O 结算完成。命令输出会被丢弃；确认失败时只保留一个有界整数退出状态。 +使用外部工作区存储的托管计算替换后，仅持久 Environment 文件系统和已通过资格验证的 Harness 私有原生历史会保留。Setup 命令不会重放。Setup 和工具可以写入启动用户有权访问的其他位置，但冷续接不保留临时系统盘上的修改、进程内存或后台进程。持久输出应写入工作区，软件包应使用已声明的安装路径；[Sandbox Provider 生命周期](../../../docs/zh/sandbox-provider.md#suspension)定义何时可以替换计算。 + ### 显式本地工具环境 {#explicit-local-tool-environment} 安装器的 `--tool-env-file`（`OAC_RUNTIME_TOOL_ENV_FILE`）提供 Runtime 操作者的基础工具变量。准备过程会将这些值复制到其私有初始化快照中，并由 Session 的 `env` 键覆盖。Runtime 绝不会重写源文件，也不会继承无关的环境凭据。设置、能力解析和 Harness 执行都会读取同一份已准备快照。即使操作者编辑了文件，重连仍会保留该快照；新 Session 会读取当前文件。Harness profile 可以引用由 Runtime 所有的文件，但不得持久保存其值的副本。配置的文件缺失或无效时，准备过程会失败。 diff --git a/contracts/agents-api/zh/harness-capabilities.md b/contracts/agents-api/zh/harness-capabilities.md index b947daaf8..e95d9114d 100644 --- a/contracts/agents-api/zh/harness-capabilities.md +++ b/contracts/agents-api/zh/harness-capabilities.md @@ -1,7 +1,7 @@ --- title: "Harness 能力" source: contracts/agents-api/harness-capabilities.md -source_hash: e1ffc7a260ceaec64ba377f7c0db28f2c371c9d664098b110b740cc110506506 +source_hash: a88b7238811707dd7fecae6ee8e509572677b49958e36ddabf8b783e97382c13 --- 本页列出每个 Harness 在每种部署位置支持的能力。Core 根据 `services/core/internal/engine` 中 Harness 的引擎配置决定准入，运行 Session 的 Runtime 也必须声明操作。所链接契约定义各操作；[Harness 接入](harness-onboarding.md#qualify-the-adapter)说明资格验证方法。 @@ -61,3 +61,5 @@ Claude 结构化输出要求单 Agent、medium verbosity，且无 Skill、Plugin | 网络 `disabled` 或 `restricted` | 已拒绝 | 已拒绝 | 已拒绝 | | [stdio Plugin MCP](environments.md#plugin-mcp) | 已验证：托管、自托管 | 已验证：自托管；已准入：托管 | 已验证：自托管；已准入：托管 | | HTTP Plugin MCP | 已准入，字面值头或 HTTPS bearer | 已准入，匿名或 HTTPS bearer | 已验证：自托管；已准入：托管；匿名或 HTTPS bearer | + +外部工作区存储还要求所选 Harness profile 和 Runtime 支持 `retained_native_history`；自带工作区存储不要求此能力。[Workspace Provider 指南](../../../docs/zh/workspace-provider.md)定义存储组合准入规则。 diff --git a/contracts/agents-api/zh/harness-onboarding.md b/contracts/agents-api/zh/harness-onboarding.md index 6b7204bfb..5df702061 100644 --- a/contracts/agents-api/zh/harness-onboarding.md +++ b/contracts/agents-api/zh/harness-onboarding.md @@ -1,7 +1,7 @@ --- title: "添加 Harness" source: contracts/agents-api/harness-onboarding.md -source_hash: 38955b517dae94e5ce187e72a6b2e03cc7eeb7b886baa722b5f117dd3260253f +source_hash: 779954060b3be129f858e2f57e62b1af56cfe2f925f98362b48b20d32574f681 --- **Harness** 是一种运行模型和工具循环的原生代理引擎（Codex、Claude Code、MiniMax Code）。**Harness 适配器**将 Runtime 的 Executor 和 Turn 契约转换到该引擎的 SDK 或协议。本文档定义 Runtime–Harness 协议：适配器接口及其生命周期义务、注册、Core 资格认定和验收。[Harness capabilities](harness-capabilities.md) 记录了当前每个 Harness 支持的功能。 @@ -149,7 +149,9 @@ MCP、公共函数、延迟函数发现、结构化输出、图像输入、详 ## 注册适配器 {#register-the-adapter} -注册是静态的，并且需要构建。从 `apps/daemon/internal/agent//declaration.go` 导出一个 `agent.Declaration`，然后将其添加到 [`cli/agent_discovery.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/apps/daemon/internal/cli/agent_discovery.go) 的 `harnessDeclarations` 中。声明包含 kind、完整能力描述符、共享模型 `Configuration` 和 `Discover` 函数。发现过程接收 profile 和诊断写入器，负责原生配置和可用性检查，并返回已安装的 `agent.Runtime` 及其描述符和 Executor 工厂。未配置适配器时返回 nil；已配置的前置条件失败时，返回不带工厂的不可用描述符。将版本门控和工厂选择条件保留在适配器内部。 +注册是静态的，并且需要构建。从 `apps/daemon/internal/agent//declaration.go` 导出一个 `agent.Declaration`，然后将其添加到 [`cli/agent_discovery.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/apps/daemon/internal/cli/agent_discovery.go) 的 `harnessDeclarations` 中。声明包含 kind、完整能力描述符、共享模型 `Configuration` 和 `Discover` 函数。发现过程接收 profile、诊断写入器以及 Runtime 解析后的 `DiscoveryOptions.StateRoot` 和实例私有的 `DiscoveryOptions.RuntimeRoot`，负责原生配置和可用性检查，并返回已安装的 `agent.Runtime` 及其描述符和 Executor 工厂。未配置适配器时返回 nil；已配置的前置条件失败时，返回不带工厂的不可用描述符。将版本门控和工厂选择条件保留在适配器内部。 + +Runtime 统一解析[状态目录](../../../docs/zh/configuration.md#runtime-resource-directories)，并显式传给每个适配器。支持保留原生历史的适配器在该根目录下按各自的命名空间布局存放历史，不从环境变量选择回退根目录，也不把历史放进工作区。不支持该能力的适配器将原生状态保持为实例私有，不能仅因收到状态根目录就宣称历史可迁移。私有启动主目录、临时数据及实例私有原生状态使用传入的 `DiscoveryOptions.RuntimeRoot`。设备凭据和连接身份必须留在保留状态之外。仅有状态路径并不意味着计算资源替换后的原生续接已通过资格验证：适配器必须保持原生所有权，在提交模型输入之前拒绝缺失或外来历史，并在另一个写入者挂载前完成原生关闭。 [`cli/agent_registration.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/apps/daemon/internal/cli/agent_registration.go) 遍历已发现的 Runtime，并调用 `agent/harness.go` 中的 `Registry.Register`。它验证发现过程是否保留了声明的 kind，并按以下顺序注册该 Runtime： diff --git a/contracts/agents-api/zh/index.md b/contracts/agents-api/zh/index.md index 85acfdf9f..9d55afff7 100644 --- a/contracts/agents-api/zh/index.md +++ b/contracts/agents-api/zh/index.md @@ -1,7 +1,7 @@ --- title: "Agents API 覆盖台账" source: contracts/agents-api/index.md -source_hash: f4a4bf113b88eb25ebccea95cc3125a7413e40493066e60bf2215da86471a5c8 +source_hash: 2394633ee55b7f86d374d5ab3827d0168d3dcbae8b962115063613e288b4cfda --- Core 旨在以下方固定版本为准支持完整的 OpenAI Agents API（[public API rule](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#public-api)）。本台账记录 Core 对各项资源实现了哪些内容、哪些契约保存其详细信息，并列出相对于 OpenAI 服务的所有已知差异和所有未解决缺口。[API namespaces and credentials](../../../docs/zh/api/index.md) 说明谁调用哪些 API；[Agents API guide](../../../docs/zh/api/public-agent-api.md) 介绍使用方法。 @@ -130,6 +130,7 @@ Core 自身字段位于 `x_agents_core` 中（[Core extensions](../../../docs/zh - Runtime 不会实施 `disabled` 或 `restricted` 网络，因此需要这些网络的 Session 会被拒绝（[restricted network policy](environments.md#restricted-network)）。 - `packages.system` 会被拒绝；系统软件包必须预先安装。 +- 冷续接不恢复进程内存、后台进程或临时系统盘修改；已完成的 setup 不会重放。参见 [Environment 持久化](environments.md#runtime-capability-preparation)。跨节点内存恢复及接管未确认停止的旧写入者尚未通过资格验证。 **Files 和 Environment files** diff --git a/docs/architecture.md b/docs/architecture.md index 4159953a7..96351610d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,7 +2,9 @@ title: "Architecture" --- -OpenAgentCore separates orchestration, compute and native execution. Core owns the API and durable state. Sandbox Providers manage compute. Independent workspace filesystem adapters manage durable Environment storage through a separate protocol. A Runtime daemon prepares an Environment and runs the selected Harness, whose native SDK or protocol owns the model and tool loop. +OpenAgentCore is an Agent execution platform that brings Harnesses, models, tools, Sessions and execution environments together through replaceable protocol implementations. It separates orchestration, compute and native execution. Core owns the API and durable state. Sandbox Providers manage compute. Independent workspace filesystem adapters manage durable Environment storage through a separate protocol. A Runtime daemon prepares an Environment and runs the selected Harness, whose native SDK or protocol owns the model and tool loop. + +The recommended deployment choices are E2B-managed compute and self-managed microsandbox with independent persistent workspace storage. Both use the same Core orchestration and capability checks. Docker remains an available Provider. Compute and filesystem adapters declare their supported combinations; a deployment choice does not introduce a separate Agent execution flow. See [Sandbox Providers](./sandbox-provider.md) and [workspace filesystem providers](./workspace-provider.md) for their contracts and supported capabilities. ```mermaid flowchart TB diff --git a/docs/configuration.md b/docs/configuration.md index 02ca6682e..e3248bc5e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -72,6 +72,20 @@ An unset or empty value selects the default. Edit `.env`, then run `oac apply`. | `insecure` | `false` | `true` is required for an `http` endpoint and rejected for `https` | | `headers` | none | Request headers for the endpoint. `Host`, `Content-Length`, `Content-Type` and `Content-Encoding` are reserved | +### Runtime resource directories + +The Runtime resolves its resource directories before discovering Harness adapters. `` is the Runtime home selected by `OAC_RUNTIME_HOME`. `OAC_RUNTIME_STATE_DIRECTORY` defaults only when unset; an empty, relative or noncanonical explicit value is rejected. Directory settings have no secondary file or environment fallback. + +| Runtime process setting | Default | Packaged Linux images | +| --- | --- | --- | +| `OAC_RUNTIME_INITIALIZATION_DIRECTORY` | `/initialization` | `/environment/initialization` | +| `OAC_RUNTIME_PACKAGE_DIRECTORY` | `/packages` | `/environment/packages` | +| `OAC_RUNTIME_STATE_DIRECTORY` | `` | `/environment/runtime-state` | + +The state directory contains retained native Harness history and capability-installation completion records. It is separate from the declared workspace directory. Selecting a separate state directory does not relocate Runtime device credentials or connection identity from their instance-private paths in the Runtime home; they must not be copied with native history. [Harness onboarding](../contracts/agents-api/harness-onboarding.md#register-the-adapter) defines the adapter boundary, and [Environment preparation](../contracts/agents-api/environments.md) owns initialization and package behavior. + +Placing state on an independent filesystem preserves those files when compute is replaced. Native continuation still requires a qualified Harness, matching ownership and confirmed stop of the previous writer. It does not preserve process memory or arbitrary files on the compute root disk. + ## Runtime settings: Web Runtime settings live in Core's database. Change them in Web; scripts use the same Core API with the Core key. diff --git a/docs/runtime-protocol.md b/docs/runtime-protocol.md index 606a62031..b99b6b668 100644 --- a/docs/runtime-protocol.md +++ b/docs/runtime-protocol.md @@ -41,6 +41,7 @@ A declaration describes what the Runtime can do. Core admits a public feature on | `local_environment`, `workspace_read_preparation`, `workspace_output_export` | The Environment type is `openai_hosted` or `self_hosted` | | `workspace_read_preparation` | An idle Files directory read needs a read-only preparation | | `native_session_recovery` | A Session with a started Turn has no recorded native Session ID | +| `retained_native_history` | The Environment uses an external workspace binding; owned workspace storage does not require this capability | | `web_search_control`, `text_verbosity` | The Harness's engine profile declares that control | | `structured_output` and `message_items` | The Agent requests `json_schema` output | | `subagent_observations` | `multi_agent.enabled` is true | @@ -53,6 +54,8 @@ A declaration describes what the Runtime can do. Core admits a public feature on Core has no admission rule for `usage` and `resume`. +`retained_native_history` means the composed Runtime and Harness can reopen the same native Session from its private retained state after confirmed native shutdown. It does not promise process memory, background processes or external connection recovery. The selected filesystem must separately provide retained storage. Core uses the previous device's persisted declaration to determine whether compute can be released without terminating the Session, and validates the replacement Runtime before admitting execution. Missing or invalid native history fails explicitly; it never authorizes a new native root or replay of prior input. + The `execution_prepare` configuration carries the opt-ins Core sets for each Run: | Field | Set by Core | @@ -123,6 +126,8 @@ Preparation and start run outside the receive loop and router lock. An admission Idle expiry of an Executor is a Runtime resource policy, separate from Core's active-Turn concurrency. On shutdown the Runtime closes active and idle Executors, keeps any target whose close failed and allows a later serialized retry. An ordinary disconnection closes the failed transport and keeps the exact router until shutdown succeeds; a wait timeout or failed cleanup never authorizes reconnection, and process shutdown keeps waiting rather than discarding owned native resources. Workspace operations keep their binding and settlement rules across Turn boundaries and Executor closure. +Suspension closes admission, drains admitted work and receipts, and confirms closure of every idle Executor before acknowledging `environment_quiesced`. A failed or unconfirmed native close prevents acknowledgement and retains cleanup ownership. The Sandbox Provider remains responsible for the filesystem flush and compute-stop guarantees of its checkpoint implementation; Runtime quiescence alone does not establish those guarantees. + ## Active input receipts Core delivers active input as `prompt_steer` with `durable_receipt: true`, one input at a time per Run, and waits for its receipt before sending the next: diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 0eaa49ff4..58f49dd7f 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -164,7 +164,7 @@ The allocation, its dedicated daemon credential digest and the exact Session bin With a configured provider, the Worker scans committed pending hosted Environments that have no allocation, which covers idle Session creation and recovery after an interruption between commit and bootstrap; an existing allocation never re-enters that path. The scan is bounded and serialized by the lifecycle owner and needs no caller action. An initial reservation without a Turn leaves its Session idle, and a daemon connection is never treated as native readiness. The same scan publishes authenticated connection observations with durable generations, after verifying the exact Session and device binding and a settled bootstrap. -Between Turns, Core checks that connected, observed compute is still its Session's running allocation; the check changes nothing and never revives a cleanup request. Running compute never expires: explicit deletion and the snapshot retention authorize its cleanup. A stopped or missing container never authorizes discarding retained workspace or history. Disabling the provider stops new hosted admission and bootstrap but never blocks cancellation, function results or input retry outcomes of existing Sessions. +Between Turns, Core checks that connected, observed compute is still its Session's running allocation; the check changes nothing and never revives a cleanup request. Running compute never expires: explicit lifecycle cleanup and snapshot retention govern reclamation. A stopped or missing container never authorizes discarding retained workspace or history or selecting replacement compute. Disabling the provider stops new hosted admission and bootstrap but never blocks cancellation, function results or input retry outcomes of existing Sessions. Terminal cleanup atomically revokes the device's authority, records the Environment's failure or expiry, settles pending input and requests cancellation, and only then calls `Kill`; original input deadlines and retry outcomes are kept. Temporary provider outages, unknown Create results and stopped compute never prove a permanent failure. After public Session deletion Core keeps the allocation and marks it released only after owned compute and volume cleanup and proof that the original Create settled; an unknown creation keeps cleanup ownership even after an absence observation, and bounded scans continue to catch late resources without another `Create`. @@ -172,23 +172,31 @@ Terminal cleanup atomically revokes the device's authority, records the Environm Each registered node has one serial lifecycle worker that owns its gate, allocation and pending cursors, connections and wake hints; E2B allocations share one serial lifecycle without a node. A thin coordinator discovers nodes and shuts workers down, and never holds its map mutex during database, provider or wait operations. Workers advance independently, so a stuck provider on one online node never stalls another: lifecycle concurrency is one operation per node and grows with the node count. Offline workers stay, so their retained resources remain observable after reconnection. -Allocation scans filter by node before their 32-row page limit, and pending scans join the unreleased committed placement. Each node advances its own cursor, including past failed observations, and wraps once at the end. Direct provisioning resolves the tenant-scoped placement before entering that node's gate, and an existing allocation must agree with it; Core never chooses another node. +Allocation scans filter by node before their 32-row page limit, and pending scans join the unreleased committed placement. Each node advances its own cursor, including past failed observations, and wraps once at the end. Direct provisioning resolves the tenant-scoped placement before entering that node's gate, and an active allocation must agree with it. Only confirmed release permits a new placement; a disconnect never moves an active allocation. Before releasing the execution lease, the coordinator stops accepting work and cancels and drains every node worker and direct caller. Lease loss affects everything; ordinary provider failures stay within their node. A planned deployment drain or node retirement cancels lifecycle contexts synchronously between leased operations, through the lease gate with its five-second bound, including an active manual reconcile, and never cancels an in-flight leased query just to change configuration. A failed cancellation fence closes manager admission and reports owner failure. A failed retirement keeps the original lifecycle identity and gate until owner shutdown, and the drain barrier stays closed. Session locks, deployment capacity transactions and revision-checked receipts stay authoritative, and no external operation holds a database lock. ### Placement and capacity -Placement is automatic: the environment-to-node placement commits with Session creation and its retry identity, callers cannot choose a node, and a retry keeps its original node even while it is offline. Node capacity counts pending reservations and unresolved resources, and new placement and a suspended-to-restoring transition share a database lock. Unknown operations keep their reservations, source teardown must be confirmed before active capacity is released, and confirmed cleanup releases placement capacity. Retained ownership needs exact provider evidence: a socket path, a missing instance or an empty listing never proves cleanup or authorizes a replacement. +Placement is automatic: the Environment-to-node placement commits with Session creation and its retry identity, callers cannot choose a node, and an active allocation keeps its original node even while it is offline. After confirmed allocation release, eligible retained Sessions can reserve compatible capacity again, including on another node. Node capacity counts pending reservations and unresolved resources, and new placement and a suspended-to-restoring transition share a database lock. Unknown operations keep their reservations, source teardown must be confirmed before active capacity is released, and confirmed cleanup releases placement capacity. Retained ownership needs exact provider evidence: a socket path, a missing instance or an empty listing never proves cleanup or authorizes a replacement. -The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and the snapshot retention bound each node. There is no node-level drain, cross-node Session migration, multi-active Core, autoscaling or snapshot replication. Node removal is refused while the node holds allocations, snapshots, reservations, unknown results or cleanup, and offline ownership is kept. +The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and the snapshot retention bound each node. Cold replacement after confirmed release does not provide node-level drain, cross-node memory restore, failover from an unreachable writer, concurrent writers, multi-active Core, autoscaling or snapshot replication. Node removal is refused while the node holds allocations, snapshots, reservations, unknown results or cleanup, and offline ownership is kept. ### Suspension -Core suspends the idle work of every provider that declares checkpoint support, with one fixed policy: it suspends work idle for 5 minutes (300 seconds) and keeps the snapshot for 24 hours (86400 seconds). The deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) reports these values. Core suspends only after at least one Turn is terminal, when no root or Subagent Turn is queued, in progress or waiting, no input, file operation or initialization is pending, and real activity has been idle for that time. For node allocations Core records the first root or child terminal transition with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial snapshot retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. +Core suspends the idle work of every provider that declares checkpoint support, with one fixed policy: it suspends work idle for 5 minutes (300 seconds) and keeps the snapshot for 24 hours (86400 seconds). The deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) reports these values. Core suspends after initialization completes, when no root or Subagent Turn is queued, in progress or waiting, no input or file operation is pending, and real activity has been idle for that time. A Session need not have run a Turn. The idle clock starts no earlier than the allocation entering the running compute phase, including after a wake. For node allocations Core records the first root or child terminal transition with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial snapshot retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. The Worker lease, the Session lock and the per-node gates own suspension for every provider. New Turn claims, file-write intents and capture admission serialize under the Session lock and share one compute-phase check; new pending work cancels a capture and wakes the same source. Normal preparation waits for the compute phase to be running, after the authenticated resume handshake, and pending input stays pending when its promotion conflicts with a lifecycle transition. Compute phases and revision-checked receipts live on the allocation. Core persists quiesce, capture and restore intent before the effect, only a fresh receipt performs a capture or restore, and recovery observes the exact attempt without retrying an unknown creation, capture or restore. A consumed snapshot never rolls a running generation back. Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap, and unknown cleanup identities are kept until owned resources are confirmed absent. Consumed artifacts and old compute are deleted, so suspension cycles never build a chain of writable disks. -Queued work and live Environment file access wake a suspended Environment; history and published Artifact reads do not. Planned suspension uses an Environment and suspension token on the daemon connection. A PID and start-time fenced local control signal (`RunCommandCompute`) wakes the parked daemon, which authenticates again before admitting work. A transient disconnect before confirmation retries the same armed suspension with bounded attempts and backoff; a permanent authentication or protocol rejection closes it. Core owns the snapshot's retention deadline, and the daemon has no timer for it. A lost quiesce acknowledgement may thaw the same source through explicit rollback but never authorizes capturing it. +Queued work and live Environment file access wake a suspended Environment; history and published Artifact reads do not. Live file requests wait for the current Runtime’s credential-authorized connection and Harness declaration before entering the file work queue; a completed compute Create alone is not Runtime readiness. Planned suspension uses an Environment and suspension token on the daemon connection. A PID and start-time fenced local control signal (`RunCommandCompute`) wakes the parked daemon, which authenticates again before admitting work. A transient disconnect before confirmation retries the same armed suspension with bounded attempts and backoff; a permanent authentication or protocol rejection closes it. The retained checkpoint resumes on its original node. Core owns the snapshot's retention deadline, and the daemon has no timer for it. A lost quiesce acknowledgement may thaw the same source through explicit rollback but never authorizes capturing it. + +### Cold replacement after checkpoint retention + +Snapshot retention bounds compute resources, not the lifetime of an eligible retained Session. After the deadline, Core can clean up the allocation without terminating the Session only when its independent filesystem object is ready, initialization is complete, the Session and Environment remain nonterminal, and the previous authenticated Runtime has a persisted `retained_native_history` declaration. The capability and native recovery obligations are defined by the [Core–Runtime protocol](runtime-protocol.md). Filesystem retention alone is insufficient. + +Core confirms the original creation is settled and all owned compute and snapshots are cleaned up before releasing allocation and placement ownership. The old device's authority is revoked before replacement admission. Until Create, Kill or DeleteSnapshot has a confirmed outcome, it keeps ownership and admits no replacement writer. A recoverable Session and Environment remain disconnected with the same filesystem object and initialization result. Archive, reset and deletion take precedence and cannot be undone by a wake request. + +New input or live Environment file access reserves a new allocation on a compatible node and bootstraps a fresh Runtime device. Preparation reuses the same filesystem and completed initialization rather than replaying initialization. Execution resumes the same native Session from retained history after validating the replacement Runtime's declaration. Missing or invalid native history fails explicitly; Core never substitutes a new native Session or replays completed input. History and published Artifact reads do not reserve compute. This path does not preserve VM memory, background processes or open connections, and it does not qualify an undeclared Runtime, Harness or filesystem combination. ### Reset and archive @@ -240,4 +248,4 @@ The node uses the explicit Unix socket in its [provider configuration](./configu `DeploymentPolicy.Workspace` declares supported attachment requirements; absence means external storage is unsupported. The derived immutable `DeploymentSpec.Workspace` receipt selects the generation mode and capabilities; its [deployment workflow](../contracts/agents-api/sandbox-deployment.md#resources) never copies filesystem configuration into the generation. Microsandbox requires `host_directory` and `user_xattr`; Docker and E2B reject external attachments. `ValidateWorkspacePolicy` uses the filesystem protocol's combination validator: an external filesystem without an enforced quota rejects a positive `EnvironmentDiskMiB`, while zero requests no quota. Root disk capacity is still required. Without an external declaration, the existing owned disk bounds apply. -Microsandbox resolves each Create and Resume binding, including observation of an interrupted restore, and passes only the local directory and immutable ObjectID to its private helper. The helper binds the whole `/environment`, including workspace, staging, initialization and packages; private HOME and Harness history remain in the VM's checkpointed root. Explicit mode, object and path labels qualify native resources; mount shape alone never selects a mode. Restore verifies the snapshot's object, remaps `/environment` through SDK `Volumes`, requires strict external mount policy and rejects every restore warning. Partial restore errors retain the exact target identity for cleanup and never establish readiness. Unknown outcomes remain observations of the original operation. Native Bind checkpoint and guest-local flock continuity evidence does not establish cross-VM fencing. +Microsandbox resolves each Create and Resume binding, including observation of an interrupted restore, and passes only the local directory and immutable ObjectID to its private helper. The helper binds the whole `/environment`; its retained contents and private native history follow the [workspace filesystem scope](workspace-provider.md#lifetime-and-filesystem-scope) and [Runtime resource directories](configuration.md#runtime-resource-directories). Private HOME, credentials and the compute root are not a durable history store. Explicit mode, object and path labels qualify native resources; mount shape alone never selects a mode. Same-node checkpoint restore verifies the snapshot's object, remaps `/environment` through SDK `Volumes`, requires strict external mount policy and rejects every restore warning. Partial restore errors retain the exact target identity for cleanup and never establish readiness. Unknown outcomes remain observations of the original operation. Native Bind checkpoint and guest-local flock continuity evidence does not establish cross-VM fencing or cross-node memory restore. diff --git a/docs/workspace-provider.md b/docs/workspace-provider.md index de3ccb7e9..564a9b854 100644 --- a/docs/workspace-provider.md +++ b/docs/workspace-provider.md @@ -41,9 +41,9 @@ Workspace access is single-writer. Core must serialize compute ownership and con ## Lifetime and filesystem scope -The object contains the whole `/environment` tree, including the workspace, staging files, initialization state and package content. They remain on the same filesystem so filesystem operations within Environment preparation preserve their semantics. A Harness's private native history/configuration remains checkpoint state; it is not redirected into the workspace object. +The object contains the whole `/environment` tree, including the workspace, staging files, initialization state, package content and the private native history of a qualified Runtime and Harness. Preparation data remains on the same filesystem so its filesystem operations preserve their semantics. Native history is private state outside the workspace directory; placement and settings are owned by [Runtime resource directories](configuration.md#runtime-resource-directories). Credentials, disposable HOME and other compute-local state have no persistence guarantee. Retained native execution requires the declared capability in the [Core–Runtime protocol](runtime-protocol.md); mounting persistent storage alone does not establish it. -Workspace storage is retained with its Session until explicit Session deletion. Archival, expiration, reset and compute checkpoint deletion do not authorize workspace deletion. Core calls `Delete` only after explicit Session deletion and confirmed compute stop; unresolved compute or storage mutations retain ownership. Failed creation does not authorize deleting storage while its Session is retained. Snapshot-expiry recovery is not defined by this contract. +Workspace storage is retained with its Session until explicit Session deletion. Archival, expiration, reset and compute checkpoint deletion do not authorize workspace deletion. Core calls `Delete` only after explicit Session deletion and confirmed compute stop; unresolved compute or storage mutations retain ownership. Failed creation does not authorize deleting storage while its Session is retained. Eligible nonterminal Sessions can survive checkpoint retention expiry and later attach the same object to fresh compute under the [cold replacement lifecycle](sandbox-provider.md#cold-replacement-after-checkpoint-retention). This does not revive archived, reset or deleted execution. ## Kernel NFS adapter @@ -57,4 +57,6 @@ The declaration is `host_directory`, `user_xattr: true`, `capacity_quota: false` Each object owns `objects//identity`, `staging/`, `live/`, `trash/` and, after deletion begins, `deleted-marker`. The permanent identity binds the tenant, Environment and object UUIDs. Create prepares and syncs a private unique staging envelope before atomically publishing its nonempty directory as `live`; replay cannot overwrite existing live data. Delete first records a durable object-local terminal marker, retires live data to unique object-local trash and removes data before its ownership markers. Concurrent deleters converge. Minimal identity and deletion metadata remain intentionally; deletion does not mean every metadata file disappears. Create and Resolve refuse the retired identity. A late in-flight Create can leave empty controlled metadata requiring a repeated Delete after its syscall settles; it cannot authorize a new writer or reuse the object identity. Cleanup examines only that object's staging and trash, without a namespace-wide sweep or background service. -NFS storage does not supply compute fencing or automatic distributed failover. Guest `flock` is not a cross-VM writer lock. The single-writer and explicit-deletion requirements above remain mandatory. The shared filesystem preserves `/environment`, while a Harness's private native history remains in its compute checkpoint; expiration of a native snapshot can therefore remove private history even while workspace files survive. Workspace retention alone is not a promise of resumable native execution after snapshot TTL expiry. +NFS storage does not supply compute fencing or automatic distributed failover. Guest `flock` is not a cross-VM writer lock. The single-writer and explicit-deletion requirements above remain mandatory. Confirmed release can permit a fresh VM on another compatible node to attach the same object, but an offline node or an unknown Create, Kill or snapshot-cleanup result never proves that the old writer stopped. Native history and database behavior require qualification of the selected Runtime, Harness and filesystem combination; file persistence alone is not a guarantee of native Session recovery, cross-node memory restore or arbitrary crash durability. + +Core requires the selected Harness profile to support `retained_native_history` when the selected deployment generation uses external workspace storage. Creation validates that generation while holding the deployment lock, before committing the Session or compute reservation. Existing Sessions use their immutable workspace binding for Runtime admission and the final execution check. Owned workspace storage does not require this capability. diff --git a/docs/zh/architecture.md b/docs/zh/architecture.md index 4be5d241a..e1739e792 100644 --- a/docs/zh/architecture.md +++ b/docs/zh/architecture.md @@ -1,10 +1,12 @@ --- title: "架构" source: docs/architecture.md -source_hash: 75efb1d2899314719492ca4f2cd8bb5dbcdc14f54afb72ac33060a15da309d31 +source_hash: 061593c891cc4dd8b9077311bec8b8b93051c1cba9e387fb7db40a7ffc3a3de5 --- -OpenAgentCore 将编排、计算资源和原生执行分开。Core 负责 API 和持久状态。Sandbox Provider 管理计算资源。独立工作区文件系统 adapter 通过单独的协议管理 Environment 持久存储。Runtime daemon 准备 Environment 并运行选定的 Harness；Harness 的原生 SDK 或协议负责模型与工具循环。 +OpenAgentCore 是 Agent 运行平台，通过可替换的协议实现统一组织 Harness、模型、工具、Session 和执行环境。它将编排、计算资源和原生执行分开。Core 负责 API 和持久状态。Sandbox Provider 管理计算资源。独立工作区文件系统 adapter 通过单独的协议管理 Environment 持久存储。Runtime daemon 准备 Environment 并运行选定的 Harness；Harness 的原生 SDK 或协议负责模型与工具循环。 + +推荐的部署选择是 E2B 托管计算，以及 microsandbox 配合独立持久化工作区存储的自部署方案。两者使用相同的 Core 编排和能力检查。Docker 仍是可用的 Provider。计算和文件系统 adapter 声明其支持的组合；部署选择不会引入独立的 Agent 执行流程。具体契约和支持能力见 [Sandbox Provider](./sandbox-provider.md) 和[工作区文件系统 Provider](./workspace-provider.md)。 ```mermaid flowchart TB diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 793535d52..edbaefa62 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: eeb840aefb1eab56b16b5185c9fb6b3fcba5f0e9e666eaa32cf9e02f1e2fe134 +source_hash: 75ead1a81873a6fd67451b9beacb530bf156cdeefcc8dd0add74d5c5d4404484 --- Core 安装的每项设置都恰好只有一个归属位置，分属以下三类： @@ -76,6 +76,20 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 | `insecure` | `false` | `http` 端点必须设为 `true`，`https` 端点不允许设为 `true` | | `headers` | 无 | 发往端点的请求标头。`Host`、`Content-Length`、`Content-Type` 和 `Content-Encoding` 为保留标头 | +### Runtime 资源目录 {#runtime-resource-directories} + +Runtime 在发现 Harness 适配器之前解析资源目录。`` 是由 `OAC_RUNTIME_HOME` 选择的 Runtime 主目录。`OAC_RUNTIME_STATE_DIRECTORY` 仅在未设置时采用默认值；显式空值、相对路径或非规范路径会被拒绝。目录设置没有其他文件或环境变量回退来源。 + +| Runtime 进程设置 | 默认值 | 打包的 Linux 镜像 | +| --- | --- | --- | +| `OAC_RUNTIME_INITIALIZATION_DIRECTORY` | `/initialization` | `/environment/initialization` | +| `OAC_RUNTIME_PACKAGE_DIRECTORY` | `/packages` | `/environment/packages` | +| `OAC_RUNTIME_STATE_DIRECTORY` | `` | `/environment/runtime-state` | + +状态目录包含保留的原生 Harness 历史和能力安装完成记录，与声明的工作区目录分开。选择独立状态目录不会将 Runtime 设备凭据或连接身份移出 Runtime 主目录中的实例私有路径；不得随原生历史复制这些数据。[Harness 接入指南](../../contracts/agents-api/zh/harness-onboarding.md#register-the-adapter)定义适配器边界，[Environment 准备](../../contracts/agents-api/zh/environments.md)负责初始化和包的行为。 + +将状态放在独立文件系统上，可以在替换计算资源时保留这些文件。原生续接仍要求 Harness 已通过资格验证、所有权匹配，并确认前一个写入者已停止。这不保留进程内存或计算资源根磁盘上的任意文件。 + ## 运行时设置：Web {#runtime-settings-web} 运行时设置存储在 Core 的数据库中。请在 Web 中修改；脚本使用同一个 Core API 和 Core 密钥。 diff --git a/docs/zh/runtime-protocol.md b/docs/zh/runtime-protocol.md index 2278a9fd7..92de66cca 100644 --- a/docs/zh/runtime-protocol.md +++ b/docs/zh/runtime-protocol.md @@ -1,7 +1,7 @@ --- title: "Core–Runtime 协议" source: docs/runtime-protocol.md -source_hash: d2edb668d82f40ccf9094b85fcba67852bd14bc226f982d8b4e8918015c79ef8 +source_hash: 59633fbfa54d9071ed6a5ef510328d9216f4d8572a0cc068de65c93ecabefc59 --- 此协议在 Runtime daemon 获取机器凭据后连接 Core 与 daemon，定义 daemon 连接上消息的含义和顺序。wire 类型、限制和验证器仅在 [`internal/agentdaemon/proto`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/internal/agentdaemon/proto) 中定义一次；Core 的 [gateway](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/runtimegateway) 与参考 Runtime 的 [dispatcher](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/apps/daemon/internal/dispatch) 都使用它们，因此无需同步第二套 payload schema。签发凭据和打开连接的 HTTP 路由见[机器连接 API](../../contracts/agents-api/zh/machine-api.md)。 @@ -43,6 +43,7 @@ wire 上每个字段都是 JSON boolean，所有字段都必须出现，包括 ` | `local_environment`, `workspace_read_preparation`, `workspace_output_export` | Environment 类型为 `openai_hosted` 或 `self_hosted` | | `workspace_read_preparation` | 空闲 Files 目录读取需要只读 preparation | | `native_session_recovery` | Session 已启动过 Turn，但未记录原生 Session ID | +| `retained_native_history` | Environment 使用外部工作区绑定；自带工作区存储不要求此能力 | | `web_search_control`, `text_verbosity` | Harness 的 engine profile 声明该控制 | | `structured_output` 和 `message_items` | Agent 请求 `json_schema` 输出 | | `subagent_observations` | `multi_agent.enabled` 为 true | @@ -53,6 +54,8 @@ wire 上每个字段都是 JSON boolean，所有字段都必须出现，包括 ` | `message_images`, `function_result_images` | 消息或 function result 携带图像 | | `mcp_http_tools`, `mcp_http_required`, `mcp_http_bearer_auth` | Agent 声明 HTTP MCP server；其中一个为 `required`；其中一个选用了 Vault 凭据 | +`retained_native_history` 表示 Runtime 与 Harness 组合可在确认原生进程关闭后，从私有持久状态重新打开同一个原生 Session。它不保证恢复进程内存、后台进程或外部连接。所选文件系统还必须独立提供持久存储。Core 根据旧设备已持久化的声明判断能否释放计算而不终结 Session，并在准入执行前验证新 Runtime。原生历史缺失或无效时明确失败，绝不授权新建原生会话或重放此前输入。 + Core 对 `usage` 和 `resume` 没有准入规则。 `execution_prepare` 的配置携带 Core 为各 Run 设置的显式启用项： @@ -125,6 +128,9 @@ preparation 和 start 在 receive loop 与 router lock 之外运行。admission Executor 空闲到期属于 Runtime 资源策略，与 Core 的活动 Turn 并发限制独立。关闭时 Runtime 关闭活动和空闲 Executor，保留关闭失败的目标，并允许稍后串行重试。普通断连会关闭失败的 transport 并保留原 router，直到 shutdown 成功；等待超时或清理失败不授权重连，进程 shutdown 继续等待，不丢弃自己拥有的原生资源。工作区操作在跨 Turn 和 Executor 关闭后仍保留绑定与结算规则。 + +挂起先关闭准入、排空已接纳工作与回执，并确认每个空闲 Executor 已关闭，随后才确认 `environment_quiesced`。原生关闭失败或结果未确认时，不得确认挂起，且保留清理责任。Sandbox Provider 仍负责其 checkpoint 实现的文件系统刷新和计算停止保证；Runtime 静止本身不证明这些保证。 + ## 活动输入回执 {#active-input-receipts} Core 通过 `prompt_steer` 交付活动输入，设置 `durable_receipt: true`，每个 Run 一次交付一个输入，并等待回执后再发送下一个： diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 67e1d5f25..73f75a0d5 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: ec6c807e262a37d2ecd456101dadf7ae4818f1bd711b4af5f08e27c9d1690d83 +source_hash: 10483462da09c4fca5d1cb04a219f7d93faa020ea6104536b33dce3deabea1d4 --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -166,7 +166,7 @@ allocation、专用 daemon credential digest 和精确 Session binding 在 `Crea 配置 provider 后，Worker 扫描已提交且没有 allocation 的 pending hosted Environment，涵盖空闲 Session 创建以及 commit 与 bootstrap 之间中断后的恢复；已有 allocation 不重新进入此路径。scan 有界，由 lifecycle owner 串行化，不需要调用方操作。没有 Turn 的初始预约让 Session 保持空闲，daemon 连接不被当作原生 readiness。同一 scan 在验证精确 Session、device binding 和已结算 bootstrap 后，发布带持久 generation 的认证连接观测。 -Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 正在运行的 allocation；该检查不做任何修改，也不复活 cleanup 请求。正在运行的计算资源不会到期：显式删除和 snapshot retention 授权其清理。停止或缺失 container 不授权丢弃保留工作区或历史。禁用 provider 停止新 hosted admission 与 bootstrap，但不阻止现有 Session 的取消、function result 或 input retry outcome。 +Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 正在运行的 allocation；该检查不做任何修改，也不复活 cleanup 请求。正在运行的计算资源不会到期：显式生命周期清理和快照保留策略管理资源回收。停止或缺失 container 不授权丢弃保留工作区或历史，也不授权选择替代计算资源。禁用 provider 停止新 hosted admission 与 bootstrap，但不阻止现有 Session 的取消、function result 或 input retry outcome。 终结清理原子撤销 device authority、记录 Environment 失败或到期、结算 pending input 并请求取消，然后才调用 `Kill`；原 input deadline 与 retry outcome 保留。临时 provider outage、未知 Create result 和停止的计算资源不证明永久失败。公开 Session 删除后 Core 保留 allocation，仅在所属 compute 与 volume 清理完成且原 Create 已结算的证明成立后标记 released；未知创建即使观察到不存在也保留 cleanup ownership，有界 scan 继续捕捉延迟资源，不再调用 `Create`。 @@ -174,23 +174,31 @@ Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 每个注册 node 有一个串行 lifecycle worker，负责 gate、allocation 与 pending cursor、connection 和 wake hint；E2B allocation 共享一个没有 node 的串行 lifecycle。薄 coordinator 发现 node 并关闭 worker，数据库、provider 或等待操作期间不持有 map mutex。worker 独立推进，因此一个在线 node 的 provider 卡住不会阻塞其他 node：lifecycle 并发为每 node 一项操作，随 node 数量增长。离线 worker 保留，因此其保留资源在重连后仍可观察。 -allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan join 尚未释放的已提交 placement。每个 node 推进自己的 cursor，包括越过失败观测，并在末尾回绕一次。direct provisioning 在进入该 node gate 前解析 tenant 范围内 placement，已有 allocation 必须与其一致；Core 不选择另一 node。 +allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan join 尚未释放的已提交 placement。每个 node 推进自己的 cursor，包括越过失败观测，并在末尾回绕一次。direct provisioning 在进入该 node gate 前解析 tenant 范围内 placement，活动 allocation 必须与其一致。只有确认释放后才能建立新的 placement；断连不会迁移活动 allocation。 释放 execution lease 前，coordinator 停止接受工作，取消并排空每个 node worker 和 direct caller。lease 丢失影响全部；普通 provider failure 限于所属 node。计划 deployment drain 或 node retirement 通过五秒有界 lease gate，在 leased operation 之间同步取消 lifecycle context，包括活动 manual reconcile，不仅为了改变配置就取消进行中的 leased query。失败的 cancellation fence 关闭 manager admission 并报告 owner failure。失败的 retirement 保留原 lifecycle identity 和 gate，直到 owner shutdown，drain barrier 保持关闭。Session lock、deployment capacity transaction 和 revision-checked receipt 仍是权威依据，外部操作不持有数据库锁。 ### Placement 与容量 {#placement-and-capacity} -placement 自动完成：environment-to-node placement 与 Session 创建及其 retry identity 一起提交，调用方不能选择 node，重试即使 node 离线也保留原 node。Node capacity 计入 pending reservation 和未决资源，新 placement 与 suspended-to-restoring 转移共享数据库锁。未知操作保留预约，source teardown 必须确认后才能释放 active capacity，确认 cleanup 后释放 placement capacity。保留所有权需要精确 provider evidence：socket path、缺失 instance 或空列表都不证明 cleanup，也不授权 replacement。 +placement 自动完成：Environment-to-node placement 与 Session 创建及其 retry identity 一起提交，调用方不能选择 node；活动 allocation 即使在 node 离线时也保留原 node。确认 allocation 释放后，符合条件的保留 Session 可以重新预约兼容容量，包括其他 node。Node capacity 计入 pending reservation 和未决资源，新 placement 与 suspended-to-restoring 转移共享数据库锁。未知操作保留预约，source teardown 必须确认后才能释放 active capacity，确认 cleanup 后释放 placement capacity。保留所有权需要精确 provider evidence：socket path、缺失 instance 或空列表都不证明 cleanup，也不授权 replacement。 -部署的 CPU、memory、disk 设置、`max_active`、`max_retained` 和 snapshot retention 限制每个 node。不提供 node-level drain、跨 node Session migration、multi-active Core、autoscaling 或 snapshot replication。node 持有 allocation、snapshot、reservation、unknown result 或 cleanup 时拒绝移除 node，离线 ownership 保留。 +部署的 CPU、memory、disk 设置、`max_active`、`max_retained` 和 snapshot retention 限制每个 node。确认释放后的冷替换不提供 node-level drain、跨 node 内存恢复、不可达写入方的故障转移、并发写入方、multi-active Core、autoscaling 或 snapshot replication。node 持有 allocation、snapshot、reservation、unknown result 或 cleanup 时拒绝移除 node，离线 ownership 保留。 ### 暂停 {#suspension} -Core 用同一个固定 policy 暂停每个声明 checkpoint 支持的 provider 的空闲工作：工作空闲 5 分钟（300 秒）后暂停，snapshot 保留 24 小时（86400 秒）。部署的 [`suspension`](../../contracts/agents-api/zh/sandbox-deployment.md#safe-response) 报告这两个值。Core 仅在至少一个 Turn 已终结、没有 root 或 Subagent Turn 排队、进行中或等待、没有 pending input、file operation 或 initialization，且真实 activity 已空闲达到该时间后暂停。对于 node allocation，Core 在同一事务中用数据库时钟记录首个 root 或 child terminal transition。candidate filter 和 Session-locked recheck 比较数据库已过时间与 idle duration，初始 snapshot retention deadline 也锚定同一数据库观测，因此 Core 与数据库主机时钟无需一致。原生 completion timestamp 在公开历史中保持不变，但不驱动 idle admission，heartbeat 不重置 activity。确认计划暂停前，daemon 关闭 admission 并排空 native cleanup、output receipt 和 file work。 +Core 用同一个固定 policy 暂停每个声明 checkpoint 支持的 provider 的空闲工作：工作空闲 5 分钟（300 秒）后暂停，snapshot 保留 24 小时（86400 秒）。部署的 [`suspension`](../../contracts/agents-api/zh/sandbox-deployment.md#safe-response) 报告这两个值。Core 在 initialization 完成、没有 root 或 Subagent Turn 排队、进行中或等待、没有 pending input 或 file operation，且真实 activity 已空闲达到该时间后暂停。Session 无需已运行过 Turn。idle clock 不早于 allocation 进入 running compute phase 的时刻开始，包括 wake 后重新进入该 phase。对于 node allocation，Core 在同一事务中用数据库时钟记录首个 root 或 child terminal transition。candidate filter 和 Session-locked recheck 比较数据库已过时间与 idle duration，初始 snapshot retention deadline 也锚定同一数据库观测，因此 Core 与数据库主机时钟无需一致。原生 completion timestamp 在公开历史中保持不变，但不驱动 idle admission，heartbeat 不重置 activity。确认计划暂停前，daemon 关闭 admission 并排空 native cleanup、output receipt 和 file work。 Worker lease、Session lock 与 per-node gate 对每个 provider 负责 suspension。新 Turn claim、file-write intent 和 capture admission 在 Session lock 下串行化，共享一个 compute-phase 检查；新 pending work 取消 capture 并唤醒同一 source。正常 preparation 在经过认证的 resume handshake 后等待 compute phase 为 running；pending input 的 promotion 与 lifecycle transition 冲突时保持 pending。compute phase 和 revision-checked receipt 位于 allocation。Core 在 effect 前持久化 quiesce、capture 和 restore intent，仅新 receipt 执行 capture 或 restore，恢复观察精确 attempt，不重试未知 creation、capture 或 restore。已消费 snapshot 不让 running generation 回滚。删除、撤销和 retention expiry 优先于 wake，一直持续到最终数据库 compare-and-swap；未知 cleanup identity 保留，直到确认所属资源不存在。已消费 artifact 和旧 compute 被删除，因此暂停循环不累积可写磁盘链。 -排队工作和实时 Environment file access 唤醒 suspended Environment；history 和已发布 Artifact read 不唤醒。计划暂停在 daemon 连接上使用 Environment 和 suspension token。受 PID 与 start-time fencing 的本地 control signal（`RunCommandCompute`）唤醒 parked daemon，daemon 在准入工作前重新认证。确认前临时断连通过有界 attempt 和 backoff 重试同一已 armed suspension；永久认证或协议拒绝则关闭。Core 负责 snapshot retention deadline，daemon 没有相应 timer。quiesce 确认丢失时可以通过明确 rollback 解冻同一 source，但不授权 capture。 +排队工作和实时 Environment file access 唤醒 suspended Environment；history 和已发布 Artifact read 不唤醒。实时文件请求在进入文件工作队列前，等待当前 Runtime 通过 credential 授权的连接及 Harness 声明；compute Create 完成并不代表 Runtime 已就绪。计划暂停在 daemon 连接上使用 Environment 和 suspension token。受 PID 与 start-time fencing 的本地 control signal（`RunCommandCompute`）唤醒 parked daemon，daemon 在准入工作前重新认证。确认前临时断连通过有界 attempt 和 backoff 重试同一已 armed suspension；永久认证或协议拒绝则关闭。保留的检查点在原 node 恢复。Core 负责 snapshot retention deadline，daemon 没有相应 timer。quiesce 确认丢失时可以通过明确 rollback 解冻同一 source，但不授权 capture。 + +### 检查点保留期后的冷替换 {#cold-replacement-after-checkpoint-retention} + +快照保留期限制计算资源，不限制符合条件的保留 Session 的生命周期。到期后，只有独立文件系统对象已就绪、初始化已完成、Session 与 Environment 均未终结，且此前已认证 Runtime 的 `retained_native_history` 声明已持久化时，Core 才能在不终结 Session 的情况下清理 allocation。能力及原生恢复义务由 [Core–Runtime 协议](runtime-protocol.md)定义。仅保留文件系统并不足够。 + +Core 在释放 allocation 和 placement 归属前，确认原始创建已结算且所属计算资源与快照均已清理。替代计算资源准入前撤销旧 device 的权限。Create、Kill 或 DeleteSnapshot 的结果尚未确认时，Core 保留归属，不准入替代写入方。可恢复的 Session 与 Environment 保持断连，保留同一文件系统对象和初始化结果。归档、重置和删除优先，唤醒请求不能撤销它们。 + +新 input 或实时 Environment 文件访问会在兼容 node 上预约新 allocation，并引导新的 Runtime device。准备流程复用同一文件系统和已完成的初始化，不重放初始化。验证替代 Runtime 的声明后，执行从保留历史续接同一原生 Session。原生历史缺失或无效时明确失败；Core 不替换为新的原生 Session，也不重放已完成的 input。读取历史和已发布 Artifact 不预约计算资源。此路径不保留 VM 内存、后台进程或开放连接，也不为未声明能力的 Runtime、Harness 或文件系统组合提供资格。 ### 重置与归档 {#reset-and-archive} @@ -242,4 +250,4 @@ node 使用 [provider 配置](configuration.md#docker-node-configuration)中的 `DeploymentPolicy.Workspace` 声明挂载要求；未声明表示不支持外部存储。派生的不可变 `DeploymentSpec.Workspace` 回执选择代次的模式和能力；[部署流程](../../contracts/agents-api/zh/sandbox-deployment.md#resources)不会将文件系统配置复制进代次。Microsandbox 要求 `host_directory` 和 `user_xattr`；Docker 和 E2B 拒绝外部挂载。`ValidateWorkspacePolicy` 使用文件系统协议的组合校验器：不强制容量配额的外部文件系统拒绝正值 `EnvironmentDiskMiB`，零表示不请求配额。根磁盘容量仍为必需。未提供外部声明时，继续使用自有磁盘的容量边界。 -Microsandbox 在每次 Create 和 Resume（包括观察中断的恢复）时解析绑定，仅向私有 helper 传递本地目录和不可变 ObjectID。Helper 挂载整个 `/environment`，包含 workspace、staging、initialization 和 packages；私有 HOME 及 Harness 历史仍保存在 VM 根磁盘检查点中。原生资源通过明确的模式、对象和路径标签校验，不根据挂载形状推断模式。恢复校验快照对象，通过 SDK `Volumes` 重映射 `/environment`，要求严格外部挂载策略，并拒绝任何恢复警告。恢复部分失败时保留精确目标身份用于清理，不证明就绪。未知结果继续观察原操作。原生 Bind 检查点及 guest 内 flock 连续性证据不证明跨 VM 隔离。 +Microsandbox 在每次 Create 和 Resume（包括观察中断的恢复）时解析绑定，仅向私有 helper 传递本地目录和不可变 ObjectID。Helper 挂载整个 `/environment`；保留内容与私有原生历史遵循[工作区文件系统范围](workspace-provider.md#lifetime-and-filesystem-scope)和 [Runtime 资源目录](configuration.md#runtime-resource-directories)。私有 HOME、凭据和计算根磁盘不是持久历史存储。原生资源通过明确的模式、对象和路径标签校验，不根据挂载形状推断模式。同 node 检查点恢复校验快照对象，通过 SDK `Volumes` 重映射 `/environment`，要求严格外部挂载策略，并拒绝任何恢复警告。恢复部分失败时保留精确目标身份用于清理，不证明就绪。未知结果继续观察原操作。原生 Bind 检查点及 guest 内 flock 连续性证据不证明跨 VM 隔离或跨 node 内存恢复。 diff --git a/docs/zh/workspace-provider.md b/docs/zh/workspace-provider.md index 489696153..16e2cbe55 100644 --- a/docs/zh/workspace-provider.md +++ b/docs/zh/workspace-provider.md @@ -1,7 +1,7 @@ --- title: 工作区文件系统 Provider source: docs/workspace-provider.md -source_hash: e9749e1aab1fc2135dcd92a843263243e4adccea393038c495fec67e114dbe83 +source_hash: 39d26eb96b553c925a46d1a2ae90fe4dc3d93fb21d5b9883e55656fd62e0f0fb --- 独立工作区文件系统边界由 [`workspacefs.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/workspacefs/workspacefs.go) 定义。本文规定必需的集成契约，并不表示所有 Sandbox Provider 或执行位置均已实现。文件系统适配器拥有存储对象与原生挂载解析职责；[Sandbox Provider](sandbox-provider.md) 拥有计算资源职责。Core 在分配任一资源前选择并验证二者的组合。 @@ -43,9 +43,9 @@ source_hash: e9749e1aab1fc2135dcd92a843263243e4adccea393038c495fec67e114dbe83 ## 生命周期与文件系统范围 {#lifetime-and-filesystem-scope} -对象包含整个 `/environment` 目录树，包括工作区、暂存文件、初始化状态和包内容。它们位于同一文件系统，以保留 Environment 准备期间文件系统操作的语义。Harness 的私有原生历史和配置仍属于检查点状态，不重定向到工作区对象。 +对象包含整个 `/environment` 目录树，包括工作区、暂存文件、初始化状态、包内容，以及已具备资格的 Runtime 与 Harness 的私有原生历史。准备数据位于同一文件系统，以保留文件系统操作的语义。原生历史是工作区目录之外的私有状态；位置与设置由 [Runtime 资源目录](configuration.md#runtime-resource-directories)定义。凭据、临时 HOME 和其他计算本地状态不保证持久化。保留原生执行需要 [Core–Runtime 协议](runtime-protocol.md)中的能力声明；仅挂载持久存储并不足以证明支持。 -工作区存储随 Session 保留，直至显式删除 Session。归档、过期、重置和删除计算检查点均不授权删除工作区。Core 仅在显式删除 Session 且确认计算资源停止后调用 `Delete`；未解决的计算或存储变更须保留归属。Session 仍被保留时，创建失败不授权删除其存储。本契约不定义快照过期恢复。 +工作区存储随 Session 保留，直至显式删除 Session。归档、过期、重置和删除计算检查点均不授权删除工作区。Core 仅在显式删除 Session 且确认计算资源停止后调用 `Delete`；未解决的计算或存储变更须保留归属。Session 仍被保留时，创建失败不授权删除其存储。符合条件且未终结的 Session 可以在检查点保留期结束后继续保留，并按[冷替换生命周期](sandbox-provider.md#cold-replacement-after-checkpoint-retention)将同一对象挂载到新的计算资源。此流程不会复活已归档、重置或删除的执行。 ## 内核 NFS 适配器 {#kernel-nfs-adapter} @@ -59,4 +59,6 @@ source_hash: e9749e1aab1fc2135dcd92a843263243e4adccea393038c495fec67e114dbe83 每个对象拥有 `objects//identity`、`staging/`、`live/`、`trash/`，以及删除开始后的 `deleted-marker`。永久标识绑定租户、Environment 和对象 UUID。Create 先准备并同步唯一的私有 staging 封装目录，再原子发布其非空目录为 `live`；重放不会覆盖现有 live 数据。Delete 先持久记录对象本地的终态标记，将 live 数据退役到唯一的对象本地 trash，再先删除数据、后删除其所有权标记。并发删除者收敛。最少量的标识和删除元数据会有意保留；删除不代表所有元数据文件均消失。Create 和 Resolve 拒绝已退役标识。迟到的在途 Create 可能留下空的受控元数据，需要在其系统调用完成后重复 Delete；它不能授权新写入者或复用对象标识。清理只查看该对象的 staging 和 trash，不全量扫描命名空间，也不使用后台服务。 -NFS 存储不提供计算 fencing 或自动分布式故障转移。Guest `flock` 不是跨 VM 写入者锁。上述单写入者和显式删除要求仍然必须遵守。共享文件系统保留 `/environment`，Harness 的私有原生历史仍在计算检查点内；因此，即使工作区文件仍然存在，原生快照过期仍可能导致私有历史丢失。保留工作区不代表原生快照 TTL 到期后仍可恢复原生执行。 +NFS 存储不提供计算 fencing 或自动分布式故障转移。Guest `flock` 不是跨 VM 写入者锁。上述单写入者和显式删除要求仍然必须遵守。确认释放后，可以允许另一兼容 node 上的新 VM 挂载同一对象，但 node 离线或 Create、Kill、快照清理结果未知均不证明旧写入方已停止。原生历史和数据库行为需要对所选 Runtime、Harness、文件系统组合进行资格验证；仅保留文件不保证原生 Session 恢复、跨 node 内存恢复或任意崩溃后的持久性。 + +所选部署代次使用外部工作区存储时，Core 要求所选 Harness profile 支持 `retained_native_history`。创建流程在持有部署锁时校验该代次，然后才提交 Session 或计算预留。既有 Session 的 Runtime 准入和最终执行检查以不可变工作区绑定为依据。自带工作区存储不要求此能力。 diff --git a/internal/agentdaemon/proto/inbound.go b/internal/agentdaemon/proto/inbound.go index ed236518f..dfbc36f2d 100644 --- a/internal/agentdaemon/proto/inbound.go +++ b/internal/agentdaemon/proto/inbound.go @@ -158,6 +158,10 @@ type AgentKindCapabilities struct { Usage CapabilitySupport `json:"usage"` Resume CapabilitySupport `json:"resume"` NativeSessionRecovery CapabilitySupport `json:"native_session_recovery"` + // RetainedNativeHistory resumes the same native Session after confirmed + // process shutdown using its retained private state, without replaying input. + // It does not promise process memory or external connection restoration. + RetainedNativeHistory CapabilitySupport `json:"retained_native_history"` Steering CapabilitySupport `json:"steering"` MessageItems CapabilitySupport `json:"message_items"` diff --git a/internal/agentdaemon/proto/prototest/capabilities.go b/internal/agentdaemon/proto/prototest/capabilities.go index 82afd39b5..69cabdaf0 100644 --- a/internal/agentdaemon/proto/prototest/capabilities.go +++ b/internal/agentdaemon/proto/prototest/capabilities.go @@ -17,6 +17,7 @@ func Capabilities(overrides proto.AgentKindCapabilities) proto.AgentKindCapabili Usage: proto.CapabilityUnsupported, Resume: proto.CapabilityUnsupported, NativeSessionRecovery: proto.CapabilityUnsupported, + RetainedNativeHistory: proto.CapabilityUnsupported, Steering: proto.CapabilityUnsupported, MessageItems: proto.CapabilityUnsupported, ToolObservations: proto.CapabilityUnsupported, diff --git a/internal/agentdaemon/proto/suspend.go b/internal/agentdaemon/proto/suspend.go index 053b6fa46..0e0ad7896 100644 --- a/internal/agentdaemon/proto/suspend.go +++ b/internal/agentdaemon/proto/suspend.go @@ -17,6 +17,9 @@ type EnvironmentSuspendPayload struct { SuspendID string `json:"suspend_id"` } +// An accepted quiesce confirms closed admission, drained work and receipts, +// and successful closure of idle native Executors. It does not confirm a +// filesystem checkpoint or that provider-owned compute has stopped. type EnvironmentSuspendResultPayload struct { EnvironmentID string `json:"environment_id"` SuspendID string `json:"suspend_id"` diff --git a/internal/agentdaemon/proto/version.go b/internal/agentdaemon/proto/version.go index 524883ee6..7f6f11783 100644 --- a/internal/agentdaemon/proto/version.go +++ b/internal/agentdaemon/proto/version.go @@ -2,7 +2,7 @@ package proto // Version identifies the complete Core–Runtime wire contract. Change it when // removing or changing a payload or its semantics; deploy both endpoints together. -const Version = "0.13.0" +const Version = "0.14.0" // VersionCompatible accepts only this contract. Patch drift, prerelease suffixes // and malformed versions do not select an implicit compatibility path. diff --git a/scripts/name-allowlist.json b/scripts/name-allowlist.json index c29a4637e..69b3837f1 100644 --- a/scripts/name-allowlist.json +++ b/scripts/name-allowlist.json @@ -263,5 +263,10 @@ "path": "scripts/ci_plan_test.py", "regex": "(?:example/parsar|@oac/parsar-example)", "reason": "The named Parsar example owns its dependency lock and component checks." + }, + { + "path": "apps/daemon/internal/cli/connect_suspend_test.go", + "regex": "\"agents-api-\"", + "reason": "The cleanup regression uses the persisted Session state-key prefix required by Runtime executor ownership validation." } ] diff --git a/services/core/deploy/claude/Dockerfile b/services/core/deploy/claude/Dockerfile index 115878e19..b4ce63d03 100644 --- a/services/core/deploy/claude/Dockerfile +++ b/services/core/deploy/claude/Dockerfile @@ -12,7 +12,8 @@ ENV HOME=/home/runtime OAC_RUNTIME_HOME=/home/runtime/.oac \ OAC_RUNTIME_CLAUDE_SDK_ENTRYPOINT=/opt/claude-sdk/dist/main.js \ OAC_RUNTIME_WORKSPACE=/environment/workspace \ OAC_RUNTIME_INITIALIZATION_DIRECTORY=/environment/initialization \ - OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages + OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages \ + OAC_RUNTIME_STATE_DIRECTORY=/environment/runtime-state USER 1000:1000 RUN node /opt/claude-sdk/dist/runtime_check.js /opt/claude-sdk/dist/main.js WORKDIR /environment/workspace diff --git a/services/core/deploy/claude/README.md b/services/core/deploy/claude/README.md index 327af0f16..f790d50b2 100644 --- a/services/core/deploy/claude/README.md +++ b/services/core/deploy/claude/README.md @@ -14,7 +14,7 @@ Claude accepts only the `anthropic` model protocol ([`harnessconfig/claudesdk`]( ## Native state and recovery -With a workspace binding, the SDK's history, home and scratch directories are `history`, `home` and `scratch` under `$OAC_RUNTIME_HOME/runtime/claude-sdk/` (mode 0700; `OAC_RUNTIME_HOME` defaults to `~/.oac`). The daemon's authentication stays under `$OAC_RUNTIME_HOME/daemon/`. These locations keep Session state apart; they do not restrict the tools. +With a workspace binding, the SDK stores retained history at `/runtime/claude-sdk/history`, where the Runtime supplies `` from the [state directory setting](../../../../docs/configuration.md#runtime-resource-directories). Without a workspace binding, native state uses `/daemon//runtime/claude-sdk`. Private launch home and scratch directories for workspace execution stay at `$OAC_RUNTIME_HOME/runtime/claude-sdk/home` and `$OAC_RUNTIME_HOME/runtime/claude-sdk/scratch` (mode 0700). The daemon's authentication stays under `$OAC_RUNTIME_HOME/daemon/`. These locations keep native history, temporary process state and device credentials separate; they do not restrict the tools. Recovery uses the SDK's history APIs ([`recovery.ts`](../../../../packages/claude-sdk-adapter/src/recovery.ts)). An explicitly supplied native Session ID must exist. When Core requires existing history but has no recorded ID, the adapter accepts only a single native Session whose recorded cwd equals the bound workspace and which has at least one message. Missing, foreign, ambiguous or empty history is rejected before any model input. @@ -43,7 +43,7 @@ The bridge ([`native_failure.ts`](../../../../packages/claude-sdk-adapter/src/na | Base | Digest-pinned `node:22.23.1-bookworm-slim` with `ca-certificates`, `bash`, `git`, `python3`, `python3-pip` and `ripgrep` | | Programs | `/usr/local/bin/oac-daemon` and the SDK bundle at `/opt/claude-sdk` | | User | UID/GID 1000 with `HOME=/home/runtime` | -| Environment | `OAC_RUNTIME_HOME=/home/runtime/.oac`, `OAC_RUNTIME_CLAUDE_SDK_NODE=/usr/local/bin/node`, `OAC_RUNTIME_CLAUDE_SDK_ENTRYPOINT=/opt/claude-sdk/dist/main.js`, `OAC_RUNTIME_WORKSPACE=/environment/workspace`, `OAC_RUNTIME_INITIALIZATION_DIRECTORY=/environment/initialization`, `OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages` | +| Environment | `OAC_RUNTIME_HOME=/home/runtime/.oac`, `OAC_RUNTIME_CLAUDE_SDK_NODE=/usr/local/bin/node`, `OAC_RUNTIME_CLAUDE_SDK_ENTRYPOINT=/opt/claude-sdk/dist/main.js`, `OAC_RUNTIME_WORKSPACE=/environment/workspace`, `OAC_RUNTIME_INITIALIZATION_DIRECTORY=/environment/initialization`, `OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages`, `OAC_RUNTIME_STATE_DIRECTORY=/environment/runtime-state` | | Entry point | `oac-daemon connect --profile default`, working directory `/environment/workspace` | The build runs the bundle's `runtime_check.js` against its entry point. The distribution copies `/opt/claude-sdk` into the combined Runtime image. Sandboxes run the image with the [Docker sandbox settings](../../../../docs/sandbox-provider.md#docker-adapter). diff --git a/services/core/deploy/codex/Dockerfile b/services/core/deploy/codex/Dockerfile index e79a93cd5..4dc401212 100644 --- a/services/core/deploy/codex/Dockerfile +++ b/services/core/deploy/codex/Dockerfile @@ -14,7 +14,8 @@ ENV HOME=/home/runtime OAC_RUNTIME_HOME=/home/runtime/.oac \ OAC_RUNTIME_CODEX_BIN=/usr/local/bin/codex \ OAC_RUNTIME_WORKSPACE=/environment/workspace \ OAC_RUNTIME_INITIALIZATION_DIRECTORY=/environment/initialization \ - OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages + OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages \ + OAC_RUNTIME_STATE_DIRECTORY=/environment/runtime-state USER 1000:1000 RUN test "$(codex --version)" = "codex-cli 0.153.4" WORKDIR /environment/workspace diff --git a/services/core/deploy/codex/README.md b/services/core/deploy/codex/README.md index ac0c9c78b..645908565 100644 --- a/services/core/deploy/codex/README.md +++ b/services/core/deploy/codex/README.md @@ -12,7 +12,7 @@ The adapter accepts only Codex `0.153.4`: installation and recovery checks requi Codex accepts only the `responses` model protocol ([`harnessconfig/codex`](../../../../internal/harnessconfig/codex/configuration.go)). A Chat Completions or Anthropic provider is rejected; nothing converts between protocols. [Model execution](../../../../contracts/agents-api/model-execution.md#deployment-defaults) owns provider selection, including the per-Harness deployment default. -Each Session has its own `CODEX_HOME` at `$OAC_RUNTIME_HOME/daemon/agent-sessions//` (`OAC_RUNTIME_HOME` defaults to `~/.oac`). Native history stays there. The adapter regenerates that directory's `config.toml` on every prompt: the Session's frozen provider bundle becomes the `[model_providers.oac]` block, with `wire_api` `responses`, and the thread is pinned to that provider, so Codex never falls back to its built-in `openai` provider. +Each Session has its own `CODEX_HOME` at `/daemon/agent-sessions//`, where the Runtime supplies `` from the [state directory setting](../../../../docs/configuration.md#runtime-resource-directories). Native history stays there; Runtime device credentials stay in the Runtime home. The state layout alone does not qualify cross-node native continuation. The adapter regenerates that directory's `config.toml` on every prompt: the Session's frozen provider bundle becomes the `[model_providers.oac]` block, with `wire_api` `responses`, and the thread is pinned to that provider, so Codex never falls back to its built-in `openai` provider. Provider bearer credentials use the native `env_key` reference and enter the model-loop child process environment; they are not written into retained `config.toml`. The adapter excludes that variable from native shell environments with `shell_environment_policy.exclude` and disables `shell_snapshot`, whose capture in the pinned native version can otherwise persist the model-loop environment. This retains native thread history without retaining native shell snapshots; it introduces no user setting. ## Execution controls @@ -66,7 +66,7 @@ Every other variant stays unclassified. | Base | Digest-pinned `node:22.23.1-bookworm-slim` with `ca-certificates`, `bash`, `git`, `python3`, `python3-pip` and `ripgrep`, the same base and package layer as the Claude and MiniMax images | | Programs | `/usr/local/bin/oac-daemon`, `/usr/local/bin/codex` (mode 0555) and `/usr/local/codex-resources` | | User | UID/GID 1000 with `HOME=/home/runtime` | -| Environment | `OAC_RUNTIME_HOME=/home/runtime/.oac`, `OAC_RUNTIME_CODEX_BIN=/usr/local/bin/codex`, `OAC_RUNTIME_WORKSPACE=/environment/workspace`, `OAC_RUNTIME_INITIALIZATION_DIRECTORY=/environment/initialization`, `OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages` | +| Environment | `OAC_RUNTIME_HOME=/home/runtime/.oac`, `OAC_RUNTIME_CODEX_BIN=/usr/local/bin/codex`, `OAC_RUNTIME_WORKSPACE=/environment/workspace`, `OAC_RUNTIME_INITIALIZATION_DIRECTORY=/environment/initialization`, `OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages`, `OAC_RUNTIME_STATE_DIRECTORY=/environment/runtime-state` | | Entry point | `oac-daemon connect --profile default`, working directory `/environment/workspace` | The build fails unless `codex --version` reports the pinned version. The image holds no credentials, workspace data or product software. The distribution copies the Codex executable and resources into the combined Runtime image; see the [maintainer guide](../../../../docs/maintainers.md#runtime-images-and-helpers). diff --git a/services/core/deploy/mcode/Dockerfile b/services/core/deploy/mcode/Dockerfile index 774c1be79..15aa27e74 100644 --- a/services/core/deploy/mcode/Dockerfile +++ b/services/core/deploy/mcode/Dockerfile @@ -13,7 +13,8 @@ ENV HOME=/home/runtime OAC_RUNTIME_HOME=/home/runtime/.oac \ OAC_RUNTIME_MCODE_WORKSPACE_BRIDGE=/opt/mcode-harness/bridge.mjs \ OAC_RUNTIME_WORKSPACE=/environment/workspace \ OAC_RUNTIME_INITIALIZATION_DIRECTORY=/environment/initialization \ - OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages + OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages \ + OAC_RUNTIME_STATE_DIRECTORY=/environment/runtime-state USER 1000:1000 RUN node /opt/mcode-harness/check.mjs && /opt/mcode-harness/native/cli.js --version WORKDIR /environment/workspace diff --git a/services/core/deploy/mcode/README.md b/services/core/deploy/mcode/README.md index 58cbbcea0..9272bd8a3 100644 --- a/services/core/deploy/mcode/README.md +++ b/services/core/deploy/mcode/README.md @@ -16,7 +16,7 @@ MiniMax Code accepts the `anthropic`, `responses` and `chat_completions` protoco ## Execution profiles -Each Session has its own native data directory, `$OAC_RUNTIME_HOME/runtime/mcode/state//` (`OAC_RUNTIME_HOME` defaults to `~/.oac`), holding the generated native configuration, instructions (`AGENTS.md`, at most 32 KiB), MCP configuration and history. The native process gets `MINIMAX_DATA_DIR`, `HOME` and `USERPROFILE` set to that directory on top of the daemon user's environment. +Each Session has its own native data directory, `$OAC_RUNTIME_HOME/runtime/mcode/state//` (`OAC_RUNTIME_HOME` defaults to `~/.oac`), holding the generated native configuration, instructions (`AGENTS.md`, at most 32 KiB), MCP configuration and history. The native process gets `MINIMAX_DATA_DIR`, `HOME` and `USERPROFILE` set to that directory on top of the daemon user's environment. This directory remains private to the compute instance: the adapter does not declare retained native history across compute replacement. | Profile | Where it runs | Native configuration | | --- | --- | --- | @@ -47,7 +47,7 @@ In the workspace profile, the frozen installation's Skills are linked into the S | Base | Digest-pinned `node:22.23.1-bookworm-slim` with `ca-certificates`, `bash`, `git`, `python3`, `python3-pip` and `ripgrep` | | Programs | `/usr/local/bin/oac-daemon` and the companion at `/opt/mcode-harness` (native CLI at `native/cli.js`, bridge at `bridge.mjs`) | | User | UID/GID 1000 with `HOME=/home/runtime` | -| Environment | `OAC_RUNTIME_HOME=/home/runtime/.oac`, `OAC_RUNTIME_MCODE_NODE`, `OAC_RUNTIME_MCODE_BIN`, `OAC_RUNTIME_MCODE_WORKSPACE_BRIDGE`, `OAC_RUNTIME_WORKSPACE=/environment/workspace`, `OAC_RUNTIME_INITIALIZATION_DIRECTORY=/environment/initialization`, `OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages` | +| Environment | `OAC_RUNTIME_HOME=/home/runtime/.oac`, `OAC_RUNTIME_MCODE_NODE`, `OAC_RUNTIME_MCODE_BIN`, `OAC_RUNTIME_MCODE_WORKSPACE_BRIDGE`, `OAC_RUNTIME_WORKSPACE=/environment/workspace`, `OAC_RUNTIME_INITIALIZATION_DIRECTORY=/environment/initialization`, `OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages`, `OAC_RUNTIME_STATE_DIRECTORY=/environment/runtime-state` | | Entry point | `oac-daemon connect --profile default`, working directory `/environment/workspace` | The build runs the companion's `check.mjs` and the native `--version`. The combined Runtime image uses this image as its base. Sandboxes run it with the [Docker sandbox settings](../../../../docs/sandbox-provider.md#docker-adapter). diff --git a/services/core/internal/api/sandbox_deployment_setup.go b/services/core/internal/api/sandbox_deployment_setup.go index e24224db3..8b2b8f77a 100644 --- a/services/core/internal/api/sandbox_deployment_setup.go +++ b/services/core/internal/api/sandbox_deployment_setup.go @@ -108,6 +108,7 @@ func (h *Handler) updateSandboxDeployment(w http.ResponseWriter, r *http.Request writeDeploymentError(w, r, err) return } + setAdminAuditSource(r, "") result, err := h.Sandboxes.DeploymentChanges.UpdateSandboxDeployment(r.Context(), selection) if err != nil { writeDeploymentError(w, r, err) diff --git a/services/core/internal/db/queries/admin_session_archive.sql b/services/core/internal/db/queries/admin_session_archive.sql index 09b58224d..3562ee949 100644 --- a/services/core/internal/db/queries/admin_session_archive.sql +++ b/services/core/internal/db/queries/admin_session_archive.sql @@ -9,5 +9,5 @@ SELECT s.id AS session_id, e.id AS environment_id, ELSE 'active' END::text AS state FROM sessions s JOIN environments e ON e.session_id = s.id -LEFT JOIN runtime_allocations a ON a.environment_id = e.id +LEFT JOIN runtime_allocations a ON a.environment_id = e.id AND a.state <> 'released' WHERE s.tenant_id = $1 AND s.id = $2 AND s.deleted_at IS NULL; diff --git a/services/core/internal/db/queries/environments.sql b/services/core/internal/db/queries/environments.sql index 38fd0896f..2e122d372 100644 --- a/services/core/internal/db/queries/environments.sql +++ b/services/core/internal/db/queries/environments.sql @@ -4,11 +4,13 @@ VALUES ($1, $2, CASE WHEN EXISTS (SELECT 1 FROM initial_environment_files WHERE OR EXISTS (SELECT 1 FROM environment_setups WHERE session_id = $2) THEN 'pending' ELSE 'complete' END); -- name: GetEnvironment :one -SELECT sqlc.embed(e), s.tenant_id, (s.configuration->'environment')::jsonb AS configuration +SELECT sqlc.embed(e), s.tenant_id, (s.configuration->'environment')::jsonb AS configuration, + EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id = e.id)::boolean AS external_workspace FROM environments e JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $1 AND e.id = $2 AND s.deleted_at IS NULL; -- name: GetSessionEnvironment :one -SELECT sqlc.embed(e), s.tenant_id, (s.configuration->'environment')::jsonb AS configuration +SELECT sqlc.embed(e), s.tenant_id, (s.configuration->'environment')::jsonb AS configuration, + EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id = e.id)::boolean AS external_workspace FROM environments e JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $1 AND s.id = $2 AND s.deleted_at IS NULL; diff --git a/services/core/internal/db/queries/local_environment_devices.sql b/services/core/internal/db/queries/local_environment_devices.sql index 75d50bfc5..d7c756b0a 100644 --- a/services/core/internal/db/queries/local_environment_devices.sql +++ b/services/core/internal/db/queries/local_environment_devices.sql @@ -4,5 +4,27 @@ SELECT sqlc.arg(id), s.tenant_id, sqlc.arg(name), sqlc.arg(credential_hash), e.i FROM environments e JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = sqlc.arg(tenant_id) AND s.id = sqlc.arg(session_id) AND e.id = sqlc.arg(environment_id) AND s.deleted_at IS NULL AND s.configuration->'environment'->>'type' = 'openai_hosted' -ON CONFLICT (environment_id) DO NOTHING +AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') +AND NOT EXISTS (SELECT 1 FROM devices previous WHERE previous.environment_id = e.id AND (previous.revoked_at IS NULL OR previous.executor_key_id IS NOT NULL)) +ON CONFLICT (environment_id) WHERE revoked_at IS NULL OR executor_key_id IS NOT NULL DO NOTHING RETURNING id; + +-- name: BindHostedSessionDevice :one +INSERT INTO session_devices(session_id, device_id) +SELECT s.id, d.id FROM sessions s JOIN devices d ON d.tenant_id = s.tenant_id +JOIN environments e ON e.id = d.environment_id AND e.session_id = s.id +WHERE s.tenant_id = $1 AND s.id = $2 AND d.id = $3 AND d.revoked_at IS NULL + AND d.executor_key_id IS NULL AND s.deleted_at IS NULL + AND s.configuration->'environment'->>'type' = 'openai_hosted' +ON CONFLICT (session_id) DO UPDATE SET device_id = EXCLUDED.device_id +WHERE session_devices.device_id = EXCLUDED.device_id OR ( + EXISTS (SELECT 1 FROM devices previous + JOIN runtime_allocations a ON a.device_id = previous.id + WHERE previous.id = session_devices.device_id AND previous.revoked_at IS NOT NULL + AND previous.executor_key_id IS NULL AND a.state = 'released' + AND previous.environment_id = (SELECT environment_id FROM devices WHERE id = EXCLUDED.device_id)) + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a + WHERE a.environment_id = (SELECT environment_id FROM devices WHERE id = EXCLUDED.device_id) + AND a.state <> 'released') +) +RETURNING device_id; diff --git a/services/core/internal/db/queries/runtime_allocations.sql b/services/core/internal/db/queries/runtime_allocations.sql index 6902e54ab..a33ccc532 100644 --- a/services/core/internal/db/queries/runtime_allocations.sql +++ b/services/core/internal/db/queries/runtime_allocations.sql @@ -2,12 +2,13 @@ INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) VALUES ($1, $2, $3, $4, $5, $6) RETURNING *; --- name: GetRuntimeAllocation :one +-- name: GetLatestRuntimeAllocation :one SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id -WHERE s.tenant_id = $1 AND a.environment_id = $2; +WHERE s.tenant_id = $1 AND a.environment_id = $2 +ORDER BY a.created_at DESC, a.id DESC LIMIT 1; -- name: ListRuntimeAllocations :many SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired @@ -21,7 +22,7 @@ ORDER BY a.id LIMIT 32; SELECT s.id, s.tenant_id FROM sessions s LEFT JOIN environments e ON e.session_id = s.id -LEFT JOIN runtime_allocations a ON a.environment_id = e.id +LEFT JOIN runtime_allocations a ON a.environment_id = e.id AND a.state <> 'released' WHERE s.id > $1 AND s.deleted_at IS NULL AND s.configuration->'environment'->>'type' = 'openai_hosted' @@ -45,3 +46,26 @@ WHERE id = $1 AND state <> 'released' RETURNING *; -- name: ReleaseRuntimeAllocation :one UPDATE runtime_allocations SET state = 'released', released_at = clock_timestamp() WHERE id = $1 AND state = 'cleanup_pending' AND create_settled RETURNING *; + +-- name: CanReplaceRuntimeAllocation :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations a JOIN devices d ON d.id = a.device_id AND d.environment_id = a.environment_id + WHERE a.id = $1 AND a.state = 'released' AND a.create_settled + AND d.revoked_at IS NOT NULL AND d.executor_key_id IS NULL + AND NOT EXISTS (SELECT 1 FROM runtime_allocations current WHERE current.environment_id = a.environment_id AND current.state <> 'released') + AND NOT EXISTS (SELECT 1 FROM devices current WHERE current.environment_id = a.environment_id AND current.revoked_at IS NULL) +)::boolean; + +-- name: CanRetainRuntimeEnvironment :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations a + JOIN devices d ON d.id = a.device_id AND d.environment_id = a.environment_id + JOIN environments e ON e.id = a.environment_id + JOIN sessions s ON s.id = e.session_id + JOIN environment_workspaces w ON w.environment_id = e.id AND w.state = 'ready' + WHERE a.id = $1 AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND e.initialization = 'complete' + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND d.supported_agent_kinds @> jsonb_build_array(jsonb_build_object( + 'kind', s.engine, 'available', true, 'capabilities', jsonb_build_object('retained_native_history', true))) +)::boolean; diff --git a/services/core/internal/db/queries/runtime_deployment.sql b/services/core/internal/db/queries/runtime_deployment.sql index 382d55726..332d6bbc3 100644 --- a/services/core/internal/db/queries/runtime_deployment.sql +++ b/services/core/internal/db/queries/runtime_deployment.sql @@ -4,10 +4,13 @@ SELECT * FROM runtime_deployment WHERE singleton = true FOR UPDATE; -- name: CountRuntimeDeploymentResources :one SELECT (SELECT count(*) FROM runtime_allocations WHERE state <> 'released')::bigint AS allocations, -(SELECT count(*) FROM environments e JOIN sessions s ON s.id = e.session_id - WHERE s.deleted_at IS NULL AND e.status = 'pending' +((SELECT count(*) FROM environments e JOIN sessions s ON s.id = e.session_id + WHERE s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND ((e.status = 'pending' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id)) + OR EXISTS (SELECT 1 FROM runtime_placements p WHERE p.environment_id = e.id AND p.released_at IS NULL)) AND s.configuration->'environment'->>'type' = 'openai_hosted' - AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id))::bigint AS pending; + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released'))::bigint + + (SELECT count(*) FROM runtime_reset_retained_environments)::bigint)::bigint AS pending; -- name: CountAddressBindings :one SELECT diff --git a/services/core/internal/db/queries/runtime_enrollment.sql b/services/core/internal/db/queries/runtime_enrollment.sql index 9fadd340c..0c68f0bcf 100644 --- a/services/core/internal/db/queries/runtime_enrollment.sql +++ b/services/core/internal/db/queries/runtime_enrollment.sql @@ -15,12 +15,12 @@ FOR SHARE OF c; -- name: EnrollRuntimeDevice :one INSERT INTO devices (id, tenant_id, name, environment_id, executor_key_id) VALUES (sqlc.arg(id), sqlc.arg(tenant_id), 'User-managed Runtime', sqlc.arg(environment_id), sqlc.arg(executor_key_id)) -ON CONFLICT (environment_id) DO UPDATE SET name = devices.name +ON CONFLICT (environment_id) WHERE revoked_at IS NULL OR executor_key_id IS NOT NULL DO UPDATE SET name = devices.name WHERE devices.executor_key_id = EXCLUDED.executor_key_id AND devices.revoked_at IS NULL RETURNING id, name, environment_id; -- name: TouchAuthenticatedDevice :execrows -UPDATE devices SET last_seen_at = clock_timestamp() +UPDATE devices SET last_seen_at = clock_timestamp(), supported_agent_kinds = sqlc.arg(supported_agent_kinds)::jsonb WHERE devices.id = sqlc.arg(id) AND EXISTS ( SELECT 1 FROM runtime_device_authority a WHERE a.id = devices.id AND a.credential_hash = sqlc.arg(credential_hash) diff --git a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql index a30354cbd..93092ac4c 100644 --- a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql +++ b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql @@ -21,16 +21,31 @@ LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM sqlc.narg(node_id)::uuid AND p.released_at IS NULL AND (SELECT reset_clear IS NULL FROM runtime_deployment) - AND e.id > sqlc.arg(after_id)::uuid AND s.deleted_at IS NULL AND e.status='pending' + AND e.id > sqlc.arg(after_id)::uuid AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') AND s.configuration->'environment'->>'type'='openai_hosted' - AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id AND a.state<>'released') ORDER BY e.id LIMIT 32; -- name: GetRuntimeLifecyclePlacement :one SELECT d.provider_kind, d.mode, a.id AS allocation_id, a.node_id AS allocation_node_id, - p.node_id AS placement_node_id, p.released_at + p.node_id AS placement_node_id, p.released_at, + CASE WHEN COALESCE(a.deployment_generation, p.deployment_generation, d.generation)=d.generation + THEN d.specification ELSE g.specification END::jsonb AS specification FROM environments e JOIN sessions s ON s.id=e.session_id CROSS JOIN runtime_deployment d -LEFT JOIN runtime_allocations a ON a.environment_id=e.id +LEFT JOIN runtime_allocations a ON a.environment_id=e.id AND a.state<>'released' LEFT JOIN runtime_placements p ON p.environment_id=e.id +LEFT JOIN runtime_deployment_generations g ON g.generation=COALESCE(a.deployment_generation, p.deployment_generation, d.generation) WHERE s.tenant_id=$1 AND e.id=$2; + +-- name: ListReplacementEnvironments :many +SELECT e.id, s.tenant_id +FROM environments e JOIN sessions s ON s.id = e.session_id +JOIN environment_workspaces w ON w.environment_id = e.id AND w.state = 'ready' +WHERE e.id > $1 AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND e.initialization = 'complete' AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND (SELECT reset_clear IS NULL FROM runtime_deployment) + AND EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state = 'released') + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') + AND EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending') +ORDER BY e.id LIMIT 32; diff --git a/services/core/internal/db/queries/runtime_nodes.sql b/services/core/internal/db/queries/runtime_nodes.sql index b6c45b2ce..6a24dbb20 100644 --- a/services/core/internal/db/queries/runtime_nodes.sql +++ b/services/core/internal/db/queries/runtime_nodes.sql @@ -14,9 +14,9 @@ SELECT n.*, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND EXISTS(SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=n.ready_generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch AND g.state='ready')::boolean AS serving_ready, COALESCE((SELECT g.state FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_state, COALESCE((SELECT g.diagnostic FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_diagnostic, - (SELECT count(*) FROM runtime_placements p LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.node_id=n.id AND p.released_at IS NULL AND (a.id IS NULL OR a.compute_phase <> 'suspended'))::bigint AS active, + (SELECT count(*) FROM runtime_placements p LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id AND a.state<>'released' WHERE p.node_id=n.id AND p.released_at IS NULL AND (a.id IS NULL OR a.compute_phase <> 'suspended'))::bigint AS active, (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL)::bigint AS retained, - (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id))::bigint AS reserved, + (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id AND a.state<>'released'))::bigint AS reserved, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='cleanup_pending')::bigint AS cleanup_pending, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='running' AND a.compute_phase IN('running','disabled'))::bigint AS running, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'snapshot' IS NOT NULL AND a.compute_state->'snapshot'<>'null'::jsonb)::bigint AS snapshots @@ -63,7 +63,7 @@ INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) VAL SELECT p.*, n.name, (EXISTS(SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=p.deployment_generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch AND g.state='ready') AND n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND n.removed_at IS NULL)::boolean AS available, COALESCE(a.observation_error,'')::text AS observation_error, COALESCE(a.state,'reserved')::text AS state, COALESCE(a.compute_phase,'disabled')::text AS compute_phase FROM runtime_placements p JOIN runtime_nodes n ON n.id=p.node_id CROSS JOIN runtime_deployment d -LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.environment_id=$1; +LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id AND a.state<>'released' WHERE p.environment_id=$1; -- name: ReleaseRuntimePlacement :exec UPDATE runtime_placements SET released_at=COALESCE(released_at,clock_timestamp()) WHERE environment_id=$1; @@ -71,7 +71,7 @@ UPDATE runtime_placements SET released_at=COALESCE(released_at,clock_timestamp() -- name: ReleaseUnallocatedRuntimePlacement :exec UPDATE runtime_placements SET released_at=COALESCE(released_at,clock_timestamp()) WHERE environment_id IN(SELECT id FROM environments WHERE session_id=$1) -AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=runtime_placements.environment_id); +AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=runtime_placements.environment_id AND a.state<>'released'); -- name: ListNodeRuntimeAllocations :many SELECT a.id,a.node_id,a.deployment_generation,a.observation_error,a.state,a.compute_phase,a.compute_phase_changed_at,e.initialization,a.created_at,a.environment_id,e.session_id,s.tenant_id @@ -84,3 +84,9 @@ SELECT id,$2,sqlc.arg(generation)::bigint FROM environments WHERE session_id=$1; -- name: SetRuntimeObservation :exec UPDATE runtime_allocations SET observation_error=$4 WHERE id=$1 AND compute_revision=$2 AND state=$3 AND state<>'released'; + +-- name: ReserveReleasedRuntimePlacement :execrows +UPDATE runtime_placements SET node_id = $2, deployment_generation = $3, + reserved_at = clock_timestamp(), released_at = NULL +WHERE runtime_placements.environment_id = $1 AND released_at IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = $1 AND a.state <> 'released'); diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index 83b5a38ca..4825047da 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -24,7 +24,7 @@ WHERE id = $1 AND compute_phase = 'running' AND compute_activity_at <= $2; -- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, - GREATEST(a.compute_activity_at, + GREATEST(a.compute_activity_at, a.compute_phase_changed_at, CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, @@ -32,15 +32,14 @@ SELECT clock_timestamp()::timestamptz AS observed_at, OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = e.session_id AND r.state = 'pending') OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = e.id AND f.state = 'pending'))::boolean AS busy, - a.compute_wake_requested, - EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.completed_at IS NOT NULL)::boolean AS has_completed_turn + a.compute_wake_requested FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE a.id = $1; -- name: RuntimeComputeBlocksAdmission :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id - WHERE e.session_id = $1 AND a.compute_phase NOT IN ('disabled', 'running') + WHERE e.session_id = $1 AND a.state <> 'released' AND a.compute_phase NOT IN ('disabled', 'running') )::boolean; -- name: RecordRuntimeTerminalActivity :exec @@ -52,5 +51,5 @@ WHERE a.environment_id = e.id AND e.session_id = $1 -- name: SessionHasRuntimeNode :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id - WHERE e.session_id = $1 AND a.node_id IS NOT NULL + WHERE e.session_id = $1 AND a.state <> 'released' AND a.node_id IS NOT NULL )::boolean; diff --git a/services/core/internal/db/queries/sandbox_reset.sql b/services/core/internal/db/queries/sandbox_reset.sql index 536219525..5e5132e96 100644 --- a/services/core/internal/db/queries/sandbox_reset.sql +++ b/services/core/internal/db/queries/sandbox_reset.sql @@ -43,7 +43,9 @@ WHERE s.deleted_at IS NULL AND s.configuration->'environment'->>'type' = 'openai AND e.status NOT IN ('failed', 'expired') AND s.id > sqlc.arg(after_id)::uuid AND (EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state NOT IN ('released', 'cleanup_pending')) - OR (e.status = 'pending' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id))) + OR (e.status = 'pending' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id)) + OR EXISTS (SELECT 1 FROM runtime_placements p WHERE p.environment_id = e.id AND p.released_at IS NULL) + OR EXISTS (SELECT 1 FROM runtime_reset_retained_environments r WHERE r.environment_id = e.id)) AND (sqlc.arg(force)::boolean OR NOT ( EXISTS (SELECT 1 FROM turns t WHERE t.session_id = s.id AND t.status IN ('in_progress', 'waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = s.id AND t.status IN ('in_progress', 'waiting')) @@ -69,9 +71,13 @@ held AS ( SELECT p.deployment_generation, p.node_id, s.id, e.id, true, false FROM environments e JOIN sessions s ON s.id = e.session_id LEFT JOIN runtime_placements p ON p.environment_id = e.id AND p.released_at IS NULL - WHERE s.deleted_at IS NULL AND e.status = 'pending' + WHERE s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND ((e.status = 'pending' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id)) OR p.environment_id IS NOT NULL) AND s.configuration->'environment'->>'type' = 'openai_hosted' - AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id) + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') + UNION ALL + SELECT r.deployment_generation, NULL::uuid, r.session_id, r.environment_id, true, false + FROM runtime_reset_retained_environments r ), classified AS ( SELECT h.*, CASE WHEN h.cleanup THEN 'cleanup' WHEN EXISTS (SELECT 1 FROM turns t WHERE t.session_id = h.session_id AND t.status IN ('in_progress', 'waiting')) diff --git a/services/core/internal/db/sqlc/admin_session_archive.sql.go b/services/core/internal/db/sqlc/admin_session_archive.sql.go index 144700226..f494fadfd 100644 --- a/services/core/internal/db/sqlc/admin_session_archive.sql.go +++ b/services/core/internal/db/sqlc/admin_session_archive.sql.go @@ -22,7 +22,7 @@ SELECT s.id AS session_id, e.id AS environment_id, ELSE 'active' END::text AS state FROM sessions s JOIN environments e ON e.session_id = s.id -LEFT JOIN runtime_allocations a ON a.environment_id = e.id +LEFT JOIN runtime_allocations a ON a.environment_id = e.id AND a.state <> 'released' WHERE s.tenant_id = $1 AND s.id = $2 AND s.deleted_at IS NULL ` diff --git a/services/core/internal/db/sqlc/environments.sql.go b/services/core/internal/db/sqlc/environments.sql.go index f1c3c45e9..6a8097c94 100644 --- a/services/core/internal/db/sqlc/environments.sql.go +++ b/services/core/internal/db/sqlc/environments.sql.go @@ -28,7 +28,8 @@ func (q *Queries) CreateEnvironment(ctx context.Context, arg CreateEnvironmentPa } const getEnvironment = `-- name: GetEnvironment :one -SELECT e.id, e.session_id, e.status, e.created_at, e.failure_reason, e.failed_at, e.failure_detail, e.initialization, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration +SELECT e.id, e.session_id, e.status, e.created_at, e.failure_reason, e.failed_at, e.failure_detail, e.initialization, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration, + EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id = e.id)::boolean AS external_workspace FROM environments e JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $1 AND e.id = $2 AND s.deleted_at IS NULL ` @@ -39,9 +40,10 @@ type GetEnvironmentParams struct { } type GetEnvironmentRow struct { - Environment Environment `json:"environment"` - TenantID pgtype.UUID `json:"tenant_id"` - Configuration []byte `json:"configuration"` + Environment Environment `json:"environment"` + TenantID pgtype.UUID `json:"tenant_id"` + Configuration []byte `json:"configuration"` + ExternalWorkspace bool `json:"external_workspace"` } func (q *Queries) GetEnvironment(ctx context.Context, arg GetEnvironmentParams) (GetEnvironmentRow, error) { @@ -58,12 +60,14 @@ func (q *Queries) GetEnvironment(ctx context.Context, arg GetEnvironmentParams) &i.Environment.Initialization, &i.TenantID, &i.Configuration, + &i.ExternalWorkspace, ) return i, err } const getSessionEnvironment = `-- name: GetSessionEnvironment :one -SELECT e.id, e.session_id, e.status, e.created_at, e.failure_reason, e.failed_at, e.failure_detail, e.initialization, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration +SELECT e.id, e.session_id, e.status, e.created_at, e.failure_reason, e.failed_at, e.failure_detail, e.initialization, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration, + EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id = e.id)::boolean AS external_workspace FROM environments e JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $1 AND s.id = $2 AND s.deleted_at IS NULL ` @@ -74,9 +78,10 @@ type GetSessionEnvironmentParams struct { } type GetSessionEnvironmentRow struct { - Environment Environment `json:"environment"` - TenantID pgtype.UUID `json:"tenant_id"` - Configuration []byte `json:"configuration"` + Environment Environment `json:"environment"` + TenantID pgtype.UUID `json:"tenant_id"` + Configuration []byte `json:"configuration"` + ExternalWorkspace bool `json:"external_workspace"` } func (q *Queries) GetSessionEnvironment(ctx context.Context, arg GetSessionEnvironmentParams) (GetSessionEnvironmentRow, error) { @@ -93,6 +98,7 @@ func (q *Queries) GetSessionEnvironment(ctx context.Context, arg GetSessionEnvir &i.Environment.Initialization, &i.TenantID, &i.Configuration, + &i.ExternalWorkspace, ) return i, err } diff --git a/services/core/internal/db/sqlc/local_environment_devices.sql.go b/services/core/internal/db/sqlc/local_environment_devices.sql.go index 72c89655c..09774b54d 100644 --- a/services/core/internal/db/sqlc/local_environment_devices.sql.go +++ b/services/core/internal/db/sqlc/local_environment_devices.sql.go @@ -11,13 +11,49 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const bindHostedSessionDevice = `-- name: BindHostedSessionDevice :one +INSERT INTO session_devices(session_id, device_id) +SELECT s.id, d.id FROM sessions s JOIN devices d ON d.tenant_id = s.tenant_id +JOIN environments e ON e.id = d.environment_id AND e.session_id = s.id +WHERE s.tenant_id = $1 AND s.id = $2 AND d.id = $3 AND d.revoked_at IS NULL + AND d.executor_key_id IS NULL AND s.deleted_at IS NULL + AND s.configuration->'environment'->>'type' = 'openai_hosted' +ON CONFLICT (session_id) DO UPDATE SET device_id = EXCLUDED.device_id +WHERE session_devices.device_id = EXCLUDED.device_id OR ( + EXISTS (SELECT 1 FROM devices previous + JOIN runtime_allocations a ON a.device_id = previous.id + WHERE previous.id = session_devices.device_id AND previous.revoked_at IS NOT NULL + AND previous.executor_key_id IS NULL AND a.state = 'released' + AND previous.environment_id = (SELECT environment_id FROM devices WHERE id = EXCLUDED.device_id)) + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a + WHERE a.environment_id = (SELECT environment_id FROM devices WHERE id = EXCLUDED.device_id) + AND a.state <> 'released') +) +RETURNING device_id +` + +type BindHostedSessionDeviceParams struct { + TenantID pgtype.UUID `json:"tenant_id"` + ID pgtype.UUID `json:"id"` + ID_2 pgtype.UUID `json:"id_2"` +} + +func (q *Queries) BindHostedSessionDevice(ctx context.Context, arg BindHostedSessionDeviceParams) (pgtype.UUID, error) { + row := q.db.QueryRow(ctx, bindHostedSessionDevice, arg.TenantID, arg.ID, arg.ID_2) + var device_id pgtype.UUID + err := row.Scan(&device_id) + return device_id, err +} + const createEnvironmentDevice = `-- name: CreateEnvironmentDevice :one INSERT INTO devices (id, tenant_id, name, credential_hash, environment_id) SELECT $1, s.tenant_id, $2, $3, e.id FROM environments e JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $4 AND s.id = $5 AND e.id = $6 AND s.deleted_at IS NULL AND s.configuration->'environment'->>'type' = 'openai_hosted' -ON CONFLICT (environment_id) DO NOTHING +AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') +AND NOT EXISTS (SELECT 1 FROM devices previous WHERE previous.environment_id = e.id AND (previous.revoked_at IS NULL OR previous.executor_key_id IS NOT NULL)) +ON CONFLICT (environment_id) WHERE revoked_at IS NULL OR executor_key_id IS NOT NULL DO NOTHING RETURNING id ` diff --git a/services/core/internal/db/sqlc/models.go b/services/core/internal/db/sqlc/models.go index 70dacf734..8fd13ca9b 100644 --- a/services/core/internal/db/sqlc/models.go +++ b/services/core/internal/db/sqlc/models.go @@ -64,6 +64,7 @@ type Device struct { EnvironmentID pgtype.UUID `json:"environment_id"` ExecutorKeyID pgtype.UUID `json:"executor_key_id"` ArchiveCancelTurnID pgtype.UUID `json:"archive_cancel_turn_id"` + SupportedAgentKinds []byte `json:"supported_agent_kinds"` } type Environment struct { @@ -351,6 +352,13 @@ type RuntimePlacement struct { DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` } +type RuntimeResetRetainedEnvironment struct { + EnvironmentID pgtype.UUID `json:"environment_id"` + SessionID pgtype.UUID `json:"session_id"` + TenantID pgtype.UUID `json:"tenant_id"` + DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` +} + type Session struct { ID pgtype.UUID `json:"id"` TenantID pgtype.UUID `json:"tenant_id"` diff --git a/services/core/internal/db/sqlc/runtime_allocations.sql.go b/services/core/internal/db/sqlc/runtime_allocations.sql.go index 4725ab1a1..3c85fc718 100644 --- a/services/core/internal/db/sqlc/runtime_allocations.sql.go +++ b/services/core/internal/db/sqlc/runtime_allocations.sql.go @@ -11,6 +11,45 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const canReplaceRuntimeAllocation = `-- name: CanReplaceRuntimeAllocation :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations a JOIN devices d ON d.id = a.device_id AND d.environment_id = a.environment_id + WHERE a.id = $1 AND a.state = 'released' AND a.create_settled + AND d.revoked_at IS NOT NULL AND d.executor_key_id IS NULL + AND NOT EXISTS (SELECT 1 FROM runtime_allocations current WHERE current.environment_id = a.environment_id AND current.state <> 'released') + AND NOT EXISTS (SELECT 1 FROM devices current WHERE current.environment_id = a.environment_id AND current.revoked_at IS NULL) +)::boolean +` + +func (q *Queries) CanReplaceRuntimeAllocation(ctx context.Context, id pgtype.UUID) (bool, error) { + row := q.db.QueryRow(ctx, canReplaceRuntimeAllocation, id) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + +const canRetainRuntimeEnvironment = `-- name: CanRetainRuntimeEnvironment :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations a + JOIN devices d ON d.id = a.device_id AND d.environment_id = a.environment_id + JOIN environments e ON e.id = a.environment_id + JOIN sessions s ON s.id = e.session_id + JOIN environment_workspaces w ON w.environment_id = e.id AND w.state = 'ready' + WHERE a.id = $1 AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND e.initialization = 'complete' + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND d.supported_agent_kinds @> jsonb_build_array(jsonb_build_object( + 'kind', s.engine, 'available', true, 'capabilities', jsonb_build_object('retained_native_history', true))) +)::boolean +` + +func (q *Queries) CanRetainRuntimeEnvironment(ctx context.Context, id pgtype.UUID) (bool, error) { + row := q.db.QueryRow(ctx, canRetainRuntimeEnvironment, id) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + const createRuntimeAllocation = `-- name: CreateRuntimeAllocation :one INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation @@ -58,20 +97,21 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime return i, err } -const getRuntimeAllocation = `-- name: GetRuntimeAllocation :one +const getLatestRuntimeAllocation = `-- name: GetLatestRuntimeAllocation :one SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.released_at, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $1 AND a.environment_id = $2 +ORDER BY a.created_at DESC, a.id DESC LIMIT 1 ` -type GetRuntimeAllocationParams struct { +type GetLatestRuntimeAllocationParams struct { TenantID pgtype.UUID `json:"tenant_id"` EnvironmentID pgtype.UUID `json:"environment_id"` } -type GetRuntimeAllocationRow struct { +type GetLatestRuntimeAllocationRow struct { RuntimeAllocation RuntimeAllocation `json:"runtime_allocation"` SessionID pgtype.UUID `json:"session_id"` TenantID pgtype.UUID `json:"tenant_id"` @@ -79,9 +119,9 @@ type GetRuntimeAllocationRow struct { Expired bool `json:"expired"` } -func (q *Queries) GetRuntimeAllocation(ctx context.Context, arg GetRuntimeAllocationParams) (GetRuntimeAllocationRow, error) { - row := q.db.QueryRow(ctx, getRuntimeAllocation, arg.TenantID, arg.EnvironmentID) - var i GetRuntimeAllocationRow +func (q *Queries) GetLatestRuntimeAllocation(ctx context.Context, arg GetLatestRuntimeAllocationParams) (GetLatestRuntimeAllocationRow, error) { + row := q.db.QueryRow(ctx, getLatestRuntimeAllocation, arg.TenantID, arg.EnvironmentID) + var i GetLatestRuntimeAllocationRow err := row.Scan( &i.RuntimeAllocation.ID, &i.RuntimeAllocation.EnvironmentID, @@ -173,7 +213,7 @@ const listRuntimeObservationSessions = `-- name: ListRuntimeObservationSessions SELECT s.id, s.tenant_id FROM sessions s LEFT JOIN environments e ON e.session_id = s.id -LEFT JOIN runtime_allocations a ON a.environment_id = e.id +LEFT JOIN runtime_allocations a ON a.environment_id = e.id AND a.state <> 'released' WHERE s.id > $1 AND s.deleted_at IS NULL AND s.configuration->'environment'->>'type' = 'openai_hosted' diff --git a/services/core/internal/db/sqlc/runtime_deployment.sql.go b/services/core/internal/db/sqlc/runtime_deployment.sql.go index 56e51e53a..8f22aa200 100644 --- a/services/core/internal/db/sqlc/runtime_deployment.sql.go +++ b/services/core/internal/db/sqlc/runtime_deployment.sql.go @@ -34,10 +34,13 @@ func (q *Queries) CountAddressBindings(ctx context.Context, publicUrl string) (C const countRuntimeDeploymentResources = `-- name: CountRuntimeDeploymentResources :one SELECT (SELECT count(*) FROM runtime_allocations WHERE state <> 'released')::bigint AS allocations, -(SELECT count(*) FROM environments e JOIN sessions s ON s.id = e.session_id - WHERE s.deleted_at IS NULL AND e.status = 'pending' +((SELECT count(*) FROM environments e JOIN sessions s ON s.id = e.session_id + WHERE s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND ((e.status = 'pending' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id)) + OR EXISTS (SELECT 1 FROM runtime_placements p WHERE p.environment_id = e.id AND p.released_at IS NULL)) AND s.configuration->'environment'->>'type' = 'openai_hosted' - AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id))::bigint AS pending + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released'))::bigint + + (SELECT count(*) FROM runtime_reset_retained_environments)::bigint)::bigint AS pending ` type CountRuntimeDeploymentResourcesRow struct { diff --git a/services/core/internal/db/sqlc/runtime_enrollment.sql.go b/services/core/internal/db/sqlc/runtime_enrollment.sql.go index 5bc1b1d0c..a34dcf9c5 100644 --- a/services/core/internal/db/sqlc/runtime_enrollment.sql.go +++ b/services/core/internal/db/sqlc/runtime_enrollment.sql.go @@ -48,7 +48,7 @@ func (q *Queries) AuthorizeRuntimeEnrollment(ctx context.Context, arg AuthorizeR const enrollRuntimeDevice = `-- name: EnrollRuntimeDevice :one INSERT INTO devices (id, tenant_id, name, environment_id, executor_key_id) VALUES ($1, $2, 'User-managed Runtime', $3, $4) -ON CONFLICT (environment_id) DO UPDATE SET name = devices.name +ON CONFLICT (environment_id) WHERE revoked_at IS NULL OR executor_key_id IS NOT NULL DO UPDATE SET name = devices.name WHERE devices.executor_key_id = EXCLUDED.executor_key_id AND devices.revoked_at IS NULL RETURNING id, name, environment_id ` @@ -121,20 +121,21 @@ func (q *Queries) ListEnrolledRuntimeBindings(ctx context.Context) ([]ListEnroll } const touchAuthenticatedDevice = `-- name: TouchAuthenticatedDevice :execrows -UPDATE devices SET last_seen_at = clock_timestamp() -WHERE devices.id = $1 AND EXISTS ( +UPDATE devices SET last_seen_at = clock_timestamp(), supported_agent_kinds = $1::jsonb +WHERE devices.id = $2 AND EXISTS ( SELECT 1 FROM runtime_device_authority a - WHERE a.id = devices.id AND a.credential_hash = $2 + WHERE a.id = devices.id AND a.credential_hash = $3 ) ` type TouchAuthenticatedDeviceParams struct { - ID pgtype.UUID `json:"id"` - CredentialHash string `json:"credential_hash"` + SupportedAgentKinds []byte `json:"supported_agent_kinds"` + ID pgtype.UUID `json:"id"` + CredentialHash string `json:"credential_hash"` } func (q *Queries) TouchAuthenticatedDevice(ctx context.Context, arg TouchAuthenticatedDeviceParams) (int64, error) { - result, err := q.db.Exec(ctx, touchAuthenticatedDevice, arg.ID, arg.CredentialHash) + result, err := q.db.Exec(ctx, touchAuthenticatedDevice, arg.SupportedAgentKinds, arg.ID, arg.CredentialHash) if err != nil { return 0, err } diff --git a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go index c1f443347..7e551e74f 100644 --- a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go @@ -13,11 +13,14 @@ import ( const getRuntimeLifecyclePlacement = `-- name: GetRuntimeLifecyclePlacement :one SELECT d.provider_kind, d.mode, a.id AS allocation_id, a.node_id AS allocation_node_id, - p.node_id AS placement_node_id, p.released_at + p.node_id AS placement_node_id, p.released_at, + CASE WHEN COALESCE(a.deployment_generation, p.deployment_generation, d.generation)=d.generation + THEN d.specification ELSE g.specification END::jsonb AS specification FROM environments e JOIN sessions s ON s.id=e.session_id CROSS JOIN runtime_deployment d -LEFT JOIN runtime_allocations a ON a.environment_id=e.id +LEFT JOIN runtime_allocations a ON a.environment_id=e.id AND a.state<>'released' LEFT JOIN runtime_placements p ON p.environment_id=e.id +LEFT JOIN runtime_deployment_generations g ON g.generation=COALESCE(a.deployment_generation, p.deployment_generation, d.generation) WHERE s.tenant_id=$1 AND e.id=$2 ` @@ -33,6 +36,7 @@ type GetRuntimeLifecyclePlacementRow struct { AllocationNodeID pgtype.UUID `json:"allocation_node_id"` PlacementNodeID pgtype.UUID `json:"placement_node_id"` ReleasedAt pgtype.Timestamptz `json:"released_at"` + Specification []byte `json:"specification"` } func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRuntimeLifecyclePlacementParams) (GetRuntimeLifecyclePlacementRow, error) { @@ -45,10 +49,49 @@ func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRunti &i.AllocationNodeID, &i.PlacementNodeID, &i.ReleasedAt, + &i.Specification, ) return i, err } +const listReplacementEnvironments = `-- name: ListReplacementEnvironments :many +SELECT e.id, s.tenant_id +FROM environments e JOIN sessions s ON s.id = e.session_id +JOIN environment_workspaces w ON w.environment_id = e.id AND w.state = 'ready' +WHERE e.id > $1 AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND e.initialization = 'complete' AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND (SELECT reset_clear IS NULL FROM runtime_deployment) + AND EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state = 'released') + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') + AND EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending') +ORDER BY e.id LIMIT 32 +` + +type ListReplacementEnvironmentsRow struct { + ID pgtype.UUID `json:"id"` + TenantID pgtype.UUID `json:"tenant_id"` +} + +func (q *Queries) ListReplacementEnvironments(ctx context.Context, id pgtype.UUID) ([]ListReplacementEnvironmentsRow, error) { + rows, err := q.db.Query(ctx, listReplacementEnvironments, id) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListReplacementEnvironmentsRow{} + for rows.Next() { + var i ListReplacementEnvironmentsRow + if err := rows.Scan(&i.ID, &i.TenantID); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listRuntimeAllocationsForNode = `-- name: ListRuntimeAllocationsForNode :many SELECT a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.released_at, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation, e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired FROM runtime_allocations a @@ -150,9 +193,9 @@ LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM $1::uuid AND p.released_at IS NULL AND (SELECT reset_clear IS NULL FROM runtime_deployment) - AND e.id > $2::uuid AND s.deleted_at IS NULL AND e.status='pending' + AND e.id > $2::uuid AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') AND s.configuration->'environment'->>'type'='openai_hosted' - AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id AND a.state<>'released') ORDER BY e.id LIMIT 32 ` diff --git a/services/core/internal/db/sqlc/runtime_nodes.sql.go b/services/core/internal/db/sqlc/runtime_nodes.sql.go index f6bc5b7d7..e28353bcf 100644 --- a/services/core/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_nodes.sql.go @@ -203,7 +203,7 @@ const getRuntimePlacement = `-- name: GetRuntimePlacement :one SELECT p.environment_id, p.node_id, p.reserved_at, p.released_at, p.deployment_generation, n.name, (EXISTS(SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=p.deployment_generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch AND g.state='ready') AND n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND n.removed_at IS NULL)::boolean AS available, COALESCE(a.observation_error,'')::text AS observation_error, COALESCE(a.state,'reserved')::text AS state, COALESCE(a.compute_phase,'disabled')::text AS compute_phase FROM runtime_placements p JOIN runtime_nodes n ON n.id=p.node_id CROSS JOIN runtime_deployment d -LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.environment_id=$1 +LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id AND a.state<>'released' WHERE p.environment_id=$1 ` type GetRuntimePlacementRow struct { @@ -383,9 +383,9 @@ SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha2 EXISTS(SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=n.ready_generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch AND g.state='ready')::boolean AS serving_ready, COALESCE((SELECT g.state FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_state, COALESCE((SELECT g.diagnostic FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_diagnostic, - (SELECT count(*) FROM runtime_placements p LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id WHERE p.node_id=n.id AND p.released_at IS NULL AND (a.id IS NULL OR a.compute_phase <> 'suspended'))::bigint AS active, + (SELECT count(*) FROM runtime_placements p LEFT JOIN runtime_allocations a ON a.environment_id=p.environment_id AND a.state<>'released' WHERE p.node_id=n.id AND p.released_at IS NULL AND (a.id IS NULL OR a.compute_phase <> 'suspended'))::bigint AS active, (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL)::bigint AS retained, - (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id))::bigint AS reserved, + (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id AND a.state<>'released'))::bigint AS reserved, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='cleanup_pending')::bigint AS cleanup_pending, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='running' AND a.compute_phase IN('running','disabled'))::bigint AS running, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'snapshot' IS NOT NULL AND a.compute_state->'snapshot'<>'null'::jsonb)::bigint AS snapshots @@ -504,7 +504,7 @@ func (q *Queries) ReleaseRuntimePlacement(ctx context.Context, environmentID pgt const releaseUnallocatedRuntimePlacement = `-- name: ReleaseUnallocatedRuntimePlacement :exec UPDATE runtime_placements SET released_at=COALESCE(released_at,clock_timestamp()) WHERE environment_id IN(SELECT id FROM environments WHERE session_id=$1) -AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=runtime_placements.environment_id) +AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=runtime_placements.environment_id AND a.state<>'released') ` func (q *Queries) ReleaseUnallocatedRuntimePlacement(ctx context.Context, sessionID pgtype.UUID) error { @@ -521,6 +521,27 @@ func (q *Queries) RemoveRuntimeNode(ctx context.Context, id pgtype.UUID) error { return err } +const reserveReleasedRuntimePlacement = `-- name: ReserveReleasedRuntimePlacement :execrows +UPDATE runtime_placements SET node_id = $2, deployment_generation = $3, + reserved_at = clock_timestamp(), released_at = NULL +WHERE runtime_placements.environment_id = $1 AND released_at IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = $1 AND a.state <> 'released') +` + +type ReserveReleasedRuntimePlacementParams struct { + EnvironmentID pgtype.UUID `json:"environment_id"` + NodeID pgtype.UUID `json:"node_id"` + DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` +} + +func (q *Queries) ReserveReleasedRuntimePlacement(ctx context.Context, arg ReserveReleasedRuntimePlacementParams) (int64, error) { + result, err := q.db.Exec(ctx, reserveReleasedRuntimePlacement, arg.EnvironmentID, arg.NodeID, arg.DeploymentGeneration) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + const setRuntimeObservation = `-- name: SetRuntimeObservation :exec UPDATE runtime_allocations SET observation_error=$4 WHERE id=$1 AND compute_revision=$2 AND state=$3 AND state<>'released' ` diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index 8b01bf155..fbfa7df77 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -29,7 +29,7 @@ func (q *Queries) ClearRuntimeWake(ctx context.Context, arg ClearRuntimeWakePara const getRuntimeActivity = `-- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, - GREATEST(a.compute_activity_at, + GREATEST(a.compute_activity_at, a.compute_phase_changed_at, CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, @@ -37,8 +37,7 @@ SELECT clock_timestamp()::timestamptz AS observed_at, OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = e.session_id AND r.state = 'pending') OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = e.id AND f.state = 'pending'))::boolean AS busy, - a.compute_wake_requested, - EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.completed_at IS NOT NULL)::boolean AS has_completed_turn + a.compute_wake_requested FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE a.id = $1 ` @@ -48,7 +47,6 @@ type GetRuntimeActivityRow struct { LastActivity pgtype.Timestamptz `json:"last_activity"` Busy bool `json:"busy"` ComputeWakeRequested bool `json:"compute_wake_requested"` - HasCompletedTurn bool `json:"has_completed_turn"` } func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRuntimeActivityRow, error) { @@ -59,7 +57,6 @@ func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRu &i.LastActivity, &i.Busy, &i.ComputeWakeRequested, - &i.HasCompletedTurn, ) return i, err } @@ -79,7 +76,7 @@ func (q *Queries) RecordRuntimeTerminalActivity(ctx context.Context, sessionID p const runtimeComputeBlocksAdmission = `-- name: RuntimeComputeBlocksAdmission :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id - WHERE e.session_id = $1 AND a.compute_phase NOT IN ('disabled', 'running') + WHERE e.session_id = $1 AND a.state <> 'released' AND a.compute_phase NOT IN ('disabled', 'running') )::boolean ` @@ -93,7 +90,7 @@ func (q *Queries) RuntimeComputeBlocksAdmission(ctx context.Context, sessionID p const sessionHasRuntimeNode = `-- name: SessionHasRuntimeNode :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id - WHERE e.session_id = $1 AND a.node_id IS NOT NULL + WHERE e.session_id = $1 AND a.state <> 'released' AND a.node_id IS NOT NULL )::boolean ` diff --git a/services/core/internal/db/sqlc/sandbox_reset.sql.go b/services/core/internal/db/sqlc/sandbox_reset.sql.go index 8b1d18f72..fcc589d64 100644 --- a/services/core/internal/db/sqlc/sandbox_reset.sql.go +++ b/services/core/internal/db/sqlc/sandbox_reset.sql.go @@ -64,9 +64,13 @@ held AS ( SELECT p.deployment_generation, p.node_id, s.id, e.id, true, false FROM environments e JOIN sessions s ON s.id = e.session_id LEFT JOIN runtime_placements p ON p.environment_id = e.id AND p.released_at IS NULL - WHERE s.deleted_at IS NULL AND e.status = 'pending' + WHERE s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND ((e.status = 'pending' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id)) OR p.environment_id IS NOT NULL) AND s.configuration->'environment'->>'type' = 'openai_hosted' - AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id) + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') + UNION ALL + SELECT r.deployment_generation, NULL::uuid, r.session_id, r.environment_id, true, false + FROM runtime_reset_retained_environments r ), classified AS ( SELECT h.deployment_generation, h.node_id, h.session_id, h.environment_id, h.pending, h.cleanup, CASE WHEN h.cleanup THEN 'cleanup' WHEN EXISTS (SELECT 1 FROM turns t WHERE t.session_id = h.session_id AND t.status IN ('in_progress', 'waiting')) @@ -166,7 +170,9 @@ WHERE s.deleted_at IS NULL AND s.configuration->'environment'->>'type' = 'openai AND e.status NOT IN ('failed', 'expired') AND s.id > $1::uuid AND (EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state NOT IN ('released', 'cleanup_pending')) - OR (e.status = 'pending' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id))) + OR (e.status = 'pending' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id)) + OR EXISTS (SELECT 1 FROM runtime_placements p WHERE p.environment_id = e.id AND p.released_at IS NULL) + OR EXISTS (SELECT 1 FROM runtime_reset_retained_environments r WHERE r.environment_id = e.id)) AND ($2::boolean OR NOT ( EXISTS (SELECT 1 FROM turns t WHERE t.session_id = s.id AND t.status IN ('in_progress', 'waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = s.id AND t.status IN ('in_progress', 'waiting')) diff --git a/services/core/internal/deployment/activity_test.go b/services/core/internal/deployment/activity_test.go index 2f308dfb2..a1b5d0c02 100644 --- a/services/core/internal/deployment/activity_test.go +++ b/services/core/internal/deployment/activity_test.go @@ -12,23 +12,22 @@ func TestRuntimeIdleAdmissionUsesDatabaseClock(t *testing.T) { t.Run(skew.String(), func(t *testing.T) { observed := coreNow.Add(skew) for _, test := range []struct { - name string - age time.Duration - busy, wake, completed bool - timeout time.Duration - want bool + name string + age time.Duration + busy, wake bool + timeout time.Duration + want bool }{ - {"recent", timeout - time.Second, false, false, true, timeout, false}, - {"boundary", timeout, false, false, true, timeout, true}, - {"elapsed", timeout + time.Second, false, false, true, timeout, true}, - {"busy", timeout + time.Second, true, false, true, timeout, false}, - {"wake", timeout + time.Second, false, true, true, timeout, false}, - {"never_completed", timeout + time.Second, false, false, false, timeout, false}, - {"missing_policy", timeout + time.Second, false, false, true, 0, false}, - {"invalid_policy", timeout + time.Second, false, false, true, -time.Second, false}, + {"recent", timeout - time.Second, false, false, timeout, false}, + {"boundary", timeout, false, false, timeout, true}, + {"elapsed", timeout + time.Second, false, false, timeout, true}, + {"busy", timeout + time.Second, true, false, timeout, false}, + {"wake", timeout + time.Second, false, true, timeout, false}, + {"missing_policy", timeout + time.Second, false, false, 0, false}, + {"invalid_policy", timeout + time.Second, false, false, -time.Second, false}, } { t.Run(test.name, func(t *testing.T) { - activity := Activity{ObservedAt: observed, LastActivity: observed.Add(-test.age), Busy: test.busy, WakeRequested: test.wake, HasCompletedTurn: test.completed} + activity := Activity{ObservedAt: observed, LastActivity: observed.Add(-test.age), Busy: test.busy, WakeRequested: test.wake} if got := activity.ReadyToSuspend(test.timeout); got != test.want { t.Fatalf("suspension admission = %v, want %v with database clock skew %s", got, test.want, skew) } diff --git a/services/core/internal/deployment/allocation.go b/services/core/internal/deployment/allocation.go index c51f2527b..d57f33ed5 100644 --- a/services/core/internal/deployment/allocation.go +++ b/services/core/internal/deployment/allocation.go @@ -76,13 +76,13 @@ type SessionDevice struct{ ID, EnvironmentID string } // Activity separates real work from the connection keepalive. ObservedAt is // the database clock when it was read. type Activity struct { - LastActivity, ObservedAt time.Time - Busy, WakeRequested, HasCompletedTurn bool + LastActivity, ObservedAt time.Time + Busy, WakeRequested bool } // ReadyToSuspend evaluates the idle policy on one database-clock observation. func (a Activity) ReadyToSuspend(idleTimeout time.Duration) bool { - return idleTimeout > 0 && a.HasCompletedTurn && !a.Busy && !a.WakeRequested && + return idleTimeout > 0 && !a.Busy && !a.WakeRequested && a.ObservedAt.Sub(a.LastActivity) >= idleTimeout } @@ -116,6 +116,8 @@ type UnallocatedEnvironment struct{ ID, TenantID string } // lifecycle reads: the deployment's provider and mode, the Environment's // allocation and the node placement its Session reserved. type LifecyclePlacement struct { + // Specification belongs to the current allocation or reserved generation. + Specification json.RawMessage Provider, Mode string // AllocationID and AllocationNodeID are empty without an allocation or // its node. diff --git a/services/core/internal/deployment/allocations.go b/services/core/internal/deployment/allocations.go index 5c6630cd8..d6ccdec2d 100644 --- a/services/core/internal/deployment/allocations.go +++ b/services/core/internal/deployment/allocations.go @@ -46,7 +46,7 @@ func (e *ExecutionOperations) ReserveAllocation(ctx context.Context, key Allocat if err != nil { return err } - if found { + if found && existing.State != "released" { if existing.ProviderKey != installation { return ErrAllocationConflict } @@ -54,6 +54,15 @@ func (e *ExecutionOperations) ReserveAllocation(ctx context.Context, key Allocat result = existing return nil } + if found { + allowed, err := tx.CanReplaceAllocation() + if err != nil { + return err + } + if !allowed { + return ErrAllocationConflict + } + } // Existing receipts replay first, so retries and cleanup continue // while admission is closed. d, err := tx.LockDeployment() @@ -177,7 +186,19 @@ func (e *ExecutionOperations) cleanup(ctx context.Context, owner Allocation, abs if revoked.SessionDeleted { err = sessions.CancelWork(ctx, tx) } else { - err = sessions.TerminateEnvironment(ctx, tx, revoked.Expired, sessions.ProvisioningFailureReason, nil) + retain := false + if revoked.Expired { + retain, err = tx.CanRetainEnvironment(revoked) + if err != nil { + return err + } + } + // Compute retention ends independently of the Session only when its + // initialized filesystem and native history can continue on new compute. + // Explicit archive/reset makes this locked eligibility check false. + if !retain { + err = sessions.TerminateEnvironment(ctx, tx, revoked.Expired, sessions.ProvisioningFailureReason, nil) + } } if err != nil { return err diff --git a/services/core/internal/deployment/allocations_test.go b/services/core/internal/deployment/allocations_test.go index dcbfa2a4e..352cf2a3d 100644 --- a/services/core/internal/deployment/allocations_test.go +++ b/services/core/internal/deployment/allocations_test.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "slices" "testing" "time" @@ -69,6 +70,19 @@ func (f *fakeReservationTx) LoadReserved() (placement.Reserved, error) { return f.loadReserved() } +func (f *fakeReservationTx) CanReplaceAllocation() (bool, error) { + unexpected(f.t, "CanReplaceAllocation") + return false, nil +} +func (f *fakeReservationTx) LoadNodes() ([]placement.Node, error) { + unexpected(f.t, "LoadNodes") + return nil, nil +} +func (f *fakeReservationTx) ReservePlacement(placement.Placement) error { + unexpected(f.t, "ReservePlacement") + return nil +} + func (f *fakeReservationTx) InsertAllocation(allocation NewAllocation) (Allocation, error) { if f.insertAllocation == nil { unexpected(f.t, "InsertAllocation") @@ -141,6 +155,7 @@ func (f *fakeAllocationTx) RecordObservation(Allocation, string) error { // methods cover a Session without an active Turn, pending input or input // activity change. type fakeCleanupTx struct { + canRetainEnvironment func(Allocation) (bool, error) *fakeAllocationTx revokeDevice func(Allocation) error requestCleanup func(Allocation) (Allocation, error) @@ -152,6 +167,13 @@ type fakeCleanupTx struct { cancelPendingInput func() error } +func (f *fakeCleanupTx) CanRetainEnvironment(current Allocation) (bool, error) { + if f.canRetainEnvironment == nil { + return false, nil + } + return f.canRetainEnvironment(current) +} + func (f *fakeCleanupTx) RevokeDevice(current Allocation) error { if f.revokeDevice == nil { unexpected(f.t, "RevokeDevice") @@ -508,3 +530,42 @@ func TestCleanupRevokesSettlesTheSessionThenReleases(t *testing.T) { }) } } + +func TestExpiredQualifiedFilesystemCleanupPreservesSessionWork(t *testing.T) { + owner := Allocation{ID: uuid.NewString(), DeviceID: uuid.NewString(), EnvironmentID: uuid.NewString(), TenantID: uuid.NewString(), ProviderKey: uuid.NewString(), State: "running", Expired: true, CreateSettled: true} + for _, deleted := range []bool{false, true} { + t.Run(fmt.Sprint("deleted=", deleted), func(t *testing.T) { + current := owner + current.SessionDeleted = deleted + checked, cancelled, revoked := false, false, false + tx := &fakeCleanupTx{fakeAllocationTx: &fakeAllocationTx{t: t, loadAllocation: func() (Allocation, error) { return current, nil }}, + canRetainEnvironment: func(Allocation) (bool, error) { + if !revoked { + t.Fatal("eligibility checked before authority revocation") + } + checked = true + return true, nil + }, + revokeDevice: func(Allocation) error { revoked = true; return nil }, + requestCleanup: func(a Allocation) (Allocation, error) { a.State = "cleanup_pending"; return a, nil }, + loadActiveTurn: func() (sessions.Turn, bool, error) { cancelled = true; return sessions.Turn{}, false, nil }, cancelPendingInput: func() error { return nil }, + } + storage := &fakeExecutionStorage{t: t, withAllocationCleanup: func(_ context.Context, _ AllocationKey, apply func(AllocationCleanupTx) error) error { + return apply(tx) + }} + operations, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL), storage) + if err != nil { + t.Fatal(err) + } + result, err := operations.RequestCleanup(t.Context(), current) + if err != nil || result.State != "cleanup_pending" || !revoked || checked == deleted || cancelled != deleted { + t.Fatal("cleanup lost lifetime separation", result, err, checked, cancelled) + } + }) + } +} + +func (f *fakeReservationTx) LoadGenerationSpecification(uint64) (GenerationSpecification, error) { + unexpected(f.t, "LoadGenerationSpecification") + return GenerationSpecification{}, nil +} diff --git a/services/core/internal/deployment/fakes_test.go b/services/core/internal/deployment/fakes_test.go index 3ccdb1d13..ac99e8612 100644 --- a/services/core/internal/deployment/fakes_test.go +++ b/services/core/internal/deployment/fakes_test.go @@ -637,3 +637,10 @@ func (f *fakeSessionReader) GetManagedSessionArchive(context.Context, string, st unexpected(f.t, "GetManagedSessionArchive") return sessions.ManagedArchive{}, nil } + +func (f *fakeReader) ReplacementEnvironments(context.Context, string) ([]UnallocatedEnvironment, error) { + panic("unexpected ReplacementEnvironments") +} +func (f *fakeReader) RetainedNativeHistory(context.Context, AllocationKey) (bool, error) { + panic("unexpected RetainedNativeHistory") +} diff --git a/services/core/internal/deployment/replacement.go b/services/core/internal/deployment/replacement.go new file mode 100644 index 000000000..57871ac8c --- /dev/null +++ b/services/core/internal/deployment/replacement.go @@ -0,0 +1,112 @@ +package deployment + +import ( + "context" + "encoding/json" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" +) + +// EnsurePlacement commits replacement demand before entering a node lifecycle. +// Its reservation survives a failed caller, while the released allocation's +// immutable receipt continues to identify the previous compute owner. +func (e *ExecutionOperations) EnsurePlacement(ctx context.Context, key AllocationKey, installation string) (placement.Reserved, error) { + key, err := key.parse() + if err != nil { + return placement.Reserved{}, err + } + installation, err = parseID(installation) + if err != nil { + return placement.Reserved{}, err + } + var result placement.Reserved + err = e.storage.WithReservation(ctx, key, func(locked sessions.LockedSession, tx ReservationTx) error { + if err := locked.Public(); err != nil { + return err + } + existing, found, err := tx.FindAllocation() + if err != nil { + return err + } + if !found || existing.State != "released" { + return ErrAllocationConflict + } + allowed, err := tx.CanReplaceAllocation() + if err != nil { + return err + } + if !allowed { + return ErrAllocationConflict + } + d, err := tx.LockDeployment() + if err != nil { + return err + } + if err := e.service.rules.CheckAdmission(d, installation); err != nil { + return err + } + if d.Mode != string(sandbox.DeploymentNodes) { + return ErrAllocationConflict + } + reserved, err := tx.LoadReserved() + if err != nil { + return err + } + if !reserved.Released { + if err := placement.CheckReserved(reserved); err != nil { + return err + } + result = reserved + return nil + } + nodes, err := tx.LoadNodes() + if err != nil { + return err + } + // Retained storage cannot move onto an owned-storage generation. Filter + // before reserving, so an incompatible ready node never traps demand. + compatible := make(map[uint64]bool) + filtered := nodes[:0] + for _, node := range nodes { + if node.ReadyGeneration == nil { + continue + } + generation := *node.ReadyGeneration + accepts, known := compatible[generation] + if !known { + specification := d.Specification + if generation != d.Generation { + stored, err := tx.LoadGenerationSpecification(generation) + if err != nil { + return err + } + specification = stored.Specification + } + var spec sandbox.DeploymentSpec + if err := json.Unmarshal(specification, &spec); err != nil { + return err + } + accepts = spec.Workspace != nil + compatible[generation] = accepts + } + if accepts { + filtered = append(filtered, node) + } + } + selected, err := e.service.rules.DecidePlacement(d, filtered) + if err != nil { + return err + } + if selected == nil { + return ErrAllocationConflict + } + if err := tx.ReservePlacement(*selected); err != nil { + return err + } + result = placement.Reserved{NodeID: selected.NodeID, Generation: selected.Generation, Available: true} + return nil + }) + return result, err +} diff --git a/services/core/internal/deployment/session_archive.go b/services/core/internal/deployment/session_archive.go index 0f6c98ad4..22f5d41d6 100644 --- a/services/core/internal/deployment/session_archive.go +++ b/services/core/internal/deployment/session_archive.go @@ -101,7 +101,7 @@ func (e *ExecutionOperations) archiveSession(ctx context.Context, tenantID, sess if err := tx.RequestArchiveCleanup(current); err != nil { return err } - } else if !allocated { + } else { if err := tx.ReleasePlacement(); err != nil { return err } diff --git a/services/core/internal/deployment/session_archive_test.go b/services/core/internal/deployment/session_archive_test.go index d325e6b83..78e853b6b 100644 --- a/services/core/internal/deployment/session_archive_test.go +++ b/services/core/internal/deployment/session_archive_test.go @@ -250,7 +250,7 @@ func TestArchiveSession(t *testing.T) { join(head, expire, []string{"RequestArchiveCleanup device allocation", "RecordArchiveAudit", "LoadArchive"})}, {"no allocation", sessions.LockedSession{}, hosted, nil, nil, join(head, expire, []string{"ReleasePlacement", "RecordArchiveAudit", "LoadArchive"})}, {"released allocation", sessions.LockedSession{}, hosted, &Allocation{ID: "allocation", ProviderKey: "previous", State: "released"}, nil, - join(head, expire, []string{"RecordArchiveAudit", "LoadArchive"})}, + join(head, expire, []string{"ReleasePlacement", "RecordArchiveAudit", "LoadArchive"})}, {"ended Environment", sessions.LockedSession{}, expired, live, nil, join(head, settle, []string{"RequestArchiveCleanup device allocation", "RecordArchiveAudit", "LoadArchive"})}, {"allocation of another installation", sessions.LockedSession{}, hosted, &Allocation{ID: "allocation", ProviderKey: "previous", State: "running"}, ErrConflict, head}, diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index 2eb526b14..51eb8f1f7 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -96,6 +96,14 @@ type ReservationTx interface { LoadReserved() (placement.Reserved, error) // InsertAllocation stores the allocation of the Session's Environment. InsertAllocation(allocation NewAllocation) (Allocation, error) + // CanReplaceAllocation verifies retained storage and settled, revoked prior ownership. + CanReplaceAllocation() (bool, error) + // LoadGenerationSpecification reads immutable storage requirements for placement. + LoadGenerationSpecification(generation uint64) (GenerationSpecification, error) + // LoadNodes reads eligible placement facts under the deployment lock. + LoadNodes() ([]placement.Node, error) + // ReservePlacement replaces the released reservation without changing historical receipts. + ReservePlacement(placement.Placement) error } // AllocationTx is one Session-locked allocation change. Each write applies to @@ -128,6 +136,9 @@ type AllocationTx interface { // AllocationCleanupTx is one Session-locked allocation cleanup. type AllocationCleanupTx interface { + // CanRetainEnvironment checks the ready retained filesystem, completed + // initialization and selected Harness's qualified history under the Session lock. + CanRetainEnvironment(current Allocation) (bool, error) AllocationTx sessions.EnvironmentTerminationTx // RevokeDevice revokes the allocation's device. @@ -210,7 +221,7 @@ type Reader interface { // AddressBindings counts what is bound to an installation address, read // in one snapshot. publicURL is the address nodes are compared against. AddressBindings(ctx context.Context, publicURL string) (AddressBindings, error) - // EnvironmentAllocation returns the Environment's allocation, including + // EnvironmentAllocation returns the Environment's latest allocation receipt, including // for a deleted Session. It returns ErrInvalidInput for a malformed // identifier and ErrNotFound for a missing allocation. EnvironmentAllocation(ctx context.Context, key AllocationKey) (Allocation, error) @@ -240,6 +251,10 @@ type Reader interface { // ID, in ID order. It reads the committed placement and never selects a // replacement node. UnallocatedEnvironments(ctx context.Context, nodeID, after string) ([]UnallocatedEnvironment, error) + // RetainedNativeHistory reports the latest owner's persisted retention eligibility. + RetainedNativeHistory(ctx context.Context, key AllocationKey) (bool, error) + // ReplacementEnvironments lists retained Environments with pending input and no compute owner. + ReplacementEnvironments(ctx context.Context, after string) ([]UnallocatedEnvironment, error) // LifecyclePlacement returns what routing the Environment to its // lifecycle reads, including for a deleted Session. A missing Environment // is sessions.ErrNotFound. diff --git a/services/core/internal/engine/claude.go b/services/core/internal/engine/claude.go index 7a19d266c..b92c435af 100644 --- a/services/core/internal/engine/claude.go +++ b/services/core/internal/engine/claude.go @@ -13,6 +13,7 @@ import ( // providers reject text without non-whitespace characters. func claudeProfile() Profile { return Profile{ + RetainedNativeHistory: proto.CapabilitySupported, ProgrammaticToolCallingDisable: proto.CapabilitySupported, MCPOrigins: []string{"service", "environment"}, Placements: []string{"none", "openai_hosted", "self_hosted"}, diff --git a/services/core/internal/engine/codex.go b/services/core/internal/engine/codex.go index a2f67bc07..7ce42df97 100644 --- a/services/core/internal/engine/codex.go +++ b/services/core/internal/engine/codex.go @@ -7,6 +7,7 @@ import ( func codexProfile() Profile { return Profile{ + RetainedNativeHistory: proto.CapabilitySupported, ProgrammaticToolCallingDisable: proto.CapabilitySupported, Placements: []string{"none", "self_hosted", "openai_hosted"}, MCPOrigins: []string{"service", "environment"}, diff --git a/services/core/internal/engine/enginetest/profile.go b/services/core/internal/engine/enginetest/profile.go index 2984c5e93..1a5801474 100644 --- a/services/core/internal/engine/enginetest/profile.go +++ b/services/core/internal/engine/enginetest/profile.go @@ -11,6 +11,7 @@ import ( // intentionally exhaustive; new fields stay unspecified until decided here. func Profile(change func(*engine.Profile)) engine.Profile { p := engine.Profile{ + RetainedNativeHistory: proto.CapabilityUnsupported, ProgrammaticToolCallingDisable: proto.CapabilityUnsupported, WebSearchControl: proto.CapabilityUnsupported, TextVerbosity: proto.CapabilityUnsupported, diff --git a/services/core/internal/engine/mcode.go b/services/core/internal/engine/mcode.go index 950cc470e..eea6c9cf0 100644 --- a/services/core/internal/engine/mcode.go +++ b/services/core/internal/engine/mcode.go @@ -12,6 +12,7 @@ import ( // with "Local message content or attachments are required.". func mcodeProfile() Profile { return Profile{ + RetainedNativeHistory: proto.CapabilityUnsupported, MCPOrigins: []string{"environment"}, Placements: []string{"none", "openai_hosted", "self_hosted"}, MCPBearer: proto.CapabilitySupported, diff --git a/services/core/internal/engine/profile.go b/services/core/internal/engine/profile.go index afb701cc7..501ef240c 100644 --- a/services/core/internal/engine/profile.go +++ b/services/core/internal/engine/profile.go @@ -16,6 +16,7 @@ var ErrInvalidInput = errors.New("invalid engine configuration") // Profile records qualified public behavior, independently of Runtime advertisements. type Profile struct { + RetainedNativeHistory proto.CapabilitySupport ProgrammaticToolCallingDisable proto.CapabilitySupport Placements []string MCPOrigins []string diff --git a/services/core/internal/execution/deployment_fixture_test.go b/services/core/internal/execution/deployment_fixture_test.go index 1cb8c27b5..fc8234e49 100644 --- a/services/core/internal/execution/deployment_fixture_test.go +++ b/services/core/internal/execution/deployment_fixture_test.go @@ -140,6 +140,8 @@ func (s *strictExecutionStorage) WithDeployment(ctx context.Context, apply func( // strictDeploymentReader runs each set func; any other call fails the test. type strictDeploymentReader struct { + replacementEnvironments func(context.Context, string) ([]deployment.UnallocatedEnvironment, error) + retainedNativeHistory func(context.Context, deployment.AllocationKey) (bool, error) t *testing.T deployment func(context.Context) (deployment.Record, error) snapshot func(context.Context) (deployment.Snapshot, error) @@ -341,3 +343,16 @@ func (r *strictDeploymentReader) Activity(ctx context.Context, allocationID stri } return r.activity(ctx, allocationID) } + +func (r *strictDeploymentReader) ReplacementEnvironments(ctx context.Context, after string) ([]deployment.UnallocatedEnvironment, error) { + if r.replacementEnvironments == nil { + return nil, unexpectedDeploymentCall(r.t, "ReplacementEnvironments") + } + return r.replacementEnvironments(ctx, after) +} +func (r *strictDeploymentReader) RetainedNativeHistory(ctx context.Context, key deployment.AllocationKey) (bool, error) { + if r.retainedNativeHistory == nil { + return false, unexpectedDeploymentCall(r.t, "RetainedNativeHistory") + } + return r.retainedNativeHistory(ctx, key) +} diff --git a/services/core/internal/execution/engine_profile_test.go b/services/core/internal/execution/engine_profile_test.go index 754e370c9..8a41be47b 100644 --- a/services/core/internal/execution/engine_profile_test.go +++ b/services/core/internal/execution/engine_profile_test.go @@ -149,3 +149,31 @@ func TestCommonOnlyValidationPreservesFunctionResults(t *testing.T) { t.Fatal("common-only result acquired a native restriction", err) } } + +func TestRetainedNativeHistoryDependsOnExternalWorkspace(t *testing.T) { + for _, required := range []bool{false, true} { + for _, supported := range []bool{false, true} { + if err := sessions.ValidateRetainedHistory(required, supported); (err == nil) != (!required || supported) { + t.Fatal(required, supported, err) + } + } + } + for _, qualified := range []bool{false, true} { + profile := enginetest.Profile(func(p *engine.Profile) { + p.Placements = []string{"openai_hosted"} + p.RetainedNativeHistory = proto.CapabilityFromBool(qualified) + }) + policy := Policy{Engines: engine.NewCatalog(map[string]engine.Profile{"test_harness": profile})} + if err := policy.ValidateSessionConfiguration("test_harness", json.RawMessage(`{"agent":{"model":"fixture"},"environment":{"type":"openai_hosted"}}`)); err != nil { + t.Fatal("generic hosted configuration acquired a storage requirement", qualified, err) + } + for _, external := range []bool{false, true} { + for _, runtimeSupported := range []bool{false, true} { + err := policy.validateEnvironmentHistory("test_harness", sessions.Environment{ExternalWorkspace: external}, runtimeSupported) + if (err == nil) != (!external || qualified && runtimeSupported) { + t.Fatal(external, qualified, runtimeSupported, err) + } + } + } + } +} diff --git a/services/core/internal/execution/prepared_dispatch.go b/services/core/internal/execution/prepared_dispatch.go index fa2b11b0b..d48d00890 100644 --- a/services/core/internal/execution/prepared_dispatch.go +++ b/services/core/internal/execution/prepared_dispatch.go @@ -49,7 +49,7 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, lease Ownership, t if err != nil { return run, err } - caps, err := d.engineCapabilities(peer, session.Engine, snapshot) + caps, err := d.sessionCapabilities(ctx, peer, session, snapshot) if err != nil { return run, err } diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 20d00602b..861c183c2 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -231,13 +231,13 @@ func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.Sandbox if state.Target == nil || state.Snapshot == nil || state.Rollback { return sandbox.ErrOwnership } + spec, err := r.workspaceSpecification(ctx, owner.Key(), owner.ID) + if err != nil { + return err + } var workspace *workspacefs.Binding - if r.config.Workspace != nil { - if r.workspaces == nil { - return workspacefs.ErrUnavailable - } - var err error - workspace, err = r.workspaces.GetReady(ctx, owner.TenantID, owner.EnvironmentID, r.config.WorkspaceRequirements, r.config.Resources.EnvironmentDiskMiB) + if spec.Workspace != nil { + workspace, err = r.workspaces.GetReady(ctx, owner.TenantID, owner.EnvironmentID, r.config.WorkspaceRequirements, spec.Resources.EnvironmentDiskMiB) if err == nil && workspace == nil { err = workspacefs.ErrNotFound } diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index 0dd7f915d..fe013d212 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -7,6 +7,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -104,43 +105,91 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Sandbox // waitRuntimeAwake is called only for live Environment file operations, before // entering the Worker's work queues. Persisted history/artifact reads bypass it. -func (w *Worker) waitRuntimeAwake(ctx context.Context, environment sessions.Environment) error { +func (w *Worker) waitRuntimeAwake(ctx context.Context, environment sessions.Environment) (err error) { + // A deadline can arrive inside a read as well as between polling ticks. + // Both paths report the same unavailable outcome to the live file caller. + defer func() { + if ctx.Err() != nil && (errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)) { + err = ErrExecutionUnavailable + } + }() key := deployment.AllocationKey{TenantID: environment.TenantID, EnvironmentID: environment.ID} - owner, err := w.dispatcher.DeploymentReader.EnvironmentAllocation(ctx, key) - if errors.Is(err, deployment.ErrNotFound) { - return nil - } - if err != nil { - return err - } - if owner.SessionDeleted || owner.Expired || owner.State == "cleanup_pending" || owner.State == "released" { - return ErrExecutionUnavailable - } - if owner.ComputePhase == "disabled" { - return nil - } - if err := w.dispatcher.Deployment.TouchActivity(ctx, environment.TenantID, environment.ID); err != nil { - return err - } - timer := time.NewTicker(100 * time.Millisecond) - defer timer.Stop() + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + touched := "" for { - owner, err = w.dispatcher.DeploymentReader.EnvironmentAllocation(ctx, key) + owner, err := w.dispatcher.DeploymentReader.EnvironmentAllocation(ctx, key) + if errors.Is(err, deployment.ErrNotFound) { + return nil + } if err != nil { return err } - if owner.SessionDeleted || owner.Expired || owner.State != "running" { + if owner.SessionDeleted { return ErrExecutionUnavailable } - if owner.ComputePhase == "running" { - return nil + if owner.State == "released" || owner.State == "cleanup_pending" || owner.Expired { + // Public archive and deletion remain terminal. A qualified compute-only + // expiry leaves the same Environment live while the old writer is fenced. + current, err := w.dispatcher.SessionsReader.GetEnvironment(ctx, environment.TenantID, environment.ID) + if err != nil { + return err + } + if current.Status == "failed" || current.Status == "expired" { + return ErrExecutionUnavailable + } + if owner.State == "released" { + qualified, err := w.dispatcher.DeploymentReader.RetainedNativeHistory(ctx, key) + if err != nil { + return err + } + if !qualified { + return ErrExecutionUnavailable + } + // The live file operation is demand. A committed placement, rather than + // another wake setting, lets maintenance recover a detached caller. + next, err := w.ProvisionEnvironment(ctx, environment.TenantID, environment.ID, owner.ProviderKey) + if err != nil && next.ID == "" && !errors.Is(err, placement.ErrNodeUnavailable) && !errors.Is(err, placement.ErrNodesPreparing) && !errors.Is(err, deployment.ErrAllocationConflict) { + return err + } + } + } else if owner.State == "running" && owner.CreateSettled && (owner.ComputePhase == "disabled" || owner.ComputePhase == "running") { + // Bootstrap completion precedes daemon registration and its first + // capability declaration. File work must wait for both receipts. + peer, peerErr := w.dispatcher.authorizedPeer(ctx, owner.DeviceID) + if peerErr == nil { + session, err := w.dispatcher.SessionsReader.GetSession(ctx, environment.TenantID, environment.SessionID) + if err != nil { + return err + } + kind, found, known := peer.AgentKindStatus(session.Engine) + if known { + if !found || !kind.Available { + return ErrExecutionUnavailable + } + return nil + } + } else if !errors.Is(peerErr, sessions.ErrNotFound) && !errors.Is(peerErr, runtimegateway.ErrDeviceNotRegistered) && !errors.Is(peerErr, runtimegateway.ErrSessionClosed) { + return peerErr + } + } else if owner.State == "running" && touched != owner.ID { + if err = w.dispatcher.Deployment.TouchActivity(ctx, environment.TenantID, environment.ID); err != nil { + return err + } + touched = owner.ID + if w.runtimes != nil { + select { + case w.runtimes.hints(owner.NodeID) <- struct{}{}: + default: + } + } } select { case <-ctx.Done(): return ErrExecutionUnavailable case <-w.stopped: return ErrExecutionUnavailable - case <-timer.C: + case <-ticker.C: } } } diff --git a/services/core/internal/execution/runtime_compute_wake_test.go b/services/core/internal/execution/runtime_compute_wake_test.go new file mode 100644 index 000000000..5e554262d --- /dev/null +++ b/services/core/internal/execution/runtime_compute_wake_test.go @@ -0,0 +1,56 @@ +package execution + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" +) + +func TestRuntimeWakeMapsCancellationDuringAllocationRead(t *testing.T) { + storageError := errors.New("allocation read failed") + for _, test := range []struct { + name string + deadline bool + cancelBeforeResult bool + result error + want error + }{ + {name: "cancelled query", want: ErrExecutionUnavailable}, + {name: "deadline during query", deadline: true, want: ErrExecutionUnavailable}, + {name: "storage failure", result: storageError, want: storageError}, + {name: "storage failure concurrent with cancellation", cancelBeforeResult: true, result: storageError, want: storageError}, + } { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + if test.deadline { + cancel() + ctx, cancel = context.WithTimeout(t.Context(), time.Millisecond) + } + defer cancel() + called := false + reader := &strictDeploymentReader{t: t, environmentAllocation: func(query context.Context, _ deployment.AllocationKey) (deployment.Allocation, error) { + called = true + if test.cancelBeforeResult { + cancel() + } + if test.result != nil { + return deployment.Allocation{}, test.result + } + if !test.deadline { + cancel() + } + <-query.Done() + return deployment.Allocation{}, fmt.Errorf("timeout: %w", query.Err()) + }} + worker := &Worker{dispatcher: &Dispatcher{DeploymentReader: reader}, stopped: make(chan struct{})} + if err := worker.waitRuntimeAwake(ctx, sessions.Environment{}); !errors.Is(err, test.want) || !called { + t.Fatalf("allocation query outcome = %v, called = %v; want %v", err, called, test.want) + } + }) + } +} diff --git a/services/core/internal/execution/runtime_initialization.go b/services/core/internal/execution/runtime_initialization.go index 4f94d4d52..8c2072578 100644 --- a/services/core/internal/execution/runtime_initialization.go +++ b/services/core/internal/execution/runtime_initialization.go @@ -126,6 +126,17 @@ func (w *Worker) prepareEnvironment(ctx context.Context, owner sessions.Environm if err != nil { return err } + placement, err := parseEnvironmentPlacement(environment.Configuration) + if err != nil { + return err + } + if placement.Type == "openai_hosted" { + info, found, known := peer.AgentKindStatus(owner.Engine) + if !known || !found || !info.Available || w.dispatcher.validateEnvironmentHistory(owner.Engine, environment, info.Capabilities.RetainedNativeHistory) != nil { + *failure = sessions.ProvisioningFailure{Step: sessions.ProvisioningHarness} + return sessions.ErrInvalidInput + } + } identity := agentcapabilities.Identity{EnvironmentID: owner.EnvironmentID, SessionID: owner.SessionID} operations := setupOperations(setup) for index := 0; index < len(cfg.Files)+len(operations); index++ { diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 49e23afd4..4c2f7bb47 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/hex" + "encoding/json" "errors" "net/url" "sync" @@ -150,9 +151,23 @@ func (w *Worker) ProvisionEnvironment(ctx context.Context, tenant, environment, if !ready { return deployment.Allocation{}, ErrExecutionUnavailable } - nodeID, err := w.runtimes.deploymentService.LifecycleNode(ctx, tenant, environment) - if err != nil { - return deployment.Allocation{}, err + key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} + existing, lookupErr := w.runtimes.deploymentReader.EnvironmentAllocation(ctx, key) + if lookupErr != nil && !errors.Is(lookupErr, deployment.ErrNotFound) { + return deployment.Allocation{}, lookupErr + } + var nodeID string + if lookupErr == nil && existing.State == "released" { + reserved, reserveErr := w.runtimes.deployment.EnsurePlacement(ctx, key, providerKey) + if reserveErr != nil { + return deployment.Allocation{}, reserveErr + } + nodeID = reserved.NodeID + } else { + nodeID, err = w.runtimes.deploymentService.LifecycleNode(ctx, tenant, environment) + if err != nil { + return deployment.Allocation{}, err + } } node, err := w.runtimes.node(nodeID) if err != nil { @@ -180,23 +195,35 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if err != nil || placement.Type != "openai_hosted" { return deployment.Allocation{}, sandbox.ErrInvalid } + spec, err := r.workspaceSpecification(ctx, deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment}, "") + if err != nil { + return deployment.Allocation{}, err + } + if environmentValue.ExternalWorkspace && spec.Workspace == nil { + return deployment.Allocation{}, workspacefs.ErrUnsupported + } var workspace *workspacefs.Binding - if r.config.Workspace != nil { - if r.workspaces == nil { - return deployment.Allocation{}, workspacefs.ErrUnavailable - } + if spec.Workspace != nil { existing, lookupErr := r.reader.EnvironmentAllocation(ctx, deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment}) if lookupErr != nil && !errors.Is(lookupErr, deployment.ErrNotFound) { return deployment.Allocation{}, lookupErr } if errors.Is(lookupErr, deployment.ErrNotFound) { - workspace, err = r.workspaces.Ensure(ctx, tenant, environment, r.config.WorkspaceRequirements, r.config.Resources.EnvironmentDiskMiB) + workspace, err = r.workspaces.Ensure(ctx, tenant, environment, r.config.WorkspaceRequirements, spec.Resources.EnvironmentDiskMiB) if err == nil && workspace == nil { err = workspacefs.ErrUnavailable } if err != nil { return deployment.Allocation{}, err } + } else if existing.State == "released" { + workspace, err = r.workspaces.GetReady(ctx, tenant, environment, r.config.WorkspaceRequirements, spec.Resources.EnvironmentDiskMiB) + if err != nil { + return deployment.Allocation{}, err + } + if workspace == nil { + return deployment.Allocation{}, workspacefs.ErrNotFound + } } else if existing.NodeID != r.nodeID { return existing, sandbox.ErrOwnership } @@ -428,3 +455,28 @@ func runtimeReference(owner deployment.Allocation) sandbox.Reference { func (w *Worker) runManagedRuntimes(ctx context.Context) error { return w.runtimes.run(ctx) } + +// workspaceSpecification reads the immutable generation selected for this +// Environment. A restore also fences the exact allocation before native I/O. +func (r *runtimeLifecycle) workspaceSpecification(ctx context.Context, key deployment.AllocationKey, allocationID string) (sandbox.DeploymentSpec, error) { + target, err := r.reader.LifecyclePlacement(ctx, key) + if err != nil { + return sandbox.DeploymentSpec{}, err + } + if allocationID != "" && target.AllocationID != allocationID { + return sandbox.DeploymentSpec{}, sandbox.ErrOwnership + } + var spec sandbox.DeploymentSpec + if err := json.Unmarshal(target.Specification, &spec); err != nil { + return sandbox.DeploymentSpec{}, sandbox.ErrInvalid + } + if spec.Workspace != nil { + if r.workspaces == nil || r.config.WorkspaceRequirements == nil { + return sandbox.DeploymentSpec{}, workspacefs.ErrUnavailable + } + if err := workspacefs.ValidateCombination(*r.config.WorkspaceRequirements, *spec.Workspace, spec.Resources.EnvironmentDiskMiB); err != nil { + return sandbox.DeploymentSpec{}, err + } + } + return spec, nil +} diff --git a/services/core/internal/execution/runtime_manager.go b/services/core/internal/execution/runtime_manager.go index 9eea4150f..16d10815d 100644 --- a/services/core/internal/execution/runtime_manager.go +++ b/services/core/internal/execution/runtime_manager.go @@ -21,6 +21,7 @@ var errRuntimeTransition = fmt.Errorf("%w: sandbox configuration is changing", E type runtimeManager struct { workspaces *workspaces.ExecutionOperations workspaceCursor string + replacementCursor string workspaceGate chan struct{} sessions sessions.Reader sessionExecution *sessions.ExecutionOperations @@ -141,6 +142,9 @@ func (m *runtimeManager) syncNodes(ctx context.Context) ([]*runtimeNode, error) return nil, m.ctx.Err() } defer func() { <-m.inventory }() + if err := m.reserveReplacementPlacements(ctx); err != nil { + return nil, err + } // A direct caller may add a newly registered node during the query. Only // entries present before this inventory snapshot can be retired by it. previous := m.snapshotNodes() diff --git a/services/core/internal/execution/runtime_pending.go b/services/core/internal/execution/runtime_pending.go index cee0c94a3..9044224d9 100644 --- a/services/core/internal/execution/runtime_pending.go +++ b/services/core/internal/execution/runtime_pending.go @@ -5,6 +5,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" ) // provisionPending shares the existing lifecycle owner and serial gate. This @@ -33,3 +34,37 @@ func (r *runtimeLifecycle) provisionPending(ctx context.Context) error { } return nil } + +// reserveReplacementPlacements runs on the manager's existing inventory clock, +// before node scans. The committed input supplies demand; placement is the +// durable retry intent after the first caller or hint is gone. No native +// operation runs here and no retained idle filesystem consumes a reservation. +func (m *runtimeManager) reserveReplacementPlacements(ctx context.Context) error { + m.mu.Lock() + configuration := m.config + m.mu.Unlock() + if m.workspaces == nil || configuration.Workspace == nil { + return nil + } + operation, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + rows, err := m.deploymentReader.ReplacementEnvironments(operation, m.replacementCursor) + if err != nil { + return err + } + if len(rows) == 0 { + m.replacementCursor = "" + return nil + } + for _, environment := range rows { + m.replacementCursor = environment.ID + _, err = m.deployment.EnsurePlacement(operation, deployment.AllocationKey{TenantID: environment.TenantID, EnvironmentID: environment.ID}, configuration.InstallationID) + if err != nil { + if ownership := m.lease.CheckOwnership(ctx); ownership != nil { + return ownership + } + log.Ctx(ctx).Warn("managed Runtime placement incomplete", "environment_id", environment.ID) + } + } + return nil +} diff --git a/services/core/internal/execution/runtime_replacement_test.go b/services/core/internal/execution/runtime_replacement_test.go new file mode 100644 index 000000000..6b3623b46 --- /dev/null +++ b/services/core/internal/execution/runtime_replacement_test.go @@ -0,0 +1,428 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" + "github.com/google/uuid" + "github.com/gorilla/websocket" +) + +type retentionProvider struct { + sandbox.SandboxProvider + killError error + creates, kills, snapshots int + bootstraps chan sandbox.Bootstrap +} + +func (p *retentionProvider) ProviderOperations() providercontract.Operations { + return microsandbox.Operations() +} +func (p *retentionProvider) Create(_ context.Context, q sandbox.Bootstrap) (sandbox.Info, error) { + p.creates++ + if p.bootstraps != nil { + p.bootstraps <- q + } + return sandbox.Info{Reference: q.Reference, ProviderID: q.AllocationID, State: "running", BootstrapComplete: true, CreateSettled: true}, nil +} +func (p *retentionProvider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { + p.kills++ + return p.killError +} +func (p *retentionProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { + p.snapshots++ + return nil +} + +func TestExpiredRetainedComputePreservesSessionAndPendingInput(t *testing.T) { + provider := &retentionProvider{killError: sandbox.ErrComputeUnconfirmed} + fixture := newWorkspaceSettlementFixture(t, provider) + r, session := fixture.lifecycle, fixture.session + owner, err := r.provision(t.Context(), session.TenantID, session.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + binding, err := fixture.storage.Get(t.Context(), session.TenantID, session.Environment.ID) + if err != nil { + t.Fatal(err) + } + // Model the committed completion and suspension receipts. Provider callbacks + // remain deterministic fixtures; these records do not claim native evidence. + if _, err = fixture.pool.Exec(t.Context(), `UPDATE environments SET initialization='complete',status='disconnected' WHERE id=$1`, owner.EnvironmentID); err != nil { + t.Fatal(err) + } + if _, err = fixture.pool.Exec(t.Context(), `UPDATE devices SET supported_agent_kinds='[{"kind":"codex","available":true,"capabilities":{"retained_native_history":true}}]' WHERE id=$1`, owner.DeviceID); err != nil { + t.Fatal(err) + } + if _, err = fixture.pool.Exec(t.Context(), `UPDATE session_devices SET native_session_id='retained-native-session' WHERE device_id=$1`, owner.DeviceID); err != nil { + t.Fatal(err) + } + compute := runtimeCompute{Current: sandbox.Compute{ID: "old-compute"}, Snapshot: &sandbox.SnapshotIdentity{ID: "old-snapshot"}} + raw, _ := json.Marshal(compute) + if _, err = fixture.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state=$2,compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID, raw); err != nil { + t.Fatal(err) + } + input, err := fixture.sessions.ReserveEnvironmentInput(t.Context(), session.TenantID, session.ID, "after-retention", []sessions.Input{{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"resume"}]}]}`)}}) + if err != nil { + t.Fatal(err) + } + for _, settled := range []bool{false, true} { + if settled { + provider.killError = nil + } + owner, err = r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + err = r.observe(t.Context(), owner) + if !settled && !errors.Is(err, sandbox.ErrComputeUnconfirmed) { + t.Fatal("unknown old writer was settled", err) + } + if settled && err != nil { + t.Fatal(err) + } + current, err := r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + want := "cleanup_pending" + if settled { + want = "released" + } + if current.State != want || current.ID != owner.ID || !current.CreateSettled { + t.Fatal("compute ownership changed without settlement", current) + } + environment, err := r.sessions.GetEnvironment(t.Context(), session.TenantID, owner.EnvironmentID) + if err != nil || environment.Status != "disconnected" || environment.Initialization != "complete" { + t.Fatal("compute expiry terminated retained Environment", environment, err) + } + pending, err := r.sessions.GetEnvironmentInputReservation(t.Context(), session.TenantID, session.ID, input.ID) + if err != nil || pending.State != sessions.EnvironmentInputPending { + t.Fatal("compute expiry consumed pending input", pending, err) + } + var nativeID string + if err = fixture.pool.QueryRow(t.Context(), `SELECT native_session_id FROM session_devices WHERE session_id=$1`, session.ID).Scan(&nativeID); err != nil || nativeID != "retained-native-session" { + t.Fatal("compute expiry discarded native identity", nativeID, err) + } + if _, err = r.workspaces.DeleteBatch(t.Context(), ""); err != nil { + t.Fatal(err) + } + retained, err := fixture.storage.Get(t.Context(), session.TenantID, owner.EnvironmentID) + if err != nil || retained.Reference != binding.Reference || retained.Configuration.ID != binding.Configuration.ID || fixture.control.deletes != 0 { + t.Fatal("compute expiry changed retained filesystem", retained, err) + } + } + if provider.creates != 1 || provider.snapshots != 1 { + t.Fatal("expiry retried creation or deleted an unconfirmed writer snapshot", provider.creates, provider.snapshots) + } +} + +func TestRetainedEnvironmentDemandRecreatesCompute(t *testing.T) { + for _, demand := range []string{"input", "live_file"} { + t.Run(demand, func(t *testing.T) { + provider := &retentionProvider{} + fixture := workspaceSettlementFixtureWithSetup(t, provider, workspaceNodeSetup(t, true)) + r, session := fixture.lifecycle, fixture.session + owner, err := r.provision(t.Context(), session.TenantID, session.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + binding, err := fixture.storage.Get(t.Context(), session.TenantID, owner.EnvironmentID) + if err != nil { + t.Fatal(err) + } + for _, statement := range []struct { + q string + arg any + }{ + {`UPDATE environments SET initialization='complete',status='disconnected' WHERE id=$1`, owner.EnvironmentID}, + {`UPDATE devices SET supported_agent_kinds='[{"kind":"codex","available":true,"capabilities":{"retained_native_history":true}}]' WHERE id=$1`, owner.DeviceID}, + {`UPDATE session_devices SET native_session_id='retained-native-session' WHERE device_id=$1`, owner.DeviceID}, + {`UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"current":{"id":"old-compute"},"snapshot":{"id":"old-snapshot"}}',compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID}, + } { + if _, err = fixture.pool.Exec(t.Context(), statement.q, statement.arg); err != nil { + t.Fatal(err) + } + } + owner, err = r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + if err = r.observe(t.Context(), owner); err != nil { + t.Fatal(err) + } + owner, err = r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil || owner.State != "released" { + t.Fatal(owner, err) + } + r.config.Mode = string(sandbox.DeploymentNodes) + manager := &runtimeManager{ctx: t.Context(), config: r.config, setupGate: make(chan struct{}, 1), nodes: make(map[string]*runtimeNode), workspaces: r.workspaces, sessions: r.sessions, sessionExecution: r.sessionExecution, deployment: r.deployment, deploymentService: r.deployments, deploymentReader: r.reader, lease: r.lease, registry: r.registry} + // Neither retained storage nor an old receipt is demand by itself. + if err = manager.reserveReplacementPlacements(t.Context()); err != nil { + t.Fatal(err) + } + candidates, err := r.reader.UnallocatedEnvironments(t.Context(), r.nodeID, "") + if err != nil || len(candidates) != 0 { + t.Fatal("idle retained Environment reserved compute", candidates, err) + } + if demand == "input" { + _, err = fixture.sessions.ReserveEnvironmentInput(t.Context(), session.TenantID, session.ID, "replacement-input", []sessions.Input{{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"resume"}]}]}`)}}) + if err != nil { + t.Fatal(err) + } + if err = manager.reserveReplacementPlacements(t.Context()); err != nil { + t.Fatal(err) + } + if err = r.provisionPending(t.Context()); err != nil { + t.Fatal(err) + } + } else { + worker := &Worker{runtimes: manager, dispatcher: &Dispatcher{Registry: r.registry, DeploymentReader: r.reader, Deployment: r.deployments, SessionsReader: r.sessions}, stopped: make(chan struct{}), directoryReads: make(chan directoryReadRequest)} + provider.bootstraps = make(chan sandbox.Bootstrap, 1) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + completed := make(chan error, 1) + go func() { _, err := worker.ReadEnvironmentDirectory(ctx, *session.Environment, ""); completed <- err }() + var bootstrap sandbox.Bootstrap + select { + case bootstrap = <-provider.bootstraps: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + assertWaiting := func() { + t.Helper() + select { + case err := <-completed: + t.Fatal("file call returned before Runtime readiness", err) + case <-worker.directoryReads: + t.Fatal("file work queued before Runtime readiness") + case <-time.After(300 * time.Millisecond): + } + } + assertWaiting() + handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Authenticator: runtimegateway.NewAuthenticator(r.sessions), Registry: r.registry}) + server := httptest.NewServer(http.HandlerFunc(handler.WS)) + defer server.Close() + query := url.Values{"device_id": {bootstrap.DeviceID}, "version": {proto.Version}} + conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http")+"?"+query.Encode(), http.Header{"Authorization": {"Bearer " + bootstrap.Credential}}) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + assertWaiting() + heartbeat, _ := proto.NewEnvelope(proto.TypeHeartbeat, "", proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: prototest.Capabilities(proto.AgentKindCapabilities{LocalEnvironment: proto.CapabilitySupported, Preparation: proto.CapabilitySupported, WorkspaceReadPreparation: proto.CapabilitySupported, RetainedNativeHistory: proto.CapabilitySupported})}}}) + if err = conn.WriteJSON(heartbeat); err != nil { + t.Fatal(err) + } + // The public file entry point may enqueue only after the actual + // credential-authorized connection has advertised its Harness. + select { + case request := <-worker.directoryReads: + request.reply(directoryReadResult{directory: proto.WorkspaceDirectoryResult{Entries: []proto.WorkspaceDirectoryEntry{}}}) + case err := <-completed: + t.Fatal("file call failed", err) + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + if err = <-completed; err != nil { + t.Fatal(err) + } + } + replacement, err := r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil || replacement.ID == owner.ID || replacement.DeviceID == owner.DeviceID || replacement.State != "running" || !replacement.CreateSettled { + t.Fatal("demand failed to create fresh owner", replacement, err) + } + environment, err := r.sessions.GetEnvironment(t.Context(), session.TenantID, owner.EnvironmentID) + if err != nil || environment.Initialization != "complete" { + t.Fatal("replacement replayed initialization", environment, err) + } + retained, err := fixture.storage.Get(t.Context(), session.TenantID, owner.EnvironmentID) + if err != nil || retained.Reference != binding.Reference || retained.Configuration.ID != binding.Configuration.ID || provider.creates != 2 { + t.Fatal("replacement changed immutable filesystem or create count", retained, provider.creates, err) + } + var nativeID string + if err = fixture.pool.QueryRow(t.Context(), `SELECT native_session_id FROM session_devices WHERE session_id=$1`, session.ID).Scan(&nativeID); err != nil || nativeID != "retained-native-session" { + t.Fatal("replacement lost native history identity", nativeID, err) + } + }) + } +} + +func workspaceNodeSetup(t *testing.T, external bool) func(Owner, *deployment.Service, deployment.Reader) (string, string) { + return func(owner Owner, service *deployment.Service, reader deployment.Reader) (string, string) { + installation := uuid.NewString() + if err := owner.Deployment.Claim(t.Context(), installation); err != nil { + t.Fatal(err) + } + specification := sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048, RootDiskMiB: 8192}, Workspace: &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true}, Runtime: &sandbox.RuntimeRelease{SourceCommit: strings.Repeat("a", 40), ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64)}} + if !external { + specification.Workspace = nil + specification.Resources.EnvironmentDiskMiB = 8192 + } + view, err := owner.Deployment.Initialize(t.Context(), installation, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: specification}) + if err != nil { + t.Fatal(err) + } + token, err := service.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + if err != nil { + t.Fatal(err) + } + node := deployment.Enrollment{NodeID: uuid.NewString(), Name: "replacement node", Credential: strings.Repeat("n", 64), Provider: view.Provider, BackendFingerprint: strings.Repeat("b", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest, CoreURL: fixturePublicURL} + if _, err = service.Enroll(t.Context(), token.Token, node); err != nil { + t.Fatal(err) + } + epoch, err := reader.OwnerEpoch(t.Context()) + if err != nil { + t.Fatal(err) + } + connection := uuid.NewString() + if err = service.ConnectNode(t.Context(), node.NodeID, connection, epoch); err != nil { + t.Fatal(err) + } + if err = service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + t.Fatal(err) + } + return installation, node.NodeID + + } +} + +func TestProvisionUsesReservedGenerationStorageDuringUpdate(t *testing.T) { + for _, external := range []bool{false, true} { + t.Run(map[bool]string{false: "owned_ready_external_target", true: "external_ready_owned_target"}[external], func(t *testing.T) { + provider := &retentionProvider{bootstraps: make(chan sandbox.Bootstrap, 1)} + f := workspaceSettlementFixtureWithSetup(t, provider, workspaceNodeSetup(t, external)) + r, s := f.lifecycle, f.session + setup, err := r.deployments.Setup(t.Context()) + if err != nil { + t.Fatal(err) + } + target := setup.Specification + target.Workspace = &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true} + target.Resources.EnvironmentDiskMiB = 0 + if external { + target.Workspace = nil + target.Resources.EnvironmentDiskMiB = 8192 + } + if _, err = r.deployment.Update(adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", RequestID: uuid.NewString(), TraceID: uuid.NewString()}), r.config.InstallationID, sandbox.Selection{Provider: setup.Provider, ExpectedGeneration: setup.Generation, DeploymentSpec: target}); err != nil { + t.Fatal(err) + } + // Neither today's target nor the lane's cached storage mode selects storage. + r.config.Workspace = target.Workspace + owner, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + if owner.DeploymentGeneration != setup.Generation { + t.Fatal("lost reserved generation", owner) + } + bootstrap := <-provider.bootstraps + if (bootstrap.Workspace != nil) != external { + t.Fatal("storage mode followed target instead of reservation", external, bootstrap.Workspace) + } + environment, err := r.sessions.GetEnvironment(t.Context(), s.TenantID, s.Environment.ID) + if err != nil || environment.ExternalWorkspace != external { + t.Fatal("immutable binding projection", environment, err) + } + }) + } +} + +func TestProvisionRejectsRetainedBindingOnOwnedGeneration(t *testing.T) { + provider := &retentionProvider{bootstraps: make(chan sandbox.Bootstrap, 1)} + f := workspaceSettlementFixtureWithSetup(t, provider, workspaceNodeSetup(t, false)) + r, s := f.lifecycle, f.session + if _, err := r.workspaces.Ensure(t.Context(), s.TenantID, s.Environment.ID, r.config.WorkspaceRequirements, 0); err != nil { + t.Fatal(err) + } + if _, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID); !errors.Is(err, workspacefs.ErrUnsupported) { + t.Fatal("binding silently dropped", err) + } + if provider.creates != 0 { + t.Fatal("incompatible compute was created") + } + if _, err := r.reader.EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: s.TenantID, EnvironmentID: s.Environment.ID}); !errors.Is(err, deployment.ErrNotFound) { + t.Fatal("incompatible compute reserved", err) + } +} + +// A deterministic provider stops after capturing Resume, before subsequent +// wake operations. The allocation, generation and filesystem binding are real. +type resumeStorageProvider struct { + retentionProvider + request *sandbox.ResumeRequest + stop error +} + +func (p *resumeStorageProvider) Resume(_ context.Context, request sandbox.ResumeRequest) (sandbox.ComputeState, error) { + p.request = &request + return sandbox.ComputeState{}, p.stop +} + +func TestRestoreUsesAllocationGenerationStorageInsteadOfCachedLane(t *testing.T) { + for _, external := range []bool{false, true} { + t.Run(map[bool]string{false: "owned_allocation_external_lane", true: "external_allocation_owned_lane"}[external], func(t *testing.T) { + stop := errors.New("resume request captured") + provider := &resumeStorageProvider{stop: stop} + f := workspaceSettlementFixtureWithSetup(t, provider, workspaceNodeSetup(t, external)) + r, s := f.lifecycle, f.session + owner, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + setup, err := r.deployments.Setup(t.Context()) + if err != nil { + t.Fatal(err) + } + target := setup.Specification + target.Workspace = &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true} + target.Resources.EnvironmentDiskMiB = 0 + if external { + target.Workspace = nil + target.Resources.EnvironmentDiskMiB = 8192 + } + audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", RequestID: uuid.NewString(), TraceID: uuid.NewString()}) + if _, err = r.deployment.Update(audit, r.config.InstallationID, sandbox.Selection{Provider: setup.Provider, ExpectedGeneration: setup.Generation, DeploymentSpec: target}); err != nil { + t.Fatal(err) + } + r.config.Workspace = target.Workspace + r.config.Resources = target.Resources + state := runtimeCompute{Target: &sandbox.Compute{ID: "replacement-compute", Generation: 2}, Snapshot: &sandbox.SnapshotIdentity{ID: "snapshot"}, RestoreID: uuid.NewString()} + if err = r.restoreCompute(t.Context(), provider, owner, state, false); !errors.Is(err, stop) { + t.Fatal(err) + } + if provider.request == nil || (provider.request.Workspace != nil) != external { + t.Fatal("Resume binding followed cached lane", external, provider.request) + } + if external { + stored, err := f.storage.Get(t.Context(), s.TenantID, s.Environment.ID) + if err != nil || provider.request.Workspace.Attachment.Reference != stored.Reference { + t.Fatal("Resume lost retained object identity", err) + } + } + provider.request = nil + stale := owner + stale.ID = uuid.NewString() + if err = r.restoreCompute(t.Context(), provider, stale, state, false); !errors.Is(err, sandbox.ErrOwnership) || provider.request != nil { + t.Fatal("stale owner reached native Resume", err) + } + }) + } +} diff --git a/services/core/internal/execution/runtime_workspace_settlement_test.go b/services/core/internal/execution/runtime_workspace_settlement_test.go index 101e76529..ce1bba4c9 100644 --- a/services/core/internal/execution/runtime_workspace_settlement_test.go +++ b/services/core/internal/execution/runtime_workspace_settlement_test.go @@ -21,6 +21,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspaces" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" ) // settledWorkspaceRefusal models the provider's pre-native filesystem rejection: @@ -42,7 +43,7 @@ type settlementWorkspaceControl struct { } func (c *settlementWorkspaceControl) Declaration() workspacefs.Declaration { - return workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory} + return workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true} } func (c *settlementWorkspaceControl) Create(_ context.Context, ref workspacefs.Reference) (workspacefs.Attachment, error) { return workspacefs.Attachment{Reference: ref, ConfigurationID: c.configuration.ID, Kind: workspacefs.AttachmentHostDirectory, Native: []byte(`{}`)}, nil @@ -62,6 +63,7 @@ func (c settlementWorkspaceControls) Normalize(config workspacefs.Configuration) } type workspaceSettlementFixture struct { + pool *pgxpool.Pool lifecycle *runtimeLifecycle sessions *sessions.Service storage *workspacepg.Store @@ -70,6 +72,10 @@ type workspaceSettlementFixture struct { } func newWorkspaceSettlementFixture(t *testing.T, provider sandbox.SandboxProvider) workspaceSettlementFixture { + return workspaceSettlementFixtureWithSetup(t, provider, nil) +} + +func workspaceSettlementFixtureWithSetup(t *testing.T, provider sandbox.SandboxProvider, setup func(Owner, *deployment.Service, deployment.Reader) (string, string)) workspaceSettlementFixture { t.Helper() pool := pgtest.OpenIsolated(t, nil) @@ -80,7 +86,12 @@ func newWorkspaceSettlementFixture(t *testing.T, provider sandbox.SandboxProvide t.Cleanup(func() { _ = lease.Close(context.Background()) }) deployments, reader, operations := testDeployment(t, pool, pgtest.CredentialKey(t), lease) owner := Owner{Lease: lease, Deployment: operations, Sessions: sessionExecution(t, lease)} - installation := initializeE2BDeployment(t, owner) + var installation, nodeID string + if setup == nil { + installation, nodeID = workspaceNodeSetup(t, true)(owner, deployments, reader) + } else { + installation, nodeID = setup(owner, deployments, reader) + } projectID := uuid.NewString() audit := adminaudit.WithSource(t.Context(), adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) management, err := projects.NewService(projectpg.New(pgunit.NewPool(pool))) @@ -92,7 +103,7 @@ func newWorkspaceSettlementFixture(t *testing.T, provider sandbox.SandboxProvide t.Fatal(err) } sessionReader, service := testSessions(t, pool, pgtest.CredentialKey(t)) - created, err := service.CreateSession(t.Context(), project.TenantID, sessions.CreateSession{Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`), ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ExecutionSourceSession}) + created, err := service.CreateSession(t.Context(), project.TenantID, sessions.CreateSession{SupportsRetainedNativeHistory: true, Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`), ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ExecutionSourceSession}) if err != nil { t.Fatal(err) } @@ -107,8 +118,8 @@ func newWorkspaceSettlementFixture(t *testing.T, provider sandbox.SandboxProvide control := &settlementWorkspaceControl{configuration: configuration} filesystems := workspaces.NewExecution(storage, writer, settlementWorkspaceControls{control}, lease) declaration := control.Declaration() - lifecycle := &runtimeLifecycle{registry: runtimegateway.NewRegistry(), sessions: sessionReader, sessionExecution: owner.Sessions, deployment: operations, deployments: deployments, reader: reader, lease: lease, workspaces: filesystems, config: RuntimeProvider{InstallationID: installation, Provider: provider, Workspace: &declaration, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} - return workspaceSettlementFixture{lifecycle: lifecycle, sessions: service, storage: storage, control: control, session: session} + lifecycle := &runtimeLifecycle{nodeID: nodeID, registry: runtimegateway.NewRegistry(), sessions: sessionReader, sessionExecution: owner.Sessions, deployment: operations, deployments: deployments, reader: reader, lease: lease, workspaces: filesystems, config: RuntimeProvider{InstallationID: installation, Provider: provider, Workspace: &declaration, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} + return workspaceSettlementFixture{pool: pool, lifecycle: lifecycle, sessions: service, storage: storage, control: control, session: session} } func TestSettledWorkspaceRefusalRetainsReceiptUntilExplicitSessionDeletion(t *testing.T) { @@ -126,7 +137,7 @@ func TestSettledWorkspaceRefusalRetainsReceiptUntilExplicitSessionDeletion(t *te t.Fatal("missing durable failure", environment.Status, err) } replay, err := lifecycle.provision(t.Context(), session.TenantID, session.Environment.ID, installation) - if err != nil || !replay.Replayed || replay.ID != allocation.ID || provider.creates != 1 { + if !errors.Is(err, deployment.ErrAllocationConflict) || replay.ID != "" || provider.creates != 1 { t.Fatal("settled failed creation retried compute", replay, err) } if _, err = filesystems.DeleteBatch(t.Context(), ""); err != nil || control.deletes != 0 { diff --git a/services/core/internal/execution/runtime_workspace_test.go b/services/core/internal/execution/runtime_workspace_test.go index 54ca2141c..cd5d1cc1a 100644 --- a/services/core/internal/execution/runtime_workspace_test.go +++ b/services/core/internal/execution/runtime_workspace_test.go @@ -93,7 +93,9 @@ func TestRuntimeWorkspaceConvergesBeforeComputeReservation(t *testing.T) { } return stop }}) - r := &runtimeLifecycle{sessions: workspaceEnvironmentReader{}, deployment: operations, reader: &strictDeploymentReader{t: t, environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + r := &runtimeLifecycle{sessions: workspaceEnvironmentReader{}, deployment: operations, reader: &strictDeploymentReader{t: t, lifecyclePlacement: func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) { + return deployment.LifecyclePlacement{Specification: []byte(`{"workspace":{"attachment":"host_directory"}}`)}, nil + }, environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { return deployment.Allocation{}, deployment.ErrNotFound }}, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{InstallationID: uuid.NewString(), Workspace: &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory}, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} _, err := r.provision(t.Context(), f.record.Reference.TenantID, f.record.Reference.EnvironmentID, r.config.InstallationID) @@ -151,7 +153,9 @@ func TestOwnedGenerationDoesNotAdoptLaterFilesystemSelection(t *testing.T) { _, operations := deploymentOperations(t, &strictDeploymentStorage{t: t}, &strictDeploymentReader{t: t}, &strictExecutionStorage{t: t, withReservation: func(context.Context, deployment.AllocationKey, func(sessions.LockedSession, deployment.ReservationTx) error) error { return stop }}) - r := &runtimeLifecycle{sessions: workspaceEnvironmentReader{}, deployment: operations, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{InstallationID: uuid.NewString()}} + r := &runtimeLifecycle{reader: &strictDeploymentReader{t: t, lifecyclePlacement: func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) { + return deployment.LifecyclePlacement{Specification: []byte(`{}`)}, nil + }}, sessions: workspaceEnvironmentReader{}, deployment: operations, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{InstallationID: uuid.NewString()}} _, err := r.provision(t.Context(), f.record.Reference.TenantID, f.record.Reference.EnvironmentID, r.config.InstallationID) if !errors.Is(err, stop) || f.binds != 0 || f.creates != 0 { t.Fatal("owned generation retroactively attached filesystem", err) diff --git a/services/core/internal/execution/support.go b/services/core/internal/execution/support.go index 388a4d544..0a264fd7e 100644 --- a/services/core/internal/execution/support.go +++ b/services/core/internal/execution/support.go @@ -1,6 +1,7 @@ package execution import ( + "context" "encoding/json" "errors" "strings" @@ -144,3 +145,32 @@ func (p Policy) engineCapabilities(peer *runtimegateway.Session, engine string, } return caps, nil } + +// sessionCapabilities shares storage-specific validation between device selection +// and the final preparation gate. The immutable binding, not today's deployment, +// determines whether this Session requires native history outside compute. +func (d *Dispatcher) sessionCapabilities(ctx context.Context, peer *runtimegateway.Session, session sessions.Session, snapshot Snapshot) (runtimedevice.KindCapabilities, error) { + caps, err := d.engineCapabilities(peer, session.Engine, snapshot) + if err != nil { + return caps, err + } + if snapshot.Environment == nil || snapshot.Environment.Type != "openai_hosted" { + return caps, nil + } + environment, err := d.SessionsReader.GetSessionEnvironment(ctx, session.TenantID, session.ID) + if err != nil { + return runtimedevice.KindCapabilities{}, err + } + if err := d.validateEnvironmentHistory(session.Engine, environment, caps.RetainedNativeHistory); err != nil { + return runtimedevice.KindCapabilities{}, err + } + return caps, nil +} + +func (p Policy) validateEnvironmentHistory(engine string, environment sessions.Environment, supported bool) error { + profile, found := p.Engines.Lookup(engine) + if !found { + return sessions.ErrInvalidInput + } + return sessions.ValidateRetainedHistory(environment.ExternalWorkspace, supported && profile.RetainedNativeHistory.IsSupported()) +} diff --git a/services/core/internal/execution/tool_search_test.go b/services/core/internal/execution/tool_search_test.go index 80a5ea7dc..314098b0f 100644 --- a/services/core/internal/execution/tool_search_test.go +++ b/services/core/internal/execution/tool_search_test.go @@ -22,6 +22,7 @@ func TestDiscoveryUsesSharedOperationQualification(t *testing.T) { workspace := strings.Replace(discoveryConfiguration, `"type":"none"`, `"type":"openai_hosted"`, 1) policy := Policy{Engines: engine.NewCatalog(map[string]engine.Profile{"new_harness": enginetest.Profile(func(p *engine.Profile) { p.Placements = []string{"openai_hosted"} + p.RetainedNativeHistory = proto.CapabilitySupported p.ToolSearch = proto.CapabilitySupported })})} if err := policy.ValidateSessionConfiguration("new_harness", json.RawMessage(workspace)); err != nil { diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 92fabc153..74d032d5d 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -133,6 +133,11 @@ func (w *Worker) CreateSession(ctx context.Context, tenant string, input session if err := w.validateCreation(ctx, input); err != nil { return sessions.Creation{}, err } + profile, ok := w.dispatcher.Engines.Lookup(input.Engine) + if !ok { + return sessions.Creation{}, sessions.ErrInvalidInput + } + input.SupportsRetainedNativeHistory = profile.RetainedNativeHistory.IsSupported() creation, err := w.dispatcher.Sessions.CreateSession(ctx, tenant, input) if err == nil && len(input.InitialInputs) > 0 { w.wakeScheduler() diff --git a/services/core/internal/execution/worker_device.go b/services/core/internal/execution/worker_device.go index 015ac962e..dcc634ddf 100644 --- a/services/core/internal/execution/worker_device.go +++ b/services/core/internal/execution/worker_device.go @@ -58,7 +58,7 @@ func (w *Worker) bindDevice(ctx context.Context, tenantID, sessionID string, inp return false, err } return w.bindSessionDevice(ctx, session, func(id string) bool { - if !w.ready(ctx, id, session.Engine, snapshot) { + if !w.ready(ctx, id, session, snapshot) { return false } if !input.HasImages() { @@ -121,11 +121,11 @@ func (w *Worker) bindSessionDevice(ctx context.Context, session sessions.Session return false, nil } -func (w *Worker) ready(ctx context.Context, deviceID, engine string, snapshot Snapshot) bool { +func (w *Worker) ready(ctx context.Context, deviceID string, session sessions.Session, snapshot Snapshot) bool { peer, err := w.dispatcher.authorizedPeer(ctx, deviceID) if err != nil { return false } - _, err = w.dispatcher.engineCapabilities(peer, engine, snapshot) + _, err = w.dispatcher.sessionCapabilities(ctx, peer, session, snapshot) return err == nil } diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index ee99f8f4d..fec458e20 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -33,7 +33,7 @@ func allocation(row sqlc.RuntimeAllocation, session, tenant pgtype.UUID, deleted // findAllocation reads the tenant's allocation of the Environment; found is // false when it has none. func findAllocation(ctx context.Context, q *sqlc.Queries, tenant, environment pgtype.UUID) (deployment.Allocation, bool, error) { - row, err := q.GetRuntimeAllocation(ctx, sqlc.GetRuntimeAllocationParams{TenantID: tenant, EnvironmentID: environment}) + row, err := q.GetLatestRuntimeAllocation(ctx, sqlc.GetLatestRuntimeAllocationParams{TenantID: tenant, EnvironmentID: environment}) if errors.Is(err, pgx.ErrNoRows) { return deployment.Allocation{}, false, nil } @@ -116,7 +116,7 @@ func (e *Execution) withAllocation(ctx context.Context, key deployment.Allocatio } return translate(e.lease.Transaction(ctx, func(ctx context.Context, tx pgx.Tx) error { q := sqlc.New(tx) - row, err := q.GetRuntimeAllocation(ctx, sqlc.GetRuntimeAllocationParams{TenantID: tenant, EnvironmentID: environment}) + row, err := q.GetLatestRuntimeAllocation(ctx, sqlc.GetLatestRuntimeAllocationParams{TenantID: tenant, EnvironmentID: environment}) if errors.Is(err, pgx.ErrNoRows) { return deployment.ErrNotFound } @@ -197,6 +197,40 @@ func (t *reservationTx) LoadReserved() (placement.Reserved, error) { return placementpg.LoadReserved(t.ctx, t.q, t.environment) } +func (t *reservationTx) CanReplaceAllocation() (bool, error) { + current, found, err := t.FindAllocation() + if err != nil || !found { + return false, err + } + id, err := parseID(current.ID) + if err != nil { + return false, err + } + allowed, err := t.q.CanReplaceRuntimeAllocation(t.ctx, id) + if err != nil || !allowed { + return false, err + } + return t.q.CanRetainRuntimeEnvironment(t.ctx, id) +} + +func (t *reservationTx) LoadNodes() ([]placement.Node, error) { + return placementpg.LoadNodes(t.ctx, t.q) +} + +func (t *reservationTx) ReservePlacement(p placement.Placement) error { + node, err := parseID(p.NodeID) + if err != nil { + return err + } + count, err := t.q.ReserveReleasedRuntimePlacement(t.ctx, sqlc.ReserveReleasedRuntimePlacementParams{ + EnvironmentID: t.environment, NodeID: node, DeploymentGeneration: pgtype.Int8{Int64: int64(p.Generation), Valid: true}, + }) + if err == nil && count != 1 { + return deployment.ErrAllocationConflict + } + return err +} + func (t *reservationTx) InsertAllocation(a deployment.NewAllocation) (deployment.Allocation, error) { id, err := parseID(a.ID) if err != nil { @@ -331,6 +365,14 @@ type cleanupTx struct { *sessionpg.SessionTx } +func (t *cleanupTx) CanRetainEnvironment(current deployment.Allocation) (bool, error) { + id, err := parseID(current.ID) + if err != nil { + return false, err + } + return t.q.CanRetainRuntimeEnvironment(t.ctx, id) +} + func (t *cleanupTx) RevokeDevice(current deployment.Allocation) error { device, err := parseID(current.DeviceID) if err != nil { @@ -352,7 +394,7 @@ func loadActivity(ctx context.Context, q *sqlc.Queries, id pgtype.UUID) (deploym if err != nil { return deployment.Activity{}, err } - return deployment.Activity{LastActivity: row.LastActivity.Time, ObservedAt: row.ObservedAt.Time, Busy: row.Busy, WakeRequested: row.ComputeWakeRequested, HasCompletedTurn: row.HasCompletedTurn}, nil + return deployment.Activity{LastActivity: row.LastActivity.Time, ObservedAt: row.ObservedAt.Time, Busy: row.Busy, WakeRequested: row.ComputeWakeRequested}, nil } func (s *Store) EnvironmentAllocation(ctx context.Context, key deployment.AllocationKey) (deployment.Allocation, error) { @@ -511,6 +553,36 @@ func (s *Store) UnallocatedEnvironments(ctx context.Context, nodeID, after strin return result, nil } +func (s *Store) RetainedNativeHistory(ctx context.Context, key deployment.AllocationKey) (bool, error) { + current, err := s.EnvironmentAllocation(ctx, key) + if err != nil { + return false, err + } + id, err := parseID(current.ID) + if err != nil { + return false, err + } + return s.pool.Queries().CanRetainRuntimeEnvironment(ctx, id) +} + +func (s *Store) ReplacementEnvironments(ctx context.Context, after string) ([]deployment.UnallocatedEnvironment, error) { + id, err := cursor(after) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, pgunit.ExecutionTimeout) + defer cancel() + rows, err := s.pool.Queries().ListReplacementEnvironments(ctx, id) + if err != nil { + return nil, err + } + result := make([]deployment.UnallocatedEnvironment, 0, len(rows)) + for _, row := range rows { + result = append(result, deployment.UnallocatedEnvironment{ID: uuidString(row.ID), TenantID: uuidString(row.TenantID)}) + } + return result, nil +} + func (s *Store) LifecyclePlacement(ctx context.Context, key deployment.AllocationKey) (deployment.LifecyclePlacement, error) { tenant, environment, err := allocationKey(key) if err != nil { @@ -525,7 +597,7 @@ func (s *Store) LifecyclePlacement(ctx context.Context, key deployment.Allocatio if err != nil { return deployment.LifecyclePlacement{}, err } - return deployment.LifecyclePlacement{Provider: row.ProviderKind, Mode: row.Mode, AllocationID: uuidString(row.AllocationID), AllocationNodeID: uuidString(row.AllocationNodeID), + return deployment.LifecyclePlacement{Specification: row.Specification, Provider: row.ProviderKind, Mode: row.Mode, AllocationID: uuidString(row.AllocationID), AllocationNodeID: uuidString(row.AllocationNodeID), PlacementNodeID: uuidString(row.PlacementNodeID), PlacementReleased: row.ReleasedAt.Valid}, nil } @@ -536,3 +608,7 @@ func (s *Store) Activity(ctx context.Context, allocationID string) (deployment.A } return loadActivity(ctx, s.pool.Queries(), id) } + +func (t *reservationTx) LoadGenerationSpecification(generation uint64) (deployment.GenerationSpecification, error) { + return (unit{ctx: t.ctx, q: t.q}).LoadGenerationSpecification(generation) +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/replacement_test.go b/services/core/internal/persistence/postgres/deploymentpg/replacement_test.go new file mode 100644 index 000000000..8b3a3d984 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/replacement_test.go @@ -0,0 +1,411 @@ +package deploymentpg_test + +import ( + "errors" + "sync" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" + "github.com/google/uuid" +) + +// releaseRetainedFixture records native cleanup completion after exercising the +// production expiry decision. It does not claim to qualify a native provider. +func releaseRetainedFixture(t *testing.T, f fixture, changes *deployment.ExecutionOperations, owner deployment.Allocation) deployment.Allocation { + t.Helper() + config := uuid.NewString() + if _, err := f.pool.Exec(t.Context(), `INSERT INTO workspace_fs_configurations(id,adapter,parameters) VALUES($1,'fixture','{}')`, config); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `INSERT INTO environment_workspaces(object_id,environment_id,configuration_id,state,attachment) VALUES($1,$2,$3,'ready','{}') ON CONFLICT(environment_id) DO NOTHING`, uuid.NewString(), owner.EnvironmentID, config); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `UPDATE devices SET supported_agent_kinds='[{"kind":"codex","available":true,"capabilities":{"retained_native_history":true}}]' WHERE id=$1`, owner.DeviceID); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET state='running',create_settled=true,compute_phase='suspended',compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID); err != nil { + t.Fatal(err) + } + owner, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + owner, err = changes.RequestCleanup(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + owner, err = changes.ReleaseAllocation(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + return owner +} + +func TestReplacementKeepsSessionIdentityAndFencesPriorOwner(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + firstNode := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, firstNode.NodeID) + secondNode := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, secondNode.NodeID) + key := hostedEnvironment(t, f.pool) + if _, err := f.pool.Exec(t.Context(), `INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) VALUES($1,$2,$3)`, key.EnvironmentID, firstNode.NodeID, view.Generation); err != nil { + t.Fatal(err) + } + first, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `UPDATE session_devices SET native_session_id='retained-native-session' WHERE device_id=$1`, first.DeviceID); err != nil { + t.Fatal(err) + } + first = releaseRetainedFixture(t, f, changes, first) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, firstNode.NodeID); err != nil { + t.Fatal(err) + } + reserved, err := changes.EnsurePlacement(t.Context(), key, installation) + if err != nil || reserved.NodeID != secondNode.NodeID { + t.Fatal(reserved, err) + } + // Replayed old cleanup must not release the new reservation, even before Create. + if _, err := changes.ReleaseAllocation(t.Context(), first); err != nil { + t.Fatal(err) + } + var released bool + if err := f.pool.QueryRow(t.Context(), `SELECT released_at IS NOT NULL FROM runtime_placements WHERE environment_id=$1`, key.EnvironmentID).Scan(&released); err != nil || released { + t.Fatal("late release removed reservation", released, err) + } + owners := make([]deployment.Allocation, 6) + errs := make([]error, len(owners)) + var wg sync.WaitGroup + for i := range owners { + wg.Go(func() { + owners[i], errs[i] = changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + }) + } + wg.Wait() + fresh := 0 + for i, owner := range owners { + if errs[i] != nil || owner.ID != owners[0].ID || owner.NodeID != secondNode.NodeID { + t.Fatal(owner, errs[i]) + } + if !owner.Replayed { + fresh++ + } + } + second := owners[0] + if fresh != 1 || second.ID == first.ID || second.DeviceID == first.DeviceID { + t.Fatal("replacement reused identity", fresh, first, second) + } + var device, native string + if err := f.pool.QueryRow(t.Context(), `SELECT device_id,native_session_id FROM session_devices WHERE session_id=$1`, second.SessionID).Scan(&device, &native); err != nil || device != second.DeviceID || native != "retained-native-session" { + t.Fatal(device, native, err) + } + for name, write := range map[string]func() (deployment.Allocation, error){"cleanup": func() (deployment.Allocation, error) { return changes.RequestCleanup(t.Context(), first) }, "release": func() (deployment.Allocation, error) { return changes.ReleaseAllocation(t.Context(), first) }, "observe": func() (deployment.Allocation, error) { return changes.ObserveRunning(t.Context(), first) }} { + if _, err := write(); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal(name, "accepted stale owner", err) + } + } + var history, current, authority int + if err := f.pool.QueryRow(t.Context(), `SELECT count(*),count(*) FILTER(WHERE state<>'released') FROM runtime_allocations WHERE environment_id=$1`, key.EnvironmentID).Scan(&history, ¤t); err != nil || history != 2 || current != 1 { + t.Fatal(history, current, err) + } + if err := f.pool.QueryRow(t.Context(), `SELECT count(*) FROM runtime_device_authority WHERE id=$1`, first.DeviceID).Scan(&authority); err != nil || authority != 0 { + t.Fatal("old credential still authorized", authority, err) + } + var oldNode string + if err := f.pool.QueryRow(t.Context(), `SELECT node_id FROM runtime_allocations WHERE id=$1`, first.ID).Scan(&oldNode); err != nil || oldNode != firstNode.NodeID { + t.Fatal("historical node changed", oldNode, err) + } + nodes, err := f.adapter.Nodes(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, node := range nodes { + if node.ID == secondNode.NodeID && (node.Active != 1 || node.Retained != 1) { + t.Fatal("history counted as compute", node) + } + } + // The database independently rejects a second live owner, even outside the domain transaction. + otherDevice := uuid.NewString() + if _, err := f.pool.Exec(t.Context(), `INSERT INTO devices(id,tenant_id,name,credential_hash) VALUES($1,$2,'fixture',$3)`, otherDevice, key.TenantID, credentialHash()); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,node_id,deployment_generation) VALUES($1,$2,$3,$4,$5,$6)`, uuid.NewString(), key.EnvironmentID, otherDevice, installation, secondNode.NodeID, view.Generation); err == nil { + t.Fatal("database accepted two current allocations") + } +} + +func TestReplacementRefusesUnsettledOwnershipAndUnknownHistory(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, node.NodeID) + key := hostedEnvironment(t, f.pool) + if _, err := f.pool.Exec(t.Context(), `INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) VALUES($1,$2,$3)`, key.EnvironmentID, node.NodeID, view.Generation); err != nil { + t.Fatal(err) + } + owner, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if err != nil { + t.Fatal(err) + } + if _, err := changes.EnsurePlacement(t.Context(), key, installation); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal("unknown writer admitted replacement", err) + } + replay, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if err != nil || !replay.Replayed || replay.ID != owner.ID { + t.Fatal("unknown Create was replayed", replay, err) + } + owner = releaseRetainedFixture(t, f, changes, owner) + for _, raw := range []string{`[]`, `[{"kind":"codex","available":true,"capabilities":{}}]`, `[{"kind":"codex","available":false,"capabilities":{"retained_native_history":true}}]`, `[{"kind":"different","available":true,"capabilities":{"retained_native_history":true}}]`} { + if _, err := f.pool.Exec(t.Context(), `UPDATE devices SET supported_agent_kinds=$2 WHERE id=$1`, owner.DeviceID, raw); err != nil { + t.Fatal(err) + } + if allowed, err := f.adapter.RetainedNativeHistory(t.Context(), key); err != nil || allowed { + t.Fatal("unknown history admitted", raw, allowed, err) + } + if _, err := changes.EnsurePlacement(t.Context(), key, installation); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal("unknown history reserved replacement", err) + } + } + if _, err := f.pool.Exec(t.Context(), `UPDATE devices SET supported_agent_kinds='[{"kind":"codex","available":true,"capabilities":{"retained_native_history":true}}]',revoked_at=NULL WHERE id=$1`, owner.DeviceID); err != nil { + t.Fatal(err) + } + if _, err := changes.EnsurePlacement(t.Context(), key, installation); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal("old device still had authority", err) + } +} + +func TestTenRetainedWorkspacesShareThreeComputeSlots(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 3, MaxRetained: 3}) + f.connect(t, node.NodeID) + keys := make([]deployment.AllocationKey, 10) + for i := range keys { + key := hostedEnvironment(t, f.pool) + keys[i] = key + if _, err := f.pool.Exec(t.Context(), `INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) VALUES($1,$2,$3)`, key.EnvironmentID, node.NodeID, view.Generation); err != nil { + t.Fatal(err) + } + owner, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if err != nil { + t.Fatal(err) + } + releaseRetainedFixture(t, f, changes, owner) + } + check := func(active, retained int64) { + t.Helper() + nodes, err := f.adapter.Nodes(t.Context()) + if err != nil || len(nodes) != 1 { + t.Fatal(nodes, err) + } + if nodes[0].Active != active || nodes[0].Retained != retained { + t.Fatal("incorrect compute accounting", nodes[0]) + } + var objects int + if err := f.pool.QueryRow(t.Context(), `SELECT count(*) FROM environment_workspaces WHERE state='ready'`).Scan(&objects); err != nil || objects != 10 { + t.Fatal("retained filesystem count", objects, err) + } + } + check(0, 0) + owners := make([]deployment.Allocation, 3) + for i := range owners { + if _, err := changes.EnsurePlacement(t.Context(), keys[i], installation); err != nil { + t.Fatal(err) + } + owner, err := changes.ReserveAllocation(t.Context(), keys[i], installation, credentialHash()) + if err != nil { + t.Fatal(err) + } + owners[i] = owner + } + check(3, 3) + if _, err := changes.EnsurePlacement(t.Context(), keys[3], installation); err == nil { + t.Fatal("fourth demand exceeded capacity") + } + var count int + if err := f.pool.QueryRow(t.Context(), `SELECT count(*) FROM runtime_allocations WHERE environment_id=$1`, keys[3].EnvironmentID).Scan(&count); err != nil || count != 1 { + t.Fatal("rejected demand created allocation", count, err) + } + check(3, 3) + releaseRetainedFixture(t, f, changes, owners[0]) + check(2, 2) + if _, err := changes.EnsurePlacement(t.Context(), keys[3], installation); err != nil { + t.Fatal(err) + } + // A committed reservation is itself retry demand for live file access. + pending, err := f.adapter.UnallocatedEnvironments(t.Context(), node.NodeID, "") + if err != nil || len(pending) != 1 || pending[0].ID != keys[3].EnvironmentID { + t.Fatal("reservation lost without pending input", pending, err) + } + if _, err := changes.ReserveAllocation(t.Context(), keys[3], installation, credentialHash()); err != nil { + t.Fatal(err) + } + check(3, 3) +} + +func TestResetArchivesRetainedSessionBeforeClearingDeployment(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, node.NodeID) + retained := make([]deployment.Allocation, 37) + for i := range retained { + key := hostedEnvironment(t, f.pool) + if _, err := f.pool.Exec(t.Context(), `INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) VALUES($1,$2,$3)`, key.EnvironmentID, node.NodeID, view.Generation); err != nil { + t.Fatal(err) + } + owner, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if err != nil { + t.Fatal(err) + } + retained[i] = releaseRetainedFixture(t, f, changes, owner) + } + target := retained[0] + // Only the exact latest ownership in this reset's installation and boundary counts. + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET provider_key=$2 WHERE id=$1`, retained[1].ID, uuid.NewString()); err != nil { + t.Fatal(err) + } + futureDevice := uuid.NewString() + if _, err := f.pool.Exec(t.Context(), `INSERT INTO devices(id,tenant_id,name,credential_hash,environment_id,revoked_at) VALUES($1,$2,'future', $3,$4,clock_timestamp())`, futureDevice, retained[2].TenantID, credentialHash(), retained[2].EnvironmentID); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,node_id,deployment_generation,state,create_settled,released_at) VALUES($1,$2,$3,$4,$5,$6,'released',true,clock_timestamp())`, uuid.NewString(), retained[2].EnvironmentID, futureDevice, installation, node.NodeID, view.Generation+1); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `UPDATE environments SET status='expired' WHERE id=$1`, retained[3].EnvironmentID); err != nil { + t.Fatal(err) + } + if err := changes.StartReset(admin(t), installation, deployment.ResetRequest{ExpectedGeneration: view.Generation, Clear: deployment.ResetForce}); err != nil { + t.Fatal(err) + } + started, err := f.service.View(t.Context()) + if err != nil || started.Reset == nil { + t.Fatal(started, err) + } + if _, err := f.pool.Exec(t.Context(), `UPDATE sessions SET created_at=$2 WHERE id=$1`, retained[4].SessionID, started.Reset.RequestedAt.Add(time.Second)); err != nil { + t.Fatal(err) + } + candidates, err := f.adapter.ResetSessions(t.Context(), "", true) + if err != nil || len(candidates) != 32 { + t.Fatal("reset scope", candidates, err) + } + snapshot, err := f.service.View(t.Context()) + if err != nil || snapshot.Resources.Pending != 33 { + t.Fatal("retained reset work missing from completion gate", snapshot.Resources, err) + } + for _, candidate := range candidates { + for _, excluded := range retained[1:5] { + if candidate.SessionID == excluded.SessionID { + t.Fatal("out-of-scope history entered reset", candidate) + } + } + } + var inUse *deployment.InUseError + if _, err := changes.CompleteReset(t.Context(), installation, view.Generation, started.Reset.RequestedAt); !errors.As(err, &inUse) { + t.Fatal("reset completed before archive", err) + } + archive := func(candidates []deployment.ResetSession) { + t.Helper() + for _, candidate := range candidates { + if _, err := f.pool.Exec(t.Context(), `INSERT INTO execution_project_scopes(tenant_id,organization_id,project_id) VALUES($1,$2,$3)`, candidate.TenantID, uuid.NewString(), uuid.NewString()); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `INSERT INTO projects(id,name,tenant_id,subject_kind,subject_id) VALUES($1,'retained',$2,'service_account','fixture')`, uuid.NewString(), candidate.TenantID); err != nil { + t.Fatal(err) + } + if _, err := changes.ArchiveResetSession(t.Context(), candidate.TenantID, candidate.SessionID, view.Generation, started.Reset.RequestedAt); err != nil { + t.Fatal(err) + } + } + } + archive(candidates) + remaining, err := f.adapter.ResetSessions(t.Context(), "", true) + if err != nil || len(remaining) != 1 { + t.Fatal("second reset page", remaining, err) + } + snapshot, err = f.service.View(t.Context()) + if err != nil || snapshot.Resources.Pending != 1 { + t.Fatal("second page did not block completion", snapshot.Resources, err) + } + if _, err := changes.CompleteReset(t.Context(), installation, view.Generation, started.Reset.RequestedAt); !errors.As(err, &inUse) { + t.Fatal("reset skipped second page", err) + } + archive(remaining) + committed, err := changes.CompleteReset(t.Context(), installation, view.Generation, started.Reset.RequestedAt) + if err != nil { + t.Fatal(err) + } + next, err := changes.Initialize(admin(t), installation, sandbox.Selection{Provider: "microsandbox", ExpectedGeneration: committed, DeploymentSpec: retainedSpecification()}) + if err != nil { + t.Fatal(err) + } + nextNode := f.enroll(t, next, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, nextNode.NodeID) + if _, err := changes.EnsurePlacement(t.Context(), target.Key(), installation); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal("reset Session revived in new deployment", err) + } + var status, fsState string + if err := f.pool.QueryRow(t.Context(), `SELECT e.status,w.state FROM environments e JOIN environment_workspaces w ON w.environment_id=e.id WHERE e.id=$1`, target.EnvironmentID).Scan(&status, &fsState); err != nil || status != "expired" || fsState != "ready" { + t.Fatal("archive lost files or kept Session executable", status, fsState, err) + } +} + +func retainedSpecification() sandbox.DeploymentSpec { + spec := testSpecification("microsandbox") + spec.Resources.RootDiskMiB = 8192 + spec.Resources.EnvironmentDiskMiB = 0 + spec.Workspace = &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true} + return spec +} + +func TestReplacementWaitsForCompatibleReadyGenerationWithoutReservingOwnedNode(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, owned := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox")}) + oldNode := f.enroll(t, owned, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, oldNode.NodeID) + external, err := changes.Update(admin(t), installation, sandbox.Selection{Provider: "microsandbox", ExpectedGeneration: owned.Generation, DeploymentSpec: retainedSpecification()}) + if err != nil { + t.Fatal(err) + } + newNode := f.enroll(t, external, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, newNode.NodeID) + key := hostedEnvironment(t, f.pool) + if _, err = f.pool.Exec(t.Context(), `INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) VALUES($1,$2,$3)`, key.EnvironmentID, newNode.NodeID, external.Generation); err != nil { + t.Fatal(err) + } + owner, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if err != nil { + t.Fatal(err) + } + releaseRetainedFixture(t, f, changes, owner) + if _, err = f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, newNode.NodeID); err != nil { + t.Fatal(err) + } + if _, err = changes.EnsurePlacement(t.Context(), key, installation); !errors.Is(err, placement.ErrNodeUnavailable) { + t.Fatal("owned generation admitted retained FS", err) + } + var released bool + if err = f.pool.QueryRow(t.Context(), `SELECT released_at IS NOT NULL FROM runtime_placements WHERE environment_id=$1`, key.EnvironmentID).Scan(&released); err != nil || !released { + t.Fatal("incompatible placement consumed capacity", released, err) + } + f.connect(t, newNode.NodeID) + reserved, err := changes.EnsurePlacement(t.Context(), key, installation) + if err != nil || reserved.NodeID != newNode.NodeID { + t.Fatal("compatible capacity did not recover demand", reserved, err) + } + fresh, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if err != nil || fresh.ID == owner.ID || fresh.NodeID != newNode.NodeID { + t.Fatal(fresh, err) + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/store.go b/services/core/internal/persistence/postgres/deploymentpg/store.go index 612af96bc..e4aee9e74 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/store.go +++ b/services/core/internal/persistence/postgres/deploymentpg/store.go @@ -116,7 +116,7 @@ func (s *Store) Allocation(ctx context.Context, ref sandbox.Reference) (deployme var result deployment.AllocationRecord err = s.pool.Snapshot(ctx, func(ctx context.Context, tx pgx.Tx) error { q := sqlc.New(tx) - row, err := q.GetRuntimeAllocation(ctx, sqlc.GetRuntimeAllocationParams{TenantID: tenant, EnvironmentID: environment}) + row, err := q.GetLatestRuntimeAllocation(ctx, sqlc.GetLatestRuntimeAllocationParams{TenantID: tenant, EnvironmentID: environment}) if errors.Is(err, pgx.ErrNoRows) { return deployment.ErrNotFound } diff --git a/services/core/internal/persistence/postgres/sessionpg/artifacts.go b/services/core/internal/persistence/postgres/sessionpg/artifacts.go index 21401d11d..a43074fee 100644 --- a/services/core/internal/persistence/postgres/sessionpg/artifacts.go +++ b/services/core/internal/persistence/postgres/sessionpg/artifacts.go @@ -173,7 +173,7 @@ type artifactStaging struct { func (t *artifactStaging) LoadEnvironment(ctx context.Context) (sessions.Environment, error) { row, err := t.q.GetSessionEnvironment(ctx, sqlc.GetSessionEnvironmentParams{TenantID: t.turn.TenantID, ID: t.turn.SessionID}) - return environmentFromRow(row.Environment, row.TenantID, row.Configuration, err) + return environmentFromRow(row.Environment, row.TenantID, row.Configuration, row.ExternalWorkspace, err) } func (t *artifactStaging) PutArtifactContent(ctx context.Context, path string, size int64, content io.Reader) error { diff --git a/services/core/internal/persistence/postgres/sessionpg/binding.go b/services/core/internal/persistence/postgres/sessionpg/binding.go index dad3703c5..6650c2552 100644 --- a/services/core/internal/persistence/postgres/sessionpg/binding.go +++ b/services/core/internal/persistence/postgres/sessionpg/binding.go @@ -116,7 +116,7 @@ func (t *SessionTx) LoadEnvironmentInput(ctx context.Context) (*sessions.Environ func (t *SessionTx) LoadEnvironment(ctx context.Context) (sessions.Environment, error) { row, err := t.q.GetSessionEnvironment(ctx, sqlc.GetSessionEnvironmentParams{TenantID: t.tenant, ID: t.session}) - return environmentFromRow(row.Environment, row.TenantID, row.Configuration, err) + return environmentFromRow(row.Environment, row.TenantID, row.Configuration, row.ExternalWorkspace, err) } // RecordEnvironmentFailure fails the Environment only while it is the @@ -185,7 +185,10 @@ func (t *SessionTx) InsertEnvironmentDevice(ctx context.Context, device sessions if err != nil { return err } - _, err = t.q.BindSessionDevice(ctx, sqlc.BindSessionDeviceParams{TenantID: t.tenant, ID: t.session, ID_2: inserted}) + _, err = t.q.BindHostedSessionDevice(ctx, sqlc.BindHostedSessionDeviceParams{TenantID: t.tenant, ID: t.session, ID_2: inserted}) + if errors.Is(err, pgx.ErrNoRows) { + return sessions.ErrDeviceBindingConflict + } return err } diff --git a/services/core/internal/persistence/postgres/sessionpg/creation.go b/services/core/internal/persistence/postgres/sessionpg/creation.go index 7861947d8..96a44b3f1 100644 --- a/services/core/internal/persistence/postgres/sessionpg/creation.go +++ b/services/core/internal/persistence/postgres/sessionpg/creation.go @@ -102,6 +102,11 @@ func (t *creationTx) LockDeployment(ctx context.Context) (placement.Deployment, return placementpg.LockDeployment(ctx, t.q) } +func (t *creationTx) LoadGenerationSpecification(ctx context.Context, generation uint64) (json.RawMessage, error) { + row, err := t.q.GetSandboxGeneration(ctx, int64(generation)) + return row.Specification, err +} + func (t *creationTx) LoadNodes(ctx context.Context) ([]placement.Node, error) { return placementpg.LoadNodes(ctx, t.q) } diff --git a/services/core/internal/persistence/postgres/sessionpg/creation_test.go b/services/core/internal/persistence/postgres/sessionpg/creation_test.go index 2165b2c07..79f125500 100644 --- a/services/core/internal/persistence/postgres/sessionpg/creation_test.go +++ b/services/core/internal/persistence/postgres/sessionpg/creation_test.go @@ -644,3 +644,65 @@ func TestCreationAudit(t *testing.T) { t.Fatal("created resources", owned, rows.Err()) } } + +// Profile admission uses the deployment selected while holding its lock, so a +// storage-mode update cannot race a preflight check and persist unsupported work. +func TestHostedExternalHistoryAdmissionRollsBackAfterDeploymentChange(t *testing.T) { + pool := pgtest.OpenIsolated(t, nil) + store, _ := creationService(t, pool) + rules, err := placement.NewRules(externalAdmissionDeclarations{}, "https://core.example") + if err != nil { + t.Fatal(err) + } + service, err := sessions.NewService(store, rules) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + tenant := uuid.NewString() + exec(t, pool, `UPDATE runtime_deployment SET installation_id=$1, backend_fingerprint=$2, provider_kind='test-direct', mode='direct', generation=1, specification='{"resources":{"cpus":2,"memory_mib":2048}}'`, uuid.New(), strings.Repeat("a", 64)) + input := sessions.CreateSession{Creator: creator, Engine: "codex", IdempotencyKey: "raced", Configuration: json.RawMessage(`{"agent":{"model":"m"},"environment":{"type":"openai_hosted"}}`)} + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + var holder int32 + if err = tx.QueryRow(ctx, "SELECT pg_backend_pid() FROM runtime_deployment FOR UPDATE").Scan(&holder); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { _, err := service.CreateSession(ctx, tenant, input); done <- err }() + awaitBlocked(ctx, t, pool, holder) + if _, err = tx.Exec(ctx, `UPDATE runtime_deployment SET specification=jsonb_set(specification,'{workspace}','{}')`); err != nil { + t.Fatal(err) + } + if err = tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if err = <-done; !errors.Is(err, sessions.ErrInvalidInput) { + t.Fatal("external combination admitted", err) + } + if count := countRows(t, pool, "SELECT count(*) FROM sessions WHERE tenant_id=$1", tenant); count != 0 { + t.Fatal("rejected admission persisted Session", count) + } + input.SupportsRetainedNativeHistory = true + created, err := service.CreateSession(ctx, tenant, input) + if err != nil { + t.Fatal(err) + } + input.SupportsRetainedNativeHistory = false + retry, err := service.CreateSession(ctx, tenant, input) + if err != nil || retry.Created || retry.Session.ID != created.Session.ID { + t.Fatal("trusted admission input changed retry identity", retry, err) + } +} + +// This fixture declares a direct provider supporting both storage modes. +type externalAdmissionDeclarations struct{} + +func (externalAdmissionDeclarations) RequiresPublicOrigin(string) (bool, error) { return false, nil } +func (externalAdmissionDeclarations) ValidateSpecification(string, sandbox.DeploymentSpec) error { + return nil +} diff --git a/services/core/internal/persistence/postgres/sessionpg/device_capabilities_test.go b/services/core/internal/persistence/postgres/sessionpg/device_capabilities_test.go new file mode 100644 index 000000000..c468bc51b --- /dev/null +++ b/services/core/internal/persistence/postgres/sessionpg/device_capabilities_test.go @@ -0,0 +1,46 @@ +package sessionpg + +import ( + "strings" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "github.com/jackc/pgx/v5/pgtype" +) + +func TestAuthenticatedCapabilitiesKeepRevokedReceipt(t *testing.T) { + pool := pgtest.OpenIsolated(t, nil) + tenant, _, _ := newEnvironment(t, pool, "self_hosted", "pending") + device := newDevice(t, pool, tenant, pgtype.UUID{}) + store := New(pgunit.NewPool(pool), nil) + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fixture", Available: true, Capabilities: runtimedevice.KindCapabilities{RetainedNativeHistory: true}}} + correct := strings.Repeat("a", 64) + if ok, err := store.TouchAuthenticatedDevice(t.Context(), device, correct, kinds); err != nil || !ok { + t.Fatal(ok, err) + } + read := func() string { + t.Helper() + var raw string + if err := pool.QueryRow(t.Context(), `SELECT supported_agent_kinds::text FROM devices WHERE id=$1`, device).Scan(&raw); err != nil { + t.Fatal(err) + } + return raw + } + before := read() + if ok, err := store.TouchAuthenticatedDevice(t.Context(), device, strings.Repeat("b", 64), nil); err != nil || ok || read() != before { + t.Fatal("wrong credential changed receipt", ok, err) + } + if ok, err := store.TouchAuthenticatedDevice(t.Context(), device, correct, nil); err != nil || !ok || read() != "[]" { + t.Fatal("unknown capabilities not cleared", ok, err) + } + if _, err := store.TouchAuthenticatedDevice(t.Context(), device, correct, kinds); err != nil { + t.Fatal(err) + } + before = read() + exec(t, pool, `UPDATE devices SET revoked_at=clock_timestamp() WHERE id=$1`, device) + if ok, err := store.TouchAuthenticatedDevice(t.Context(), device, correct, nil); err != nil || ok || read() != before { + t.Fatal("revoked device changed retained receipt", ok, err) + } +} diff --git a/services/core/internal/persistence/postgres/sessionpg/devices.go b/services/core/internal/persistence/postgres/sessionpg/devices.go index 7512586b3..188a45ef0 100644 --- a/services/core/internal/persistence/postgres/sessionpg/devices.go +++ b/services/core/internal/persistence/postgres/sessionpg/devices.go @@ -2,6 +2,7 @@ package sessionpg import ( "context" + "encoding/json" "errors" "fmt" @@ -196,12 +197,19 @@ func (s *Store) TouchDevice(ctx context.Context, device string) (bool, error) { return n > 0, err } -func (s *Store) TouchAuthenticatedDevice(ctx context.Context, device, credentialHash string) (bool, error) { +func (s *Store) TouchAuthenticatedDevice(ctx context.Context, device, credentialHash string, kinds []runtimedevice.SupportedAgentKind) (bool, error) { id, err := parseID(device) if err != nil { return false, err } - n, err := s.units.Queries().TouchAuthenticatedDevice(ctx, sqlc.TouchAuthenticatedDeviceParams{ID: id, CredentialHash: credentialHash}) + if kinds == nil { + kinds = []runtimedevice.SupportedAgentKind{} + } + raw, err := json.Marshal(kinds) + if err != nil { + return false, err + } + n, err := s.units.Queries().TouchAuthenticatedDevice(ctx, sqlc.TouchAuthenticatedDeviceParams{ID: id, CredentialHash: credentialHash, SupportedAgentKinds: raw}) return n > 0, err } diff --git a/services/core/internal/persistence/postgres/sessionpg/environment.go b/services/core/internal/persistence/postgres/sessionpg/environment.go index 830d095ab..e191fe228 100644 --- a/services/core/internal/persistence/postgres/sessionpg/environment.go +++ b/services/core/internal/persistence/postgres/sessionpg/environment.go @@ -23,7 +23,7 @@ func LoadEnvironment(ctx context.Context, q *sqlc.Queries, tenant, environment s return sessions.Environment{}, err } row, err := q.GetEnvironment(ctx, sqlc.GetEnvironmentParams{TenantID: tenantID, ID: pgunit.PathID(environment)}) - return environmentFromRow(row.Environment, row.TenantID, row.Configuration, err) + return environmentFromRow(row.Environment, row.TenantID, row.Configuration, row.ExternalWorkspace, err) } // loadSessionEnvironment reads, on q, the Environment of the tenant's Session @@ -39,7 +39,7 @@ func loadSessionEnvironment(ctx context.Context, q *sqlc.Queries, tenant, sessio return sessions.Environment{}, err } row, err := q.GetSessionEnvironment(ctx, sqlc.GetSessionEnvironmentParams{TenantID: tenantID, ID: id}) - return environmentFromRow(row.Environment, row.TenantID, row.Configuration, err) + return environmentFromRow(row.Environment, row.TenantID, row.Configuration, row.ExternalWorkspace, err) } func (s *Store) GetEnvironment(ctx context.Context, tenant, environment string) (sessions.Environment, error) { diff --git a/services/core/internal/persistence/postgres/sessionpg/rows.go b/services/core/internal/persistence/postgres/sessionpg/rows.go index d635b72bf..4cbe2ddf0 100644 --- a/services/core/internal/persistence/postgres/sessionpg/rows.go +++ b/services/core/internal/persistence/postgres/sessionpg/rows.go @@ -101,7 +101,7 @@ func sessionCreator(kind, id pgtype.Text) (*identity.Subject, error) { // environmentFromRow maps a stored Environment of the tenant with its // normalized configuration snapshot, from the result of the query that read // it: no rows is sessions.ErrNotFound. -func environmentFromRow(row sqlc.Environment, tenant pgtype.UUID, configuration []byte, err error) (sessions.Environment, error) { +func environmentFromRow(row sqlc.Environment, tenant pgtype.UUID, configuration []byte, externalWorkspace bool, err error) (sessions.Environment, error) { if errors.Is(err, pgx.ErrNoRows) { return sessions.Environment{}, sessions.ErrNotFound } @@ -115,6 +115,6 @@ func environmentFromRow(row sqlc.Environment, tenant pgtype.UUID, configuration return sessions.Environment{ ID: uuid.UUID(row.ID.Bytes).String(), SessionID: uuid.UUID(row.SessionID.Bytes).String(), TenantID: uuid.UUID(tenant.Bytes).String(), Status: row.Status, - Initialization: row.Initialization, CreatedAt: row.CreatedAt.Time, Configuration: configuration, + ExternalWorkspace: externalWorkspace, Initialization: row.Initialization, CreatedAt: row.CreatedAt.Time, Configuration: configuration, }, nil } diff --git a/services/core/internal/persistence/postgres/sessionpg/session_reads.go b/services/core/internal/persistence/postgres/sessionpg/session_reads.go index c987a5fdd..e38c2bd20 100644 --- a/services/core/internal/persistence/postgres/sessionpg/session_reads.go +++ b/services/core/internal/persistence/postgres/sessionpg/session_reads.go @@ -285,7 +285,7 @@ func loadSessionActivity(ctx context.Context, q *sqlc.Queries, session sessions. tenant, _ := parseID(session.TenantID) environment, err := q.GetSessionEnvironment(ctx, sqlc.GetSessionEnvironmentParams{TenantID: tenant, ID: id}) if err == nil { - value, err := environmentFromRow(environment.Environment, environment.TenantID, environment.Configuration, nil) + value, err := environmentFromRow(environment.Environment, environment.TenantID, environment.Configuration, environment.ExternalWorkspace, nil) if err != nil { return session, err } diff --git a/services/core/internal/runtimedevice/state.go b/services/core/internal/runtimedevice/state.go index 73f917dc1..fd27fc78d 100644 --- a/services/core/internal/runtimedevice/state.go +++ b/services/core/internal/runtimedevice/state.go @@ -65,6 +65,7 @@ type KindCapabilities struct { Usage bool `json:"usage,omitempty"` Resume bool `json:"resume,omitempty"` NativeSessionRecovery bool `json:"native_session_recovery,omitempty"` + RetainedNativeHistory bool `json:"retained_native_history,omitempty"` Steering bool `json:"steering,omitempty"` MessageItems bool `json:"message_items,omitempty"` diff --git a/services/core/internal/runtimegateway/session.go b/services/core/internal/runtimegateway/session.go index 9032c2f48..42191c76e 100644 --- a/services/core/internal/runtimegateway/session.go +++ b/services/core/internal/runtimegateway/session.go @@ -499,6 +499,7 @@ func deviceKindsFromHeartbeat(p proto.HeartbeatPayload) []runtimedevice.Supporte DurableTurns: info.Capabilities.DurableTurns.IsSupported(), DurableInputReceipts: info.Capabilities.DurableInputReceipts.IsSupported(), NativeSessionRecovery: info.Capabilities.NativeSessionRecovery.IsSupported(), + RetainedNativeHistory: info.Capabilities.RetainedNativeHistory.IsSupported(), MessageItems: info.Capabilities.MessageItems.IsSupported(), ToolObservations: info.Capabilities.ToolObservations.IsSupported(), diff --git a/services/core/internal/sessions/artifacts_test.go b/services/core/internal/sessions/artifacts_test.go index f11756353..c4213ee9d 100644 --- a/services/core/internal/sessions/artifacts_test.go +++ b/services/core/internal/sessions/artifacts_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" ) // exportFile is one entry of a test export. @@ -212,7 +213,7 @@ func (s *fakeArtifactStorage) TouchDevice(context.Context, string) (bool, error) return false, nil } -func (s *fakeArtifactStorage) TouchAuthenticatedDevice(context.Context, string, string) (bool, error) { +func (s *fakeArtifactStorage) TouchAuthenticatedDevice(context.Context, string, string, []runtimedevice.SupportedAgentKind) (bool, error) { s.t.Fatal("unexpected call to TouchAuthenticatedDevice") return false, nil } diff --git a/services/core/internal/sessions/creation.go b/services/core/internal/sessions/creation.go index effe001a4..bafb5aefb 100644 --- a/services/core/internal/sessions/creation.go +++ b/services/core/internal/sessions/creation.go @@ -18,6 +18,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/jsonobject" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/metadata" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/skills" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/writeaudit" "github.com/google/uuid" @@ -73,6 +74,8 @@ type CreationTx interface { // LockDeployment locks and reads the sandbox deployment, serializing // hosted creation with deployment changes, restore and node removal. LockDeployment(ctx context.Context) (placement.Deployment, error) + // LoadGenerationSpecification reads the retained generation selected under the deployment lock. + LoadGenerationSpecification(ctx context.Context, generation uint64) (json.RawMessage, error) LoadNodes(ctx context.Context) ([]placement.Node, error) ReservePlacement(ctx context.Context, chosen placement.Placement) error // LockSkills locks the tenant's Skills in ID order, whatever the order of @@ -224,6 +227,20 @@ func (s *Service) createResources(ctx context.Context, tx CreationTx, session Se if err != nil { return nil, err } + specification := deployment.Specification + if chosen != nil && chosen.Generation != deployment.Generation { + specification, err = tx.LoadGenerationSpecification(ctx, chosen.Generation) + if err != nil { + return nil, err + } + } + var selected sandbox.DeploymentSpec + if err := json.Unmarshal(specification, &selected); err != nil { + return nil, err + } + if err := ValidateRetainedHistory(selected.Workspace != nil, input.SupportsRetainedNativeHistory); err != nil { + return nil, err + } if chosen != nil { if err := tx.ReservePlacement(ctx, *chosen); err != nil { return nil, err diff --git a/services/core/internal/sessions/creation_test.go b/services/core/internal/sessions/creation_test.go index 16f7359ba..585c35be7 100644 --- a/services/core/internal/sessions/creation_test.go +++ b/services/core/internal/sessions/creation_test.go @@ -25,20 +25,21 @@ import ( type fakeCreationTx struct { *fakeInputTx - upsertSession func(NewSession) (Creation, error) - lockDeployment func() (placement.Deployment, error) - loadNodes func() ([]placement.Node, error) - reservePlacement func() error - lockSkills func() (map[string]skills.Skill, error) - readSkillVersion func() (skills.Content, error) - saveModelExecution func() error - saveExecutionConfiguration func() error - saveInitialFiles func() error - saveSetup func() error - createEnvironment func() (string, error) - pruneChanges func() error - auditCreation func() error - loadSession func() (Session, error) + upsertSession func(NewSession) (Creation, error) + loadGenerationSpecification func(uint64) (json.RawMessage, error) + lockDeployment func() (placement.Deployment, error) + loadNodes func() ([]placement.Node, error) + reservePlacement func() error + lockSkills func() (map[string]skills.Skill, error) + readSkillVersion func() (skills.Content, error) + saveModelExecution func() error + saveExecutionConfiguration func() error + saveInitialFiles func() error + saveSetup func() error + createEnvironment func() (string, error) + pruneChanges func() error + auditCreation func() error + loadSession func() (Session, error) } var _ CreationTx = (*fakeCreationTx)(nil) @@ -53,6 +54,11 @@ func (f *fakeCreationTx) LockDeployment(context.Context) (placement.Deployment, return f.lockDeployment() } +func (f *fakeCreationTx) LoadGenerationSpecification(_ context.Context, generation uint64) (json.RawMessage, error) { + f.record("LoadGenerationSpecification", f.loadGenerationSpecification != nil, fmt.Sprint(generation)) + return f.loadGenerationSpecification(generation) +} + func (f *fakeCreationTx) LoadNodes(context.Context) ([]placement.Node, error) { f.record("LoadNodes", f.loadNodes != nil) return f.loadNodes() @@ -448,7 +454,7 @@ func TestCreateSession(t *testing.T) { t.Run("a hosted Session is admitted under the deployment lock and placed after its Environment", func(t *testing.T) { ready := uint64(5) tx := newCreationTx(t, hostedSession, true) - tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Specification: json.RawMessage(`{}`)}) + tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Generation: 5, Specification: json.RawMessage(`{}`)}) tx.createEnvironment = returns("environment") tx.loadNodes = returns([]placement.Node{{ID: "node", Online: true, ServingReady: true, ReadyGeneration: &ready, MaxActive: 1, MaxRetained: 1, CoreURL: rules.PublicURL()}}) tx.reservePlacement = done @@ -459,6 +465,34 @@ func TestCreateSession(t *testing.T) { } }) + for _, external := range []bool{false, true} { + t.Run(fmt.Sprintf("selected generation external=%t", external), func(t *testing.T) { + ready := uint64(4) + tx := newCreationTx(t, hostedSession, true) + tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Generation: 5, Specification: json.RawMessage(`{"workspace":{}}`)}) + tx.createEnvironment = returns("environment") + tx.loadNodes = returns([]placement.Node{{ID: "node", Online: true, ServingReady: true, ReadyGeneration: &ready, MaxActive: 1, MaxRetained: 1, CoreURL: rules.PublicURL()}}) + spec := json.RawMessage(`{}`) + if external { + spec = json.RawMessage(`{"workspace":{}}`) + } + tx.loadGenerationSpecification = func(generation uint64) (json.RawMessage, error) { + if generation != 4 { + t.Fatal(generation) + } + return spec, nil + } + tx.reservePlacement = done + _, err, calls := runCreation(t, rules, tx, creationInput("openai_hosted")) + if external != errors.Is(err, ErrInvalidInput) { + t.Fatalf("selected generation admission: %v", err) + } + if external && strings.Contains(strings.Join(calls, " "), "ReservePlacement") { + t.Fatal("unsupported external placement reserved", calls) + } + }) + } + for _, test := range []struct { name string rules *placement.Rules @@ -470,7 +504,7 @@ func TestCreateSession(t *testing.T) { tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Resetting: true}) }, placement.ErrResetAdmission, []string{"UpsertSession create", "LockDeployment"}}, {"no node places the Environment", rules, func(tx *fakeCreationTx) { - tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Specification: json.RawMessage(`{}`)}) + tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Generation: 5, Specification: json.RawMessage(`{}`)}) tx.createEnvironment, tx.loadNodes = returns("environment"), returns([]placement.Node(nil)) }, placement.ErrNodeUnavailable, []string{"UpsertSession create", "LockDeployment", "CreateEnvironment", "LoadNodes"}}, {"hosted creation needs rules", nil, func(*fakeCreationTx) {}, nil, []string{"UpsertSession create"}}, diff --git a/services/core/internal/sessions/devices.go b/services/core/internal/sessions/devices.go index d7fd40cf9..1a488b93c 100644 --- a/services/core/internal/sessions/devices.go +++ b/services/core/internal/sessions/devices.go @@ -94,7 +94,7 @@ type DeviceStorage interface { TouchDevice(ctx context.Context, device string) (bool, error) // TouchAuthenticatedDevice records that the device was seen with the // credential and reports whether that credential still has authority. - TouchAuthenticatedDevice(ctx context.Context, device, credentialHash string) (bool, error) + TouchAuthenticatedDevice(ctx context.Context, device, credentialHash string, kinds []runtimedevice.SupportedAgentKind) (bool, error) // WithEnrollment authenticates the executor credential for the // Environment, then runs apply in the transaction of the Environment's // Session, with the Environment and what the Session lock shows. A @@ -148,7 +148,7 @@ func (s *Service) TouchRuntimeHeartbeat(ctx context.Context, device string) (run // credential the gateway authenticated. A credential that lost its authority // reports Deleted. func (s *Service) TouchAgentDaemonHeartbeat(ctx context.Context, heartbeat runtimedevice.Heartbeat) (runtimedevice.HeartbeatStatus, error) { - current, err := s.storage.TouchAuthenticatedDevice(ctx, heartbeat.RuntimeID, heartbeat.CredentialHash) + current, err := s.storage.TouchAuthenticatedDevice(ctx, heartbeat.RuntimeID, heartbeat.CredentialHash, heartbeat.SupportedAgentKinds) if err != nil { return runtimedevice.HeartbeatStatus{}, err } @@ -156,7 +156,8 @@ func (s *Service) TouchAgentDaemonHeartbeat(ctx context.Context, heartbeat runti } // heartbeatStatus is the liveness a heartbeat reports. Live connectivity -// belongs to the gateway Registry; only the last-seen time is stored. +// belongs to the gateway Registry. Stored declarations describe capabilities, +// not current connectivity. func heartbeatStatus(current bool) runtimedevice.HeartbeatStatus { return runtimedevice.HeartbeatStatus{Liveness: "online", Deleted: !current} } diff --git a/services/core/internal/sessions/devices_test.go b/services/core/internal/sessions/devices_test.go index 0ea72fcd4..6fa241645 100644 --- a/services/core/internal/sessions/devices_test.go +++ b/services/core/internal/sessions/devices_test.go @@ -14,8 +14,9 @@ import ( // identify it, then runs its func; a method whose func is unset fails the // test. type fakeStorage struct { - t *testing.T - calls []string + t *testing.T + calls []string + heartbeatKinds []runtimedevice.SupportedAgentKind createDevice func(DeviceRegistration) (ExecutionDevice, error) revokeDevice func() error @@ -76,8 +77,9 @@ func (s *fakeStorage) TouchDevice(_ context.Context, device string) (bool, error return s.touchDevice() } -func (s *fakeStorage) TouchAuthenticatedDevice(_ context.Context, device, credentialHash string) (bool, error) { +func (s *fakeStorage) TouchAuthenticatedDevice(_ context.Context, device, credentialHash string, kinds []runtimedevice.SupportedAgentKind) (bool, error) { s.record("TouchAuthenticatedDevice", s.touchAuthenticatedDevice != nil, device, credentialHash) + s.heartbeatKinds = kinds return s.touchAuthenticatedDevice() } @@ -233,3 +235,16 @@ func TestEnrollRuntimeBindsUnderTheSessionLock(t *testing.T) { }) } } + +func TestHeartbeatPassesRetainedHistoryDeclarationToAuthenticatedStorage(t *testing.T) { + storage := &fakeStorage{t: t, touchAuthenticatedDevice: returns(true)} + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fixture", Available: true, Capabilities: runtimedevice.KindCapabilities{RetainedNativeHistory: true}}} + _, err := deviceService(t, storage).TouchAgentDaemonHeartbeat(t.Context(), runtimedevice.Heartbeat{RuntimeID: "device", CredentialHash: credentialDigest, SupportedAgentKinds: kinds}) + if err != nil || len(storage.heartbeatKinds) != 1 || !storage.heartbeatKinds[0].Capabilities.RetainedNativeHistory { + t.Fatalf("declaration lost: %v", err) + } + _, err = deviceService(t, storage).TouchAgentDaemonHeartbeat(t.Context(), runtimedevice.Heartbeat{RuntimeID: "device", CredentialHash: credentialDigest}) + if err != nil || len(storage.heartbeatKinds) != 0 { + t.Fatalf("removed declaration retained: %v", err) + } +} diff --git a/services/core/internal/sessions/environment.go b/services/core/internal/sessions/environment.go index 2a11a64b8..dd0afb7ae 100644 --- a/services/core/internal/sessions/environment.go +++ b/services/core/internal/sessions/environment.go @@ -61,13 +61,15 @@ func createsEnvironment(configuration json.RawMessage) (bool, error) { // Environment retains execution ownership; its configuration is an internal snapshot, not a public response. type Environment struct { - Initialization string - ID string - SessionID string - TenantID string - Status string - CreatedAt time.Time - Configuration json.RawMessage + // ExternalWorkspace is a read projection of the immutable filesystem binding, not a setting. + ExternalWorkspace bool + Initialization string + ID string + SessionID string + TenantID string + Status string + CreatedAt time.Time + Configuration json.RawMessage } // EnvironmentInitialization owns preparation independently of compute ownership. @@ -135,3 +137,11 @@ type EnvironmentFileWrite struct { SettledAt *time.Time Replayed bool } + +// ValidateRetainedHistory checks the selected external filesystem and Harness combination. +func ValidateRetainedHistory(required, supported bool) error { + if required && !supported { + return fmt.Errorf("%w: external workspace storage requires retained_native_history", ErrInvalidInput) + } + return nil +} diff --git a/services/core/internal/sessions/session.go b/services/core/internal/sessions/session.go index 9ce349aa4..534ac908d 100644 --- a/services/core/internal/sessions/session.go +++ b/services/core/internal/sessions/session.go @@ -42,6 +42,9 @@ type Session struct { } type CreateSession struct { + // SupportsRetainedNativeHistory is trusted admission input from the immutable engine profile. + // It is not persisted and never contributes to the caller's retry identity. + SupportsRetainedNativeHistory bool `json:"-"` // DeploymentProviderRevision is private creation metadata, never retry identity. DeploymentProviderRevision uuid.UUID `json:"-"` ExecutionConfiguration *v1.SessionExecutionConfiguration diff --git a/services/core/migrations/000098_compute_incarnations.sql b/services/core/migrations/000098_compute_incarnations.sql new file mode 100644 index 000000000..7fb951331 --- /dev/null +++ b/services/core/migrations/000098_compute_incarnations.sql @@ -0,0 +1,40 @@ +-- +goose Up +ALTER TABLE runtime_allocations DROP CONSTRAINT runtime_allocations_environment_id_key; +CREATE UNIQUE INDEX runtime_allocations_current_environment ON runtime_allocations(environment_id) WHERE state <> 'released'; +CREATE INDEX runtime_allocations_environment_receipts ON runtime_allocations(environment_id, created_at DESC, id DESC); +ALTER TABLE runtime_allocations DROP CONSTRAINT runtime_allocation_placement_fk; +ALTER TABLE runtime_allocations ADD CONSTRAINT runtime_allocation_node_fk FOREIGN KEY(node_id) REFERENCES runtime_nodes(id); +ALTER TABLE devices DROP CONSTRAINT devices_environment_id_key; +CREATE UNIQUE INDEX devices_environment_authority ON devices(environment_id) WHERE revoked_at IS NULL OR executor_key_id IS NOT NULL; + +ALTER TABLE devices ADD COLUMN supported_agent_kinds jsonb NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(supported_agent_kinds) = 'array'); + +-- Cold retained Sessions own no compute, but a reset must end their execution lifetime. +CREATE VIEW runtime_reset_retained_environments AS +SELECT e.id AS environment_id, s.id AS session_id, s.tenant_id, a.deployment_generation +FROM environments e JOIN sessions s ON s.id = e.session_id +JOIN environment_workspaces w ON w.environment_id = e.id AND w.state = 'ready' +JOIN LATERAL ( + SELECT allocation.* FROM runtime_allocations allocation + WHERE allocation.environment_id = e.id + ORDER BY allocation.created_at DESC, allocation.id DESC LIMIT 1 +) a ON true +CROSS JOIN runtime_deployment d +WHERE d.reset_clear IS NOT NULL AND d.reset_requested_at IS NOT NULL + AND s.deleted_at IS NULL AND s.created_at <= d.reset_requested_at + AND e.status NOT IN ('failed','expired') AND e.initialization = 'complete' + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND a.state = 'released' AND a.provider_key = d.installation_id + AND a.deployment_generation <= d.generation AND a.created_at <= d.reset_requested_at + AND NOT EXISTS (SELECT 1 FROM runtime_placements p WHERE p.environment_id = e.id AND p.released_at IS NULL); + +-- +goose Down +DROP VIEW runtime_reset_retained_environments; +ALTER TABLE devices DROP COLUMN supported_agent_kinds; +DROP INDEX devices_environment_authority; +ALTER TABLE devices ADD CONSTRAINT devices_environment_id_key UNIQUE(environment_id); +ALTER TABLE runtime_allocations DROP CONSTRAINT runtime_allocation_node_fk; +ALTER TABLE runtime_allocations ADD CONSTRAINT runtime_allocation_placement_fk FOREIGN KEY(environment_id,node_id) REFERENCES runtime_placements(environment_id,node_id); +DROP INDEX runtime_allocations_environment_receipts; +DROP INDEX runtime_allocations_current_environment; +ALTER TABLE runtime_allocations ADD CONSTRAINT runtime_allocations_environment_id_key UNIQUE(environment_id); diff --git a/services/core/tests/integration/runtime_allocations_test.go b/services/core/tests/integration/runtime_allocations_test.go index cd9899680..824ab501a 100644 --- a/services/core/tests/integration/runtime_allocations_test.go +++ b/services/core/tests/integration/runtime_allocations_test.go @@ -183,8 +183,12 @@ func TestRuntimeAllocationCleanupRevokesAndKeepsIdentity(t *testing.T) { if _, err := deploymentExecution(t, w).ReleaseAllocation(t.Context(), owner); err != nil { t.Fatal(err) } - got, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, owner.ProviderKey, runtimedevice.HashCredential(uuid.NewString())) - if err != nil || !got.Replayed || got.State != "released" { - t.Fatalf("cleanup permitted replacement: %+v %v", got, err) + key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID} + if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), key, owner.ProviderKey, runtimedevice.HashCredential(uuid.NewString())); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatalf("terminal cleanup permitted replacement: %v", err) + } + got, err := deploymentStore(s).EnvironmentAllocation(t.Context(), key) + if err != nil || got.ID != owner.ID || got.DeviceID != owner.DeviceID || got.State != "released" { + t.Fatalf("cleanup changed retained ownership: %+v %v", got, err) } } diff --git a/services/core/tests/integration/runtime_compute_lifecycle_test.go b/services/core/tests/integration/runtime_compute_lifecycle_test.go index 73f5e29d6..84b7e6af0 100644 --- a/services/core/tests/integration/runtime_compute_lifecycle_test.go +++ b/services/core/tests/integration/runtime_compute_lifecycle_test.go @@ -349,7 +349,7 @@ func (f *computeLifecycleFixture) phase(tenant, environment, phase string) deplo func (f *computeLifecycleFixture) complete(owner deployment.Allocation) string { id := uuid.NewString() f.sql(`INSERT INTO turns(id,session_id,status,completed_at) VALUES($1,$2,'completed',clock_timestamp()-interval '2 minutes')`, id, owner.SessionID) - f.sql(`UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 minutes' WHERE id=$1`, owner.ID) + f.sql(`UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 minutes',compute_phase_changed_at=clock_timestamp()-interval '2 minutes' WHERE id=$1`, owner.ID) return id } func (f *computeLifecycleFixture) queued(owner deployment.Allocation) string { @@ -361,12 +361,6 @@ func (f *computeLifecycleFixture) queued(owner deployment.Allocation) string { func TestRuntimeComputeLifecycleIdleSuspendAndQueuedSameSessionWake(t *testing.T) { f := newComputeLifecycleFixture(t, 2, 4) tenant, session, env, owner := f.create() - for range 3 { - f.phase(tenant, env.ID, "running") - } - if f.provider.captures != 0 { - t.Fatal("never-used Session suspended") - } completed := f.complete(owner) suspended := f.phase(tenant, env.ID, "suspended") if f.provider.captures != 1 || f.provider.computeKills != 1 || len(f.provider.computes) != 0 || len(f.provider.snapshots) != 1 { diff --git a/services/core/tests/integration/runtime_idle_clock_test.go b/services/core/tests/integration/runtime_idle_clock_test.go index b792c00e5..e0aa96fb7 100644 --- a/services/core/tests/integration/runtime_idle_clock_test.go +++ b/services/core/tests/integration/runtime_idle_clock_test.go @@ -64,7 +64,7 @@ func verifyManagedIdleClock(t *testing.T, s, w *Store, owner deployment.Allocati observedBefore := runtimeDatabaseTime(t, s) activity, err := deploymentStore(w).Activity(t.Context(), owner.ID) observedAfter := runtimeDatabaseTime(t, s) - if err != nil || activity.ObservedAt.Before(observedBefore) || activity.ObservedAt.After(observedAfter) || activity.ReadyToSuspend(idleTimeout) || activity.LastActivity.Before(before) || activity.LastActivity.After(after) || activity.Busy || activity.WakeRequested || !activity.HasCompletedTurn { + if err != nil || activity.ObservedAt.Before(observedBefore) || activity.ObservedAt.After(observedAfter) || activity.ReadyToSuspend(idleTimeout) || activity.LastActivity.Before(before) || activity.LastActivity.After(after) || activity.Busy || activity.WakeRequested { t.Fatal("idle clock did not use committed terminal ingestion", activity, before, after, err) } until := runtimeDatabaseTime(t, s).Add(time.Hour) @@ -72,7 +72,7 @@ func verifyManagedIdleClock(t *testing.T, s, w *Store, owner deployment.Allocati t.Fatal("new completion admitted premature idle", err) } // Advance only the internal activity age; the remote public timestamp remains unchanged. - runtimeSuspensionSQL(t, s.pool, "UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 minutes' WHERE id=$1", owner.ID) + runtimeSuspensionSQL(t, s.pool, "UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 minutes',compute_phase_changed_at=clock_timestamp()-interval '2 minutes' WHERE id=$1", owner.ID) if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleTimeout); err != nil { t.Fatal("remote timestamp delayed elapsed idle timer", err) } diff --git a/services/core/tests/integration/runtime_lifecycle_nodes_test.go b/services/core/tests/integration/runtime_lifecycle_nodes_test.go index 72f26498a..a26698128 100644 --- a/services/core/tests/integration/runtime_lifecycle_nodes_test.go +++ b/services/core/tests/integration/runtime_lifecycle_nodes_test.go @@ -173,7 +173,7 @@ func TestRuntimeLifecycleNodeInventoryAndRouting(t *testing.T) { if _, err = deploymentExecution(t, w).ReleaseAllocation(t.Context(), owner); err != nil { t.Fatal(err) } - checkRoute(other, nil) + checkRoute("", placement.ErrNodeUnavailable) // Released ownership is no longer a lifecycle route. if rows, err := deploymentStore(w).LifecycleAllocations(t.Context(), other, ""); err != nil || len(rows) != 0 { t.Fatal("released allocation scanned", rows, err) } diff --git a/services/core/tests/integration/runtime_node_lifecycle_test.go b/services/core/tests/integration/runtime_node_lifecycle_test.go index 545e87866..e601d41b4 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_test.go @@ -24,7 +24,7 @@ func TestManagedNodesIsolateBlockedProviderAndInitialization(t *testing.T) { if _, err := f.pool.Exec(t.Context(), "INSERT INTO turns(id,session_id,status,completed_at) VALUES($1,$2,'completed',clock_timestamp()-interval '2 minutes')", uuid.NewString(), wakeOwner.SessionID); err != nil { t.Fatal(err) } - if _, err := f.pool.Exec(t.Context(), "UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 minutes' WHERE id=$1", wakeOwner.ID); err != nil { + if _, err := f.pool.Exec(t.Context(), "UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 minutes',compute_phase_changed_at=clock_timestamp()-interval '2 minutes' WHERE id=$1", wakeOwner.ID); err != nil { t.Fatal(err) } f.phase(wakeTenant, wakeEnv.ID, "suspended") diff --git a/services/core/tests/integration/runtime_suspension_test.go b/services/core/tests/integration/runtime_suspension_test.go index 1d5eabad8..3e2b73be3 100644 --- a/services/core/tests/integration/runtime_suspension_test.go +++ b/services/core/tests/integration/runtime_suspension_test.go @@ -64,13 +64,13 @@ func runtimeSuspensionStep(t *testing.T, w *Store, owner deployment.Allocation, return next } -func TestRuntimeSuspensionRequiresCompletedIdleAndNoPendingWork(t *testing.T) { - cases := []string{"no_completed_turn", "queued", "in_progress", "waiting", "subagent_queued", "subagent_in_progress", "subagent_waiting", "input_reservation", "file_write", "idle"} +func TestRuntimeSuspensionRequiresIdleAndNoPendingWork(t *testing.T) { + cases := []string{"no_completed_turn", "initial_input", "queued", "in_progress", "waiting", "subagent_queued", "subagent_in_progress", "subagent_waiting", "input_reservation", "file_write", "idle"} for _, kind := range cases { t.Run(kind, func(t *testing.T) { _, w, pool, owner := runtimeSuspensionFixture(t) completed := "" - if kind != "no_completed_turn" { + if kind != "no_completed_turn" && kind != "initial_input" { completed = runtimeSuspensionCompleted(t, pool, owner) } switch kind { @@ -81,6 +81,8 @@ func TestRuntimeSuspensionRequiresCompletedIdleAndNoPendingWork(t *testing.T) { runtimeSuspensionSQL(t, pool, `INSERT INTO turn_events(session_id,turn_id,ordinal,kind,payload) VALUES($1,$2,1,'subagent','{}')`, owner.SessionID, completed) runtimeSuspensionSQL(t, pool, `INSERT INTO subagent_identities(id,session_id,device_id,engine,native_id,parent_native_id,native_created_at,first_turn_id,first_event_ordinal) VALUES($1,$2,$3,'codex','child','root',1,$4,1)`, child, owner.SessionID, owner.DeviceID, completed) runtimeSuspensionSQL(t, pool, `INSERT INTO subagent_turns(id,session_id,subagent_id,native_id,status,created_at) VALUES($1,$2,$3,'child-turn',$4,clock_timestamp())`, uuid.NewString(), owner.SessionID, child, strings.TrimPrefix(kind, "subagent_")) + case "initial_input": + runtimeSuspensionSQL(t, pool, `INSERT INTO environment_input_reservations(id,session_id,idempotency_key,batch,is_initial,created_at,deadline) VALUES($1,$2,'initial','[{}]',true,clock_timestamp(),clock_timestamp()+interval '1 minute')`, uuid.NewString(), owner.SessionID) case "input_reservation": runtimeSuspensionSQL(t, pool, `INSERT INTO environment_input_reservations(id,session_id,idempotency_key,batch,created_at,deadline) VALUES($1,$2,'pending','[{}]',clock_timestamp(),clock_timestamp()+interval '1 minute')`, uuid.NewString(), owner.SessionID) case "file_write": @@ -91,12 +93,12 @@ func TestRuntimeSuspensionRequiresCompletedIdleAndNoPendingWork(t *testing.T) { t.Fatal(err) } wantBusy := kind != "idle" && kind != "no_completed_turn" - if activity.Busy != wantBusy || activity.HasCompletedTurn != (kind != "no_completed_turn") { + if activity.Busy != wantBusy { t.Fatalf("activity lost pending work: %+v", activity) } until := time.Now().Add(time.Hour) _, err = deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Nanosecond) - if kind == "idle" { + if kind == "idle" || kind == "no_completed_turn" { if err != nil { t.Fatal(err) } @@ -264,7 +266,7 @@ func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { func TestRuntimeSuspensionIdleStartsAfterLastCompletion(t *testing.T) { _, w, pool, owner := runtimeSuspensionFixture(t) turn := runtimeSuspensionCompleted(t, pool, owner) - runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 hours',compute_phase_changed_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) var completed time.Time if err := pool.QueryRow(t.Context(), `SELECT completed_at FROM turns WHERE id=$1`, turn).Scan(&completed); err != nil { t.Fatal(err) @@ -328,7 +330,7 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { t.Run(kind, func(t *testing.T) { s, w, pool, owner := runtimeSuspensionFixture(t) turn := runtimeSuspensionCompleted(t, pool, owner) - runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '20 minutes' WHERE id=$1`, owner.ID) + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '20 minutes',compute_phase_changed_at=clock_timestamp()-interval '20 minutes' WHERE id=$1`, owner.ID) var err error owner, err = deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: owner.TenantID, EnvironmentID: owner.EnvironmentID}) if err != nil { @@ -425,3 +427,32 @@ func TestRuntimeComputePhaseChangedAtInNodeAllocations(t *testing.T) { t.Fatal("an unknown phase time was not null", string(encoded)) } } + +func TestNeverStartedSessionIdleStartsAfterInitialization(t *testing.T) { + _, w, pool, owner := runtimeSuspensionFixture(t) + const timeout = 5 * time.Minute + until := time.Now().Add(time.Hour) + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '1 hour',created_at=clock_timestamp()-interval '1 hour' WHERE id=$1`, owner.ID) + activity, err := deploymentStore(w).Activity(t.Context(), owner.ID) + if err != nil || activity.ReadyToSuspend(timeout) { + t.Fatal("creation age bypassed ready-time idle clock", activity, err) + } + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_phase_changed_at=clock_timestamp()-interval '6 minutes' WHERE id=$1`, owner.ID) + activity, err = deploymentStore(w).Activity(t.Context(), owner.ID) + if err != nil || !activity.ReadyToSuspend(timeout) { + t.Fatal("never-started initialized Session cannot idle", activity, err) + } + runtimeSuspensionSQL(t, pool, `UPDATE environments SET initialization='running' WHERE id=$1`, owner.EnvironmentID) + if _, err = deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, timeout); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal("unfinished initialization admitted suspension", err) + } + runtimeSuspensionSQL(t, pool, `UPDATE environments SET initialization='complete' WHERE id=$1`, owner.EnvironmentID) + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_wake_requested=true WHERE id=$1`, owner.ID) + if _, err = deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, timeout); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal("wake lost to suspension", err) + } + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_wake_requested=false WHERE id=$1`, owner.ID) + if _, err = deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, timeout); err != nil { + t.Fatal("idle Session with no Turn was blocked", err) + } +} diff --git a/services/core/tests/integration/sandbox_deployment_audit_http_test.go b/services/core/tests/integration/sandbox_deployment_audit_http_test.go new file mode 100644 index 000000000..1f2ee5ba7 --- /dev/null +++ b/services/core/tests/integration/sandbox_deployment_audit_http_test.go @@ -0,0 +1,83 @@ +package integration + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// The HTTP boundary must supply provenance; this bridge passes its untouched +// context into the actual deployment transaction, including its audit write. +type auditedDeploymentChanges struct { + strictStandIn + operations *deployment.ExecutionOperations + installation string +} + +func (d auditedDeploymentChanges) UpdateSandboxDeployment(ctx context.Context, input sandbox.Selection) (deployment.View, error) { + return d.operations.Update(ctx, d.installation, input) +} + +func TestSandboxDeploymentUpdateHTTPCommitsWithAdministratorAudit(t *testing.T) { + store, writer, initial, input := webSpecificationFixture(t, "docker") + changes := auditedDeploymentChanges{strictStandIn{t}, deploymentExecution(t, writer), initial.InstallationID} + handler, err := publicHandler(t, store, newTestAuthenticator(t, nil), "codex", func(d *api.Dependencies) { d.Sandboxes.DeploymentChanges = changes }) + if err != nil { + t.Fatal(err) + } + call := func(generation uint64, cpus uint32, key, actor string, status int) *httptest.ResponseRecorder { + t.Helper() + resources := input.Resources + resources.CPUs = cpus + body, err := json.Marshal(map[string]any{"expected_generation": generation, "provider": "docker", "configuration": map[string]any{}, "resources": resources, "runtime": input.Runtime}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodPut, "/core/v1/sandbox/deployment", strings.NewReader(string(body))) + request.Header.Set("Authorization", "Bearer "+key) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("X-Core-Console-Actor", actor) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != status { + t.Fatalf("update status=%d want=%d body=%s", response.Code, status, response.Body) + } + return response + } + call(1, 3, "invalid", "operator", http.StatusUnauthorized) + response := call(1, 3, "admin", "operator", http.StatusOK) + current, err := deploymentService(t, store).View(t.Context()) + if err != nil || current.Generation != 2 || current.Specification.Resources.CPUs != 3 { + t.Fatal("deployment did not commit", current, err) + } + var credential, actor, requestID, traceID, action, kind, resource string + var project, tenant *string + err = store.pool.QueryRow(t.Context(), `SELECT admin_credential_id,actor_label,request_id,trace_id,action,resource_type,resource_id,project_id,tenant_id FROM admin_audit_log WHERE resource_type='sandbox_deployment' AND resource_id=$1`, initial.InstallationID).Scan(&credential, &actor, &requestID, &traceID, &action, &kind, &resource, &project, &tenant) + if err != nil { + t.Fatal(err) + } + digest := sha256.Sum256([]byte("admin")) + if credential != hex.EncodeToString(digest[:])[:8] || actor != "operator" || requestID == "" || requestID != response.Header().Get("x-request-id") || traceID == "" || action != "change" || kind != "sandbox_deployment" || resource != initial.InstallationID || project != nil || tenant != nil { + t.Fatal("administrator provenance differs", credential, actor, requestID, traceID, project, tenant) + } + // Invalid audit provenance still rolls the mutation back atomically. + call(2, 4, "admin", strings.Repeat("a", 129), http.StatusBadRequest) + call(1, 4, "admin", "operator", http.StatusConflict) + current, err = deploymentService(t, store).View(t.Context()) + if err != nil || current.Generation != 2 || current.Specification.Resources.CPUs != 3 { + t.Fatal("rejected update changed deployment", current, err) + } + var count int + if err = store.pool.QueryRow(t.Context(), `SELECT count(*) FROM admin_audit_log WHERE resource_type='sandbox_deployment' AND resource_id=$1`, initial.InstallationID).Scan(&count); err != nil || count != 1 { + t.Fatal("rejected request wrote audit", count, err) + } +} diff --git a/services/core/tests/integration/session_execution_configuration_test.go b/services/core/tests/integration/session_execution_configuration_test.go index 54dea62c3..e4a1c168c 100644 --- a/services/core/tests/integration/session_execution_configuration_test.go +++ b/services/core/tests/integration/session_execution_configuration_test.go @@ -282,7 +282,7 @@ func TestSessionExecutionConfigurationSurvivesSuspendResume(t *testing.T) { t.Fatal("runtime transition changed execution projection", phase, err) } after, err := deploymentStore(w).Activity(t.Context(), owner.ID) - if err != nil || !before.LastActivity.Equal(after.LastActivity) || before.WakeRequested != after.WakeRequested || before.Busy != after.Busy || before.HasCompletedTurn != after.HasCompletedTurn { + if err != nil || !before.LastActivity.Equal(after.LastActivity) || before.WakeRequested != after.WakeRequested || before.Busy != after.Busy { t.Fatal("configuration read changed runtime activity", err) } } From f3b50eeb20f42a1c47c6271592c8fec6c43eaaa5 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 10 Oct 2026 23:58:44 +0800 Subject: [PATCH 08/22] feat(sandbox): support native E2B suspension and retained capacity --- .../agents-api/node-generation-protocol.md | 15 +- .../agents-api/zh/node-generation-protocol.md | 17 +- deploy/compose/compose.yaml | 2 + deploy/compose/test_compose.py | 9 +- docs/configuration.md | 8 +- docs/getting-started/operations.md | 4 - docs/runtime-bootstrap.md | 4 + docs/sandbox-provider.md | 44 ++-- docs/zh/configuration.md | 10 +- docs/zh/getting-started/operations.md | 6 +- docs/zh/runtime-bootstrap.md | 6 +- docs/zh/sandbox-provider.md | 48 ++-- internal/runtimebootstrap/bootstrap.go | 3 + .../server/managed_generation_operations.go | 20 +- .../cmd/server/managed_generations_test.go | 8 +- services/core/cmd/server/managed_nodes.go | 2 +- services/core/cmd/server/managed_setup.go | 4 +- .../server/managed_setup_preflight_test.go | 21 +- .../core/cmd/server/managed_setup_test.go | 15 +- .../core/cmd/server/sandbox_capacity_test.go | 16 ++ services/core/deploy/e2b/build-template.py | 2 + .../core/deploy/e2b/build_template_test.py | 3 + .../deploy/e2b/helper_contract_generated.py | 13 +- services/core/deploy/e2b/init.py | 2 + services/core/deploy/e2b/init_test.py | 4 +- services/core/deploy/e2b/managed_init.py | 7 + services/core/deploy/e2b/managed_init_test.py | 8 +- .../db/queries/runtime_allocations.sql | 4 +- .../db/queries/runtime_suspension.sql | 25 +- .../db/sqlc/runtime_allocations.sql.go | 6 +- .../db/sqlc/runtime_suspension.sql.go | 51 +++- .../core/internal/deployment/activity_test.go | 27 +-- .../core/internal/deployment/allocation.go | 6 +- .../core/internal/deployment/execution.go | 14 ++ .../core/internal/deployment/fakes_test.go | 67 +++-- services/core/internal/deployment/service.go | 2 +- services/core/internal/deployment/storage.go | 8 + .../archive_cancellation_cleanup_test.go | 2 +- .../execution/deployment_fixture_test.go | 58 +++-- .../provider_operations_fixture_test.go | 13 +- .../internal/execution/runtime_compute.go | 154 ++++++++---- .../runtime_compute_compatibility_test.go | 60 +++++ .../execution/runtime_compute_wake.go | 17 +- .../execution/runtime_fresh_hint_test.go | 44 ++-- .../internal/execution/runtime_lifecycle.go | 48 +++- .../runtime_workspace_settlement_test.go | 2 +- .../execution/runtime_workspace_test.go | 10 +- services/core/internal/execution/worker.go | 4 + .../postgres/deploymentpg/allocations.go | 20 +- .../persistence/postgres/deploymentpg/tx.go | 4 + .../core/internal/processconfig/config.go | 6 + .../processconfig/sandbox_capacity.go | 41 ++++ .../processconfig/sandbox_capacity_test.go | 53 ++++ .../internal/sandbox/docker/operations.go | 13 +- .../internal/sandbox/e2b/helper_contract.go | 31 ++- .../sandbox/e2b/installed_paths_test.go | 3 +- .../sandbox/e2b/internal/contractgen/main.go | 40 ++- .../core/internal/sandbox/e2b/operations.go | 49 +--- .../core/internal/sandbox/e2b/process_test.go | 9 +- .../core/internal/sandbox/e2b/provider.go | 12 +- .../core/internal/sandbox/e2b/suspension.go | 115 +++++++++ .../internal/sandbox/e2b/suspension_test.go | 48 ++++ .../internal/sandbox/e2b/workspace_test.go | 8 + .../microsandbox/creation_settlement_test.go | 2 +- .../sandbox/microsandbox/operations.go | 3 +- .../sandbox/microsandbox/process_test.go | 6 +- .../internal/sandbox/microsandbox/provider.go | 33 +-- .../sandbox/microsandbox/provider_test.go | 14 +- .../sandbox/microsandbox/suspension.go | 146 +++++++++++ .../microsandbox/suspension_contract_test.go | 57 +++++ .../internal/sandbox/microsandbox/types.go | 54 ++++- .../sandbox/microsandbox/workspace_test.go | 10 +- .../internal/sandbox/node/generation_json.go | 4 +- .../sandbox/node/generation_json_test.go | 4 +- .../core/internal/sandbox/node/node_test.go | 6 +- .../core/internal/sandbox/node/operations.go | 3 +- .../internal/sandbox/node/operations_test.go | 56 ++++- .../node/provider_operations_fixture_test.go | 16 +- services/core/internal/sandbox/node/proxy.go | 12 +- services/core/internal/sandbox/node/wire.go | 41 ++-- .../core/internal/sandbox/operations_test.go | 2 +- .../sandbox/providers/registration.go | 5 - .../sandbox/providers/registration_test.go | 8 +- .../internal/sandbox/providers/registry.go | 4 +- .../sandbox/providers/registry_test.go | 10 +- .../core/internal/sandbox/sandbox_provider.go | 97 ++++++-- .../internal/sandbox/sandbox_provider_test.go | 2 +- .../core/internal/sandbox/suspension_test.go | 55 +++++ .../admin_session_archive_worker_http_test.go | 2 +- .../archived_cancellation_migration_test.go | 2 + .../provider_operations_fixture_test.go | 18 +- .../provider_registration_migration_test.go | 2 + .../runtime_compute_lifecycle_test.go | 216 +++++++++++++++-- .../integration/runtime_idle_clock_test.go | 23 +- .../runtime_node_lifecycle_fixture_test.go | 12 +- .../integration/runtime_suspension_test.go | 72 +++++- .../runtime_wake_hint_integration_test.go | 6 +- .../runtime_worker_recovery_test.go | 19 +- .../sandbox_deployment_view_test.go | 2 +- .../integration/sandbox_generations_test.go | 7 +- .../session_execution_configuration_test.go | 2 +- services/core/tools/e2b-provider/README.md | 12 +- .../e2b-provider/helper_contract_generated.py | 13 +- .../tools/e2b-provider/observation_test.py | 3 +- services/core/tools/e2b-provider/provider.py | 11 +- .../core/tools/e2b-provider/provider_test.py | 2 +- .../core/tools/e2b-provider/state_test.py | 3 +- .../core/tools/e2b-provider/suspension.py | 228 ++++++++++++++++++ .../tools/e2b-provider/suspension_test.py | 170 +++++++++++++ .../tools/e2b-provider/testdata/contract.json | 86 ++++--- .../tools/microsandbox-provider/README.md | 10 +- .../tools/microsandbox-provider/bootstrap.go | 5 +- 112 files changed, 2405 insertions(+), 546 deletions(-) create mode 100644 services/core/cmd/server/sandbox_capacity_test.go create mode 100644 services/core/internal/execution/runtime_compute_compatibility_test.go create mode 100644 services/core/internal/processconfig/sandbox_capacity.go create mode 100644 services/core/internal/processconfig/sandbox_capacity_test.go create mode 100644 services/core/internal/sandbox/e2b/suspension.go create mode 100644 services/core/internal/sandbox/e2b/suspension_test.go create mode 100644 services/core/internal/sandbox/microsandbox/suspension.go create mode 100644 services/core/internal/sandbox/microsandbox/suspension_contract_test.go create mode 100644 services/core/internal/sandbox/suspension_test.go create mode 100644 services/core/tools/e2b-provider/suspension.py create mode 100644 services/core/tools/e2b-provider/suspension_test.py diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index 5b7892cac..cc2d0f76f 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -44,20 +44,21 @@ Each operation carries its own arguments and returns the following result on suc | `command` | `RunCommand` | `command` | `command` | | `observe` | `Observe` | `observation` | `sample` | | `initial` | `Initial` | None | `compute` | -| `new_compute` | `NewCompute` | Positive compute `generation` and optional `snapshot` | `compute` | +| `new_compute` | `NewCompute` | Positive `generation` and optional `retained` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | Exact current `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | None | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` and `command` | `command` | | `suspend` | `Suspend` | `suspend` | `state` | | `resume` | `Resume` | `resume` | `state` | -| `delete_snapshot` | `DeleteSnapshot` | `snapshot` | None | +| `delete_retained` | `DeleteRetained` | `retained` | None | -A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. A node without generation management accepts only its enrolled `deployment_generation`; a generation-managing node runs the request on that generation's provider and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not observe-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. +A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. A node without generation management accepts only its enrolled `deployment_generation`; a generation-managing node runs the request on that generation's provider and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not reconciliation-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. The budget is relative: the node anchors `timeout_ms` to its own clock on receipt and consumes it while the request waits in its queue, so the hosts' clocks need not agree. Core still bounds its own wait. A full node queue closes the connection. -The `response` frame carries `id` and `connection_id`. A successful response carries the result named in the operation table, with no result field for `kill`, `kill_compute` or `delete_snapshot`. A failed response carries an `error_code`: +The `response` frame carries `id` and `connection_id`. A successful response carries the result named in the operation table, with no result field for `kill`, `kill_compute` or `delete_retained`. A failed response carries an `error_code`: | `error_code` | Meaning | | --- | --- | @@ -67,7 +68,7 @@ The `response` frame carries `id` and `connection_id`. A successful response car | `unsupported` | The operation is declared unsupported; see below | | `unconfirmed`, or any other value | The outcome is unknown | -A failed response carries no result, except an `info` that is an exact-reference `CreateSettled` receipt: a confirmed native Create that failed a later check can still prove that the attempt settled. A timeout, a lost response or a disconnect is unavailable or uncertain, never evidence of absence, and Core never replays a mutation after one; it observes the original operation instead. The [Sandbox Provider guide](../../docs/sandbox-provider.md#operation-outcomes-and-retries) defines each outcome. +A failed response carries no result except an exact-provenance Resume target for cleanup (as specified below), or an `info` that is an exact-reference `CreateSettled` receipt: a confirmed native Create that failed a later check can still prove that the attempt settled. A timeout, a lost response or a disconnect is unavailable or uncertain, never evidence of absence, and Core never replays a mutation after one; it observes the original operation instead. The [Sandbox Provider guide](../../docs/sandbox-provider.md#operation-outcomes-and-retries) defines each outcome. Node startup and generation loading validate complete Provider operation declarations before accepting work, and the Core proxy uses the same registered declaration, so an unsupported operation rejects before node resolution or native I/O. The [operation contract](../../docs/sandbox-provider.md#explicit-operation-contracts) owns the inventory. An `unsupported` response carries an `unsupported` object with the exact method `operation` and an authored safe `reason`; the proxy checks both against the request. Missing, malformed or mismatched evidence is an unconfirmed result, never proof that a mutation was rejected. Unsupported stays distinct from observation unavailability and unknown compute or command results, and it neither settles resource ownership nor authorizes a replay. @@ -123,6 +124,6 @@ The readiness classes, one exported error and one code each, are authored in `se Preparation diagnostics keep fixed typed causes. Only artifact transfer, checksum or release-provenance failures report `runtime_download_failed`; the private preparer signals that class through its exit category, without Core or the node parsing stderr. Provider, ownership, cancellation and unclassified failures keep their typed code or `provider_unavailable`. No raw provider text crosses the protocol. -The current wire version is 6. Create bootstrap and Resume requests may carry an optional workspace filesystem binding. The node validates its tenant and Environment against the allocation reference, its immutable ObjectID, and its attachment configuration ID against the supplied configuration before forwarding it. The filesystem resolver validates adapter-native ownership. A missing binding selects owned storage; a present binding cannot fall back. Error responses may retain a Resume target only when its nonempty native ID, name, generation and snapshot provenance match the requested target; this is cleanup evidence, never successful restore. +Creation carries the Session-selected `Bootstrap.Harness` into Runtime bootstrap version 2. Core and nodes use protocol version 7 and require a coordinated upgrade. `Bootstrap.Harness` is required. All ten suspension operations belong to the same `SandboxProvider` and are declared supported or unsupported together; dispatch uses no optional interface. `Observe` reads one allocation per request. -Creation carries the Session-selected `Bootstrap.Harness` into Runtime bootstrap version 2. Core and nodes use protocol version 6; upgrade matching nodes before publishing this Core release. +The current wire version is 7. Create bootstrap and Resume requests may carry an optional workspace filesystem binding. The node validates its tenant and Environment against the allocation reference, its immutable ObjectID, and its attachment configuration ID against the supplied configuration before forwarding it. The filesystem resolver validates adapter-native ownership. A missing binding selects owned storage; a present binding cannot fall back. Error responses may retain a Resume target only when its nonempty native ID, name, generation and retained-state provenance match the requested target; this is cleanup evidence, never successful restore. diff --git a/contracts/agents-api/zh/node-generation-protocol.md b/contracts/agents-api/zh/node-generation-protocol.md index b24278ee8..99998679f 100644 --- a/contracts/agents-api/zh/node-generation-protocol.md +++ b/contracts/agents-api/zh/node-generation-protocol.md @@ -1,7 +1,7 @@ --- title: "沙箱节点协议" source: contracts/agents-api/node-generation-protocol.md -source_hash: 961c7993fa9c75ddadb6604fece87fb556bbcc40d4f190d03d2e90927c5be7bb +source_hash: e9a9e3d1027d797bdf467d867023fa11792866ea93252f94904a66f7be45af1e --- 沙箱节点在其主机上运行 Docker 或 microsandbox Provider，并通过一个 WebSocket 与 Core 相连。Core 通过该连接发送 Provider 操作；节点针对本地 Provider 执行这些操作，并报告就绪状态、主机测量值及其持有的部署代次。Core 始终是唯一的生命周期所有者：节点绝不重试变更操作或调度工作。帧和校验器位于 [`services/core/internal/sandbox/node`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/sandbox/node)（`wire.go`、`generation_wire.go`）；节点用于注册和读取配置的 HTTP 路由位于[机器连接 API](machine-api.md#node-routes)。 @@ -46,20 +46,21 @@ Core 发送包含以下内容的 `request` 帧： | `command` | `RunCommand` | `command` | `command` | | `observe` | `Observe` | `observation` | `sample` | | `initial` | `Initial` | 无 | `compute` | -| `new_compute` | `NewCompute` | 大于零的计算 `generation` 和可选的 `snapshot` | `compute` | +| `new_compute` | `NewCompute` | 大于零的计算 `generation` 和可选的 `retained` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | 精确的当前 `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | 无 | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` 和 `command` | `command` | | `suspend` | `Suspend` | `suspend` | `state` | | `resume` | `Resume` | `resume` | `state` | -| `delete_snapshot` | `DeleteSnapshot` | `snapshot` | 无 | +| `delete_retained` | `DeleteRetained` | `retained` | 无 | -只要 `connection_id`、`owner_epoch` 或 `sequence` 中任一值不匹配，请求就会关闭连接。格式错误的请求会得到 `invalid` 响应。未启用代次管理的节点仅接受其登记的 `deployment_generation`；支持代次管理的节点在对应代次的 Provider 上运行请求，无法运行时回复 `unconfirmed`。Core 仅向节点上已就绪的代次发送 `create` 和非 observe-only 的 `resume`，并且每条连接最多保留 32 个待处理请求。 +只要 `connection_id`、`owner_epoch` 或 `sequence` 中任一值不匹配，请求就会关闭连接。格式错误的请求会得到 `invalid` 响应。未启用代次管理的节点仅接受其登记的 `deployment_generation`；支持代次管理的节点在对应代次的 Provider 上运行请求，无法运行时回复 `unconfirmed`。Core 仅向节点上已就绪的代次发送 `create` 和非 reconciliation-only 的 `resume`，并且每条连接最多保留 32 个待处理请求。 预算采用相对计时：节点收到请求时以自己的时钟为基准锚定 `timeout_ms`，并在请求排队等待期间持续消耗该预算，因此各主机的时钟无需保持一致。Core 仍会限制自身等待时长。节点队列已满时会关闭连接。 -`response` 帧包含 `id` 和 `connection_id`。成功响应携带操作表中指定的结果；对于 `kill`、`kill_compute` 或 `delete_snapshot`，响应不含结果字段。失败响应携带一个 `error_code`： +`response` 帧包含 `id` 和 `connection_id`。成功响应携带操作表中指定的结果；对于 `kill`、`kill_compute` 或 `delete_retained`，响应不含结果字段。失败响应携带一个 `error_code`： | `error_code` | 含义 | | --- | --- | @@ -69,7 +70,7 @@ Core 发送包含以下内容的 `request` 帧： | `unsupported` | 该操作被声明为不支持；见下文 | | `unconfirmed` 或任何其他值 | 结果未知 | -失败响应不携带结果，唯一的例外是作为精确引用 `CreateSettled` 回执的 `info` 结果：即便已确认的原生 Create 在后续检查中失败，仍可证明该尝试已有确定结果。超时、响应丢失或断连属于不可用或不确定情况，绝不能证明资源不存在；发生这些情况后，Core 绝不重放变更操作，而是改为观察原始操作。[Sandbox Provider 指南](../../../docs/zh/sandbox-provider.md#operation-outcomes-and-retries) 定义了每种结果。 +失败响应不携带结果；例外包括下文规定的精确来源 Resume 目标清理证据，以及作为精确引用 `CreateSettled` 回执的 `info` 结果：即便已确认的原生 Create 在后续检查中失败，仍可证明该尝试已有确定结果。超时、响应丢失或断连属于不可用或不确定情况，绝不能证明资源不存在；发生这些情况后，Core 绝不重放变更操作，而是改为观察原始操作。[Sandbox Provider 指南](../../../docs/zh/sandbox-provider.md#operation-outcomes-and-retries) 定义了每种结果。 节点启动和代次加载会在接受工作前验证完整的 Provider 操作声明，Core 代理使用同一份已注册声明，因此不支持的操作会在节点解析或原生 I/O 之前被拒绝。操作清单由[操作契约](../../../docs/zh/sandbox-provider.md#explicit-operation-contracts)维护。`unsupported` 响应包含一个 `unsupported` 对象，其中有精确的方法 `operation` 和经作者编写且安全的 `reason`；代理会将两者与请求进行核对。证据缺失、格式错误或不匹配会得到 `unconfirmed` 结果，而绝不会证明变更操作被拒绝。`unsupported` 始终不同于观察不可用，也不同于计算或命令结果未知；它既不确定资源所有权，也不授权重放。 @@ -125,6 +126,6 @@ Runtime 字节缺失时，绝不将固定的放置实例迁移到当前 Runtime 准备诊断使用固定的类型化原因。只有制品传输、校验和或版本来源验证失败才会报告 `runtime_download_failed`；私有准备器通过退出类别指示这一类失败，Core 和节点都不解析 stderr。Provider 故障、所有权故障、取消和未分类故障保留其类型化代码，或使用 `provider_unavailable`。协议中不会传输任何 Provider 原始文本。 -当前线协议版本为 6。Create 引导和 Resume 请求可携带可选的工作区文件系统绑定。节点在转发前校验其租户及 Environment 与分配引用一致、ObjectID 不可变且有效，以及挂载配置 ID 与所传配置一致。文件系统解析器负责适配器原生所有权校验。未提供绑定时选择自有存储；已提供绑定时不得回退。错误响应仅可保留原生 ID 非空且名称、代次、快照来源均匹配请求的 Resume 目标；这仅为清理证据，不代表恢复成功。 +创建操作通过 `Bootstrap.Harness` 将会话选择传递到 Runtime 启动协议版本 2。Core 和节点使用协议版本 7，需协调升级配套组件。`Bootstrap.Harness` 是必填字段。十项暂停操作均属于同一个 `SandboxProvider`，必须完整声明支持或不支持；不通过可选接口分派。`Observe` 每次只观测一个 allocation。 -创建操作通过 `Bootstrap.Harness` 将会话选择传递到 Runtime 启动协议版本 2。Core 和节点使用协议版本 6；发布此 Core 版本前需升级配套节点。 +当前线协议版本为 7。Create 引导和 Resume 请求可携带可选的工作区文件系统绑定。节点在转发前校验其租户及 Environment 与分配引用一致、ObjectID 不可变且有效，以及挂载配置 ID 与所传配置一致。文件系统解析器负责适配器原生所有权校验。未提供绑定时选择自有存储；已提供绑定时不得回退。错误响应仅可保留原生 ID 非空且名称、代次、保留状态来源均匹配请求的 Resume 目标；这仅为清理证据，不代表恢复成功。 diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml index 70471dc31..d91f0d8cd 100644 --- a/deploy/compose/compose.yaml +++ b/deploy/compose/compose.yaml @@ -59,6 +59,8 @@ services: OAC_CREDENTIAL_KEY_FILE: /run/oac/credential.key OAC_CORE_KEY_DIGESTS_FILE: /run/oac/core-key-digests.json OAC_EXECUTION_CONCURRENCY: ${OAC_EXECUTION_CONCURRENCY:-} + OAC_SANDBOX_MAX_ACTIVE: ${OAC_SANDBOX_MAX_ACTIVE:-} + OAC_SANDBOX_MAX_RETAINED: ${OAC_SANDBOX_MAX_RETAINED:-} OAC_DEFAULT_HARNESS: ${OAC_DEFAULT_HARNESS:-} OAC_HARNESSES: ${OAC_HARNESSES:-} OAC_WRITE_AUDIT_RETENTION: ${OAC_WRITE_AUDIT_RETENTION:-} diff --git a/deploy/compose/test_compose.py b/deploy/compose/test_compose.py index 4f14ef726..7a4fcc6b3 100644 --- a/deploy/compose/test_compose.py +++ b/deploy/compose/test_compose.py @@ -71,9 +71,16 @@ def test_compose_uses_private_services_and_ordered_initialization(self): self.assertEqual(services['web']['healthcheck']['test'], ['CMD', '/usr/local/bin/oac-web', 'healthcheck']) self.assertNotIn('python3', json.dumps(self.compose)) self.assertEqual(services['init']['environment']['OAC_REVISION'], 'd' * 40) - for name in ('OAC_EXECUTION_CONCURRENCY', 'OAC_DEFAULT_HARNESS', 'OAC_HARNESSES', 'OAC_WRITE_AUDIT_RETENTION', 'OAC_LOG_LEVEL'): + for name in ('OAC_SANDBOX_MAX_ACTIVE', 'OAC_SANDBOX_MAX_RETAINED', 'OAC_EXECUTION_CONCURRENCY', 'OAC_DEFAULT_HARNESS', 'OAC_HARNESSES', 'OAC_WRITE_AUDIT_RETENTION', 'OAC_LOG_LEVEL'): self.assertEqual(services['core']['environment'][name], '', name) + def test_direct_sandbox_capacity_reaches_core(self): + from unittest.mock import patch + with patch.dict(os.environ, {'OAC_SANDBOX_MAX_ACTIVE': '17', 'OAC_SANDBOX_MAX_RETAINED': '61'}): + configured = self.render()['services']['core']['environment'] + self.assertEqual(configured['OAC_SANDBOX_MAX_ACTIVE'], '17') + self.assertEqual(configured['OAC_SANDBOX_MAX_RETAINED'], '61') + def test_public_url_can_be_configured_after_initial_startup(self): for value in (None, '', 'https://oac.example.test', 'http://localhost:9080'): with self.subTest(public_url=value): diff --git a/docs/configuration.md b/docs/configuration.md index 02ca6682e..3f4f38c0f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -50,6 +50,8 @@ Model providers are not process settings; see [Default models](#default-models). | `OAC_LOG_FORMAT` | `auto` | `auto`, `text` or `json` | | `OAC_LOG_ADD_SOURCE` | unset | `1` adds source locations | | `OAC_EXECUTION_CONCURRENCY` | `4` | Concurrent execution work, from 1 to 1024 | +| `OAC_SANDBOX_MAX_ACTIVE` | `100` | Active sandbox limit for direct Providers with suspension, from 1 to 100000. Independent of execution concurrency and node capacity | +| `OAC_SANDBOX_MAX_RETAINED` | `400` | Retained sandbox limit for direct Providers, from 1 to 100000, including active, suspended and unconfirmed cleanup. Must be at least the active limit | | `OAC_DEFAULT_HARNESS` | `codex` | Harness used when a request does not name one | | `OAC_HARNESSES` | Every registered Harness | Comma-separated Harnesses to enable besides the default one. Unknown names stop startup | | `OAC_WRITE_AUDIT_RETENTION` | `2160h` | Minimum `1h` | @@ -87,6 +89,10 @@ Runtime settings live in Core's database. Change them in Web; scripts use the sa Which harnesses are enabled, and the default one, are process settings (`core.harnesses`, `core.default_harness`); System shows them read-only. The [Core administration API](../contracts/agents-api/admin-api.md) lists every Core API route, and the [deployment contract](../contracts/agents-api/sandbox-deployment.md) defines the sandbox fields, limits and change rules. +### Direct Provider capacity + +Set `OAC_SANDBOX_MAX_ACTIVE` and `OAC_SANDBOX_MAX_RETAINED` in `.env`, then run `oac apply`. Core uses these limits for direct Providers with suspension enabled. Every unreleased allocation consumes retained capacity. Lowering a limit stops no existing sandbox; new allocations wait until usage falls below both limits. These settings are independent of `OAC_EXECUTION_CONCURRENCY` and enrolled node capacity. + ### Independent workspace storage The initial supported independent filesystem combination is microsandbox with the [kernel NFS adapter](./workspace-provider.md#kernel-nfs-adapter). Mount the same NFSv4.2 export on the Linux Core host and every participating node before starting their services, at the same absolute path, for example `/srv/oac-workspaces`. The operator manages the export, mount availability and service startup ordering. Use a trusted-client AUTH_SYS export with `root_squash`; do not enable `no_root_squash` or broaden permissions to make a check pass. Prepare the namespace and service principals according to the adapter's [ownership requirements](./workspace-provider.md#kernel-nfs-adapter). @@ -193,7 +199,7 @@ Core reads its process environment. Compose interpolates `.env` into it and moun | `OAC_CREDENTIAL_KEY_FILE` | Required. `/run/oac/credential.key`: a base64-encoded random 32-byte key. Core seals stored credentials with it | | `OAC_CORE_KEY_DIGESTS_FILE` | Required. `/run/oac/core-key-digests.json`: a JSON array with the SHA-256 of the Core key | | `OAC_INSTALLATION_ID_FILE` | Required. `/run/oac/installation.id`: the installation ID, a canonical UUID. Core refuses an ID other than the one its database recorded | -| `OAC_EXECUTION_CONCURRENCY`, `OAC_DEFAULT_HARNESS`, `OAC_HARNESSES`, `OAC_WRITE_AUDIT_RETENTION`, `OAC_OAUTH_TRUSTED_ORIGINS`, `OAC_HISTORY_SETTINGS_FILE`, `OAC_LOG_LEVEL`, `OAC_LOG_FORMAT`, `OAC_LOG_ADD_SOURCE` | The matching [process settings](#settings). Web reads the three log settings too | +| `OAC_SANDBOX_MAX_ACTIVE`, `OAC_SANDBOX_MAX_RETAINED`, `OAC_EXECUTION_CONCURRENCY`, `OAC_DEFAULT_HARNESS`, `OAC_HARNESSES`, `OAC_WRITE_AUDIT_RETENTION`, `OAC_OAUTH_TRUSTED_ORIGINS`, `OAC_HISTORY_SETTINGS_FILE`, `OAC_LOG_LEVEL`, `OAC_LOG_FORMAT`, `OAC_LOG_ADD_SOURCE` | The matching [process settings](#settings). Web reads the three log settings too | | `OAC_PROVIDER_ROOT` | Absolute adapter artifact root. The Core image sets `/opt/oac`. Each adapter owns its helper paths beneath this root. Core serves self-hosted daemon installers from its `native-installers/` directory when that holds a `catalog.json`, after checking the catalog against its own release. Adapter state lives at `/state`, the data volume's [`state/`](#compose-installations) | Core logs the history file path it loads, never environment values or file contents. diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index 0ea13768f..023581438 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -190,7 +190,3 @@ Installation and mutating `oac` commands share the [installation lock](../config Web signs administrators in with the Core key, checks the origin of every request, and forwards signed-in `/core/v1` requests to Core with the Core key, which stays on the server. It forwards `/v1` and `/api/v1` to Core unchanged, with the caller's own credential, serves only the non-secret node payload at `/node-install/`, and has no Docker or KVM access. Machine routes under `/api/v1` use their own enrollment and connection credentials. No service receives a Docker socket. Sandboxes are the isolation boundary ([Runtime and outer isolation](../concepts.md#runtime-and-outer-isolation)). Docker sandboxes share the node's kernel, and a Docker node is [root-equivalent](./nodes.md#what-the-installer-sets-up) on its host; microsandbox gives each sandbox a microVM with an explicit [network policy](./nodes.md#what-the-installer-sets-up). Core itself has no Docker socket or KVM access. - -## Runtime availability probes - -Codex CLI availability emits `runtime version probe` records for `process_spawn` and `process_wait`, with duration and outcome only. The wait interval includes executable loading and the version command; neither interval is model execution. diff --git a/docs/runtime-bootstrap.md b/docs/runtime-bootstrap.md index 8284d9f51..5a788ddf5 100644 --- a/docs/runtime-bootstrap.md +++ b/docs/runtime-bootstrap.md @@ -34,6 +34,10 @@ The Runtime validates the input and owns authentication and connection. A succes Self-hosted executors and operator-provisioned devices get their daemon identity in other ways; the [machine connection API](../contracts/agents-api/machine-api.md#credentials) lists every credential source. All of them enter the same Runtime execution loop. +## Hosted suspension control + +The private hosted park/wake control-file path is authored as `SuspendControlFile` in `internal/runtimebootstrap/bootstrap.go`. Core recovery and native Go adapters read that value; the E2B helper contract generator projects it into the template builder. Managed startup prepares its private directory and supplies `OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE` to enable Runtime suspension. This is a packaged protocol setting. The shared Sandbox Provider registration owns idle and retention defaults; the adapter owns its native lease timeout. + ## Verification `go test ./internal/runtimebootstrap ./apps/daemon/internal/cli` covers the input contract, the exclusivity of credential sources and restart behavior. Provider tests verify delivery and file permissions without relying on the Runtime's private storage. diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 149b7c352..6554d8360 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -25,7 +25,7 @@ Core owns durable Environment, allocation, placement and cleanup state; the Prov ## Implement the interface -`sandbox_provider.go` holds the Core–Sandbox Provider protocol: the `SandboxProvider` interface for allocation, checkpoint and observation, its request and result types, and the setup-time `ConfigurationAdapter` with its typed errors. Value types that Core also uses beyond this boundary, such as `DeploymentSpec` and `CallFence`, live in their own files of the same package. Every method is required at compile time, and `ProviderOperations()` declares which ones the Provider supports. Five operations are always supported: +`sandbox_provider.go` holds the Core–Sandbox Provider protocol: the `SandboxProvider` interface for allocation, suspension and observation, its request and result types, and the setup-time `ConfigurationAdapter` with its typed errors. Value types that Core also uses beyond this boundary, such as `DeploymentSpec` and `CallFence`, live in their own files of the same package. Every method is required at compile time, and `ProviderOperations()` declares which ones the Provider supports. Five operations are always supported: | Operation | Purpose | | --- | --- | @@ -45,7 +45,7 @@ Every provider returns a complete `ProviderOperations()` declaration with one en | --- | --- | --- | | `Create`, `GetInfo`, `Renew`, `Kill`, `RunCommand` | Supported | Allocation lifecycle and bounded commands | | `Observe` | Explicit decision | Ownership-checked read-only observation of one allocation | -| `Initial`, `NewCompute`, `GetCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute` | The same decision for every checkpoint method; supported only by a `nodes` registration | Exact compute incarnations, capture and restore, retained-source resume and cleanup | +| `Initial`, `NewCompute`, `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteRetained`, `RunCommandCompute` | The same decision for all ten suspension methods in `nodes` or `direct` mode | Exact compute incarnations, capture and restore, retained-source resume and cleanup | `Initial` and `NewCompute` construct compute references without allocating, `ResumeCompute` thaws only the same resident instance after an aborted pause, and `RunCommandCompute` runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)). @@ -89,7 +89,7 @@ Every call receives a bounded context. Expiry or cancellation ends the caller's `ErrInvalid`, `ErrOwnership`, `ErrExists`, `ErrNotFound`, `ErrComputeUnconfirmed` and `ErrCommandUnconfirmed` keep their defined meanings. An unclassified native or transport error is unknown, never permission to retry a mutation. Core never reads provider diagnostics as lifecycle truth or exposes native error text or credentials; the node transport maps errors to fixed codes, and direct SDK details stay private. -Checkpoint support adds `Compute` generation, name and ID and `SnapshotIdentity`; persist operation IDs and the provider's snapshot provenance unchanged. `ObserveOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and snapshot, not whatever instance now has the same name. Read [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) and its failure tests before declaring checkpoint support. +Suspension support adds `Compute` generation, name and ID and `RetainedState`; persist operation IDs and the provider's retained-state provenance unchanged. `ReconcileOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and retained state, not whatever instance now has the same name. Read [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) and its failure tests before declaring suspension support. ### Four distinct readiness facts @@ -112,7 +112,7 @@ A new provider takes these steps: 2. Add its specification and resource validators. 3. Implement `sandbox.ConfigurationAdapter` over a typed native configuration. `DecodeInput` strictly parses the separate public `configuration` and write-only `credential` objects of a request. `Encode` produces whitelisted public selectors, read-only observations and separate secret bytes, and never passes request JSON through. `Decode` restores stored selectors, and keeps access to owned resources, without remote admission or new template validation. `Normalize` copies its input before changing it. `ResolveChange`, `Equal` and `WithCredential` own inheritance, identity and credential composition. `Requirements` declares whether a credential and a public Core origin are required, and which setup operations are supported: `Discovery` for `DiscoverConfiguration`, `SelectionDiscovery` for `DiscoverSelection` and `CredentialVerification` for `VerifyCredential`. `DiscoverConfiguration` validates the query and returns a safe catalog, never a mutation or an admission decision, while Core keeps authorization, input limits and deadlines. `DiscoverSelection` resolves a candidate's omitted native values before commit, and `VerifyCredential` verifies a credential's access to owned resources without mutation. Both receive the candidate's `sandbox.DirectConfig` and build any native client for that call only. A node provider accepts only an empty public object, rejects credentials and returns Unsupported for every setup operation and for credential replacement. 4. For a node adapter, export from its package the `BuildLocal` constructor, the typed `native` object it decodes and the native files it adds to the shared node artifacts. Node-local settings, such as host paths, live only in that object; resources and the Runtime release are read from the node configuration's `specification`. -5. Register its constructor, policies, configuration adapter and operation declaration in `providers/registry.go`. Its key is the provider kind, which also labels the Provider's observations, and checkpoint support reads this entry. The generated projections combine each registered mode and deployment policy with the shared field bounds in `sandbox/deployment_contract.go`: the installer reads them from `deploy/node/node_spec.py`, and the TypeScript client and Web from `packages/agents-client/src/deployment-contract.ts`, so Web reads these declarations instead of comparing provider kinds. Regenerate both with `go run ./services/core/cmd/specification-contract -write`. +5. Register its constructor, policies, configuration adapter and operation declaration in `providers/registry.go`. Its key is the provider kind, which also labels the Provider's observations, and suspension support reads this entry. The generated projections combine each registered mode and deployment policy with the shared field bounds in `sandbox/deployment_contract.go`: the installer reads them from `deploy/node/node_spec.py`, and the TypeScript client and Web from `packages/agents-client/src/deployment-contract.ts`, so Web reads these declarations instead of comparing provider kinds. Regenerate both with `go run ./services/core/cmd/specification-contract -write`. 6. Supply the distribution artifacts for the adapter and its helper, and offer the provider to operators through the registered configuration contract. **Known design gap:** Web still names providers in the setup wizard's backend choice, the Docker confirmation and E2B's configuration fields wherever Web shows or parses them (the setup step with its service presets, the deployment summary and the client's deployment projection), because the protocol declares no configuration fields yet. Exposing another provider through that surface currently requires a shared Web edit. This coupling does not meet [Complexity stays in the adapter](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#complexity-stays-in-the-adapter); new integrations must express their configuration through the protocol and keep vendor-specific behavior in the adapter. Never add a Session or Turn scheduling path, a vendor column or API field, or a vendor switch in the store. @@ -126,7 +126,7 @@ A node configuration, `sandbox.NodeConfig`, holds `provider`, `generation`, `ins - A `nodes` registration has only `BuildLocal`, and a `direct` registration only `BuildDirect`; missing, mixed or unknown modes are rejected. - The specification and resource validators, the configuration adapter and a complete operation declaration are mandatory, so an incomplete registration cannot publish a partial projection. A declared default size must pass the adapter's resource validator. - The Runtime input policy either accepts the pinned Runtime or gives the adapter's fixed reason for rejecting it, never both. -- Checkpoint is admitted only for a `nodes` registration, because the common lifecycle suspends only node allocations; registration rejects a `direct` Provider that declares it. A registration carries no suspension values: Core applies its one [suspension policy](#suspension) to every Provider that declares checkpoint support. +- Suspension is admitted in both `nodes` and `direct` modes when the complete lifecycle is declared supported. A registration carries no suspension values: Core applies its one [suspension policy](#suspension) to every supported Provider. The configuration adapter must be non-nil, including its concrete value. Every `ConfigurationRequirements` field needs an explicit valid decision: `Credential` and `PublicOrigin` are `Required` or `NotRequired`, and `Discovery`, `SelectionDiscovery` and `CredentialVerification` use the shared supported or unsupported declaration with a safe reason. A new requirement field needs an explicit validation update and never inherits an existing decision. Requiring a credential does not promise the `VerifyCredential` operation. These checks establish complete registration, not correct native SDK behavior; constructor and adapter contract tests still apply. @@ -136,7 +136,7 @@ Preview and persistence use `providers.Normalize` and `providers.Describe`. `pro A direct adapter with a credential verifies all retained generations and allocation references before a key is replaced. The common `sandbox.CallFence` excludes native calls and waits for helper completion, including calls whose callers timed out; execution invokes the prepared verification and fencing callbacks without branching on a vendor. -Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `sandbox.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and a `Quiescent` check when a helper can outlive its caller; generation collection waits for it. The factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes checkpoint operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through the checkpoint declaration, never through a provider name. +Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `sandbox.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and a `Quiescent` check when a helper can outlive its caller; generation collection waits for it. The factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes suspension operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through the suspension declaration, never through a provider name. The backend fingerprint identifies a native resource namespace, not capacity. Core keeps deployment generations so that owned allocations keep resolving to their original backend; never repoint retained allocations at a replacement backend. @@ -166,15 +166,15 @@ The allocation, its dedicated daemon credential digest and the exact Session bin With a configured provider, the Worker scans committed pending hosted Environments that have no allocation, which covers idle Session creation and recovery after an interruption between commit and bootstrap; an existing allocation never re-enters that path. The scan is bounded and serialized by the lifecycle owner and needs no caller action. An initial reservation without a Turn leaves its Session idle, and a daemon connection is never treated as native readiness. The same scan publishes authenticated connection observations with durable generations, after verifying the exact Session and device binding and a settled bootstrap. -Between Turns, Core checks that connected, observed compute is still its Session's running allocation; the check changes nothing and never revives a cleanup request. Running compute never expires: explicit deletion and the snapshot retention authorize its cleanup. A stopped or missing container never authorizes discarding retained workspace or history. Disabling the provider stops new hosted admission and bootstrap but never blocks cancellation, function results or input retry outcomes of existing Sessions. +Between Turns, Core checks that connected, observed compute is still its Session's running allocation; the check changes nothing and never revives a cleanup request. Running compute never expires: explicit deletion and the retained-state retention authorize its cleanup. A stopped or missing container never authorizes discarding retained workspace or history. Disabling the provider stops new hosted admission and bootstrap but never blocks cancellation, function results or input retry outcomes of existing Sessions. Terminal cleanup atomically revokes the device's authority, records the Environment's failure or expiry, settles pending input and requests cancellation, and only then calls `Kill`; original input deadlines and retry outcomes are kept. Temporary provider outages, unknown Create results and stopped compute never prove a permanent failure. After public Session deletion Core keeps the allocation and marks it released only after owned compute and volume cleanup and proof that the original Create settled; an unknown creation keeps cleanup ownership even after an absence observation, and bounded scans continue to catch late resources without another `Create`. ### Per-node lifecycle workers -Each registered node has one serial lifecycle worker that owns its gate, allocation and pending cursors, connections and wake hints; E2B allocations share one serial lifecycle without a node. A thin coordinator discovers nodes and shuts workers down, and never holds its map mutex during database, provider or wait operations. Workers advance independently, so a stuck provider on one online node never stalls another: lifecycle concurrency is one operation per node and grows with the node count. Offline workers stay, so their retained resources remain observable after reconnection. +After a hosted Environment commits, Core sends a bounded hint to the lifecycle worker for its placement. Committed pending input also sends a hint to recover a missed creation notification. Hints reuse the serial gate, execution lease, capacity checks and one-shot allocation receipt; admission handlers never create a sandbox directly. Each normal maintenance period admits at most one extra hinted scan, and periodic scans recover missed or coalesced hints. Existing allocations are observed before pending provisioning, so a slow observation on the same node can still delay a fresh Environment. -After a hosted Environment commits, Core sends a bounded hint to its placement’s lifecycle worker. A committed pending input also sends a hint, recovering a missed creation notification. Hints use the existing serial gate, lease, capacity checks and one-shot allocation receipts; they do not provision inline in the admission handler. Each normal maintenance period permits at most one extra hinted scan, and periodic scans recover missed or coalesced hints. Existing allocation observations still precede pending provisioning, so a slow observation on the same node can delay a fresh Environment. +Each registered node has one serial lifecycle worker that owns its gate, allocation and pending cursors, connections and wake hints; E2B allocations share one serial lifecycle without a node. A thin coordinator discovers nodes and shuts workers down, and never holds its map mutex during database, provider or wait operations. Workers advance independently, so a stuck provider on one online node never stalls another: lifecycle concurrency is one operation per node and grows with the node count. Offline workers stay, so their retained resources remain observable after reconnection. Allocation scans filter by node before their 32-row page limit, and pending scans join the unreleased committed placement. Each node advances its own cursor, including past failed observations, and wraps once at the end. Direct provisioning resolves the tenant-scoped placement before entering that node's gate, and an existing allocation must agree with it; Core never chooses another node. @@ -184,15 +184,31 @@ Before releasing the execution lease, the coordinator stops accepting work and c Placement is automatic: the environment-to-node placement commits with Session creation and its retry identity, callers cannot choose a node, and a retry keeps its original node even while it is offline. Node capacity counts pending reservations and unresolved resources, and new placement and a suspended-to-restoring transition share a database lock. Unknown operations keep their reservations, source teardown must be confirmed before active capacity is released, and confirmed cleanup releases placement capacity. Retained ownership needs exact provider evidence: a socket path, a missing instance or an empty listing never proves cleanup or authorizes a replacement. -The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and the snapshot retention bound each node. There is no node-level drain, cross-node Session migration, multi-active Core, autoscaling or snapshot replication. Node removal is refused while the node holds allocations, snapshots, reservations, unknown results or cleanup, and offline ownership is kept. +The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and the retained-state retention bound each node. There is no node-level drain, cross-node Session migration, multi-active Core, autoscaling or snapshot replication. Node removal is refused while the node holds allocations, retained states, reservations, unknown results or cleanup, and offline ownership is kept. ### Suspension -Core suspends the idle work of every provider that declares checkpoint support, with one fixed policy: it suspends work idle for 5 minutes (300 seconds) and keeps the snapshot for 24 hours (86400 seconds). The deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) reports these values. Core suspends only after at least one Turn is terminal, when no root or Subagent Turn is queued, in progress or waiting, no input, file operation or initialization is pending, and real activity has been idle for that time. For node allocations Core records the first root or child terminal transition with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial snapshot retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. +The single `SandboxProvider` contract declares `Initial`, `NewCompute`, `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `KillCompute`, `DeleteRetained`, `RunCommandCompute` and `ResumeCompute` together: all are supported or all are unsupported. No optional suspension interface or provider-name branch selects this lifecycle. + +`RetainedState` is an opaque adapter-owned envelope with `Reference`, `ID`, `Data`, `OperationID`, `SourceGeneration`, `SourceName` and `SourceID`. `Data` is nonempty and at most 64 KiB. Core preserves the envelope unchanged and validates operation and source ownership; the immutable allocation provider/generation binding selects the only adapter allowed to interpret it. A retained state need not be an independent snapshot. Microsandbox retains its complete native snapshot identity in `Data`; E2B retains its versioned native pause receipt and cannot promise a separate disk image. Each adapter validates its native payload and ownership before any effect. + +`Compute.RestoredFrom`, `ComputeState.Retained`, `ResourcesReleased`, `SuspendSettled` and recovery's `ReconcileOnly` are authoritative settlement evidence. A missing resource is not proof that an unknown suspension settled. `RenewCompute` renews only the exact incarnation; an uncertain result never authorizes a second create, capture or restore. Durable `runtime_compute` uses `protocol_version: "1"`; startup rejects missing or unknown versions. Version-1 envelopes matching the current retained-state field shape remain readable without a rewrite. Snapshot-only durable shapes are rejected even when labeled version 1; equal version numbers do not establish compatibility. Upgrade performs no migration, synthetic replay or retained-state discard. + +`ResumeRequest.Workspace` carries the allocation's independently owned filesystem binding. Core obtains a ready binding before restore. Suspension must prove the source released active execution before a target can become a writer. Adapters validate the binding before native effects; an adapter without independent filesystem support rejects a non-null binding. `DeleteRetained` deletes only adapter-owned retained compute state, never the workspace. Explicit Session deletion first settles compute cleanup and then deletes independent storage, preserving a single active writer throughout recovery. + +Core suspends the idle work of every provider that declares suspension support, with one fixed policy: it suspends work idle for 5 minutes (300 seconds) and keeps retained state for 24 hours (86400 seconds). The deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) reports these values. Core suspends initialized Environments, including those with no Turn yet, when no root or Subagent Turn is queued, in progress or waiting, no input, file operation or initialization is pending, and real activity has been idle for that time. For allocations Core records initialization completion and root or child terminal transitions with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial retained-state retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. + +The Worker lease, the Session lock and the per-node gates own suspension for every provider. New Turn claims, file-write intents and capture admission serialize under the Session lock and share one compute-phase check; new pending work cancels a capture and wakes the same source. Normal preparation waits for the compute phase to be running, after the authenticated resume handshake, and pending input stays pending when its promotion conflicts with a lifecycle transition. Compute phases and revision-checked receipts live on the allocation. Core persists quiesce, capture and restore intent before the effect, only a fresh receipt performs a capture or restore, and recovery observes the exact attempt without retrying an unknown creation, capture or restore. Consumed retained state never rolls a running generation back. Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap, and unknown cleanup identities are kept until owned resources are confirmed absent. Consumed artifacts and old compute are deleted, so suspension cycles never build a chain of writable disks. + +Queued work and live Environment file access wake a suspended Environment; history and published Artifact reads do not. Planned suspension uses an Environment and suspension token on the daemon connection. A PID and start-time fenced local control signal (`RunCommandCompute`) wakes the parked daemon, which authenticates again before admitting work. A transient disconnect before confirmation retries the same armed suspension with bounded attempts and backoff; a permanent authentication or protocol rejection closes it. Core owns the retained state's retention deadline, and the daemon has no timer for it. A lost quiesce acknowledgement may thaw the same source through explicit rollback but never authorizes capturing it. + +`Suspend` owns native resource release. Core releases active capacity only after receiving a bound retained handle, suspended state, `ResourcesReleased` and `SuspendSettled`. `ReconcileOnly` prohibits replaying the original capture or pause but permits adapter cleanup justified by durable retained evidence. An outcome without retained state permits rollback only with `SuspendSettled` and an exact source that can resume; other uncertainty retains ownership and closes admission. Core never unconditionally destroys the source after `Suspend`. + +`Resume` consumes retained state once into the precommitted target. Recovery observes that same attempt. Core persists `waking`, authenticates and resumes the daemon, deletes consumed retained resources, then commits `running` and admits work. `DeleteRetained` is idempotent cleanup that preserves running compute. Cleanup failure keeps `waking` and cannot trigger another restore. `KillCompute` remains destructive; old-generation cleanup must not kill a newer active incarnation sharing its native ID. Every unreleased allocation, including a running one, consumes `max_retained`. Every reservation records the shared compute protocol version, including allocations whose compute phase is `disabled`. Activation rejects an unreleased allocation with a missing or different version; the old version must complete its normal cleanup before upgrade. Session history remains. -The Worker lease, the Session lock and the per-node gates own suspension for every provider. New Turn claims, file-write intents and capture admission serialize under the Session lock and share one compute-phase check; new pending work cancels a capture and wakes the same source. Normal preparation waits for the compute phase to be running, after the authenticated resume handshake, and pending input stays pending when its promotion conflicts with a lifecycle transition. Compute phases and revision-checked receipts live on the allocation. Core persists quiesce, capture and restore intent before the effect, only a fresh receipt performs a capture or restore, and recovery observes the exact attempt without retrying an unknown creation, capture or restore. A consumed snapshot never rolls a running generation back. Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap, and unknown cleanup identities are kept until owned resources are confirmed absent. Consumed artifacts and old compute are deleted, so suspension cycles never build a chain of writable disks. +### Coordinated protocol upgrade -Queued work and live Environment file access wake a suspended Environment; history and published Artifact reads do not. Planned suspension uses an Environment and suspension token on the daemon connection. A PID and start-time fenced local control signal (`RunCommandCompute`) wakes the parked daemon, which authenticates again before admitting work. A transient disconnect before confirmation retries the same armed suspension with bounded attempts and backoff; a permanent authentication or protocol rejection closes it. Core owns the snapshot's retention deadline, and the daemon has no timer for it. A lost quiesce acknowledgement may thaw the same source through explicit rollback but never authorizes capturing it. +Upgrade Core and nodes together with node wire version 7, the E2B helper with private wire version 4 and its rebuilt template containing hosted suspension control, and the microsandbox helper with private wire version 5. All Providers deliver Runtime bootstrap version 2. Each boundary validates its own contract; version numbers are not interchangeable across boundaries. Before activation, use the previous compatible release to complete normal cleanup of incompatible unreleased allocations. The startup fence preserves their stored receipts and Session history and makes no Provider calls to migrate them. ### Reset and archive @@ -200,7 +216,7 @@ A [reset](../contracts/agents-api/sandbox-deployment.md#reset) is durable execut One snapshot and timestamp partition the held resources. Offline ownership comes from the allocation's or active placement's node, with the same 45-second connection and owner-epoch predicate as online presence, independently of provider readiness; no cleanup failure, offline state or empty read authorizes a synthetic release. At zero held resources, Core drains outside database transactions, rechecks under the deployment lock and clears the deployment in one transaction, then publishes a generation-bearing empty provider without fallible work. If the final write or drain fails, it restores the committed provider with a bounded owner context before releasing the mutation gate; if that recovery fails, admission stays fenced and the owner stops. -An administrator [Session archive](../contracts/agents-api/admin-api.md#session-archive) keeps its Project scope and hosted eligibility checks in a Session-first transaction, together with Environment expiry, cancellation, Runtime authority revocation and audit; a reset's background archive reconstructs the actual Project scope from trusted records and keeps the requester's provenance. The ordinary provider lifecycle releases compute and snapshots. Archive cancellation keeps a healthy receipt path until the terminal commit: only the archive that first revokes a device records its exact `archive_cancel_turn_id` (ordinary revocation clears it, and repeated cleanup keeps it), and the existing authenticated delivery may drain that cancellation for at most 20 seconds from the Turn's original `cancel_requested_at`. Core tracks the delivery through `done`, the cancellation acknowledgement and the terminal commit, independently of subscription removal, and grants no new connection, input, file or MCP authority or lease renewal. No transaction or lifecycle gate waits for the receipt, and a lost peer, expiry or restart falls back to ordinary failure and cleanup, never a fabricated cancelled outcome. +An administrator [Session archive](../contracts/agents-api/admin-api.md#session-archive) keeps its Project scope and hosted eligibility checks in a Session-first transaction, together with Environment expiry, cancellation, Runtime authority revocation and audit; a reset's background archive reconstructs the actual Project scope from trusted records and keeps the requester's provenance. The ordinary provider lifecycle releases compute and retained states. Archive cancellation keeps a healthy receipt path until the terminal commit: only the archive that first revokes a device records its exact `archive_cancel_turn_id` (ordinary revocation clears it, and repeated cleanup keeps it), and the existing authenticated delivery may drain that cancellation for at most 20 seconds from the Turn's original `cancel_requested_at`. Core tracks the delivery through `done`, the cancellation acknowledgement and the terminal commit, independently of subscription removal, and grants no new connection, input, file or MCP authority or lease renewal. No transaction or lifecycle gate waits for the receipt, and a lost peer, expiry or restart falls back to ordinary failure and cleanup, never a fabricated cancelled outcome. ## Validate the integration diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 793535d52..18a33cf31 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: eeb840aefb1eab56b16b5185c9fb6b3fcba5f0e9e666eaa32cf9e02f1e2fe134 +source_hash: 2c38905154d559c2a8def738e4514b4b32ca563f34d20d84407af8ad87b210b3 --- Core 安装的每项设置都恰好只有一个归属位置，分属以下三类： @@ -54,6 +54,8 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 | `OAC_LOG_FORMAT` | `auto` | `auto`, `text` or `json` | | `OAC_LOG_ADD_SOURCE` | unset | `1` adds source locations | | `OAC_EXECUTION_CONCURRENCY` | `4` | Concurrent execution work, from 1 to 1024 | +| `OAC_SANDBOX_MAX_ACTIVE` | `100` | 启用暂停能力的直接 Provider 的活跃沙箱上限，范围 1–100000；独立于执行并发度和节点容量 | +| `OAC_SANDBOX_MAX_RETAINED` | `400` | 直接 Provider 的保留沙箱上限，范围 1–100000，包括活跃、已暂停以及尚未确认清理完成的分配；必须不小于活跃上限 | | `OAC_DEFAULT_HARNESS` | `codex` | Harness used when a request does not name one | | `OAC_HARNESSES` | Every registered Harness | Comma-separated Harnesses to enable besides the default one. Unknown names stop startup | | `OAC_WRITE_AUDIT_RETENTION` | `2160h` | Minimum `1h` | @@ -91,6 +93,10 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 哪些 Harness 已启用以及默认 Harness 属于进程设置（`core.harnesses`、`core.default_harness`）；System 会以只读方式显示它们。[Core 管理 API](../../contracts/agents-api/zh/admin-api.md) 列出了所有 Core API 路由，[部署契约](../../contracts/agents-api/zh/sandbox-deployment.md) 定义了沙箱字段、限制和更改规则。 +### 直接 Provider 容量 {#direct-provider-capacity} + +在 `.env` 中设置 `OAC_SANDBOX_MAX_ACTIVE` 和 `OAC_SANDBOX_MAX_RETAINED`，然后运行 `oac apply`。Core 将这些限制用于启用暂停能力的直接 Provider。每个尚未释放的分配都占用保留容量。降低上限不会终止已有沙箱；新分配会等待使用量低于两项上限。这些设置独立于 `OAC_EXECUTION_CONCURRENCY` 和已注册节点的容量。 + ### 独立工作区存储 {#independent-workspace-storage} 首个支持的独立文件系统组合是 microsandbox 与[内核 NFS 适配器](./workspace-provider.md#kernel-nfs-adapter)。启动服务前，在 Linux Core 主机和所有参与节点上将同一个 NFSv4.2 导出挂载到相同的绝对路径，例如 `/srv/oac-workspaces`。运维人员负责导出、挂载可用性和服务启动顺序。使用带有 `root_squash` 的受信任客户端 AUTH_SYS 导出；不要启用 `no_root_squash` 或放宽权限来使检查通过。按照适配器的[所有权要求](./workspace-provider.md#kernel-nfs-adapter)准备命名空间和服务身份。 @@ -197,7 +203,7 @@ Core 读取进程环境。Compose 将 `.env` 插值到环境中，并把机密 | `OAC_CREDENTIAL_KEY_FILE` | 必填。`/run/oac/credential.key`：Base64 编码的 32 字节随机密钥。Core 用它加密存储的凭据 | | `OAC_CORE_KEY_DIGESTS_FILE` | 必填。`/run/oac/core-key-digests.json`：一个包含 Core 密钥 SHA-256 的 JSON 数组 | | `OAC_INSTALLATION_ID_FILE` | 必填。`/run/oac/installation.id`：安装 ID，采用规范 UUID 格式。如果 ID 与数据库记录的 ID 不一致，Core 会拒绝它 | -| `OAC_EXECUTION_CONCURRENCY`、`OAC_DEFAULT_HARNESS`、`OAC_HARNESSES`、`OAC_WRITE_AUDIT_RETENTION`、`OAC_OAUTH_TRUSTED_ORIGINS`、`OAC_HISTORY_SETTINGS_FILE`、`OAC_LOG_LEVEL`、`OAC_LOG_FORMAT`、`OAC_LOG_ADD_SOURCE` | 对应的[进程设置](#settings)。Web 也读取三个日志设置 | +| `OAC_SANDBOX_MAX_ACTIVE`、`OAC_SANDBOX_MAX_RETAINED`、`OAC_EXECUTION_CONCURRENCY`、`OAC_DEFAULT_HARNESS`、`OAC_HARNESSES`、`OAC_WRITE_AUDIT_RETENTION`、`OAC_OAUTH_TRUSTED_ORIGINS`、`OAC_HISTORY_SETTINGS_FILE`、`OAC_LOG_LEVEL`、`OAC_LOG_FORMAT`、`OAC_LOG_ADD_SOURCE` | 对应的[进程设置](#settings)。Web 也读取三个日志设置 | | `OAC_PROVIDER_ROOT` | 适配器构件的绝对根目录。Core 镜像设置为 `/opt/oac`。每个适配器都拥有此根目录下的辅助路径。当其中的 `native-installers/` 目录包含 `catalog.json` 时，Core 在核对该目录清单与自身发行版后提供自托管守护进程安装程序。适配器状态位于 `/state`，即数据卷的 [`state/`](#compose-installations) | Core 会记录所加载的历史文件路径，但绝不记录环境变量的值或文件内容。 diff --git a/docs/zh/getting-started/operations.md b/docs/zh/getting-started/operations.md index cfe626664..8a24fb8d9 100644 --- a/docs/zh/getting-started/operations.md +++ b/docs/zh/getting-started/operations.md @@ -1,7 +1,7 @@ --- title: "运维" source: docs/getting-started/operations.md -source_hash: f6ce9d76fdd074d9e7e7da4a69da6a4583a67848b86aa641ad09f127fcae346b +source_hash: c04a4d8fedef58355dfe5e7537415d81aed55420346bc99263157c04c8b60daf --- 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。 @@ -193,7 +193,3 @@ cd && rm -rf ~/.oac/core Web 使用 Core 密钥让管理员登录，检查每个请求来源，并用保留在服务器上的 Core 密钥将已登录的 `/core/v1` 请求转发到 Core。它把 `/v1` 和 `/api/v1` 原样转发给 Core，使用调用方自己的凭据；Web 仅在 `/node-install/` 提供不含密钥的节点文件，没有 Docker 或 KVM 访问权限。`/api/v1` 机器路由使用独立注册和连接凭据。没有服务持有 Docker 套接字。 沙箱是隔离边界（[Runtime 与外层隔离](../concepts.md#runtime-and-outer-isolation)）。Docker 沙箱共享节点内核，Docker 节点在主机上[等同于 root 权限](nodes.md#what-the-installer-sets-up)；microsandbox 为每个沙箱提供具有显式[网络策略](nodes.md#what-the-installer-sets-up)的 microVM。Core 自身无 Docker 套接字或 KVM 访问权限。 - -## Runtime 可用性探测 {#runtime-availability-probes} - -Codex CLI 可用性探测为 `process_spawn` 和 `process_wait` 输出 `runtime version probe` 记录，只包含耗时和结果。等待区间包含可执行文件加载和版本命令，两者都不是模型执行。 diff --git a/docs/zh/runtime-bootstrap.md b/docs/zh/runtime-bootstrap.md index 5bd813902..aeecb495e 100644 --- a/docs/zh/runtime-bootstrap.md +++ b/docs/zh/runtime-bootstrap.md @@ -1,7 +1,7 @@ --- title: "Runtime 引导" source: docs/runtime-bootstrap.md -source_hash: 73fb9968007f83d35f1e6fe185d9e376fa81fc7683fe822426b290632ca8de24 +source_hash: 537197f388a57bb983c0e7e0b8a3bfe66b1e8e0ef7f4f1fef572ad7dcf724cb3 --- Sandbox Provider 通过交付一个引导文件来启动托管 Runtime。本文负责 Provider 到 Runtime 的启动输入。类型与验证器位于 [`internal/runtimebootstrap`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/runtimebootstrap/bootstrap.go)；Go provider 使用 [`runtime_bootstrap.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/runtime_bootstrap.go) 中的 `sandbox.Bootstrap.RuntimeConnection()` 构造输入，SDK helper 原样转发序列化对象。provider 不读取或写入 Runtime 的私有认证存储。 @@ -36,6 +36,10 @@ Runtime 验证输入，并负责认证与连接。启动成功仅证明交付完 自托管 executor 和运维人员供应的设备通过其他方式获取 daemon 身份；[机器连接 API](../../contracts/agents-api/zh/machine-api.md#credentials) 列出所有凭据来源。它们都进入同一 Runtime 执行循环。 +## 托管暂停控制 {#hosted-suspension-control} + +私有托管暂停/唤醒控制文件的路径在 `internal/runtimebootstrap/bootstrap.go` 中以 `SuspendControlFile` 定义。Core 恢复逻辑和原生 Go adapter 读取该值；E2B helper 契约生成器将其投射到模板构建器。托管启动准备私有目录并提供 `OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE`，以启用 Runtime 暂停。这是随软件包发布的协议设置。共享 Sandbox Provider 注册负责空闲和保留默认值；adapter 负责自身原生租约超时。 + ## 验证 {#verification} `go test ./internal/runtimebootstrap ./apps/daemon/internal/cli` 覆盖输入契约、凭据来源互斥规则和重启行为。Provider 测试验证交付与文件权限，不依赖 Runtime 的私有存储。 diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 90b262e43..13ca28591 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: a69ba63ef2b5bd883e0a235807df9d270c23dab9b6df11cf6019534281ced8e9 +source_hash: e49b04aa281a5f8bba5838b14b9a3a7ab607a5484d79cd451fcd8bbcc77142d5 --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -27,7 +27,7 @@ Core 拥有持久 Environment、allocation、placement 和 cleanup 状态；Prov ## 实现接口 {#implement-the-interface} -`sandbox_provider.go` 包含 Core–Sandbox Provider 协议：负责 allocation、checkpoint 和观测的 `SandboxProvider` 接口及其请求与结果类型，以及 setup 阶段的 `ConfigurationAdapter` 及其类型化错误。Core 在此边界之外也使用的值类型，例如 `DeploymentSpec` 和 `CallFence`，位于同一 package 的独立文件中。每个方法在编译期都必须实现，`ProviderOperations()` 声明 Provider 支持哪些方法。以下五项操作始终支持： +`sandbox_provider.go` 包含 Core–Sandbox Provider 协议：负责 allocation、暂停和观测的 `SandboxProvider` 接口及其请求与结果类型，以及 setup 阶段的 `ConfigurationAdapter` 及其类型化错误。Core 在此边界之外也使用的值类型，例如 `DeploymentSpec` 和 `CallFence`，位于同一 package 的独立文件中。每个方法在编译期都必须实现，`ProviderOperations()` 声明 Provider 支持哪些方法。以下五项操作始终支持： | 操作 | 用途 | | --- | --- | @@ -47,7 +47,7 @@ Docker 等没有原生可续期租约的 backend 仍遵守 Core 的 hosted expir | --- | --- | --- | | `Create`、`GetInfo`、`Renew`、`Kill`、`RunCommand` | 支持 | Allocation 生命周期与有界命令 | | `Observe` | 明确决定 | 检查所有权、只读地观测一个 allocation | -| `Initial`、`NewCompute`、`GetCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute` | 所有 checkpoint 方法决定一致；仅 `nodes` 注册可以支持 | 精确计算实例、捕获与恢复、保留源恢复和清理 | +| `Initial`、`NewCompute`、`GetCompute`、`RenewCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteRetained`、`RunCommandCompute` | 全部十项暂停方法决定一致；`nodes` 与 `direct` 模式均可支持 | 精确计算实例、捕获与恢复、保留源恢复和清理 | `Initial` 和 `NewCompute` 构造 compute reference，不分配资源；`ResumeCompute` 在暂停中止后仅解冻同一驻留实例；`RunCommandCompute` 在一个精确 compute incarnation 中运行有界命令。Core 使用 `RunCommandCompute` 在恢复后唤醒 parked daemon（[`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)）。 @@ -91,7 +91,7 @@ Microsandbox 在返回创建不存在回执前永久关闭初始 Create 准入 `ErrInvalid`、`ErrOwnership`、`ErrExists`、`ErrNotFound`、`ErrComputeUnconfirmed` 和 `ErrCommandUnconfirmed` 保持其定义含义。未分类原生或 transport error 表示未知，不授权重试 mutation。Core 不将 provider diagnostics 读作生命周期事实，也不暴露原生错误文本或凭据；node transport 将错误映射为固定 code，直接 SDK 细节保持私有。 -Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity`；原样持久化 operation ID 与 provider snapshot provenance。suspend 或 resume 的 `ObserveOnly` 仅观察上次尝试，不启动另一 capture 或 restore。`ResumeCompute` 仅解冻保留源，不冷启动已停止源。清理针对精确 compute incarnation 和 snapshot，不针对当前同名实例。声明 checkpoint 支持前阅读 [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) 及其失败测试。 +暂停支持增加精确的 `Compute` 代次、名称和 ID，以及不透明的 `RetainedState` 收据。原样持久化操作 ID 和 Provider 保留状态的来源。暂停或恢复中的 `ReconcileOnly` 观察原始尝试，不会开始另一次捕获或恢复。`ResumeCompute` 仅解冻保留的源实例，绝不冷启动已停止的源实例。清理针对精确计算实例和保留状态，而非当前恰好同名的实例。声明支持前请阅读 [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) 及其失败测试。 ### 四个独立就绪事实 {#four-distinct-readiness-facts} @@ -114,7 +114,7 @@ Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity` 2. 添加 specification 和 resource validator。 3. 基于类型化原生配置实现 `sandbox.ConfigurationAdapter`。`DecodeInput` 严格解析请求中独立的公开 `configuration` 与只写 `credential` 对象。`Encode` 生成白名单公开 selector、只读观测和独立 secret bytes，不透传请求 JSON。`Decode` 恢复已存储 selector 并保留对所属资源的访问，不做远程 admission 或新模板验证。`Normalize` 修改前复制输入。`ResolveChange`、`Equal` 和 `WithCredential` 负责继承、身份与凭据组合。`Requirements` 声明是否需要凭据和公开 Core origin，以及支持哪些 setup 操作：`Discovery` 对应 `DiscoverConfiguration`，`SelectionDiscovery` 对应 `DiscoverSelection`，`CredentialVerification` 对应 `VerifyCredential`。`DiscoverConfiguration` 验证 query 并返回安全 catalog，不做 mutation 或 admission decision；Core 保留授权、输入限制与 deadline。`DiscoverSelection` 在提交前解析候选项省略的原生值，`VerifyCredential` 验证凭据对所属资源的访问，不修改资源。两者都接收候选项的 `sandbox.DirectConfig`，原生 client 只为该次调用构造。node provider 仅接受空公开对象，拒绝凭据，对每项 setup 操作和 credential replacement 返回 Unsupported。 4. node adapter 从自己的包中导出 `BuildLocal` constructor、它解码的类型化 `native` 对象，以及它在共享 node artifact 之外添加的原生文件。node-local 设置（如主机路径）只存放在该对象中；resources 和 Runtime release 从 node 配置的 `specification` 读取。 -5. 在 `providers/registry.go` 中注册 constructor、policy、configuration adapter 和 operation 声明。其键即 provider kind，也用于标记该 Provider 的观测；checkpoint 支持读取此项。生成的投影组合每个已注册的部署模式和 deployment policy 与 `sandbox/deployment_contract.go` 中的共享 field bound：installer 从 `deploy/node/node_spec.py` 读取，TypeScript 客户端和 Web 从 `packages/agents-client/src/deployment-contract.ts` 读取，因此 Web 读取这些声明，而不比较 provider kind。通过 `go run ./services/core/cmd/specification-contract -write` 重新生成两者。 +5. 在 `providers/registry.go` 中注册 constructor、policy、configuration adapter 和 operation 声明。其键即 provider kind，也用于标记该 Provider 的观测；暂停支持读取此项。生成的投影组合每个已注册的部署模式和 deployment policy 与 `sandbox/deployment_contract.go` 中的共享 field bound：installer 从 `deploy/node/node_spec.py` 读取，TypeScript 客户端和 Web 从 `packages/agents-client/src/deployment-contract.ts` 读取，因此 Web 读取这些声明，而不比较 provider kind。通过 `go run ./services/core/cmd/specification-contract -write` 重新生成两者。 6. 提供 adapter 和 helper 的发行产物，通过已注册 configuration 契约向运维人员提供 provider。 **已知设计缺口：** Web 仍在 setup 向导的后端选择、Docker 确认，以及所有显示或解析 E2B 配置字段的地方（带服务预设的 setup 步骤、部署摘要和客户端的部署投影）中指名 provider，因为协议尚未声明配置字段。通过该界面提供另一 provider 目前需要修改共享的 Web。此耦合不符合[复杂性留在 adapter 内](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#complexity-stays-in-the-adapter)；新集成必须通过协议表达配置，把厂商专有行为留在 adapter。不得添加 Session 或 Turn 调度路径、厂商专有 column 或 API field，或 store 中的厂商 switch。 @@ -128,7 +128,7 @@ node 配置 `sandbox.NodeConfig` 包含 `provider`、`generation`、`installatio - `nodes` 注册仅有 `BuildLocal`，`direct` 注册仅有 `BuildDirect`；缺失、混合或未知 mode 被拒绝。 - specification 和 resource validator、configuration adapter 与完整 operation 声明都是必需项，因此不完整注册不能发布部分投影。声明的默认大小必须通过 adapter 的 resource validator。 - Runtime input policy 要么接受固定 Runtime，要么给出 adapter 拒绝它的固定原因，不能两者兼有。 -- Checkpoint 仅准入 `nodes` 注册，因为公共 lifecycle 只暂停 node allocation；声明 checkpoint 的 `direct` Provider 会被注册拒绝。注册不携带 suspension 数值：Core 对每个声明 checkpoint 支持的 Provider 应用同一个 [suspension policy](#suspension)。 +- 完整生命周期声明支持时，`nodes` 与 `direct` 模式都准入暂停。注册不携带 suspension 数值；Core 对所有支持的 Provider 应用同一个 [暂停策略](#suspension)。 configuration adapter 必须非 nil，包括其具体值。每个 `ConfigurationRequirements` 字段都需要明确有效的决定：`Credential` 和 `PublicOrigin` 为 `Required` 或 `NotRequired`，`Discovery`、`SelectionDiscovery` 和 `CredentialVerification` 使用共享 supported 或 unsupported 声明并携带安全 reason。新增 requirement field 需要明确更新验证，不继承已有决定。要求凭据不承诺支持 `VerifyCredential` 操作。这些检查证明注册完整，不证明原生 SDK 行为正确；constructor 和 adapter 契约测试仍然适用。 @@ -138,7 +138,7 @@ configuration adapter 必须非 nil，包括其具体值。每个 `Configuration 具有凭据的 direct adapter 在替换 key 前验证全部保留 generation 与 allocation reference。公共 `sandbox.CallFence` 排除原生调用并等待 helper 完成，包括调用方已超时的调用；execution 调用已准备的 verification 和 fencing callback，不按厂商分支。 -厂商部署验证和 SDK setup 留在构造边界，构造不创建 Environment。node-local adapter 的 `sandbox.Built` 返回 provider、probe、installation identity、backend fingerprint 和 specification digest；helper 可能比调用方存活更久时，还返回 `Quiescent` 检查，generation 回收会等待它。factory 还返回 close 函数。`execution.RuntimeProvider` 将 adapter 绑定到 kind、installation ID、backend fingerprint、generation、mode 和 node ownership；选择由数据库负责，内存副本不构成另一权限来源。Docker 与 microsandbox 在 node 上运行，E2B 直接构造。node proxy 仅对注册声明支持 checkpoint 的 backend 暴露 checkpoint 操作，公共 lifecycle 通过 checkpoint 声明准入 suspension，不通过 provider name。 +厂商部署验证和 SDK setup 留在构造边界，构造不创建 Environment。node-local adapter 的 `sandbox.Built` 返回 provider、probe、installation identity、backend fingerprint 和 specification digest；helper 可能比调用方存活更久时，还返回 `Quiescent` 检查，generation 回收会等待它。factory 还返回 close 函数。`execution.RuntimeProvider` 将 adapter 绑定到 kind、installation ID、backend fingerprint、generation、mode 和 node ownership；选择由数据库负责，内存副本不构成另一权限来源。Docker 与 microsandbox 在 node 上运行，E2B 直接构造。node proxy 仅对注册声明支持 暂停的 backend 暴露暂停操作，公共 lifecycle 通过暂停声明准入 suspension，不通过 provider name。 backend fingerprint 标识原生资源命名空间，不表示容量。Core 保留部署 generation，使所属 allocation 继续解析到原 backend；不要将保留 allocation 重新指向替代 backend。 @@ -168,15 +168,15 @@ allocation、专用 daemon credential digest 和精确 Session binding 在 `Crea 配置 provider 后，Worker 扫描已提交且没有 allocation 的 pending hosted Environment，涵盖空闲 Session 创建以及 commit 与 bootstrap 之间中断后的恢复；已有 allocation 不重新进入此路径。scan 有界，由 lifecycle owner 串行化，不需要调用方操作。没有 Turn 的初始预约让 Session 保持空闲，daemon 连接不被当作原生 readiness。同一 scan 在验证精确 Session、device binding 和已结算 bootstrap 后，发布带持久 generation 的认证连接观测。 -Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 正在运行的 allocation；该检查不做任何修改，也不复活 cleanup 请求。正在运行的计算资源不会到期：显式删除和 snapshot retention 授权其清理。停止或缺失 container 不授权丢弃保留工作区或历史。禁用 provider 停止新 hosted admission 与 bootstrap，但不阻止现有 Session 的取消、function result 或 input retry outcome。 +Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 正在运行的 allocation；该检查不做任何修改，也不复活 cleanup 请求。正在运行的计算资源不会到期：显式删除和保留状态期限授权其清理。停止或缺失 container 不授权丢弃保留工作区或历史。禁用 provider 停止新 hosted admission 与 bootstrap，但不阻止现有 Session 的取消、function result 或 input retry outcome。 终结清理原子撤销 device authority、记录 Environment 失败或到期、结算 pending input 并请求取消，然后才调用 `Kill`；原 input deadline 与 retry outcome 保留。临时 provider outage、未知 Create result 和停止的计算资源不证明永久失败。公开 Session 删除后 Core 保留 allocation，仅在所属 compute 与 volume 清理完成且原 Create 已结算的证明成立后标记 released；未知创建即使观察到不存在也保留 cleanup ownership，有界 scan 继续捕捉延迟资源，不再调用 `Create`。 ### 每节点生命周期 worker {#per-node-lifecycle-workers} -每个注册 node 有一个串行 lifecycle worker，负责 gate、allocation 与 pending cursor、connection 和 wake hint；E2B allocation 共享一个没有 node 的串行 lifecycle。薄 coordinator 发现 node 并关闭 worker，数据库、provider 或等待操作期间不持有 map mutex。worker 独立推进，因此一个在线 node 的 provider 卡住不会阻塞其他 node：lifecycle 并发为每 node 一项操作，随 node 数量增长。离线 worker 保留，因此其保留资源在重连后仍可观察。 +托管 Environment 提交后，Core 会向其放置节点的生命周期 worker 发送有界提示。已提交的待处理输入也会发送提示，以恢复遗漏的创建通知。提示复用现有串行门控、执行租约、容量检查和一次性分配凭据，不会在准入处理器内直接创建沙箱。每个正常维护周期最多允许一次额外提示扫描；周期扫描负责恢复遗漏或合并的提示。已有分配的观察仍先于待创建资源处理，因此同一节点上的慢观察仍可能延迟新 Environment。 -托管 Environment 提交后，Core 向其 placement 的生命周期 worker 发送有界提示。已提交的待处理输入也会发送提示，以恢复遗漏的创建通知。提示沿用串行 gate、租约、容量检查和一次性 allocation 收据，不在准入处理器内直接创建资源。每个正常维护周期最多允许一次额外提示扫描；周期扫描恢复遗漏或合并的提示。现有 allocation 观察仍先于待创建资源扫描，因此同一节点上缓慢的观察可能延迟新 Environment。 +每个注册 node 有一个串行 lifecycle worker，负责 gate、allocation 与 pending cursor、connection 和 wake hint；E2B allocation 共享一个没有 node 的串行 lifecycle。薄 coordinator 发现 node 并关闭 worker，数据库、provider 或等待操作期间不持有 map mutex。worker 独立推进，因此一个在线 node 的 provider 卡住不会阻塞其他 node：lifecycle 并发为每 node 一项操作，随 node 数量增长。离线 worker 保留，因此其保留资源在重连后仍可观察。 allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan join 尚未释放的已提交 placement。每个 node 推进自己的 cursor，包括越过失败观测，并在末尾回绕一次。direct provisioning 在进入该 node gate 前解析 tenant 范围内 placement，已有 allocation 必须与其一致；Core 不选择另一 node。 @@ -186,15 +186,33 @@ allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan jo placement 自动完成：environment-to-node placement 与 Session 创建及其 retry identity 一起提交，调用方不能选择 node，重试即使 node 离线也保留原 node。Node capacity 计入 pending reservation 和未决资源，新 placement 与 suspended-to-restoring 转移共享数据库锁。未知操作保留预约，source teardown 必须确认后才能释放 active capacity，确认 cleanup 后释放 placement capacity。保留所有权需要精确 provider evidence：socket path、缺失 instance 或空列表都不证明 cleanup，也不授权 replacement。 -部署的 CPU、memory、disk 设置、`max_active`、`max_retained` 和 snapshot retention 限制每个 node。不提供 node-level drain、跨 node Session migration、multi-active Core、autoscaling 或 snapshot replication。node 持有 allocation、snapshot、reservation、unknown result 或 cleanup 时拒绝移除 node，离线 ownership 保留。 +部署的 CPU、memory、disk 设置、`max_active`、`max_retained` 和保留状态期限限制每个 node。不提供 node-level drain、跨 node Session migration、multi-active Core、autoscaling 或 snapshot replication。node 持有 allocation、保留状态、reservation、unknown result 或 cleanup 时拒绝移除 node，离线 ownership 保留。 ### 暂停 {#suspension} -Core 用同一个固定 policy 暂停每个声明 checkpoint 支持的 provider 的空闲工作：工作空闲 5 分钟（300 秒）后暂停，snapshot 保留 24 小时（86400 秒）。部署的 [`suspension`](../../contracts/agents-api/zh/sandbox-deployment.md#safe-response) 报告这两个值。Core 仅在至少一个 Turn 已终结、没有 root 或 Subagent Turn 排队、进行中或等待、没有 pending input、file operation 或 initialization，且真实 activity 已空闲达到该时间后暂停。对于 node allocation，Core 在同一事务中用数据库时钟记录首个 root 或 child terminal transition。candidate filter 和 Session-locked recheck 比较数据库已过时间与 idle duration，初始 snapshot retention deadline 也锚定同一数据库观测，因此 Core 与数据库主机时钟无需一致。原生 completion timestamp 在公开历史中保持不变，但不驱动 idle admission，heartbeat 不重置 activity。确认计划暂停前，daemon 关闭 admission 并排空 native cleanup、output receipt 和 file work。 +单一 `SandboxProvider` 一并声明 `Initial`、`NewCompute`、`GetCompute`、`RenewCompute`、`Suspend`、`Resume`、`KillCompute`、`DeleteRetained`、`RunCommandCompute` 和 `ResumeCompute`：十项全部支持或全部不支持。生命周期不通过可选暂停接口或 Provider 名称分支选择。 + +`RetainedState` 是 adapter 拥有的不透明信封，包含 `Reference`、`ID`、`Data`、`OperationID`、`SourceGeneration`、`SourceName` 和 `SourceID`。`Data` 必须非空且不超过 64 KiB。Core 原样保留信封并校验操作与源实例归属；不可变的 allocation Provider/代次绑定决定唯一有权解释它的 adapter。保留状态不必是独立快照：microsandbox 将完整原生快照身份放入 `Data`，E2B 保留带版本的原生暂停收据，不承诺独立磁盘镜像。adapter 在任何副作用前校验原生内容及归属。 + +`Compute.RestoredFrom`、`ComputeState.Retained`、`ResourcesReleased`、`SuspendSettled` 和恢复的 `ReconcileOnly` 是结算证据。资源缺失不证明未知暂停已结束。`RenewCompute` 只续租精确实例；不确定结果不授权第二次创建、捕获或恢复。持久 `runtime_compute` 使用 `protocol_version: "1"`；启动拒绝缺失或未知版本。仅匹配当前 retained-state 字段形状的版本 1 信封可直接读取，无需改写。旧 snapshot-only 形状即使标记相同的版本 1 也会被拒绝；数字相同不证明互通。升级不做迁移、合成重放或丢弃保留状态。 + +`ResumeRequest.Workspace` 携带 allocation 独立拥有的文件系统绑定；Core 在恢复前获取 ready 绑定。源实例必须已释放活跃执行能力，目标才可成为写者。adapter 在原生副作用前校验绑定；不支持独立文件系统的 adapter 拒绝非空绑定。`DeleteRetained` 仅删除 adapter 的计算保留状态，绝不删除 workspace。显式 Session 删除先完成 compute 清理，再删除独立存储，恢复期间保持单一活跃写者。 + +Core 对所有声明暂停支持的 Provider 应用统一固定策略：工作空闲 5 分钟（300 秒）后暂停，保留状态期限为 24 小时（86400 秒）；部署的 [`suspension`](../../contracts/agents-api/zh/sandbox-deployment.md#safe-response) 返回这些值。Core 暂停已经初始化的 Environment，包括尚未执行 Turn 的 Environment，前提是没有 root 或 Subagent Turn 排队、执行或等待，没有输入、文件操作或初始化待处理，且真实活动已空闲达到该时长。Core 在同一事务中使用数据库时钟记录 allocation 的初始化完成和 root 或 child 终结转换。候选筛选和持有 Session 锁的复查比较数据库经过时间与空闲时长，初始保留状态期限也锚定同一数据库观测，因此 Core 与数据库主机的时钟无需一致。公开历史中的原生完成时间戳保持不变，但不驱动空闲准入，心跳也不重置活动时间。确认计划暂停前，daemon 关闭准入并排空原生清理、输出收据和文件工作。 + +Worker 租约、Session 锁和每个 node 的 gate 负责所有 Provider 的暂停。新 Turn claim、文件写入意图和捕获准入在 Session 锁下串行化，共享计算阶段检查；新待处理工作取消捕获并唤醒同一源实例。正常准备在经过认证的恢复握手后等待计算阶段变为 running；待处理输入的提升与生命周期转换冲突时，输入保持 pending。计算阶段和经 revision 校验的收据存储在 allocation 中。Core 在副作用前持久化静止、捕获和恢复意图，仅新收据执行捕获或恢复；恢复流程观察精确尝试，不重试结果未知的创建、捕获或恢复。已消费的保留状态不会使运行中的代次回滚。删除、撤销和保留期到期始终优先于唤醒，直至最后的数据库 compare-and-swap；未知清理身份会一直保留，直到确认所属资源已不存在。已消费产物和旧计算实例会被删除，因此暂停循环不会累积可写磁盘链。 + +排队工作和实时 Environment 文件访问会唤醒暂停的 Environment；历史和已发布 Artifact 的读取不会唤醒。计划暂停在 daemon 连接上使用 Environment 与 suspension token。由 PID 和启动时间隔离的本地控制信号（`RunCommandCompute`）唤醒 parked daemon，daemon 在准入工作前重新认证。确认前的临时断连通过有界尝试和退避重试同一已准备好的暂停；永久认证或协议拒绝会将其关闭。Core 负责保留状态的到期期限，daemon 没有相应定时器。静止确认丢失时可以通过明确回滚解冻同一源实例，但不授权捕获。 + +`Suspend` 负责原生资源释放，返回绑定的保留句柄、suspended 状态、`ResourcesReleased` 和 `SuspendSettled` 后，Core 才释放活跃容量。`ReconcileOnly` 禁止重放原始捕获或暂停，但允许完成由持久保留产物证明安全的 adapter 清理。无保留状态的结果只有在带有 `SuspendSettled` 且源实例处于可恢复的运行或暂停状态时才允许回滚。其他所有不确定结果均保留所有权并关闭准入。Core 从不在 `Suspend` 后无条件销毁源实例。 + +`Resume` 将保留状态恰好消费一次并恢复到预先提交的目标。恢复逻辑观察同一次尝试。Core 持久化 `waking`、认证并恢复 daemon、删除已消费的保留资源，然后提交 `running` 并准入工作。`DeleteRetained` 是幂等产物清理，会保留运行中的计算资源。清理失败会保持 `waking` 阶段，不能触发再次恢复。`KillCompute` 仍然是破坏性操作；清理旧代次时不得终止共享原生 ID 的较新活跃实例。 + +现有 Session 锁、生命周期租约、空闲规则、容量查询和清理顺序继续作为权威。每个尚未释放的分配都占用 `max_retained`，包括运行中的分配。每个分配在预留时都写入共享计算协议版本，包括暂停阶段为 `disabled` 的分配。激活会拒绝任何协议版本缺失或不同的未释放分配；升级前必须由旧版本完成普通清理。Session 历史保留。 -Worker lease、Session lock 与 per-node gate 对每个 provider 负责 suspension。新 Turn claim、file-write intent 和 capture admission 在 Session lock 下串行化，共享一个 compute-phase 检查；新 pending work 取消 capture 并唤醒同一 source。正常 preparation 在经过认证的 resume handshake 后等待 compute phase 为 running；pending input 的 promotion 与 lifecycle transition 冲突时保持 pending。compute phase 和 revision-checked receipt 位于 allocation。Core 在 effect 前持久化 quiesce、capture 和 restore intent，仅新 receipt 执行 capture 或 restore，恢复观察精确 attempt，不重试未知 creation、capture 或 restore。已消费 snapshot 不让 running generation 回滚。删除、撤销和 retention expiry 优先于 wake，一直持续到最终数据库 compare-and-swap；未知 cleanup identity 保留，直到确认所属资源不存在。已消费 artifact 和旧 compute 被删除，因此暂停循环不累积可写磁盘链。 +### 协调协议升级 {#coordinated-protocol-upgrade} -排队工作和实时 Environment file access 唤醒 suspended Environment；history 和已发布 Artifact read 不唤醒。计划暂停在 daemon 连接上使用 Environment 和 suspension token。受 PID 与 start-time fencing 的本地 control signal（`RunCommandCompute`）唤醒 parked daemon，daemon 在准入工作前重新认证。确认前临时断连通过有界 attempt 和 backoff 重试同一已 armed suspension；永久认证或协议拒绝则关闭。Core 负责 snapshot retention deadline，daemon 没有相应 timer。quiesce 确认丢失时可以通过明确 rollback 解冻同一 source，但不授权 capture。 +Core 与 node 一同升级到 node wire version 7；E2B helper 使用 private wire version 4，并重建包含托管暂停控制的模板；microsandbox helper 使用 private wire version 5。所有 Provider 交付 Runtime bootstrap version 2。每个边界独立验证自身契约，不同边界的版本号不能互换。激活前，使用此前兼容的版本对不兼容的未释放 allocation 完成正常清理。启动栅栏保留其持久收据及 Session 历史，不通过 Provider 调用迁移它们。 ### 重置与归档 {#reset-and-archive} @@ -202,7 +220,7 @@ Worker lease、Session lock 与 per-node gate 对每个 provider 负责 suspensi 一个 snapshot 和 timestamp 对持有资源分区。离线 ownership 来自 allocation 或 active placement 的 node，与 online presence 使用同一 45 秒 connection 和 owner-epoch predicate，独立于 provider readiness；cleanup failure、offline state 或空 read 都不授权合成 release。held resource 为零时，Core 在数据库事务外 drain，在 deployment lock 下复查，并在一个事务中清空 deployment，再发布携带 generation 的空 provider，没有可失败工作。最终 write 或 drain 失败时，在释放 mutation gate 前使用有界 owner context 恢复已提交 provider；恢复失败时 admission 保持 fenced，owner 停止。 -管理员 [Session archive](../../contracts/agents-api/zh/admin-api.md#session-archive) 在 Session-first transaction 中保留 Project scope 与 hosted eligibility 检查，并一起处理 Environment expiry、cancellation、Runtime authority revocation 和 audit；reset 的后台 archive 从可信记录重建实际 Project scope，保留请求方 provenance。普通 provider lifecycle 释放 compute 和 snapshot。archive cancellation 在 terminal commit 前保留健康 receipt path：仅首次撤销 device 的 archive 记录精确 `archive_cancel_turn_id`（普通 revocation 清空它，重复 cleanup 保留它），现有已认证 delivery 可从 Turn 原 `cancel_requested_at` 起最多 20 秒排空该 cancellation。Core 独立于 subscription removal，通过 `done`、cancellation acknowledgement 和 terminal commit 跟踪 delivery，不授予新 connection、input、file 或 MCP authority，也不续期 lease。事务或 lifecycle gate 不等待 receipt，peer 丢失、到期或重启回到普通 failure 与 cleanup，不虚构 cancelled outcome。 +管理员 [Session archive](../../contracts/agents-api/zh/admin-api.md#session-archive) 在 Session-first transaction 中保留 Project scope 与 hosted eligibility 检查，并一起处理 Environment expiry、cancellation、Runtime authority revocation 和 audit；reset 的后台 archive 从可信记录重建实际 Project scope，保留请求方 provenance。普通 provider lifecycle 释放 compute 和保留状态。archive cancellation 在 terminal commit 前保留健康 receipt path：仅首次撤销 device 的 archive 记录精确 `archive_cancel_turn_id`（普通 revocation 清空它，重复 cleanup 保留它），现有已认证 delivery 可从 Turn 原 `cancel_requested_at` 起最多 20 秒排空该 cancellation。Core 独立于 subscription removal，通过 `done`、cancellation acknowledgement 和 terminal commit 跟踪 delivery，不授予新 connection、input、file 或 MCP authority，也不续期 lease。事务或 lifecycle gate 不等待 receipt，peer 丢失、到期或重启回到普通 failure 与 cleanup，不虚构 cancelled outcome。 ## 验证集成 {#validate-the-integration} diff --git a/internal/runtimebootstrap/bootstrap.go b/internal/runtimebootstrap/bootstrap.go index 1beabb40f..183f5b054 100644 --- a/internal/runtimebootstrap/bootstrap.go +++ b/internal/runtimebootstrap/bootstrap.go @@ -14,6 +14,9 @@ import ( "github.com/google/uuid" ) +// SuspendControlFile is the packaged private hosted Runtime park/wake location. +const SuspendControlFile = "/run/oac/daemon-suspend.json" + const Version = 2 const MaxBytes = 16 * 1024 diff --git a/services/core/cmd/server/managed_generation_operations.go b/services/core/cmd/server/managed_generation_operations.go index 2f7c99b65..aba23a97c 100644 --- a/services/core/cmd/server/managed_generation_operations.go +++ b/services/core/cmd/server/managed_generation_operations.go @@ -17,7 +17,7 @@ func (p *generationRouter) Initial(ctx context.Context, r sandbox.Reference) (sa defer done() return v.Initial(ctx, r) } -func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, snapshot *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, snapshot *sandbox.RetainedState) (sandbox.Compute, error) { if err := providercontract.Require(p, "NewCompute"); err != nil { return sandbox.Compute{}, err } @@ -72,8 +72,8 @@ func (p *generationRouter) KillCompute(ctx context.Context, r sandbox.Reference, defer done() return v.KillCompute(ctx, r, c) } -func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Reference, snapshot sandbox.SnapshotIdentity) error { - if err := providercontract.Require(p, "DeleteSnapshot"); err != nil { +func (p *generationRouter) DeleteRetained(ctx context.Context, r sandbox.Reference, snapshot sandbox.RetainedState) error { + if err := providercontract.Require(p, "DeleteRetained"); err != nil { return err } v, done, err := p.route(ctx, r) @@ -81,7 +81,7 @@ func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Referen return err } defer done() - return v.DeleteSnapshot(ctx, r, snapshot) + return v.DeleteRetained(ctx, r, snapshot) } func (p *generationRouter) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { if err := providercontract.Require(p, "RunCommandCompute"); err != nil { @@ -105,3 +105,15 @@ func (p *generationRouter) ResumeCompute(ctx context.Context, r sandbox.Referenc defer done() return v.ResumeCompute(ctx, r, c) } + +func (p *generationRouter) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + if err := providercontract.Require(p, "RenewCompute"); err != nil { + return sandbox.ComputeState{}, err + } + v, done, err := p.route(ctx, r) + if err != nil { + return sandbox.ComputeState{}, err + } + defer done() + return v.RenewCompute(ctx, r, c) +} diff --git a/services/core/cmd/server/managed_generations_test.go b/services/core/cmd/server/managed_generations_test.go index 4b1c57aac..341d27cd5 100644 --- a/services/core/cmd/server/managed_generations_test.go +++ b/services/core/cmd/server/managed_generations_test.go @@ -29,7 +29,7 @@ q=json.load(sys.stdin) with (pathlib.Path(q['Config']['StateDir'])/'requests').open('a') as f: f.write(json.dumps(q)+'\n') info=dict(q['Reference'],State='running',ProviderID='owned',CreateSettled=True) if q['Operation']=='kill': info['State']='absent' -print(json.dumps({'Version':2,'Info':info})) +print(json.dumps({'Version':q['Version'],'Info':info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -54,7 +54,7 @@ print(json.dumps({'Version':2,'Info':info})) value.Configuration = &key return value, nil } - setup := &managedSetup{processPaths: paths, registry: providers.Builtin(), deployment: &fakeDeploymentSetups{t: t, allocationSetup: allocation}, installationID: id} + setup := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), deployment: &fakeDeploymentSetups{t: t, allocationSetup: allocation}, installationID: id} // A facade retained by a generation-one lifecycle still reads current credentials. router := &generationRouter{setup: setup} ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) @@ -132,7 +132,7 @@ code = '' if k == 'revoked': code = 'unauthorized' elif k == 'unconfirmed': code = 'unconfirmed' elif not (k.startswith('team-a') and template in ('owned-a', 'new-a') or k == 'team-b' and template == 'public-b'): code = 'team_mismatch' -result = {'Version': 2, 'ErrorCode': code} +result = {'Version': q['Version'], 'ErrorCode': code} if not code: result['DeploymentValid'] = True if q['Operation'] == 'validate_deployment': @@ -151,7 +151,7 @@ print(json.dumps(result)) setups := &fakeDeploymentSetups{t: t, setup: committedSetup(&committed), withCredential: credentialService(t), generationPage: func(context.Context, int64) ([]deployment.Setup, error) { return nil, nil }} allocations := &fakeGenerationAllocations{t: t, credentialAllocations: func(context.Context, string) ([]deployment.Allocation, error) { return nil, nil }} - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: setups, allocations: allocations} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: setups, allocations: allocations} loaded, err := s.load(t.Context()) if err != nil { t.Fatal(err) diff --git a/services/core/cmd/server/managed_nodes.go b/services/core/cmd/server/managed_nodes.go index b908bbab6..78a5639f7 100644 --- a/services/core/cmd/server/managed_nodes.go +++ b/services/core/cmd/server/managed_nodes.go @@ -70,7 +70,7 @@ func configureManagedNodes(nodes *deployment.Service, reader deployment.Reader, return nodes.Heartbeat(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health)) }, }) - result.setup = &managedSetup{processPaths: config.ProviderPaths, registry: registry, deployment: nodes, allocations: reader, hub: result.hub, installationID: config.InstallationID, runtimeAPI: config.PublicOrigin.RuntimeAPI()} + result.setup = &managedSetup{capacity: config.SandboxCapacity, processPaths: config.ProviderPaths, registry: registry, deployment: nodes, allocations: reader, hub: result.hub, installationID: config.InstallationID, runtimeAPI: config.PublicOrigin.RuntimeAPI()} result.runtime = execution.NewDeferredRuntimeProvider(config.InstallationID, result.setup.load, result.setup.prepare) result.runtime.PublishUnconfigured = result.setup.publishUnconfigured return result diff --git a/services/core/cmd/server/managed_setup.go b/services/core/cmd/server/managed_setup.go index 758ff5db0..f913b508c 100644 --- a/services/core/cmd/server/managed_setup.go +++ b/services/core/cmd/server/managed_setup.go @@ -11,6 +11,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/processconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -22,6 +23,7 @@ import ( // observation. The database owns the selection; this cache is never a writer. type managedSetup struct { processPaths sandbox.ProcessPaths + capacity processconfig.SandboxLimits // registry builds the selected direct provider and discovers configuration; // the deployment setup reports what the registration declares. registry *providers.Registry @@ -159,7 +161,7 @@ func (s *managedSetup) configuration(setup deployment.Setup) (execution.Prepared Resources: setup.Specification.Resources, WorkspaceRequirements: adapter.Policy.Workspace, Workspace: setup.Specification.Workspace} if setup.Suspension != nil { selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.Suspension.IdleSeconds) * time.Second, - Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second} + Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second, MaxActive: s.capacity.MaxActive, MaxRetained: s.capacity.MaxRetained} } return execution.PreparedRuntimeDeployment{Config: selected, Publish: s.publish}, nil } diff --git a/services/core/cmd/server/managed_setup_preflight_test.go b/services/core/cmd/server/managed_setup_preflight_test.go index 884b89385..959fb49fd 100644 --- a/services/core/cmd/server/managed_setup_preflight_test.go +++ b/services/core/cmd/server/managed_setup_preflight_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -19,7 +20,7 @@ import ( func TestE2BRejectedSpecificationHasSafeActionableDiagnostic(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(e2b.ProtocolVersion)+",\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") @@ -28,7 +29,7 @@ func TestE2BRejectedSpecificationHasSafeActionableDiagnostic(t *testing.T) { } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id} selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 3, MemoryMiB: 3072}}, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} _, err := s.prepare(t.Context(), selection) if !errors.Is(err, sandbox.ErrConfigurationSelection) || strings.Contains(err.Error(), "synthetic-private-key") || s.selected.Load() != nil { @@ -49,12 +50,12 @@ op = q['Operation'] with (pathlib.Path(q['Config']['StateDir']) / 'operations').open('a') as log: log.write(op + '\n') if op == 'validate_deployment': - print(json.dumps({'Version': 2, 'ErrorCode': 'invalid'})) + print(json.dumps({'Version': q['Version'], 'ErrorCode': 'invalid'})) else: info = dict(q['Reference'], ProviderID='owned-compute', State='stopped', CreateSettled=True) if op == 'kill': info.update(ProviderID='', State='absent') - print(json.dumps({'Version': 2, 'Info': info})) + print(json.dumps({'Version': q['Version'], 'Info': info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -69,7 +70,7 @@ else: Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} allocation := func(context.Context, sandbox.Reference) (deployment.Setup, error) { return selection, nil } - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: committedSetup(&selection), allocationSetup: allocation}} if _, err := s.prepare(t.Context(), selection); err == nil || s.selected.Load() != nil { t.Fatal("invalid new template selection was published", err) @@ -101,13 +102,13 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { } helper := filepath.Join(t.TempDir(), "provider") requests := filepath.Join(state, "requests") - script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" + script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":" + strconv.Itoa(e2b.ProtocolVersion) + ",\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} candidate, err := s.prepare(t.Context(), selection) disk := int32(24063) @@ -133,7 +134,7 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(e2b.ProtocolVersion)+",\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") @@ -142,7 +143,7 @@ func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-team-a", Template: "public-team-b:" + uuid.NewString()}} if _, err := s.prepare(t.Context(), selection); !errors.Is(err, sandbox.ErrCredentialOwnership) || s.selected.Load() != nil { t.Fatal("public readability accepted as team ownership", err) @@ -150,7 +151,7 @@ func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { } func TestManagedSetupRoutesProviderWithoutCredentialRequirement(t *testing.T) { - s := &managedSetup{} + s := &managedSetup{capacity: testSandboxCapacity(t)} config := &execution.RuntimeProvider{ProviderKind: "docker"} candidate, err := s.routeGenerations( execution.PreparedRuntimeDeployment{Config: config}, diff --git a/services/core/cmd/server/managed_setup_test.go b/services/core/cmd/server/managed_setup_test.go index aaaf9cd02..d2f95a334 100644 --- a/services/core/cmd/server/managed_setup_test.go +++ b/services/core/cmd/server/managed_setup_test.go @@ -122,7 +122,7 @@ func TestManagedSetupNeverReusesAnotherGenerationOrUnverifiedState(t *testing.T) value := deployment.Setup{InstallationID: "installation", Provider: "docker", Mode: "nodes", Generation: 1} var loadErr error setups := &fakeDeploymentSetups{t: t, setup: func(context.Context) (deployment.Setup, error) { return value, loadErr }} - s := &managedSetup{registry: providers.Builtin(), deployment: setups, allocations: &fakeGenerationAllocations{t: t}, installationID: "installation"} + s := &managedSetup{capacity: testSandboxCapacity(t), registry: providers.Builtin(), deployment: setups, allocations: &fakeGenerationAllocations{t: t}, installationID: "installation"} cached := &execution.RuntimeProvider{InstallationID: "installation", ProviderKind: "docker", Generation: 1} s.publish(cached) if got, err := s.load(t.Context()); err != nil || got != cached { @@ -153,7 +153,7 @@ func TestMissingE2BHelperReportsProviderUnavailable(t *testing.T) { id := uuid.NewString() committed := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-key", Template: "runtime:" + uuid.NewString()}} - s := &managedSetup{processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id, + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: committedSetup(&committed)}} if _, err := s.load(t.Context()); !errors.Is(err, execution.ErrExecutionUnavailable) { t.Fatal("missing local helper must leave administrative recovery available", err) @@ -164,10 +164,12 @@ func TestMissingE2BHelperReportsProviderUnavailable(t *testing.T) { } func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "7") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "31") id := uuid.NewString() hub := node.NewHub(node.HubOptions{}) defer hub.Close() - s := &managedSetup{registry: providers.Builtin(), installationID: id, hub: hub, deployment: &fakeDeploymentSetups{t: t}, allocations: &fakeGenerationAllocations{t: t}, runtimeAPI: "https://core.example/api/v1"} + s := &managedSetup{capacity: testSandboxCapacity(t), registry: providers.Builtin(), installationID: id, hub: hub, deployment: &fakeDeploymentSetups{t: t}, allocations: &fakeGenerationAllocations{t: t}, runtimeAPI: "https://core.example/api/v1"} previous := &execution.RuntimeProvider{InstallationID: id, Generation: 1, ProviderKind: "docker"} s.publish(previous) candidate, err := s.prepare(t.Context(), deployment.Setup{InstallationID: id, Provider: "microsandbox", Mode: "nodes", Operations: microsandbox.Operations(), Suspension: &deployment.Suspension{IdleSeconds: 300, RetentionSeconds: 86400}}) @@ -177,6 +179,9 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { if s.selected.Load().Config != previous || candidate.Config.ProviderKind != "microsandbox" || candidate.Config.Suspension == nil || candidate.Config.CoreURL != "https://core.example/api/v1" { t.Fatal("preparation published or lost candidate configuration") } + if candidate.Config.Suspension.MaxActive != 7 || candidate.Config.Suspension.MaxRetained != 31 { + t.Fatal("configured capacity was not propagated") + } committed := *candidate.Config committed.Generation = 2 candidate.Publish(&committed) @@ -187,7 +192,7 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { func TestManagedSetupRejectedCandidateRetainsSelection(t *testing.T) { id := uuid.NewString() - s := &managedSetup{processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id} previous := &execution.RuntimeProvider{InstallationID: id, Generation: 1, ProviderKind: "docker"} s.publish(previous) _, err := s.prepare(t.Context(), deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, @@ -210,7 +215,7 @@ func TestManagedSetupResetTombstoneRejectsDelayedProviderLoad(t *testing.T) { return deployment.Setup{}, ctx.Err() } } - s := &managedSetup{registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: delayed}} + s := &managedSetup{capacity: testSandboxCapacity(t), registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: delayed}} done := make(chan error, 1) go func() { provider, err := s.load(t.Context()) diff --git a/services/core/cmd/server/sandbox_capacity_test.go b/services/core/cmd/server/sandbox_capacity_test.go new file mode 100644 index 000000000..46ad1cc0d --- /dev/null +++ b/services/core/cmd/server/sandbox_capacity_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/processconfig" +) + +func testSandboxCapacity(t testing.TB) processconfig.SandboxLimits { + t.Helper() + c, err := processconfig.SandboxCapacity() + if err != nil { + t.Fatal(err) + } + return c +} diff --git a/services/core/deploy/e2b/build-template.py b/services/core/deploy/e2b/build-template.py index d78926213..30616a94d 100644 --- a/services/core/deploy/e2b/build-template.py +++ b/services/core/deploy/e2b/build-template.py @@ -10,6 +10,7 @@ import tempfile from e2b import Template +from helper_contract_generated import SUSPEND_CONTROL_FILE BASE = 'node:22.23.1-bookworm-slim@sha256:8607a9064d4a571140998ae9e52a3b3fcf9cff361d04642d5971e6cd76d39e27' parser = argparse.ArgumentParser() @@ -27,6 +28,7 @@ if value.startswith(('HOME=', 'OAC_'))) if environment.get('OAC_RUNTIME_WORKSPACE') != '/environment/workspace': parser.error('Image does not use the colocated Runtime layout') +environment['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'] = SUSPEND_CONTROL_FILE dev_home = Path(os.environ.get('OAC_DEV_HOME') or Path.home() / '.oac') if not dev_home.is_absolute(): parser.error('OAC_DEV_HOME must be absolute') diff --git a/services/core/deploy/e2b/build_template_test.py b/services/core/deploy/e2b/build_template_test.py index 9a3b3034f..3e4f90508 100644 --- a/services/core/deploy/e2b/build_template_test.py +++ b/services/core/deploy/e2b/build_template_test.py @@ -64,6 +64,9 @@ def build(instance, **kwargs): self.assertIs(instance, template) context = Path(factory.call_args.kwargs['file_context_path']) self.assertEqual(stat.S_IMODE(context.stat().st_mode), 0o700) + from helper_contract_generated import SUSPEND_CONTROL_FILE + runtime_env = json.loads((context / 'runtime-env.json').read_text()) + self.assertEqual(runtime_env['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'], SUSPEND_CONTROL_FILE) with tarfile.open(context / 'runtime.tar.gz') as archive: modes = {m.name: stat.S_IMODE(m.mode) for m in archive.getmembers()} for parent in ('usr', 'usr/local', 'etc'): diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index 0b7635c78..7f420e0aa 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -1,5 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" +COMPUTE_FIELDS = ["Generation","Name","ID","RestoredFrom"] ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] @@ -10,9 +11,13 @@ MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 2 +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute_info","compute_renew","suspend","resume","compute_kill","delete_retained","compute_command","resume_compute"] +PROTOCOL_VERSION = 4 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Retained","Deadline"] +RESPONSE_FIELDS = ["Version","State","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +RESUME_FIELDS = ["Workspace","Reference","OperationID","Retained","Target","ReconcileOnly"] +RETAINED_FIELDS = ["Reference","ID","Data","OperationID","SourceGeneration","SourceName","SourceID"] SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" +SUSPEND_FIELDS = ["Reference","OperationID","Source","Retained","ReconcileOnly"] diff --git a/services/core/deploy/e2b/init.py b/services/core/deploy/e2b/init.py index 75d8e1f07..fa8171c29 100644 --- a/services/core/deploy/e2b/init.py +++ b/services/core/deploy/e2b/init.py @@ -106,6 +106,8 @@ def initialize(): # Claim before any side effect. An interrupted attempt must never start twice. write_private(ROOT / 'launch.json', identity) environment = prepare_runtime() + # Application-owned startup does not participate in Core-managed suspension. + environment.pop('OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE', None) credential = PROFILE.parent / 'executor-key.json' write_private(credential, payload['executor_key'], owner=1000) source.unlink() diff --git a/services/core/deploy/e2b/init_test.py b/services/core/deploy/e2b/init_test.py index 3c416c13f..31a1489d3 100644 --- a/services/core/deploy/e2b/init_test.py +++ b/services/core/deploy/e2b/init_test.py @@ -37,7 +37,8 @@ def test_launch_handoff_is_private_and_cannot_replay(self): profile = Path(temporary) / 'private/default' environment_file = Path(temporary) / 'image.json' environment_file.write_text(json.dumps({'HOME': '/home/runtime', - 'OAC_RUNTIME_HOME': '/home/runtime/.oac', 'OAC_RUNTIME_WORKSPACE': '/environment/workspace'})) + 'OAC_RUNTIME_HOME': '/home/runtime/.oac', 'OAC_RUNTIME_WORKSPACE': '/environment/workspace', + 'OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE': '/private/control/fixture.json'})) (root / 'bootstrap.json').write_text(json.dumps(PAYLOAD)) real_chmod = Path.chmod @@ -57,6 +58,7 @@ def chmod(path, mode): self.assertEqual(argv[argv.index('--remote') + 1], PAYLOAD['remote_url']) self.assertNotIn('test-private-key', repr(popen.call_args)) self.assertNotIn('OAC_RUNTIME_SESSION_ID', options['env']) + self.assertNotIn('OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE', options['env']) self.assertEqual((options['user'], options['group'], options['extra_groups']), (1000, 1000, [])) self.assertEqual(options['umask'], 0o077) key = profile.parent / 'executor-key.json' diff --git a/services/core/deploy/e2b/managed_init.py b/services/core/deploy/e2b/managed_init.py index 588f69d80..8c004102e 100644 --- a/services/core/deploy/e2b/managed_init.py +++ b/services/core/deploy/e2b/managed_init.py @@ -49,6 +49,13 @@ def initialize(): binding = identity(payload) shared.write_private(root / 'managed-launch.json', binding) environment = shared.prepare_runtime() + control_file = Path(environment['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE']) + if not control_file.is_absolute() or '..' in control_file.parts: + raise ValueError('Absolute private suspend control file required') + control_directory = control_file.parent + control_directory.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chown(control_directory, 1000, 1000) + control_directory.chmod(0o700) environment.update(OAC_RUNTIME_ENVIRONMENT_ID=payload['EnvironmentID'], OAC_RUNTIME_SESSION_ID=payload['SessionID'], OAC_RUNTIME_NETWORK_ACCESS=payload['NetworkAccess'], diff --git a/services/core/deploy/e2b/managed_init_test.py b/services/core/deploy/e2b/managed_init_test.py index f38a92ffd..c6d6a957e 100644 --- a/services/core/deploy/e2b/managed_init_test.py +++ b/services/core/deploy/e2b/managed_init_test.py @@ -63,9 +63,11 @@ def exercise(self, failed=False): if failed: process.side_effect = RuntimeError('private process diagnostic') image_env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'OAC_RUNTIME_HOME': str(Path(temporary) / '.oac'), - 'OAC_RUNTIME_WORKSPACE': '/environment/workspace'} + 'OAC_RUNTIME_WORKSPACE': '/environment/workspace', + 'OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE': str(Path(temporary) / 'control' / 'custom-suspend.json')} with patch.object(managed_init.shared, 'ROOT', root), patch.object(managed_init.shared, 'PROFILE', profile), \ patch.object(managed_init.shared, 'prepare_runtime', return_value=image_env), \ + patch.object(managed_init.os, 'chown') as chown, \ patch.object(managed_init.os, 'fchown'), patch.object(managed_init.subprocess, 'Popen', process): if failed: with self.assertRaises(RuntimeError): @@ -83,6 +85,10 @@ def exercise(self, failed=False): self.assertNotIn(data['RuntimeBootstrap']['credential'], json.dumps(process.call_args.kwargs['env'])) self.assertEqual(process.call_args.kwargs['env']['OAC_RUNTIME_ENVIRONMENT_ID'], data['EnvironmentID']) self.assertEqual(process.call_args.kwargs['user'], 1000) + control = Path(temporary) / 'control' + self.assertEqual(control.stat().st_mode & 0o777, 0o700) + chown.assert_called_once_with(control, 1000, 1000) + self.assertEqual(process.call_args.kwargs['env']['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'], str(control / 'custom-suspend.json')) if failed: self.assertFalse((root / 'managed-ready.json').exists()) else: diff --git a/services/core/internal/db/queries/runtime_allocations.sql b/services/core/internal/db/queries/runtime_allocations.sql index 6902e54ab..4eeccef61 100644 --- a/services/core/internal/db/queries/runtime_allocations.sql +++ b/services/core/internal/db/queries/runtime_allocations.sql @@ -1,6 +1,6 @@ -- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) -VALUES ($1, $2, $3, $4, $5, $6) RETURNING *; +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, compute_state) +VALUES ($1, $2, $3, $4, $5, $6, jsonb_build_object('protocol_version', sqlc.arg(protocol_version)::text)) RETURNING *; -- name: GetRuntimeAllocation :one SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index 83b5a38ca..379687964 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -25,15 +25,12 @@ WHERE id = $1 AND compute_phase = 'running' AND compute_activity_at <= $2; -- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, - CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, - CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = e.session_id AND r.state = 'pending') OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = e.id AND f.state = 'pending'))::boolean AS busy, - a.compute_wake_requested, - EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.completed_at IS NOT NULL)::boolean AS has_completed_turn + a.compute_wake_requested FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE a.id = $1; @@ -47,10 +44,28 @@ SELECT EXISTS ( UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e WHERE a.environment_id = e.id AND e.session_id = $1 - AND a.node_id IS NOT NULL AND a.state = 'running'; + AND a.state = 'running'; -- name: SessionHasRuntimeNode :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE e.session_id = $1 AND a.node_id IS NOT NULL )::boolean; + +-- name: HasIncompatibleRuntimeComputeState :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations + WHERE state <> 'released' + AND ((compute_state->>'protocol_version') IS DISTINCT FROM sqlc.arg(protocol_version)::text + OR compute_state ? 'snapshot' + OR (compute_state #> '{current,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot'] + OR (compute_state #> '{target,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot']) +)::boolean; + +-- name: CountRuntimeComputeReservations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended'; + +-- name: CountRuntimeRetainedAllocations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released'; diff --git a/services/core/internal/db/sqlc/runtime_allocations.sql.go b/services/core/internal/db/sqlc/runtime_allocations.sql.go index 4725ab1a1..2eed749c1 100644 --- a/services/core/internal/db/sqlc/runtime_allocations.sql.go +++ b/services/core/internal/db/sqlc/runtime_allocations.sql.go @@ -12,8 +12,8 @@ import ( ) const createRuntimeAllocation = `-- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) -VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, compute_state) +VALUES ($1, $2, $3, $4, $5, $6, jsonb_build_object('protocol_version', $7::text)) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` type CreateRuntimeAllocationParams struct { @@ -23,6 +23,7 @@ type CreateRuntimeAllocationParams struct { ProviderKey pgtype.UUID `json:"provider_key"` NodeID pgtype.UUID `json:"node_id"` DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` + ProtocolVersion string `json:"protocol_version"` } func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntimeAllocationParams) (RuntimeAllocation, error) { @@ -33,6 +34,7 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime arg.ProviderKey, arg.NodeID, arg.DeploymentGeneration, + arg.ProtocolVersion, ) var i RuntimeAllocation err := row.Scan( diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index 8b01bf155..f0ac40ce3 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -27,18 +27,39 @@ func (q *Queries) ClearRuntimeWake(ctx context.Context, arg ClearRuntimeWakePara return err } +const countRuntimeComputeReservations = `-- name: CountRuntimeComputeReservations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended' +` + +func (q *Queries) CountRuntimeComputeReservations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { + row := q.db.QueryRow(ctx, countRuntimeComputeReservations, providerKey) + var count int64 + err := row.Scan(&count) + return count, err +} + +const countRuntimeRetainedAllocations = `-- name: CountRuntimeRetainedAllocations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' +` + +func (q *Queries) CountRuntimeRetainedAllocations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { + row := q.db.QueryRow(ctx, countRuntimeRetainedAllocations, providerKey) + var count int64 + err := row.Scan(&count) + return count, err +} + const getRuntimeActivity = `-- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, - CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, - CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = e.session_id AND r.state = 'pending') OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = e.id AND f.state = 'pending'))::boolean AS busy, - a.compute_wake_requested, - EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.completed_at IS NOT NULL)::boolean AS has_completed_turn + a.compute_wake_requested FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE a.id = $1 ` @@ -48,7 +69,6 @@ type GetRuntimeActivityRow struct { LastActivity pgtype.Timestamptz `json:"last_activity"` Busy bool `json:"busy"` ComputeWakeRequested bool `json:"compute_wake_requested"` - HasCompletedTurn bool `json:"has_completed_turn"` } func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRuntimeActivityRow, error) { @@ -59,16 +79,33 @@ func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRu &i.LastActivity, &i.Busy, &i.ComputeWakeRequested, - &i.HasCompletedTurn, ) return i, err } +const hasIncompatibleRuntimeComputeState = `-- name: HasIncompatibleRuntimeComputeState :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations + WHERE state <> 'released' + AND ((compute_state->>'protocol_version') IS DISTINCT FROM $1::text + OR compute_state ? 'snapshot' + OR (compute_state #> '{current,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot'] + OR (compute_state #> '{target,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot']) +)::boolean +` + +func (q *Queries) HasIncompatibleRuntimeComputeState(ctx context.Context, protocolVersion string) (bool, error) { + row := q.db.QueryRow(ctx, hasIncompatibleRuntimeComputeState, protocolVersion) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + const recordRuntimeTerminalActivity = `-- name: RecordRuntimeTerminalActivity :exec UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e WHERE a.environment_id = e.id AND e.session_id = $1 - AND a.node_id IS NOT NULL AND a.state = 'running' + AND a.state = 'running' ` func (q *Queries) RecordRuntimeTerminalActivity(ctx context.Context, sessionID pgtype.UUID) error { diff --git a/services/core/internal/deployment/activity_test.go b/services/core/internal/deployment/activity_test.go index 2f308dfb2..a1b5d0c02 100644 --- a/services/core/internal/deployment/activity_test.go +++ b/services/core/internal/deployment/activity_test.go @@ -12,23 +12,22 @@ func TestRuntimeIdleAdmissionUsesDatabaseClock(t *testing.T) { t.Run(skew.String(), func(t *testing.T) { observed := coreNow.Add(skew) for _, test := range []struct { - name string - age time.Duration - busy, wake, completed bool - timeout time.Duration - want bool + name string + age time.Duration + busy, wake bool + timeout time.Duration + want bool }{ - {"recent", timeout - time.Second, false, false, true, timeout, false}, - {"boundary", timeout, false, false, true, timeout, true}, - {"elapsed", timeout + time.Second, false, false, true, timeout, true}, - {"busy", timeout + time.Second, true, false, true, timeout, false}, - {"wake", timeout + time.Second, false, true, true, timeout, false}, - {"never_completed", timeout + time.Second, false, false, false, timeout, false}, - {"missing_policy", timeout + time.Second, false, false, true, 0, false}, - {"invalid_policy", timeout + time.Second, false, false, true, -time.Second, false}, + {"recent", timeout - time.Second, false, false, timeout, false}, + {"boundary", timeout, false, false, timeout, true}, + {"elapsed", timeout + time.Second, false, false, timeout, true}, + {"busy", timeout + time.Second, true, false, timeout, false}, + {"wake", timeout + time.Second, false, true, timeout, false}, + {"missing_policy", timeout + time.Second, false, false, 0, false}, + {"invalid_policy", timeout + time.Second, false, false, -time.Second, false}, } { t.Run(test.name, func(t *testing.T) { - activity := Activity{ObservedAt: observed, LastActivity: observed.Add(-test.age), Busy: test.busy, WakeRequested: test.wake, HasCompletedTurn: test.completed} + activity := Activity{ObservedAt: observed, LastActivity: observed.Add(-test.age), Busy: test.busy, WakeRequested: test.wake} if got := activity.ReadyToSuspend(test.timeout); got != test.want { t.Fatalf("suspension admission = %v, want %v with database clock skew %s", got, test.want, skew) } diff --git a/services/core/internal/deployment/allocation.go b/services/core/internal/deployment/allocation.go index c51f2527b..820c398f2 100644 --- a/services/core/internal/deployment/allocation.go +++ b/services/core/internal/deployment/allocation.go @@ -76,13 +76,13 @@ type SessionDevice struct{ ID, EnvironmentID string } // Activity separates real work from the connection keepalive. ObservedAt is // the database clock when it was read. type Activity struct { - LastActivity, ObservedAt time.Time - Busy, WakeRequested, HasCompletedTurn bool + LastActivity, ObservedAt time.Time + Busy, WakeRequested bool } // ReadyToSuspend evaluates the idle policy on one database-clock observation. func (a Activity) ReadyToSuspend(idleTimeout time.Duration) bool { - return idleTimeout > 0 && a.HasCompletedTurn && !a.Busy && !a.WakeRequested && + return idleTimeout > 0 && !a.Busy && !a.WakeRequested && a.ObservedAt.Sub(a.LastActivity) >= idleTimeout } diff --git a/services/core/internal/deployment/execution.go b/services/core/internal/deployment/execution.go index 6af321418..e2541d406 100644 --- a/services/core/internal/deployment/execution.go +++ b/services/core/internal/deployment/execution.go @@ -289,3 +289,17 @@ func (e *ExecutionOperations) recordMetadata(tx DeploymentTx, input sandbox.Sele } return tx.RecordConfigurationMetadata(record.Metadata) } + +// CheckRuntimeComputeProtocol refuses incompatible allocation receipts before activation. +func (e *ExecutionOperations) CheckRuntimeComputeProtocol(ctx context.Context, version string) error { + return e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { + incompatible, err := tx.HasIncompatibleComputeState(version) + if err != nil { + return err + } + if incompatible { + return errors.New("incompatible retained runtime state: use the previous release to archive allocations before upgrading; history is preserved") + } + return nil + }) +} diff --git a/services/core/internal/deployment/fakes_test.go b/services/core/internal/deployment/fakes_test.go index 3ccdb1d13..0317764a1 100644 --- a/services/core/internal/deployment/fakes_test.go +++ b/services/core/internal/deployment/fakes_test.go @@ -76,27 +76,29 @@ func (f *fakeExecutionStorage) WithDeployment(ctx context.Context, apply func(De } type fakeReader struct { - t testing.TB - deployment func(context.Context) (Record, error) - snapshot func(context.Context) (Snapshot, error) - ownerEpoch func(context.Context) (uint64, error) - allocation func(context.Context, sandbox.Reference) (AllocationRecord, error) - generations func(context.Context, int64) ([]GenerationRecord, error) - nodes func(context.Context) ([]NodeRecord, error) - nodeHistory func(context.Context, string, coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) - readNodes func(context.Context, func(NodeReads) error) error - resetSessions func(context.Context, string, bool) ([]ResetSession, error) - addressBindings func(context.Context, string) (AddressBindings, error) - environmentAllocation func(context.Context, AllocationKey) (Allocation, error) - credentialAllocations func(context.Context, string) ([]Allocation, error) - observationSessions func(context.Context, string, int) (ObservationSessionPage, error) - nodeAllocations func(context.Context, string) ([]NodeAllocation, error) - nodeOnline func(context.Context, string) (bool, error) - lifecycleNodes func(context.Context) ([]string, error) - lifecycleAllocations func(context.Context, string, string) ([]Allocation, error) - unallocatedEnvironments func(context.Context, string, string) ([]UnallocatedEnvironment, error) - lifecyclePlacement func(context.Context, AllocationKey) (LifecyclePlacement, error) - activity func(context.Context, string) (Activity, error) + countRetainedAllocations func(context.Context, string) (int64, error) + countComputeReservations func(context.Context, string) (int64, error) + t testing.TB + deployment func(context.Context) (Record, error) + snapshot func(context.Context) (Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (AllocationRecord, error) + generations func(context.Context, int64) ([]GenerationRecord, error) + nodes func(context.Context) ([]NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) + readNodes func(context.Context, func(NodeReads) error) error + resetSessions func(context.Context, string, bool) ([]ResetSession, error) + addressBindings func(context.Context, string) (AddressBindings, error) + environmentAllocation func(context.Context, AllocationKey) (Allocation, error) + credentialAllocations func(context.Context, string) ([]Allocation, error) + observationSessions func(context.Context, string, int) (ObservationSessionPage, error) + nodeAllocations func(context.Context, string) ([]NodeAllocation, error) + nodeOnline func(context.Context, string) (bool, error) + lifecycleNodes func(context.Context) ([]string, error) + lifecycleAllocations func(context.Context, string, string) ([]Allocation, error) + unallocatedEnvironments func(context.Context, string, string) ([]UnallocatedEnvironment, error) + lifecyclePlacement func(context.Context, AllocationKey) (LifecyclePlacement, error) + activity func(context.Context, string) (Activity, error) } func (f *fakeReader) Deployment(ctx context.Context) (Record, error) { @@ -346,6 +348,7 @@ func (f *fakeNodeTx) RefreshServingReadiness(nodeID string, protocol int) error } type fakeDeploymentTx struct { + hasIncompatibleComputeState func(string) (bool, error) t testing.TB loadDeployment func() (Record, error) loadSnapshot func() (Snapshot, error) @@ -637,3 +640,25 @@ func (f *fakeSessionReader) GetManagedSessionArchive(context.Context, string, st unexpected(f.t, "GetManagedSessionArchive") return sessions.ManagedArchive{}, nil } + +func (f *fakeDeploymentTx) HasIncompatibleComputeState(version string) (bool, error) { + if f.hasIncompatibleComputeState == nil { + f.t.Fatal("unexpected HasIncompatibleComputeState") + return false, nil + } + return f.hasIncompatibleComputeState(version) +} + +func (f *fakeReader) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { + if f.countComputeReservations == nil { + unexpected(f.t, "CountComputeReservations") + } + return f.countComputeReservations(ctx, installationID) +} + +func (f *fakeReader) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { + if f.countRetainedAllocations == nil { + unexpected(f.t, "CountRetainedAllocations") + } + return f.countRetainedAllocations(ctx, installationID) +} diff --git a/services/core/internal/deployment/service.go b/services/core/internal/deployment/service.go index 022a0d359..582b67960 100644 --- a/services/core/internal/deployment/service.go +++ b/services/core/internal/deployment/service.go @@ -253,7 +253,7 @@ func (s *Service) SetupForSelection(installationID string, input sandbox.Selecti // suspension returns Core's idle suspension policy for a provider that // declares checkpoint support, and nil for any other provider. func (s *Service) suspension(provider string) (*Suspension, error) { - checkpoint, err := s.registry.SupportsCheckpoint(provider) + checkpoint, err := s.registry.SupportsSuspension(provider) if err != nil || !checkpoint { return nil, err } diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index 2eb526b14..2364910bb 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -247,6 +247,12 @@ type Reader interface { // Activity returns the allocation's activity; a missing allocation is // ErrNotFound. Activity(ctx context.Context, allocationID string) (Activity, error) + // CountComputeReservations counts the installation's allocations that + // reserve active compute. + CountComputeReservations(ctx context.Context, installationID string) (int64, error) + // CountRetainedAllocations counts the installation's allocations that + // retain resources. + CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) } // NodeReads loads node authentication facts. @@ -292,6 +298,8 @@ type NodeTx interface { // DeploymentTx is one leased deployment change. type DeploymentTx interface { + // HasIncompatibleComputeState checks all unreleased allocation protocol receipts. + HasIncompatibleComputeState(version string) (bool, error) LoadDeployment() (Record, error) LoadSnapshot() (Snapshot, error) CountResources() (Resources, error) diff --git a/services/core/internal/execution/archive_cancellation_cleanup_test.go b/services/core/internal/execution/archive_cancellation_cleanup_test.go index 3ac684477..0823c6cc0 100644 --- a/services/core/internal/execution/archive_cancellation_cleanup_test.go +++ b/services/core/internal/execution/archive_cancellation_cleanup_test.go @@ -107,7 +107,7 @@ func TestArchiveWaitingCleanupReceiptBarrier(t *testing.T) { } currentCompute := sandbox.Compute{ID: uuid.NewString(), Name: owner.ID + "-g0"} if checkpoint { - state, _ := json.Marshal(runtimeCompute{Current: currentCompute}) + state, _ := json.Marshal(runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: currentCompute}) owner, err = leased.Deployment.SetCompute(t.Context(), owner, "running", state, nil, 0) if err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/deployment_fixture_test.go b/services/core/internal/execution/deployment_fixture_test.go index 1cb8c27b5..67716db1b 100644 --- a/services/core/internal/execution/deployment_fixture_test.go +++ b/services/core/internal/execution/deployment_fixture_test.go @@ -140,27 +140,29 @@ func (s *strictExecutionStorage) WithDeployment(ctx context.Context, apply func( // strictDeploymentReader runs each set func; any other call fails the test. type strictDeploymentReader struct { - t *testing.T - deployment func(context.Context) (deployment.Record, error) - snapshot func(context.Context) (deployment.Snapshot, error) - ownerEpoch func(context.Context) (uint64, error) - allocation func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) - generations func(context.Context, int64) ([]deployment.GenerationRecord, error) - nodes func(context.Context) ([]deployment.NodeRecord, error) - nodeHistory func(context.Context, string, coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) - readNodes func(context.Context, func(deployment.NodeReads) error) error - resetSessions func(context.Context, string, bool) ([]deployment.ResetSession, error) - addressBindings func(context.Context, string) (deployment.AddressBindings, error) - environmentAllocation func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) - credentialAllocations func(context.Context, string) ([]deployment.Allocation, error) - observationSessions func(context.Context, string, int) (deployment.ObservationSessionPage, error) - nodeAllocations func(context.Context, string) ([]deployment.NodeAllocation, error) - nodeOnline func(context.Context, string) (bool, error) - lifecycleNodes func(context.Context) ([]string, error) - lifecycleAllocations func(context.Context, string, string) ([]deployment.Allocation, error) - unallocatedEnvironments func(context.Context, string, string) ([]deployment.UnallocatedEnvironment, error) - lifecyclePlacement func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) - activity func(context.Context, string) (deployment.Activity, error) + countRetainedAllocations func(context.Context, string) (int64, error) + countComputeReservations func(context.Context, string) (int64, error) + t *testing.T + deployment func(context.Context) (deployment.Record, error) + snapshot func(context.Context) (deployment.Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) + generations func(context.Context, int64) ([]deployment.GenerationRecord, error) + nodes func(context.Context) ([]deployment.NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) + readNodes func(context.Context, func(deployment.NodeReads) error) error + resetSessions func(context.Context, string, bool) ([]deployment.ResetSession, error) + addressBindings func(context.Context, string) (deployment.AddressBindings, error) + environmentAllocation func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) + credentialAllocations func(context.Context, string) ([]deployment.Allocation, error) + observationSessions func(context.Context, string, int) (deployment.ObservationSessionPage, error) + nodeAllocations func(context.Context, string) ([]deployment.NodeAllocation, error) + nodeOnline func(context.Context, string) (bool, error) + lifecycleNodes func(context.Context) ([]string, error) + lifecycleAllocations func(context.Context, string, string) ([]deployment.Allocation, error) + unallocatedEnvironments func(context.Context, string, string) ([]deployment.UnallocatedEnvironment, error) + lifecyclePlacement func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) + activity func(context.Context, string) (deployment.Activity, error) } func (r *strictDeploymentReader) Deployment(ctx context.Context) (deployment.Record, error) { @@ -341,3 +343,17 @@ func (r *strictDeploymentReader) Activity(ctx context.Context, allocationID stri } return r.activity(ctx, allocationID) } + +func (r *strictDeploymentReader) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { + if r.countComputeReservations == nil { + return 0, unexpectedDeploymentCall(r.t, "CountComputeReservations") + } + return r.countComputeReservations(ctx, installationID) +} + +func (r *strictDeploymentReader) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { + if r.countRetainedAllocations == nil { + return 0, unexpectedDeploymentCall(r.t, "CountRetainedAllocations") + } + return r.countRetainedAllocations(ctx, installationID) +} diff --git a/services/core/internal/execution/provider_operations_fixture_test.go b/services/core/internal/execution/provider_operations_fixture_test.go index 98e951c04..c1a132c1a 100644 --- a/services/core/internal/execution/provider_operations_fixture_test.go +++ b/services/core/internal/execution/provider_operations_fixture_test.go @@ -17,11 +17,12 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -30,7 +31,7 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { func (*lifecycleOnlySandbox) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*lifecycleOnlySandbox) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*lifecycleOnlySandbox) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*lifecycleOnlySandbox) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (*lifecycleOnlySandbox) Resume(context.Context, sandbox.ResumeRequest) (san func (*lifecycleOnlySandbox) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*lifecycleOnlySandbox) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*lifecycleOnlySandbox) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*lifecycleOnlySandbox) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -57,3 +58,7 @@ func (*lifecycleOnlySandbox) ResumeCompute(context.Context, sandbox.Reference, s func (*lifecycleOnlySandbox) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } + +func (*lifecycleOnlySandbox) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 20d00602b..cbf208557 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -1,9 +1,11 @@ package execution import ( + "bytes" "context" "encoding/json" "errors" + "io" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" @@ -17,20 +19,24 @@ import ( // RuntimeSuspensionPolicy is the idle suspension policy of a provider that // suspends sandboxes. type RuntimeSuspensionPolicy struct { + MaxActive int + MaxRetained int IdleTimeout time.Duration Retention time.Duration } type runtimeCompute struct { - Current sandbox.Compute `json:"current"` - Target *sandbox.Compute `json:"target,omitempty"` - Snapshot *sandbox.SnapshotIdentity `json:"snapshot,omitempty"` - SuspendID string `json:"suspend_id,omitempty"` - RestoreID string `json:"restore_id,omitempty"` - Rollback bool `json:"rollback,omitempty"` + Version string `json:"protocol_version"` + Current sandbox.Compute `json:"current"` + Target *sandbox.Compute `json:"target,omitempty"` + Retained *sandbox.RetainedState `json:"retained,omitempty"` + SuspendID string `json:"suspend_id,omitempty"` + RestoreID string `json:"restore_id,omitempty"` + Rollback bool `json:"rollback,omitempty"` } func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner deployment.Allocation, phase string, state runtimeCompute, until *time.Time) (deployment.Allocation, error) { + state.Version = sandbox.SuspensionStateVersion raw, err := json.Marshal(state) if err != nil { return owner, err @@ -71,7 +77,7 @@ func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment. return err } var state runtimeCompute - if json.Unmarshal(owner.ComputeState, &state) != nil || state.Current.ID == "" { + if decodeRuntimeCompute(owner.ComputeState, &state) != nil || state.Current.ID == "" { return sandbox.ErrOwnership } if owner.SessionDeleted || owner.Expired || owner.State == "cleanup_pending" { @@ -113,6 +119,13 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro if compute.Status != "running" || !compute.BootstrapComplete { return sandbox.ErrComputeUnconfirmed } + renewed, err := p.RenewCompute(ctx, runtimeReference(owner), state.Current) + if err != nil { + return err + } + if sandbox.ValidateComputeResult(state.Current, renewed.Compute) != nil || renewed.Status != "running" || !renewed.BootstrapComplete { + return sandbox.ErrComputeUnconfirmed + } peer, err := authorizedRuntimePeer(ctx, r.sessions, r.registry, owner.DeviceID) if err != nil { return err @@ -173,15 +186,15 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro } func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) error { - result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Snapshot: state.Snapshot, ObserveOnly: observeOnly}) + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Retained: state.Retained, ReconcileOnly: observeOnly}) if err != nil { return err } - if result.Compute.ID != state.Current.ID { - return sandbox.ErrOwnership + if err := sandbox.ValidateSuspendResult(sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Retained: state.Retained, ReconcileOnly: observeOnly}, result); err != nil { + return err } - if result.Snapshot == nil { - if !observeOnly || result.SourceStopped || (result.Status != "running" && result.Status != "paused") { + if result.Retained == nil { + if !observeOnly || !result.SuspendSettled || result.ResourcesReleased || (result.Status != "running" && result.Status != "paused") { return sandbox.ErrComputeUnconfirmed } state.Rollback = true @@ -191,17 +204,8 @@ func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.Sandbox } return r.wakeCompute(ctx, p, next, state) } - state.Snapshot = result.Snapshot - // Store the verified artifact before any recovery-path kill. Snapshot failure - // or an unknown result cannot silently fall back to a cold Environment. - next, err := r.saveCompute(ctx, owner, "suspending", state, owner.ComputeRetainedUntil) - if err != nil { - return err - } - if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { - return err - } - _, err = r.saveCompute(ctx, next, "suspended", state, next.ComputeRetainedUntil) + state.Retained = result.Retained + _, err = r.saveCompute(ctx, owner, "suspended", state, owner.ComputeRetainedUntil) return err } @@ -213,13 +217,19 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San if !activity.Busy && !activity.WakeRequested { return nil } - if state.Snapshot == nil || state.Target != nil { + if err := r.computeCapacityForAllocation(ctx, owner); err != nil { + return err + } + if state.Retained == nil || state.Target != nil { return sandbox.ErrOwnership } - target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Snapshot) + target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Retained) if err != nil { return err } + if target.Name == "" || target.Generation != state.Current.Generation+1 || target.RestoredFrom == nil || *target.RestoredFrom != *state.Retained { + return sandbox.ErrOwnership + } state.Target, state.RestoreID = &target, uuid.NewString() next, err := r.saveCompute(ctx, owner, "restoring", state, owner.ComputeRetainedUntil) if err != nil { @@ -228,28 +238,18 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San return r.restoreCompute(ctx, p, next, state, false) } func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) error { - if state.Target == nil || state.Snapshot == nil || state.Rollback { + if state.Target == nil || state.Retained == nil || state.Rollback { return sandbox.ErrOwnership } - var workspace *workspacefs.Binding - if r.config.Workspace != nil { - if r.workspaces == nil { - return workspacefs.ErrUnavailable - } - var err error - workspace, err = r.workspaces.GetReady(ctx, owner.TenantID, owner.EnvironmentID, r.config.WorkspaceRequirements, r.config.Resources.EnvironmentDiskMiB) - if err == nil && workspace == nil { - err = workspacefs.ErrNotFound - } - if err != nil { - return err - } + workspace, err := r.restoreWorkspace(ctx, owner) + if err != nil { + return err } - result, err := p.Resume(ctx, sandbox.ResumeRequest{Workspace: workspace, Reference: runtimeReference(owner), OperationID: state.RestoreID, Snapshot: *state.Snapshot, Target: *state.Target, ObserveOnly: observeOnly}) + result, err := p.Resume(ctx, sandbox.ResumeRequest{Workspace: workspace, Reference: runtimeReference(owner), OperationID: state.RestoreID, Retained: *state.Retained, Target: *state.Target, ReconcileOnly: observeOnly}) if err != nil { return err } - if result.Status != "running" || result.Compute.ID == "" { + if result.Status != "running" || !result.BootstrapComplete || sandbox.ValidateComputeResult(*state.Target, result.Compute) != nil { return sandbox.ErrComputeUnconfirmed } state.Current, state.Target = result.Compute, nil @@ -268,3 +268,75 @@ func ignoreComputeAbsent(err error) error { } return err } + +// Node-backed restores reserve capacity atomically in SetCompute. +func (r *runtimeLifecycle) computeCapacityForAllocation(ctx context.Context, owner deployment.Allocation) error { + if owner.NodeID != "" { + return nil + } + return r.computeCapacity(ctx, owner.ProviderKey) +} + +func decodeRuntimeCompute(raw []byte, state *runtimeCompute) error { + d := json.NewDecoder(bytes.NewReader(raw)) + d.DisallowUnknownFields() + if err := d.Decode(state); err != nil { + return err + } + if d.Decode(new(any)) != io.EOF || state.Version != sandbox.SuspensionStateVersion { + return sandbox.ErrOwnership + } + return nil +} + +func (r *runtimeLifecycle) computeCapacity(ctx context.Context, key string) error { + policy := r.config.Suspension + if key != r.config.InstallationID { + return sandbox.ErrOwnership + } + if policy == nil { + return nil + } + count, err := r.reader.CountComputeReservations(ctx, key) + if err != nil { + return err + } + if count >= int64(policy.MaxActive) { + return ErrExecutionUnavailable + } + return nil +} + +// restoreWorkspace follows the allocation's immutable generation, never the +// current deployment's filesystem selection or sizing. +func (r *runtimeLifecycle) restoreWorkspace(ctx context.Context, owner deployment.Allocation) (*workspacefs.Binding, error) { + record, err := r.reader.Allocation(ctx, runtimeReference(owner)) + if err != nil { + return nil, err + } + if record.ID != owner.ID || record.InstallationID != owner.ProviderKey || record.Generation != owner.DeploymentGeneration || record.Released { + return nil, sandbox.ErrOwnership + } + raw := record.Deployment.Specification + if record.Generation != record.Deployment.Generation { + if record.Retained == nil || record.Retained.Generation != record.Generation { + return nil, sandbox.ErrOwnership + } + raw = record.Retained.Specification + } + var spec sandbox.DeploymentSpec + if err := json.Unmarshal(raw, &spec); err != nil { + return nil, sandbox.ErrInvalid + } + if spec.Workspace == nil { + return nil, nil + } + if r.workspaces == nil { + return nil, workspacefs.ErrUnavailable + } + binding, err := r.workspaces.GetReady(ctx, owner.TenantID, owner.EnvironmentID, r.config.WorkspaceRequirements, spec.Resources.EnvironmentDiskMiB) + if err == nil && binding == nil { + return nil, workspacefs.ErrNotFound + } + return binding, err +} diff --git a/services/core/internal/execution/runtime_compute_compatibility_test.go b/services/core/internal/execution/runtime_compute_compatibility_test.go new file mode 100644 index 000000000..cdf5ba6eb --- /dev/null +++ b/services/core/internal/execution/runtime_compute_compatibility_test.go @@ -0,0 +1,60 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" + "reflect" + "testing" +) + +// This literal freezes the deployed beta v1 shape independently of current types. +func TestBetaRetainedV1RoundTripPreservesEveryField(t *testing.T) { + const raw = `{"protocol_version":"1","current":{"Generation":4,"Name":"owned-source","ID":"native-source","RestoredFrom":{"Reference":"owned/reference","ID":"native-parent","Data":"{\"version\":1,\"sandbox_id\":\"native-source\"}","OperationID":"prior-operation","SourceGeneration":3,"SourceName":"prior-source","SourceID":"native-parent"}},"target":{"Generation":5,"Name":"owned-target","ID":"","RestoredFrom":{"Reference":"owned/reference","ID":"native-source","Data":"{\"version\":1,\"sandbox_id\":\"native-source\"}","OperationID":"pause-operation","SourceGeneration":4,"SourceName":"owned-source","SourceID":"native-source"}},"retained":{"Reference":"owned/reference","ID":"native-source","Data":"{\"version\":1,\"sandbox_id\":\"native-source\"}","OperationID":"pause-operation","SourceGeneration":4,"SourceName":"owned-source","SourceID":"native-source"},"suspend_id":"pause-operation","restore_id":"resume-operation","rollback":true}` + var state runtimeCompute + if err := decodeRuntimeCompute([]byte(raw), &state); err != nil { + t.Fatal(err) + } + out, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + var before, after any + if json.Unmarshal([]byte(raw), &before) != nil || json.Unmarshal(out, &after) != nil || !reflect.DeepEqual(before, after) { + t.Fatal("beta v1 field loss or rewrite") + } + for _, invalid := range []string{`{}`, `{"protocol_version":"2"}`, `{"protocol_version":"1","snapshot":{}}`, `{"protocol_version":"1","current":{"RestoredFrom":{"Digest":"legacy"}}}`, `{"protocol_version":"1","target":{"RestoredFrom":{"CheckpointRoot":"legacy"}}}`, raw + `{}`} { + if decodeRuntimeCompute([]byte(invalid), new(runtimeCompute)) == nil { + t.Fatal("incompatible receipt accepted") + } + } +} + +func TestRestoreUsesAllocationGenerationWorkspaceAfterDeploymentChange(t *testing.T) { + owner := deployment.Allocation{ID: "allocation", ProviderKey: "installation", DeploymentGeneration: 4} + for _, originalExternal := range []bool{false, true} { + t.Run(fmt.Sprint(originalExternal), func(t *testing.T) { + original, current := sandbox.DeploymentSpec{}, sandbox.DeploymentSpec{Workspace: &workspacefs.Declaration{}} + if originalExternal { + original, current = current, original + } + prior, _ := json.Marshal(original) + latest, _ := json.Marshal(current) + r := runtimeLifecycle{config: RuntimeProvider{Workspace: current.Workspace}, reader: &strictDeploymentReader{t: t, allocation: func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) { + return deployment.AllocationRecord{ID: owner.ID, InstallationID: owner.ProviderKey, Generation: 4, Deployment: deployment.Record{Generation: 5, Specification: latest}, Retained: &deployment.GenerationRecord{Generation: 4, Specification: prior}}, nil + }}} + binding, err := r.restoreWorkspace(t.Context(), owner) + if originalExternal { + if !errors.Is(err, workspacefs.ErrUnavailable) { + t.Fatal("original external storage was bypassed", err) + } + } else if err != nil || binding != nil { + t.Fatal("old owned disk depended on new workspace", err) + } + }) + } +} diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index 0dd7f915d..3d485e21d 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -6,6 +6,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -25,7 +26,7 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxPro } // This idempotent control signal is fenced by guest PID/start time and the // suspension token. It cannot execute or replay an agent request. - result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", "/run/oac/daemon-suspend.json", "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) + result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", runtimebootstrap.SuspendControlFile, "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) if err != nil { return err } @@ -55,8 +56,8 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxPro } // The artifact has been consumed. Never restore it after this generation // admits work, even if garbage collection or the final database commit fails. - if state.Snapshot != nil { - if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + if state.Retained != nil { + if err := ignoreComputeAbsent(p.DeleteRetained(ctx, runtimeReference(owner), *state.Retained)); err != nil { return err } } @@ -76,13 +77,13 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Sandbox } // An uncommitted artifact is found by its persisted attempt, never a directory // glob. The helper's allocation lock also waits for an earlier unknown call. - if owner.ComputePhase == "suspending" && state.Snapshot == nil { - result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, ObserveOnly: true}) + if owner.ComputePhase == "suspending" && state.Retained == nil { + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, ReconcileOnly: true}) if err != nil && !errors.Is(err, sandbox.ErrNotFound) { return err } if err == nil { - state.Snapshot = result.Snapshot + state.Retained = result.Retained } } if state.Target != nil { @@ -93,8 +94,8 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Sandbox if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { return err } - if state.Snapshot != nil { - if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + if state.Retained != nil { + if err := ignoreComputeAbsent(p.DeleteRetained(ctx, runtimeReference(owner), *state.Retained)); err != nil { return err } } diff --git a/services/core/internal/execution/runtime_fresh_hint_test.go b/services/core/internal/execution/runtime_fresh_hint_test.go index 334052b4e..fec4b3817 100644 --- a/services/core/internal/execution/runtime_fresh_hint_test.go +++ b/services/core/internal/execution/runtime_fresh_hint_test.go @@ -4,11 +4,6 @@ import ( "context" "encoding/json" "errors" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" - projectpg "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/projectpg" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/projects" "sync" "sync/atomic" "testing" @@ -17,6 +12,7 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -43,11 +39,10 @@ func TestFreshHintRoutesAndPreservesLifecycleGuards(t *testing.T) { t.Run(scenario, func(t *testing.T) { m := testRuntimeManager(t) m.setupGate = make(chan struct{}, 1) - m.config = RuntimeProvider{InstallationID: uuid.NewString(), Mode: "direct", Provider: &freshHintProvider{}} + m.config = RuntimeProvider{InstallationID: uuid.NewString(), ProviderKind: "e2b", Mode: "direct", Provider: &freshHintProvider{}} nodeID := "" if scenario == "node" || scenario == "released_placement" { - m.config.ProviderKind = "docker" - m.config.Mode = "nodes" + m.config.ProviderKind, m.config.Mode = "docker", "nodes" nodeID = uuid.NewString() } node, err := m.node(nodeID) @@ -56,7 +51,7 @@ func TestFreshHintRoutesAndPreservesLifecycleGuards(t *testing.T) { } environment := sessions.Environment{ID: uuid.NewString(), Configuration: json.RawMessage(`{"type":"openai_hosted"}`)} if scenario == "self_hosted" { - environment.Configuration = json.RawMessage(`{"type":"self_hosted","workspace_directory":"/workspace"}`) + environment.Configuration = json.RawMessage(`{"type":"self_hosted"}`) } if scenario == "suspended" { environment.Initialization = "complete" @@ -73,7 +68,7 @@ func TestFreshHintRoutesAndPreservesLifecycleGuards(t *testing.T) { return deployment.Allocation{}, deployment.ErrNotFound }, lifecyclePlacement: func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) { - return deployment.LifecyclePlacement{Provider: m.config.ProviderKind, PlacementNodeID: nodeID, PlacementReleased: scenario == "released_placement"}, nil + return deployment.LifecyclePlacement{Provider: m.config.ProviderKind, Mode: m.config.Mode, PlacementNodeID: nodeID, PlacementReleased: scenario == "released_placement"}, nil }, } m.deploymentService, _ = deploymentOperations(t, &strictDeploymentStorage{t: t}, reader, &strictExecutionStorage{t: t}) @@ -110,19 +105,19 @@ func (p *freshHintProvider) Create(_ context.Context, b sandbox.Bootstrap) (sand } func TestFreshEnvironmentHintProvisionsWithoutMaintenanceTick(t *testing.T) { - for _, mode := range []string{"create", "recovered_input", "recovered_initial"} { + for _, mode := range []string{"create", "create_initial", "recovered_input", "recovered_initial"} { t.Run(mode, func(t *testing.T) { owner, deployments, reader, pool := resetManagerDB(t, nil) installation := initializeE2BDeployment(t, owner) - sessionReader, s := testSessions(t, pool, pgtest.CredentialKey(t)) + sessionReader, sessionService := testSessions(t, pool, pgtest.CredentialKey(t)) provider := &freshHintProvider{created: make(chan struct{}, 1)} m := testRuntimeManager(t) m.setupGate = make(chan struct{}, 1) m.sessions, m.sessionExecution = sessionReader, owner.Sessions m.deployment, m.deploymentService, m.deploymentReader = owner.Deployment, deployments, reader m.lease, m.registry = owner.Lease, runtimegateway.NewRegistry() - m.config = RuntimeProvider{Mode: "direct", InstallationID: installation, CoreURL: fixturePublicURL + "/api/v1", Provider: provider} - worker := &Worker{lease: owner.Lease, runtimes: m, dispatcher: &Dispatcher{Sessions: s, sessionExecution: owner.Sessions, SessionsReader: sessionReader, DeploymentReader: reader, notifications: &executionNotifications{}}, scheduleWake: make(chan struct{}, 1)} + m.config = RuntimeProvider{InstallationID: installation, Generation: 1, ProviderKind: "e2b", Mode: "direct", CoreURL: fixturePublicURL + "/api/v1", Provider: provider} + worker := &Worker{lease: owner.Lease, runtimes: m, dispatcher: &Dispatcher{Sessions: sessionService, SessionsReader: sessionReader, DeploymentReader: reader, notifications: &executionNotifications{}}, scheduleWake: make(chan struct{}, 1)} node, err := m.node("") if err != nil { t.Fatal(err) @@ -140,33 +135,24 @@ func TestFreshEnvironmentHintProvisionsWithoutMaintenanceTick(t *testing.T) { }() t.Cleanup(func() { cancel(); <-done }) <-scans // Startup scan finishes before any Session is committed. - projectID := uuid.NewString() - audit := adminaudit.WithSource(ctx, adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) - management, err := projects.NewService(projectpg.New(pgunit.NewPool(pool))) - if err != nil { - t.Fatal(err) - } - project, err := management.CreateProject(audit, projects.CreateProject{ID: projectID, Name: "Fresh hint"}) - if err != nil { - t.Fatal(err) - } - tenant := project.TenantID + tenant := uuid.NewString() input := sessions.CreateSession{Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted"}}`), ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ExecutionSourceSession} messages := []sessions.Input{{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"fixture"}]}]}`)}} - if mode == "recovered_initial" { + if mode == "recovered_initial" || mode == "create_initial" { input.InitialInputs = messages } var creation sessions.Creation switch mode { - case "create": + case "create", "create_initial": + // Ordinary and streaming HTTP creation share this Worker entrypoint. creation, err = worker.CreateSession(ctx, tenant, input) case "recovered_input", "recovered_initial": - creation, err = s.CreateSession(ctx, tenant, input) + creation, err = sessionService.CreateSession(ctx, tenant, input) } + session := creation.Session if err != nil { t.Fatal(err) } - session := creation.Session if mode == "recovered_input" || mode == "recovered_initial" { key := uuid.NewString() if mode == "recovered_initial" { diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 8b0eea006..feca79400 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -101,12 +101,9 @@ func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway. if copied.ProviderKind == "" || (copied.Mode != string(sandbox.DeploymentNodes) && copied.Mode != string(sandbox.DeploymentDirect)) { return RuntimeProvider{}, sandbox.ErrInvalid } - if copied.Mode == string(sandbox.DeploymentDirect) && copied.Suspension != nil { - return RuntimeProvider{}, sandbox.ErrInvalid - } if config.Suspension != nil { policy := *config.Suspension - if !sandbox.SupportsCheckpoint(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second { + if !sandbox.SupportsSuspension(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second || (copied.Mode == string(sandbox.DeploymentDirect) && (policy.MaxActive < 1 || policy.MaxRetained < policy.MaxActive)) { return RuntimeProvider{}, sandbox.ErrInvalid } copied.Suspension = &policy @@ -169,6 +166,10 @@ func (w *Worker) ProvisionEnvironment(ctx context.Context, tenant, environment, // provision runs under the lifecycle gate and uses the durable one-shot receipt. func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, providerKey string) (deployment.Allocation, error) { provider := r.config.Provider + // The lifecycle lane, not provider configuration, owns node identity. + if (r.nodeID == "") != (r.config.Mode == string(sandbox.DeploymentDirect)) { + return deployment.Allocation{}, sandbox.ErrOwnership + } if providerKey != r.config.InstallationID { return deployment.Allocation{}, sandbox.ErrInvalid } @@ -180,6 +181,26 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if err != nil || placement.Type != "openai_hosted" { return deployment.Allocation{}, sandbox.ErrInvalid } + key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} + if _, err := r.reader.EnvironmentAllocation(ctx, key); errors.Is(err, deployment.ErrNotFound) { + if r.config.Generation == 0 { + return deployment.Allocation{}, ErrExecutionUnavailable + } + if err := r.computeFreshCapacity(ctx, providerKey); err != nil { + return deployment.Allocation{}, err + } + if policy := r.config.Suspension; policy != nil && (r.config.ProviderKind == "" || r.config.Mode == "direct") { + count, err := r.reader.CountRetainedAllocations(ctx, providerKey) + if err != nil { + return deployment.Allocation{}, err + } + if count >= int64(policy.MaxRetained) { + return deployment.Allocation{}, ErrExecutionUnavailable + } + } + } else if err != nil { + return deployment.Allocation{}, err + } var workspace *workspacefs.Binding if r.config.Workspace != nil { if r.workspaces == nil { @@ -201,11 +222,6 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p return existing, sandbox.ErrOwnership } } - key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} - session, err := r.sessions.GetSession(ctx, tenant, environmentValue.SessionID) - if err != nil { - return deployment.Allocation{}, err - } secret := make([]byte, 32) if _, err := rand.Read(secret); err != nil { return deployment.Allocation{}, err @@ -224,9 +240,13 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if err := r.lease.CheckOwnership(ctx); err != nil { return owner, err } + session, err := r.sessions.GetSession(ctx, tenant, environmentValue.SessionID) + if err != nil { + return deployment.Allocation{}, err + } info, err := provider.Create(ctx, sandbox.Bootstrap{ Reference: runtimeReference(owner), SessionID: owner.SessionID, DeviceID: owner.DeviceID, - Workspace: workspace, CoreURL: r.config.CoreURL, Credential: token, Harness: session.Engine, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, + Workspace: workspace, Harness: session.Engine, CoreURL: r.config.CoreURL, Credential: token, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, }) if info.Reference == runtimeReference(owner) && info.CreateSettled && info.State == "absent" { record, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) @@ -432,3 +452,11 @@ func runtimeReference(owner deployment.Allocation) sandbox.Reference { func (w *Worker) runManagedRuntimes(ctx context.Context) error { return w.runtimes.run(ctx) } + +// Manager deployments reserve capacity with Session placement before provisioning. +func (r *runtimeLifecycle) computeFreshCapacity(ctx context.Context, key string) error { + if r.config.ProviderKind != "" && r.config.Mode != "direct" { + return nil + } + return r.computeCapacity(ctx, key) +} diff --git a/services/core/internal/execution/runtime_workspace_settlement_test.go b/services/core/internal/execution/runtime_workspace_settlement_test.go index 101e76529..d4cf55108 100644 --- a/services/core/internal/execution/runtime_workspace_settlement_test.go +++ b/services/core/internal/execution/runtime_workspace_settlement_test.go @@ -107,7 +107,7 @@ func newWorkspaceSettlementFixture(t *testing.T, provider sandbox.SandboxProvide control := &settlementWorkspaceControl{configuration: configuration} filesystems := workspaces.NewExecution(storage, writer, settlementWorkspaceControls{control}, lease) declaration := control.Declaration() - lifecycle := &runtimeLifecycle{registry: runtimegateway.NewRegistry(), sessions: sessionReader, sessionExecution: owner.Sessions, deployment: operations, deployments: deployments, reader: reader, lease: lease, workspaces: filesystems, config: RuntimeProvider{InstallationID: installation, Provider: provider, Workspace: &declaration, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} + lifecycle := &runtimeLifecycle{registry: runtimegateway.NewRegistry(), sessions: sessionReader, sessionExecution: owner.Sessions, deployment: operations, deployments: deployments, reader: reader, lease: lease, workspaces: filesystems, config: RuntimeProvider{InstallationID: installation, Mode: "direct", Generation: 1, Provider: provider, Workspace: &declaration, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} return workspaceSettlementFixture{lifecycle: lifecycle, sessions: service, storage: storage, control: control, session: session} } diff --git a/services/core/internal/execution/runtime_workspace_test.go b/services/core/internal/execution/runtime_workspace_test.go index 8c52d590f..0999b7c99 100644 --- a/services/core/internal/execution/runtime_workspace_test.go +++ b/services/core/internal/execution/runtime_workspace_test.go @@ -16,10 +16,6 @@ import ( type workspaceEnvironmentReader struct{ sessions.Reader } -func (workspaceEnvironmentReader) GetSession(context.Context, string, string) (sessions.Session, error) { - return sessions.Session{Engine: "codex"}, nil -} - func (workspaceEnvironmentReader) GetEnvironment(context.Context, string, string) (sessions.Environment, error) { return sessions.Environment{Configuration: []byte(`{"type":"openai_hosted"}`)}, nil } @@ -99,7 +95,7 @@ func TestRuntimeWorkspaceConvergesBeforeComputeReservation(t *testing.T) { }}) r := &runtimeLifecycle{sessions: workspaceEnvironmentReader{}, deployment: operations, reader: &strictDeploymentReader{t: t, environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { return deployment.Allocation{}, deployment.ErrNotFound - }}, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{InstallationID: uuid.NewString(), Workspace: &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory}, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} + }}, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{Mode: "direct", Generation: 1, InstallationID: uuid.NewString(), Workspace: &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory}, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} _, err := r.provision(t.Context(), f.record.Reference.TenantID, f.record.Reference.EnvironmentID, r.config.InstallationID) if fail { if !errors.Is(err, workspacefs.ErrUnavailable) || reserved { @@ -155,7 +151,9 @@ func TestOwnedGenerationDoesNotAdoptLaterFilesystemSelection(t *testing.T) { _, operations := deploymentOperations(t, &strictDeploymentStorage{t: t}, &strictDeploymentReader{t: t}, &strictExecutionStorage{t: t, withReservation: func(context.Context, deployment.AllocationKey, func(sessions.LockedSession, deployment.ReservationTx) error) error { return stop }}) - r := &runtimeLifecycle{sessions: workspaceEnvironmentReader{}, deployment: operations, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{InstallationID: uuid.NewString()}} + r := &runtimeLifecycle{sessions: workspaceEnvironmentReader{}, deployment: operations, reader: &strictDeploymentReader{t: t, environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + return deployment.Allocation{}, deployment.ErrNotFound + }}, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{Mode: "direct", Generation: 1, InstallationID: uuid.NewString()}} _, err := r.provision(t.Context(), f.record.Reference.TenantID, f.record.Reference.EnvironmentID, r.config.InstallationID) if !errors.Is(err, stop) || f.binds != 0 || f.creates != 0 { t.Fatal("owned generation retroactively attached filesystem", err) diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 6d67ff3cb..fd4328c8a 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -75,6 +76,9 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher, owner Owner) (_ *W if owner.Deployment == nil { return nil, errors.New("execution worker requires the deployment execution operations") } + if err := owner.Deployment.CheckRuntimeComputeProtocol(ctx, sandbox.SuspensionStateVersion); err != nil { + return nil, err + } owned.notifications = &executionNotifications{} worker := &Worker{concurrency: dispatcher.MaxConcurrentExecutions, dispatcher: owned, lease: owner.Lease, directoryReads: make(chan directoryReadRequest), fileWrites: make(chan fileWriteRequest), stopped: make(chan struct{}), scheduleWake: make(chan struct{}, 1), enrolledConnections: make(map[string]*runtimeConnection)} worker.runtimes, err = newRuntimeManager(owner, owned.Deployment, owned.DeploymentReader, owned.SessionsReader, owned.Registry, owned.ManagedRuntimes) diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index ee99f8f4d..b65e310ee 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -14,6 +14,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/placementpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/sessionpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -217,6 +218,7 @@ func (t *reservationTx) InsertAllocation(a deployment.NewAllocation) (deployment row, err := t.q.CreateRuntimeAllocation(t.ctx, sqlc.CreateRuntimeAllocationParams{ ID: id, EnvironmentID: t.environment, DeviceID: device, ProviderKey: provider, NodeID: node, DeploymentGeneration: pgtype.Int8{Int64: int64(a.Generation), Valid: true}, + ProtocolVersion: sandbox.SuspensionStateVersion, }) if err != nil { return deployment.Allocation{}, err @@ -352,7 +354,7 @@ func loadActivity(ctx context.Context, q *sqlc.Queries, id pgtype.UUID) (deploym if err != nil { return deployment.Activity{}, err } - return deployment.Activity{LastActivity: row.LastActivity.Time, ObservedAt: row.ObservedAt.Time, Busy: row.Busy, WakeRequested: row.ComputeWakeRequested, HasCompletedTurn: row.HasCompletedTurn}, nil + return deployment.Activity{LastActivity: row.LastActivity.Time, ObservedAt: row.ObservedAt.Time, Busy: row.Busy, WakeRequested: row.ComputeWakeRequested}, nil } func (s *Store) EnvironmentAllocation(ctx context.Context, key deployment.AllocationKey) (deployment.Allocation, error) { @@ -536,3 +538,19 @@ func (s *Store) Activity(ctx context.Context, allocationID string) (deployment.A } return loadActivity(ctx, s.pool.Queries(), id) } + +func (s *Store) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { + id, err := parseID(installationID) + if err != nil { + return 0, err + } + return s.pool.Queries().CountRuntimeComputeReservations(ctx, id) +} + +func (s *Store) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { + id, err := parseID(installationID) + if err != nil { + return 0, err + } + return s.pool.Queries().CountRuntimeRetainedAllocations(ctx, id) +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/tx.go b/services/core/internal/persistence/postgres/deploymentpg/tx.go index 28f35e1c5..7e865c437 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/tx.go +++ b/services/core/internal/persistence/postgres/deploymentpg/tx.go @@ -354,3 +354,7 @@ func (t *deploymentTx) RecordAudit(action, installationID string) error { func (t *deploymentTx) RecordAuditAs(source adminaudit.Source, action, installationID string) error { return auditpg.RecordDeploymentMutation(adminaudit.WithSource(t.ctx, source), t.q, action, "sandbox_deployment", installationID) } + +func (t *deploymentTx) HasIncompatibleComputeState(version string) (bool, error) { + return t.q.HasIncompatibleRuntimeComputeState(t.ctx, version) +} diff --git a/services/core/internal/processconfig/config.go b/services/core/internal/processconfig/config.go index 6a5b75edf..6bc5c9b72 100644 --- a/services/core/internal/processconfig/config.go +++ b/services/core/internal/processconfig/config.go @@ -41,6 +41,7 @@ const ( // Config is Core's process configuration. type Config struct { + SandboxCapacity SandboxLimits // Addr is OAC_ADDR, the listener address. Addr string // PublicOrigin is OAC_PUBLIC_URL. @@ -102,6 +103,9 @@ type runtimeHistoryFile struct { func Load() (Config, error) { var c Config var err error + if c.SandboxCapacity, err = SandboxCapacity(); err != nil { + return Config{}, err + } if c.Log, err = log.LoadConfig(); err != nil { return Config{}, err } @@ -170,6 +174,8 @@ func (c Config) Settings() []api.InstallationSetting { setting("log.level", strings.ToLower(c.Log.Level.String()), "info", []api.InstallationService{api.InstallationCore, api.InstallationWeb}), setting("log.format", format, "auto", []api.InstallationService{api.InstallationCore, api.InstallationWeb}), setting("log.add_source", c.Log.AddSource, false, []api.InstallationService{api.InstallationCore, api.InstallationWeb}), + setting("core.sandbox_capacity.max_active", c.SandboxCapacity.MaxActive, defaultSandboxMaxActive, []api.InstallationService{api.InstallationCore}), + setting("core.sandbox_capacity.max_retained", c.SandboxCapacity.MaxRetained, defaultSandboxMaxRetained, []api.InstallationService{api.InstallationCore}), setting("core.execution_concurrency", c.ExecutionConcurrency, execution.DefaultExecutionConcurrency, []api.InstallationService{api.InstallationCore}), setting("core.harnesses", c.Harnesses, (engine.Catalog{}).Kinds(), []api.InstallationService{api.InstallationCore}), setting("core.default_harness", c.DefaultHarness, defaultHarness, []api.InstallationService{api.InstallationCore}), diff --git a/services/core/internal/processconfig/sandbox_capacity.go b/services/core/internal/processconfig/sandbox_capacity.go new file mode 100644 index 000000000..742b057ff --- /dev/null +++ b/services/core/internal/processconfig/sandbox_capacity.go @@ -0,0 +1,41 @@ +package processconfig + +import ( + "os" + "strconv" +) + +const defaultSandboxMaxActive = 100 +const defaultSandboxMaxRetained = 400 + +// SandboxLimits bounds direct Providers with suspension independently of +// execution concurrency and each enrolled node's capacity. +type SandboxLimits struct { + MaxActive int + MaxRetained int +} + +// SandboxCapacity reads the process-owned active and retained limits. Unset or +// empty variables select the same defaults as the other process settings. +func SandboxCapacity() (SandboxLimits, error) { + capacity := SandboxLimits{MaxActive: defaultSandboxMaxActive, MaxRetained: defaultSandboxMaxRetained} + for _, setting := range []struct { + name string + target *int + }{ + {"OAC_SANDBOX_MAX_ACTIVE", &capacity.MaxActive}, + {"OAC_SANDBOX_MAX_RETAINED", &capacity.MaxRetained}, + } { + if raw := os.Getenv(setting.name); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100000 { + return SandboxLimits{}, configError(setting.name + " must be an integer between 1 and 100000") + } + *setting.target = value + } + } + if capacity.MaxRetained < capacity.MaxActive { + return SandboxLimits{}, configError("OAC_SANDBOX_MAX_RETAINED must be at least OAC_SANDBOX_MAX_ACTIVE") + } + return capacity, nil +} diff --git a/services/core/internal/processconfig/sandbox_capacity_test.go b/services/core/internal/processconfig/sandbox_capacity_test.go new file mode 100644 index 000000000..6c06fd7f0 --- /dev/null +++ b/services/core/internal/processconfig/sandbox_capacity_test.go @@ -0,0 +1,53 @@ +package processconfig + +import ( + "strings" + "testing" +) + +func TestSandboxCapacityDefaultsAndSettings(t *testing.T) { + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "") + capacity, err := SandboxCapacity() + if err != nil || capacity.MaxActive != 100 || capacity.MaxRetained != 400 { + t.Fatal("empty Compose settings lost defaults", capacity, err) + } + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "7") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "31") + capacity, err = SandboxCapacity() + settings := (Config{SandboxCapacity: capacity}).Settings() + if err != nil { + t.Fatal(err) + } + found := map[string]any{} + for _, setting := range settings { + found[setting.Key] = setting.Value + } + if found["core.sandbox_capacity.max_active"] != 7 || found["core.sandbox_capacity.max_retained"] != 31 { + t.Fatal("installation omitted effective direct capacity", found) + } +} + +func TestCheckValidatesSandboxCapacity(t *testing.T) { + for _, key := range []string{"OAC_SANDBOX_MAX_ACTIVE", "OAC_SANDBOX_MAX_RETAINED"} { + for _, tc := range []struct { + value string + valid bool + }{{"0", false}, {"-1", false}, {"1.5", false}, {"synthetic-secret-value", false}, {"100001", false}, {"100000", true}, {"1", true}, {"", true}} { + t.Run(key+"/"+tc.value, func(t *testing.T) { + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "1") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "100000") + t.Setenv(key, tc.value) + _, err := SandboxCapacity() + if (err == nil) != tc.valid || err != nil && (!strings.Contains(err.Error(), key) || strings.Contains(err.Error(), "synthetic-secret-value")) { + t.Fatal("wrong process validation", err) + } + }) + } + } + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "100") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "99") + if _, err := SandboxCapacity(); err == nil { + t.Fatal("accepted retained limit below active limit") + } +} diff --git a/services/core/internal/sandbox/docker/operations.go b/services/core/internal/sandbox/docker/operations.go index 2722da0dd..0d479f3f1 100644 --- a/services/core/internal/sandbox/docker/operations.go +++ b/services/core/internal/sandbox/docker/operations.go @@ -16,11 +16,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "NewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "GetCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Suspend": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Resume": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "KillCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Observe": {State: providercontract.Supported}, @@ -30,7 +31,7 @@ func (*Provider) ProviderOperations() providercontract.Operations { return Opera func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} } -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} } func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Compu func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} } -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} +func (p *Provider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: Operations()["DeleteRetained"].Reason} } func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} @@ -54,3 +55,7 @@ func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} } + +func (p *Provider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: Operations()["RenewCompute"].Reason} +} diff --git a/services/core/internal/sandbox/e2b/helper_contract.go b/services/core/internal/sandbox/e2b/helper_contract.go index f6fe533b4..517357622 100644 --- a/services/core/internal/sandbox/e2b/helper_contract.go +++ b/services/core/internal/sandbox/e2b/helper_contract.go @@ -11,7 +11,7 @@ import ( //go:generate go run ./internal/contractgen // This adapter-private boundary is documented in tools/e2b-provider/README.md. -const ProtocolVersion = 2 +const ProtocolVersion = 4 const MaxOutputBytes = 1024 * 1024 const MaxRequestBytes = 72 * 1024 * 1024 const MaxResponseBytes = 16 * 1024 * 1024 @@ -20,7 +20,7 @@ const MaxCommandInputBytes = sandbox.MaxCommandInputBytes // HelperOperations declares the complete set of one-shot helper operations. func HelperOperations() []string { - return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential"} + return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential", "compute_info", "compute_renew", "suspend", "resume", "compute_kill", "delete_retained", "compute_command", "resume_compute"} } // HelperErrors are sanitized wire outcomes; an empty code denotes success. @@ -56,6 +56,28 @@ func (q Request) Validate() error { return sandbox.ErrInvalid } } + + switch q.Operation { + case "compute_info", "compute_renew", "compute_kill", "compute_command", "resume_compute": + if q.Compute == nil || q.Compute.Name != q.Reference.AllocationID || (q.Operation != "compute_info" && q.Compute.ID == "") { + return sandbox.ErrInvalid + } + if q.Operation == "compute_command" && q.Command == nil { + return sandbox.ErrInvalid + } + case "suspend": + if q.Suspend == nil || q.Suspend.Reference != q.Reference || !validID(q.Suspend.OperationID) || q.Suspend.Source.Name != q.Reference.AllocationID || q.Suspend.Source.ID == "" { + return sandbox.ErrInvalid + } + case "resume": + if q.Resume == nil || q.Resume.Workspace != nil || q.Resume.Reference != q.Reference || !validID(q.Resume.OperationID) || sandbox.ValidateRetained(q.Resume.Retained) != nil || q.Resume.Retained.Reference != q.Reference.AllocationID { + return sandbox.ErrInvalid + } + case "delete_retained": + if q.Retained == nil || sandbox.ValidateRetained(*q.Retained) != nil || q.Retained.Reference != q.Reference.AllocationID { + return sandbox.ErrInvalid + } + } return nil } @@ -69,10 +91,15 @@ type Request struct { Bootstrap *sandbox.Bootstrap `json:",omitempty"` RuntimeBootstrap *runtimebootstrap.Connection `json:",omitempty"` Command *sandbox.Command `json:",omitempty"` + Compute *sandbox.Compute `json:",omitempty"` + Suspend *sandbox.SuspendRequest `json:",omitempty"` + Resume *sandbox.ResumeRequest `json:",omitempty"` + Retained *sandbox.RetainedState `json:",omitempty"` Deadline time.Time } type Response struct { Version int + State *sandbox.ComputeState `json:",omitempty"` Info *sandbox.Info `json:",omitempty"` Command *sandbox.CommandResult `json:",omitempty"` ErrorCode string diff --git a/services/core/internal/sandbox/e2b/installed_paths_test.go b/services/core/internal/sandbox/e2b/installed_paths_test.go index 60d5b467b..35a813ef9 100644 --- a/services/core/internal/sandbox/e2b/installed_paths_test.go +++ b/services/core/internal/sandbox/e2b/installed_paths_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "strconv" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -31,7 +32,7 @@ func TestConfigurationDiscoveryUsesSuppliedProcessPaths(t *testing.T) { if err := os.MkdirAll(filepath.Dir(binary), 0700); err != nil { t.Fatal(err) } - if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"Templates\":[]}'\n"), 0700); err != nil { + if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(ProtocolVersion)+",\"Templates\":[]}'\n"), 0700); err != nil { t.Fatal(err) } input := sandbox.ConfigurationDiscoveryInput{Credential: json.RawMessage(`{"api_key":"synthetic-key"}`)} diff --git a/services/core/internal/sandbox/e2b/internal/contractgen/main.go b/services/core/internal/sandbox/e2b/internal/contractgen/main.go index a7a9c788f..de3685b1e 100644 --- a/services/core/internal/sandbox/e2b/internal/contractgen/main.go +++ b/services/core/internal/sandbox/e2b/internal/contractgen/main.go @@ -85,11 +85,17 @@ func main() { identity = append(identity, "InstallationID") values := map[string]any{ "PROTOCOL_VERSION": e2b.ProtocolVersion, "SDK_VERSION": sdk, - "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, - "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, - "OPERATIONS": e2b.HelperOperations(), "ERROR_CODES": e2b.HelperErrors(), + "SUSPEND_CONTROL_FILE": runtimebootstrap.SuspendControlFile, + "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, + "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, + "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, + "OPERATIONS": e2b.HelperOperations(), "ERROR_CODES": e2b.HelperErrors(), "REQUEST_FIELDS": fields(reflect.TypeFor[e2b.Request]()), "RESPONSE_FIELDS": fields(reflect.TypeFor[e2b.Response]()), "REFERENCE_FIELDS": fields(reflect.TypeFor[sandbox.Reference]()), + "COMPUTE_FIELDS": fields(reflect.TypeFor[sandbox.Compute]()), + "RETAINED_FIELDS": fields(reflect.TypeFor[sandbox.RetainedState]()), + "SUSPEND_FIELDS": fields(reflect.TypeFor[sandbox.SuspendRequest]()), + "RESUME_FIELDS": fields(reflect.TypeFor[sandbox.ResumeRequest]()), "MANAGED_BOOTSTRAP_FIELDS": bootstrap, "MANAGED_BOOTSTRAP_REQUIRED_FIELDS": requiredBootstrap, "MANAGED_IDENTITY_FIELDS": identity, "NETWORK_ACCESS": networkValues(filepath.Join(root, "internal/agentnetwork/policy.go")), } @@ -132,7 +138,24 @@ func fixtures() []byte { } for _, operation := range e2b.HelperOperations() { copy := q + copy.Operation = operation + c := sandbox.Compute{Name: r.AllocationID, ID: "native-fixture"} + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: installation, OperationID: installation, SourceName: c.Name, SourceID: c.ID, Data: "opaque"} + switch operation { + case "compute_info", "compute_renew", "compute_kill", "compute_command", "resume_compute": + copy.Compute = &c + if operation == "compute_command" { + copy.Command = &sandbox.Command{Args: []string{"true"}} + } + case "suspend": + copy.Suspend = &sandbox.SuspendRequest{Reference: r, OperationID: installation, Source: c} + case "resume": + target := sandbox.Compute{Generation: 1, Name: c.Name, ID: c.ID, RestoredFrom: &retained} + copy.Resume = &sandbox.ResumeRequest{Reference: r, OperationID: installation, Retained: retained, Target: target} + case "delete_retained": + copy.Retained = &retained + } if operation == "verify_credential" { copy.References = []sandbox.Reference{r} } @@ -157,6 +180,17 @@ func fixtures() []byte { value["Bootstrap"] = bootstrapValue add("request", fmt.Sprintf("workspace-%v", workspace), workspace == nil, value) } + for _, workspace := range []any{nil, map[string]any{}} { + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: installation, OperationID: installation, SourceName: r.AllocationID, SourceID: "native-fixture", Data: "opaque"} + copy := q + copy.Operation = "resume" + copy.Resume = &sandbox.ResumeRequest{Reference: r, OperationID: installation, Retained: retained, Target: sandbox.Compute{Generation: 1, Name: r.AllocationID, ID: "native-fixture", RestoredFrom: &retained}} + value := object(copy) + resume := object(copy.Resume) + resume["Workspace"] = workspace + value["Resume"] = resume + add("request", fmt.Sprintf("resume-workspace-%v", workspace), workspace == nil, value) + } for _, count := range []int{0, e2b.MaxCredentialReferences, e2b.MaxCredentialReferences + 1} { copy := q copy.Operation = "verify_credential" diff --git a/services/core/internal/sandbox/e2b/operations.go b/services/core/internal/sandbox/e2b/operations.go index a3da4688e..7bb90e068 100644 --- a/services/core/internal/sandbox/e2b/operations.go +++ b/services/core/internal/sandbox/e2b/operations.go @@ -1,11 +1,12 @@ package e2b import ( - "context" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) +var _ sandbox.SandboxProvider = (*Provider)(nil) + // Operations is this adapter's complete authored resource contract. func Operations() providercontract.Operations { return providercontract.Operations{ @@ -14,43 +15,17 @@ func Operations() providercontract.Operations { "Renew": {State: providercontract.Supported}, "Kill": {State: providercontract.Supported}, "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Suspend": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Resume": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "Initial": {State: providercontract.Supported}, + "NewCompute": {State: providercontract.Supported}, + "GetCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, + "Suspend": {State: providercontract.Supported}, + "Resume": {State: providercontract.Supported}, + "KillCompute": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, + "RunCommandCompute": {State: providercontract.Supported}, + "ResumeCompute": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, } } func (*Provider) ProviderOperations() providercontract.Operations { return Operations() } -func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} -} -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} -} -func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "GetCompute", Reason: Operations()["GetCompute"].Reason} -} -func (p *Provider) Suspend(context.Context, sandbox.SuspendRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Suspend", Reason: Operations()["Suspend"].Reason} -} -func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Resume", Reason: Operations()["Resume"].Reason} -} -func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { - return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} -} -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} -} -func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { - return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} -} -func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} -} diff --git a/services/core/internal/sandbox/e2b/process_test.go b/services/core/internal/sandbox/e2b/process_test.go index a2e90a5b4..695333885 100644 --- a/services/core/internal/sandbox/e2b/process_test.go +++ b/services/core/internal/sandbox/e2b/process_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -12,7 +13,7 @@ import ( ) func TestProcessWireRejectsUnknownAndTrailingOutput(t *testing.T) { - for _, output := range []string{`{"Version":2,"Secret":"private"}`, `{"Version":2} {"Version":2}`} { + for _, output := range []string{`{"Version":1,"Secret":"private"}`, `{"Version":1} {"Version":1}`} { root := t.TempDir() binary := filepath.Join(root, "helper") if err := os.WriteFile(binary, []byte("#!/bin/sh\nprintf '%s' '"+output+"'\n"), 0700); err != nil { @@ -29,7 +30,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { marker := filepath.Join(root, "settled") // The private test path contains no shell syntax; the real adapter never puts // credentials or request contents in argv or inherited environment. - script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":2}'\n" + script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -50,7 +51,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { func TestEnvironmentDropsProviderSelectorsAndCredentials(t *testing.T) { root := t.TempDir() binary := filepath.Join(root, "helper") - script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":2}'\n" + script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -67,7 +68,7 @@ func TestCredentialFenceWaitsForActualHelperExitAfterCancellation(t *testing.T) binary := filepath.Join(root, "helper") marker := filepath.Join(root, "started") finish := filepath.Join(root, "finish") - script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":2}'\n" + script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } diff --git a/services/core/internal/sandbox/e2b/provider.go b/services/core/internal/sandbox/e2b/provider.go index 81e74cdba..4c022070e 100644 --- a/services/core/internal/sandbox/e2b/provider.go +++ b/services/core/internal/sandbox/e2b/provider.go @@ -195,12 +195,20 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen } connection = &value } - out, err := p.caller.Call(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) + return p.callRequest(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) +} + +func (p *Provider) callRequest(ctx context.Context, q Request) (Response, error) { + operation, r := q.Operation, q.Reference + if q.Validate() != nil { + return Response{}, sandbox.ErrInvalid + } + out, err := p.caller.Call(ctx, q) if errors.Is(err, errHelperNotStarted) { return unstarted(operation, r), sandbox.ErrComputeUnconfirmed } if err != nil || out.Version != ProtocolVersion { - if operation == "command" { + if operation == "command" || operation == "compute_command" { return Response{}, sandbox.ErrCommandUnconfirmed } return Response{}, sandbox.ErrComputeUnconfirmed diff --git a/services/core/internal/sandbox/e2b/suspension.go b/services/core/internal/sandbox/e2b/suspension.go new file mode 100644 index 000000000..5237ea082 --- /dev/null +++ b/services/core/internal/sandbox/e2b/suspension.go @@ -0,0 +1,115 @@ +package e2b + +import ( + "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +func (p *Provider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { + if !validReference(r) { + return sandbox.Compute{}, sandbox.ErrInvalid + } + return sandbox.Compute{Name: r.AllocationID}, nil +} +func (p *Provider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, retained *sandbox.RetainedState) (sandbox.Compute, error) { + if !validReference(r) || retained == nil || sandbox.ValidateRetained(*retained) != nil || retained.Reference != r.AllocationID || retained.SourceName != r.AllocationID || generation == 0 || generation != retained.SourceGeneration+1 { + return sandbox.Compute{}, sandbox.ErrInvalid + } + value := *retained + return sandbox.Compute{Generation: generation, Name: r.AllocationID, ID: retained.SourceID, RestoredFrom: &value}, nil +} +func (p *Provider) computeCall(ctx context.Context, q Request) (Response, error) { + deadline, ok := ctx.Deadline() + if !ok || ctx.Err() != nil { + return Response{}, sandbox.ErrInvalid + } + q.Version, q.Config, q.Deadline = ProtocolVersion, p.config, deadline + return p.callRequest(ctx, q) +} +func (p *Provider) computeState(ctx context.Context, operation string, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + if c.Name != r.AllocationID { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + out, err := p.computeCall(ctx, Request{Operation: operation, Reference: r, Compute: &c}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State == nil || sandbox.ValidateComputeResult(c, out.State.Compute) != nil { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return *out.State, nil +} +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "compute_info", r, c) +} +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "compute_renew", r, c) +} +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "resume_compute", r, c) +} +func (p *Provider) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + if !validID(q.OperationID) || q.Source.Name != q.Reference.AllocationID || q.Source.ID == "" { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State == nil { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + if err = sandbox.ValidateSuspendResult(q, *out.State); err != nil { + return sandbox.ComputeState{}, err + } + if out.State.Retained != nil && out.State.Retained.Reference != q.Reference.AllocationID { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + return *out.State, nil +} +func (p *Provider) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + if q.Workspace != nil { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Resume", Reason: "external_workspace_unsupported"} + } + planned, err := p.NewCompute(ctx, q.Reference, q.Target.Generation, &q.Retained) + if err != nil || sandbox.ValidateComputeResult(planned, q.Target) != nil || !validID(q.OperationID) { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State == nil || sandbox.ValidateComputeResult(q.Target, out.State.Compute) != nil || out.State.Status != "running" || !out.State.BootstrapComplete { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return *out.State, nil +} +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { + if c.Name != r.AllocationID || c.ID == "" { + return sandbox.ErrInvalid + } + _, err := p.computeCall(ctx, Request{Operation: "compute_kill", Reference: r, Compute: &c}) + return err +} +func (p *Provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + if sandbox.ValidateRetained(s) != nil || s.Reference != r.AllocationID { + return sandbox.ErrInvalid + } + _, err := p.computeCall(ctx, Request{Operation: "delete_retained", Reference: r, Retained: &s}) + return err +} +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { + if c.Name != r.AllocationID || c.ID == "" { + return sandbox.CommandResult{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "compute_command", Reference: r, Compute: &c, Command: &command}) + if err != nil { + return sandbox.CommandResult{}, err + } + if out.Command == nil { + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + return *out.Command, nil +} diff --git a/services/core/internal/sandbox/e2b/suspension_test.go b/services/core/internal/sandbox/e2b/suspension_test.go new file mode 100644 index 000000000..d4a0c0c06 --- /dev/null +++ b/services/core/internal/sandbox/e2b/suspension_test.go @@ -0,0 +1,48 @@ +package e2b + +import ( + "errors" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" +) + +func TestSuspensionPlansSameNativeIDWithoutCallingHelper(t *testing.T) { + p, f, r := fixture(t) + initial, err := p.Initial(bounded(t), r) + if err != nil || initial.Name != r.AllocationID || initial.ID != "" { + t.Fatal(initial, err) + } + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: uuid.NewString(), OperationID: uuid.NewString(), SourceName: r.AllocationID, SourceID: "native-id", Data: "opaque"} + target, err := p.NewCompute(bounded(t), r, 1, &retained) + if err != nil || target.ID != retained.SourceID || target.Generation != 1 || *target.RestoredFrom != retained || len(f.requests) != 0 { + t.Fatal(target, err) + } + if _, err = p.NewCompute(bounded(t), r, 2, &retained); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("generation jump accepted", err) + } + retained.Reference = uuid.NewString() + if _, err = p.NewCompute(bounded(t), r, 1, &retained); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("foreign allocation accepted", err) + } +} +func TestSuspensionRejectsUnsettledAndForeignHelperOutcomes(t *testing.T) { + p, f, r := fixture(t) + current := sandbox.Compute{Name: r.AllocationID, ID: "native-id"} + q := sandbox.SuspendRequest{Reference: r, Source: current, OperationID: uuid.NewString()} + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: q.OperationID, OperationID: q.OperationID, SourceName: current.Name, SourceID: current.ID, Data: "opaque"} + f.response.State = &sandbox.ComputeState{Compute: current, Status: "suspended", BootstrapComplete: true, Retained: &retained, ResourcesReleased: true, SuspendSettled: true} + if _, err := p.Suspend(bounded(t), q); err != nil { + t.Fatal(err) + } + f.response.State.SuspendSettled = false + if _, err := p.Suspend(bounded(t), q); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { + t.Fatal(err) + } + f.response.State.SuspendSettled = true + f.response.State.Compute.Generation = 1 + if _, err := p.Suspend(bounded(t), q); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal(err) + } +} diff --git a/services/core/internal/sandbox/e2b/workspace_test.go b/services/core/internal/sandbox/e2b/workspace_test.go index 0f6ed46d2..5acc16786 100644 --- a/services/core/internal/sandbox/e2b/workspace_test.go +++ b/services/core/internal/sandbox/e2b/workspace_test.go @@ -15,3 +15,11 @@ func TestCreateExplicitlyRejectsExternalWorkspace(t *testing.T) { t.Fatal("external workspace not rejected before native access", err) } } + +func TestResumeExplicitlyRejectsExternalWorkspace(t *testing.T) { + provider := new(Provider) + _, err := provider.Resume(context.Background(), sandbox.ResumeRequest{Workspace: &workspacefs.Binding{}}) + if reason, ok := providercontract.UnsupportedReason(err, "Resume"); !ok || reason != "external_workspace_unsupported" { + t.Fatal("external workspace not rejected before native access", err) + } +} diff --git a/services/core/internal/sandbox/microsandbox/creation_settlement_test.go b/services/core/internal/sandbox/microsandbox/creation_settlement_test.go index 6fab743b3..f088e8308 100644 --- a/services/core/internal/sandbox/microsandbox/creation_settlement_test.go +++ b/services/core/internal/sandbox/microsandbox/creation_settlement_test.go @@ -150,7 +150,7 @@ func TestOrdinaryComputeInspectionDoesNotRequestInitialSettlement(t *testing.T) if err != nil { t.Fatal(err) } - _, err = provider.GetCompute(deadline(t), ref, Compute{Name: Name(config, ref, 0)}) + _, err = provider.nativeGetCompute(deadline(t), ref, Compute{Name: Name(config, ref, 0)}) if !errors.Is(err, sandbox.ErrNotFound) { t.Fatal(err) } diff --git a/services/core/internal/sandbox/microsandbox/operations.go b/services/core/internal/sandbox/microsandbox/operations.go index 13e81ff96..e3742747e 100644 --- a/services/core/internal/sandbox/microsandbox/operations.go +++ b/services/core/internal/sandbox/microsandbox/operations.go @@ -12,11 +12,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Supported}, "NewCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "GetCompute": {State: providercontract.Supported}, "Suspend": {State: providercontract.Supported}, "Resume": {State: providercontract.Supported}, "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, diff --git a/services/core/internal/sandbox/microsandbox/process_test.go b/services/core/internal/sandbox/microsandbox/process_test.go index 68021cd2d..894568e83 100644 --- a/services/core/internal/sandbox/microsandbox/process_test.go +++ b/services/core/internal/sandbox/microsandbox/process_test.go @@ -335,7 +335,11 @@ identity={"installation_id":"test-installation","generation":1,"specification_di with g.collection_lease(Path(sys.argv[2]),1,i,identity,initialize=sys.argv[3]=="init"): pass ` run := func(mode string) error { - return exec.Command("python3", "-c", script, installer, root, mode, testLeaseIdentity().SpecificationDigest).Run() + output, err := exec.Command("python3", "-c", script, installer, root, mode, testLeaseIdentity().SpecificationDigest).CombinedOutput() + if err != nil { + t.Log(string(output)) + } + return err } if err := run("init"); err != nil { t.Fatal("Python initialization", err) diff --git a/services/core/internal/sandbox/microsandbox/provider.go b/services/core/internal/sandbox/microsandbox/provider.go index 0b9f62800..88ce87292 100644 --- a/services/core/internal/sandbox/microsandbox/provider.go +++ b/services/core/internal/sandbox/microsandbox/provider.go @@ -26,7 +26,7 @@ func NewWithCaller(c Config, caller Caller) (*Provider, error) { c.Network.Rules = append([]NetworkRule(nil), c.Network.Rules...) return &Provider{config: c, caller: caller}, nil } -func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (Compute, error) { +func (p *Provider) nativeInitial(ctx context.Context, r sandbox.Reference) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err } @@ -93,7 +93,7 @@ func (p *Provider) responseState(ctx context.Context, q Request, out Response) ( } want := q.Compute if q.Operation == "create" { - want, e = p.Initial(ctx, q.Reference) + want, e = p.nativeInitial(ctx, q.Reference) if e != nil { return State{}, e } @@ -166,7 +166,7 @@ func (p *Provider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Inf return result, err } func (p *Provider) GetInfo(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { - c, e := p.Initial(ctx, r) + c, e := p.nativeInitial(ctx, r) if e != nil { return sandbox.Info{}, e } @@ -185,34 +185,37 @@ func (p *Provider) Renew(ctx context.Context, r sandbox.Reference) (sandbox.Info return p.GetInfo(ctx, r) } func (p *Provider) Kill(ctx context.Context, r sandbox.Reference) error { - c, e := p.Initial(ctx, r) + c, e := p.nativeInitial(ctx, r) if e != nil { return e } - return p.KillCompute(ctx, r, c) + return p.nativeKillCompute(ctx, r, c) } func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { - compute, e := p.Initial(ctx, r) + compute, e := p.nativeInitial(ctx, r) if e != nil { return sandbox.CommandResult{}, e } - return p.RunCommandCompute(ctx, r, compute, c) + return p.nativeRunCommandCompute(ctx, r, compute, c) } -func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { +func (p *Provider) nativeGetCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { return p.state(ctx, Request{Operation: "inspect", Reference: r, Compute: c}) } -func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c Compute) error { +func (p *Provider) nativeKillCompute(ctx context.Context, r sandbox.Reference, c Compute) error { _, e := p.call(ctx, Request{Operation: "kill", Reference: r, Compute: c}) return e } -func (p *Provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s SnapshotIdentity) error { +func (p *Provider) nativeDeleteSnapshot(ctx context.Context, r sandbox.Reference, s SnapshotIdentity) error { _, e := p.call(ctx, Request{Operation: "delete_snapshot", Reference: r, Snapshot: &s}) return e } -func (p *Provider) Suspend(ctx context.Context, q SuspendRequest) (State, error) { +func (p *Provider) nativeSuspend(ctx context.Context, q SuspendRequest) (State, error) { return p.state(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}) } -func (p *Provider) Resume(ctx context.Context, q ResumeRequest) (State, error) { +func (p *Provider) nativeResume(ctx context.Context, q ResumeRequest) (State, error) { + if p.config.ExternalWorkspace != (q.Workspace != nil) { + return State{}, sandbox.ErrInvalid + } workspace, err := p.resolveWorkspace(ctx, q.Reference, q.Workspace) if err != nil { return State{}, err @@ -220,7 +223,7 @@ func (p *Provider) Resume(ctx context.Context, q ResumeRequest) (State, error) { q.Workspace = nil return p.state(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q, Workspace: workspace}) } -func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c Compute, command sandbox.Command) (sandbox.CommandResult, error) { +func (p *Provider) nativeRunCommandCompute(ctx context.Context, r sandbox.Reference, c Compute, command sandbox.Command) (sandbox.CommandResult, error) { out, e := p.call(ctx, Request{Operation: "command", Reference: r, Compute: c, Command: &command}) if e != nil { return sandbox.CommandResult{}, e @@ -233,11 +236,11 @@ func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c // ResumeCompute thaws the exact resident source after an aborted suspension. // It never starts stopped compute or restores a checkpoint. -func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { +func (p *Provider) nativeResumeCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { return p.state(ctx, Request{Operation: "resume_compute", Reference: r, Compute: c}) } -func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { +func (p *Provider) nativeNewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err } diff --git a/services/core/internal/sandbox/microsandbox/provider_test.go b/services/core/internal/sandbox/microsandbox/provider_test.go index 94bfa9881..42f955dc1 100644 --- a/services/core/internal/sandbox/microsandbox/provider_test.go +++ b/services/core/internal/sandbox/microsandbox/provider_test.go @@ -44,7 +44,7 @@ func TestRejectsChangedComputeIdentity(t *testing.T) { if e != nil { t.Fatal(e) } - _, e = p.GetCompute(deadline(t), r, Compute{Name: Name(c, r, 0), ID: "local:4"}) + _, e = p.nativeGetCompute(deadline(t), r, Compute{Name: Name(c, r, 0), ID: "local:4"}) if !errors.Is(e, sandbox.ErrOwnership) { t.Fatalf("foreign identity accepted: %v", e) } @@ -60,7 +60,7 @@ func TestRestoreMustRetainExactProvenance(t *testing.T) { got.RestoredFrom = &v return Response{Version: ProtocolVersion, State: &State{Compute: got, Status: "running", BootstrapComplete: true}}, nil })) - _, e := p.Resume(deadline(t), ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: target}) + _, e := p.nativeResume(deadline(t), ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: target}) if !errors.Is(e, sandbox.ErrOwnership) { t.Fatalf("different snapshot accepted: %v", e) } @@ -70,7 +70,7 @@ func TestRejectsSnapshotPathBeforeHelper(t *testing.T) { s.Reference = "/foreign/checkpoint" calls := 0 p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) - if e := p.DeleteSnapshot(deadline(t), r, s); !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { + if e := p.nativeDeleteSnapshot(deadline(t), r, s); !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { t.Fatalf("e=%v calls=%d", e, calls) } } @@ -83,7 +83,7 @@ func TestObserveOnlyPreservesCapturedButResidentState(t *testing.T) { } return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: "paused", Snapshot: &s}}, nil })) - got, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + got, e := p.nativeSuspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) if e != nil || got.SourceStopped || got.Status != "paused" { t.Fatalf("state=%+v error=%v", got, e) } @@ -117,11 +117,11 @@ func TestNoDeadlineOrForeignAllocationNeverCallsHelper(t *testing.T) { r := testRef() foreign := r foreign.EnvironmentID = "77777777-7777-4777-8777-777777777777" - initial, initialErr := p.Initial(deadline(t), r) + initial, initialErr := p.nativeInitial(deadline(t), r) if initialErr != nil { t.Fatal(initialErr) } - _, e = p.GetCompute(deadline(t), foreign, initial) + _, e = p.nativeGetCompute(deadline(t), foreign, initial) if !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { t.Fatalf("e=%v calls=%d", e, calls) } @@ -146,7 +146,7 @@ func TestMissingSnapshotObservationAllowsOnlyIntactSourceRollback(t *testing.T) p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: status, BootstrapComplete: true}}, nil })) - _, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + _, e := p.nativeSuspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) if status == "running" || status == "paused" { if e != nil { t.Fatal(e) diff --git a/services/core/internal/sandbox/microsandbox/suspension.go b/services/core/internal/sandbox/microsandbox/suspension.go new file mode 100644 index 000000000..46954f196 --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/suspension.go @@ -0,0 +1,146 @@ +package microsandbox + +import ( + "context" + "encoding/json" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// retained translates a verified native snapshot without exposing its schema to Core. +func retained(s SnapshotIdentity) sandbox.RetainedState { + raw, _ := json.Marshal(s) + return sandbox.RetainedState{Reference: s.Reference, ID: s.ID, OperationID: s.OperationID, SourceGeneration: s.SourceGeneration, SourceName: s.SourceName, SourceID: s.SourceID, Data: string(raw)} +} +func (p *Provider) snapshot(r sandbox.Reference, s sandbox.RetainedState) (SnapshotIdentity, error) { + var native SnapshotIdentity + if sandbox.ValidateRetained(s) != nil || json.Unmarshal([]byte(s.Data), &native) != nil || retained(native) != s || ValidateSnapshot(p.config, r, native) != nil { + return native, sandbox.ErrOwnership + } + return native, nil +} +func compute(c Compute) sandbox.Compute { + out := sandbox.Compute{Generation: c.Generation, Name: c.Name, ID: c.ID} + if c.RestoredFrom != nil { + s := retained(*c.RestoredFrom) + out.RestoredFrom = &s + } + return out +} +func (p *Provider) nativeCompute(r sandbox.Reference, c sandbox.Compute) (Compute, error) { + out := Compute{Generation: c.Generation, Name: c.Name, ID: c.ID} + if c.RestoredFrom != nil { + s, e := p.snapshot(r, *c.RestoredFrom) + if e != nil { + return out, e + } + out.RestoredFrom = &s + } + if ValidateCompute(p.config, r, out) != nil { + return out, sandbox.ErrOwnership + } + return out, nil +} +func state(s State) sandbox.ComputeState { + out := sandbox.ComputeState{Compute: compute(s.Compute), Status: s.Status, BootstrapComplete: s.BootstrapComplete, ResourcesReleased: s.SourceStopped} + if s.Snapshot != nil { + v := retained(*s.Snapshot) + out.Retained = &v + } + return out +} +func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Compute, error) { + c, e := p.nativeInitial(ctx, r) + return compute(c), e +} +func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.RetainedState) (sandbox.Compute, error) { + var snap *SnapshotIdentity + if s != nil { + v, e := p.snapshot(r, *s) + if e != nil { + return sandbox.Compute{}, e + } + snap = &v + } + c, e := p.nativeNewCompute(ctx, r, g, snap) + return compute(c), e +} +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.ComputeState{}, e + } + s, e := p.nativeGetCompute(ctx, r, n) + return state(s), e +} +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.GetCompute(ctx, r, c) +} +func (p *Provider) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + c, e := p.nativeCompute(q.Reference, q.Source) + if e != nil { + return sandbox.ComputeState{}, e + } + n := SuspendRequest{Reference: q.Reference, OperationID: q.OperationID, Source: c, ObserveOnly: q.ReconcileOnly} + if q.Retained != nil { + s, e := p.snapshot(q.Reference, *q.Retained) + if e != nil { + return sandbox.ComputeState{}, e + } + n.Snapshot = &s + } + s, e := p.nativeSuspend(ctx, n) + if e == nil && s.Snapshot != nil { + // Complete the same exact-source cleanup previously performed by Core. + // Native KillCompute still verifies ownership and removes its disks. + e = p.nativeKillCompute(ctx, q.Reference, c) + if e == nil { + s.SourceStopped = true + s.Status = "suspended" + } + } + out := state(s) + // The helper holds the allocation lock until the original native work settles. + out.SuspendSettled = e == nil + return out, e +} +func (p *Provider) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + c, e := p.nativeCompute(q.Reference, q.Target) + if e != nil { + return sandbox.ComputeState{}, e + } + s, e := p.snapshot(q.Reference, q.Retained) + if e != nil { + return sandbox.ComputeState{}, e + } + v, e := p.nativeResume(ctx, ResumeRequest{Workspace: q.Workspace, Reference: q.Reference, OperationID: q.OperationID, Snapshot: s, Target: c, ObserveOnly: q.ReconcileOnly}) + return state(v), e +} +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { + n, e := p.nativeCompute(r, c) + if e != nil { + return e + } + return p.nativeKillCompute(ctx, r, n) +} +func (p *Provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + n, e := p.snapshot(r, s) + if e != nil { + return e + } + return p.nativeDeleteSnapshot(ctx, r, n) +} +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, q sandbox.Command) (sandbox.CommandResult, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.CommandResult{}, e + } + return p.nativeRunCommandCompute(ctx, r, n, q) +} +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.ComputeState{}, e + } + v, e := p.nativeResumeCompute(ctx, r, n) + return state(v), e +} diff --git a/services/core/internal/sandbox/microsandbox/suspension_contract_test.go b/services/core/internal/sandbox/microsandbox/suspension_contract_test.go new file mode 100644 index 000000000..284bc2159 --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/suspension_contract_test.go @@ -0,0 +1,57 @@ +package microsandbox + +import ( + "context" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "testing" +) + +func TestSharedSuspendSettlesExactSourceCleanupInsideAdapter(t *testing.T) { + for _, reconcile := range []bool{false, true} { + for _, cleanupFails := range []bool{false, true} { + c, r, snap := testConfig(), testRef(), testSnapshot() + source := Compute{Name: snap.SourceName, ID: snap.SourceID} + calls := []string{} + p, e := NewWithCaller(c, callerFunc(func(_ context.Context, q Request) (Response, error) { + calls = append(calls, q.Operation) + if q.Operation == "suspend" { + if q.Suspend.ObserveOnly != reconcile { + t.Fatal("reconcile intent changed") + } + return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: "suspended", BootstrapComplete: true, Snapshot: &snap, SourceStopped: !reconcile}}, nil + } + if q.Operation != "kill" || q.Compute.ID != source.ID { + t.Fatal("wrong source cleanup", q.Operation) + } + if cleanupFails { + return Response{}, errors.New("lost cleanup") + } + return Response{Version: ProtocolVersion}, nil + })) + if e != nil { + t.Fatal(e) + } + q := sandbox.SuspendRequest{Reference: r, OperationID: snap.OperationID, Source: compute(source), ReconcileOnly: reconcile} + got, e := p.Suspend(deadline(t), q) + if len(calls) != 2 || calls[0] != "suspend" || calls[1] != "kill" { + t.Fatal(calls) + } + if cleanupFails { + if e == nil || got.SuspendSettled { + t.Fatal("failed cleanup released capacity") + } + continue + } + if e != nil { + t.Fatal(e) + } + if e = sandbox.ValidateSuspendResult(q, got); e != nil { + t.Fatal(e) + } + if got.Retained == nil || got.Retained.Data == "" { + t.Fatal("missing native proof") + } + } + } +} diff --git a/services/core/internal/sandbox/microsandbox/types.go b/services/core/internal/sandbox/microsandbox/types.go index 0a04f9e53..a947b3b47 100644 --- a/services/core/internal/sandbox/microsandbox/types.go +++ b/services/core/internal/sandbox/microsandbox/types.go @@ -4,6 +4,7 @@ package microsandbox import ( "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -42,12 +43,53 @@ type NetworkPolicy struct { } type NetworkRule struct{ Action, Direction, Destination, Protocol, Port string } -// These aliases keep the helper wire private while Core uses provider-neutral types. -type Compute = sandbox.Compute -type SnapshotIdentity = sandbox.SnapshotIdentity -type State = sandbox.ComputeState -type SuspendRequest = sandbox.SuspendRequest -type ResumeRequest = sandbox.ResumeRequest +// Native snapshot wire types belong to this adapter. +type Compute struct { + Generation uint64 + Name string + ID string + RestoredFrom *SnapshotIdentity +} + +// SnapshotIdentity is provider evidence from a verified full snapshot. Core +// persists it unchanged and records consumption separately; it never invents +// paths, checksums, native checkpoint fields, or source identity. +type SnapshotIdentity struct { + Reference string + ID string + Digest string + CheckpointID string + CheckpointRoot string + OperationID string + SourceGeneration uint64 + SourceName string + SourceID string +} + +type State struct { + Compute Compute + Status string + BootstrapComplete bool + Snapshot *SnapshotIdentity + SourceStopped bool +} +type SuspendRequest struct { + Reference sandbox.Reference + OperationID string + Source Compute + Snapshot *SnapshotIdentity + // Recovery observes the previous attempt and never starts a new capture. + ObserveOnly bool +} +type ResumeRequest struct { + Workspace *workspacefs.Binding `json:",omitempty"` + Reference sandbox.Reference + OperationID string + Snapshot SnapshotIdentity + Target Compute + // Recovery observes the previous target and never starts a new restore. + ObserveOnly bool +} // Request and Response are the finite, private helper boundary. Confidential // Bootstrap and Command bytes travel only through stdin and are never logged. diff --git a/services/core/internal/sandbox/microsandbox/workspace_test.go b/services/core/internal/sandbox/microsandbox/workspace_test.go index f345df118..bc74b2aec 100644 --- a/services/core/internal/sandbox/microsandbox/workspace_test.go +++ b/services/core/internal/sandbox/microsandbox/workspace_test.go @@ -42,17 +42,17 @@ func TestWorkspaceResolveBeforeRestoreAndRetainPartialTarget(t *testing.T) { return workspacefs.Directory{Path: "/resolved/environment"}, nil }) q := ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: Compute{Generation: 1, Name: Name(c, r, 1), RestoredFrom: &s}, Workspace: workspaceBinding()} - state, err := p.Resume(deadline(t), q) + state, err := p.nativeResume(deadline(t), q) if !errors.Is(err, ErrUnconfirmed) || state.Compute.ID != "local:partial" || resolves != 1 || calls != 1 { t.Fatal(state, err, resolves, calls) } q.ObserveOnly = true - _, _ = p.Resume(deadline(t), q) + _, _ = p.nativeResume(deadline(t), q) if resolves != 2 { t.Fatal("restore observation did not resolve original binding") } q.Workspace.Attachment.Reference.EnvironmentID = r.TenantID - _, err = p.Resume(deadline(t), q) + _, err = p.nativeResume(deadline(t), q) if !errors.Is(err, workspacefs.ErrOwnership) || calls != 2 { t.Fatal("foreign binding reached helper", err, calls) } @@ -62,7 +62,7 @@ func TestWorkspaceDoesNotFallbackWithoutResolver(t *testing.T) { config := testConfig() config.ExternalWorkspace = true p, _ := NewWithCaller(config, callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) - _, err := p.Create(deadline(t), sandbox.Bootstrap{Harness: "codex", Reference: testRef(), SessionID: testRef().TenantID, DeviceID: testRef().EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()}) + _, err := p.Create(deadline(t), sandbox.Bootstrap{Reference: testRef(), SessionID: testRef().TenantID, DeviceID: testRef().EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "disabled", Workspace: workspaceBinding()}) if !errors.Is(err, workspacefs.ErrUnsupported) || calls != 0 { t.Fatal(err, calls) } @@ -81,7 +81,7 @@ func TestPreNativeWorkspaceFailureSettlesAbsentCreation(t *testing.T) { provider.workspace = workspaceResolverFunc(func(context.Context, workspacefs.Binding) (workspacefs.Directory, error) { return workspacefs.Directory{}, workspacefs.ErrUnavailable }) - bootstrap := sandbox.Bootstrap{Harness: "codex", Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()} + bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "disabled", Workspace: workspaceBinding()} ctx := deadline(t) switch failure { case "mode": diff --git a/services/core/internal/sandbox/node/generation_json.go b/services/core/internal/sandbox/node/generation_json.go index 661d177b2..75e85ea36 100644 --- a/services/core/internal/sandbox/node/generation_json.go +++ b/services/core/internal/sandbox/node/generation_json.go @@ -130,12 +130,12 @@ func validateGenerationJSON(raw []byte, kind string) error { } } if value := values["request"]; value != nil { - request, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume snapshot observation", "") + request, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume retained observation", "") if err != nil { return err } if bootstrap := request["bootstrap"]; bootstrap != nil { - if _, err := generationObject(bootstrap, "TenantID EnvironmentID AllocationID SessionID DeviceID CoreURL Credential Harness NetworkAccess AllowedDomains", "Workspace", "AllowedDomains"); err != nil { + if _, err := generationObject(bootstrap, "TenantID EnvironmentID AllocationID SessionID DeviceID CoreURL Credential Harness NetworkAccess AllowedDomains", "Workspace", "AllowedDomains Workspace"); err != nil { return err } } diff --git a/services/core/internal/sandbox/node/generation_json_test.go b/services/core/internal/sandbox/node/generation_json_test.go index e9f52cbf9..8fa769f8e 100644 --- a/services/core/internal/sandbox/node/generation_json_test.go +++ b/services/core/internal/sandbox/node/generation_json_test.go @@ -63,7 +63,7 @@ func TestGenerationHealthRejectsUnboundedOrAmbiguousNumbers(t *testing.T) { } func TestNodeProtocolRejectsHistoricalVersions(t *testing.T) { - for _, version := range []int{1, 2, 3, 4} { + for _, version := range []int{1, 2, 3, 4, 5, 6} { raw, _ := json.Marshal(frame{Version: version, Type: "hello", Identity: new(Identity), Health: &Health{ObservedAt: time.Now().UTC()}}) if _, err := decodeFrame(raw); err == nil { t.Fatalf("accepted historical protocol %d", version) @@ -106,7 +106,7 @@ func TestNodeBootstrapRejectsAmbiguousHarness(t *testing.T) { t.Fatal("ambiguous selection reached dispatch", replacement) } } - if _, err := decodeFrame([]byte(strings.Replace(good, `,"Harness":"codex"`, "", 1))); err == nil { + if _, err := decodeFrame([]byte(strings.Replace(good, `"Harness":"codex",`, "", 1))); err == nil { t.Fatal("missing Harness accepted") } } diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index a8a21fbce..1b21674ca 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -169,11 +169,11 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing } } -func TestOfflineIsUnknownAndDockerDoesNotAdvertiseCheckpoint(t *testing.T) { +func TestOfflineIsUnknownAndDockerDoesNotAdvertiseSuspension(t *testing.T) { h := NewHub(HubOptions{OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) p := h.Proxy(uuid.NewString(), docker.Operations(), 1) - if sandbox.SupportsCheckpoint(p) { - t.Fatal("docker advertised checkpoint") + if sandbox.SupportsSuspension(p) { + t.Fatal("docker advertised suspension") } _, err := p.GetInfo(context.Background(), reference()) if !errors.Is(err, sandbox.ErrComputeUnconfirmed) || errors.Is(err, sandbox.ErrNotFound) { diff --git a/services/core/internal/sandbox/node/operations.go b/services/core/internal/sandbox/node/operations.go index 8f016664b..aca02dcb5 100644 --- a/services/core/internal/sandbox/node/operations.go +++ b/services/core/internal/sandbox/node/operations.go @@ -10,10 +10,11 @@ var operationMethods = map[string]string{ "initial": "Initial", "new_compute": "NewCompute", "compute": "GetCompute", + "renew_compute": "RenewCompute", "suspend": "Suspend", "resume": "Resume", "kill_compute": "KillCompute", - "delete_snapshot": "DeleteSnapshot", + "delete_retained": "DeleteRetained", "command_compute": "RunCommandCompute", "resume_compute": "ResumeCompute", "observe": "Observe", diff --git a/services/core/internal/sandbox/node/operations_test.go b/services/core/internal/sandbox/node/operations_test.go index e508988b5..f0567ad1c 100644 --- a/services/core/internal/sandbox/node/operations_test.go +++ b/services/core/internal/sandbox/node/operations_test.go @@ -49,9 +49,63 @@ func TestUnsupportedProxyRejectsBeforeNodeResolution(t *testing.T) { } } func TestNodeOperationMappingCoversForwardedMethods(t *testing.T) { - for _, method := range []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand", "Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute", "Observe"} { + for _, method := range []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand", "Initial", "NewCompute", "GetCompute", "RenewCompute", "Suspend", "Resume", "KillCompute", "DeleteRetained", "RunCommandCompute", "ResumeCompute", "Observe"} { if wire := operationWire(method); wire == "" || operationMethod(wire) != method { t.Fatal(method) } } } + +// Retained handles and renew are part of the allocation protocol, including on +// nodes whose implementation embeds no optional lifecycle interface. +type retainedWireProvider struct { + fakeProvider + renewed sandbox.Compute + deleted sandbox.RetainedState +} + +func (p *retainedWireProvider) ProviderOperations() providercontract.Operations { + operations := p.fakeProvider.ProviderOperations() + operations["RenewCompute"] = providercontract.Support{State: providercontract.Supported} + operations["DeleteRetained"] = providercontract.Support{State: providercontract.Supported} + return operations +} +func (p *retainedWireProvider) RenewCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + p.renewed = c + return sandbox.ComputeState{Compute: c, Status: "running"}, nil +} +func (p *retainedWireProvider) DeleteRetained(_ context.Context, _ sandbox.Reference, retained sandbox.RetainedState) error { + p.deleted = retained + return nil +} +func TestRetainedLifecycleDispatchSurvivesStrictWire(t *testing.T) { + ref := reference() + retained := sandbox.RetainedState{Reference: ref.AllocationID, ID: uuid.NewString(), OperationID: uuid.NewString(), SourceName: ref.AllocationID, SourceID: "native", Data: "opaque"} + compute := sandbox.Compute{Generation: 1, Name: ref.AllocationID, ID: "native", RestoredFrom: &retained} + p := new(retainedWireProvider) + for _, operation := range []string{"renew_compute", "delete_retained"} { + q := request{DeploymentGeneration: 1, ID: uuid.NewString(), Sequence: 1, ConnectionID: uuid.NewString(), OwnerEpoch: 1, Operation: operation, TimeoutMillis: 1000, Reference: ref} + if operation == "renew_compute" { + q.Compute = &compute + } else { + q.Retained = &retained + } + raw, err := json.Marshal(frame{Version: ProtocolVersion, Type: "request", Request: &q}) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeFrame(raw) + if err != nil { + t.Fatal(err) + } + if err := decoded.Request.validate(); err != nil { + t.Fatal(err) + } + if out := execute(t.Context(), p, *decoded.Request); responseError(out) != nil { + t.Fatal(out) + } + } + if p.renewed.ID != compute.ID || p.renewed.RestoredFrom == nil || *p.renewed.RestoredFrom != retained || p.deleted != retained { + t.Fatal("retained evidence lost in transport") + } +} diff --git a/services/core/internal/sandbox/node/provider_operations_fixture_test.go b/services/core/internal/sandbox/node/provider_operations_fixture_test.go index 3f892e9bb..fdcb82c42 100644 --- a/services/core/internal/sandbox/node/provider_operations_fixture_test.go +++ b/services/core/internal/sandbox/node/provider_operations_fixture_test.go @@ -17,11 +17,12 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -30,7 +31,7 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { func (*fakeProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*fakeProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*fakeProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*fakeProvider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (*fakeProvider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Com func (*fakeProvider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*fakeProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*fakeProvider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*fakeProvider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -66,13 +67,18 @@ func (*observationProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Supported}, } } + +func (*fakeProvider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/sandbox/node/proxy.go b/services/core/internal/sandbox/node/proxy.go index e064f5b38..9271fb418 100644 --- a/services/core/internal/sandbox/node/proxy.go +++ b/services/core/internal/sandbox/node/proxy.go @@ -105,8 +105,8 @@ func (p *provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Co } return *out.Compute, nil } -func (p *provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.SnapshotIdentity) (sandbox.Compute, error) { - out, e := p.call(ctx, request{Operation: "new_compute", Reference: r, Generation: g, Snapshot: s}) +func (p *provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.RetainedState) (sandbox.Compute, error) { + out, e := p.call(ctx, request{Operation: "new_compute", Reference: r, Generation: g, Retained: s}) if e != nil { return sandbox.Compute{}, e } @@ -141,8 +141,8 @@ func (p *provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandb _, e := p.call(ctx, request{Operation: "kill_compute", Reference: r, Compute: &c}) return e } -func (p *provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s sandbox.SnapshotIdentity) error { - _, e := p.call(ctx, request{Operation: "delete_snapshot", Reference: r, Snapshot: &s}) +func (p *provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + _, e := p.call(ctx, request{Operation: "delete_retained", Reference: r, Retained: &s}) return e } func (p *provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, v sandbox.Command) (sandbox.CommandResult, error) { @@ -158,3 +158,7 @@ func (p *provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c san func (h *Hub) GenerationProvider(declared providercontract.Operations, resolve func(context.Context, sandbox.Reference) (string, uint64, error)) sandbox.SandboxProvider { return &provider{hub: h, operations: maps.Clone(declared), resolveGeneration: resolve} } + +func (p *provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, request{Operation: "renew_compute", Reference: r, Compute: &c}) +} diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index e7a9d80f2..58d51b97d 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -19,7 +19,7 @@ import ( "github.com/gorilla/websocket" ) -const ProtocolVersion = 6 +const ProtocolVersion = 7 const MaxControlFrameBytes = 32 * 1024 const MaxFrameBytes = 72 * 1024 * 1024 const maxPending = 32 @@ -87,15 +87,15 @@ type request struct { TimeoutMillis int64 `json:"timeout_ms"` // deadline is anchored to the receiving host and never crosses the wire. deadline time.Time - Reference sandbox.Reference `json:"reference"` - Bootstrap *sandbox.Bootstrap `json:"bootstrap,omitempty"` - Compute *sandbox.Compute `json:"compute,omitempty"` - Generation uint64 `json:"generation,omitempty"` - Command *sandbox.Command `json:"command,omitempty"` - Suspend *sandbox.SuspendRequest `json:"suspend,omitempty"` - Resume *sandbox.ResumeRequest `json:"resume,omitempty"` - Snapshot *sandbox.SnapshotIdentity `json:"snapshot,omitempty"` - Observation *runtimeobs.Target `json:"observation,omitempty"` + Reference sandbox.Reference `json:"reference"` + Bootstrap *sandbox.Bootstrap `json:"bootstrap,omitempty"` + Compute *sandbox.Compute `json:"compute,omitempty"` + Generation uint64 `json:"generation,omitempty"` + Command *sandbox.Command `json:"command,omitempty"` + Suspend *sandbox.SuspendRequest `json:"suspend,omitempty"` + Resume *sandbox.ResumeRequest `json:"resume,omitempty"` + Retained *sandbox.RetainedState `json:"retained,omitempty"` + Observation *runtimeobs.Target `json:"observation,omitempty"` } type response struct { @@ -244,7 +244,7 @@ func (q request) validate() error { return sandbox.ErrInvalid } count := 0 - for _, ok := range []bool{q.Bootstrap != nil, q.Compute != nil, q.Command != nil, q.Suspend != nil, q.Resume != nil, q.Snapshot != nil, q.Observation != nil} { + for _, ok := range []bool{q.Bootstrap != nil, q.Compute != nil, q.Command != nil, q.Suspend != nil, q.Resume != nil, q.Retained != nil, q.Observation != nil} { if ok { count++ } @@ -263,10 +263,10 @@ func (q request) validate() error { return nil } case "new_compute": - if count == 0 || count == 1 && q.Snapshot != nil { + if count == 0 || count == 1 && q.Retained != nil { return nil } - case "compute", "kill_compute", "resume_compute": + case "compute", "renew_compute", "kill_compute", "resume_compute": if count == 1 && q.Compute != nil { return nil } @@ -286,8 +286,8 @@ func (q request) validate() error { if count == 1 && q.Resume != nil && q.Resume.Reference == q.Reference && sandbox.ValidateWorkspaceBinding(q.Reference, q.Resume.Workspace) == nil { return nil } - case "delete_snapshot": - if count == 1 && q.Snapshot != nil { + case "delete_retained": + if count == 1 && q.Retained != nil { return nil } } @@ -337,11 +337,14 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response compute, err = p.Initial(ctx, q.Reference) out.Compute = &compute case "new_compute": - compute, err = p.NewCompute(ctx, q.Reference, q.Generation, q.Snapshot) + compute, err = p.NewCompute(ctx, q.Reference, q.Generation, q.Retained) out.Compute = &compute case "compute": state, err = p.GetCompute(ctx, q.Reference, *q.Compute) out.State = &state + case "renew_compute": + state, err = p.RenewCompute(ctx, q.Reference, *q.Compute) + out.State = &state case "suspend": state, err = p.Suspend(ctx, *q.Suspend) out.State = &state @@ -350,8 +353,8 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response out.State = &state case "kill_compute": err = p.KillCompute(ctx, q.Reference, *q.Compute) - case "delete_snapshot": - err = p.DeleteSnapshot(ctx, q.Reference, *q.Snapshot) + case "delete_retained": + err = p.DeleteRetained(ctx, q.Reference, *q.Retained) case "resume_compute": state, err = p.ResumeCompute(ctx, q.Reference, *q.Compute) out.State = &state @@ -382,7 +385,7 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response } func requiresReady(q request) bool { - return q.Operation == "create" || q.Operation == "resume" && q.Resume != nil && !q.Resume.ObserveOnly + return q.Operation == "create" || q.Operation == "resume" && q.Resume != nil && !q.Resume.ReconcileOnly } // setTimeout consumes sender queue time without comparing clocks across hosts. diff --git a/services/core/internal/sandbox/operations_test.go b/services/core/internal/sandbox/operations_test.go index 62d2c5b3f..c8f494397 100644 --- a/services/core/internal/sandbox/operations_test.go +++ b/services/core/internal/sandbox/operations_test.go @@ -24,7 +24,7 @@ func TestDeclarationsRejectMissingUnknownAndContradictoryOperations(t *testing.T name string change func(providercontract.Operations) }{ - {"omitted", func(o providercontract.Operations) { delete(o, "DeleteSnapshot") }}, + {"omitted", func(o providercontract.Operations) { delete(o, "DeleteRetained") }}, {"zero", func(o providercontract.Operations) { o["Observe"] = providercontract.Support{} }}, {"unknown", func(o providercontract.Operations) { o["FutureOperation"] = providercontract.Support{State: providercontract.Supported} diff --git a/services/core/internal/sandbox/providers/registration.go b/services/core/internal/sandbox/providers/registration.go index 5a5668d35..eb7c455a7 100644 --- a/services/core/internal/sandbox/providers/registration.go +++ b/services/core/internal/sandbox/providers/registration.go @@ -55,11 +55,6 @@ func ValidateRegistration(a Adapter) error { if err := sandbox.ValidateOperations(operations); err != nil { return err } - // The common lifecycle suspends only node allocations, so only a nodes - // registration may declare checkpoint support. - if operations["Initial"].State == providercontract.Supported && a.Mode != sandbox.DeploymentNodes { - return invalid("checkpoint support outside nodes mode") - } if a.Policy.DefaultResources != nil && a.ValidateResources(*a.Policy.DefaultResources) != nil { return invalid("default resources") } diff --git a/services/core/internal/sandbox/providers/registration_test.go b/services/core/internal/sandbox/providers/registration_test.go index 592bdf313..e5c20bc80 100644 --- a/services/core/internal/sandbox/providers/registration_test.go +++ b/services/core/internal/sandbox/providers/registration_test.go @@ -173,12 +173,12 @@ func TestRegistrationRejectsInvalidDefaultResources(t *testing.T) { } } -// Only a nodes registration may declare checkpoint support. -func TestRegistrationCheckpointRequiresNodes(t *testing.T) { +// Complete suspension support is independent of deployment placement. +func TestRegistrationSuspensionSupportsDirect(t *testing.T) { registry := Builtin() a := registry.adapters["microsandbox"] a.Mode, a.BuildLocal, a.NodeArtifacts, a.BuildDirect = "direct", nil, nil, registry.adapters["e2b"].BuildDirect - if err := ValidateRegistration(a); !errors.Is(err, providercontract.ErrContract) || !strings.Contains(err.Error(), "checkpoint") { - t.Fatal("direct checkpoint registration accepted", err) + if err := ValidateRegistration(a); err != nil { + t.Fatal("direct suspension registration rejected", err) } } diff --git a/services/core/internal/sandbox/providers/registry.go b/services/core/internal/sandbox/providers/registry.go index a39c0429f..bd13da340 100644 --- a/services/core/internal/sandbox/providers/registry.go +++ b/services/core/internal/sandbox/providers/registry.go @@ -88,8 +88,8 @@ func (r *Registry) BuildDirect(c sandbox.DirectConfig) (sandbox.SandboxProvider, return p, nil } -// SupportsCheckpoint reports whether the provider declares checkpoint suspension. -func (r *Registry) SupportsCheckpoint(kind string) (bool, error) { +// SupportsSuspension reports whether the provider declares checkpoint suspension. +func (r *Registry) SupportsSuspension(kind string) (bool, error) { a, err := r.Lookup(kind) if err != nil { return false, err diff --git a/services/core/internal/sandbox/providers/registry_test.go b/services/core/internal/sandbox/providers/registry_test.go index 47a0629bb..3edefa87e 100644 --- a/services/core/internal/sandbox/providers/registry_test.go +++ b/services/core/internal/sandbox/providers/registry_test.go @@ -19,7 +19,7 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { }{ {"docker", "nodes", "nodes", false}, {"microsandbox", "nodes", "nodes", true}, - {"e2b", "direct", "e2b", false}, + {"e2b", "direct", "e2b", true}, } { t.Run(tc.kind, func(t *testing.T) { d, err := registry.Describe(tc.kind, installation) @@ -27,7 +27,7 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { t.Fatalf("wrong mode or namespace: %+v %v", d, err) } a, err := registry.Lookup(tc.kind) - checkpoint, checkpointErr := registry.SupportsCheckpoint(tc.kind) + checkpoint, checkpointErr := registry.SupportsSuspension(tc.kind) if err != nil || checkpointErr != nil || checkpoint != tc.checkpoint || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { t.Fatal("inconsistent construction/capability registration", err, checkpointErr) } @@ -70,7 +70,7 @@ func TestNewRegistrationDoesNotNeedCoreDispatchChanges(t *testing.T) { // Registration is test-local: production registrations are fixed, never plugins. registry.adapters[kind] = registry.adapters["docker"] s, err := registry.Normalize(sandbox.Selection{Provider: kind, DeploymentSpec: validRegistrationSpec()}) - checkpoint, checkpointErr := registry.SupportsCheckpoint(kind) + checkpoint, checkpointErr := registry.SupportsSuspension(kind) if err != nil || checkpointErr != nil || s.Provider != kind || checkpoint { t.Fatal("new entry did not follow shared boundary", err, checkpointErr) } @@ -111,8 +111,8 @@ func TestCapabilityLookupsReportFailures(t *testing.T) { invalid.Operations = nil registry.adapters["invalid-registration"] = invalid for kind, want := range map[string]error{"unregistered": ErrUnknownProvider, "invalid-registration": providercontract.ErrContract} { - if _, err := registry.SupportsCheckpoint(kind); !errors.Is(err, want) { - t.Fatal(kind, "SupportsCheckpoint", err) + if _, err := registry.SupportsSuspension(kind); !errors.Is(err, want) { + t.Fatal(kind, "SupportsSuspension", err) } if _, err := registry.RetainedLimit(kind, 1, 2); !errors.Is(err, want) { t.Fatal(kind, "RetainedLimit", err) diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index e5770790e..b95182b6e 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -52,10 +52,11 @@ type Reference struct{ TenantID, EnvironmentID, AllocationID string } type Bootstrap struct { Workspace *workspacefs.Binding `json:",omitempty"` + Harness string Reference - SessionID, DeviceID, CoreURL, Credential, Harness string - NetworkAccess string - AllowedDomains []string + SessionID, DeviceID, CoreURL, Credential string + NetworkAccess string + AllowedDomains []string } // Info describes compute only. Running does not establish daemon authentication, @@ -108,12 +109,13 @@ type SandboxProvider interface { // The checkpoint lifecycle supplies exact-incarnation operations; Worker and // Store remain the lifecycle owner. Its operations share one declaration. Initial(context.Context, Reference) (Compute, error) - NewCompute(context.Context, Reference, uint64, *SnapshotIdentity) (Compute, error) + NewCompute(context.Context, Reference, uint64, *RetainedState) (Compute, error) GetCompute(context.Context, Reference, Compute) (ComputeState, error) + RenewCompute(context.Context, Reference, Compute) (ComputeState, error) Suspend(context.Context, SuspendRequest) (ComputeState, error) Resume(context.Context, ResumeRequest) (ComputeState, error) KillCompute(context.Context, Reference, Compute) error - DeleteSnapshot(context.Context, Reference, SnapshotIdentity) error + DeleteRetained(context.Context, Reference, RetainedState) error RunCommandCompute(context.Context, Reference, Compute, Command) (CommandResult, error) // ResumeCompute thaws only the same resident instance after an aborted pause. ResumeCompute(context.Context, Reference, Compute) (ComputeState, error) @@ -126,9 +128,17 @@ type SandboxProvider interface { // requiredOperations are supported by every Provider. var requiredOperations = []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand"} -// checkpointOperations are all supported or all unsupported: partial cleanup or +// suspensionOperations are all supported or all unsupported: partial cleanup or // restore support cannot safely own a compute incarnation. -var checkpointOperations = []string{"Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute"} +var suspensionOperations = []string{"Initial", "NewCompute", "GetCompute", "RenewCompute", "Suspend", "Resume", "KillCompute", "DeleteRetained", "RunCommandCompute", "ResumeCompute"} + +// ValidateRetained checks the shared envelope; only its adapter interprets Data. +func ValidateRetained(s RetainedState) error { + if s.Reference == "" || s.ID == "" || s.OperationID == "" || s.SourceID == "" || s.SourceName == "" || len(s.Data) == 0 || len(s.Data) > 64*1024 { + return ErrInvalid + } + return nil +} // Compute identifies one incarnation of an allocation. Name is provider-derived. // ID is empty only until the original create or restore result is observed. @@ -136,18 +146,15 @@ type Compute struct { Generation uint64 Name string ID string - RestoredFrom *SnapshotIdentity + RestoredFrom *RetainedState } -// SnapshotIdentity is provider evidence from a verified full snapshot. Core -// persists it unchanged and records consumption separately; it never invents -// paths, checksums, native checkpoint fields, or source identity. -type SnapshotIdentity struct { +// RetainedState is adapter-owned recoverable state. Data is opaque to Core. +// A retained state does not imply an independent snapshot. +type RetainedState struct { Reference string ID string - Digest string - CheckpointID string - CheckpointRoot string + Data string OperationID string SourceGeneration uint64 SourceName string @@ -158,27 +165,67 @@ type ComputeState struct { Compute Compute Status string BootstrapComplete bool - Snapshot *SnapshotIdentity - SourceStopped bool + Retained *RetainedState + ResourcesReleased bool + SuspendSettled bool } type SuspendRequest struct { Reference Reference OperationID string Source Compute - Snapshot *SnapshotIdentity - // Recovery observes the previous attempt and never starts a new capture. - ObserveOnly bool + Retained *RetainedState + // Recovery settles the previous attempt without another capture. + // Ownership-verified cleanup of a durable retained artifact may complete. + ReconcileOnly bool } type ResumeRequest struct { Workspace *workspacefs.Binding `json:",omitempty"` Reference Reference OperationID string - Snapshot SnapshotIdentity + Retained RetainedState Target Compute // Recovery observes the previous target and never starts a new restore. - ObserveOnly bool + ReconcileOnly bool +} + +// ValidateComputeResult binds an observation to its precommitted incarnation. +func ValidateComputeResult(want, got Compute) error { + if got.ID == "" || got.Name != want.Name || got.Generation != want.Generation || (want.ID != "" && got.ID != want.ID) || (want.RestoredFrom == nil) != (got.RestoredFrom == nil) { + return ErrOwnership + } + if want.RestoredFrom != nil && *want.RestoredFrom != *got.RestoredFrom { + return ErrOwnership + } + return nil +} + +// ValidateSuspendResult distinguishes settled rollback from uncertain native work. +func ValidateSuspendResult(q SuspendRequest, s ComputeState) error { + if ValidateComputeResult(q.Source, s.Compute) != nil { + return ErrOwnership + } + if !s.SuspendSettled || !s.BootstrapComplete { + return ErrComputeUnconfirmed + } + if s.Retained == nil { + if !q.ReconcileOnly || s.ResourcesReleased || (s.Status != "running" && s.Status != "paused") { + return ErrComputeUnconfirmed + } + return nil + } + v := s.Retained + if ValidateRetained(*v) != nil || v.OperationID != q.OperationID || v.SourceID != q.Source.ID || v.SourceName != q.Source.Name || v.SourceGeneration != q.Source.Generation || (q.Retained != nil && *q.Retained != *v) { + return ErrOwnership + } + if !s.ResourcesReleased || s.Status != "suspended" { + return ErrComputeUnconfirmed + } + return nil } +// SuspensionStateVersion fences incompatible durable lifecycle shapes. +const SuspensionStateVersion = "1" + // ValidateProvider checks a constructed Provider's declaration. func ValidateProvider(p SandboxProvider) error { if p == nil { @@ -213,15 +260,15 @@ func ValidateOperations(operations providercontract.Operations) error { return fmt.Errorf("%w: required operation %s", providercontract.ErrContract, name) } } - for _, name := range checkpointOperations { + for _, name := range suspensionOperations { if operations[name].State != operations["Initial"].State { - return fmt.Errorf("%w: incomplete checkpoint lifecycle", providercontract.ErrContract) + return fmt.Errorf("%w: incomplete suspension lifecycle", providercontract.ErrContract) } } return nil } -func SupportsCheckpoint(p SandboxProvider) bool { +func SupportsSuspension(p SandboxProvider) bool { return providercontract.Require(p, "Initial") == nil } diff --git a/services/core/internal/sandbox/sandbox_provider_test.go b/services/core/internal/sandbox/sandbox_provider_test.go index 56c23b330..59806b0f3 100644 --- a/services/core/internal/sandbox/sandbox_provider_test.go +++ b/services/core/internal/sandbox/sandbox_provider_test.go @@ -16,7 +16,7 @@ func TestOperationGroupsPartitionTheInterface(t *testing.T) { methods = append(methods, name) } } - groups := slices.Concat(requiredOperations, checkpointOperations, []string{"Observe"}) + groups := slices.Concat(requiredOperations, suspensionOperations, []string{"Observe"}) slices.Sort(groups) if !slices.Equal(methods, groups) { t.Fatalf("SandboxProvider operations %v, groups %v", methods, groups) diff --git a/services/core/internal/sandbox/suspension_test.go b/services/core/internal/sandbox/suspension_test.go new file mode 100644 index 000000000..add0fb28b --- /dev/null +++ b/services/core/internal/sandbox/suspension_test.go @@ -0,0 +1,55 @@ +package sandbox + +import ( + "strings" + "testing" +) + +func TestSuspensionRequiresBoundSettledResourceRelease(t *testing.T) { + c := Compute{ID: "native", Name: "source", Generation: 2} + r := RetainedState{Reference: "allocation", ID: "retained", OperationID: "op", SourceGeneration: 2, SourceName: "source", SourceID: "native", Data: "private-proof"} + q := SuspendRequest{OperationID: "op", Source: c} + valid := ComputeState{Compute: c, Status: "suspended", BootstrapComplete: true, Retained: &r, ResourcesReleased: true, SuspendSettled: true} + if err := ValidateSuspendResult(q, valid); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + change func(*ComputeState) + }{ + {"unsettled", func(s *ComputeState) { s.SuspendSettled = false }}, + {"capacity still held", func(s *ComputeState) { s.ResourcesReleased = false }}, + {"running", func(s *ComputeState) { s.Status = "running" }}, + {"foreign source", func(s *ComputeState) { s.Compute.ID = "foreign" }}, + {"foreign handle", func(s *ComputeState) { v := *s.Retained; v.OperationID = "other"; s.Retained = &v }}, + {"oversized", func(s *ComputeState) { v := *s.Retained; v.Data = strings.Repeat("x", 65537); s.Retained = &v }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := valid + tc.change(&s) + if ValidateSuspendResult(q, s) == nil { + t.Fatal("invalid suspension accepted") + } + }) + } + rollback := ComputeState{Compute: c, Status: "running", BootstrapComplete: true, SuspendSettled: true} + if ValidateSuspendResult(q, rollback) == nil { + t.Fatal("fresh dispatch accepted rollback") + } + q.ReconcileOnly = true + if err := ValidateSuspendResult(q, rollback); err != nil { + t.Fatal(err) + } + rollback.SuspendSettled = false + if ValidateSuspendResult(q, rollback) == nil { + t.Fatal("running observation inferred settlement") + } +} +func TestComputeResultFencesSameNativeIDAcrossGenerations(t *testing.T) { + old := Compute{ID: "same-native", Name: "allocation", Generation: 1} + next := old + next.Generation++ + if ValidateComputeResult(old, next) == nil { + t.Fatal("stale logical generation accepted") + } +} diff --git a/services/core/tests/integration/admin_session_archive_worker_http_test.go b/services/core/tests/integration/admin_session_archive_worker_http_test.go index fb536b9e3..ac3b6761c 100644 --- a/services/core/tests/integration/admin_session_archive_worker_http_test.go +++ b/services/core/tests/integration/admin_session_archive_worker_http_test.go @@ -126,7 +126,7 @@ func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { t.Fatal("archive did not commit administrator audit", audits, err) } stop() - // Snapshot after shutdown: cancellation and lifecycle draining are complete. + // Retained after shutdown: cancellation and lifecycle draining are complete. // A stale handler must not fall back to the still-open admission Store. snapshot := func() string { t.Helper() diff --git a/services/core/tests/integration/archived_cancellation_migration_test.go b/services/core/tests/integration/archived_cancellation_migration_test.go index 705a695d6..805bc4591 100644 --- a/services/core/tests/integration/archived_cancellation_migration_test.go +++ b/services/core/tests/integration/archived_cancellation_migration_test.go @@ -22,6 +22,8 @@ func TestArchivedCancellationMigrationDoesNotAdoptOldRevocations(t *testing.T) { if _, err := deploymentExecution(t, w).ArchiveSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 1); err != nil { t.Fatal(err) } + // Exercise the historical migration with the target schema's disabled + // suspension policy, independently of the current provider defaults. db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() provider, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) diff --git a/services/core/tests/integration/provider_operations_fixture_test.go b/services/core/tests/integration/provider_operations_fixture_test.go index f528d5ae0..7f207632c 100644 --- a/services/core/tests/integration/provider_operations_fixture_test.go +++ b/services/core/tests/integration/provider_operations_fixture_test.go @@ -17,11 +17,12 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -30,7 +31,7 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { func (*lifecycleProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*lifecycleProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*lifecycleProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*lifecycleProvider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (*lifecycleProvider) Resume(context.Context, sandbox.ResumeRequest) (sandbo func (*lifecycleProvider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*lifecycleProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*lifecycleProvider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*lifecycleProvider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -57,7 +58,7 @@ func (*lifecycleProvider) ResumeCompute(context.Context, sandbox.Reference, sand func (*lifecycleProvider) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } -func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations { +func (*fakeSuspensionProvider) ProviderOperations() providercontract.Operations { return providercontract.Operations{ "Create": {State: providercontract.Supported}, "GetInfo": {State: providercontract.Supported}, @@ -66,13 +67,18 @@ func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Supported}, "NewCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "GetCompute": {State: providercontract.Supported}, "Suspend": {State: providercontract.Supported}, "Resume": {State: providercontract.Supported}, "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, } } + +func (*lifecycleProvider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/tests/integration/provider_registration_migration_test.go b/services/core/tests/integration/provider_registration_migration_test.go index bbc4cfb1e..1f10e1aea 100644 --- a/services/core/tests/integration/provider_registration_migration_test.go +++ b/services/core/tests/integration/provider_registration_migration_test.go @@ -29,6 +29,8 @@ func TestProviderRegistrationDowngradePreservesCustomEndpoints(t *testing.T) { if _, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, input); err != nil { t.Fatal(err) } + // Exercise the historical migration with the target schema's disabled + // suspension policy, independently of the current provider defaults. db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() migrations, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) diff --git a/services/core/tests/integration/runtime_compute_lifecycle_test.go b/services/core/tests/integration/runtime_compute_lifecycle_test.go index 73f5e29d6..f188a5448 100644 --- a/services/core/tests/integration/runtime_compute_lifecycle_test.go +++ b/services/core/tests/integration/runtime_compute_lifecycle_test.go @@ -15,6 +15,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" + db "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" @@ -26,11 +27,11 @@ import ( // The controlled provider records external effects independently of DB phases. // Lost replies retain those effects so recovery must use observation, not replay. -type fakeCheckpointProvider struct { +type fakeSuspensionProvider struct { preparation *initializationPeer lifecycleProvider computes map[string]sandbox.ComputeState - snapshots map[string]sandbox.SnapshotIdentity + snapshots map[string]sandbox.RetainedState bootstraps map[string]sandbox.Bootstrap peers map[string]*websocket.Conn registry *runtimegateway.Registry @@ -41,15 +42,16 @@ type fakeCheckpointProvider struct { quiesces, resumes atomic.Int32 loseCapture, loseRestore, rejectQuiesce bool beforeQuiesce func() + renewals atomic.Int32 } -func (p *fakeCheckpointProvider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { +func (p *fakeSuspensionProvider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{Name: r.AllocationID + "-g0"}, nil } -func (p *fakeCheckpointProvider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, parent *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *fakeSuspensionProvider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, parent *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{Generation: generation, Name: fmt.Sprintf("%s-g%d", r.AllocationID, generation), RestoredFrom: parent}, nil } -func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { +func (p *fakeSuspensionProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { info, err := p.lifecycleProvider.Create(ctx, b) p.mu.Lock() defer p.mu.Unlock() @@ -59,7 +61,7 @@ func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap p.bootstraps[b.AllocationID] = b return info, err } -func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[c.Name] @@ -71,24 +73,28 @@ func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Referen } return state, nil } -func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[q.Source.Name] if !ok { + if snapshot, exists := p.snapshots[q.OperationID]; exists && q.ReconcileOnly { + p.captureObservations++ + return sandbox.ComputeState{Compute: q.Source, Status: "suspended", Retained: &snapshot, BootstrapComplete: true, ResourcesReleased: true, SuspendSettled: true}, nil + } return sandbox.ComputeState{}, sandbox.ErrNotFound } if state.Compute.ID != q.Source.ID { return sandbox.ComputeState{}, sandbox.ErrOwnership } - if q.ObserveOnly { + if q.ReconcileOnly { p.captureObservations++ } else { p.captures++ if _, exists := p.snapshots[q.OperationID]; exists { return sandbox.ComputeState{}, errors.New("capture replayed") } - p.snapshots[q.OperationID] = sandbox.SnapshotIdentity{Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Digest: "verified", CheckpointID: "checkpoint", CheckpointRoot: "private", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} + p.snapshots[q.OperationID] = sandbox.RetainedState{Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Data: "verified-native-state", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} state.Status = "paused" p.computes[q.Source.Name] = state if p.loseCapture { @@ -97,14 +103,20 @@ func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendReq } } if snapshot, exists := p.snapshots[q.OperationID]; exists { - state.Snapshot = &snapshot - } + state.Retained = &snapshot + state.SuspendSettled = true + state.ResourcesReleased = true + state.Status = "suspended" + delete(p.computes, q.Source.Name) + p.computeKills++ + } + state.SuspendSettled = true return state, nil } -func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() - if q.ObserveOnly { + if q.ReconcileOnly { p.restoreObservations++ state, ok := p.computes[q.Target.Name] if !ok { @@ -126,7 +138,7 @@ func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeReque } return state, nil } -func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { +func (p *fakeSuspensionProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[c.Name] @@ -140,7 +152,7 @@ func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Refere delete(p.computes, c.Name) return nil } -func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Reference, s sandbox.SnapshotIdentity) error { +func (p *fakeSuspensionProvider) DeleteRetained(_ context.Context, _ sandbox.Reference, s sandbox.RetainedState) error { p.mu.Lock() defer p.mu.Unlock() old, ok := p.snapshots[s.OperationID] @@ -154,7 +166,7 @@ func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Ref delete(p.snapshots, s.OperationID) return nil } -func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { state, err := p.GetCompute(ctx, r, c) if err != nil { return state, err @@ -165,7 +177,7 @@ func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Re p.computes[c.Name] = state return state, nil } -func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { +func (p *fakeSuspensionProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { if _, err := p.GetCompute(ctx, r, c); err != nil { return sandbox.CommandResult{}, err } @@ -178,7 +190,7 @@ func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbo p.mu.Unlock() return sandbox.CommandResult{}, p.connect(ctx, b) } -func (p *fakeCheckpointProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { +func (p *fakeSuspensionProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { header := http.Header{"Authorization": []string{"Bearer " + b.Credential}} conn, _, err := websocket.DefaultDialer.DialContext(ctx, p.endpoint+"?device_id="+b.DeviceID+"&version="+proto.Version, header) if err != nil { @@ -256,7 +268,7 @@ func (p *fakeCheckpointProvider) connect(ctx context.Context, b sandbox.Bootstra type computeLifecycleFixture struct { t *testing.T store *Store - provider *fakeCheckpointProvider + provider *fakeSuspensionProvider worker *execution.Worker stop func() key string @@ -265,10 +277,13 @@ type computeLifecycleFixture struct { } func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *computeLifecycleFixture { + return computeFixtureForMode(t, maxActive, maxRetained, false) +} +func computeFixtureForMode(t *testing.T, maxActive, maxRetained int, direct bool) *computeLifecycleFixture { t.Helper() s, _ := newManagedTestStore(t) registry := runtimegateway.NewRegistry() - p := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} + p := &fakeSuspensionProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.RetainedState{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Authenticator: runtimegateway.NewAuthenticator(sessionAdapter(s)), Registry: registry}) server := httptest.NewServer(http.HandlerFunc(handler.WS)) p.endpoint = "ws" + strings.TrimPrefix(server.URL, "http") @@ -280,12 +295,18 @@ func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *compu } server.Close() }) - f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, "microsandbox"), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour}} + kind := "microsandbox" + if direct { + kind = "e2b" + } + f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, kind), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour, MaxActive: maxActive, MaxRetained: maxRetained}} view, err := deploymentService(t, s).View(t.Context()) if err != nil { t.Fatal(err) } - f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID + if !direct { + f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID + } f.start() return f } @@ -294,7 +315,9 @@ func (f *computeLifecycleFixture) start() { t.Helper() w := startWebWorker(t, f.store, f.provider.registry, f.key, f.provider, &f.policy) // The Worker's claim starts a new owner epoch, in which the node reconnects. - onlineManagerNode(t, f.store, f.node) + if f.node != "" { + onlineManagerNode(t, f.store, f.node) + } var once sync.Once stop := func() { once.Do(func() { ctx, cancel := context.WithCancel(context.Background()); cancel(); _ = w.Run(ctx) }) @@ -365,7 +388,7 @@ func TestRuntimeComputeLifecycleIdleSuspendAndQueuedSameSessionWake(t *testing.T f.phase(tenant, env.ID, "running") } if f.provider.captures != 0 { - t.Fatal("never-used Session suspended") + t.Fatal("recently-created Session suspended") } completed := f.complete(owner) suspended := f.phase(tenant, env.ID, "suspended") @@ -389,6 +412,28 @@ func TestRuntimeComputeLifecycleIdleSuspendAndQueuedSameSessionWake(t *testing.T } } +func TestRuntimeComputeLifecycleUnusedSessionSuspendsAndExpires(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, session, env, owner := f.create() + f.sql(`UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 minutes' WHERE id=$1`, owner.ID) + suspended := f.phase(tenant, env.ID, "suspended") + if suspended.ComputeRetainedUntil == nil || f.provider.captures != 1 || f.provider.computeKills != 1 || len(f.provider.computes) != 0 || len(f.provider.snapshots) != 1 { + t.Fatal("unused Session did not suspend and release active compute") + } + var turns int + if err := f.store.pool.QueryRow(t.Context(), `SELECT count(*) FROM turns WHERE session_id=$1`, session.ID).Scan(&turns); err != nil || turns != 0 { + t.Fatal("unused Session gained a Turn", turns, err) + } + f.sql(`UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID) + reconcileManagedState(t, f.worker, f.store, tenant, env.ID, "released") + if len(f.provider.computes) != 0 || len(f.provider.snapshots) != 0 || f.provider.snapshotDeletes != 1 || f.provider.promptFrames.Load() != 0 { + t.Fatal("unused Session expiry retained resources or sent execution input") + } + if _, err := sessionAdapter(f.store).GetSession(t.Context(), tenant, session.ID); err != nil { + t.Fatal("resource cleanup removed Session history", err) + } +} + func TestRuntimeComputeLifecycleLostCaptureAndRestoreObserveWithoutReplay(t *testing.T) { f := newComputeLifecycleFixture(t, 1, 2) tenant, _, env, owner := f.create() @@ -514,3 +559,126 @@ func TestRuntimeComputeLifecycleSuspendedDeletionAndExpiryCleanup(t *testing.T) }) } } + +func TestRuntimeComputeLifecycleCapacityBoundsActiveAndRetained(t *testing.T) { + f := computeFixtureForMode(t, 1, 2, true) + tenant, _, env, owner := f.create() + tenant2, _, env2 := managedSession(t, f.store) + if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { + t.Fatalf("active capacity ignored: %v", err) + } + if f.provider.creates != 1 { + t.Fatal("capacity rejection allocated compute") + } + f.complete(owner) + f.phase(tenant, env.ID, "suspended") + second, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key) + if err != nil { + t.Fatal(err) + } + pending := f.queued(owner) + for range 4 { + f.worker.ReconcileManagedRuntimes(t.Context()) + } + first, err := deploymentStore(f.store).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}) + if err != nil || first.ComputePhase != "suspended" || f.provider.restores != 0 { + t.Fatal("wake exceeded active capacity", err) + } + f.sql(`UPDATE turns SET status='cancelled',completed_at=clock_timestamp() WHERE id=$1`, pending) + f.provider.mu.Lock() + b := f.provider.bootstraps[second.ID] + f.provider.mu.Unlock() + if err := f.provider.connect(t.Context(), b); err != nil { + t.Fatal(err) + } + second = f.phase(tenant2, env2.ID, "running") + f.complete(second) + f.phase(tenant2, env2.ID, "suspended") + // Both retained allocations count even when their source VMs are gone. + tenant3, _, env3 := managedSession(t, f.store) + if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant3, env3.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { + t.Fatalf("retained capacity ignored: %v", err) + } + if f.provider.creates != 2 { + t.Fatal("retained limit created a third allocation") + } +} + +func (p *fakeSuspensionProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + p.renewals.Add(1) + return p.GetCompute(ctx, r, c) +} + +func TestRuntimeComputeProtocolUpgradeRefusesOldReceiptsBeforeCleanup(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + f.complete(owner) + retained := f.phase(tenant, env.ID, "suspended") + f.stop() + for _, versioned := range []bool{false, true} { + f.sql(`UPDATE runtime_allocations SET compute_state=($2::jsonb-'retained') || jsonb_build_object('snapshot',$2::jsonb->'retained') WHERE id=$1`, owner.ID, retained.ComputeState) + if !versioned { + f.sql(`UPDATE runtime_allocations SET compute_state=compute_state-'protocol_version' WHERE id=$1`, owner.ID) + } + deletes := f.provider.snapshotDeletes + w, err := startNextWorker(t, t.Context(), f.store, &execution.Dispatcher{Registry: f.provider.registry, ManagedRuntimes: webRuntimes(t, f.store, f.key, f.provider, &f.policy)}) + if err == nil || w != nil || !strings.Contains(err.Error(), "previous release") { + t.Fatalf("incompatible state activated: %v", err) + } + if f.provider.snapshotDeletes != deletes || len(f.provider.snapshots) != 1 { + t.Fatal("upgrade lost owned artifact") + } + var state []byte + if err := f.store.pool.QueryRow(t.Context(), `SELECT compute_state FROM runtime_allocations WHERE id=$1`, owner.ID).Scan(&state); err != nil { + t.Fatal(err) + } + if !strings.Contains(string(state), `"snapshot"`) { + t.Fatal("old receipt was rewritten") + } + } + // A consumed legacy snapshot can survive only in current/target provenance. + for _, field := range []string{"current", "target"} { + f.sql(`UPDATE runtime_allocations SET compute_state=$2::jsonb || jsonb_build_object($3::text, jsonb_build_object('RestoredFrom', jsonb_build_object('Digest', 'legacy'))) WHERE id=$1`, owner.ID, retained.ComputeState, field) + incompatible, err := db.New(f.store.pool).HasIncompatibleRuntimeComputeState(t.Context(), sandbox.SuspensionStateVersion) + if err != nil || !incompatible { + t.Fatalf("legacy %s provenance accepted: incompatible=%v err=%v", field, incompatible, err) + } + } + // Adapter-owned data may contain any private field names. The activation + // fence must inspect only the shared envelope, never this opaque string. + f.sql(`UPDATE runtime_allocations SET compute_state=jsonb_set($2::jsonb, '{retained,Data}', to_jsonb($3::text)) WHERE id=$1`, owner.ID, retained.ComputeState, `{"snapshot":{"Digest":"private","CheckpointID":"native","CheckpointRoot":"owned"}}`) + incompatible, err := db.New(f.store.pool).HasIncompatibleRuntimeComputeState(t.Context(), sandbox.SuspensionStateVersion) + if err != nil || incompatible { + t.Fatalf("opaque native payload affected activation: incompatible=%v err=%v", incompatible, err) + } + f.sql(`UPDATE runtime_allocations SET compute_state=$2::jsonb WHERE id=$1`, owner.ID, retained.ComputeState) + f.start() +} + +func TestRuntimeComputeRepeatedWakeStillRenewsCurrentIncarnation(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + before := f.provider.renewals.Load() + for range 3 { + f.sql(`UPDATE runtime_allocations SET compute_wake_requested=true,compute_activity_at=clock_timestamp() WHERE id=$1`, owner.ID) + f.phase(tenant, env.ID, "running") + } + if f.provider.renewals.Load() < before+3 { + t.Fatal("wake requests bypassed native lease renewal") + } +} + +func TestRuntimeComputeProtocolRejectsOldDisabledAllocation(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + _, _, _, owner := f.create() + f.stop() + f.sql(`UPDATE runtime_allocations SET compute_phase='disabled',compute_state='{}'::jsonb WHERE id=$1`, owner.ID) + before := f.provider.renewals.Load() + w, err := startNextWorker(t, t.Context(), f.store, &execution.Dispatcher{Registry: f.provider.registry, ManagedRuntimes: webRuntimes(t, f.store, f.key, f.provider, &f.policy)}) + if err == nil || w != nil || !strings.Contains(err.Error(), "previous release") { + t.Fatal("old disabled allocation activated", err) + } + if f.provider.renewals.Load() != before || len(f.provider.computes) != 1 { + t.Fatal("upgrade changed owned native compute") + } +} diff --git a/services/core/tests/integration/runtime_idle_clock_test.go b/services/core/tests/integration/runtime_idle_clock_test.go index b792c00e5..d2cfdffc5 100644 --- a/services/core/tests/integration/runtime_idle_clock_test.go +++ b/services/core/tests/integration/runtime_idle_clock_test.go @@ -64,7 +64,7 @@ func verifyManagedIdleClock(t *testing.T, s, w *Store, owner deployment.Allocati observedBefore := runtimeDatabaseTime(t, s) activity, err := deploymentStore(w).Activity(t.Context(), owner.ID) observedAfter := runtimeDatabaseTime(t, s) - if err != nil || activity.ObservedAt.Before(observedBefore) || activity.ObservedAt.After(observedAfter) || activity.ReadyToSuspend(idleTimeout) || activity.LastActivity.Before(before) || activity.LastActivity.After(after) || activity.Busy || activity.WakeRequested || !activity.HasCompletedTurn { + if err != nil || activity.ObservedAt.Before(observedBefore) || activity.ObservedAt.After(observedAfter) || activity.ReadyToSuspend(idleTimeout) || activity.LastActivity.Before(before) || activity.LastActivity.After(after) || activity.Busy || activity.WakeRequested { t.Fatal("idle clock did not use committed terminal ingestion", activity, before, after, err) } until := runtimeDatabaseTime(t, s).Add(time.Hour) @@ -225,3 +225,24 @@ func TestManagedIdleClockReconnectPreservesReceipts(t *testing.T) { t.Fatal(err) } } + +func TestDirectManagedIdleClockIgnoresNativeClockSkew(t *testing.T) { + for _, skew := range []time.Duration{-269 * time.Second, 269 * time.Second} { + t.Run(skew.String(), func(t *testing.T) { + s, w, owner := managedIdleClockFixture(t) + runtimeSuspensionSQL(t, s.pool, "UPDATE runtime_allocations SET node_id=NULL WHERE id=$1", owner.ID) + owner.NodeID = "" + turn := uuid.NewString() + runtimeSuspensionSQL(t, s.pool, "INSERT INTO turns(id,session_id,status,started_at) VALUES($1,$2,'in_progress',clock_timestamp())", turn, owner.SessionID) + source := runtimeDatabaseTime(t, s).Add(skew).UnixMilli() + outcome := json.RawMessage(fmt.Sprintf(`{"done":{"source_completed_at_ms":%d}}`, source)) + before := runtimeDatabaseTime(t, s) + completed, err := sessionExecution(t, w.lease).CompleteExecution(t.Context(), owner.TenantID, owner.SessionID, turn, sessions.TurnCompleted, outcome, "", 0) + after := runtimeDatabaseTime(t, s) + if err != nil || completed.CompletedAt.UnixMilli() != source { + t.Fatal(completed, err) + } + verifyManagedIdleClock(t, s, w, owner, before, after) + }) + } +} diff --git a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go index 055ac6383..837cb4279 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go @@ -25,7 +25,7 @@ import ( ) type nodeIsolationProvider struct { - *fakeCheckpointProvider + *fakeSuspensionProvider blockMu sync.Mutex blocked map[string]bool mode string @@ -49,7 +49,7 @@ func (p *nodeIsolationProvider) block(ctx context.Context, r sandbox.Reference, return ctx.Err() } func (p *nodeIsolationProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { - info, err := p.fakeCheckpointProvider.Create(ctx, b) + info, err := p.fakeSuspensionProvider.Create(ctx, b) if err == nil { err = p.connect(ctx, b) } @@ -59,13 +59,13 @@ func (p *nodeIsolationProvider) GetInfo(ctx context.Context, r sandbox.Reference if err := p.block(ctx, r, "observe"); err != nil { return sandbox.Info{}, err } - return p.fakeCheckpointProvider.GetInfo(ctx, r) + return p.fakeSuspensionProvider.GetInfo(ctx, r) } func (p *nodeIsolationProvider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { if err := p.block(ctx, r, "observe"); err != nil { return sandbox.ComputeState{}, err } - return p.fakeCheckpointProvider.GetCompute(ctx, r, c) + return p.fakeSuspensionProvider.GetCompute(ctx, r, c) } func (p *nodeIsolationProvider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { return p.preparation.RunCommand(ctx, r, c) @@ -90,8 +90,8 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { t.Helper() s, pool := newManagedTestStore(t) registry := runtimegateway.NewRegistry() - cp := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} - p := &nodeIsolationProvider{fakeCheckpointProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} + cp := &fakeSuspensionProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.RetainedState{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} + p := &nodeIsolationProvider{fakeSuspensionProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} preparationContext, cancelPreparation := context.WithCancel(t.Context()) t.Cleanup(cancelPreparation) cp.preparation = &initializationPeer{t: t, apply: func(request proto.RuntimePreparePayload, data []byte) proto.RuntimePrepareResultPayload { diff --git a/services/core/tests/integration/runtime_suspension_test.go b/services/core/tests/integration/runtime_suspension_test.go index 1d5eabad8..3e2a9cf4c 100644 --- a/services/core/tests/integration/runtime_suspension_test.go +++ b/services/core/tests/integration/runtime_suspension_test.go @@ -64,7 +64,7 @@ func runtimeSuspensionStep(t *testing.T, w *Store, owner deployment.Allocation, return next } -func TestRuntimeSuspensionRequiresCompletedIdleAndNoPendingWork(t *testing.T) { +func TestRuntimeSuspensionRequiresIdleAndNoPendingWork(t *testing.T) { cases := []string{"no_completed_turn", "queued", "in_progress", "waiting", "subagent_queued", "subagent_in_progress", "subagent_waiting", "input_reservation", "file_write", "idle"} for _, kind := range cases { t.Run(kind, func(t *testing.T) { @@ -91,12 +91,12 @@ func TestRuntimeSuspensionRequiresCompletedIdleAndNoPendingWork(t *testing.T) { t.Fatal(err) } wantBusy := kind != "idle" && kind != "no_completed_turn" - if activity.Busy != wantBusy || activity.HasCompletedTurn != (kind != "no_completed_turn") { + if activity.Busy != wantBusy { t.Fatalf("activity lost pending work: %+v", activity) } until := time.Now().Add(time.Hour) _, err = deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Nanosecond) - if kind == "idle" { + if kind == "idle" || kind == "no_completed_turn" { if err != nil { t.Fatal(err) } @@ -261,17 +261,60 @@ func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { } } +func TestRuntimeSuspensionCountsUncertainCapacityUntilReleased(t *testing.T) { + s, provider := configuredStore(t) + pool := s.pool + w := executionWriter(t, s) + cases := []struct { + state, phase string + count bool + }{ + {"creating", "disabled", true}, {"running", "running", true}, {"running", "quiescing", true}, {"running", "suspending", true}, {"running", "suspended", false}, {"running", "restoring", true}, {"running", "waking", true}, {"cleanup_pending", "restoring", true}, {"released", "running", false}, + } + want := int64(0) + wantRetained := int64(0) + for _, item := range cases { + tenant := uuid.NewString() + _, env := localEnvironment(t, s, tenant) + owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}, provider, runtimedevice.HashCredential(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET state=$2,compute_phase=$3,create_settled=($2<>'creating'),released_at=CASE WHEN $2='released' THEN clock_timestamp() END WHERE id=$1`, owner.ID, item.state, item.phase) + if item.count { + want++ + } + if item.state != "released" { + wantRetained++ + } + retained, err := deploymentStore(w).CountRetainedAllocations(t.Context(), provider) + if err != nil || retained != wantRetained { + t.Fatalf("retained capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, retained, wantRetained, err) + } + got, err := deploymentStore(w).CountComputeReservations(t.Context(), provider) + if err != nil || got != want { + t.Fatalf("capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, got, want, err) + } + } + got, err := deploymentStore(w).CountComputeReservations(t.Context(), uuid.NewString()) + if err != nil || got != 0 { + t.Fatal("capacity crossed installation boundary", got, err) + } +} + func TestRuntimeSuspensionIdleStartsAfterLastCompletion(t *testing.T) { - _, w, pool, owner := runtimeSuspensionFixture(t) - turn := runtimeSuspensionCompleted(t, pool, owner) + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) - var completed time.Time - if err := pool.QueryRow(t.Context(), `SELECT completed_at FROM turns WHERE id=$1`, turn).Scan(&completed); err != nil { + before := runtimeDatabaseTime(t, s) + id, _ := parseID(owner.SessionID) + if err := s.queries.RecordRuntimeTerminalActivity(t.Context(), id); err != nil { t.Fatal(err) } + after := runtimeDatabaseTime(t, s) activity, err := deploymentStore(w).Activity(t.Context(), owner.ID) - if err != nil || !activity.LastActivity.Equal(completed) { - t.Fatal("long Turn completion did not restart idle interval", activity, completed, err) + if err != nil || activity.LastActivity.Before(before) || activity.LastActivity.After(after) || activity.ReadyToSuspend(time.Minute) { + t.Fatal("long Turn completion did not restart the ingestion idle interval", activity, before, after, err) } } @@ -352,6 +395,12 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { default: runtimeSuspensionSQL(t, pool, `INSERT INTO environment_file_writes(id,environment_id,device_id,request_sha256,state,created_at,settled_at) VALUES($1,$2,$3,$4,$5,clock_timestamp()-interval '10 minutes',clock_timestamp())`, uuid.NewString(), owner.EnvironmentID, owner.DeviceID, strings.Repeat("a", 64), strings.TrimPrefix(kind, "file_")) } + if kind == "root" || kind == "subagent" { + id, _ := parseID(owner.SessionID) + if err := s.queries.RecordRuntimeTerminalActivity(t.Context(), id); err != nil { + t.Fatal(err) + } + } until := time.Now().Add(time.Hour) if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleTimeout); !errors.Is(err, deployment.ErrAllocationConflict) { t.Fatal("completion after idle observation did not fence quiesce", err) @@ -363,6 +412,11 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 0); !errors.Is(err, deployment.ErrInvalidInput) { t.Fatal("missing idle timeout accepted", err) } + // Re-observe the allocation after terminal ingestion advanced its activity fence. + owner, err = deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: owner.TenantID, EnvironmentID: owner.EnvironmentID}) + if err != nil { + t.Fatal(err) + } if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Nanosecond); err != nil { t.Fatal("elapsed idle timeout rejected", err) } diff --git a/services/core/tests/integration/runtime_wake_hint_integration_test.go b/services/core/tests/integration/runtime_wake_hint_integration_test.go index aedeca68c..ebb976bd7 100644 --- a/services/core/tests/integration/runtime_wake_hint_integration_test.go +++ b/services/core/tests/integration/runtime_wake_hint_integration_test.go @@ -16,7 +16,7 @@ import ( ) type wakeHintScanProvider struct { - *fakeCheckpointProvider + *fakeSuspensionProvider sentinel string release chan struct{} scans chan int @@ -35,7 +35,7 @@ func (p *wakeHintScanProvider) GetCompute(ctx context.Context, reference sandbox } } } - return p.fakeCheckpointProvider.GetCompute(ctx, reference, compute) + return p.fakeSuspensionProvider.GetCompute(ctx, reference, compute) } type wakeHintIntegrationTarget struct { @@ -70,7 +70,7 @@ func newWakeHintIntegration(t *testing.T) *wakeHintIntegration { target.owner = f.phase(target.tenant, target.environment.ID, "suspended") f.stop() provider := &wakeHintScanProvider{ - fakeCheckpointProvider: f.provider, sentinel: sentinel.owner.ID, + fakeSuspensionProvider: f.provider, sentinel: sentinel.owner.ID, release: make(chan struct{}), scans: make(chan int, 16), } worker := startWebWorker(t, f.store, f.provider.registry, f.key, provider, &f.policy) diff --git a/services/core/tests/integration/runtime_worker_recovery_test.go b/services/core/tests/integration/runtime_worker_recovery_test.go index 7cb9c7f68..5fce9b5dd 100644 --- a/services/core/tests/integration/runtime_worker_recovery_test.go +++ b/services/core/tests/integration/runtime_worker_recovery_test.go @@ -2,12 +2,14 @@ package integration import ( "context" + "encoding/json" "errors" "testing" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" @@ -15,11 +17,24 @@ import ( func insertWorkerRuntimeAllocation(t *testing.T, pool *pgxpool.Pool, h *dispatchHarness, phase string) { t.Helper() - _, err := pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,compute_phase,compute_retained_until,deployment_generation) - VALUES($1,$2,$3,$4,'running',true,$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment))`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase) + state, err := json.Marshal(map[string]any{"protocol_version": sandbox.SuspensionStateVersion, "current": sandbox.Compute{Name: h.device.EnvironmentID, ID: h.device.EnvironmentID}}) if err != nil { t.Fatal(err) } + _, err = pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,compute_phase,compute_retained_until,deployment_generation,compute_state) + VALUES($1,$2,$3,$4,'running',true,$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment),$6)`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase, state) + if err != nil { + t.Fatal(err) + } + // This synthetic allocation must not outlive the test in the shared fixture + // database, where the next worker validates every retained protocol receipt. + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := pool.Exec(ctx, "DELETE FROM runtime_allocations WHERE environment_id=$1", h.device.EnvironmentID); err != nil { + t.Error(err) + } + }) } func runtimeWorkerHarness(t *testing.T) (*dispatchHarness, *pgxpool.Pool) { diff --git a/services/core/tests/integration/sandbox_deployment_view_test.go b/services/core/tests/integration/sandbox_deployment_view_test.go index ca1ed70b7..9f8051f31 100644 --- a/services/core/tests/integration/sandbox_deployment_view_test.go +++ b/services/core/tests/integration/sandbox_deployment_view_test.go @@ -38,7 +38,7 @@ func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { for _, want := range []string{ `"specification":{"resources":{"cpus":2,"memory_mib":2048}}`, `"template_build":{"status":"ready","resources":{"cpus":2,"memory_mib":2048,"root_disk_mib":24063}}`, - `"suspension":null`, + `"suspension":{"idle_seconds":300,"retention_seconds":86400}`, } { if !bytes.Contains(raw, []byte(want)) { t.Fatalf("E2B view lacks %s: %s", want, raw) diff --git a/services/core/tests/integration/sandbox_generations_test.go b/services/core/tests/integration/sandbox_generations_test.go index 580459b2a..da94f3457 100644 --- a/services/core/tests/integration/sandbox_generations_test.go +++ b/services/core/tests/integration/sandbox_generations_test.go @@ -3,6 +3,7 @@ package integration import ( "database/sql" "os" + "strings" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" @@ -215,14 +216,16 @@ func TestGenerationDowngradeRefusesOldAllocation(t *testing.T) { if _, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, input); err != nil { t.Fatal(err) } + // Isolate the generation downgrade guard using the target schema's disabled + // suspension policy. Active suspension itself is not representable there. db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() migrations, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) if err != nil { t.Fatal(err) } - if _, err = migrations.DownTo(t.Context(), 80); err == nil { - t.Fatal("downgrade erased old owned allocation") + if _, err = migrations.DownTo(t.Context(), 80); err == nil || !strings.Contains(err.Error(), "Cannot downgrade while retained ownership") { + t.Fatal("generation ownership guard did not reject downgrade", err) } // Earlier down migrations can commit before the generation guard vetoes // downgrade. Restore the current schema before invoking current Store code. diff --git a/services/core/tests/integration/session_execution_configuration_test.go b/services/core/tests/integration/session_execution_configuration_test.go index 54dea62c3..e4a1c168c 100644 --- a/services/core/tests/integration/session_execution_configuration_test.go +++ b/services/core/tests/integration/session_execution_configuration_test.go @@ -282,7 +282,7 @@ func TestSessionExecutionConfigurationSurvivesSuspendResume(t *testing.T) { t.Fatal("runtime transition changed execution projection", phase, err) } after, err := deploymentStore(w).Activity(t.Context(), owner.ID) - if err != nil || !before.LastActivity.Equal(after.LastActivity) || before.WakeRequested != after.WakeRequested || before.Busy != after.Busy || before.HasCompletedTurn != after.HasCompletedTurn { + if err != nil || !before.LastActivity.Equal(after.LastActivity) || before.WakeRequested != after.WakeRequested || before.Busy != after.Busy { t.Fatal("configuration read changed runtime activity", err) } } diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index 46f1e9f15..2f8b9a8a0 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -30,7 +30,7 @@ Compatible endpoints must return the SDK 2.51.0 template-list and template-build Run `go generate ./services/core/internal/sandbox/e2b` from the repository root after changing these declarations. `make check-e2b-provider` and the Go adapter tests reject stale projections; both languages consume generated valid and invalid exchanges covering wire types, extra fields, operation/reference bounds and managed-bootstrap fields. The helper build copies those fixtures with its source before running the pinned-SDK suite. -The boundary has version 2. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. +The boundary has version 4. Managed bootstrap and Resume accept an omitted or null `Workspace`; E2B rejects non-null external workspace bindings before native calls. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. ## Receipts and state directory @@ -50,7 +50,7 @@ An unknown Create is never repeated. A Create whose connection material was lost ## Inspection and cleanup -Inspection uses SDK metadata and ID reads only. SDK `connect` is never used because it can resume paused compute. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. +Inspection uses SDK metadata and ID reads only. Inspection never uses SDK `connect`, which is reserved for the explicit managed Resume operation. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. `CreateSettled` proves that the original Create and bootstrap can no longer mutate; it is independent of `BootstrapComplete`. An empty lookup never settles an unknown Create. Kill destroys every matching sandbox, confirms that none remains and only then records a settled tombstone; it returns `State=absent` with `CreateSettled`. A settled rejected Create with no sandbox IDs proves absence without a cloud request, so GetInfo and Kill still succeed when the key is invalid. Ordinary missing compute has no such proof. @@ -71,3 +71,11 @@ make check-e2b-provider ``` With `OAC_TEST_E2B_SDK_PYTHON` pointing at the pinned SDK environment, this runs this directory's tests and the [template scripts' tests](../../deploy/e2b/README.md#tests). The helper build runs this directory's suite and checks the relocated helper's `--check` report. These tests create no cloud resources. + +## Managed suspension + +The adapter implements the complete shared suspension group using opaque retained state. A native pause retains the same sandbox; it is not a portable filesystem snapshot. Core's common activity rule quiesces the Runtime, then the adapter calls the pinned SDK's memory-preserving `Sandbox.pause`. The private allocation receipt commits the operation before native I/O and reports resource release only after observing the exact sandbox paused. + +Resume calls `Sandbox.connect` once with `on_resume=restore` and the existing native timeout. It preserves the native sandbox ID while advancing the shared logical generation. Fresh connection material is persisted before returning running. Unknown pause and connect outcomes are observed without replay; a missing connect receipt keeps execution unavailable. The adapter fences stale generations before commands and deletion. Consumed pause receipt cleanup preserves running compute. + +The shared registration owns the 300-second idle duration and 86400-second retention default. Native timeout remains 3600 seconds and automatic native resume remains disabled. SDK calls use the configured official-compatible endpoint. Generated declarations and fixtures own the private helper shape. diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index 0b7635c78..7f420e0aa 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -1,5 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" +COMPUTE_FIELDS = ["Generation","Name","ID","RestoredFrom"] ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] @@ -10,9 +11,13 @@ MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 2 +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute_info","compute_renew","suspend","resume","compute_kill","delete_retained","compute_command","resume_compute"] +PROTOCOL_VERSION = 4 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Retained","Deadline"] +RESPONSE_FIELDS = ["Version","State","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +RESUME_FIELDS = ["Workspace","Reference","OperationID","Retained","Target","ReconcileOnly"] +RETAINED_FIELDS = ["Reference","ID","Data","OperationID","SourceGeneration","SourceName","SourceID"] SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" +SUSPEND_FIELDS = ["Reference","OperationID","Source","Retained","ReconcileOnly"] diff --git a/services/core/tools/e2b-provider/observation_test.py b/services/core/tools/e2b-provider/observation_test.py index f72554e8d..3a773ca24 100644 --- a/services/core/tools/e2b-provider/observation_test.py +++ b/services/core/tools/e2b-provider/observation_test.py @@ -7,6 +7,7 @@ from provider import Provider from provider_test import ProviderTest from state import Failure +from helper_contract_generated import PROTOCOL_VERSION class ObservationTest(ProviderTest): @@ -15,7 +16,7 @@ def observe(self, reference=None): try: return Provider(request).execute() except Failure as error: - return {'Version': 2, 'ErrorCode': error.code} + return {'Version': PROTOCOL_VERSION, 'ErrorCode': error.code} def listing(self, *pages): pages = [list(page) for page in pages] or [[]] diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index 264b39489..a75d09a67 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -105,6 +105,9 @@ def __init__(self, request): bootstrap = request.get('Bootstrap') if bootstrap is not None and (not isinstance(bootstrap, dict) or bootstrap.get('Workspace') is not None): raise Failure('invalid') + resume = request.get('Resume') + if resume is not None and (not isinstance(resume, dict) or resume.get('Workspace') is not None): + raise Failure('invalid') deadline = datetime.fromisoformat(request['Deadline'].replace('Z', '+00:00')) self.deadline = time.monotonic() + (deadline - datetime.now(timezone.utc)).total_seconds() self.metadata = {PREFIX + field.lower(): value for field, value in @@ -238,7 +241,9 @@ def create(self): if definitely_rejected(error): self.receipt.save(status='rejected', settled=True) raise Failure('unconfirmed') from None - self.receipt.save(status='created', ids=[cloud.sandbox_id], connection=connection_material(cloud)) + self.receipt.save(status='created', ids=[cloud.sandbox_id], connection=connection_material(cloud), + compute={'Generation': 0, 'Name': self.reference['AllocationID'], + 'ID': cloud.sandbox_id, 'RestoredFrom': None}) # A create response must not steer envd traffic to an unrelated host. self.check_domain(cloud) # SDK Create returns connection material, but no metadata or resources. @@ -413,6 +418,10 @@ def execute(self): with Receipt(self.q, self.remaining) as self.receipt: try: operation = self.q['Operation'] + if operation in ('compute_info', 'compute_renew', 'suspend', 'resume', 'compute_kill', + 'delete_retained', 'compute_command', 'resume_compute'): + from suspension import Suspension + return Suspension(self, Sandbox, connection_material, run).execute() if operation == 'kill': self.kill() return {'Version': PROTOCOL_VERSION, 'Info': self.info(absent=True), 'ErrorCode': ''} diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 240e05eab..9bd122dea 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -65,7 +65,7 @@ def setUp(self): self.reference = {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']} self.config = {'StateDir': self.temporary.name, 'InstallationID': str(uuid4()), 'APIKey': 'private-account-secret', 'Template': 'test:' + str(uuid4()), 'TimeoutSeconds': 120} - self.request = {'Version': 2, 'Operation': 'create', 'Config': self.config, + self.request = {'Version': PROTOCOL_VERSION, 'Operation': 'create', 'Config': self.config, 'Reference': self.reference, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=30)).isoformat(), 'Bootstrap': dict(self.reference, SessionID=str(uuid4()), DeviceID=str(uuid4()), CoreURL='https://core.example/api/v1', Credential='private-runtime-secret', diff --git a/services/core/tools/e2b-provider/state_test.py b/services/core/tools/e2b-provider/state_test.py index 9ff5b89b4..7de903990 100644 --- a/services/core/tools/e2b-provider/state_test.py +++ b/services/core/tools/e2b-provider/state_test.py @@ -11,6 +11,7 @@ from unittest.mock import patch from provider import Provider +from helper_contract_generated import PROTOCOL_VERSION from state import Failure, Receipt @@ -18,7 +19,7 @@ class StateTest(unittest.TestCase): def setUp(self): self.root = tempfile.TemporaryDirectory() self.addCleanup(self.root.cleanup) - self.request = {'Version': 2, 'Operation': 'kill', + self.request = {'Version': PROTOCOL_VERSION, 'Operation': 'kill', 'Config': {'StateDir': self.root.name, 'InstallationID': str(uuid4()), 'APIKey': 'test'}, 'Reference': {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']}, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=2)).isoformat()} diff --git a/services/core/tools/e2b-provider/suspension.py b/services/core/tools/e2b-provider/suspension.py new file mode 100644 index 000000000..d37b860da --- /dev/null +++ b/services/core/tools/e2b-provider/suspension.py @@ -0,0 +1,228 @@ +"""E2B's implementation of exact-incarnation suspension under the allocation lock.""" +import json + +from helper_contract_generated import PROTOCOL_VERSION, COMPUTE_FIELDS, SUSPEND_FIELDS, RESUME_FIELDS +from state import Failure + + +class Suspension: + def __init__(self, provider, sdk, material, command): + self.p, self.sdk, self.material, self.command = provider, sdk, material, command + + @property + def record(self): + record = self.p.receipt.data + if not record or record.get('settled') is not True: + raise Failure('unconfirmed') + return record + + def current(self): + current = self.record.get('compute') + if not isinstance(current, dict) or not current.get('ID'): + raise Failure('unconfirmed') + return current + + def matches(self, wanted, actual, unresolved=False): + if (not isinstance(wanted, dict) or set(wanted) != set(COMPUTE_FIELDS) or + type(wanted['Generation']) is not int or wanted['Generation'] < 0 or + wanted['Name'] != self.p.reference['AllocationID'] or + any(wanted[k] != actual[k] for k in ('Generation', 'Name', 'RestoredFrom')) or + (wanted['ID'] != actual['ID'] and not (unresolved and wanted['ID'] == ''))): + raise Failure('ownership') + + def cloud(self, execution=True): + found = self.p.discover() + if len(found) != 1: + raise Failure('unconfirmed') + cloud = found[0] + if cloud.sandbox_id != self.current()['ID']: + raise Failure('ownership') + if execution: + self.p.qualified(cloud) + return cloud + + def state(self, current, cloud): + return {'Compute': current, 'Status': cloud.state, + 'BootstrapComplete': self.record.get('bootstrap_complete') is True, + 'Retained': None, 'ResourcesReleased': False, 'SuspendSettled': False} + + def retained(self, operation, current): + return {'Reference': self.p.reference['AllocationID'], 'ID': operation, 'OperationID': operation, + 'SourceGeneration': current['Generation'], 'SourceName': current['Name'], + 'SourceID': current['ID'], + 'Data': json.dumps({'version': 1, 'sandbox_id': current['ID']}, sort_keys=True, separators=(',', ':'))} + + def request(self, field): + q = self.p.q.get(field) + fields = set(SUSPEND_FIELDS if field == 'Suspend' else RESUME_FIELDS) + required = fields - ({'Workspace'} if field == 'Resume' else set()) + if (not isinstance(q, dict) or set(q) - fields or not required <= set(q) or + (field == 'Resume' and q.get('Workspace') is not None) or + q.get('Reference') != self.p.reference): + raise Failure('invalid') + # UUID parsing is shared with the allocation envelope validator. + from provider import valid_id + if not valid_id(q.get('OperationID')) or type(q.get('ReconcileOnly')) is not bool: + raise Failure('invalid') + return q + + def inspect(self, operation): + current = self.current() + self.matches(self.p.q.get('Compute'), current, unresolved=operation == 'compute_info') + if self.record.get('status') == 'killed': + raise Failure('not_found') + cloud = self.cloud() + if operation != 'compute_info': + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + pending = self.record.get('suspension') + if pending and pending['phase'] not in ('resumed', 'consumed'): + raise Failure('unconfirmed') + if operation == 'compute_renew': + self.sdk.set_timeout(cloud.sandbox_id, self.p.config['TimeoutSeconds'], **self.p.options()) + cloud = self.cloud() + if operation == 'compute_command': + return {'Command': self.command(self.p.client(cloud), self.p.q['Command'], self.p.remaining)} + return {'State': self.state(current, cloud)} + + def suspend(self): + q = self.request('Suspend') + current = self.current() + self.matches(q.get('Source'), current) + handle = self.retained(q['OperationID'], current) + if q.get('Retained') not in (None, handle): + raise Failure('ownership') + entry = self.record.get('suspension') + if entry and entry['retained'] == handle: + if entry['phase'] not in ('pause_pending', 'paused'): + raise Failure('ownership') + elif q['ReconcileOnly']: + # No durable native intent: the allocation lock proves this helper + # never issued pause. Only a qualified running source can roll back. + if entry and entry['phase'] != 'consumed': + raise Failure('ownership') + cloud = self.cloud() + if cloud.state != 'running': + raise Failure('unconfirmed') + state = self.state(current, cloud) + state['SuspendSettled'] = True + return {'State': state} + else: + if entry and entry['phase'] != 'consumed': + raise Failure('ownership') + cloud = self.cloud() + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + entry = {'retained': handle, 'phase': 'pause_pending'} + self.p.receipt.save(suspension=entry) + # An error remains pending. A later observation may prove paused; + # a running observation never proves a timed-out pause cannot land. + self.sdk.pause(current['ID'], keep_memory=True, **self.p.options()) + cloud = self.cloud() + if cloud.state != 'paused': + raise Failure('unconfirmed') + self.p.receipt.save(suspension=dict(entry, phase='paused')) + state = self.state(current, cloud) + state.update(Status='suspended', Retained=handle, ResourcesReleased=True, SuspendSettled=True) + return {'State': state} + + def resume(self): + q = self.request('Resume') + entry = self.record.get('suspension') + if not entry or entry['retained'] != q.get('Retained'): + raise Failure('ownership') + retained = entry['retained'] + target = {'Generation': retained['SourceGeneration'] + 1, 'Name': retained['SourceName'], + 'ID': retained['SourceID'], 'RestoredFrom': retained} + self.matches(q.get('Target'), target) + if entry['phase'] in ('resume_pending', 'resumed', 'consumed'): + if entry.get('resume_id') != q['OperationID'] or entry.get('target') != target: + raise Failure('ownership') + elif entry['phase'] == 'paused' and not q['ReconcileOnly']: + cloud = self.cloud() + if cloud.state != 'paused': + raise Failure('unconfirmed') + entry = dict(entry, phase='resume_pending', resume_id=q['OperationID'], target=target, connected=False) + self.p.receipt.save(suspension=entry) + connected = self.sdk.connect(target['ID'], timeout=self.p.config['TimeoutSeconds'], + on_resume='restore', **self.p.options()) + if connected.sandbox_id != target['ID']: + raise Failure('ownership') + self.p.check_domain(connected) + entry = dict(entry, connected=True) + self.p.receipt.save(suspension=entry, connection=self.material(connected)) + else: + raise Failure('unconfirmed') + # Never repeat connect after an uncertain reply. Fresh connection material + # must have been durably received before execution can resume. + if not entry.get('connected'): + raise Failure('unconfirmed') + cloud = self.cloud() + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + if entry['phase'] != 'consumed': + self.p.receipt.save(compute=target, suspension=dict(entry, phase='resumed')) + return {'State': self.state(target, cloud)} + + def kill(self): + wanted = self.p.q.get('Compute') + current = self.current() + entry = self.record.get('suspension') + target = (entry or {}).get('target') + # Target-first cleanup may arrive before Resume was ever dispatched. + planned = None + if entry: + r = entry['retained'] + planned = {'Generation': r['SourceGeneration'] + 1, 'Name': r['SourceName'], + 'ID': r['SourceID'], 'RestoredFrom': r} + if wanted == planned and target is None and entry['phase'] == 'paused': + return {} + if wanted != current and wanted != target: + # Once destruction is confirmed, stale source cleanup is harmless. + if self.record.get('status') == 'killed' and isinstance(wanted, dict): + if wanted['ID'] == current['ID'] and wanted['Name'] == current['Name'] and wanted['Generation'] < current['Generation']: + return {} + raise Failure('ownership') + if entry and entry['phase'] == 'resume_pending' and entry.get('connected'): + # The SDK reply already settled the original connect. Cleanup owns + # recovery once Core leaves restoring, so reconcile that saved target + # here without another connect or a resource-qualification gate. + if wanted != target: + raise Failure('ownership') + found = self.p.discover() + if len(found) > 1 or any(cloud.sandbox_id != target['ID'] for cloud in found): + raise Failure('ownership') + entry = dict(entry, phase='resumed') + self.p.receipt.save(compute=target, suspension=entry) + if entry and entry['phase'] in ('pause_pending', 'resume_pending'): + # No successful native reply or settled pause has been observed. + raise Failure('unconfirmed') + self.p.kill() + return {} + + def delete(self): + wanted = self.p.q.get('Retained') + entry = self.record.get('suspension') + if not entry or wanted != entry['retained']: + raise Failure('ownership') + if entry['phase'] == 'consumed': + return {} + if self.record.get('status') != 'killed': + if entry['phase'] != 'resumed': + raise Failure('unconfirmed') + self.matches(entry['target'], self.current()) + if self.cloud().state != 'running': + raise Failure('unconfirmed') + # E2B consumed the pause image on restore. Keep its minimal generation + # receipt to reject delayed resume/cleanup; never kill running compute. + self.p.receipt.save(suspension=dict(entry, phase='consumed')) + return {} + + def execute(self): + operation = self.p.q['Operation'] + if operation in ('compute_info', 'compute_renew', 'compute_command', 'resume_compute'): + result = self.inspect(operation) + else: + result = {'suspend': self.suspend, 'resume': self.resume, + 'compute_kill': self.kill, 'delete_retained': self.delete}[operation]() + return dict(result, Version=PROTOCOL_VERSION, ErrorCode='') diff --git a/services/core/tools/e2b-provider/suspension_test.py b/services/core/tools/e2b-provider/suspension_test.py new file mode 100644 index 000000000..875c0d69e --- /dev/null +++ b/services/core/tools/e2b-provider/suspension_test.py @@ -0,0 +1,170 @@ +"""Native lifecycle effects and durable recovery are tested through the helper.""" +import unittest +from uuid import uuid4 + +from provider import Provider +import provider_test + + +class SuspensionTest(unittest.TestCase): + setUp = provider_test.ProviderTest.setUp + call = provider_test.ProviderTest.call + record = provider_test.ProviderTest.record + + def prepare(self): + self.assertEqual(self.call('create')['ErrorCode'], '') + self.api.pause.side_effect = lambda *a, **k: setattr(self.cloud, 'state', 'paused') + def connect(*args, **kwargs): + self.cloud.state = 'running' + return self.cloud + self.api.connect.side_effect = connect + return self.record()['compute'] + + def invoke(self, operation, **payload): + return Provider(dict(self.request, Operation=operation, **payload)).execute() + + def pause(self, source): + q = {'Reference': self.reference, 'OperationID': str(uuid4()), 'Source': source, + 'Retained': None, 'ReconcileOnly': False} + response = self.invoke('suspend', Suspend=q) + return q, response + + def resume_request(self, retained): + target = {'Generation': retained['SourceGeneration'] + 1, 'Name': retained['SourceName'], + 'ID': retained['SourceID'], 'RestoredFrom': retained} + return {'Reference': self.reference, 'OperationID': str(uuid4()), 'Target': target, + 'Retained': retained, 'ReconcileOnly': False} + + def test_resume_rejects_external_workspace_before_sdk(self): + from state import Failure + with self.assertRaises(Failure) as raised: + Provider(dict(self.request, Operation='resume', Resume={'Workspace': {}})) + self.assertEqual(raised.exception.code, 'invalid') + self.api.connect.assert_not_called() + + def test_resume_accepts_null_workspace(self): + current = self.prepare() + _, paused = self.pause(current) + request = dict(self.resume_request(paused['State']['Retained']), Workspace=None) + self.assertEqual(self.invoke('resume', Resume=request)['ErrorCode'], '') + + def test_two_cycles_keep_native_id_and_fence_old_generation(self): + current = self.prepare() + for generation in range(2): + old = current + q, paused = self.pause(current) + self.assertEqual(paused['ErrorCode'], '') + self.assertTrue(paused['State']['ResourcesReleased']) + retained = paused['State']['Retained'] + request = self.resume_request(retained) + result = self.invoke('resume', Resume=request) + self.assertEqual(result['ErrorCode'], '') + current = result['State']['Compute'] + self.assertEqual(current['ID'], old['ID']) + self.assertEqual(current['Generation'], generation + 1) + self.assertEqual(self.invoke('compute_kill', Compute=old)['ErrorCode'], 'ownership') + self.api.kill.assert_not_called() + # Core wakes the parked daemon before deleting the consumed receipt. + self.assertEqual(self.invoke('compute_command', Compute=current, + Command={'Args': ['oac-daemon', 'resume']})['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.assertEqual(self.invoke('compute_renew', Compute=current)['ErrorCode'], '') + self.assertEqual(self.api.pause.call_count, 2) + self.assertEqual(self.api.connect.call_count, 2) + self.assertEqual(self.api.connect.call_args.kwargs['on_resume'], 'restore') + self.assertEqual(self.api.connect.call_args.kwargs['timeout'], self.config['TimeoutSeconds']) + self.assertTrue(self.api.pause.call_args.kwargs['keep_memory']) + + def test_lost_pause_reply_observes_without_replay(self): + current = self.prepare() + def lost(*args, **kwargs): + self.cloud.state = 'paused' + raise TimeoutError() + self.api.pause.side_effect = lost + q, result = self.pause(current) + self.assertEqual(result['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('suspend', Suspend=dict(q, ReconcileOnly=True))['ErrorCode'], '') + self.api.pause.assert_called_once() + + def test_pending_pause_running_cannot_roll_back_or_delete(self): + current = self.prepare() + self.api.pause.side_effect = TimeoutError() + q, result = self.pause(current) + self.assertEqual(result['ErrorCode'], 'unconfirmed') + for _ in range(2): + self.assertEqual(self.invoke('suspend', Suspend=dict(q, ReconcileOnly=True))['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], 'unconfirmed') + self.api.pause.assert_called_once() + self.api.kill.assert_not_called() + + def test_missing_native_pause_intent_can_settle_running_rollback(self): + current = self.prepare() + q = {'Reference': self.reference, 'OperationID': str(uuid4()), 'Source': current, + 'Retained': None, 'ReconcileOnly': True} + state = self.invoke('suspend', Suspend=q)['State'] + self.assertTrue(state['SuspendSettled']) + self.assertFalse(state['ResourcesReleased']) + self.assertIsNone(state['Retained']) + self.api.pause.assert_not_called() + + def test_lost_resume_reply_never_replays_connect(self): + current = self.prepare() + _, paused = self.pause(current) + q = self.resume_request(paused['State']['Retained']) + def lost(*a, **k): + self.cloud.state = 'running' + raise TimeoutError() + self.api.connect.side_effect = lost + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('resume', Resume=dict(q, ReconcileOnly=True))['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('compute_kill', Compute=q['Target'])['ErrorCode'], 'unconfirmed') + self.api.connect.assert_called_once() + self.api.kill.assert_not_called() + + def test_lost_core_resume_reply_adopts_saved_target_without_connect(self): + current = self.prepare() + _, paused = self.pause(current) + q = self.resume_request(paused['State']['Retained']) + first = self.invoke('resume', Resume=q) + self.assertEqual(first['ErrorCode'], '') + second = self.invoke('resume', Resume=dict(q, ReconcileOnly=True)) + self.assertEqual(first, second) + self.api.connect.assert_called_once() + + def test_foreign_retained_and_incarnation_cannot_mutate(self): + current = self.prepare() + _, paused = self.pause(current) + retained = paused['State']['Retained'] + q = self.resume_request(dict(retained, SourceID='foreign-id')) + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'ownership') + self.assertEqual(self.invoke('delete_retained', Retained=dict(retained, ID='foreign'))['ErrorCode'], 'ownership') + self.api.connect.assert_not_called() + self.api.kill.assert_not_called() + + def test_archive_recovers_saved_connect_reply_before_cleanup(self): + from e2b.exceptions import SandboxNotFoundException + current = self.prepare() + _, paused = self.pause(current) + retained = paused['State']['Retained'] + q = self.resume_request(retained) + def connected_then_observation_lost(*args, **kwargs): + self.cloud.state = 'running' + self.api.get_info.side_effect = TimeoutError() + return self.cloud + self.api.connect.side_effect = connected_then_observation_lost + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'unconfirmed') + self.assertTrue(self.record()['suspension']['connected']) + self.assertEqual(self.record()['suspension']['phase'], 'resume_pending') + self.api.get_info.side_effect = None + # The old incarnation remains fenced until exact target cleanup settles. + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], 'ownership') + def killed(*args, **kwargs): + self.api.get_info.side_effect = SandboxNotFoundException('gone') + self.api.kill.side_effect = killed + self.assertEqual(self.invoke('compute_kill', Compute=q['Target'])['ErrorCode'], '') + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.api.connect.assert_called_once() + self.api.kill.assert_called_once() + self.assertEqual(self.record()['status'], 'killed') diff --git a/services/core/tools/e2b-provider/testdata/contract.json b/services/core/tools/e2b-provider/testdata/contract.json index 0efaa314d..bd22db5b3 100644 --- a/services/core/tools/e2b-provider/testdata/contract.json +++ b/services/core/tools/e2b-provider/testdata/contract.json @@ -21,46 +21,56 @@ {"kind":"managed","name":"restricted","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":["example.com"],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"restricted","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, {"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, {"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, -{"kind":"request","name":"command","payload":{"Version":2,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"create","payload":{"Version":2,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"inspect","payload":{"Version":2,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"kill","payload":{"Version":2,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_builds","payload":{"Version":2,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_templates","payload":{"Version":2,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe","payload":{"Version":2,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"renew","payload":{"Version":2,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"validate_deployment","payload":{"Version":2,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-0","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-32","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-33","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"command","payload":{"Version":4,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_command","payload":{"Version":4,"Operation":"compute_command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Command":{"Args":["true"],"Directory":"","Stdin":null},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_info","payload":{"Version":4,"Operation":"compute_info","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_kill","payload":{"Version":4,"Operation":"compute_kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_renew","payload":{"Version":4,"Operation":"compute_renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"create","payload":{"Version":4,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"delete_retained","payload":{"Version":4,"Operation":"delete_retained","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Retained":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"inspect","payload":{"Version":4,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill","payload":{"Version":4,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_builds","payload":{"Version":4,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_templates","payload":{"Version":4,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"observe","payload":{"Version":4,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"renew","payload":{"Version":4,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume","payload":{"Version":4,"Operation":"resume","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Resume":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"66666666-6666-4666-8666-666666666666","Retained":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"},"Target":{"Generation":1,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"}},"ReconcileOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume-workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"resume","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Resume":{"OperationID":"66666666-6666-4666-8666-666666666666","ReconcileOnly":false,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Retained":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"},"Target":{"Generation":1,"ID":"native-fixture","Name":"33333333-3333-4333-8333-333333333333","RestoredFrom":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"}},"Workspace":null},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":true}, +{"kind":"request","name":"resume-workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"resume","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Resume":{"OperationID":"66666666-6666-4666-8666-666666666666","ReconcileOnly":false,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Retained":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"},"Target":{"Generation":1,"ID":"native-fixture","Name":"33333333-3333-4333-8333-333333333333","RestoredFrom":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"}},"Workspace":{}},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"resume_compute","payload":{"Version":4,"Operation":"resume_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"suspend","payload":{"Version":4,"Operation":"suspend","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Suspend":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"66666666-6666-4666-8666-666666666666","Source":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Retained":null,"ReconcileOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"validate_deployment","payload":{"Version":4,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-0","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-32","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-33","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, {"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":true},"valid":false}, {"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":null},"valid":false}, {"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":"1"},"valid":false}, -{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":3},"valid":false}, -{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":true}, -{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"response","name":"error-","payload":{"Version":2,"ErrorCode":""},"valid":true}, -{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":2,"ErrorCode":"command_unconfirmed"},"valid":true}, -{"kind":"response","name":"error-exists","payload":{"Version":2,"ErrorCode":"exists"},"valid":true}, -{"kind":"response","name":"error-invalid","payload":{"Version":2,"ErrorCode":"invalid"},"valid":true}, -{"kind":"response","name":"error-not_found","payload":{"Version":2,"ErrorCode":"not_found"},"valid":true}, -{"kind":"response","name":"error-ownership","payload":{"Version":2,"ErrorCode":"ownership"},"valid":true}, -{"kind":"response","name":"error-team_mismatch","payload":{"Version":2,"ErrorCode":"team_mismatch"},"valid":true}, -{"kind":"response","name":"error-template_invalid","payload":{"Version":2,"ErrorCode":"template_invalid"},"valid":true}, -{"kind":"response","name":"error-unauthorized","payload":{"Version":2,"ErrorCode":"unauthorized"},"valid":true}, -{"kind":"response","name":"error-unconfirmed","payload":{"Version":2,"ErrorCode":"unconfirmed"},"valid":true}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":2},"valid":false}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":2},"valid":false}, -{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":2},"valid":false}, +{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":5},"valid":false}, +{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":true}, +{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"response","name":"error-","payload":{"Version":4,"ErrorCode":""},"valid":true}, +{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":4,"ErrorCode":"command_unconfirmed"},"valid":true}, +{"kind":"response","name":"error-exists","payload":{"Version":4,"ErrorCode":"exists"},"valid":true}, +{"kind":"response","name":"error-invalid","payload":{"Version":4,"ErrorCode":"invalid"},"valid":true}, +{"kind":"response","name":"error-not_found","payload":{"Version":4,"ErrorCode":"not_found"},"valid":true}, +{"kind":"response","name":"error-ownership","payload":{"Version":4,"ErrorCode":"ownership"},"valid":true}, +{"kind":"response","name":"error-team_mismatch","payload":{"Version":4,"ErrorCode":"team_mismatch"},"valid":true}, +{"kind":"response","name":"error-template_invalid","payload":{"Version":4,"ErrorCode":"template_invalid"},"valid":true}, +{"kind":"response","name":"error-unauthorized","payload":{"Version":4,"ErrorCode":"unauthorized"},"valid":true}, +{"kind":"response","name":"error-unconfirmed","payload":{"Version":4,"ErrorCode":"unconfirmed"},"valid":true}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":4},"valid":false}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":4},"valid":false}, +{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":4},"valid":false}, {"kind":"response","name":"invalid-Version","payload":{"ErrorCode":"","Version":true},"valid":false} ] diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index 58e190c59..a487b26f6 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -1,6 +1,6 @@ # microsandbox Sandbox Provider helper -microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 5; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. +microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 5; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the suspension operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. [Add a Sandbox Provider](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources, Runtime release and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy. @@ -29,14 +29,14 @@ A helper response carries `CreateSettled` with a configuration rejection only af Core persists operation IDs, source and target generations, exact identities and snapshot evidence before it depends on them. `Initial` and `NewCompute` only construct references and allocate nothing. - **Suspend** pauses the exact VM, captures a full snapshot under the persisted operation's derived group and member, verifies the complete checkpoint closure, then force-stops the source. Pausing alone does not release memory. A completed matching artifact is inspected instead of captured again. A full snapshot records a resource proof only after the source's limits match. -- **KillCompute** checks the precise incarnation before it stops the VM and removes its writable disks. Core calls it after it has persisted the verified snapshot, even when Suspend already stopped the source, so no chain of old writable disks grows across suspension cycles. +- **KillCompute** checks the precise incarnation before it stops the VM and removes its writable disks. Suspend completes native source cleanup under the allocation lock before reporting resource release, including recovery of a captured artifact. Core uses KillCompute for allocation cleanup. - **Restore** verifies the exact artifact and creates the precommitted target name. An existing target is adopted only when its immutable ID, if known, and its persisted `snapshot_parent` agree. Upstream restore defaults to public networking, so restore passes the same explicit host policy as creation, and no undeclared host resource or mount is inherited. - Native restore leaves the managed root size unset because the target inherits the verified full snapshot. The helper accepts that only with a matching snapshot resource proof and the exact source and target identities, and it checks the target's CPU, memory and Environment disk before keeping the inherited proof. A missing root size never counts as unlimited capacity, and retained state is never resized. - Fresh restore and retry share one completion: verify the original artifact and resource proof, inspect the running target's resources and ancestry, persist its missing derived resource-proof label, then strictly reread the same native ID ([`restore_completion.go`](restore_completion.go)). A conflicting proof is an error. Native restore does not copy the source's ownership labels; ancestry supplies that evidence. There is no ordinary Start, replacement, disk-only restore or cold boot. - **ResumeCompute** thaws the same resident source after an aborted suspension. The pinned SDK handle method is name-based; the allocation lock and ID checks before and after the call fence every managed replacement. Manual lifecycle changes in the managed namespace are unsupported. -- **DeleteSnapshot** accepts only the derived operation selector and the matching full artifact identity, never an arbitrary path. Core owns retention, consumed snapshot generations and cleanup order. A checkpoint never rolls back work admitted after its first restore. +- **DeleteRetained** accepts only the derived operation selector and the matching full artifact identity, never an arbitrary path. Core owns retention, consumed snapshot generations and cleanup order. A checkpoint never rolls back work admitted after its first restore. -After a lost response Core uses `ObserveOnly`. It never starts a capture or restore, and observing a suspend operation never kills its source. Observation checks artifact integrity and source ownership independently of resource checks, so resource drift cannot hide a retained artifact from cleanup; Core can persist recovered snapshot evidence before KillCompute. If the artifact is absent but the exact source is still running or paused with a settled bootstrap, observation returns the source without a snapshot and Core can abort the suspension; thawing and further execution still require the resource checks. For an interrupted restore, `ObserveOnly` may finish the missing resource proof on the exact target but never restarts a stopped target, changes resources or restores again. Missing state never authorizes a replay. +After a lost response Core uses `ReconcileOnly`. The adapter observes the original capture or restore and never starts it again. When a complete matching snapshot exists, reconciliation finishes exact source cleanup before reporting settled suspension and released resources. Artifact integrity and source ownership are checked independently of resource qualification, so resource drift cannot prevent owned cleanup. If capture is settled without an artifact and the exact source is still running or paused with a settled bootstrap, the adapter reports that outcome so Core can abort suspension; thawing and further execution still require resource checks. For an interrupted restore, reconciliation may finish the missing resource proof on the exact target but never restarts a stopped target, changes resources or restores again. Missing state never authorizes a replay. GetCompute, commands, cleanup and the next suspension verify restored provenance from the persisted VM configuration after the consumed artifact is deleted. @@ -65,4 +65,4 @@ The helper returns only the native observation time, exact uptime, cumulative vC `make check-microsandbox-provider`, part of `make check`, runs the pure-Go adapter tests everywhere and this module's tests on Linux; other hosts print an explicit skip for the Linux-only module. -The native helper wire uses version 5 and carries the Session-selected Harness into Runtime bootstrap version 2. Upgrade the helper with its node and Core. +The native helper wire uses version 5 and carries the Session-selected Harness into Runtime bootstrap version 2. Upgrade the helper with its node and Core. The Go adapter wraps native snapshot identity in the shared opaque retained-state handle and completes exact-source cleanup through the existing ownership-checked helper operations. Reconciliation never repeats snapshot capture. SuspendSettled is reported only after these serialized operations complete; captured-artifact cleanup does not require execution resource qualification. diff --git a/services/core/tools/microsandbox-provider/bootstrap.go b/services/core/tools/microsandbox-provider/bootstrap.go index d1753e4f4..e6779414f 100644 --- a/services/core/tools/microsandbox-provider/bootstrap.go +++ b/services/core/tools/microsandbox-provider/bootstrap.go @@ -8,6 +8,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentnetwork" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" wire "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" sdk "github.com/superradcompany/microsandbox/sdk/go" @@ -18,7 +19,7 @@ import ( const bootstrapScript = ` import ctypes,json,os,stat,subprocess,sys b=json.load(sys.stdin) -for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages','/run/oac']: +for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages',os.path.dirname(os.environ['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'])]: os.makedirs(p,mode=0o700,exist_ok=True) if not stat.S_ISDIR(os.lstat(p).st_mode): raise RuntimeError('invalid bootstrap directory') os.chmod(p,0o700);os.chown(p,1000,1000) @@ -71,7 +72,7 @@ func (b backend) create(ctx context.Context) (wire.Response, error) { "HOME": "/home/runtime", "OAC_RUNTIME_HOME": "/home/runtime/.oac", "OAC_RUNTIME_ENVIRONMENT_ID": bootstrap.EnvironmentID, "OAC_RUNTIME_SESSION_ID": bootstrap.SessionID, "OAC_RUNTIME_NETWORK_ACCESS": policy.Access, "OAC_RUNTIME_ALLOWED_DOMAINS": string(domains), - "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": "/run/oac/daemon-suspend.json", + "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": runtimebootstrap.SuspendControlFile, })) if e != nil { return wire.Response{}, e From 45621aebe69f4a2315abef87fd6405e20d2cafdb Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 00:00:36 +0800 Subject: [PATCH 09/22] Document independent workspace recovery (#645) --- docs/configuration.md | 6 ++-- docs/getting-started/operations.md | 42 ++++++++++++++++++++------ docs/zh/configuration.md | 8 ++--- docs/zh/getting-started/operations.md | 43 ++++++++++++++++++++------- 4 files changed, 73 insertions(+), 26 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index e3248bc5e..fb83fb72c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -103,7 +103,7 @@ Which harnesses are enabled, and the default one, are process settings (`core.ha ### Independent workspace storage -The initial supported independent filesystem combination is microsandbox with the [kernel NFS adapter](./workspace-provider.md#kernel-nfs-adapter). Mount the same NFSv4.2 export on the Linux Core host and every participating node before starting their services, at the same absolute path, for example `/srv/oac-workspaces`. The operator manages the export, mount availability and service startup ordering. Use a trusted-client AUTH_SYS export with `root_squash`; do not enable `no_root_squash` or broaden permissions to make a check pass. Prepare the namespace and service principals according to the adapter's [ownership requirements](./workspace-provider.md#kernel-nfs-adapter). +The initial supported independent filesystem combination is microsandbox with the [kernel NFS adapter](./workspace-provider.md#kernel-nfs-adapter). For a new installation, prepare storage and service accounts first, expose the mount to Core, select storage, configure microsandbox, then enroll nodes. Mount the same NFSv4.2 export on the Linux Core host and every participating node before starting their services, at the same absolute path, for example `/srv/oac-workspaces`. The operator manages the export, mount availability and service startup ordering. Use a trusted-client AUTH_SYS export with `root_squash`; do not enable `no_root_squash` or broaden permissions to make a check pass. Prepare the namespace and service principals according to the adapter's [ownership requirements](./workspace-provider.md#kernel-nfs-adapter). Core's Compose service already runs as UID 65532. Before adding a node, prepare `oac-node` with the same UID, home `/var/lib/oac-node`, a nologin shell and a nonzero primary group. Its supplementary groups may contain only its primary group, `docker` and `kvm`; an existing home must belong to that account. The installer adopts this account. Without a precreated account it allocates a system UID, which need not match Core. Resolve an existing UID or account conflict as a host administration task before enrollment; changing a running account's UID is not part of storage setup. @@ -121,7 +121,7 @@ services: create_host_path: false ``` -Select storage using `PUT /core/v1/workspace-storage` with the Core key as described in the [administration API](../contracts/agents-api/admin-api.md#workspace-storage). Generate separate canonical UUIDs for the immutable configuration `id` and the namespace marker, then submit this shape with your actual values: +Select storage using `PUT /core/v1/workspace-storage` with the Core key as described in the [administration API](../contracts/agents-api/admin-api.md#workspace-storage). For first-time setup, generate separate canonical UUIDs for the immutable configuration `id` and the namespace marker, then submit this shape with your actual values. Restores retain their existing identities. Selection runs the adapter's availability, ownership and xattr checks in Core; it does not prove that every node can resolve the mount: ```json { @@ -135,7 +135,7 @@ Select storage using `PUT /core/v1/workspace-storage` with the Core key as descr } ``` -Then create or update the microsandbox deployment through the [sandbox deployment API](../contracts/agents-api/sandbox-deployment.md#routes), setting `resources.environment_disk_mib` to `0` and retaining the required compute resources, Runtime and provider configuration. A positive value requests a quota that this adapter does not enforce and is rejected. Deployment workspace requirements and each Session's attachment are derived from the selected database configuration; do not add a filesystem field to the deployment or a second storage configuration to node files. Web has no workspace storage editor. +Then create or update the microsandbox deployment through the [sandbox deployment API](../contracts/agents-api/sandbox-deployment.md#routes), setting `resources.environment_disk_mib` to `0` and retaining the required compute resources, Runtime and provider configuration. A positive value requests a quota that this adapter does not enforce and is rejected. Deployment workspace requirements and each Session's attachment are derived from the selected database configuration; do not add a filesystem field to the deployment or a second storage configuration to node files. Web has no workspace storage editor. After the filesystem and deployment are selected, use **Nodes → Add node** to enroll the prepared hosts. Connected, ready compute alone does not qualify an external filesystem; validate actual Session creation and access through each participating node before admitting application traffic. Include the independent namespace in the [stopped-write backup and restore procedure](./getting-started/operations.md#back-up). ### Node capacity diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index 55760e76b..33cadfe6a 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -121,19 +121,43 @@ Core records which key made each public resource write; the retention of that hi ## Back up -Back up these together; a restore needs all of them: +A recoverable backup contains one consistent, stopped-write set. A database dump or Core volume export alone does not include independent workspace storage or node compute state. Use the same release when restoring; see [installation version policy](#installation-version-policy). -- the Docker volume `_data`, including its `database/`, `secrets/` and `state/` directories. It holds Projects, key digests, nodes, default models, encrypted credentials and all execution history, including large objects. A logical dump: +### Backup contents - ```sh - docker compose -f "$HOME/.oac/core/compose.yaml" exec -T database \ - pg_dump -U agents_api agents_api > oac-backup.sql - ``` +- The Docker volume `_data`, including `database/`, `secrets/` and `state/`. It holds Projects, key digests, nodes, default models, encrypted credentials and execution history, including large objects. Keep `secrets/core/credential.key` with its matching database or stored credentials cannot be decrypted. +- The installation directory, including `.env`, `compose.yaml`, Compose overrides and the management command. +- Every independent workspace namespace selected by current or retained objects, even when it is outside the installation volume. Back up its entire root, including `.oac-storage-root`, object identities, staging, live data, trash and deletion markers. Preserve numeric ownership, modes, links and extended attributes, including every `user.*` attribute. Copying only `live/data` loses lifecycle evidence and can resurrect deleted identities. [Workspace storage](../workspace-provider.md#kernel-nfs-adapter) owns the layout and ownership requirements. +- Each node's state directory, `/var/lib/oac-node/.oac/nodes//`, and its provider storage: Docker volumes or microsandbox's store. Include all compute and snapshot resources still referenced by the database. See [when a node host fails](./nodes.md#when-a-node-host-fails). -- the installation directory containing `.env`, `compose.yaml` and the command. The data volume's `secrets/core/credential.key` must stay with the database, or stored credentials cannot be decrypted. -- each node's state directory on its host, `/var/lib/oac-node/.oac/nodes//`, with its provider storage: Docker volumes or microsandbox's store. See [when a node host fails](./nodes.md#when-a-node-host-fails) for restoring them. +### Establish a stopped-write window -Stop with `docker compose stop`, export the complete data volume and archive the installation directory, then `docker compose start`. Docker Desktop supports volume export from its **Volumes** view. A SQL dump alone does not include the encryption key or Provider state. +1. Block new application input, live file access and administrative mutations at the installation's ingress. Let active execution, initialization, file operations and native cleanup finish. Stop other applications or host processes that can write to the same filesystem. +2. While Core and nodes remain available, confirm that every owned writer has stopped and all Create, Kill, snapshot cleanup and filesystem mutations have settled. An offline node, timeout, missing instance or stopped service is insufficient. A confirmed suspension stops its source compute but retains a checkpoint that must be included in the backup. For the simplest cold recovery baseline, wait until eligible retained Sessions have completed checkpoint expiry cleanup and their allocations are released; retain their filesystem objects and native history under the [cold replacement contract](../sandbox-provider.md#cold-replacement-after-checkpoint-retention). +3. Stop Core and Web, then stop the node control services to prevent new lifecycle work. Recheck the exact native resources and keep all writers stopped until every part of the backup is complete. **Stopping Core or a node service does not stop its sandboxes.** The node service uses `KillMode=process`; microVMs and other provider-managed compute may continue to run independently. +4. Flush the stopped filesystem's pending writes using the storage platform's supported procedure and take its snapshot or export. Export the Core data volume and installation directory, plus required node state and provider stores, within the same window. Stop PostgreSQL before copying its `database/` files. Record the release, backup time, component inventory and checksums together, and keep the backup outside the installation being backed up. +5. Restore service availability only after all copies finish. Bring up storage before Core and nodes, verify readiness, then reopen ingress. Do not replay uncertain native work to make a backup pass. + +There is no one-command, lossless maintenance drain. Archive and deployment reset change Session lifecycle state; they are not substitutes for a backup pause that preserves resumability. If any writer or mutation remains unknown, keep the backup incomplete and resolve that ownership first. + +For a logical database copy during the stopped-write window, leave only the database service running and use: + +```sh +docker compose -f "$HOME/.oac/core/compose.yaml" exec -T database \ + pg_dump -U agents_api agents_api > oac-backup.sql +``` + +A logical dump still needs the matching secrets, state, independent filesystems and required node resources. Docker Desktop's **Volumes** export is a way to copy a stopped data volume; it does not coordinate these other components. + +### Restore and rehearse + +Keep application ingress and Core/node services stopped while restoring. Fence or shut down every old writer before exposing the restored filesystem; an unreachable old host is not a fence. A rehearsal must use an isolated installation and storage copy that cannot connect to the original nodes or share their writable namespace. + +Restore matching secrets and database state, the complete independent filesystem namespaces and any required node identities/provider stores before starting Core. Preserve the database's configuration and namespace UUIDs, object identities and deletion markers; do not run first-time namespace initialization over restored storage. Reestablish the configured mount paths and service UIDs, then validate the mounted filesystem, ownership and xattrs from each participating service context. [Independent workspace storage](../configuration.md#independent-workspace-storage) owns deployment setup. + +The cold released baseline resumes only qualified native history after new compute admission. A backup containing retained checkpoints also depends on the matching native runtime, node identity, provider store and external filesystem identities. File-level restore can change inode identity and prevent strict checkpoint restore even when file bytes match. This procedure does not promise portable memory snapshots or restoration onto an arbitrary replacement node; preserve unresolved ownership rather than substituting a new sandbox. + +Start the database, Core and node services only after their restored dependencies are complete; keep ingress restricted while checking them. In an isolated rehearsal, verify a retained Session's files and native continuation on fresh compute where qualified, confirm that deleted objects cannot be reopened, and check that no old writer can access the restored namespace. Record the results before relying on the backup for recovery. ## Uninstall diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index edbaefa62..7cf303515 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: 75ead1a81873a6fd67451b9beacb530bf156cdeefcc8dd0add74d5c5d4404484 +source_hash: 6bfcaa201c332aa1907f6ab7b0e0ffde4c8f635b363dbea2d0633947615e7264 --- Core 安装的每项设置都恰好只有一个归属位置，分属以下三类： @@ -107,7 +107,7 @@ Runtime 在发现 Harness 适配器之前解析资源目录。`` ### 独立工作区存储 {#independent-workspace-storage} -首个支持的独立文件系统组合是 microsandbox 与[内核 NFS 适配器](./workspace-provider.md#kernel-nfs-adapter)。启动服务前，在 Linux Core 主机和所有参与节点上将同一个 NFSv4.2 导出挂载到相同的绝对路径，例如 `/srv/oac-workspaces`。运维人员负责导出、挂载可用性和服务启动顺序。使用带有 `root_squash` 的受信任客户端 AUTH_SYS 导出；不要启用 `no_root_squash` 或放宽权限来使检查通过。按照适配器的[所有权要求](./workspace-provider.md#kernel-nfs-adapter)准备命名空间和服务身份。 +首个支持的独立文件系统组合是 microsandbox 与[内核 NFS 适配器](./workspace-provider.md#kernel-nfs-adapter)。新安装应先准备存储和服务账户，再向 Core 暴露挂载、选择存储、配置 microsandbox，最后注册节点。启动服务前，在 Linux Core 主机和所有参与节点上将同一个 NFSv4.2 导出挂载到相同的绝对路径，例如 `/srv/oac-workspaces`。运维人员负责导出、挂载可用性和服务启动顺序。使用带有 `root_squash` 的受信任客户端 AUTH_SYS 导出；不要启用 `no_root_squash` 或放宽权限来使检查通过。按照适配器的[所有权要求](./workspace-provider.md#kernel-nfs-adapter)准备命名空间和服务身份。 Core 的 Compose 服务已使用 UID 65532 运行。添加节点前，创建同 UID 的 `oac-node`，主目录为 `/var/lib/oac-node`，使用 nologin shell 和非零主组。附加组只能包含其主组、`docker` 和 `kvm`；已有主目录必须属于该账户。安装器会接管此账户。如果没有预创建账户，安装器分配的系统 UID 不一定与 Core 一致。注册前应通过主机管理流程解决已有 UID 或账户冲突；修改运行中账户的 UID 不属于存储配置步骤。 @@ -125,7 +125,7 @@ services: create_host_path: false ``` -按[管理 API](../../contracts/agents-api/zh/admin-api.md#workspace-storage)说明，使用 Core key 调用 `PUT /core/v1/workspace-storage` 选择存储。分别为不可变配置 `id` 和命名空间标记生成规范 UUID，然后使用实际值提交以下结构： +按[管理 API](../../contracts/agents-api/zh/admin-api.md#workspace-storage)说明，使用 Core key 调用 `PUT /core/v1/workspace-storage` 选择存储。首次设置时，分别为不可变配置 `id` 和命名空间标记生成规范 UUID，然后使用实际值提交以下结构。恢复时保留已有标识。选择操作会在 Core 中运行适配器的可用性、所有权和 xattr 检查；它不证明每个节点均能解析该挂载： ```json { @@ -139,7 +139,7 @@ services: } ``` -随后通过[沙箱部署 API](../../contracts/agents-api/zh/sandbox-deployment.md#routes)创建或更新 microsandbox 部署，将 `resources.environment_disk_mib` 设为 `0`，并保留所需计算资源、Runtime 和 Provider 配置。正数表示请求配额，本适配器不实施此配额，因此会拒绝。部署的工作区要求和每个 Session 的挂载凭据均从已选数据库配置派生；不要向部署添加文件系统字段，也不要在节点文件中添加第二份存储配置。Web 没有工作区存储编辑器。 +随后通过[沙箱部署 API](../../contracts/agents-api/zh/sandbox-deployment.md#routes)创建或更新 microsandbox 部署，将 `resources.environment_disk_mib` 设为 `0`，并保留所需计算资源、Runtime 和 Provider 配置。正数表示请求配额，本适配器不实施此配额，因此会拒绝。部署的工作区要求和每个 Session 的挂载凭据均从已选数据库配置派生；不要向部署添加文件系统字段，也不要在节点文件中添加第二份存储配置。Web 没有工作区存储编辑器。选择文件系统和部署后，通过 **Nodes → Add node** 注册已准备的主机。计算资源已连接且就绪并不足以证明外部文件系统具备资格；准入应用流量前，通过每个参与节点验证实际 Session 创建和访问。将独立命名空间纳入[停止写入后的备份恢复流程](./getting-started/operations.md#back-up)。 ### 节点容量 {#node-capacity} diff --git a/docs/zh/getting-started/operations.md b/docs/zh/getting-started/operations.md index e1b4d1112..8ab281776 100644 --- a/docs/zh/getting-started/operations.md +++ b/docs/zh/getting-started/operations.md @@ -1,7 +1,7 @@ --- title: "运维" source: docs/getting-started/operations.md -source_hash: 97f5d49b9b39441a78026433d662120d46471ef5f9a0b6b4508998a3b4491d09 +source_hash: acd6e73b0bdf43f3bb5adfc375c8938d1b8169a326620076242c844ab8b4766b --- 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。 @@ -123,20 +123,43 @@ Core 记录每次公开资源写入所使用的密钥；历史保留策略为 [` ## 备份 {#back-up} -一起备份这些内容；恢复时全部需要： +可恢复的备份必须是一组在停止写入期间取得的一致数据。仅数据库转储或 Core 卷导出不包含独立工作区存储及节点计算状态。恢复时使用相同版本；参见[安装版本策略](#installation-version-policy)。 -- Docker 卷 `_data`，包括其中的 `database/`、`secrets/` 和 `state/` 目录。其中包含 Project、密钥摘要、节点、默认模型、加密凭据和全部执行历史（含大对象）。逻辑备份： +### 备份内容 {#backup-contents} - ```sh - docker compose -f "$HOME/.oac/core/compose.yaml" exec -T database \ - pg_dump -U agents_api agents_api > oac-backup.sql - ``` +- Docker 卷 `_data`，包括 `database/`、`secrets/` 和 `state/`。其中保存 Project、密钥摘要、节点、默认模型、加密凭据和执行历史（含大对象）。`secrets/core/credential.key` 必须与匹配的数据库一起保留，否则存储的凭据无法解密。 +- 安装目录，包括 `.env`、`compose.yaml`、Compose 覆盖文件和管理命令。 +- 当前或保留对象所使用的每个独立工作区命名空间，即使其位于安装卷之外。备份整个根目录，包括 `.oac-storage-root`、对象标识、staging、live 数据、trash 和删除标记。保留数值所有者、权限、链接及扩展属性，包括全部 `user.*` 属性。仅复制 `live/data` 会丢失生命周期证据，并可能使已删除标识复活。[工作区存储](../workspace-provider.md#kernel-nfs-adapter)负责定义布局与所有权要求。 +- 各节点状态目录 `/var/lib/oac-node/.oac/nodes//` 及其 Provider 存储：Docker 卷或 microsandbox 存储。包含数据库仍然引用的全部计算资源和快照。参见[节点主机故障时](./nodes.md#when-a-node-host-fails)。 -- 安装目录中的 `.env`、`compose.yaml` 和管理命令。数据卷内的 `secrets/core/credential.key` 必须与数据库一起保留，否则存储的凭据无法解密。 +### 建立停止写入窗口 {#establish-a-stopped-write-window} -- 各节点主机上的状态目录 `/var/lib/oac-node/.oac/nodes//` 及提供商存储：Docker 卷或 microsandbox 存储。恢复方法见[节点主机故障时](nodes.md#when-a-node-host-fails)。 +1. 在安装入口阻止新应用 input、实时文件访问和管理变更。等待活动执行、初始化、文件操作和原生清理完成。停止其他能够写入同一文件系统的应用或主机进程。 +2. 保持 Core 和节点可用，确认每个所属写入方均已停止，且所有 Create、Kill、快照清理及文件系统变更均已结算。节点离线、超时、实例缺失或服务停止均不足以证明这一点。确认挂起会停止源计算资源，但保留的检查点也必须纳入备份。最简单的冷恢复基线是等待符合条件的保留 Session 完成检查点到期清理、allocation 已释放，再按[冷替换契约](../sandbox-provider.md#cold-replacement-after-checkpoint-retention)保留文件系统对象及原生历史。 +3. 停止 Core 和 Web，再停止节点控制服务，以防止新的生命周期操作。重新核实精确的原生资源，保持所有写入方停止，直到备份各部分全部完成。**停止 Core 或节点服务不会停止其沙箱。** 节点服务使用 `KillMode=process`；microVM 及其他由 Provider 管理的计算资源可能继续独立运行。 +4. 使用存储平台支持的流程将已停止写入的文件系统中的待写数据落盘，并创建其快照或导出。在同一窗口内导出 Core 数据卷、安装目录及所需的节点状态和 Provider 存储。复制 PostgreSQL 的 `database/` 文件前必须停止 PostgreSQL。一起记录版本、备份时间、组成清单和校验值，并将备份保存在被备份安装之外。 +5. 所有复制完成后才恢复服务。先恢复存储，再启动 Core 和节点，验证就绪后开放入口。不要通过重放不确定的原生操作来让备份通过。 -运行 `docker compose stop`，导出完整数据卷并归档安装目录，再运行 `docker compose start`。Docker Desktop 的 **Volumes** 页面支持导出数据卷。SQL 转储不包含加密密钥和 Provider 状态。 +当前没有一条命令即可完成且无损的维护排空流程。归档和部署重置会改变 Session 生命周期状态，不能替代保持续接能力的备份暂停。任何写入方或变更仍然未知时，应将备份保留为未完成状态，先解决相应归属。 + +若在停止写入窗口中进行数据库逻辑复制，仅保持 database 服务运行，执行： + +```sh +docker compose -f "$HOME/.oac/core/compose.yaml" exec -T database \ + pg_dump -U agents_api agents_api > oac-backup.sql +``` + +逻辑转储仍然需要匹配的 secrets、state、独立文件系统及所需节点资源。Docker Desktop 的 **Volumes** 导出可用于复制已停止的数据卷，但不会协调这些其他部分。 + +### 恢复与演练 {#restore-and-rehearse} + +恢复期间保持应用入口及 Core/节点服务关闭。暴露恢复后的文件系统前，必须隔离或关闭每一个旧写入方；旧主机不可达不代表已隔离。演练必须使用隔离的安装和存储副本，不能连接原节点，也不能共享其可写命名空间。 + +启动 Core 前，恢复相匹配的 secrets 和数据库状态、完整的独立文件系统命名空间，以及任何所需的节点标识和 Provider 存储。保留数据库中的配置和命名空间 UUID、对象标识及删除标记；不要对恢复后的存储执行首次命名空间初始化。重新建立配置中的挂载路径和服务 UID，然后从每个参与服务的上下文验证所挂载文件系统、所有权及 xattr。[独立工作区存储](../configuration.md#independent-workspace-storage)负责定义部署设置。 + +已释放计算资源的冷恢复基线仅在新计算资源准入后续接具备资格的原生历史。包含保留检查点的备份还依赖匹配的原生 Runtime、节点标识、Provider 存储和外部文件系统标识。文件级恢复可能改变 inode 标识，即使文件字节一致，也可能阻止严格检查点恢复。此流程不承诺可移植内存快照，也不承诺恢复到任意替代节点；应保留未解决的归属，而不是替换为新沙箱。 + +恢复依赖全部完整后才启动 database、Core 和节点服务；检查期间保持入口受限。在隔离演练中，验证保留 Session 的文件和具备资格时在新计算资源上的原生续接，确认已删除对象无法重新打开，并检查旧写入方均无法访问恢复后的命名空间。在依赖备份进行恢复前记录结果。 ## 卸载 {#uninstall} From 40766bf3e540bba247db4ee634884becb25c6ad5 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 00:51:05 +0800 Subject: [PATCH 10/22] Document workspace capacity isolation limits (#648) --- docs/workspace-provider.md | 2 ++ docs/zh/workspace-provider.md | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/workspace-provider.md b/docs/workspace-provider.md index 564a9b854..9900d0d26 100644 --- a/docs/workspace-provider.md +++ b/docs/workspace-provider.md @@ -55,6 +55,8 @@ Construction validates and canonicalizes JSON without filesystem I/O. `Check` ve The declaration is `host_directory`, `user_xattr: true`, `capacity_quota: false`. The native attachment contains only the namespace identity; resolution validates the complete binding against the immutable configuration and filesystem ownership before returning `objects//live/data` beneath the root. Only this data directory is exposed to the guest. Neither tenant nor application input supplies a host path. +The NFS adapter enforces no per-Environment or per-tenant byte or inode quota. One object can exhaust the shared filesystem, preventing writes to other objects and the durable terminal metadata required for safe deletion. Operators must monitor free bytes and inodes and maintain headroom for lifecycle metadata and cleanup. A capacity bound on the whole namespace does not isolate capacity between tenants or objects. + Each object owns `objects//identity`, `staging/`, `live/`, `trash/` and, after deletion begins, `deleted-marker`. The permanent identity binds the tenant, Environment and object UUIDs. Create prepares and syncs a private unique staging envelope before atomically publishing its nonempty directory as `live`; replay cannot overwrite existing live data. Delete first records a durable object-local terminal marker, retires live data to unique object-local trash and removes data before its ownership markers. Concurrent deleters converge. Minimal identity and deletion metadata remain intentionally; deletion does not mean every metadata file disappears. Create and Resolve refuse the retired identity. A late in-flight Create can leave empty controlled metadata requiring a repeated Delete after its syscall settles; it cannot authorize a new writer or reuse the object identity. Cleanup examines only that object's staging and trash, without a namespace-wide sweep or background service. NFS storage does not supply compute fencing or automatic distributed failover. Guest `flock` is not a cross-VM writer lock. The single-writer and explicit-deletion requirements above remain mandatory. Confirmed release can permit a fresh VM on another compatible node to attach the same object, but an offline node or an unknown Create, Kill or snapshot-cleanup result never proves that the old writer stopped. Native history and database behavior require qualification of the selected Runtime, Harness and filesystem combination; file persistence alone is not a guarantee of native Session recovery, cross-node memory restore or arbitrary crash durability. diff --git a/docs/zh/workspace-provider.md b/docs/zh/workspace-provider.md index 16e2cbe55..8c681069a 100644 --- a/docs/zh/workspace-provider.md +++ b/docs/zh/workspace-provider.md @@ -1,7 +1,7 @@ --- title: 工作区文件系统 Provider source: docs/workspace-provider.md -source_hash: 39d26eb96b553c925a46d1a2ae90fe4dc3d93fb21d5b9883e55656fd62e0f0fb +source_hash: 21eb220ac59a477a8e799a6de20e01b339536901e8c9685695d2ac498b1b7bdb --- 独立工作区文件系统边界由 [`workspacefs.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/workspacefs/workspacefs.go) 定义。本文规定必需的集成契约，并不表示所有 Sandbox Provider 或执行位置均已实现。文件系统适配器拥有存储对象与原生挂载解析职责；[Sandbox Provider](sandbox-provider.md) 拥有计算资源职责。Core 在分配任一资源前选择并验证二者的组合。 @@ -57,6 +57,8 @@ source_hash: 39d26eb96b553c925a46d1a2ae90fe4dc3d93fb21d5b9883e55656fd62e0f0fb 能力声明为 `host_directory`、`user_xattr: true`、`capacity_quota: false`。原生挂载凭据仅包含命名空间标识；解析时根据不可变配置和文件系统所有权验证完整 binding，随后返回根目录下的 `objects//live/data`。仅此数据目录暴露给 guest。租户和应用输入都不提供主机路径。 +NFS 适配器不实施按 Environment 或租户划分的字节或 inode 配额。单个对象可能耗尽共享文件系统，阻止其他对象写入，也可能阻止写入安全删除所需的持久终态元数据。运维人员必须监测可用字节和 inode，并为生命周期元数据和清理保留余量。对整个命名空间设置容量上限，并不提供租户或对象之间的容量隔离。 + 每个对象拥有 `objects//identity`、`staging/`、`live/`、`trash/`，以及删除开始后的 `deleted-marker`。永久标识绑定租户、Environment 和对象 UUID。Create 先准备并同步唯一的私有 staging 封装目录，再原子发布其非空目录为 `live`；重放不会覆盖现有 live 数据。Delete 先持久记录对象本地的终态标记，将 live 数据退役到唯一的对象本地 trash，再先删除数据、后删除其所有权标记。并发删除者收敛。最少量的标识和删除元数据会有意保留；删除不代表所有元数据文件均消失。Create 和 Resolve 拒绝已退役标识。迟到的在途 Create 可能留下空的受控元数据，需要在其系统调用完成后重复 Delete；它不能授权新写入者或复用对象标识。清理只查看该对象的 staging 和 trash，不全量扫描命名空间，也不使用后台服务。 NFS 存储不提供计算 fencing 或自动分布式故障转移。Guest `flock` 不是跨 VM 写入者锁。上述单写入者和显式删除要求仍然必须遵守。确认释放后，可以允许另一兼容 node 上的新 VM 挂载同一对象，但 node 离线或 Create、Kill、快照清理结果未知均不证明旧写入方已停止。原生历史和数据库行为需要对所选 Runtime、Harness、文件系统组合进行资格验证；仅保留文件不保证原生 Session 恢复、跨 node 内存恢复或任意崩溃后的持久性。 From ba4047d3abff73d6d0f6ff2c40c0bf40f1b0a151 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 00:56:32 +0800 Subject: [PATCH 11/22] Queue sessions for available compute (#646) * Queue sessions for available compute * Align hosted creation test with deferred placement --- contracts/agents-api/environments.md | 2 + contracts/agents-api/sandbox-deployment.md | 4 +- contracts/agents-api/zh/environments.md | 4 +- contracts/agents-api/zh/sandbox-deployment.md | 6 +- docs/sandbox-provider.md | 8 +- docs/zh/sandbox-provider.md | 10 +- services/core/IMPLEMENTATION.md | 4 +- services/core/cmd/server/main.go | 2 +- .../db/queries/runtime_lifecycle_nodes.sql | 32 +- .../db/queries/runtime_suspension.sql | 4 +- .../queries/runtime_suspension_pressure.sql | 19 + services/core/internal/db/queries/turns.sql | 2 +- .../db/sqlc/runtime_lifecycle_nodes.sql.go | 65 +++- .../db/sqlc/runtime_suspension.sql.go | 7 +- .../sqlc/runtime_suspension_pressure.sql.go | 74 ++++ services/core/internal/db/sqlc/turns.sql.go | 5 +- .../core/internal/deployment/allocation.go | 19 + .../core/internal/deployment/allocations.go | 23 +- .../internal/deployment/allocations_test.go | 120 +++++- services/core/internal/deployment/errors.go | 2 + .../core/internal/deployment/execution.go | 6 +- .../core/internal/deployment/fakes_test.go | 4 +- .../core/internal/deployment/replacement.go | 102 +++-- .../internal/deployment/retention_pressure.go | 44 +++ .../core/internal/deployment/service_test.go | 7 +- .../deployment/session_archive_test.go | 3 +- services/core/internal/deployment/storage.go | 20 +- .../deployment/suspension_pressure.go | 124 +++++++ .../execution/deployment_fixture_test.go | 13 +- .../internal/execution/runtime_compute.go | 8 +- .../execution/runtime_compute_wake.go | 11 +- .../internal/execution/runtime_lifecycle.go | 19 +- .../internal/execution/runtime_manager.go | 9 +- .../internal/execution/runtime_pending.go | 59 ++- .../runtime_placement_demand_test.go | 146 ++++++++ .../execution/runtime_replacement_test.go | 6 +- .../runtime_workspace_settlement_test.go | 5 + services/core/internal/execution/worker.go | 2 +- .../core/internal/execution/worker_wakeup.go | 6 + .../internal/execution/worker_wakeup_test.go | 11 +- .../postgres/deploymentpg/allocations.go | 63 +++- .../postgres/deploymentpg/fixture_test.go | 3 +- .../deploymentpg/placement_demand_test.go | 215 +++++++++++ .../deploymentpg/retention_pressure_test.go | 143 +++++++ .../postgres/deploymentpg/setup_test.go | 3 +- .../deploymentpg/suspension_pressure.go | 123 ++++++ .../deploymentpg/suspension_pressure_test.go | 351 ++++++++++++++++++ .../persistence/postgres/pgtest/lease.go | 3 +- .../persistence/postgres/pgtest/lease_test.go | 26 ++ .../persistence/postgres/pgunit/lease_test.go | 3 +- .../postgres/sessionpg/creation.go | 13 - .../postgres/sessionpg/creation_test.go | 22 +- .../sessionpg/execution_inputs_test.go | 3 +- .../persistence/postgres/sessionpg/session.go | 2 +- .../providers/configuration_flow_test.go | 3 +- services/core/internal/sessions/creation.go | 33 +- .../core/internal/sessions/creation_test.go | 153 ++++---- .../core/internal/sessions/transaction.go | 2 + .../integration/admin_session_archive_test.go | 9 + .../deployment_public_fixture_test.go | 3 +- .../device_bootstrap_binding_test.go | 2 +- .../core/tests/integration/execution_test.go | 3 +- .../integration/runtime_idle_clock_test.go | 5 +- .../runtime_lifecycle_nodes_test.go | 14 +- .../runtime_node_generations_test.go | 37 +- .../runtime_node_lifecycle_fixture_test.go | 8 +- .../tests/integration/runtime_nodes_test.go | 96 ++++- .../integration/runtime_observation_test.go | 3 + .../integration/runtime_suspension_test.go | 5 +- .../sandbox_deployment_worker_test.go | 8 +- .../integration/sandbox_generations_test.go | 3 + .../tests/integration/sandbox_reset_test.go | 3 + .../sandbox_specification_lifecycle_test.go | 23 +- .../integration/session_deletion_test.go | 3 + .../session_devices_fixture_test.go | 1 + 75 files changed, 2065 insertions(+), 342 deletions(-) create mode 100644 services/core/internal/db/queries/runtime_suspension_pressure.sql create mode 100644 services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go create mode 100644 services/core/internal/deployment/retention_pressure.go create mode 100644 services/core/internal/deployment/suspension_pressure.go create mode 100644 services/core/internal/execution/runtime_placement_demand_test.go create mode 100644 services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go create mode 100644 services/core/internal/persistence/postgres/deploymentpg/retention_pressure_test.go create mode 100644 services/core/internal/persistence/postgres/deploymentpg/suspension_pressure.go create mode 100644 services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go create mode 100644 services/core/internal/persistence/postgres/pgtest/lease_test.go diff --git a/contracts/agents-api/environments.md b/contracts/agents-api/environments.md index 195c5e3ff..099243c5f 100644 --- a/contracts/agents-api/environments.md +++ b/contracts/agents-api/environments.md @@ -92,6 +92,8 @@ The application owns the machine. It creates the Session with a clean absolute ` Session input goes through `POST /agents/sessions/{id}/events` in ordered batches of 1–64 events. On an Environment-bearing Session, Core reserves input that cannot start yet until the Environment is connected and prepared. +A valid node-backed creation can wait for compute capacity after committing. Initial input uses the same reservation and deadline while waiting; creating without input still requests Environment preparation. [Placement and generation selection](./sandbox-deployment.md#generation-ownership-and-rollout) define the scheduling rules. + ### Initial input Creation accepts initial text as a string or an ordered array of user messages. Omitted or null input creates no Turn and no connection action. diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 855e90d22..b3b79721e 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -122,7 +122,7 @@ No provider text or credential is returned. The write and its `change` or `repla ## Generation ownership and rollout -`runtime_deployment` holds the current specification. Superseded rows keep only immutable specification, build and endpoint metadata, never another E2B key. An E2B allocation binds its generation at reservation; a node placement binds at Session admission, and its allocation copies that generation, even across later updates. Inspection, renewal, commands and cleanup use the allocation's original specification and endpoint with the current key; a missing generation never falls back to the current specification. Released generation identifiers stay reserved. +`runtime_deployment` holds the current specification. Superseded rows keep only immutable specification, build and endpoint metadata, never another E2B key. An E2B allocation binds its generation at reservation; a node placement binds when the scheduler reserves compatible capacity, and its allocation copies that generation, even across later updates. Inspection, renewal, commands and cleanup use the allocation's original specification and endpoint with the current key; a missing generation never falls back to the current specification. Released generation identifiers stay reserved. A generation is retained while it is current, referenced by an unreleased allocation or placement, or pinned by a node that is not removed. A node's durable serving pin survives offline periods and zero resources and is separate from its current readiness. Collection shares the deployment lock with updates and admission and deletes at most 32 eligible generation rows per pass. Reset retires pins and clears superseded rows only after confirmed release. @@ -142,7 +142,7 @@ A node is online while it is connected under the current owner epoch with a hear Each node adds `rollout: {state, ready_generation, diagnostic?}`, where `ready_generation` is the nullable durable serving pin and `diagnostic` a fixed code for the target generation; allocation items add `deployment_generation`. Poll every five seconds only while `rollout.state` is `preparing` or `reset` is not null; old Sessions and failed, update-required or offline nodes alone do not keep polling active. -New admission filters nodes by online presence, exact serving-generation readiness, address and shared capacity before it prefers the newest qualifying pin, so a full newest node never hides a free older one. Without a candidate, admission creates no provisional Session or placement: an online node that is actually preparing with free capacity gives 503 `sandbox_nodes_preparing`, and a full or offline fleet gives `runtime_node_unavailable`. +Node-backed creation validates the target deployment and commits the Session, pending Environment and any initial input without reserving compute. A full, offline or preparing fleet leaves that accepted work waiting; an unconfigured deployment, reset or unsupported combination still rejects admission. The common scheduler uses bounded rotating scans of unplaced demand, with pages ordered by recorded time and Environment ID and a fixed time boundary for each scan so continuous arrivals cannot prevent it from revisiting older work. It checks online presence, exact serving-generation readiness, address, shared capacity and the selected generation's Harness/filesystem compatibility, then prefers the newest qualifying pin. Placement is immutable once reserved. A bounded scan skips temporarily unavailable demand and resumes after a restart. Existing suspended allocations restore on their original node through the same capacity lock; this is not a global fairness guarantee across hot restores and unplaced work. Input retains its [original five-minute deadline](./environments.md#reservations), including time waiting for capacity. ## Reset diff --git a/contracts/agents-api/zh/environments.md b/contracts/agents-api/zh/environments.md index 35b847a49..570681196 100644 --- a/contracts/agents-api/zh/environments.md +++ b/contracts/agents-api/zh/environments.md @@ -1,7 +1,7 @@ --- title: "环境与模板" source: contracts/agents-api/environments.md -source_hash: ac67b8c7a651bae18588cd5c845b40ab4d9c89382ae9f2599a039a27f300c1c2 +source_hash: 48b9d8679a983adc5eb9164c9a94e1137bf75800fbb1a0f81a0ae1b3cfccdafb --- Environment 是 Session 的执行资源，包括 Harness 运行所在的机器、工作区以及已完成准备的能力。Session 通过其 `environment` 配置创建 Environment；不存在独立的 create 调用。Environment Template 是 Session 创建时解析的可复用准备配置。本契约涵盖这两类资源、两种放置方式、输入接纳、能力准备、Skills、Plugins 和 MCP 连接来源。 @@ -94,6 +94,8 @@ Core 在 Session 创建事务中创建 Environment 记录；Session upsert 会 Session 输入通过 `POST /agents/sessions/{id}/events` 以每次 1–64 个事件的有序批次提交。对于带 Environment 的 Session，Core 会将尚不能启动的输入预留，直到 Environment 完成连接和准备。 +有效的节点模式创建可以在提交后等待计算容量。初始输入在等待期间仍使用同一 reservation 和期限；无输入创建也会请求 Environment 准备。[Placement 与代次选择](./sandbox-deployment.md#generation-ownership-and-rollout) 定义调度规则。 + ### 初始输入 {#initial-input} 创建操作接受字符串形式的初始文本，或由用户消息构成的有序数组。省略或传入 null 输入不会创建 Turn 或连接操作。 diff --git a/contracts/agents-api/zh/sandbox-deployment.md b/contracts/agents-api/zh/sandbox-deployment.md index b886dab3e..db6d81c6c 100644 --- a/contracts/agents-api/zh/sandbox-deployment.md +++ b/contracts/agents-api/zh/sandbox-deployment.md @@ -1,7 +1,7 @@ --- title: "沙箱部署" source: contracts/agents-api/sandbox-deployment.md -source_hash: 39ac8d674a737096598c9c31e8459b60a7e3a58dffc3f5b0258714ad5b05801e +source_hash: 245f1791cf2e42b3a40c91aa0df4f390a44709db5e0ffbcae1e12dc4170344fe --- 沙箱部署为 Core 管理的 `openai_hosted` 执行选择 Sandbox Provider、每个沙箱的资源以及不可变的 Runtime 发行版。PostgreSQL 为每个安装维护一个当前有效选择；Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径，且不能覆盖其资源或 Runtime。该选择独立于 Harness。部署可以保持未配置状态，没有节点；此时它拒绝托管准入。 @@ -125,7 +125,7 @@ POST 会在持久保存候选配置之前对其进行验证，并且不会创建 ## 代次所有权与推出 {#generation-ownership-and-rollout} -`runtime_deployment` 保存当前 specification。被取代的行仅保留不可变的 specification、构建和端点元数据，绝不保留另一个 E2B 密钥。E2B 分配在预留时绑定其代次；节点放置在 Session 准入时绑定，其分配会复制该代次，即使之后发生更新也是如此。检查、续期、命令和清理使用分配的原始 specification 和端点以及当前密钥；缺失代次绝不会回退到当前 specification。已释放代次的标识仍会保留。 +`runtime_deployment` 保存当前 specification。被取代的行仅保留不可变的 specification、构建和端点元数据，绝不保留另一个 E2B 密钥。E2B 分配在预留时绑定其代次；节点放置在调度器预留兼容容量时绑定，其分配会复制该代次，即使之后发生更新也是如此。检查、续期、命令和清理使用分配的原始 specification 和端点以及当前密钥；缺失代次绝不会回退到当前 specification。已释放代次的标识仍会保留。 当一个代次为当前代次、被尚未释放的分配或放置引用，或者被尚未移除的节点固定时，该代次会保留。节点的持久服务固定状态可跨离线时段和零资源状态保留，并与当前就绪状态相互独立。回收操作与更新和准入共享部署锁，每轮最多删除 32 个符合条件的代次行。只有确认释放后，重置才会停用固定状态并清除被取代的行。 @@ -145,7 +145,7 @@ POST 会在持久保存候选配置之前对其进行验证，并且不会创建 每个节点会添加 `rollout: {state, ready_generation, diagnostic?}`，其中 `ready_generation` 是可为 null 的持久服务固定状态，`diagnostic` 是目标代次的固定代码；分配项会添加 `deployment_generation`。仅当 `rollout.state` 为 `preparing` 或 `reset` 非 null 时，才每五秒轮询一次；旧 Session 以及失败、需要更新或离线的节点本身均不会使轮询保持活动状态。 -新的准入操作会先按在线状态、精确的服务代次就绪情况、地址和共享容量筛选节点，再优先选择最新的合格固定状态，因此最新的节点已满时不会掩盖仍有空闲资源的较旧节点。没有候选项时，准入操作不会创建临时 Session 或放置：在线节点若确实正在准备且有空闲容量，则返回 503 `sandbox_nodes_preparing`；节点集群全部已满或离线，则返回 `runtime_node_unavailable`。 +节点模式创建先验证目标部署，提交 Session、pending Environment 及初始输入，但不预留计算资源。节点全部已满、离线或准备中时，已接受的工作继续等待；未配置部署、reset 或不支持的组合仍拒绝准入。共同调度器对尚未放置的需求进行有界循环扫描，分页按需求记录时间和 Environment ID 排序，每轮使用固定时间上界，避免持续的新需求阻止重新访问较早的工作。它检查在线状态、精确服务代次就绪情况、地址、共享容量和所选代次的 Harness/文件系统兼容性，然后优先选择最新的合格固定代次。预留后的 placement 保持不可变。有界扫描跳过暂时不可调度的需求，并在重启后继续。已有 suspended allocation 通过同一容量锁在原节点恢复；这不构成热恢复和未放置工作之间的全局公平性保证。输入保留[原始五分钟期限](./environments.md#reservations)，等待容量的时间也计入其中。 ## 重置 {#reset} diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 58f49dd7f..8e7780e91 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -178,13 +178,15 @@ Before releasing the execution lease, the coordinator stops accepting work and c ### Placement and capacity -Placement is automatic: the Environment-to-node placement commits with Session creation and its retry identity, callers cannot choose a node, and an active allocation keeps its original node even while it is offline. After confirmed allocation release, eligible retained Sessions can reserve compatible capacity again, including on another node. Node capacity counts pending reservations and unresolved resources, and new placement and a suspended-to-restoring transition share a database lock. Unknown operations keep their reservations, source teardown must be confirmed before active capacity is released, and confirmed cleanup releases placement capacity. Retained ownership needs exact provider evidence: a socket path, a missing instance or an empty listing never proves cleanup or authorizes a replacement. +Placement is automatic: Session creation commits durable pending work, and the common scheduler reserves a compatible node before allocation. The [deployment contract](../contracts/agents-api/sandbox-deployment.md#generation-ownership-and-rollout) owns waiting, ordering and generation selection. Callers cannot choose a node, and an active allocation keeps its original node even while it is offline. After confirmed allocation release, eligible retained Sessions can reserve compatible capacity again, including on another node. Node capacity counts pending reservations and unresolved resources, and new placement and a suspended-to-restoring transition share a database lock. Unknown operations keep their reservations, source teardown must be confirmed before active capacity is released, and confirmed cleanup releases placement capacity. Retained ownership needs exact provider evidence: a socket path, a missing instance or an empty listing never proves cleanup or authorizes a replacement. The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and the snapshot retention bound each node. Cold replacement after confirmed release does not provide node-level drain, cross-node memory restore, failover from an unreachable writer, concurrent writers, multi-active Core, autoscaling or snapshot replication. Node removal is refused while the node holds allocations, snapshots, reservations, unknown results or cleanup, and offline ownership is kept. ### Suspension -Core suspends the idle work of every provider that declares checkpoint support, with one fixed policy: it suspends work idle for 5 minutes (300 seconds) and keeps the snapshot for 24 hours (86400 seconds). The deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) reports these values. Core suspends after initialization completes, when no root or Subagent Turn is queued, in progress or waiting, no input or file operation is pending, and real activity has been idle for that time. A Session need not have run a Turn. The idle clock starts no earlier than the allocation entering the running compute phase, including after a wake. For node allocations Core records the first root or child terminal transition with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial snapshot retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. +Core suspends the idle work of every provider that declares checkpoint support, with one shared policy: normally it suspends work idle for 5 minutes (300 seconds) and keeps the snapshot for 24 hours (86400 seconds). The deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) reports these values. Core suspends after initialization completes, when no root or Subagent Turn is queued, in progress or waiting, no input or file operation is pending, and real activity has been idle for that time. A Session need not have run a Turn. The idle clock starts no earlier than the allocation entering the running compute phase, including after a wake. For node allocations Core records the first root or child terminal transition with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial snapshot retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. + +Capacity pressure can suspend an already idle node allocation before the normal timeout, after a 15-second inactivity grace. The Session-locked decision rechecks pending work, wake requests and activity, then takes the shared deployment capacity lock and confirms compatible waiting demand. It never interrupts an active Turn or releases capacity before the ordinary quiesce, capture and source-stop proofs complete. Pressure-triggered reclamation is conservative: an in-flight reclamation delays another only when its node is currently eligible to serve the same demand. Unavailable or incompatible nodes keep their ownership receipts without blocking healthy capacity. If suspension cannot make capacity usable, the waiting request keeps its original deadline. The Worker lease, the Session lock and the per-node gates own suspension for every provider. New Turn claims, file-write intents and capture admission serialize under the Session lock and share one compute-phase check; new pending work cancels a capture and wakes the same source. Normal preparation waits for the compute phase to be running, after the authenticated resume handshake, and pending input stays pending when its promotion conflicts with a lifecycle transition. Compute phases and revision-checked receipts live on the allocation. Core persists quiesce, capture and restore intent before the effect, only a fresh receipt performs a capture or restore, and recovery observes the exact attempt without retrying an unknown creation, capture or restore. A consumed snapshot never rolls a running generation back. Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap, and unknown cleanup identities are kept until owned resources are confirmed absent. Consumed artifacts and old compute are deleted, so suspension cycles never build a chain of writable disks. @@ -194,6 +196,8 @@ Queued work and live Environment file access wake a suspended Environment; histo Snapshot retention bounds compute resources, not the lifetime of an eligible retained Session. After the deadline, Core can clean up the allocation without terminating the Session only when its independent filesystem object is ready, initialization is complete, the Session and Environment remain nonterminal, and the previous authenticated Runtime has a persisted `retained_native_history` declaration. The capability and native recovery obligations are defined by the [Core–Runtime protocol](runtime-protocol.md). Filesystem retention alone is insufficient. +When compatible demand cannot obtain a retained slot, Core may end an idle, stopped allocation’s checkpoint retention early. It rechecks native-history eligibility, pending work and capacity under the Session and deployment locks. The same expiry cleanup must confirm resource deletion before the waiting demand can reserve that slot. The 24-hour retention is an upper bound, not a guarantee of memory preservation under capacity pressure; files and qualified native history remain retained. + Core confirms the original creation is settled and all owned compute and snapshots are cleaned up before releasing allocation and placement ownership. The old device's authority is revoked before replacement admission. Until Create, Kill or DeleteSnapshot has a confirmed outcome, it keeps ownership and admits no replacement writer. A recoverable Session and Environment remain disconnected with the same filesystem object and initialization result. Archive, reset and deletion take precedence and cannot be undone by a wake request. New input or live Environment file access reserves a new allocation on a compatible node and bootstraps a fresh Runtime device. Preparation reuses the same filesystem and completed initialization rather than replaying initialization. Execution resumes the same native Session from retained history after validating the replacement Runtime's declaration. Missing or invalid native history fails explicitly; Core never substitutes a new native Session or replays completed input. History and published Artifact reads do not reserve compute. This path does not preserve VM memory, background processes or open connections, and it does not qualify an undeclared Runtime, Harness or filesystem combination. diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 73f75a0d5..24631a0d6 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: 10483462da09c4fca5d1cb04a219f7d93faa020ea6104536b33dce3deabea1d4 +source_hash: 4a7ce15e8fc2c9519b4974a60c746fbd436b24bbce5ab2ef461e442bea3e5f86 --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -180,13 +180,15 @@ allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan jo ### Placement 与容量 {#placement-and-capacity} -placement 自动完成：Environment-to-node placement 与 Session 创建及其 retry identity 一起提交，调用方不能选择 node；活动 allocation 即使在 node 离线时也保留原 node。确认 allocation 释放后，符合条件的保留 Session 可以重新预约兼容容量，包括其他 node。Node capacity 计入 pending reservation 和未决资源，新 placement 与 suspended-to-restoring 转移共享数据库锁。未知操作保留预约，source teardown 必须确认后才能释放 active capacity，确认 cleanup 后释放 placement capacity。保留所有权需要精确 provider evidence：socket path、缺失 instance 或空列表都不证明 cleanup，也不授权 replacement。 +placement 自动完成：Session 创建提交持久的 pending 工作，共同调度器在 allocation 之前预留兼容节点。[部署契约](../../contracts/agents-api/zh/sandbox-deployment.md#generation-ownership-and-rollout) 定义等待、排序与代次选择。调用方不能选择 node；活动 allocation 即使在 node 离线时也保留原 node。确认 allocation 释放后，符合条件的保留 Session 可以重新预约兼容容量，包括其他 node。Node capacity 计入 pending reservation 和未决资源，新 placement 与 suspended-to-restoring 转移共享数据库锁。未知操作保留预约，source teardown 必须确认后才能释放 active capacity，确认 cleanup 后释放 placement capacity。保留所有权需要精确 provider evidence：socket path、缺失 instance 或空列表都不证明 cleanup，也不授权 replacement。 部署的 CPU、memory、disk 设置、`max_active`、`max_retained` 和 snapshot retention 限制每个 node。确认释放后的冷替换不提供 node-level drain、跨 node 内存恢复、不可达写入方的故障转移、并发写入方、multi-active Core、autoscaling 或 snapshot replication。node 持有 allocation、snapshot、reservation、unknown result 或 cleanup 时拒绝移除 node，离线 ownership 保留。 ### 暂停 {#suspension} -Core 用同一个固定 policy 暂停每个声明 checkpoint 支持的 provider 的空闲工作：工作空闲 5 分钟（300 秒）后暂停，snapshot 保留 24 小时（86400 秒）。部署的 [`suspension`](../../contracts/agents-api/zh/sandbox-deployment.md#safe-response) 报告这两个值。Core 在 initialization 完成、没有 root 或 Subagent Turn 排队、进行中或等待、没有 pending input 或 file operation，且真实 activity 已空闲达到该时间后暂停。Session 无需已运行过 Turn。idle clock 不早于 allocation 进入 running compute phase 的时刻开始，包括 wake 后重新进入该 phase。对于 node allocation，Core 在同一事务中用数据库时钟记录首个 root 或 child terminal transition。candidate filter 和 Session-locked recheck 比较数据库已过时间与 idle duration，初始 snapshot retention deadline 也锚定同一数据库观测，因此 Core 与数据库主机时钟无需一致。原生 completion timestamp 在公开历史中保持不变，但不驱动 idle admission，heartbeat 不重置 activity。确认计划暂停前，daemon 关闭 admission 并排空 native cleanup、output receipt 和 file work。 +Core 用同一个共享 policy 暂停每个声明 checkpoint 支持的 provider 的空闲工作：通常在工作空闲 5 分钟（300 秒）后暂停，snapshot 保留 24 小时（86400 秒）。部署的 [`suspension`](../../contracts/agents-api/zh/sandbox-deployment.md#safe-response) 报告这两个值。Core 在 initialization 完成、没有 root 或 Subagent Turn 排队、进行中或等待、没有 pending input 或 file operation，且真实 activity 已空闲达到该时间后暂停。Session 无需已运行过 Turn。idle clock 不早于 allocation 进入 running compute phase 的时刻开始，包括 wake 后重新进入该 phase。对于 node allocation，Core 在同一事务中用数据库时钟记录首个 root 或 child terminal transition。candidate filter 和 Session-locked recheck 比较数据库已过时间与 idle duration，初始 snapshot retention deadline 也锚定同一数据库观测，因此 Core 与数据库主机时钟无需一致。原生 completion timestamp 在公开历史中保持不变，但不驱动 idle admission，heartbeat 不重置 activity。确认计划暂停前，daemon 关闭 admission 并排空 native cleanup、output receipt 和 file work。 + +容量压力可以让已经空闲的 node allocation 在通常超时之前暂停，但仍需经过 15 秒无活动宽限期。Session 锁内的决策重新检查 pending work、wake request 和 activity，再获取共享 deployment 容量锁并确认兼容的等待需求。它不会中断活动 Turn，也不会在普通 quiesce、capture 和 source-stop 证明完成前释放容量。压力触发的回收采用保守策略：只有进行中回收所属的节点当前有资格服务同一需求时，它才会推迟另一次回收。不可用或不兼容的节点保留其 ownership receipt，但不会阻挡健康节点的容量。如果暂停无法使容量可用，等待请求仍保留原始期限。 Worker lease、Session lock 与 per-node gate 对每个 provider 负责 suspension。新 Turn claim、file-write intent 和 capture admission 在 Session lock 下串行化，共享一个 compute-phase 检查；新 pending work 取消 capture 并唤醒同一 source。正常 preparation 在经过认证的 resume handshake 后等待 compute phase 为 running；pending input 的 promotion 与 lifecycle transition 冲突时保持 pending。compute phase 和 revision-checked receipt 位于 allocation。Core 在 effect 前持久化 quiesce、capture 和 restore intent，仅新 receipt 执行 capture 或 restore，恢复观察精确 attempt，不重试未知 creation、capture 或 restore。已消费 snapshot 不让 running generation 回滚。删除、撤销和 retention expiry 优先于 wake，一直持续到最终数据库 compare-and-swap；未知 cleanup identity 保留，直到确认所属资源不存在。已消费 artifact 和旧 compute 被删除，因此暂停循环不累积可写磁盘链。 @@ -196,6 +198,8 @@ Worker lease、Session lock 与 per-node gate 对每个 provider 负责 suspensi 快照保留期限制计算资源，不限制符合条件的保留 Session 的生命周期。到期后，只有独立文件系统对象已就绪、初始化已完成、Session 与 Environment 均未终结，且此前已认证 Runtime 的 `retained_native_history` 声明已持久化时，Core 才能在不终结 Session 的情况下清理 allocation。能力及原生恢复义务由 [Core–Runtime 协议](runtime-protocol.md)定义。仅保留文件系统并不足够。 +兼容需求无法获得 retained slot 时，Core 可以提前结束空闲且已停止 allocation 的检查点保留期。它在 Session 和 deployment 锁内重新检查原生历史资格、pending work 和容量。等待需求预留该 slot 前，同一 expiry cleanup 必须确认资源已删除。24 小时保留期是上限，不保证容量压力下仍保留内存；文件与已验证的原生历史继续保留。 + Core 在释放 allocation 和 placement 归属前，确认原始创建已结算且所属计算资源与快照均已清理。替代计算资源准入前撤销旧 device 的权限。Create、Kill 或 DeleteSnapshot 的结果尚未确认时，Core 保留归属，不准入替代写入方。可恢复的 Session 与 Environment 保持断连，保留同一文件系统对象和初始化结果。归档、重置和删除优先，唤醒请求不能撤销它们。 新 input 或实时 Environment 文件访问会在兼容 node 上预约新 allocation，并引导新的 Runtime device。准备流程复用同一文件系统和已完成的初始化，不重放初始化。验证替代 Runtime 的声明后，执行从保留历史续接同一原生 Session。原生历史缺失或无效时明确失败；Core 不替换为新的原生 Session，也不重放已完成的 input。读取历史和已发布 Artifact 不预约计算资源。此路径不保留 VM 内存、后台进程或开放连接，也不为未声明能力的 Runtime、Harness 或文件系统组合提供资格。 diff --git a/services/core/IMPLEMENTATION.md b/services/core/IMPLEMENTATION.md index 86892c4c7..f9a5bc27c 100644 --- a/services/core/IMPLEMENTATION.md +++ b/services/core/IMPLEMENTATION.md @@ -35,10 +35,10 @@ Domain owners, each with its PostgreSQL adapter under `internal/persistence/post - `environmenttemplates` (`templatepg`): Environment Templates, their validation and default network, their sealed setup, initial files, Skills and Plugins, and the resolved Template that Session creation composes into its Environment. - `modelconfiguration` (`modelconfigurationpg`): each Harness's deployment default model configuration and its last-use observations. - `skills` (`skillpg`): Skills and their immutable versions: archive checks, the default and latest pointers, version selection and deletion, and each version's sealed archive. Session creation freezes selected versions inside its own transaction: `skillpg.LockSkills` locks the selected Skills in ID order, `skillpg.ReadVersionForFreeze` opens and verifies a version, and `sessions` selects each version with the `skills` rules. -- `deployment` and its subpackage `deployment/placement` (`deploymentpg`, `placementpg`): the sandbox deployment and its nodes: provider configuration and the sealed credential, the specification and retained generations, setup, update and switch, node enrollment, identity and authentication, generation configuration, capacity, presence, status, host history, reset, and the counts of nodes and sandboxes bound to the public address; and hosted runtime allocations: reservation with the dedicated device, compute ownership and settlement, observation diagnostics, activity, compute phases, wake receipts and cleanup, with the reads that schedule, discover and authorize them. It interprets Sandbox Provider declarations through the `providers.Registry` it is given, whose lookups return typed errors. `cmd/server` builds that registry and calls it only to build a direct Provider and to discover a Provider's configuration; it takes each setup's mode and declared operations from the `Setup` that `deployment` returns. `deployment/placement` owns hosted admission and placement: `cmd/server` builds one `placement.Rules` from the registry and the public URL, both fixed while Core runs, and gives it to `deployment.Service` and `sessions.Service`; its pure decisions admit a hosted Session, choose its node, and admit an allocation's reserved node and a restore on it, and its errors keep one status, code and message through `writeSandboxError`. `placementpg` loads the facts those decisions read and applies them on the caller's transaction-bound queries; it has no Store or transaction runner. The only other caller is Session creation: `sessions` decides admission and placement with those rules over the `placementpg` participants inside the creation transaction. Deployment changes and allocation writes run through `deployment.ExecutionOperations` on `deploymentpg.NewExecution(lease, …)`, which the Worker receives as `execution.Owner.Deployment`. Each allocation write locks the owning Session, decides in `deployment`, applies in `deploymentpg` and prunes the Session's change journal in the same leased transaction; cleanup settles the Session through the `sessions` procedures on a `sessionpg.SessionTx` bound to that transaction. The Worker reads the deployment, prepares a selection's setup and records live-compute activity through the pooled `deployment.Service` in `execution.Dispatcher.Deployment`, and lists the Sessions a reset still has to archive, schedules node lifecycles and reads allocations through the pooled `deployment.Reader` in `execution.Dispatcher.DeploymentReader`; a pooled read carries no lease, and the leased write that follows it rechecks the allocation's owner. Node management and reads use the pooled `deploymentpg.Store`, which `cmd/server` also reads the owner epoch from and runs the host-history sampler on. Provider calls run outside transactions, and the final transaction rechecks the expected generation. Session archive crosses the Session and the deployment, so it is a `deployment.ExecutionOperations` operation: `ArchiveSession`, which the administrator's archive route calls through `api.Execution.SessionArchive`, and `ArchiveResetSession`, which a reset calls, lock the Session through `deploymentpg`'s `WithSessionArchive`, check the deployment's generation and the running reset in `deployment`, then expire the Environment and cancel its work through the `sessions` procedures, revoke the allocation's device and request its cleanup, and record the audit entry in one leased transaction. `deployment.ObservationResolver` resolves a Session's Runtime observation target for `runtimeobs` from `sessions.SessionReader` and `deployment.Reader`. +- `deployment` and its subpackage `deployment/placement` (`deploymentpg`, `placementpg`): the sandbox deployment and its nodes: provider configuration and the sealed credential, the specification and retained generations, setup, update and switch, node enrollment, identity and authentication, generation configuration, capacity, presence, status, host history, reset, and the counts of nodes and sandboxes bound to the public address; and hosted runtime allocations: reservation with the dedicated device, compute ownership and settlement, observation diagnostics, activity, compute phases, wake receipts and cleanup, with the reads that schedule, discover and authorize them. It interprets Sandbox Provider declarations through the `providers.Registry` it is given, whose lookups return typed errors. `cmd/server` builds that registry and calls it only to build a direct Provider and to discover a Provider's configuration; it takes each setup's mode and declared operations from the `Setup` that `deployment` returns. `deployment/placement` owns hosted admission and placement: `cmd/server` builds one `placement.Rules` from the registry and the public URL, both fixed while Core runs, and gives it to `deployment.Service` and `sessions.Service`; its pure decisions admit a hosted Session, choose its node, and admit an allocation's reserved node and a restore on it, and its errors keep one status, code and message through `writeSandboxError`. `placementpg` loads the facts those decisions read and applies them on the caller's transaction-bound queries; it has no Store or transaction runner. Session creation uses those rules and the `placementpg` participants only for durable admission inside the creation transaction; it does not reserve compute. The leased execution manager owns the common placement entry point for first compute and replacement compute through `deployment.ExecutionOperations.EnsurePlacement`. Deployment changes and allocation writes run through `deployment.ExecutionOperations` on `deploymentpg.NewExecution(lease, …)`, which the Worker receives as `execution.Owner.Deployment`. Each allocation write locks the owning Session, decides in `deployment`, applies in `deploymentpg` and prunes the Session's change journal in the same leased transaction; cleanup settles the Session through the `sessions` procedures on a `sessionpg.SessionTx` bound to that transaction. The Worker reads the deployment, prepares a selection's setup and records live-compute activity through the pooled `deployment.Service` in `execution.Dispatcher.Deployment`, and lists the Sessions a reset still has to archive, schedules node lifecycles and reads allocations through the pooled `deployment.Reader` in `execution.Dispatcher.DeploymentReader`; a pooled read carries no lease, and the leased write that follows it rechecks the allocation's owner. Node management and reads use the pooled `deploymentpg.Store`, which `cmd/server` also reads the owner epoch from and runs the host-history sampler on. Provider calls run outside transactions, and the final transaction rechecks the expected generation. Session archive crosses the Session and the deployment, so it is a `deployment.ExecutionOperations` operation: `ArchiveSession`, which the administrator's archive route calls through `api.Execution.SessionArchive`, and `ArchiveResetSession`, which a reset calls, lock the Session through `deploymentpg`'s `WithSessionArchive`, check the deployment's generation and the running reset in `deployment`, then expire the Environment and cancel its work through the `sessions` procedures, revoke the allocation's device and request its cleanup, and record the audit entry in one leased transaction. `deployment.ObservationResolver` resolves a Session's Runtime observation target for `runtimeobs` from `sessions.SessionReader` and `deployment.Reader`. - `coremetrics` (`coremetricspg`): the Core metrics PostgreSQL holds: the root Turn queue counts, the root Turn history, read from one read-only snapshot, and the database size. `cmd/server`'s Core metrics source adds them to the process, pool, Worker and daemon registry measurements. - `runtimehistory` (`runtimehistorypg`): Runtime history samples: the periodic export, scoped reads and retention, which also prunes node-host samples. `runtimehistory.Service` scopes a read to the Session's hosted Environment through `sessions.EnvironmentReader`. -- `sessions` (`sessionpg`): Session use cases and reads. The pooled `sessions.Service` runs the use cases on `sessionpg.Store`, which implements `sessions.Storage`, and plain reads use `sessions.Reader`, which `sessionpg.Store` also implements, directly. These cover Sessions (their creation, reads, the change journal and stream snapshot, diagnostics, the frozen execution configuration, measured usage and archive state, metadata updates, deletion and the public write audit), Turns (their reads and the execution work scan), input admission (a public input batch, Environment input reservation and the expiry of one reservation, with the reads of a Turn's admitted inputs, a reservation and the Environment input work scan), the model provider a Session froze, which `sessionpg.Store` opens with the credential key, root Items and Subagents (their reads), Session Artifacts (their reads, deletion and the staging of a Turn's export), Environments (their reads, the initialization list and the frozen setup and initial files, which `sessionpg.Store` opens with the credential key), devices (their reads, which include a Session's execution binding and the execution device list, their creation, revocation, heartbeats and Runtime enrollment, and the archived cancellation receipt the daemon gateway reads), the administrator's views across Projects (a Project's asset counts and Sessions for the summary, and the Sessions whose Runtime the administrator observes), executor credentials (authentication and a Project's credential state through `sessions.ExecutorCredentialReader`, and issuance, rotation and revocation; the Core-key Project operations record their audit entry in the same transaction) and native installation authorization and claims, whose tokens `sessionpg.Store` signs with the credential key. Session creation runs in one pooled `sessions.CreationTx`: `sessions` validates the request, computes its retry identity, with the provider key fingerprinted by `sessionpg.Store` under the credential key, and orders the upsert, hosted admission, the Skill freeze, the sealed model provider, execution configuration, initial files and setup, the Environment, placement and the initial input; `FindSessionCreation` finds an earlier creation by its recorded intent. `cmd/server` builds one `sessionpg.Store` with the credential key and the Service with its `placement.Rules`, and wires the Service and the Store into `execution.Dispatcher.Sessions` and `SessionsReader`, into the api fields, into Runtime enrollment and into the daemon gateway; the Worker creates Sessions through the Service after its execution checks, waking the scheduler only after the commit, and stages Artifacts through it; `cmd/environment-key` builds one without the key, and the Service without placement rules, for its credential commands. Turn transitions, execution completion, which alone publishes or discards a Turn's staged Artifacts, the start of Artifact capture, function calls and their application receipts, the Turn execution journal, Environment initialization, connection observations and their reconciliation, Session device binding, file-write reservation and settlement, and the promotion, failure and bulk expiry of Environment input reservations run through `sessions.ExecutionOperations` on `sessionpg.NewExecution(lease)`, which the Worker receives as `execution.Owner.Sessions`. +- `sessions` (`sessionpg`): Session use cases and reads. The pooled `sessions.Service` runs the use cases on `sessionpg.Store`, which implements `sessions.Storage`, and plain reads use `sessions.Reader`, which `sessionpg.Store` also implements, directly. These cover Sessions (their creation, reads, the change journal and stream snapshot, diagnostics, the frozen execution configuration, measured usage and archive state, metadata updates, deletion and the public write audit), Turns (their reads and the execution work scan), input admission (a public input batch, Environment input reservation and the expiry of one reservation, with the reads of a Turn's admitted inputs, a reservation and the Environment input work scan), the model provider a Session froze, which `sessionpg.Store` opens with the credential key, root Items and Subagents (their reads), Session Artifacts (their reads, deletion and the staging of a Turn's export), Environments (their reads, the initialization list and the frozen setup and initial files, which `sessionpg.Store` opens with the credential key), devices (their reads, which include a Session's execution binding and the execution device list, their creation, revocation, heartbeats and Runtime enrollment, and the archived cancellation receipt the daemon gateway reads), the administrator's views across Projects (a Project's asset counts and Sessions for the summary, and the Sessions whose Runtime the administrator observes), executor credentials (authentication and a Project's credential state through `sessions.ExecutorCredentialReader`, and issuance, rotation and revocation; the Core-key Project operations record their audit entry in the same transaction) and native installation authorization and claims, whose tokens `sessionpg.Store` signs with the credential key. Session creation runs in one pooled `sessions.CreationTx`: `sessions` validates the request, computes its retry identity, with the provider key fingerprinted by `sessionpg.Store` under the credential key, and orders the upsert, hosted admission, the Skill freeze, the sealed model provider, execution configuration, initial files and setup, the Environment and the initial input; `FindSessionCreation` finds an earlier creation by its recorded intent. `cmd/server` builds one `sessionpg.Store` with the credential key and the Service with its `placement.Rules`, and wires the Service and the Store into `execution.Dispatcher.Sessions` and `SessionsReader`, into the api fields, into Runtime enrollment and into the daemon gateway; the Worker creates Sessions through the Service after its execution checks, waking the scheduler only after the commit, and stages Artifacts through it; `cmd/environment-key` builds one without the key, and the Service without placement rules, for its credential commands. Turn transitions, execution completion, which alone publishes or discards a Turn's staged Artifacts, the start of Artifact capture, function calls and their application receipts, the Turn execution journal, Environment initialization, connection observations and their reconciliation, Session device binding, file-write reservation and settlement, and the promotion, failure and bulk expiry of Environment input reservations run through `sessions.ExecutionOperations` on `sessionpg.NewExecution(lease)`, which the Worker receives as `execution.Owner.Sessions`. ## Request handling diff --git a/services/core/cmd/server/main.go b/services/core/cmd/server/main.go index 6c358db24..c9be5a7ff 100644 --- a/services/core/cmd/server/main.go +++ b/services/core/cmd/server/main.go @@ -235,7 +235,7 @@ func run(config processconfig.Config) error { if err != nil { return err } - deploymentExecution, err := deployment.NewExecutionOperations(deploymentService, deploymentpg.NewExecution(lease, credentialKey)) + deploymentExecution, err := deployment.NewExecutionOperations(deploymentService, deploymentpg.NewExecution(lease, credentialKey), dispatcher.Engines) if err != nil { return errors.Join(err, lease.Close(ctx)) } diff --git a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql index 93092ac4c..d73c69e1f 100644 --- a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql +++ b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql @@ -19,6 +19,7 @@ SELECT e.id, s.tenant_id FROM environments e JOIN sessions s ON s.id=e.session_id LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM sqlc.narg(node_id)::uuid + AND (p.environment_id IS NOT NULL OR (SELECT mode = 'direct' FROM runtime_deployment)) AND p.released_at IS NULL AND (SELECT reset_clear IS NULL FROM runtime_deployment) AND e.id > sqlc.arg(after_id)::uuid AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') @@ -38,14 +39,27 @@ LEFT JOIN runtime_placements p ON p.environment_id=e.id LEFT JOIN runtime_deployment_generations g ON g.generation=COALESCE(a.deployment_generation, p.deployment_generation, d.generation) WHERE s.tenant_id=$1 AND e.id=$2; --- name: ListReplacementEnvironments :many -SELECT e.id, s.tenant_id +-- name: ListPlacementDemand :many +WITH demand AS ( +SELECT e.id, s.tenant_id, s.engine, (latest.id IS NOT NULL)::boolean AS retained, + CASE WHEN latest.id IS NULL THEN s.created_at + ELSE LEAST((SELECT min(r.created_at) FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()), CASE WHEN latest.compute_wake_requested THEN latest.compute_activity_at END) END::timestamptz AS demanded_at FROM environments e JOIN sessions s ON s.id = e.session_id -JOIN environment_workspaces w ON w.environment_id = e.id AND w.state = 'ready' -WHERE e.id > $1 AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') - AND e.initialization = 'complete' AND s.configuration->'environment'->>'type' = 'openai_hosted' - AND (SELECT reset_clear IS NULL FROM runtime_deployment) - AND EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state = 'released') +LEFT JOIN LATERAL (SELECT a.id, a.state, a.compute_wake_requested, a.compute_activity_at FROM runtime_allocations a WHERE a.environment_id = e.id ORDER BY a.created_at DESC, a.id DESC LIMIT 1) latest ON true +WHERE s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND (SELECT reset_clear IS NULL AND mode = 'nodes' FROM runtime_deployment) + AND NOT EXISTS (SELECT 1 FROM runtime_placements p WHERE p.environment_id = e.id AND p.released_at IS NULL) AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') - AND EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending') -ORDER BY e.id LIMIT 32; + AND ( + (e.initialization IN ('pending','complete') AND latest.id IS NULL) + OR (e.initialization = 'complete' + AND EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id = e.id AND w.state = 'ready') + AND EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state = 'released') + AND (latest.compute_wake_requested OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()))) + ) +) +SELECT id, tenant_id, engine, retained, demanded_at, COALESCE(sqlc.narg(until_time)::timestamptz, statement_timestamp())::timestamptz AS scan_until FROM demand +WHERE (demanded_at, id) > (sqlc.arg(after_time)::timestamptz, sqlc.arg(after_id)::uuid) +AND demanded_at <= COALESCE(sqlc.narg(until_time)::timestamptz, statement_timestamp()) +ORDER BY demanded_at, id LIMIT 32; diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index 4825047da..5bb294a8d 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -11,11 +11,11 @@ RETURNING *; -- name: TouchRuntimeActivity :exec UPDATE runtime_allocations a -SET compute_activity_at = clock_timestamp(), compute_wake_requested = true +SET compute_activity_at = CASE WHEN a.state = 'released' AND a.compute_wake_requested THEN a.compute_activity_at ELSE clock_timestamp() END, compute_wake_requested = true FROM environments e JOIN sessions s ON s.id = e.session_id WHERE a.environment_id = e.id AND s.tenant_id = sqlc.arg(tenant_id) AND e.id = sqlc.arg(environment_id) AND s.deleted_at IS NULL - AND a.state = 'running' AND a.compute_phase <> 'disabled'; + AND ((a.state = 'running' AND a.compute_phase <> 'disabled') OR a.id = sqlc.narg(retained_id)::uuid); -- name: ClearRuntimeWake :exec UPDATE runtime_allocations diff --git a/services/core/internal/db/queries/runtime_suspension_pressure.sql b/services/core/internal/db/queries/runtime_suspension_pressure.sql new file mode 100644 index 000000000..e46b2fd1f --- /dev/null +++ b/services/core/internal/db/queries/runtime_suspension_pressure.sql @@ -0,0 +1,19 @@ +-- name: ListRuntimeSuspensionInFlightNodes :many +SELECT DISTINCT a.node_id +FROM runtime_allocations a +WHERE a.node_id IS NOT NULL AND a.state <> 'released' + AND (a.compute_phase IN ('quiescing', 'suspending') + OR (a.compute_retained_until <= clock_timestamp() AND a.compute_phase <> 'disabled')); + +-- name: ListWaitingRuntimeRestoreGenerations :many +SELECT DISTINCT a.deployment_generation +FROM runtime_allocations a +JOIN environments e ON e.id = a.environment_id +JOIN sessions s ON s.id = e.session_id +WHERE a.node_id = $1 AND a.state = 'running' AND a.compute_phase = 'suspended' + AND a.compute_retained_until > clock_timestamp() + AND s.deleted_at IS NULL AND e.status NOT IN ('failed', 'expired') + AND (a.compute_wake_requested OR EXISTS ( + SELECT 1 FROM environment_input_reservations r + WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp() + )); diff --git a/services/core/internal/db/queries/turns.sql b/services/core/internal/db/queries/turns.sql index 0caa8416c..4fab9fdd3 100644 --- a/services/core/internal/db/queries/turns.sql +++ b/services/core/internal/db/queries/turns.sql @@ -1,5 +1,5 @@ -- name: LockSession :one -SELECT id, deleted_at FROM sessions WHERE tenant_id = $1 AND id = $2 FOR UPDATE; +SELECT id, deleted_at, engine FROM sessions WHERE tenant_id = $1 AND id = $2 FOR UPDATE; -- name: GetActiveTurn :one SELECT * FROM turns WHERE session_id = $1 AND status IN ('queued', 'in_progress', 'waiting'); diff --git a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go index 7e551e74f..0a1cb8e65 100644 --- a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go @@ -54,34 +54,64 @@ func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRunti return i, err } -const listReplacementEnvironments = `-- name: ListReplacementEnvironments :many -SELECT e.id, s.tenant_id +const listPlacementDemand = `-- name: ListPlacementDemand :many +WITH demand AS ( +SELECT e.id, s.tenant_id, s.engine, (latest.id IS NOT NULL)::boolean AS retained, + CASE WHEN latest.id IS NULL THEN s.created_at + ELSE LEAST((SELECT min(r.created_at) FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()), CASE WHEN latest.compute_wake_requested THEN latest.compute_activity_at END) END::timestamptz AS demanded_at FROM environments e JOIN sessions s ON s.id = e.session_id -JOIN environment_workspaces w ON w.environment_id = e.id AND w.state = 'ready' -WHERE e.id > $1 AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') - AND e.initialization = 'complete' AND s.configuration->'environment'->>'type' = 'openai_hosted' - AND (SELECT reset_clear IS NULL FROM runtime_deployment) - AND EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state = 'released') +LEFT JOIN LATERAL (SELECT a.id, a.state, a.compute_wake_requested, a.compute_activity_at FROM runtime_allocations a WHERE a.environment_id = e.id ORDER BY a.created_at DESC, a.id DESC LIMIT 1) latest ON true +WHERE s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND s.configuration->'environment'->>'type' = 'openai_hosted' + AND (SELECT reset_clear IS NULL AND mode = 'nodes' FROM runtime_deployment) + AND NOT EXISTS (SELECT 1 FROM runtime_placements p WHERE p.environment_id = e.id AND p.released_at IS NULL) AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') - AND EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending') -ORDER BY e.id LIMIT 32 + AND ( + (e.initialization IN ('pending','complete') AND latest.id IS NULL) + OR (e.initialization = 'complete' + AND EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id = e.id AND w.state = 'ready') + AND EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state = 'released') + AND (latest.compute_wake_requested OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()))) + ) +) +SELECT id, tenant_id, engine, retained, demanded_at, COALESCE($1::timestamptz, statement_timestamp())::timestamptz AS scan_until FROM demand +WHERE (demanded_at, id) > ($2::timestamptz, $3::uuid) +AND demanded_at <= COALESCE($1::timestamptz, statement_timestamp()) +ORDER BY demanded_at, id LIMIT 32 ` -type ListReplacementEnvironmentsRow struct { - ID pgtype.UUID `json:"id"` - TenantID pgtype.UUID `json:"tenant_id"` +type ListPlacementDemandParams struct { + UntilTime pgtype.Timestamptz `json:"until_time"` + AfterTime pgtype.Timestamptz `json:"after_time"` + AfterID pgtype.UUID `json:"after_id"` } -func (q *Queries) ListReplacementEnvironments(ctx context.Context, id pgtype.UUID) ([]ListReplacementEnvironmentsRow, error) { - rows, err := q.db.Query(ctx, listReplacementEnvironments, id) +type ListPlacementDemandRow struct { + ID pgtype.UUID `json:"id"` + TenantID pgtype.UUID `json:"tenant_id"` + Engine string `json:"engine"` + Retained bool `json:"retained"` + DemandedAt pgtype.Timestamptz `json:"demanded_at"` + ScanUntil pgtype.Timestamptz `json:"scan_until"` +} + +func (q *Queries) ListPlacementDemand(ctx context.Context, arg ListPlacementDemandParams) ([]ListPlacementDemandRow, error) { + rows, err := q.db.Query(ctx, listPlacementDemand, arg.UntilTime, arg.AfterTime, arg.AfterID) if err != nil { return nil, err } defer rows.Close() - items := []ListReplacementEnvironmentsRow{} + items := []ListPlacementDemandRow{} for rows.Next() { - var i ListReplacementEnvironmentsRow - if err := rows.Scan(&i.ID, &i.TenantID); err != nil { + var i ListPlacementDemandRow + if err := rows.Scan( + &i.ID, + &i.TenantID, + &i.Engine, + &i.Retained, + &i.DemandedAt, + &i.ScanUntil, + ); err != nil { return nil, err } items = append(items, i) @@ -191,6 +221,7 @@ SELECT e.id, s.tenant_id FROM environments e JOIN sessions s ON s.id=e.session_id LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM $1::uuid + AND (p.environment_id IS NOT NULL OR (SELECT mode = 'direct' FROM runtime_deployment)) AND p.released_at IS NULL AND (SELECT reset_clear IS NULL FROM runtime_deployment) AND e.id > $2::uuid AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index fbfa7df77..64fba0e08 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -155,19 +155,20 @@ func (q *Queries) SetRuntimeCompute(ctx context.Context, arg SetRuntimeComputePa const touchRuntimeActivity = `-- name: TouchRuntimeActivity :exec UPDATE runtime_allocations a -SET compute_activity_at = clock_timestamp(), compute_wake_requested = true +SET compute_activity_at = CASE WHEN a.state = 'released' AND a.compute_wake_requested THEN a.compute_activity_at ELSE clock_timestamp() END, compute_wake_requested = true FROM environments e JOIN sessions s ON s.id = e.session_id WHERE a.environment_id = e.id AND s.tenant_id = $1 AND e.id = $2 AND s.deleted_at IS NULL - AND a.state = 'running' AND a.compute_phase <> 'disabled' + AND ((a.state = 'running' AND a.compute_phase <> 'disabled') OR a.id = $3::uuid) ` type TouchRuntimeActivityParams struct { TenantID pgtype.UUID `json:"tenant_id"` EnvironmentID pgtype.UUID `json:"environment_id"` + RetainedID pgtype.UUID `json:"retained_id"` } func (q *Queries) TouchRuntimeActivity(ctx context.Context, arg TouchRuntimeActivityParams) error { - _, err := q.db.Exec(ctx, touchRuntimeActivity, arg.TenantID, arg.EnvironmentID) + _, err := q.db.Exec(ctx, touchRuntimeActivity, arg.TenantID, arg.EnvironmentID, arg.RetainedID) return err } diff --git a/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go new file mode 100644 index 000000000..da19d0c89 --- /dev/null +++ b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go @@ -0,0 +1,74 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: runtime_suspension_pressure.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const listRuntimeSuspensionInFlightNodes = `-- name: ListRuntimeSuspensionInFlightNodes :many +SELECT DISTINCT a.node_id +FROM runtime_allocations a +WHERE a.node_id IS NOT NULL AND a.state <> 'released' + AND (a.compute_phase IN ('quiescing', 'suspending') + OR (a.compute_retained_until <= clock_timestamp() AND a.compute_phase <> 'disabled')) +` + +func (q *Queries) ListRuntimeSuspensionInFlightNodes(ctx context.Context) ([]pgtype.UUID, error) { + rows, err := q.db.Query(ctx, listRuntimeSuspensionInFlightNodes) + if err != nil { + return nil, err + } + defer rows.Close() + items := []pgtype.UUID{} + for rows.Next() { + var node_id pgtype.UUID + if err := rows.Scan(&node_id); err != nil { + return nil, err + } + items = append(items, node_id) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listWaitingRuntimeRestoreGenerations = `-- name: ListWaitingRuntimeRestoreGenerations :many +SELECT DISTINCT a.deployment_generation +FROM runtime_allocations a +JOIN environments e ON e.id = a.environment_id +JOIN sessions s ON s.id = e.session_id +WHERE a.node_id = $1 AND a.state = 'running' AND a.compute_phase = 'suspended' + AND a.compute_retained_until > clock_timestamp() + AND s.deleted_at IS NULL AND e.status NOT IN ('failed', 'expired') + AND (a.compute_wake_requested OR EXISTS ( + SELECT 1 FROM environment_input_reservations r + WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp() + )) +` + +func (q *Queries) ListWaitingRuntimeRestoreGenerations(ctx context.Context, nodeID pgtype.UUID) ([]pgtype.Int8, error) { + rows, err := q.db.Query(ctx, listWaitingRuntimeRestoreGenerations, nodeID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []pgtype.Int8{} + for rows.Next() { + var deployment_generation pgtype.Int8 + if err := rows.Scan(&deployment_generation); err != nil { + return nil, err + } + items = append(items, deployment_generation) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/services/core/internal/db/sqlc/turns.sql.go b/services/core/internal/db/sqlc/turns.sql.go index b3093ccbc..0100959ee 100644 --- a/services/core/internal/db/sqlc/turns.sql.go +++ b/services/core/internal/db/sqlc/turns.sql.go @@ -235,7 +235,7 @@ func (q *Queries) ListTurnInputs(ctx context.Context, arg ListTurnInputsParams) } const lockSession = `-- name: LockSession :one -SELECT id, deleted_at FROM sessions WHERE tenant_id = $1 AND id = $2 FOR UPDATE +SELECT id, deleted_at, engine FROM sessions WHERE tenant_id = $1 AND id = $2 FOR UPDATE ` type LockSessionParams struct { @@ -246,12 +246,13 @@ type LockSessionParams struct { type LockSessionRow struct { ID pgtype.UUID `json:"id"` DeletedAt pgtype.Timestamptz `json:"deleted_at"` + Engine string `json:"engine"` } func (q *Queries) LockSession(ctx context.Context, arg LockSessionParams) (LockSessionRow, error) { row := q.db.QueryRow(ctx, lockSession, arg.TenantID, arg.ID) var i LockSessionRow - err := row.Scan(&i.ID, &i.DeletedAt) + err := row.Scan(&i.ID, &i.DeletedAt, &i.Engine) return i, err } diff --git a/services/core/internal/deployment/allocation.go b/services/core/internal/deployment/allocation.go index d57f33ed5..50909468e 100644 --- a/services/core/internal/deployment/allocation.go +++ b/services/core/internal/deployment/allocation.go @@ -112,6 +112,25 @@ type ObservationSessionPage struct { // an existing one. type UnallocatedEnvironment struct{ ID, TenantID string } +// PlacementDemand is committed work awaiting a node reservation. Demand time +// belongs to Session creation or its oldest pending input, not to a retry. +type PlacementDemand struct { + UnallocatedEnvironment + Engine string + Retained bool + At time.Time +} + +// PlacementDemandCursor continues the bounded, oldest-demand-first scan. +// An empty EnvironmentID ends a page scan; Until may still describe a +// nonempty final page for callers that yield before processing every row. +type PlacementDemandCursor struct { + // Until fixes the database-clock horizon of one sweep, so new arrivals cannot prevent wraparound. + Until time.Time + At time.Time + EnvironmentID string +} + // LifecyclePlacement is what routing a hosted Environment to its node // lifecycle reads: the deployment's provider and mode, the Environment's // allocation and the node placement its Session reserved. diff --git a/services/core/internal/deployment/allocations.go b/services/core/internal/deployment/allocations.go index d6ccdec2d..8a1aa92ae 100644 --- a/services/core/internal/deployment/allocations.go +++ b/services/core/internal/deployment/allocations.go @@ -223,8 +223,8 @@ func (e *ExecutionOperations) cleanup(ctx context.Context, owner Allocation, abs // SetCompute commits a compute phase before its external effects. The owner's // revision and the Session lock fence a stale lifecycle observation. Moving -// running compute to quiescing requires the idle timeout, which the -// transaction checks again against the activity it reads. +// running compute to quiescing requires idle compute and either the idle +// timeout or capacity pressure, rechecked in the transaction. func (e *ExecutionOperations) SetCompute(ctx context.Context, owner Allocation, phase string, state json.RawMessage, retainedUntil *time.Time, idleTimeout time.Duration) (Allocation, error) { if !computeTransition(owner.ComputePhase, phase) || !json.Valid(state) || (owner.ComputePhase == "running" && phase == "quiescing" && idleTimeout <= 0) { return Allocation{}, ErrInvalidInput @@ -248,8 +248,23 @@ func (e *ExecutionOperations) SetCompute(ctx context.Context, owner Allocation, if activity.Busy || activity.WakeRequested { return Allocation{}, ErrAllocationConflict } - if phase == "quiescing" && (!activity.ReadyToSuspend(idleTimeout) || current.ComputeActivityAt.After(owner.ComputeActivityAt)) { - return Allocation{}, ErrAllocationConflict + if phase == "quiescing" { + if current.ComputeActivityAt.After(owner.ComputeActivityAt) { + return Allocation{}, ErrAllocationConflict + } + if !activity.ReadyToSuspend(idleTimeout) { + // A short quiet period also backs off a rejected daemon quiesce. + if !activity.ReadyToSuspend(15*time.Second) || current.NodeID == "" { + return Allocation{}, ErrNotIdle + } + allowed, err := e.canSuspendForDemand(tx, current) + if err != nil { + return Allocation{}, err + } + if !allowed { + return Allocation{}, ErrNotIdle + } + } } } // A node-backed restore reserves capacity on its original node. diff --git a/services/core/internal/deployment/allocations_test.go b/services/core/internal/deployment/allocations_test.go index 352cf2a3d..4de111df1 100644 --- a/services/core/internal/deployment/allocations_test.go +++ b/services/core/internal/deployment/allocations_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "slices" "testing" "time" @@ -91,11 +92,15 @@ func (f *fakeReservationTx) InsertAllocation(allocation NewAllocation) (Allocati } type fakeAllocationTx struct { - t testing.TB - loadAllocation func() (Allocation, error) - loadSessionDevice func() (SessionDevice, bool, error) - settleCreation func(Allocation) (Allocation, error) - release func(Allocation) (Allocation, error) + canRetainEnvironment func(Allocation) (bool, error) + t testing.TB + loadActivity func(Allocation) (Activity, error) + loadSuspensionDemand func(Allocation) (SuspensionDemand, error) + setCompute func(Allocation, ComputeChange) (Allocation, error) + loadAllocation func() (Allocation, error) + loadSessionDevice func() (SessionDevice, bool, error) + settleCreation func(Allocation) (Allocation, error) + release func(Allocation) (Allocation, error) } func (f *fakeAllocationTx) LoadAllocation() (Allocation, error) { @@ -112,9 +117,28 @@ func (f *fakeAllocationTx) LoadSessionDevice() (SessionDevice, bool, error) { return f.loadSessionDevice() } -func (f *fakeAllocationTx) LoadActivity(Allocation) (Activity, error) { - unexpected(f.t, "LoadActivity") - return Activity{}, nil +func (f *fakeAllocationTx) LoadActivity(current Allocation) (Activity, error) { + if f.loadActivity == nil { + unexpected(f.t, "LoadActivity") + } + return f.loadActivity(current) +} + +func (f *fakeAllocationTx) LoadSuspensionDemand(current Allocation) (SuspensionDemand, error) { + if f.loadSuspensionDemand == nil { + unexpected(f.t, "LoadSuspensionDemand") + } + return f.loadSuspensionDemand(current) +} + +func (f *fakeAllocationTx) PlacementDemand(PlacementDemandCursor) ([]PlacementDemand, PlacementDemandCursor, error) { + unexpected(f.t, "PlacementDemand") + return nil, PlacementDemandCursor{}, nil +} + +func (f *fakeAllocationTx) LoadGenerationSpecification(uint64) (GenerationSpecification, error) { + unexpected(f.t, "LoadGenerationSpecification") + return GenerationSpecification{}, nil } func (f *fakeAllocationTx) LoadRestore(Allocation) (placement.Restore, error) { @@ -141,9 +165,11 @@ func (f *fakeAllocationTx) Release(current Allocation) (Allocation, error) { return f.release(current) } -func (f *fakeAllocationTx) SetCompute(Allocation, ComputeChange) (Allocation, error) { - unexpected(f.t, "SetCompute") - return Allocation{}, nil +func (f *fakeAllocationTx) SetCompute(current Allocation, change ComputeChange) (Allocation, error) { + if f.setCompute == nil { + unexpected(f.t, "SetCompute") + } + return f.setCompute(current, change) } func (f *fakeAllocationTx) RecordObservation(Allocation, string) error { @@ -280,7 +306,7 @@ func allocationOperations(t *testing.T, reservation *fakeReservationTx, locked s return apply(allocation) } } - result, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL), storage) + result, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL), storage, engine.Catalog{}) if err != nil { t.Fatal(err) } @@ -515,7 +541,7 @@ func TestCleanupRevokesSettlesTheSessionThenReleases(t *testing.T) { storage := &fakeExecutionStorage{t: t, withAllocationCleanup: func(_ context.Context, _ AllocationKey, apply func(AllocationCleanupTx) error) error { return apply(tx) }} - operations, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL), storage) + operations, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL), storage, engine.Catalog{}) if err != nil { t.Fatal(err) } @@ -553,7 +579,7 @@ func TestExpiredQualifiedFilesystemCleanupPreservesSessionWork(t *testing.T) { storage := &fakeExecutionStorage{t: t, withAllocationCleanup: func(_ context.Context, _ AllocationKey, apply func(AllocationCleanupTx) error) error { return apply(tx) }} - operations, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL), storage) + operations, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL), storage, engine.Catalog{}) if err != nil { t.Fatal(err) } @@ -569,3 +595,69 @@ func (f *fakeReservationTx) LoadGenerationSpecification(uint64) (GenerationSpeci unexpected(f.t, "LoadGenerationSpecification") return GenerationSpecification{}, nil } + +func TestSetComputeRechecksIdlePressureAndActivity(t *testing.T) { + now := time.Now() + owner := Allocation{ID: uuid.NewString(), DeviceID: uuid.NewString(), TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), NodeID: uuid.NewString(), ProviderKey: uuid.NewString(), State: "running", ComputePhase: "running", ComputeActivityAt: now.Add(-time.Minute)} + for _, test := range []struct { + name string + idle time.Duration + busy, wake, changed, pressure bool + want error + checkPressure bool + }{ + {name: "idle timeout", idle: 6 * time.Minute}, + {name: "early pressure", idle: time.Minute, pressure: true, checkPressure: true}, + {name: "no demand", idle: time.Minute, checkPressure: true, want: ErrNotIdle}, + {name: "quiet grace", idle: time.Second, pressure: true, want: ErrNotIdle}, + {name: "turn became busy", idle: time.Minute, busy: true, pressure: true, want: ErrAllocationConflict}, + {name: "wake arrived", idle: time.Minute, wake: true, pressure: true, want: ErrAllocationConflict}, + {name: "activity changed", idle: time.Minute, changed: true, pressure: true, want: ErrAllocationConflict}, + } { + t.Run(test.name, func(t *testing.T) { + current := owner + if test.changed { + current.ComputeActivityAt = now + } + pressureChecks, writes := 0, 0 + tx := &fakeAllocationTx{t: t, + loadAllocation: func() (Allocation, error) { return current, nil }, + loadSessionDevice: func() (SessionDevice, bool, error) { + return SessionDevice{ID: owner.DeviceID, EnvironmentID: owner.EnvironmentID}, true, nil + }, + loadActivity: func(Allocation) (Activity, error) { + return Activity{ObservedAt: now, LastActivity: now.Add(-test.idle), Busy: test.busy, WakeRequested: test.wake}, nil + }, + loadSuspensionDemand: func(Allocation) (SuspensionDemand, error) { + pressureChecks++ + return SuspensionDemand{Deployment: placement.Deployment{InstallationID: owner.ProviderKey, Mode: "nodes"}, Nodes: []placement.Node{{ID: owner.NodeID, Online: true, Active: 1, MaxActive: 1}}, RestoreWaiting: test.pressure}, nil + }, + setCompute: func(a Allocation, c ComputeChange) (Allocation, error) { + writes++ + a.ComputePhase = c.Phase + return a, nil + }, + } + storage := &fakeExecutionStorage{t: t, withAllocation: func(_ context.Context, _ AllocationKey, apply func(AllocationTx) error) error { return apply(tx) }} + operations, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL), storage, engine.Catalog{}) + if err != nil { + t.Fatal(err) + } + until := now.Add(time.Hour) + _, err = operations.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute) + if !errors.Is(err, test.want) { + t.Fatalf("SetCompute: %v, want %v", err, test.want) + } + if (pressureChecks == 1) != test.checkPressure || (writes == 1) != (test.want == nil) { + t.Fatalf("pressure checks=%d writes=%d", pressureChecks, writes) + } + }) + } +} + +func (f *fakeAllocationTx) CanRetainEnvironment(current Allocation) (bool, error) { + if f.canRetainEnvironment == nil { + return false, nil + } + return f.canRetainEnvironment(current) +} diff --git a/services/core/internal/deployment/errors.go b/services/core/internal/deployment/errors.go index e9001c419..d3d30bb50 100644 --- a/services/core/internal/deployment/errors.go +++ b/services/core/internal/deployment/errors.go @@ -27,6 +27,8 @@ var ( // matches the stored allocation, device binding, state or compute revision, // or a replay for another installation. ErrAllocationConflict = errors.New("the sandbox allocation changed") + // ErrNotIdle means neither idle expiry nor current demand requires suspension. + ErrNotIdle = errors.New("sandbox compute is not ready to suspend") // ErrNodeAddressMismatch rejects an enrollment whose Core address is not // the installation public URL. The token stays unconsumed. ErrNodeAddressMismatch = errors.New("sandbox node Core address differs from the public URL") diff --git a/services/core/internal/deployment/execution.go b/services/core/internal/deployment/execution.go index 6af321418..7390deacd 100644 --- a/services/core/internal/deployment/execution.go +++ b/services/core/internal/deployment/execution.go @@ -6,6 +6,7 @@ import ( "errors" "math" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) @@ -16,15 +17,16 @@ import ( type ExecutionOperations struct { service *Service storage ExecutionStorage + engines engine.Catalog } // NewExecutionOperations binds the deployment changes to the lease-bound // storage of one execution owner. -func NewExecutionOperations(service *Service, storage ExecutionStorage) (*ExecutionOperations, error) { +func NewExecutionOperations(service *Service, storage ExecutionStorage, engines engine.Catalog) (*ExecutionOperations, error) { if service == nil || storage == nil { return nil, errors.New("deployment execution operations require the deployment service and execution storage") } - return &ExecutionOperations{service: service, storage: storage}, nil + return &ExecutionOperations{service: service, storage: storage, engines: engines}, nil } // Claim reserves the installation for Web setup, once per execution owner diff --git a/services/core/internal/deployment/fakes_test.go b/services/core/internal/deployment/fakes_test.go index ac99e8612..5b3e69dfa 100644 --- a/services/core/internal/deployment/fakes_test.go +++ b/services/core/internal/deployment/fakes_test.go @@ -638,8 +638,8 @@ func (f *fakeSessionReader) GetManagedSessionArchive(context.Context, string, st return sessions.ManagedArchive{}, nil } -func (f *fakeReader) ReplacementEnvironments(context.Context, string) ([]UnallocatedEnvironment, error) { - panic("unexpected ReplacementEnvironments") +func (f *fakeReader) PlacementDemand(context.Context, PlacementDemandCursor) ([]PlacementDemand, PlacementDemandCursor, error) { + panic("unexpected PlacementDemand") } func (f *fakeReader) RetainedNativeHistory(context.Context, AllocationKey) (bool, error) { panic("unexpected RetainedNativeHistory") diff --git a/services/core/internal/deployment/replacement.go b/services/core/internal/deployment/replacement.go index 57871ac8c..980ba5bd2 100644 --- a/services/core/internal/deployment/replacement.go +++ b/services/core/internal/deployment/replacement.go @@ -9,9 +9,9 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -// EnsurePlacement commits replacement demand before entering a node lifecycle. -// Its reservation survives a failed caller, while the released allocation's -// immutable receipt continues to identify the previous compute owner. +// EnsurePlacement reserves first or retained demand before entering a node lifecycle. +// The immutable engine profile and actual selected generation are validated +// before committing its reservation. func (e *ExecutionOperations) EnsurePlacement(ctx context.Context, key AllocationKey, installation string) (placement.Reserved, error) { key, err := key.parse() if err != nil { @@ -30,14 +30,26 @@ func (e *ExecutionOperations) EnsurePlacement(ctx context.Context, key Allocatio if err != nil { return err } - if !found || existing.State != "released" { - return ErrAllocationConflict - } - allowed, err := tx.CanReplaceAllocation() + environment, err := tx.LoadEnvironment(ctx) if err != nil { return err } - if !allowed { + kind, err := sessions.EnvironmentType(environment.Configuration) + if err != nil || kind != "openai_hosted" || environment.Status == "failed" || environment.Status == "expired" { + return ErrAllocationConflict + } + if found { + if existing.State != "released" { + return ErrAllocationConflict + } + allowed, err := tx.CanReplaceAllocation() + if err != nil { + return err + } + if !allowed { + return ErrAllocationConflict + } + } else if environment.Initialization != "pending" && environment.Initialization != "complete" { return ErrAllocationConflict } d, err := tx.LockDeployment() @@ -65,36 +77,11 @@ func (e *ExecutionOperations) EnsurePlacement(ctx context.Context, key Allocatio if err != nil { return err } - // Retained storage cannot move onto an owned-storage generation. Filter - // before reserving, so an incompatible ready node never traps demand. - compatible := make(map[uint64]bool) - filtered := nodes[:0] - for _, node := range nodes { - if node.ReadyGeneration == nil { - continue - } - generation := *node.ReadyGeneration - accepts, known := compatible[generation] - if !known { - specification := d.Specification - if generation != d.Generation { - stored, err := tx.LoadGenerationSpecification(generation) - if err != nil { - return err - } - specification = stored.Specification - } - var spec sandbox.DeploymentSpec - if err := json.Unmarshal(specification, &spec); err != nil { - return err - } - accepts = spec.Workspace != nil - compatible[generation] = accepts - } - if accepts { - filtered = append(filtered, node) - } + filtered, err := e.eligibleNodes(d, nodes, locked.Engine, found, tx.LoadGenerationSpecification) + if err != nil { + return err } + selected, err := e.service.rules.DecidePlacement(d, filtered) if err != nil { return err @@ -110,3 +97,44 @@ func (e *ExecutionOperations) EnsurePlacement(ctx context.Context, key Allocatio }) return result, err } + +// eligibleNodes applies one generation and Harness compatibility decision for +// allocation admission and capacity-pressure decisions. Retained files require +// another external-storage generation; new Sessions may select either kind. +func (e *ExecutionOperations) eligibleNodes(d placement.Deployment, nodes []placement.Node, engineKind string, retained bool, loadGeneration func(uint64) (GenerationSpecification, error)) ([]placement.Node, error) { + profile, known := e.engines.Lookup(engineKind) + if !known { + return nil, sessions.ErrInvalidInput + } + compatible := make(map[uint64]bool) + filtered := make([]placement.Node, 0, len(nodes)) + for _, node := range nodes { + if node.ReadyGeneration == nil { + // Placement cannot select it, but preserves its preparing diagnosis. + filtered = append(filtered, node) + continue + } + generation := *node.ReadyGeneration + accepts, known := compatible[generation] + if !known { + specification := d.Specification + if generation != d.Generation { + stored, err := loadGeneration(generation) + if err != nil { + return nil, err + } + specification = stored.Specification + } + var spec sandbox.DeploymentSpec + if err := json.Unmarshal(specification, &spec); err != nil { + return nil, err + } + accepts = (!retained || spec.Workspace != nil) && sessions.ValidateRetainedHistory(spec.Workspace != nil, profile.RetainedNativeHistory.IsSupported()) == nil + compatible[generation] = accepts + } + if accepts { + filtered = append(filtered, node) + } + } + return filtered, nil +} diff --git a/services/core/internal/deployment/retention_pressure.go b/services/core/internal/deployment/retention_pressure.go new file mode 100644 index 000000000..aeef016b1 --- /dev/null +++ b/services/core/internal/deployment/retention_pressure.go @@ -0,0 +1,44 @@ +package deployment + +import ( + "context" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// EndRetentionForDemand brings forward the existing retention deadline only +// when a stopped, idle owner can keep its native history and release a needed +// retained slot. The ordinary expiry cleanup still proves every resource gone. +func (e *ExecutionOperations) EndRetentionForDemand(ctx context.Context, owner Allocation) (Allocation, error) { + if owner.NodeID == "" || owner.ComputePhase != "suspended" { + return owner, nil + } + return e.change(ctx, owner, true, func(tx AllocationTx, current Allocation) (Allocation, error) { + if current.State != "running" || current.ComputePhase != "suspended" || current.ComputeRevision != owner.ComputeRevision || current.Expired { + return Allocation{}, ErrAllocationConflict + } + activity, err := tx.LoadActivity(current) + if err != nil || activity.Busy || activity.WakeRequested { + return current, err + } + retained, err := tx.CanRetainEnvironment(current) + if err != nil || !retained { + return current, err + } + pressure, err := tx.LoadSuspensionDemand(current) + if err != nil { + return current, err + } + if pressure.Deployment.Resetting || pressure.Deployment.Mode != string(sandbox.DeploymentNodes) || pressure.Deployment.InstallationID != current.ProviderKey { + return current, nil + } + needed, err := e.placementNeedsCapacity(tx, pressure, current.NodeID, false, true) + if err != nil { + return current, err + } + if needed { + return tx.SetCompute(current, ComputeChange{Phase: "suspended", State: current.ComputeState, RetainedUntil: &activity.ObservedAt}) + } + return current, nil + }) +} diff --git a/services/core/internal/deployment/service_test.go b/services/core/internal/deployment/service_test.go index e3554720f..ae703bb6a 100644 --- a/services/core/internal/deployment/service_test.go +++ b/services/core/internal/deployment/service_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "strings" "testing" "time" @@ -46,7 +47,7 @@ func operations(t *testing.T, publicURL string, tx *fakeDeploymentTx) *Execution if tx != nil { storage.withDeployment = func(_ context.Context, apply func(DeploymentTx) error) error { return apply(tx) } } - result, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, publicURL), storage) + result, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, publicURL), storage, engine.Catalog{}) if err != nil { t.Fatal(err) } @@ -89,10 +90,10 @@ func TestNewServiceAndOperationsRejectNilDependencies(t *testing.T) { } } service := newService(t, storage, reader, testPublicURL) - if result, err := NewExecutionOperations(nil, &fakeExecutionStorage{t: t}); result != nil || err == nil { + if result, err := NewExecutionOperations(nil, &fakeExecutionStorage{t: t}, engine.Catalog{}); result != nil || err == nil { t.Errorf("NewExecutionOperations without the service = %v, %v", result, err) } - if result, err := NewExecutionOperations(service, nil); result != nil || err == nil { + if result, err := NewExecutionOperations(service, nil, engine.Catalog{}); result != nil || err == nil { t.Errorf("NewExecutionOperations without storage = %v, %v", result, err) } } diff --git a/services/core/internal/deployment/session_archive_test.go b/services/core/internal/deployment/session_archive_test.go index 78e853b6b..781905129 100644 --- a/services/core/internal/deployment/session_archive_test.go +++ b/services/core/internal/deployment/session_archive_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "slices" "strings" "testing" @@ -216,7 +217,7 @@ func archiveOperations(t *testing.T, tx *fakeArchiveTx, locked sessions.LockedSe } return apply(ctx, locked, tx) }} - operations, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL), storage) + operations, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL), storage, engine.Catalog{}) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index 51eb8f1f7..d25af71cf 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -102,7 +102,7 @@ type ReservationTx interface { LoadGenerationSpecification(generation uint64) (GenerationSpecification, error) // LoadNodes reads eligible placement facts under the deployment lock. LoadNodes() ([]placement.Node, error) - // ReservePlacement replaces the released reservation without changing historical receipts. + // ReservePlacement inserts a first reservation or replaces a released reservation without changing historical receipts. ReservePlacement(placement.Placement) error } @@ -110,6 +110,10 @@ type ReservationTx interface { // current, the allocation LoadAllocation returned, and returns it changed. A // write whose stored guard no longer holds is ErrAllocationConflict. type AllocationTx interface { + // CanRetainEnvironment checks the ready retained filesystem, completed + // initialization and selected Harness's qualified history under the Session lock. + CanRetainEnvironment(current Allocation) (bool, error) + // LoadAllocation returns the Environment's allocation. LoadAllocation() (Allocation, error) // LoadSessionDevice returns the device the Session is bound to and @@ -117,6 +121,13 @@ type AllocationTx interface { LoadSessionDevice() (SessionDevice, bool, error) // LoadActivity returns the allocation's activity. LoadActivity(current Allocation) (Activity, error) + // LoadSuspensionDemand locks the deployment and reads demand and capacity. + LoadSuspensionDemand(current Allocation) (SuspensionDemand, error) + // PlacementDemand reads one qualified page; next advances past all inspected + // rows, including ineligible receipts. An empty next.EnvironmentID ends the scan. + PlacementDemand(after PlacementDemandCursor) ([]PlacementDemand, PlacementDemandCursor, error) + // LoadGenerationSpecification reads immutable generation requirements. + LoadGenerationSpecification(generation uint64) (GenerationSpecification, error) // LoadRestore locks the deployment and returns what restoring the // allocation's suspended compute on its node reads. LoadRestore(current Allocation) (placement.Restore, error) @@ -136,9 +147,6 @@ type AllocationTx interface { // AllocationCleanupTx is one Session-locked allocation cleanup. type AllocationCleanupTx interface { - // CanRetainEnvironment checks the ready retained filesystem, completed - // initialization and selected Harness's qualified history under the Session lock. - CanRetainEnvironment(current Allocation) (bool, error) AllocationTx sessions.EnvironmentTerminationTx // RevokeDevice revokes the allocation's device. @@ -253,8 +261,8 @@ type Reader interface { UnallocatedEnvironments(ctx context.Context, nodeID, after string) ([]UnallocatedEnvironment, error) // RetainedNativeHistory reports the latest owner's persisted retention eligibility. RetainedNativeHistory(ctx context.Context, key AllocationKey) (bool, error) - // ReplacementEnvironments lists retained Environments with pending input and no compute owner. - ReplacementEnvironments(ctx context.Context, after string) ([]UnallocatedEnvironment, error) + // PlacementDemand lists up to 32 unreserved first or retained requests in demand-time and ID order. + PlacementDemand(ctx context.Context, after PlacementDemandCursor) ([]PlacementDemand, PlacementDemandCursor, error) // LifecyclePlacement returns what routing the Environment to its // lifecycle reads, including for a deleted Session. A missing Environment // is sessions.ErrNotFound. diff --git a/services/core/internal/deployment/suspension_pressure.go b/services/core/internal/deployment/suspension_pressure.go new file mode 100644 index 000000000..8d40e7e1f --- /dev/null +++ b/services/core/internal/deployment/suspension_pressure.go @@ -0,0 +1,124 @@ +package deployment + +import ( + "errors" + "slices" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// SuspensionDemand is a transaction-bound capacity observation. It is not a +// reservation: only a confirmed suspension frees an active slot. +type SuspensionDemand struct { + Deployment placement.Deployment + Nodes []placement.Node + InFlightNodes []string + RestoreWaiting bool +} + +func (e *ExecutionOperations) canSuspendForDemand(tx AllocationTx, owner Allocation) (bool, error) { + pressure, err := tx.LoadSuspensionDemand(owner) + if err != nil { + return false, err + } + if pressure.Deployment.Resetting || pressure.Deployment.Mode != string(sandbox.DeploymentNodes) || pressure.Deployment.InstallationID != owner.ProviderKey { + return false, nil + } + var source *placement.Node + for i := range pressure.Nodes { + if pressure.Nodes[i].ID == owner.NodeID { + source = &pressure.Nodes[i] + break + } + } + if source == nil || !source.Online || source.Active < int64(source.MaxActive) { + return false, nil + } + // A retained restore already owns its retained slot and may use an older + // prepared generation than the node's serving generation. + if pressure.RestoreWaiting { + return !slices.Contains(pressure.InFlightNodes, owner.NodeID), nil + } + reclaimRetained := source.Retained >= int64(source.MaxRetained) + if reclaimRetained { + retained, err := tx.CanRetainEnvironment(owner) + if err != nil || !retained { + return false, err + } + } + return e.placementNeedsCapacity(tx, pressure, owner.NodeID, true, reclaimRetained) +} + +// placementNeedsCapacity scans bounded pages under the caller's deployment +// lock and transaction deadline. Suspension and early retention expiry use the +// same admission decision; an ineligible old receipt never supplies pressure. +func (e *ExecutionOperations) placementNeedsCapacity(tx AllocationTx, pressure SuspensionDemand, nodeID string, freeActive, freeRetained bool) (bool, error) { + var after PlacementDemandCursor + for { + demands, next, err := tx.PlacementDemand(after) + if err != nil { + return false, err + } + for _, demand := range demands { + nodes, err := e.eligibleNodes(pressure.Deployment, pressure.Nodes, demand.Engine, demand.Retained, tx.LoadGenerationSpecification) + if err != nil { + return false, err + } + if _, err := e.service.rules.DecidePlacement(pressure.Deployment, nodes); err == nil { + // Let the demand scan consume real free capacity first. + return false, nil + } else if !errors.Is(err, placement.ErrNodeUnavailable) && !errors.Is(err, placement.ErrNodesPreparing) { + return false, err + } + inFlight, err := e.reclamationInFlight(pressure, nodes) + if err != nil { + return false, err + } + if inFlight { + continue + } + for i := range nodes { + if nodes[i].ID == nodeID { + if freeActive { + nodes[i].Active-- + } + if freeRetained && nodes[i].Retained >= int64(nodes[i].MaxRetained) { + nodes[i].Retained-- + } + } + } + chosen, err := e.service.rules.DecidePlacement(pressure.Deployment, nodes) + if err == nil && chosen != nil && chosen.NodeID == nodeID { + return true, nil + } + if err != nil && !errors.Is(err, placement.ErrNodeUnavailable) && !errors.Is(err, placement.ErrNodesPreparing) { + return false, err + } + } + if next.EnvironmentID == "" { + return false, nil + } + after = next + } +} + +// reclamationInFlight coordinates only resources whose nodes can serve this +// demand. Unavailable or incompatible nodes retain their receipts without +// stopping pressure recovery on a healthy node. +func (e *ExecutionOperations) reclamationInFlight(pressure SuspensionDemand, nodes []placement.Node) (bool, error) { + for _, node := range nodes { + if !slices.Contains(pressure.InFlightNodes, node.ID) { + continue + } + // Capacity is what the in-flight operation may release; all other + // placement requirements must hold now, using the normal admission rule. + node.Active, node.Retained = 0, 0 + if chosen, err := e.service.rules.DecidePlacement(pressure.Deployment, []placement.Node{node}); err == nil && chosen != nil { + return true, nil + } else if err != nil && !errors.Is(err, placement.ErrNodeUnavailable) && !errors.Is(err, placement.ErrNodesPreparing) { + return false, err + } + } + return false, nil +} diff --git a/services/core/internal/execution/deployment_fixture_test.go b/services/core/internal/execution/deployment_fixture_test.go index fc8234e49..9132a9292 100644 --- a/services/core/internal/execution/deployment_fixture_test.go +++ b/services/core/internal/execution/deployment_fixture_test.go @@ -3,6 +3,7 @@ package execution import ( "context" "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "testing" "time" @@ -68,7 +69,7 @@ func deploymentOperations(t *testing.T, storage deployment.Storage, reader deplo if err != nil { t.Fatal(err) } - operations, err := deployment.NewExecutionOperations(service, execution) + operations, err := deployment.NewExecutionOperations(service, execution, engine.Catalog{}) if err != nil { t.Fatal(err) } @@ -140,7 +141,7 @@ func (s *strictExecutionStorage) WithDeployment(ctx context.Context, apply func( // strictDeploymentReader runs each set func; any other call fails the test. type strictDeploymentReader struct { - replacementEnvironments func(context.Context, string) ([]deployment.UnallocatedEnvironment, error) + placementDemand func(context.Context, deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) retainedNativeHistory func(context.Context, deployment.AllocationKey) (bool, error) t *testing.T deployment func(context.Context) (deployment.Record, error) @@ -344,11 +345,11 @@ func (r *strictDeploymentReader) Activity(ctx context.Context, allocationID stri return r.activity(ctx, allocationID) } -func (r *strictDeploymentReader) ReplacementEnvironments(ctx context.Context, after string) ([]deployment.UnallocatedEnvironment, error) { - if r.replacementEnvironments == nil { - return nil, unexpectedDeploymentCall(r.t, "ReplacementEnvironments") +func (r *strictDeploymentReader) PlacementDemand(ctx context.Context, after deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) { + if r.placementDemand == nil { + return nil, deployment.PlacementDemandCursor{}, unexpectedDeploymentCall(r.t, "PlacementDemand") } - return r.replacementEnvironments(ctx, after) + return r.placementDemand(ctx, after) } func (r *strictDeploymentReader) RetainedNativeHistory(ctx context.Context, key deployment.AllocationKey) (bool, error) { if r.retainedNativeHistory == nil { diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 861c183c2..9512d6765 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -129,12 +129,15 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro return r.deployment.ClearWake(ctx, owner, owner.ComputeActivityAt) } policy := r.config.Suspension - if policy == nil || !activity.ReadyToSuspend(policy.IdleTimeout) { + if policy == nil || activity.Busy { return nil } state.SuspendID, state.RestoreID, state.Rollback = uuid.NewString(), "", false until := activity.ObservedAt.Add(policy.Retention) next, err := r.saveCompute(ctx, owner, "quiescing", state, &until) + if errors.Is(err, deployment.ErrNotIdle) { + return nil + } if err != nil { return err } @@ -211,7 +214,8 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San return err } if !activity.Busy && !activity.WakeRequested { - return nil + _, err := r.deployment.EndRetentionForDemand(ctx, owner) + return err } if state.Snapshot == nil || state.Target != nil { return sandbox.ErrOwnership diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index fe013d212..315b08c37 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -146,8 +146,15 @@ func (w *Worker) waitRuntimeAwake(ctx context.Context, environment sessions.Envi if !qualified { return ErrExecutionUnavailable } - // The live file operation is demand. A committed placement, rather than - // another wake setting, lets maintenance recover a detached caller. + // A qualified released allocation retains the existing wake intent. + // The common placement scan orders it with pending Session input. + if touched != owner.ID { + if err := w.dispatcher.Deployment.TouchActivity(ctx, environment.TenantID, environment.ID); err != nil { + return err + } + touched = owner.ID + w.wakeScheduler() + } next, err := w.ProvisionEnvironment(ctx, environment.TenantID, environment.ID, owner.ProviderKey) if err != nil && next.ID == "" && !errors.Is(err, placement.ErrNodeUnavailable) && !errors.Is(err, placement.ErrNodesPreparing) && !errors.Is(err, deployment.ErrAllocationConflict) { return err diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 4c2f7bb47..68e678d6e 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -74,7 +74,7 @@ func newRuntimeManager(owner Owner, deployments *deployment.Service, deploymentR return nil, sandbox.ErrInvalid } ctx, stop := context.WithCancel(context.Background()) - return &runtimeManager{workspaces: owner.Workspaces, workspaceGate: make(chan struct{}, 1), sessions: sessionReader, sessionExecution: owner.Sessions, deployment: owner.Deployment, deploymentService: deployments, deploymentReader: deploymentReader, lease: owner.Lease, registry: registry, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, prepareDeployment: config.prepareDeployment, publishUnconfigured: config.PublishUnconfigured, setupGate: make(chan struct{}, 1), mutationGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)}, nil + return &runtimeManager{workspaces: owner.Workspaces, workspaceGate: make(chan struct{}, 1), sessions: sessionReader, sessionExecution: owner.Sessions, deployment: owner.Deployment, deploymentService: deployments, deploymentReader: deploymentReader, lease: owner.Lease, registry: registry, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, prepareDeployment: config.prepareDeployment, publishUnconfigured: config.PublishUnconfigured, setupGate: make(chan struct{}, 1), mutationGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1), placementWake: make(chan struct{}, 1)}, nil } func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway.Registry) (RuntimeProvider, error) { @@ -156,19 +156,16 @@ func (w *Worker) ProvisionEnvironment(ctx context.Context, tenant, environment, if lookupErr != nil && !errors.Is(lookupErr, deployment.ErrNotFound) { return deployment.Allocation{}, lookupErr } - var nodeID string - if lookupErr == nil && existing.State == "released" { - reserved, reserveErr := w.runtimes.deployment.EnsurePlacement(ctx, key, providerKey) - if reserveErr != nil { - return deployment.Allocation{}, reserveErr - } - nodeID = reserved.NodeID - } else { - nodeID, err = w.runtimes.deploymentService.LifecycleNode(ctx, tenant, environment) - if err != nil { + if errors.Is(lookupErr, deployment.ErrNotFound) || existing.State == "released" { + // The common ordered inventory is the only placement entry point. + if _, err := w.runtimes.syncNodes(ctx); err != nil { return deployment.Allocation{}, err } } + nodeID, err := w.runtimes.deploymentService.LifecycleNode(ctx, tenant, environment) + if err != nil { + return deployment.Allocation{}, err + } node, err := w.runtimes.node(nodeID) if err != nil { return deployment.Allocation{}, err diff --git a/services/core/internal/execution/runtime_manager.go b/services/core/internal/execution/runtime_manager.go index 16d10815d..080a04547 100644 --- a/services/core/internal/execution/runtime_manager.go +++ b/services/core/internal/execution/runtime_manager.go @@ -21,7 +21,8 @@ var errRuntimeTransition = fmt.Errorf("%w: sandbox configuration is changing", E type runtimeManager struct { workspaces *workspaces.ExecutionOperations workspaceCursor string - replacementCursor string + placementCursor deployment.PlacementDemandCursor + placementWake chan struct{} workspaceGate chan struct{} sessions sessions.Reader sessionExecution *sessions.ExecutionOperations @@ -142,7 +143,7 @@ func (m *runtimeManager) syncNodes(ctx context.Context) ([]*runtimeNode, error) return nil, m.ctx.Err() } defer func() { <-m.inventory }() - if err := m.reserveReplacementPlacements(ctx); err != nil { + if err := m.reservePlacements(ctx); err != nil { return nil, err } // A direct caller may add a newly registered node during the query. Only @@ -272,6 +273,10 @@ func (m *runtimeManager) run(ctx context.Context) error { return m.ctx.Err() case err := <-m.failed: return err + case <-m.placementWake: + if _, err := m.syncNodes(ctx); err != nil && !errors.Is(err, errRuntimeTransition) { + return err + } case <-ticker.C: if err := m.deployment.CollectGenerations(ctx); err != nil { return err diff --git a/services/core/internal/execution/runtime_pending.go b/services/core/internal/execution/runtime_pending.go index 9044224d9..3e1db28ba 100644 --- a/services/core/internal/execution/runtime_pending.go +++ b/services/core/internal/execution/runtime_pending.go @@ -2,10 +2,12 @@ package execution import ( "context" + "errors" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) // provisionPending shares the existing lifecycle owner and serial gate. This @@ -35,36 +37,73 @@ func (r *runtimeLifecycle) provisionPending(ctx context.Context) error { return nil } -// reserveReplacementPlacements runs on the manager's existing inventory clock, -// before node scans. The committed input supplies demand; placement is the -// durable retry intent after the first caller or hint is gone. No native -// operation runs here and no retained idle filesystem consumes a reservation. -func (m *runtimeManager) reserveReplacementPlacements(ctx context.Context) error { +// reservePlacements runs under the inventory gate before node scans. Committed +// Sessions and inputs supply demand; a placement survives a disconnected caller. +// No native operation runs here or consumes a reservation for retained idle files. +func (m *runtimeManager) reservePlacements(ctx context.Context) error { m.mu.Lock() configuration := m.config m.mu.Unlock() - if m.workspaces == nil || configuration.Workspace == nil { + if configuration.Mode != string(sandbox.DeploymentNodes) { return nil } operation, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - rows, err := m.deploymentReader.ReplacementEnvironments(operation, m.replacementCursor) + rows, next, err := m.deploymentReader.PlacementDemand(operation, m.placementCursor) if err != nil { + // A bounded inventory read may time out while the execution owner is + // still healthy. Keep its cursor for the next hint or maintenance tick. + if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { + return m.lease.CheckOwnership(ctx) + } return err } if len(rows) == 0 { - m.replacementCursor = "" + m.placementCursor = deployment.PlacementDemandCursor{} return nil } for _, environment := range rows { - m.replacementCursor = environment.ID - _, err = m.deployment.EnsurePlacement(operation, deployment.AllocationKey{TenantID: environment.TenantID, EnvironmentID: environment.ID}, configuration.InstallationID) + if operation.Err() != nil { + if ownership := m.lease.CheckOwnership(ctx); ownership != nil { + return ownership + } + return ctx.Err() + } + reserved, err := m.deployment.EnsurePlacement(operation, deployment.AllocationKey{TenantID: environment.TenantID, EnvironmentID: environment.ID}, configuration.InstallationID) + // A timed-out attempt yields its position, but later rows have not + // been attempted and must receive a fresh budget on the next scan. + m.placementCursor = deployment.PlacementDemandCursor{At: environment.At, EnvironmentID: environment.ID, Until: next.Until} if err != nil { if ownership := m.lease.CheckOwnership(ctx); ownership != nil { return ownership } + if ctx.Err() != nil { + return ctx.Err() + } + if operation.Err() != nil || errors.Is(err, context.DeadlineExceeded) { + return nil + } log.Ctx(ctx).Warn("managed Runtime placement incomplete", "environment_id", environment.ID) + continue } + node, err := m.node(reserved.NodeID) + if err != nil { + return err + } + select { + case node.lifecycle.wakeHints <- struct{}{}: + default: + } + if operation.Err() != nil { + if ownership := m.lease.CheckOwnership(ctx); ownership != nil { + return ownership + } + return ctx.Err() + } + } + m.placementCursor = next + if next.EnvironmentID == "" { + m.placementCursor = deployment.PlacementDemandCursor{} } return nil } diff --git a/services/core/internal/execution/runtime_placement_demand_test.go b/services/core/internal/execution/runtime_placement_demand_test.go new file mode 100644 index 000000000..f4d5faa85 --- /dev/null +++ b/services/core/internal/execution/runtime_placement_demand_test.go @@ -0,0 +1,146 @@ +package execution + +import ( + "context" + "errors" + "fmt" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" + "github.com/google/uuid" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" +) + +func TestPlacementInventoryTimeoutRetainsOwnerAndCursor(t *testing.T) { + for _, tc := range []struct { + name string + failure error + fatal bool + }{ + {"bounded read", context.DeadlineExceeded, false}, + {"database failure", errors.New("database unavailable"), true}, + {"cancellation", context.Canceled, true}, + } { + t.Run(tc.name, func(t *testing.T) { + m := testRuntimeManager(t) + before := deployment.PlacementDemandCursor{EnvironmentID: "cursor"} + m.placementCursor = before + m.deploymentReader = &strictDeploymentReader{t: t, placementDemand: func(context.Context, deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) { + return nil, deployment.PlacementDemandCursor{}, tc.failure + }} + err := m.reservePlacements(t.Context()) + if (err != nil) != tc.fatal { + t.Fatal(err) + } + if m.placementCursor != before { + t.Fatal("failed page advanced cursor", m.placementCursor) + } + }) + } +} + +func TestPlacementInventoryTimeoutNeverMasksParentCancellationOrLostLease(t *testing.T) { + for _, canceled := range []bool{false, true} { + m := testRuntimeManager(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if canceled { + cancel() + } else { + m.lease = lostLease{} + } + m.deploymentReader = &strictDeploymentReader{t: t, placementDemand: func(context.Context, deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) { + return nil, deployment.PlacementDemandCursor{}, context.DeadlineExceeded + }} + if err := m.reservePlacements(ctx); err == nil { + t.Fatal("timeout masked cancellation or lost ownership") + } + } +} + +func TestPlacementReservationTimeoutYieldsOnlyAttemptedDemand(t *testing.T) { + for _, size := range []int{2, 32} { + t.Run(fmt.Sprint(size), func(t *testing.T) { + m := testRuntimeManager(t) + m.config.InstallationID = uuid.NewString() + horizon := time.Now().UTC() + rows := make([]deployment.PlacementDemand, size) + for i := range rows { + rows[i] = deployment.PlacementDemand{UnallocatedEnvironment: deployment.UnallocatedEnvironment{ID: uuid.NewString(), TenantID: uuid.NewString()}, At: horizon.Add(time.Duration(i-size) * time.Second)} + } + reads := 0 + m.deploymentReader = &strictDeploymentReader{t: t, placementDemand: func(ctx context.Context, after deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) { + reads++ + if ctx.Err() != nil { + t.Fatal("page read inherited expired budget", ctx.Err()) + } + if reads == 1 { + next := deployment.PlacementDemandCursor{Until: horizon} + if size == 32 { + next.At = rows[size-1].At + next.EnvironmentID = rows[size-1].ID + } + return rows, next, nil + } + want := deployment.PlacementDemandCursor{At: rows[0].At, EnvironmentID: rows[0].ID, Until: horizon} + if after != want { + t.Fatalf("later unattempted rows were skipped or horizon changed: got %#v, want %#v", after, want) + } + return rows[1:], deployment.PlacementDemandCursor{Until: horizon}, nil + }} + attempts := []string{} + _, m.deployment = deploymentOperations(t, &strictDeploymentStorage{t: t}, &strictDeploymentReader{t: t}, &strictExecutionStorage{t: t, withReservation: func(ctx context.Context, key deployment.AllocationKey, _ func(sessions.LockedSession, deployment.ReservationTx) error) error { + if ctx.Err() != nil { + t.Fatal("reservation attempted with spent context", ctx.Err()) + } + attempts = append(attempts, key.EnvironmentID) + if len(attempts) == 1 { + return context.DeadlineExceeded + } + return placement.ErrNodeUnavailable + }}) + if err := m.reservePlacements(t.Context()); err != nil { + t.Fatal(err) + } + if len(attempts) != 1 { + t.Fatal("spent reservation budget advanced later rows", attempts) + } + if err := m.reservePlacements(t.Context()); err != nil { + t.Fatal(err) + } + if len(attempts) != size { + t.Fatal("later rows did not get a fresh attempt", attempts) + } + for i, id := range attempts { + if id != rows[i].ID { + t.Fatal("attempt order changed", attempts) + } + } + if m.placementCursor != (deployment.PlacementDemandCursor{}) { + t.Fatal("completed short page did not wrap", m.placementCursor) + } + }) + } +} + +func TestPlacementBudgetEndsBeforeAttemptPreservesCursor(t *testing.T) { + m := testRuntimeManager(t) + before := deployment.PlacementDemandCursor{EnvironmentID: uuid.NewString(), At: time.Now().Add(-time.Minute), Until: time.Now()} + m.placementCursor = before + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + m.deploymentReader = &strictDeploymentReader{t: t, placementDemand: func(context.Context, deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) { + cancel() + return []deployment.PlacementDemand{{UnallocatedEnvironment: deployment.UnallocatedEnvironment{ID: uuid.NewString(), TenantID: uuid.NewString()}}}, deployment.PlacementDemandCursor{Until: before.Until}, nil + }} + // There is deliberately no deployment executor: no reservation may start + // after the read exhausts the parent budget. + if err := m.reservePlacements(ctx); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if m.placementCursor != before { + t.Fatal("unattempted demand advanced cursor", m.placementCursor) + } +} diff --git a/services/core/internal/execution/runtime_replacement_test.go b/services/core/internal/execution/runtime_replacement_test.go index 6b3623b46..182ddb17f 100644 --- a/services/core/internal/execution/runtime_replacement_test.go +++ b/services/core/internal/execution/runtime_replacement_test.go @@ -173,9 +173,9 @@ func TestRetainedEnvironmentDemandRecreatesCompute(t *testing.T) { t.Fatal(owner, err) } r.config.Mode = string(sandbox.DeploymentNodes) - manager := &runtimeManager{ctx: t.Context(), config: r.config, setupGate: make(chan struct{}, 1), nodes: make(map[string]*runtimeNode), workspaces: r.workspaces, sessions: r.sessions, sessionExecution: r.sessionExecution, deployment: r.deployment, deploymentService: r.deployments, deploymentReader: r.reader, lease: r.lease, registry: r.registry} + manager := &runtimeManager{ctx: t.Context(), config: r.config, setupGate: make(chan struct{}, 1), inventory: make(chan struct{}, 1), nodes: make(map[string]*runtimeNode), workspaces: r.workspaces, sessions: r.sessions, sessionExecution: r.sessionExecution, deployment: r.deployment, deploymentService: r.deployments, deploymentReader: r.reader, lease: r.lease, registry: r.registry} // Neither retained storage nor an old receipt is demand by itself. - if err = manager.reserveReplacementPlacements(t.Context()); err != nil { + if err = manager.reservePlacements(t.Context()); err != nil { t.Fatal(err) } candidates, err := r.reader.UnallocatedEnvironments(t.Context(), r.nodeID, "") @@ -187,7 +187,7 @@ func TestRetainedEnvironmentDemandRecreatesCompute(t *testing.T) { if err != nil { t.Fatal(err) } - if err = manager.reserveReplacementPlacements(t.Context()); err != nil { + if err = manager.reservePlacements(t.Context()); err != nil { t.Fatal(err) } if err = r.provisionPending(t.Context()); err != nil { diff --git a/services/core/internal/execution/runtime_workspace_settlement_test.go b/services/core/internal/execution/runtime_workspace_settlement_test.go index ce1bba4c9..6207c92df 100644 --- a/services/core/internal/execution/runtime_workspace_settlement_test.go +++ b/services/core/internal/execution/runtime_workspace_settlement_test.go @@ -108,6 +108,11 @@ func workspaceSettlementFixtureWithSetup(t *testing.T, provider sandbox.SandboxP t.Fatal(err) } session := created.Session + if nodeID != "" { + if _, err := operations.EnsurePlacement(t.Context(), deployment.AllocationKey{TenantID: session.TenantID, EnvironmentID: session.Environment.ID}, installation); err != nil { + t.Fatal(err) + } + } configuration := workspacefs.Configuration{ID: uuid.NewString(), Adapter: "fixture", Parameters: []byte(`{}`)} storage := workspacepg.New(pgunit.NewPool(pool)) writer := workspacepg.NewExecution(lease) diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 74d032d5d..bf9c77434 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -139,7 +139,7 @@ func (w *Worker) CreateSession(ctx context.Context, tenant string, input session } input.SupportsRetainedNativeHistory = profile.RetainedNativeHistory.IsSupported() creation, err := w.dispatcher.Sessions.CreateSession(ctx, tenant, input) - if err == nil && len(input.InitialInputs) > 0 { + if err == nil { w.wakeScheduler() } return creation, err diff --git a/services/core/internal/execution/worker_wakeup.go b/services/core/internal/execution/worker_wakeup.go index 153a6b962..c210235d2 100644 --- a/services/core/internal/execution/worker_wakeup.go +++ b/services/core/internal/execution/worker_wakeup.go @@ -9,6 +9,12 @@ import ( // wakeScheduler only hints at committed work. The existing loop retains lease, // capacity and Session ownership; polling recovers absent or coalesced hints. func (w *Worker) wakeScheduler() { + if w.runtimes != nil { + select { + case w.runtimes.placementWake <- struct{}{}: + default: + } + } select { case w.scheduleWake <- struct{}{}: default: diff --git a/services/core/internal/execution/worker_wakeup_test.go b/services/core/internal/execution/worker_wakeup_test.go index 8a0a48f15..cb32b3e46 100644 --- a/services/core/internal/execution/worker_wakeup_test.go +++ b/services/core/internal/execution/worker_wakeup_test.go @@ -6,7 +6,7 @@ import ( ) func TestSchedulerWakeCoalescesConcurrentAdmissionsAndKeepsNextHint(t *testing.T) { - worker := &Worker{scheduleWake: make(chan struct{}, 1)} + worker := &Worker{scheduleWake: make(chan struct{}, 1), runtimes: &runtimeManager{placementWake: make(chan struct{}, 1)}} var callers sync.WaitGroup for range 100 { callers.Go(func() { @@ -19,10 +19,19 @@ func TestSchedulerWakeCoalescesConcurrentAdmissionsAndKeepsNextHint(t *testing.T if got := len(worker.scheduleWake); got != 1 { t.Fatalf("queued wakeups = %d, want one", got) } + if got := len(worker.runtimes.placementWake); got != 1 { + t.Fatalf("placement wakeups = %d, want one", got) + } + <-worker.runtimes.placementWake <-worker.scheduleWake // A commit while a previous scan is running needs a subsequent scan. worker.wakeScheduler() select { + case <-worker.runtimes.placementWake: + default: + t.Fatal("admission during placement scan lost its wakeup") + } + select { case <-worker.scheduleWake: default: t.Fatal("admission during a scan lost its wakeup") diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index fec458e20..8572502c5 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -163,7 +163,37 @@ func (s *Store) WithActivity(ctx context.Context, key deployment.AllocationKey, return err } touch := activityTx(func() error { - return q.TouchRuntimeActivity(ctx, sqlc.TouchRuntimeActivityParams{TenantID: tenant, EnvironmentID: environment}) + retained := pgtype.UUID{} + current, found, err := findAllocation(ctx, q, tenant, environment) + if err != nil { + return err + } + if found && current.State == "released" { + d, err := placementpg.LockDeployment(ctx, q) + if err != nil { + return err + } + if d.Resetting { + return placement.ErrResetAdmission + } + id, err := parseID(current.ID) + if err != nil { + return err + } + replaceable, err := q.CanReplaceRuntimeAllocation(ctx, id) + if err != nil { + return err + } + qualified, err := q.CanRetainRuntimeEnvironment(ctx, id) + if err != nil { + return err + } + if !replaceable || !qualified { + return deployment.ErrAllocationConflict + } + retained = id + } + return q.TouchRuntimeActivity(ctx, sqlc.TouchRuntimeActivityParams{TenantID: tenant, EnvironmentID: environment, RetainedID: retained}) }) if err := apply(locked, touch); err != nil { return err @@ -194,7 +224,11 @@ func (t *reservationTx) LockDeployment() (placement.Deployment, error) { } func (t *reservationTx) LoadReserved() (placement.Reserved, error) { - return placementpg.LoadReserved(t.ctx, t.q, t.environment) + reserved, err := placementpg.LoadReserved(t.ctx, t.q, t.environment) + if errors.Is(err, pgx.ErrNoRows) { + return placement.Reserved{Released: true}, nil + } + return reserved, err } func (t *reservationTx) CanReplaceAllocation() (bool, error) { @@ -218,6 +252,13 @@ func (t *reservationTx) LoadNodes() ([]placement.Node, error) { } func (t *reservationTx) ReservePlacement(p placement.Placement) error { + _, found, err := t.FindAllocation() + if err != nil { + return err + } + if !found { + return placementpg.ReservePlacement(t.ctx, t.q, t.session, p) + } node, err := parseID(p.NodeID) if err != nil { return err @@ -365,7 +406,7 @@ type cleanupTx struct { *sessionpg.SessionTx } -func (t *cleanupTx) CanRetainEnvironment(current deployment.Allocation) (bool, error) { +func (t *allocationTx) CanRetainEnvironment(current deployment.Allocation) (bool, error) { id, err := parseID(current.ID) if err != nil { return false, err @@ -565,22 +606,10 @@ func (s *Store) RetainedNativeHistory(ctx context.Context, key deployment.Alloca return s.pool.Queries().CanRetainRuntimeEnvironment(ctx, id) } -func (s *Store) ReplacementEnvironments(ctx context.Context, after string) ([]deployment.UnallocatedEnvironment, error) { - id, err := cursor(after) - if err != nil { - return nil, err - } +func (s *Store) PlacementDemand(ctx context.Context, after deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) { ctx, cancel := context.WithTimeout(ctx, pgunit.ExecutionTimeout) defer cancel() - rows, err := s.pool.Queries().ListReplacementEnvironments(ctx, id) - if err != nil { - return nil, err - } - result := make([]deployment.UnallocatedEnvironment, 0, len(rows)) - for _, row := range rows { - result = append(result, deployment.UnallocatedEnvironment{ID: uuidString(row.ID), TenantID: uuidString(row.TenantID)}) - } - return result, nil + return loadPlacementDemand(ctx, s.pool.Queries(), after) } func (s *Store) LifecyclePlacement(ctx context.Context, key deployment.AllocationKey) (deployment.LifecyclePlacement, error) { diff --git a/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go b/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go index 45c48d97a..8c49f92f0 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go @@ -3,6 +3,7 @@ package deploymentpg_test import ( "bytes" "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "strings" "testing" @@ -77,7 +78,7 @@ func (f fixture) execution(t *testing.T) (*deployment.ExecutionOperations, *pgun t.Fatal(err) } t.Cleanup(func() { _ = lease.Close(context.Background()) }) - operations, err := deployment.NewExecutionOperations(f.service, deploymentpg.NewExecution(lease, f.cipher)) + operations, err := deployment.NewExecutionOperations(f.service, deploymentpg.NewExecution(lease, f.cipher), engine.Catalog{}) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go b/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go new file mode 100644 index 000000000..1e2f410b6 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go @@ -0,0 +1,215 @@ +package deploymentpg_test + +import ( + "errors" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +func TestPlacementDemandAdmitsOnlyAvailableCompute(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 3, MaxRetained: 10}) + f.connect(t, node.NodeID) + keys := make([]deployment.AllocationKey, 10) + for i := range keys { + keys[i] = pendingHostedEnvironment(t, f) + if i%2 == 0 { + if _, err := f.pool.Exec(t.Context(), `UPDATE environments SET initialization='complete' WHERE id=$1`, keys[i].EnvironmentID); err != nil { + t.Fatal(err) + } + } + } + demand, _, err := f.adapter.PlacementDemand(t.Context(), deployment.PlacementDemandCursor{}) + if err != nil || len(demand) != 10 { + t.Fatal(demand, err) + } + for i, request := range demand { + _, err := changes.EnsurePlacement(t.Context(), deployment.AllocationKey{TenantID: request.TenantID, EnvironmentID: request.ID}, installation) + if i < 3 && err != nil { + t.Fatal(i, err) + } + if i >= 3 && !errors.Is(err, placement.ErrNodeUnavailable) { + t.Fatal(i, err) + } + } + var reserved, devices, allocations int + if err := f.pool.QueryRow(t.Context(), `SELECT (SELECT count(*) FROM runtime_placements WHERE released_at IS NULL),(SELECT count(*) FROM devices),(SELECT count(*) FROM runtime_allocations)`).Scan(&reserved, &devices, &allocations); err != nil { + t.Fatal(err) + } + if reserved != 3 || devices != 0 || allocations != 0 { + t.Fatal(reserved, devices, allocations) + } + // A pending node request must never enter the direct Provider's unallocated lane. + direct, err := f.adapter.UnallocatedEnvironments(t.Context(), "", "") + if err != nil || len(direct) != 0 { + t.Fatal("node demand entered direct lane", direct, err) + } + remaining, _, err := f.adapter.PlacementDemand(t.Context(), deployment.PlacementDemandCursor{}) + if err != nil || len(remaining) != 7 { + t.Fatal(remaining, err) + } + // A retried reservation consumes no additional slot or identity. + first := demand[0] + if _, err := changes.EnsurePlacement(t.Context(), deployment.AllocationKey{TenantID: first.TenantID, EnvironmentID: first.ID}, installation); err != nil { + t.Fatal(err) + } +} + +func TestPlacementDemandPagesByOriginalDemandTime(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + at := time.Now().UTC().Add(-time.Hour).Truncate(time.Microsecond) + for i := 0; i < 40; i++ { + key := pendingHostedEnvironment(t, f) + // UUIDs are random; demand order must follow durable acceptance time. + if _, err := f.pool.Exec(t.Context(), `UPDATE sessions SET created_at=$2 WHERE id=(SELECT session_id FROM environments WHERE id=$1)`, key.EnvironmentID, at.Add(time.Duration(i)*time.Second)); err != nil { + t.Fatal(err) + } + } + first, next, err := f.adapter.PlacementDemand(t.Context(), deployment.PlacementDemandCursor{}) + if err != nil || len(first) != 32 { + t.Fatal(len(first), err) + } + for i, item := range first { + if !item.At.Equal(at.Add(time.Duration(i) * time.Second)) { + t.Fatal(i, item.At) + } + } + second, end, err := f.adapter.PlacementDemand(t.Context(), next) + if err != nil || len(second) != 8 { + t.Fatal(len(second), err) + } + if end.EnvironmentID != "" || !end.Until.Equal(next.Until) { + t.Fatal("short page lost its scan horizon", end, next) + } + if !second[0].At.Equal(at.Add(32 * time.Second)) { + t.Fatal(second[0]) + } +} + +func TestFirstPlacementChecksActualGenerationHistorySupport(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, node.NodeID) + key := pendingHostedEnvironment(t, f) + if _, err := f.pool.Exec(t.Context(), `UPDATE sessions SET engine='mcode' WHERE id=(SELECT session_id FROM environments WHERE id=$1)`, key.EnvironmentID); err != nil { + t.Fatal(err) + } + if _, err := changes.EnsurePlacement(t.Context(), key, installation); !errors.Is(err, placement.ErrNodeUnavailable) { + t.Fatal("unsupported Harness reserved external filesystem", err) + } + pending, _, err := f.adapter.PlacementDemand(t.Context(), deployment.PlacementDemandCursor{}) + if err != nil || len(pending) != 1 { + t.Fatal(pending, err) + } + if _, err := f.pool.Exec(t.Context(), `UPDATE sessions SET engine='codex' WHERE id=(SELECT session_id FROM environments WHERE id=$1)`, key.EnvironmentID); err != nil { + t.Fatal(err) + } + if _, err := changes.EnsurePlacement(t.Context(), key, installation); err != nil { + t.Fatal(err) + } + // Terminal state is rechecked under the Session lock, after the demand read. + another := pendingHostedEnvironment(t, f) + if _, err := f.pool.Exec(t.Context(), `UPDATE environments SET status='expired' WHERE id=$1`, another.EnvironmentID); err != nil { + t.Fatal(err) + } + if _, err := changes.EnsurePlacement(t.Context(), another, installation); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal(err) + } +} + +func pendingHostedEnvironment(t *testing.T, f fixture) deployment.AllocationKey { + t.Helper() + key := hostedEnvironment(t, f.pool) + if _, err := f.pool.Exec(t.Context(), `UPDATE environments SET initialization='pending' WHERE id=$1`, key.EnvironmentID); err != nil { + t.Fatal(err) + } + return key +} + +func TestReleasedFileWakeUsesFirstDemandAndLatestOwner(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + f.connect(t, node.NodeID) + key := hostedEnvironment(t, f.pool) + if _, err := changes.EnsurePlacement(t.Context(), key, installation); err != nil { + t.Fatal(err) + } + owner, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if err != nil { + t.Fatal(err) + } + owner = releaseRetainedFixture(t, f, changes, owner) + if err := f.service.TouchActivity(t.Context(), key.TenantID, key.EnvironmentID); err != nil { + t.Fatal(err) + } + first, _, err := f.adapter.PlacementDemand(t.Context(), deployment.PlacementDemandCursor{}) + if err != nil || len(first) != 1 || !first[0].Retained { + t.Fatal(first, err) + } + if err := f.service.TouchActivity(t.Context(), key.TenantID, key.EnvironmentID); err != nil { + t.Fatal(err) + } + replay, _, err := f.adapter.PlacementDemand(t.Context(), deployment.PlacementDemandCursor{}) + if err != nil || len(replay) != 1 || !replay[0].At.Equal(first[0].At) { + t.Fatal(replay, err) + } + if _, err := changes.EnsurePlacement(t.Context(), key, installation); err != nil { + t.Fatal(err) + } + next, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if err != nil { + t.Fatal(err) + } + if next.ID == owner.ID { + t.Fatal("wake reused released owner") + } + releaseRetainedFixture(t, f, changes, next) + idle, _, err := f.adapter.PlacementDemand(t.Context(), deployment.PlacementDemandCursor{}) + if err != nil || len(idle) != 0 { + t.Fatal("historical wake leaked into later owner", idle, err) + } +} + +func TestPlacementDemandWrapsDespiteContinuousNewArrivals(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + add := func(n int) { + for range n { + pendingHostedEnvironment(t, f) + } + } + add(64) + first, next, err := f.adapter.PlacementDemand(t.Context(), deployment.PlacementDemandCursor{}) + if err != nil || len(first) != 32 || next.Until.IsZero() { + t.Fatal(len(first), next, err) + } + horizon := next.Until + add(64) + second, next, err := f.adapter.PlacementDemand(t.Context(), next) + if err != nil || len(second) != 32 || !next.Until.Equal(horizon) { + t.Fatal(len(second), next, err) + } + add(64) + end, next, err := f.adapter.PlacementDemand(t.Context(), next) + if err != nil || len(end) != 0 || next != (deployment.PlacementDemandCursor{}) { + t.Fatal(len(end), next, err) + } + // Still-unplaced old demand is retried before the subsequent arrivals. + again, _, err := f.adapter.PlacementDemand(t.Context(), next) + if err != nil || len(again) != 32 || again[0].ID != first[0].ID { + t.Fatal(again, err) + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/retention_pressure_test.go b/services/core/internal/persistence/postgres/deploymentpg/retention_pressure_test.go new file mode 100644 index 000000000..bbccedac9 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/retention_pressure_test.go @@ -0,0 +1,143 @@ +package deploymentpg_test + +import ( + "encoding/json" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" +) + +func qualifyPressureRetention(t *testing.T, f fixture, owner deployment.Allocation) deployment.Allocation { + t.Helper() + config := uuid.NewString() + for _, statement := range []struct { + sql string + args []any + }{ + {`INSERT INTO workspace_fs_configurations(id,adapter,parameters) VALUES($1,'fixture','{}')`, []any{config}}, + {`INSERT INTO environment_workspaces(object_id,environment_id,configuration_id,state,attachment) VALUES($1,$2,$3,'ready','{}')`, []any{uuid.NewString(), owner.EnvironmentID, config}}, + {`UPDATE devices SET supported_agent_kinds='[{"kind":"codex","available":true,"capabilities":{"retained_native_history":true}}]' WHERE id=$1`, []any{owner.DeviceID}}, + {`UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{}',compute_retained_until=clock_timestamp()+interval '24 hours' WHERE id=$1`, []any{owner.ID}}, + } { + if _, err := f.pool.Exec(t.Context(), statement.sql, statement.args...); err != nil { + t.Fatal(err) + } + } + current, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + return current +} + +func TestRetentionPressureKeepsHistoryUntilOrdinaryCleanupSettles(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, node.NodeID) + owner := qualifyPressureRetention(t, f, runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)) + waiting := pendingPressureDemand(t, f) + ended, err := changes.EndRetentionForDemand(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + if ended.ComputeRetainedUntil == nil || !ended.ComputeRetainedUntil.Before(*owner.ComputeRetainedUntil) { + t.Fatal("retention was not ended", ended) + } + // The database deadline is only an intent. It is not a cleanup receipt. + nodes, err := f.adapter.Nodes(t.Context()) + if err != nil || nodes[0].Retained != 1 { + t.Fatal("released capacity before cleanup", nodes, err) + } + if _, err = changes.EnsurePlacement(t.Context(), waiting, installation); err == nil { + t.Fatal("reserved before cleanup") + } + current, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil || !current.Expired { + t.Fatal("database expiry", current, err) + } + pending, err := changes.RequestCleanup(t.Context(), current) + if err != nil { + t.Fatal(err) + } + var status, filesystem string + if err = f.pool.QueryRow(t.Context(), `SELECT e.status,w.state FROM environments e JOIN environment_workspaces w ON w.environment_id=e.id WHERE e.id=$1`, owner.EnvironmentID).Scan(&status, &filesystem); err != nil || status == "expired" || status == "failed" || filesystem != "ready" { + t.Fatal("lost retained environment", status, filesystem, err) + } + // This fixture supplies the native cleanup completion that a real lifecycle + // must obtain before invoking ReleaseAllocation. + if _, err = changes.ReleaseAllocation(t.Context(), pending); err != nil { + t.Fatal(err) + } + if _, err = changes.EnsurePlacement(t.Context(), waiting, installation); err != nil { + t.Fatal("waiting demand did not progress", err) + } +} + +func TestRetentionPressureGuards(t *testing.T) { + for _, name := range []string{"no demand", "retained headroom", "wake requested", "missing native history", "free other node", "already ending"} { + t.Run(name, func(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + limit := 1 + if name == "retained headroom" || name == "already ending" { + limit = 2 + } + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: limit}) + f.connect(t, node.NodeID) + owner := qualifyPressureRetention(t, f, runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)) + if name != "no demand" { + pendingPressureDemand(t, f) + } + var err error + switch name { + case "wake requested": + _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_wake_requested=true WHERE id=$1`, owner.ID) + case "missing native history": + _, err = f.pool.Exec(t.Context(), `UPDATE devices SET supported_agent_kinds='[]' WHERE id=$1`, owner.DeviceID) + case "free other node": + other := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + f.connect(t, other.NodeID) + case "already ending": + other := qualifyPressureRetention(t, f, runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)) + _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, other.ID) + } + if err != nil { + t.Fatal(err) + } + next, err := changes.EndRetentionForDemand(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + if next.ComputeRevision != owner.ComputeRevision || next.ComputeRetainedUntil == nil || !next.ComputeRetainedUntil.Equal(*owner.ComputeRetainedUntil) { + t.Fatal("ended retention despite guard", next) + } + }) + } +} + +func TestPressureSuspendsRetainableOwnerWhenRetainedSlotsAreFull(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, node.NodeID) + owner := qualifyPressureRetention(t, f, runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='running',compute_retained_until=NULL WHERE id=$1`, owner.ID); err != nil { + t.Fatal(err) + } + owner, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + pendingPressureDemand(t, f) + until := time.Now().Add(24 * time.Hour) + if _, err = changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute); err != nil { + t.Fatal(err) + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/setup_test.go b/services/core/internal/persistence/postgres/deploymentpg/setup_test.go index 6fee76efc..972395dd1 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/setup_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/setup_test.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "sync" "testing" @@ -56,7 +57,7 @@ func setupClosedExecution(t *testing.T, f fixture) *deployment.ExecutionOperatio t.Fatal(err) } awaitRelease() - operations, err := deployment.NewExecutionOperations(f.service, deploymentpg.NewExecution(lease, f.cipher)) + operations, err := deployment.NewExecutionOperations(f.service, deploymentpg.NewExecution(lease, f.cipher), engine.Catalog{}) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure.go b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure.go new file mode 100644 index 000000000..1f51fb9f8 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure.go @@ -0,0 +1,123 @@ +package deploymentpg + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/placementpg" +) + +func (t *allocationTx) LoadGenerationSpecification(generation uint64) (deployment.GenerationSpecification, error) { + return (unit{ctx: t.ctx, q: t.q}).LoadGenerationSpecification(generation) +} + +func (t *allocationTx) LoadSuspensionDemand(current deployment.Allocation) (deployment.SuspensionDemand, error) { + var result deployment.SuspensionDemand + var err error + result.Deployment, err = placementpg.LockDeployment(t.ctx, t.q) + if err != nil { + return result, err + } + inFlight, err := t.q.ListRuntimeSuspensionInFlightNodes(t.ctx) + if err != nil || result.Deployment.Resetting { + return result, err + } + for _, node := range inFlight { + result.InFlightNodes = append(result.InFlightNodes, uuidString(node)) + } + result.Nodes, err = placementpg.LoadNodes(t.ctx, t.q) + if err != nil { + return result, err + } + node, err := parseID(current.NodeID) + if err != nil { + return result, err + } + generations, err := t.q.ListWaitingRuntimeRestoreGenerations(t.ctx, node) + if err != nil { + return result, err + } + for _, generation := range generations { + ready, err := t.q.NodeGenerationReady(t.ctx, sqlc.NodeGenerationReadyParams{NodeID: node, Generation: generation.Int64}) + if err != nil { + return result, err + } + if ready { + result.RestoreWaiting = true + return result, nil + } + } + return result, nil +} + +func (t *allocationTx) PlacementDemand(after deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) { + rows, next, err := loadPlacementDemand(t.ctx, t.q, after) + if err != nil { + return nil, deployment.PlacementDemandCursor{}, err + } + qualified := rows[:0] + for _, row := range rows { + if row.Retained { + tenant, environment, err := allocationKey(deployment.AllocationKey{TenantID: row.TenantID, EnvironmentID: row.ID}) + if err != nil { + return nil, next, err + } + owner, found, err := findAllocation(t.ctx, t.q, tenant, environment) + if err != nil { + return nil, next, err + } + if !found { + continue + } + id, err := parseID(owner.ID) + if err != nil { + return nil, next, err + } + replaceable, err := t.q.CanReplaceRuntimeAllocation(t.ctx, id) + if err != nil { + return nil, next, err + } + if !replaceable { + continue + } + retained, err := t.q.CanRetainRuntimeEnvironment(t.ctx, id) + if err != nil { + return nil, next, err + } + if !retained { + continue + } + } + qualified = append(qualified, row) + } + return qualified, next, nil +} + +func loadPlacementDemand(ctx context.Context, q *sqlc.Queries, after deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) { + var next deployment.PlacementDemandCursor + id, err := cursor(after.EnvironmentID) + if err != nil { + return nil, next, err + } + rows, err := q.ListPlacementDemand(ctx, sqlc.ListPlacementDemandParams{AfterID: id, AfterTime: pgtype.Timestamptz{Time: after.At, Valid: true}, UntilTime: pgtype.Timestamptz{Time: after.Until, Valid: !after.Until.IsZero()}}) + if err != nil { + return nil, next, err + } + result := make([]deployment.PlacementDemand, 0, len(rows)) + for _, row := range rows { + result = append(result, deployment.PlacementDemand{UnallocatedEnvironment: deployment.UnallocatedEnvironment{ID: uuidString(row.ID), TenantID: uuidString(row.TenantID)}, Engine: row.Engine, Retained: row.Retained, At: row.DemandedAt.Time}) + } + // Even a short final page carries its database horizon, so a caller + // yielding partway through can resume without admitting new arrivals. + if len(rows) > 0 { + next.Until = rows[0].ScanUntil.Time + } + if len(rows) == 32 { + last := rows[len(rows)-1] + next = deployment.PlacementDemandCursor{At: last.DemandedAt.Time, EnvironmentID: uuidString(last.ID), Until: last.ScanUntil.Time} + } + return result, next, nil +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go new file mode 100644 index 000000000..a5e9c6308 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go @@ -0,0 +1,351 @@ +package deploymentpg_test + +import ( + "encoding/json" + "errors" + "github.com/google/uuid" + "sync" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +func runningPressureOwner(t *testing.T, f fixture, changes *deployment.ExecutionOperations, installation, node string, generation uint64) deployment.Allocation { + t.Helper() + key := hostedEnvironment(t, f.pool) + if _, err := f.pool.Exec(t.Context(), `INSERT INTO runtime_placements(environment_id,node_id,deployment_generation) VALUES($1,$2,$3)`, key.EnvironmentID, node, generation); err != nil { + t.Fatal(err) + } + owner, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if err != nil { + t.Fatal(err) + } + if _, err = f.pool.Exec(t.Context(), `UPDATE environments SET initialization='complete' WHERE id=$1`, key.EnvironmentID); err != nil { + t.Fatal(err) + } + if _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET state='running',create_settled=true,compute_phase='running',compute_activity_at=clock_timestamp()-interval '1 minute',compute_phase_changed_at=clock_timestamp()-interval '1 minute' WHERE id=$1`, owner.ID); err != nil { + t.Fatal(err) + } + owner, err = f.adapter.EnvironmentAllocation(t.Context(), key) + if err != nil { + t.Fatal(err) + } + return owner +} + +func TestPressureSuspensionRequiresUsefulCapacity(t *testing.T) { + for _, test := range []struct { + name string + retained int + demand, freeNode bool + want error + }{ + {"no demand", 3, false, false, deployment.ErrNotIdle}, + {"retained full", 1, true, false, deployment.ErrNotIdle}, + {"other node free", 3, true, true, deployment.ErrNotIdle}, + {"waiting first session", 3, true, false, nil}, + } { + t.Run(test.name, func(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox")}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: test.retained}) + f.connect(t, node.NodeID) + owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) + if test.demand { + pendingPressureDemand(t, f) + } + if test.freeNode { + other := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 3}) + f.connect(t, other.NodeID) + } + until := time.Now().Add(time.Hour) + _, err := changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute) + if !errors.Is(err, test.want) { + t.Fatalf("quiesce: %v, want %v", err, test.want) + } + nodes, err := f.adapter.Nodes(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, n := range nodes { + if n.ID == node.NodeID && (n.Active != 1 || n.Retained != 1) { + t.Fatalf("unconfirmed suspension freed capacity: %#v", n) + } + } + }) + } +} + +func TestPressureSuspensionSerializesAcrossNodes(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox")}) + owners := make([]deployment.Allocation, 2) + for i := range owners { + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 3}) + f.connect(t, node.NodeID) + owners[i] = runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) + } + pendingPressureDemand(t, f) + var wg sync.WaitGroup + errs := make([]error, len(owners)) + for i, owner := range owners { + wg.Go(func() { + until := time.Now().Add(time.Hour) + _, errs[i] = changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute) + }) + } + wg.Wait() + success := 0 + for _, err := range errs { + if err == nil { + success++ + } else if !errors.Is(err, deployment.ErrNotIdle) { + t.Fatal(err) + } + } + if success != 1 { + t.Fatalf("one waiting Session started %d suspensions: %v", success, errs) + } + var phases int + if err := f.pool.QueryRow(t.Context(), `SELECT count(*) FROM runtime_allocations WHERE compute_phase='quiescing'`).Scan(&phases); err != nil || phases != 1 { + t.Fatal(phases, err) + } +} + +func pendingPressureDemand(t *testing.T, f fixture) deployment.AllocationKey { + t.Helper() + key := hostedEnvironment(t, f.pool) + if _, err := f.pool.Exec(t.Context(), `UPDATE environments SET initialization='pending' WHERE id=$1`, key.EnvironmentID); err != nil { + t.Fatal(err) + } + return key +} + +func TestPressureSuspensionServesRestoreOnItsOriginalNode(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox")}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + f.connect(t, node.NodeID) + waiting := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_retained_until=clock_timestamp()+interval '1 hour',compute_wake_requested=true WHERE id=$1`, waiting.ID); err != nil { + t.Fatal(err) + } + owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) + // Free capacity on another node cannot restore this retained compute. + other := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + f.connect(t, other.NodeID) + until := time.Now().Add(time.Hour) + if _, err := changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute); err != nil { + t.Fatal(err) + } + var wake bool + if err := f.pool.QueryRow(t.Context(), `SELECT compute_wake_requested FROM runtime_allocations WHERE id=$1`, waiting.ID).Scan(&wake); err != nil || !wake { + t.Fatal("pressure consumed the waiting wake", wake, err) + } +} + +func TestPressureDemandScansPastIncompatiblePage(t *testing.T) { + for _, retained := range []bool{false, true} { + t.Run(map[bool]string{false: "active", true: "retained"}[retained], func(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + capacity := deployment.Capacity{MaxActive: 1, MaxRetained: 2} + if retained { + capacity.MaxRetained = 1 + } + node := f.enroll(t, view, capacity) + f.connect(t, node.NodeID) + owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) + if retained { + owner = qualifyPressureRetention(t, f, owner) + } + for range 32 { + key := pendingPressureDemand(t, f) + if _, err := f.pool.Exec(t.Context(), `UPDATE sessions SET engine='mcode' WHERE id=(SELECT session_id FROM environments WHERE id=$1)`, key.EnvironmentID); err != nil { + t.Fatal(err) + } + } + pendingPressureDemand(t, f) + until := time.Now().Add(time.Hour) + var result deployment.Allocation + var err error + if retained { + result, err = changes.EndRetentionForDemand(t.Context(), owner) + } else { + result, err = changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute) + } + if err != nil { + t.Fatal(err) + } + if retained && (result.ComputeRetainedUntil == nil || !result.ComputeRetainedUntil.Before(*owner.ComputeRetainedUntil)) { + t.Fatal("later compatible demand did not end retention") + } + if !retained && result.ComputePhase != "quiescing" { + t.Fatal("later compatible demand did not suspend compute") + } + }) + } +} + +func TestPressureDemandRejectsUnqualifiedRetainedReceipt(t *testing.T) { + for _, badReceipt := range []string{"history", "credential"} { + t.Run(badReceipt, func(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + f.connect(t, node.NodeID) + count := 1 + if badReceipt == "history" { + count = 32 + } + for range count { + old := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) + old = releaseRetainedFixture(t, f, changes, old) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_wake_requested=true WHERE id=$1`, old.ID); err != nil { + t.Fatal(err) + } + statement := `UPDATE devices SET supported_agent_kinds='[]' WHERE id=$1` + if badReceipt == "credential" { + statement = `UPDATE devices SET revoked_at=NULL WHERE id=$1` + } + if _, err := f.pool.Exec(t.Context(), statement, old.DeviceID); err != nil { + t.Fatal(err) + } + } + owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) + until := time.Now().Add(time.Hour) + if _, err := changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute); !errors.Is(err, deployment.ErrNotIdle) { + t.Fatal("unqualified receipt suspended healthy owner", err) + } + // An empty qualified page must not stop the cursor before a later + // valid request, even though the raw first page was nonempty. + pendingPressureDemand(t, f) + if _, err := changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute); err != nil { + t.Fatal("later valid demand was hidden", err) + } + }) + } +} + +func TestPressureReclamationIgnoresUnavailableNodeReceipts(t *testing.T) { + for _, obstruction := range []string{"offline expired", "offline quiescing", "offline suspending", "different address", "incompatible generation"} { + for _, retention := range []bool{false, true} { + t.Run(obstruction+map[bool]string{false: "/suspend", true: "/retention"}[retention], func(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + spec := retainedSpecification() + if obstruction == "incompatible generation" { + spec = testSpecification("microsandbox") + } + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: spec}) + badNode := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + f.connect(t, badNode.NodeID) + stuck := runningPressureOwner(t, f, changes, installation, badNode.NodeID, view.Generation) + phase := "quiescing" + if obstruction == "offline suspending" { + phase = "suspending" + } + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase=$2,compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, stuck.ID, phase); err != nil { + t.Fatal(err) + } + switch obstruction { + case "offline expired", "offline quiescing", "offline suspending": + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, badNode.NodeID); err != nil { + t.Fatal(err) + } + if obstruction == "offline expired" { + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 minute' WHERE id=$1`, stuck.ID); err != nil { + t.Fatal(err) + } + } + case "different address": + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET core_url='https://other.example' WHERE id=$1`, badNode.NodeID); err != nil { + t.Fatal(err) + } + case "incompatible generation": + var err error + view, err = changes.Update(admin(t), installation, sandbox.Selection{Provider: "microsandbox", ExpectedGeneration: view.Generation, DeploymentSpec: retainedSpecification()}) + if err != nil { + t.Fatal(err) + } + } + limit := 2 + if retention { + limit = 1 + } + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: limit}) + f.connect(t, node.NodeID) + var waiting deployment.AllocationKey + if obstruction == "incompatible generation" { + old := releaseRetainedFixture(t, f, changes, runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)) + waiting = old.Key() + } else { + waiting = pendingPressureDemand(t, f) + } + // A real pending reservation supplies demand and retains its original deadline. + if _, err := f.pool.Exec(t.Context(), `INSERT INTO environment_input_reservations(id,session_id,idempotency_key,batch,created_at,deadline) SELECT $1,session_id,'pressure','[{}]',clock_timestamp(),clock_timestamp()+interval '5 minutes' FROM environments WHERE id=$2`, uuid.NewString(), waiting.EnvironmentID); err != nil { + t.Fatal(err) + } + owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) + if retention { + owner = qualifyPressureRetention(t, f, owner) + } + until := time.Now().Add(time.Hour) + if retention { + if _, err := changes.EndRetentionForDemand(t.Context(), owner); err != nil { + t.Fatal(err) + } + current, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil || !current.Expired { + t.Fatal("healthy retention did not advance", current, err) + } + pending, err := changes.RequestCleanup(t.Context(), current) + if err != nil { + t.Fatal(err) + } + // Supply the native deletion receipt for the healthy fixture only. + if _, err = changes.ReleaseAllocation(t.Context(), pending); err != nil { + t.Fatal(err) + } + } else { + current, err := changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute) + if err != nil { + t.Fatal("healthy suspension blocked", err) + } + // Supply healthy native quiesce/capture/stop receipts through the + // existing fenced phase transitions; never settle the stuck owner. + for _, next := range []string{"suspending", "suspended"} { + current, err = changes.SetCompute(t.Context(), current, next, json.RawMessage(`{}`), &until, 5*time.Minute) + if err != nil { + t.Fatal(err) + } + } + } + reserved, err := changes.EnsurePlacement(t.Context(), waiting, installation) + if err != nil || reserved.NodeID != node.NodeID { + t.Fatal("waiting input did not advance on healthy node", reserved, err) + } + observed, err := f.adapter.EnvironmentAllocation(t.Context(), stuck.Key()) + if err != nil || observed.ID != stuck.ID || observed.DeviceID != stuck.DeviceID || observed.State != "running" || observed.ComputePhase != phase { + t.Fatal("changed unknown receipt", observed, err) + } + nodes, err := f.adapter.Nodes(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, n := range nodes { + if n.ID == badNode.NodeID && (n.Active != 1 || n.Retained != 1) { + t.Fatal("released unknown capacity", n) + } + } + }) + } + } +} diff --git a/services/core/internal/persistence/postgres/pgtest/lease.go b/services/core/internal/persistence/postgres/pgtest/lease.go index 551c16f22..8119a50c4 100644 --- a/services/core/internal/persistence/postgres/pgtest/lease.go +++ b/services/core/internal/persistence/postgres/pgtest/lease.go @@ -14,7 +14,8 @@ func ObserveExecutionLeaseRelease(t *testing.T, pool *pgxpool.Pool) func() { t.Helper() // Match the single-bigint key in queries/scheduling.sql, scoped to this DB. const lock = `locktype='advisory' AND granted AND objsubid=1 - AND classid::bigint * 4294967296 + objid::bigint = 706172736172 + AND classid::bigint = (706172736172::bigint >> 32) + AND objid::bigint = (706172736172::bigint & 4294967295) AND database=(SELECT oid FROM pg_database WHERE datname=current_database())` ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) defer cancel() diff --git a/services/core/internal/persistence/postgres/pgtest/lease_test.go b/services/core/internal/persistence/postgres/pgtest/lease_test.go new file mode 100644 index 000000000..443fb6210 --- /dev/null +++ b/services/core/internal/persistence/postgres/pgtest/lease_test.go @@ -0,0 +1,26 @@ +package pgtest + +import "testing" + +func TestObserveExecutionLeaseReleaseWithNegativeAdvisoryLock(t *testing.T) { + pool := OpenIsolated(t, nil) + conn, err := pool.Acquire(t.Context()) + if err != nil { + t.Fatal(err) + } + defer conn.Release() + // Negative single-bigint keys occupy high unsigned pg_locks.classid bits. + // Other tests may hold one on the same PostgreSQL server while we observe + // this database's positive execution key. + if _, err := conn.Exec(t.Context(), `SELECT pg_advisory_lock(-1::bigint),pg_advisory_lock(706172736172::bigint)`); err != nil { + t.Fatal(err) + } + released := ObserveExecutionLeaseRelease(t, pool) + if _, err := conn.Exec(t.Context(), `SELECT pg_advisory_unlock(706172736172::bigint)`); err != nil { + t.Fatal(err) + } + released() + if _, err := conn.Exec(t.Context(), `SELECT pg_advisory_unlock(-1::bigint)`); err != nil { + t.Fatal(err) + } +} diff --git a/services/core/internal/persistence/postgres/pgunit/lease_test.go b/services/core/internal/persistence/postgres/pgunit/lease_test.go index c5377bc66..d1df04098 100644 --- a/services/core/internal/persistence/postgres/pgunit/lease_test.go +++ b/services/core/internal/persistence/postgres/pgunit/lease_test.go @@ -33,7 +33,8 @@ func awaitLeaseRelease(t *testing.T, pool *pgxpool.Pool) { var held bool // Match the single-bigint key in queries/scheduling.sql, scoped to this database. err := pool.QueryRow(t.Context(), `SELECT EXISTS (SELECT 1 FROM pg_locks WHERE locktype='advisory' AND granted AND objsubid=1 - AND classid::bigint * 4294967296 + objid::bigint = 706172736172 + AND classid::bigint = (706172736172::bigint >> 32) + AND objid::bigint = (706172736172::bigint & 4294967295) AND database=(SELECT oid FROM pg_database WHERE datname=current_database()))`).Scan(&held) if err != nil { t.Fatal("observe execution lease release", err) diff --git a/services/core/internal/persistence/postgres/sessionpg/creation.go b/services/core/internal/persistence/postgres/sessionpg/creation.go index 96a44b3f1..127fe6752 100644 --- a/services/core/internal/persistence/postgres/sessionpg/creation.go +++ b/services/core/internal/persistence/postgres/sessionpg/creation.go @@ -102,19 +102,6 @@ func (t *creationTx) LockDeployment(ctx context.Context) (placement.Deployment, return placementpg.LockDeployment(ctx, t.q) } -func (t *creationTx) LoadGenerationSpecification(ctx context.Context, generation uint64) (json.RawMessage, error) { - row, err := t.q.GetSandboxGeneration(ctx, int64(generation)) - return row.Specification, err -} - -func (t *creationTx) LoadNodes(ctx context.Context) ([]placement.Node, error) { - return placementpg.LoadNodes(ctx, t.q) -} - -func (t *creationTx) ReservePlacement(ctx context.Context, chosen placement.Placement) error { - return placementpg.ReservePlacement(ctx, t.q, t.session, chosen) -} - func (t *creationTx) LockSkills(ctx context.Context, ids []string) (map[string]skills.Skill, error) { locked, err := skillpg.LockSkills(ctx, t.q, t.tenant, ids) return locked, skillError(err) diff --git a/services/core/internal/persistence/postgres/sessionpg/creation_test.go b/services/core/internal/persistence/postgres/sessionpg/creation_test.go index 79f125500..bc2b5326b 100644 --- a/services/core/internal/persistence/postgres/sessionpg/creation_test.go +++ b/services/core/internal/persistence/postgres/sessionpg/creation_test.go @@ -464,10 +464,9 @@ func TestEnvironmentCreationReservesItsInitialInput(t *testing.T) { } // Hosted creation checks admission on the locked deployment, so it sees a -// reset committed while it waited, and places after creating the -// Environment, rolling both back when no node is available. A retry admits -// nothing and still returns its Session. -func TestHostedCreationAdmitsAndPlacesUnderTheDeploymentLock(t *testing.T) { +// reset committed while it waited. Accepted Environments wait for compute +// without a placement; a retry admits nothing and returns its Session. +func TestHostedCreationAdmitsUnderDeploymentLockAndLeavesComputeUnreserved(t *testing.T) { pool := pgtest.OpenIsolated(t, nil) _, service := creationService(t, pool) ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) @@ -518,11 +517,18 @@ func TestHostedCreationAdmitsAndPlacesUnderTheDeploymentLock(t *testing.T) { t.Fatal("retry ran admission", retry, err) } exec(t, pool, "UPDATE runtime_deployment SET reset_clear=NULL, reset_requested_at=NULL, reset_forced_at=NULL, reset_audit=NULL") - if _, err := service.CreateSession(ctx, tenant, hosted("unplaced")); !errors.Is(err, placement.ErrNodeUnavailable) { - t.Fatal("placement without a node", err) + queued, err := service.CreateSession(ctx, tenant, hosted("unplaced")) + if err != nil || !queued.Created { + t.Fatal("creation without available compute was not accepted", queued, err) } - if count := countRows(t, pool, "SELECT count(*) FROM sessions s JOIN environments e ON e.session_id=s.id WHERE s.tenant_id=$1", tenant); count != 1 { - t.Fatal("failed hosted creation left work", count) + if count := countRows(t, pool, "SELECT count(*) FROM sessions s JOIN environments e ON e.session_id=s.id WHERE s.tenant_id=$1", tenant); count != 2 { + t.Fatal("accepted hosted creation did not preserve both Sessions", count) + } + if count := countRows(t, pool, "SELECT count(*) FROM runtime_placements WHERE environment_id=$1", queued.Session.Environment.ID); count != 0 { + t.Fatal("creation reserved a node", count) + } + if count := countRows(t, pool, "SELECT count(*) FROM runtime_allocations WHERE environment_id=$1", queued.Session.Environment.ID); count != 0 { + t.Fatal("creation allocated compute", count) } } diff --git a/services/core/internal/persistence/postgres/sessionpg/execution_inputs_test.go b/services/core/internal/persistence/postgres/sessionpg/execution_inputs_test.go index 80af513a5..0cedc377b 100644 --- a/services/core/internal/persistence/postgres/sessionpg/execution_inputs_test.go +++ b/services/core/internal/persistence/postgres/sessionpg/execution_inputs_test.go @@ -34,7 +34,8 @@ func terminateLeaseOwner(t *testing.T, pool *pgxpool.Pool) { t.Helper() var killed bool err := pool.QueryRow(t.Context(), `SELECT pg_terminate_backend(pid, 1000) FROM pg_locks WHERE locktype='advisory' AND granted AND objsubid=1 - AND classid::bigint * 4294967296 + objid::bigint = 706172736172 + AND classid::bigint = (706172736172::bigint >> 32) + AND objid::bigint = (706172736172::bigint & 4294967295) AND database=(SELECT oid FROM pg_database WHERE datname=current_database())`).Scan(&killed) if err != nil || !killed { t.Fatal("terminate execution lease owner", killed, err) diff --git a/services/core/internal/persistence/postgres/sessionpg/session.go b/services/core/internal/persistence/postgres/sessionpg/session.go index 2daebf41a..222db15d4 100644 --- a/services/core/internal/persistence/postgres/sessionpg/session.go +++ b/services/core/internal/persistence/postgres/sessionpg/session.go @@ -24,7 +24,7 @@ func LockSession(ctx context.Context, q *sqlc.Queries, tenant, session pgtype.UU if err != nil { return sessions.LockedSession{}, err } - return sessions.LockedSession{Deleted: row.DeletedAt.Valid}, nil + return sessions.LockedSession{Deleted: row.DeletedAt.Valid, Engine: row.Engine}, nil } // WithSession runs one Session transaction on runner: it locks the tenant's diff --git a/services/core/internal/sandbox/providers/configuration_flow_test.go b/services/core/internal/sandbox/providers/configuration_flow_test.go index abc454120..3d87a8926 100644 --- a/services/core/internal/sandbox/providers/configuration_flow_test.go +++ b/services/core/internal/sandbox/providers/configuration_flow_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "net/http/httptest" "strings" "testing" @@ -115,7 +116,7 @@ func TestAdditionalConfigurationProviderUsesCommonAPIAndStore(t *testing.T) { t.Fatal(err) } defer lease.Close(context.Background()) - changes, err := deployment.NewExecutionOperations(service, deploymentpg.NewExecution(lease, pgtest.CredentialKey(t))) + changes, err := deployment.NewExecutionOperations(service, deploymentpg.NewExecution(lease, pgtest.CredentialKey(t)), engine.Catalog{}) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/sessions/creation.go b/services/core/internal/sessions/creation.go index bafb5aefb..56957dad3 100644 --- a/services/core/internal/sessions/creation.go +++ b/services/core/internal/sessions/creation.go @@ -74,10 +74,6 @@ type CreationTx interface { // LockDeployment locks and reads the sandbox deployment, serializing // hosted creation with deployment changes, restore and node removal. LockDeployment(ctx context.Context) (placement.Deployment, error) - // LoadGenerationSpecification reads the retained generation selected under the deployment lock. - LoadGenerationSpecification(ctx context.Context, generation uint64) (json.RawMessage, error) - LoadNodes(ctx context.Context) ([]placement.Node, error) - ReservePlacement(ctx context.Context, chosen placement.Placement) error // LockSkills locks the tenant's Skills in ID order, whatever the order of // ids, and returns them by ID. A missing one is ErrNotFound. LockSkills(ctx context.Context, ids []string) (map[string]skills.Skill, error) @@ -117,7 +113,7 @@ type CreationTx interface { // CreateSession creates a Session under a tenant-scoped key, so retries, // including concurrent ones, return the stored Session. The same key with // different input or another creator is ErrIdempotencyConflict. Only the new -// Session freezes resources, takes a placement and admits its initial inputs, +// Session freezes resources and admits its initial inputs, // so a retry after completion or later Turns never submits them again. func (s *Service) CreateSession(ctx context.Context, tenant string, input CreateSession) (Creation, error) { session, batch, encoded, err := prepareCreation(input, s.storage.FingerprintProviderKey) @@ -219,33 +215,18 @@ func (s *Service) createResources(ctx context.Context, tx CreationTx, session Se created = append(created, writeaudit.Resource{Type: writeaudit.ResourceEnvironment, ID: environment, ParentID: session.ID}) } if hosted { - nodes, err := tx.LoadNodes(ctx) - if err != nil { + if err := s.rules.CheckPublicOrigin(deployment.Provider); err != nil { return nil, err } - chosen, err := s.rules.DecidePlacement(deployment, nodes) - if err != nil { + var target sandbox.DeploymentSpec + if err := json.Unmarshal(deployment.Specification, &target); err != nil { return nil, err } - specification := deployment.Specification - if chosen != nil && chosen.Generation != deployment.Generation { - specification, err = tx.LoadGenerationSpecification(ctx, chosen.Generation) - if err != nil { - return nil, err - } - } - var selected sandbox.DeploymentSpec - if err := json.Unmarshal(specification, &selected); err != nil { + if err := ValidateRetainedHistory(target.Workspace != nil, input.SupportsRetainedNativeHistory); err != nil { return nil, err } - if err := ValidateRetainedHistory(selected.Workspace != nil, input.SupportsRetainedNativeHistory); err != nil { - return nil, err - } - if chosen != nil { - if err := tx.ReservePlacement(ctx, *chosen); err != nil { - return nil, err - } - } + // Node placement belongs to the common scheduler, including when + // capacity is available, so new Sessions cannot bypass waiting work. } if len(batch) == 0 { return created, nil diff --git a/services/core/internal/sessions/creation_test.go b/services/core/internal/sessions/creation_test.go index 585c35be7..7cbc81e34 100644 --- a/services/core/internal/sessions/creation_test.go +++ b/services/core/internal/sessions/creation_test.go @@ -25,21 +25,18 @@ import ( type fakeCreationTx struct { *fakeInputTx - upsertSession func(NewSession) (Creation, error) - loadGenerationSpecification func(uint64) (json.RawMessage, error) - lockDeployment func() (placement.Deployment, error) - loadNodes func() ([]placement.Node, error) - reservePlacement func() error - lockSkills func() (map[string]skills.Skill, error) - readSkillVersion func() (skills.Content, error) - saveModelExecution func() error - saveExecutionConfiguration func() error - saveInitialFiles func() error - saveSetup func() error - createEnvironment func() (string, error) - pruneChanges func() error - auditCreation func() error - loadSession func() (Session, error) + upsertSession func(NewSession) (Creation, error) + lockDeployment func() (placement.Deployment, error) + lockSkills func() (map[string]skills.Skill, error) + readSkillVersion func() (skills.Content, error) + saveModelExecution func() error + saveExecutionConfiguration func() error + saveInitialFiles func() error + saveSetup func() error + createEnvironment func() (string, error) + pruneChanges func() error + auditCreation func() error + loadSession func() (Session, error) } var _ CreationTx = (*fakeCreationTx)(nil) @@ -54,21 +51,6 @@ func (f *fakeCreationTx) LockDeployment(context.Context) (placement.Deployment, return f.lockDeployment() } -func (f *fakeCreationTx) LoadGenerationSpecification(_ context.Context, generation uint64) (json.RawMessage, error) { - f.record("LoadGenerationSpecification", f.loadGenerationSpecification != nil, fmt.Sprint(generation)) - return f.loadGenerationSpecification(generation) -} - -func (f *fakeCreationTx) LoadNodes(context.Context) ([]placement.Node, error) { - f.record("LoadNodes", f.loadNodes != nil) - return f.loadNodes() -} - -func (f *fakeCreationTx) ReservePlacement(_ context.Context, chosen placement.Placement) error { - f.record("ReservePlacement", f.reservePlacement != nil, chosen.NodeID, fmt.Sprint(chosen.Generation)) - return f.reservePlacement() -} - func (f *fakeCreationTx) LockSkills(_ context.Context, ids []string) (map[string]skills.Skill, error) { f.record("LockSkills", f.lockSkills != nil, ids...) return f.lockSkills() @@ -151,9 +133,9 @@ var errFingerprint = errors.New("fingerprint failed") func failing(string) (string, error) { return "", errFingerprint } // declarations accept every provider and specification. -type declarations struct{} +type declarations struct{ publicOriginRequired bool } -func (declarations) RequiresPublicOrigin(string) (bool, error) { return false, nil } +func (d declarations) RequiresPublicOrigin(string) (bool, error) { return d.publicOriginRequired, nil } func (declarations) ValidateSpecification(string, sandbox.DeploymentSpec) error { return nil } @@ -451,44 +433,18 @@ func TestCreateSession(t *testing.T) { } }) - t.Run("a hosted Session is admitted under the deployment lock and placed after its Environment", func(t *testing.T) { - ready := uint64(5) - tx := newCreationTx(t, hostedSession, true) - tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Generation: 5, Specification: json.RawMessage(`{}`)}) - tx.createEnvironment = returns("environment") - tx.loadNodes = returns([]placement.Node{{ID: "node", Online: true, ServingReady: true, ReadyGeneration: &ready, MaxActive: 1, MaxRetained: 1, CoreURL: rules.PublicURL()}}) - tx.reservePlacement = done - _, err, calls := runCreation(t, rules, tx, creationInput("openai_hosted")) - want := []string{"UpsertSession create", "LockDeployment", "CreateEnvironment", "LoadNodes", "ReservePlacement node 5", "AuditCreation session:session environment:environment:session", "LoadSession"} - if err != nil || strings.Join(calls, "\n") != strings.Join(want, "\n") { - t.Fatalf("calls %q, %v", calls, err) - } - }) - for _, external := range []bool{false, true} { - t.Run(fmt.Sprintf("selected generation external=%t", external), func(t *testing.T) { - ready := uint64(4) + t.Run(fmt.Sprintf("admission validates target generation external=%t", external), func(t *testing.T) { tx := newCreationTx(t, hostedSession, true) - tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Generation: 5, Specification: json.RawMessage(`{"workspace":{}}`)}) - tx.createEnvironment = returns("environment") - tx.loadNodes = returns([]placement.Node{{ID: "node", Online: true, ServingReady: true, ReadyGeneration: &ready, MaxActive: 1, MaxRetained: 1, CoreURL: rules.PublicURL()}}) spec := json.RawMessage(`{}`) if external { spec = json.RawMessage(`{"workspace":{}}`) } - tx.loadGenerationSpecification = func(generation uint64) (json.RawMessage, error) { - if generation != 4 { - t.Fatal(generation) - } - return spec, nil - } - tx.reservePlacement = done - _, err, calls := runCreation(t, rules, tx, creationInput("openai_hosted")) + tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Mode: string(sandbox.DeploymentNodes), Generation: 5, Specification: spec}) + tx.createEnvironment = returns("environment") + _, err, _ := runCreation(t, rules, tx, creationInput("openai_hosted")) if external != errors.Is(err, ErrInvalidInput) { - t.Fatalf("selected generation admission: %v", err) - } - if external && strings.Contains(strings.Join(calls, " "), "ReservePlacement") { - t.Fatal("unsupported external placement reserved", calls) + t.Fatalf("target generation admission: %v", err) } }) } @@ -503,10 +459,12 @@ func TestCreateSession(t *testing.T) { {"a reset closes hosted admission", rules, func(tx *fakeCreationTx) { tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Resetting: true}) }, placement.ErrResetAdmission, []string{"UpsertSession create", "LockDeployment"}}, - {"no node places the Environment", rules, func(tx *fakeCreationTx) { - tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Generation: 5, Specification: json.RawMessage(`{}`)}) - tx.createEnvironment, tx.loadNodes = returns("environment"), returns([]placement.Node(nil)) - }, placement.ErrNodeUnavailable, []string{"UpsertSession create", "LockDeployment", "CreateEnvironment", "LoadNodes"}}, + {"an unconfigured deployment still rejects admission", rules, func(tx *fakeCreationTx) { + tx.lockDeployment = returns(placement.Deployment{Mode: string(sandbox.DeploymentNodes)}) + }, placement.ErrNodeUnavailable, []string{"UpsertSession create", "LockDeployment"}}, + {"an invalid deployment still rejects admission", rules, func(tx *fakeCreationTx) { + tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Mode: string(sandbox.DeploymentNodes), Specification: json.RawMessage(`[]`)}) + }, placement.ErrAdmissionClosed, []string{"UpsertSession create", "LockDeployment"}}, {"hosted creation needs rules", nil, func(*fakeCreationTx) {}, nil, []string{"UpsertSession create"}}, } { t.Run(test.name, func(t *testing.T) { @@ -551,6 +509,67 @@ func TestCreateSession(t *testing.T) { }) } +func TestCreateSessionDefersNodePlacement(t *testing.T) { + rules, err := placement.NewRules(declarations{}, "https://core.example") + if err != nil { + t.Fatal(err) + } + for _, mode := range []sandbox.DeploymentMode{sandbox.DeploymentNodes, sandbox.DeploymentDirect} { + for _, initialInput := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/initial=%t", mode, initialInput), func(t *testing.T) { + input := creationInput("openai_hosted") + input.SupportsRetainedNativeHistory = true + if initialInput { + input.InitialInputs = []Input{messageInput("hi")} + } + session := Session{ID: "session", Engine: "codex", Configuration: input.Configuration} + tx := newCreationTx(t, session, true) + tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "fixture", Mode: string(mode), Generation: 5, Specification: json.RawMessage(`{"workspace":{}}`)}) + tx.createEnvironment = returns("environment") + if initialInput { + tx.loadEnvironmentInput, tx.createInputReservation, tx.pruneChanges = inputs(), returns(EnvironmentInputReservation{}), done + } + result, err, calls := runCreation(t, rules, tx, input) + if err != nil || !result.Created { + t.Fatalf("creation %+v, %v", result, err) + } + want := []string{"UpsertSession create", "LockDeployment", "CreateEnvironment"} + if initialInput { + want = append(want, "LoadEnvironmentInput", `CreateInputReservation [{"kind":"message","payload":`+hi+`}] initial`, "LoadEnvironmentInput", "PruneChanges") + } + want = append(want, "AuditCreation session:session environment:environment:session", "LoadSession") + if strings.Join(calls, "\n") != strings.Join(want, "\n") { + t.Fatalf("calls %q, want %q", calls, want) + } + // A retry reads the durable Session without reserving input again. + retry := newCreationTx(t, session, false) + result, err, calls = runCreation(t, rules, retry, input) + if err != nil || result.Created || result.Session.ID != session.ID || strings.Join(calls, "\n") != "UpsertSession create\nAuditCreation\nLoadSession" { + t.Fatalf("retry %+v, calls %q, %v", result, calls, err) + } + }) + } + } +} + +func TestCreateSessionStillChecksPublicOriginBeforePlacement(t *testing.T) { + rules, err := placement.NewRules(declarations{publicOriginRequired: true}, "http://localhost:8080") + if err != nil { + t.Fatal(err) + } + for _, mode := range []sandbox.DeploymentMode{sandbox.DeploymentNodes, sandbox.DeploymentDirect} { + t.Run(string(mode), func(t *testing.T) { + input := creationInput("openai_hosted") + tx := newCreationTx(t, Session{ID: "session", Configuration: input.Configuration}, true) + tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "fixture", Mode: string(mode), Specification: json.RawMessage(`{}`)}) + tx.createEnvironment = returns("environment") + if _, err, _ := runCreation(t, rules, tx, input); !errors.Is(err, placement.ErrPublicURLUnreachable) { + t.Fatalf("unreachable public origin admitted: %v", err) + } + }) + } +} + func TestFindSessionCreation(t *testing.T) { request := json.RawMessage(`{"agent_id":"agent"}`) hash, err := intentHash(request, fingerprints) diff --git a/services/core/internal/sessions/transaction.go b/services/core/internal/sessions/transaction.go index 10e58f34c..6d5b84542 100644 --- a/services/core/internal/sessions/transaction.go +++ b/services/core/internal/sessions/transaction.go @@ -14,6 +14,8 @@ import ( // LockedSession is what locking a Session row shows. type LockedSession struct { + // Engine is the immutable Harness selected when the Session was created. + Engine string // Deleted reports that the Session was publicly deleted. Its row stays so // that remaining execution can settle. Deleted bool diff --git a/services/core/tests/integration/admin_session_archive_test.go b/services/core/tests/integration/admin_session_archive_test.go index 810a5ee48..5101af9b1 100644 --- a/services/core/tests/integration/admin_session_archive_test.go +++ b/services/core/tests/integration/admin_session_archive_test.go @@ -69,6 +69,15 @@ func managedArchiveSession(t *testing.T, s *Store, input sessions.CreateSession) func archiveAllocation(t *testing.T, w *Store, tenant string, session sessions.Session, installation string) deployment.Allocation { t.Helper() + setup, err := deploymentService(t, w).Setup(t.Context()) + if err != nil { + t.Fatal(err) + } + if setup.Mode == "nodes" { + if err := reserveSessionPlacement(t, w, w, session); err != nil { + t.Fatal(err) + } + } owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())) if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/deployment_public_fixture_test.go b/services/core/tests/integration/deployment_public_fixture_test.go index b1ce77186..565cdab05 100644 --- a/services/core/tests/integration/deployment_public_fixture_test.go +++ b/services/core/tests/integration/deployment_public_fixture_test.go @@ -1,6 +1,7 @@ package integration import ( + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" @@ -34,6 +35,6 @@ func fixtureDeploymentExecution(s *Store, lease *pgunit.Lease) (*deployment.Serv if err != nil { return nil, nil, err } - changes, err := deployment.NewExecutionOperations(service, deploymentpg.NewExecution(lease, s.credentialCipher)) + changes, err := deployment.NewExecutionOperations(service, deploymentpg.NewExecution(lease, s.credentialCipher), engine.Catalog{}) return service, changes, err } diff --git a/services/core/tests/integration/device_bootstrap_binding_test.go b/services/core/tests/integration/device_bootstrap_binding_test.go index 4d85bc1c9..f6187833c 100644 --- a/services/core/tests/integration/device_bootstrap_binding_test.go +++ b/services/core/tests/integration/device_bootstrap_binding_test.go @@ -28,7 +28,7 @@ func TestDeviceCredentialCarriesPersistedAllocationNode(t *testing.T) { for _, nodeID := range []string{d.NodeID, remote} { t.Run(nodeID, func(t *testing.T) { tenant, bearer := uuid.NewString(), uuid.NewString() - session, err := createSessionOnNode(t, s, tenant, managerSessionInput(uuid.NewString()), nodeID) + session, err := createSessionOnNode(t, s, writer, tenant, managerSessionInput(uuid.NewString()), nodeID) if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/execution_test.go b/services/core/tests/integration/execution_test.go index 8e73078db..66a31026c 100644 --- a/services/core/tests/integration/execution_test.go +++ b/services/core/tests/integration/execution_test.go @@ -99,7 +99,8 @@ func executionOwnerPID(t *testing.T, pool *pgxpool.Pool) int32 { t.Helper() var pid int32 err := pool.QueryRow(t.Context(), `SELECT pid FROM pg_locks WHERE locktype='advisory' AND granted AND objsubid=1 - AND classid::bigint * 4294967296 + objid::bigint = 706172736172 + AND classid::bigint = (706172736172::bigint >> 32) + AND objid::bigint = (706172736172::bigint & 4294967295) AND database=(SELECT oid FROM pg_database WHERE datname=current_database())`).Scan(&pid) if err != nil { t.Fatal("observe execution lease owner", err) diff --git a/services/core/tests/integration/runtime_idle_clock_test.go b/services/core/tests/integration/runtime_idle_clock_test.go index e0aa96fb7..298b2e082 100644 --- a/services/core/tests/integration/runtime_idle_clock_test.go +++ b/services/core/tests/integration/runtime_idle_clock_test.go @@ -29,6 +29,9 @@ func managedIdleClockFixture(t *testing.T) (*Store, *Store, deployment.Allocatio if err != nil { t.Fatal(err) } + if err := reserveSessionPlacement(t, s, w, session); err != nil { + t.Fatal(err) + } owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, d.InstallationID, runtimedevice.HashCredential("runtime")) if err != nil { t.Fatal(err) @@ -68,7 +71,7 @@ func verifyManagedIdleClock(t *testing.T, s, w *Store, owner deployment.Allocati t.Fatal("idle clock did not use committed terminal ingestion", activity, before, after, err) } until := runtimeDatabaseTime(t, s).Add(time.Hour) - if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleTimeout); !errors.Is(err, deployment.ErrAllocationConflict) { + if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleTimeout); !errors.Is(err, deployment.ErrNotIdle) { t.Fatal("new completion admitted premature idle", err) } // Advance only the internal activity age; the remote public timestamp remains unchanged. diff --git a/services/core/tests/integration/runtime_lifecycle_nodes_test.go b/services/core/tests/integration/runtime_lifecycle_nodes_test.go index a26698128..b87368a27 100644 --- a/services/core/tests/integration/runtime_lifecycle_nodes_test.go +++ b/services/core/tests/integration/runtime_lifecycle_nodes_test.go @@ -28,10 +28,10 @@ func lifecycleTestNode(t *testing.T, s *Store) string { onlineManagerNode(t, s, id) return id } -func lifecycleTestSession(t *testing.T, s *Store, node string) (string, sessions.Session) { +func lifecycleTestSession(t *testing.T, s, w *Store, node string) (string, sessions.Session) { t.Helper() tenant := uuid.NewString() - session, err := createSessionOnNode(t, s, tenant, managerSessionInput(uuid.NewString()), node) + session, err := createSessionOnNode(t, s, w, tenant, managerSessionInput(uuid.NewString()), node) if err != nil { t.Fatal(err) } @@ -39,7 +39,7 @@ func lifecycleTestSession(t *testing.T, s *Store, node string) (string, sessions } func lifecycleTestAllocation(t *testing.T, s, w *Store, d managerNode, node string) deployment.Allocation { t.Helper() - tenant, session := lifecycleTestSession(t, s, node) + tenant, session := lifecycleTestSession(t, s, w, node) allocation, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, d.InstallationID, runtimedevice.HashCredential(uuid.NewString())) if err != nil { t.Fatal(err) @@ -53,11 +53,11 @@ func TestRuntimeLifecycleNodePagesAreIndependent(t *testing.T) { var allocated, pending []string for range 34 { allocated = append(allocated, lifecycleTestAllocation(t, s, w, d, d.NodeID).ID) - _, session := lifecycleTestSession(t, s, d.NodeID) + _, session := lifecycleTestSession(t, s, w, d.NodeID) pending = append(pending, session.Environment.ID) } second := lifecycleTestAllocation(t, s, w, d, other) - _, secondPending := lifecycleTestSession(t, s, other) + _, secondPending := lifecycleTestSession(t, s, w, other) // Offline and unresolved cleanup remain discoverable without changing placement. if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.NodeID); err != nil { t.Fatal(err) @@ -126,7 +126,7 @@ func TestRuntimeLifecycleNodePagesAreIndependent(t *testing.T) { func TestRuntimeLifecycleNodeInventoryAndRouting(t *testing.T) { s, w, d := managerFixture(t, 100, 100) other := lifecycleTestNode(t, s) - tenant, session := lifecycleTestSession(t, s, other) + tenant, session := lifecycleTestSession(t, s, w, other) environment := session.Environment.ID checkRoute := func(want string, wantErr error) { t.Helper() @@ -190,7 +190,7 @@ func TestRuntimeLifecycleNodeRejectsMissingOrReleasedPlacement(t *testing.T) { for _, mutation := range []string{"DELETE FROM runtime_placements WHERE environment_id=$1", "UPDATE runtime_placements SET released_at=clock_timestamp() WHERE environment_id=$1"} { t.Run(mutation[:6], func(t *testing.T) { s, w, d := managerFixture(t, 4, 4) - tenant, session := lifecycleTestSession(t, s, d.NodeID) + tenant, session := lifecycleTestSession(t, s, w, d.NodeID) if _, err := s.pool.Exec(t.Context(), mutation, session.Environment.ID); err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/runtime_node_generations_test.go b/services/core/tests/integration/runtime_node_generations_test.go index f491d98cf..8af333c9c 100644 --- a/services/core/tests/integration/runtime_node_generations_test.go +++ b/services/core/tests/integration/runtime_node_generations_test.go @@ -35,8 +35,11 @@ func generationHeartbeat(t *testing.T, s *Store, node deployment.Enrollment, con } } -func placedGeneration(t *testing.T, s *Store, session sessions.Session) (string, int64) { +func placedGeneration(t *testing.T, s, w *Store, session sessions.Session) (string, int64) { t.Helper() + if err := reserveSessionPlacement(t, s, w, session); err != nil { + t.Fatal(err) + } var node string var generation int64 if err := s.pool.QueryRow(t.Context(), "SELECT node_id::text,deployment_generation FROM runtime_placements WHERE environment_id=$1", session.Environment.ID).Scan(&node, &generation); err != nil { @@ -59,7 +62,7 @@ func TestNodeGenerationsCapacityFallbackAndImmutablePending(t *testing.T) { if err != nil { t.Fatal(err) } - pendingNode, pendingGeneration := placedGeneration(t, s, pending) + pendingNode, pendingGeneration := placedGeneration(t, s, w, pending) token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 2})) if err != nil { t.Fatal(err) @@ -76,7 +79,7 @@ func TestNodeGenerationsCapacityFallbackAndImmutablePending(t *testing.T) { if err != nil { t.Fatal("target preparation suppressed fallback", err) } - if _, g := placedGeneration(t, s, fallback); g != 1 { + if _, g := placedGeneration(t, s, w, fallback); g != 1 { t.Fatal(g) } generationHeartbeat(t, s, a, ca, second, "ready") @@ -84,7 +87,7 @@ func TestNodeGenerationsCapacityFallbackAndImmutablePending(t *testing.T) { if err != nil { t.Fatal(err) } - if n, g := placedGeneration(t, s, newest); n != a.NodeID || g != 2 { + if n, g := placedGeneration(t, s, w, newest); n != a.NodeID || g != 2 { t.Fatal(n, g) } // Capacity remains shared across generations. A full newest pin cannot hide B. @@ -95,7 +98,7 @@ func TestNodeGenerationsCapacityFallbackAndImmutablePending(t *testing.T) { if err != nil { t.Fatal(err) } - if n, g := placedGeneration(t, s, old); n != b.NodeID || g != 1 { + if n, g := placedGeneration(t, s, w, old); n != b.NodeID || g != 1 { t.Fatal("newest full hid older free node", n, g) } third, _ := changeNodeTarget(t, w, second, input) @@ -163,7 +166,11 @@ func TestNodeGenerationsReconnectAndV1Fallback(t *testing.T) { if err != nil || nodes[0].ProviderReady || *nodes[0].Rollout.ReadyGeneration != 1 { t.Fatal("reconnect inherited readiness or lost pin", nodes, err) } - if _, err = s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, placement.ErrNodeUnavailable) { + queued, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + if err := reserveSessionPlacement(t, s, w, queued); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("unconfirmed connection admitted", err) } if err = service.Heartbeat(t.Context(), node.NodeID, connection, first.OwnerEpoch, deployment.NodeHealth{ProviderReady: true}); err != nil { @@ -175,7 +182,7 @@ func TestNodeGenerationsReconnectAndV1Fallback(t *testing.T) { } func TestNodeGenerationPreparationRefusalCreatesNoProvisionalOwnership(t *testing.T) { - s, _, first, _ := webSpecificationFixture(t, "docker") + s, w, first, _ := webSpecificationFixture(t, "docker") node := specificationNode(t, s, first) connection := uuid.NewString() if err := deploymentService(t, s).ConnectNode(t.Context(), node.NodeID, connection, first.OwnerEpoch); err != nil { @@ -183,15 +190,23 @@ func TestNodeGenerationPreparationRefusalCreatesNoProvisionalOwnership(t *testin } generationHeartbeat(t, s, node, connection, first, "preparing") tenant := uuid.NewString() - if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); !errors.Is(err, placement.ErrNodesPreparing) { + queued, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + if err := reserveSessionPlacement(t, s, w, queued); !errors.Is(err, placement.ErrNodesPreparing) { t.Fatal("actual preparation was not identified", err) } var sessions, placements int - if err := s.pool.QueryRow(t.Context(), "SELECT (SELECT count(*) FROM sessions WHERE tenant_id=$1),(SELECT count(*) FROM runtime_placements)", tenant).Scan(&sessions, &placements); err != nil || sessions != 0 || placements != 0 { - t.Fatal("refusal left provisional ownership", sessions, placements, err) + if err := s.pool.QueryRow(t.Context(), "SELECT (SELECT count(*) FROM sessions WHERE tenant_id=$1),(SELECT count(*) FROM runtime_placements)", tenant).Scan(&sessions, &placements); err != nil || sessions != 1 || placements != 0 { + t.Fatal("waiting request consumed provisional compute", sessions, placements, err) } generationHeartbeat(t, s, node, connection, first, "failed") - if _, err := s.CreateSession(t.Context(), tenant, managerSessionInput(uuid.NewString())); !errors.Is(err, placement.ErrNodeUnavailable) { + if err := reserveSessionPlacement(t, s, w, queued); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("failed preparation advertised active work", err) } + generationHeartbeat(t, s, node, connection, first, "ready") + if err := reserveSessionPlacement(t, s, w, queued); err != nil { + t.Fatal("ready node did not admit waiting Session", err) + } } diff --git a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go index 055ac6383..f6803bf48 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go @@ -77,6 +77,7 @@ type nodeIsolationFixture struct { nodes *deployment.Service pool *pgxpool.Pool worker *execution.Worker + placement *deployment.ExecutionOperations provider *nodeIsolationProvider key, nodeA, nodeB string epoch uint64 @@ -119,7 +120,9 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { f := &nodeIsolationFixture{initializationCancel: cancelPreparation, t: t, store: s, nodes: deploymentService(t, s), pool: pool, provider: p, key: webDeployment(t, s, "microsandbox"), nodeA: uuid.NewString(), nodeB: uuid.NewString()} // Keep restored compute awake throughout the isolation assertions. // The suspension setup explicitly dates its activity two minutes in the past. - f.worker = startWebWorker(t, s, registry, f.key, p, &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Minute, Retention: time.Hour}) + owner := executionOwner(t, s) + f.placement = owner.Deployment + f.worker = startOwnedWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: webRuntimes(t, s, f.key, p, &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Minute, Retention: time.Hour})}, owner) t.Cleanup(f.stop) f.epoch = fixtureOwnerEpoch(t, s) f.enroll(f.nodeA) @@ -185,6 +188,9 @@ func (f *nodeIsolationFixture) session(node string, initialize bool) (string, se } } session, err := f.store.CreateSession(f.t.Context(), tenant, input) + if err == nil { + _, err = f.placement.EnsurePlacement(f.t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, f.key) + } for _, value := range others { if err := f.nodes.Heartbeat(context.WithoutCancel(f.t.Context()), value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { f.t.Fatal(err) diff --git a/services/core/tests/integration/runtime_nodes_test.go b/services/core/tests/integration/runtime_nodes_test.go index 3ab2694c2..4150a0681 100644 --- a/services/core/tests/integration/runtime_nodes_test.go +++ b/services/core/tests/integration/runtime_nodes_test.go @@ -44,8 +44,8 @@ func managerSessionInput(key string) sessions.CreateSession { } // createSessionOnNode steers automatic placement in multi-node tests: only node -// stays provider-ready while the Session is created. -func createSessionOnNode(t *testing.T, s *Store, tenant string, input sessions.CreateSession, node string) (sessions.Session, error) { +// stays provider-ready while the scheduler reserves the created Session. +func createSessionOnNode(t *testing.T, s, w *Store, tenant string, input sessions.CreateSession, node string) (sessions.Session, error) { t.Helper() // Use the same authenticated readiness observations as the scheduler. The // compatibility provider_ready column alone is not admission authority. @@ -83,7 +83,21 @@ func createSessionOnNode(t *testing.T, s *Store, tenant string, input sessions.C } } }() - return s.CreateSession(t.Context(), tenant, input) + session, err := s.CreateSession(t.Context(), tenant, input) + if err == nil { + err = reserveSessionPlacement(t, s, w, session) + } + return session, err +} + +func reserveSessionPlacement(t *testing.T, s, w *Store, session sessions.Session) error { + t.Helper() + setup, err := deploymentService(t, s).Setup(t.Context()) + if err != nil { + return err + } + _, err = deploymentExecution(t, w).EnsurePlacement(t.Context(), deployment.AllocationKey{TenantID: session.TenantID, EnvironmentID: session.Environment.ID}, setup.InstallationID) + return err } type sessionPlacement struct { @@ -115,7 +129,7 @@ func sessionRuntimePlacement(ctx context.Context, s *Store, tenant, session stri return placement, err } func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { - s, _, d := managerFixture(t, 1, 4) + s, w, d := managerFixture(t, 1, 4) tenant := uuid.NewString() var wg sync.WaitGroup results := make(chan sessions.Session, 16) @@ -132,23 +146,46 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { wg.Wait() close(results) close(failures) - successes := 0 for err := range failures { - if err == nil { - successes++ - } else if !errors.Is(err, placement.ErrNodeUnavailable) { - t.Fatal(err) + if err != nil { + t.Fatal("capacity rejected durable Session creation", err) } } - if successes != 1 { - t.Fatal("overbooked node", successes) + var created []sessions.Session + for session := range results { + created = append(created, session) + } + if len(created) != 16 { + t.Fatal("lost accepted Sessions", len(created)) + } + type admission struct { + session sessions.Session + err error + } + reserved := make(chan admission, len(created)) + for _, session := range created { + wg.Add(1) + go func() { defer wg.Done(); reserved <- admission{session, reserveSessionPlacement(t, s, w, session)} }() } + wg.Wait() + close(reserved) var retained sessions.Session - for session := range results { - if session.ID != "" { - retained = session + var waiting []sessions.Session + for result := range reserved { + if result.err == nil { + if retained.ID != "" { + t.Fatal("overbooked node") + } + retained = result.session + } else if errors.Is(result.err, placement.ErrNodeUnavailable) { + waiting = append(waiting, result.session) + } else { + t.Fatal(result.err) } } + if retained.ID == "" || len(waiting) != 15 { + t.Fatal("capacity admission", retained.ID, len(waiting)) + } service := deploymentService(t, s) nodes, err := service.ListNodes(t.Context()) if err != nil || len(nodes) != 1 || nodes[0].Active != 1 || nodes[0].Retained != 1 || nodes[0].Reserved != 1 { @@ -160,11 +197,20 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: retained.ID}); err != nil { t.Fatal(err) } + if err := reserveSessionPlacement(t, s, w, waiting[0]); err != nil { + t.Fatal("released slot did not admit waiting Session", err) + } + if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: waiting[0].ID}); err != nil { + t.Fatal(err) + } input := managerSessionInput("retry") first, err := s.CreateSession(t.Context(), tenant, input) if err != nil { t.Fatal(err) } + if err := reserveSessionPlacement(t, s, w, first); err != nil { + t.Fatal(err) + } if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.NodeID); err != nil { t.Fatal(err) } @@ -225,7 +271,11 @@ func TestRuntimeNodesEnrollmentAndEpoch(t *testing.T) { if err := nodes.Heartbeat(t.Context(), input.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("old epoch heartbeat revived node", err) } - if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput("stale")); !errors.Is(err, placement.ErrNodeUnavailable) { + queued, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput("stale")) + if err != nil { + t.Fatal(err) + } + if err := reserveSessionPlacement(t, s, w, queued); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("stale node admitted", err) } onlineManagerNode(t, s, d.NodeID) @@ -244,6 +294,9 @@ func TestRuntimeNodesRetention(t *testing.T) { if err != nil { t.Fatal(err) } + if err := reserveSessionPlacement(t, s, w, first); err != nil { + t.Fatal(err) + } retained, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: first.Environment.ID}, next.InstallationID, runtimedevice.HashCredential("runtime")) if err != nil { t.Fatal(err) @@ -286,6 +339,9 @@ func TestRuntimeNodesRestoreAndCreationShareCapacity(t *testing.T) { if err != nil { t.Fatal(err) } + if err := reserveSessionPlacement(t, s, w, session); err != nil { + t.Fatal(err) + } allocation, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, d.InstallationID, runtimedevice.HashCredential("runtime")) if err != nil { t.Fatal(err) @@ -298,6 +354,10 @@ func TestRuntimeNodesRestoreAndCreationShareCapacity(t *testing.T) { t.Fatal(err) } until := time.Now().Add(time.Hour) + second, err := s.CreateSession(t.Context(), tenant, managerSessionInput("second")) + if err != nil { + t.Fatal(err) + } start := make(chan struct{}) results := make(chan error, 2) go func() { @@ -307,8 +367,7 @@ func TestRuntimeNodesRestoreAndCreationShareCapacity(t *testing.T) { }() go func() { <-start - _, err := s.CreateSession(t.Context(), tenant, managerSessionInput("second")) - results <- err + results <- reserveSessionPlacement(t, s, w, second) }() close(start) success := 0 @@ -344,6 +403,9 @@ func TestRuntimeNodesLongOfflineRetainsExactAllocation(t *testing.T) { if err != nil { t.Fatal(err) } + if err := reserveSessionPlacement(t, s, w, session); err != nil { + t.Fatal(err) + } owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, d.InstallationID, runtimedevice.HashCredential("runtime")) if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/runtime_observation_test.go b/services/core/tests/integration/runtime_observation_test.go index 5ad2acf2b..1c84c5c95 100644 --- a/services/core/tests/integration/runtime_observation_test.go +++ b/services/core/tests/integration/runtime_observation_test.go @@ -16,6 +16,9 @@ func TestRuntimeNodeObservationRetainsResourcesAndFencesStaleResults(t *testing. if err != nil { t.Fatal(err) } + if err := reserveSessionPlacement(t, s, w, session); err != nil { + t.Fatal(err) + } owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, d.InstallationID, runtimedevice.HashCredential("runtime")) if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/runtime_suspension_test.go b/services/core/tests/integration/runtime_suspension_test.go index 3e2b73be3..2f76e9ffc 100644 --- a/services/core/tests/integration/runtime_suspension_test.go +++ b/services/core/tests/integration/runtime_suspension_test.go @@ -355,7 +355,7 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { runtimeSuspensionSQL(t, pool, `INSERT INTO environment_file_writes(id,environment_id,device_id,request_sha256,state,created_at,settled_at) VALUES($1,$2,$3,$4,$5,clock_timestamp()-interval '10 minutes',clock_timestamp())`, uuid.NewString(), owner.EnvironmentID, owner.DeviceID, strings.Repeat("a", 64), strings.TrimPrefix(kind, "file_")) } until := time.Now().Add(time.Hour) - if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleTimeout); !errors.Is(err, deployment.ErrAllocationConflict) { + if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleTimeout); !errors.Is(err, deployment.ErrNotIdle) { t.Fatal("completion after idle observation did not fence quiesce", err) } activity, err := deploymentStore(w).Activity(t.Context(), owner.ID) @@ -407,6 +407,9 @@ func TestRuntimeComputePhaseChangedAtInNodeAllocations(t *testing.T) { if err != nil { t.Fatal(err) } + if err := reserveSessionPlacement(t, s, w, session); err != nil { + t.Fatal(err) + } allocation, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, d.InstallationID, runtimedevice.HashCredential("runtime")) if err != nil { t.Fatal(err) diff --git a/services/core/tests/integration/sandbox_deployment_worker_test.go b/services/core/tests/integration/sandbox_deployment_worker_test.go index 7e76740a7..b2143dc1e 100644 --- a/services/core/tests/integration/sandbox_deployment_worker_test.go +++ b/services/core/tests/integration/sandbox_deployment_worker_test.go @@ -50,8 +50,12 @@ func TestSandboxDeploymentWorkerActivatesWithoutRestart(t *testing.T) { if _, err := w.InitializeSandboxDeployment(t.Context(), sandbox.Selection{DeploymentSpec: SandboxDeploymentTestSpec("docker"), Provider: "docker"}); err != nil { t.Fatal(err) } - if _, err := s.CreateSession(t.Context(), uuid.NewString(), input); !errors.Is(err, placement.ErrNodeUnavailable) { - t.Fatal("zero-node deployment admitted Session", err) + queued, err := s.CreateSession(t.Context(), uuid.NewString(), input) + if err != nil { + t.Fatal("zero-node deployment rejected waiting Session", err) + } + if _, err := w.ProvisionEnvironment(t.Context(), queued.TenantID, queued.Environment.ID, id); !errors.Is(err, placement.ErrNodeUnavailable) { + t.Fatal("zero-node deployment provisioned compute", err) } token, err := EnrollmentTestToken(deployments.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 4, MaxRetained: 16})) if err != nil { diff --git a/services/core/tests/integration/sandbox_generations_test.go b/services/core/tests/integration/sandbox_generations_test.go index 580459b2a..2b08131df 100644 --- a/services/core/tests/integration/sandbox_generations_test.go +++ b/services/core/tests/integration/sandbox_generations_test.go @@ -110,6 +110,9 @@ func TestPendingPlacementGenerationSurvivesRepeatedUpdates(t *testing.T) { s, w, view, _ := webSpecificationFixture(t, "docker") node := specificationNode(t, s, view) tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + if err := reserveSessionPlacement(t, s, w, session); err != nil { + t.Fatal(err) + } for generation := uint64(1); generation <= 2; generation++ { tx, err := s.pool.Begin(t.Context()) if err != nil { diff --git a/services/core/tests/integration/sandbox_reset_test.go b/services/core/tests/integration/sandbox_reset_test.go index cc37abdc4..18936ec6b 100644 --- a/services/core/tests/integration/sandbox_reset_test.go +++ b/services/core/tests/integration/sandbox_reset_test.go @@ -253,6 +253,9 @@ func TestSandboxResetAuditFailureRollsBackPauseAndCompletion(t *testing.T) { func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { s, w, d := managerFixture(t, 10, 10) _, pending := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + if err := reserveSessionPlacement(t, s, w, pending); err != nil { + t.Fatal(err) + } tenant, suspended := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) allocation := archiveAllocation(t, w, tenant, suspended, d.InstallationID) runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_allocations SET compute_phase='suspended',compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, allocation.ID) diff --git a/services/core/tests/integration/sandbox_specification_lifecycle_test.go b/services/core/tests/integration/sandbox_specification_lifecycle_test.go index 6c27b7e03..f3a87dc6c 100644 --- a/services/core/tests/integration/sandbox_specification_lifecycle_test.go +++ b/services/core/tests/integration/sandbox_specification_lifecycle_test.go @@ -132,7 +132,7 @@ func TestSandboxSpecificationChangesPreserveEveryRetainedResource(t *testing.T) changes, deployments := deploymentExecution(t, w), deploymentService(t, s) node := specificationNode(t, s, view) tenant := uuid.NewString() - session, err := createSessionOnNode(t, s, tenant, managerSessionInput(uuid.NewString()), node.NodeID) + session, err := createSessionOnNode(t, s, w, tenant, managerSessionInput(uuid.NewString()), node.NodeID) if err != nil { t.Fatal(err) } @@ -310,24 +310,35 @@ func TestSandboxSpecificationAllocationRaceWithMaintenance(t *testing.T) { // A node keeps the public URL it enrolled with. After the public URL changes it // receives no new sandboxes until it is re-added. func TestNodeBoundToAnotherPublicURLGetsNoNewSandboxes(t *testing.T) { - s, _, view, _ := webSpecificationFixture(t, "docker") + s, w, view, _ := webSpecificationFixture(t, "docker") s.SetPlacement(placementRules(t, "https://old.example")) + w.SetPlacement(s.placement) node := specificationNode(t, s, view) nodes, err := deploymentService(t, s).ListNodes(t.Context()) if err != nil || len(nodes) != 1 || nodes[0].ID != node.NodeID || nodes[0].CoreURL != "https://old.example" { t.Fatal("enrollment did not record the node's address", nodes, err) } s.SetPlacement(placementRules(t, "https://new.example")) - if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); !errors.Is(err, placement.ErrNodeUnavailable) { + w.SetPlacement(s.placement) + queued, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + if err := reserveSessionPlacement(t, s, w, queued); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("placed a new sandbox on a node bound to the old address", err) } bindings, err := deploymentService(t, s).AddressBindings(t.Context()) - if err != nil || bindings.Nodes != 1 || bindings.NodesOnOtherAddress != 1 || bindings.HostedSandboxes != 0 { + if err != nil || bindings.Nodes != 1 || bindings.NodesOnOtherAddress != 1 || bindings.HostedSandboxes != 1 { t.Fatal(bindings, err) } + var placements int + if err := s.pool.QueryRow(t.Context(), "SELECT count(*) FROM runtime_placements WHERE environment_id=$1", queued.Environment.ID).Scan(&placements); err != nil || placements != 0 { + t.Fatal("waiting Session acquired an incompatible placement", placements, err) + } s.SetPlacement(placementRules(t, "https://old.example")) - if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); err != nil { - t.Fatal("node on the current address rejected placement", err) + w.SetPlacement(s.placement) + if err := reserveSessionPlacement(t, s, w, queued); err != nil { + t.Fatal("node on current address rejected waiting placement", err) } } diff --git a/services/core/tests/integration/session_deletion_test.go b/services/core/tests/integration/session_deletion_test.go index f23b20593..f58924b7d 100644 --- a/services/core/tests/integration/session_deletion_test.go +++ b/services/core/tests/integration/session_deletion_test.go @@ -387,6 +387,9 @@ func TestSessionDeletionKeepsProvisioningInputPlacementUntilSettled(t *testing.T if err != nil { t.Fatal(err) } + if err := reserveSessionPlacement(t, s, w, session); err != nil { + t.Fatal(err) + } type state struct { deleted, released pgtype.Timestamptz retained, reserved int64 diff --git a/services/core/tests/integration/session_devices_fixture_test.go b/services/core/tests/integration/session_devices_fixture_test.go index 9dc9380e5..9181b28a3 100644 --- a/services/core/tests/integration/session_devices_fixture_test.go +++ b/services/core/tests/integration/session_devices_fixture_test.go @@ -20,6 +20,7 @@ func bindSessionDevice(t *testing.T, s *Store, tenant, session, device string) e if err != nil { return err } + defer func() { _ = lease.Close(context.Background()) }() released := pgtest.ObserveExecutionLeaseRelease(t, s.pool) operations, err := sessions.NewExecutionOperations(sessionpg.NewExecution(lease)) if err == nil { From c8dd06a9b10aebd6492ac515fd0e5cbfc18c32fb Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 01:15:43 +0800 Subject: [PATCH 12/22] Honor initialization execution budget (#647) --- .../internal/dispatch/runtime_preparation.go | 29 ++++--- .../dispatch/runtime_preparation_test.go | 81 ++++++++++++++++++- .../runtime_initialization_process.go | 8 +- .../runtime_initialization_test.go | 24 +++++- .../localworkspace/snapshot_marker_test.go | 4 +- contracts/agents-api/environments.md | 2 +- contracts/agents-api/zh/environments.md | 4 +- docs/runtime-protocol.md | 2 +- docs/zh/runtime-protocol.md | 4 +- internal/agentdaemon/proto/runtime_prepare.go | 18 +++-- .../agentdaemon/proto/runtime_prepare_test.go | 26 +++++- internal/agentdaemon/proto/version.go | 2 +- .../execution/runtime_capabilities_test.go | 6 +- .../execution/runtime_initialization.go | 15 ++-- .../core/internal/execution/runtime_setup.go | 7 ++ .../internal/execution/runtime_setup_test.go | 40 ++++++++- .../runtimegateway/runtime_prepare.go | 12 ++- .../runtimegateway/runtime_prepare_test.go | 78 +++++++++++++++++- 18 files changed, 302 insertions(+), 60 deletions(-) diff --git a/apps/daemon/internal/dispatch/runtime_preparation.go b/apps/daemon/internal/dispatch/runtime_preparation.go index 1c777f0fe..926ddc5f9 100644 --- a/apps/daemon/internal/dispatch/runtime_preparation.go +++ b/apps/daemon/internal/dispatch/runtime_preparation.go @@ -13,19 +13,18 @@ import ( "github.com/google/uuid" ) -const runtimePreparationTimeout = 120 * time.Second - // Router.mu protects one connection-local transfer. Partial installation data // belongs to the bound Environment and is never removed by transfer cleanup. type runtimePreparationTransfer struct { - envelope proto.Envelope - request proto.RuntimePreparePayload - data []byte - ready chan struct{} - cancel context.CancelFunc - finished bool - apply bool - uncertain bool + transferDeadline time.Time + envelope proto.Envelope + request proto.RuntimePreparePayload + data []byte + ready chan struct{} + cancel context.CancelFunc + finished bool + apply bool + uncertain bool } func (r *Router) handleRuntimePrepare(ctx context.Context, env proto.Envelope) error { @@ -65,9 +64,10 @@ func (r *Router) handleRuntimePrepare(ctx context.Context, env proto.Envelope) e r.mu.Unlock() return r.sendRuntimePrepareResult(ctx, env.ID, rejectedRuntimePreparation("resource_unavailable")) } - owner, cancel := context.WithTimeout(context.WithoutCancel(ctx), runtimePreparationTimeout) + owner, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Duration(request.BudgetMS)*time.Millisecond) u := &runtimePreparationTransfer{ - envelope: env, request: request, data: make([]byte, 0, request.SizeBytes), + transferDeadline: time.Now().Add(time.Duration(proto.RuntimePrepareTransferBudgetMS) * time.Millisecond), + envelope: env, request: request, data: make([]byte, 0, request.SizeBytes), ready: make(chan struct{}), cancel: cancel, } r.runtimePreparation = u @@ -100,7 +100,7 @@ func (r *Router) handleRuntimePrepare(ctx context.Context, env proto.Envelope) e return nil } apply := false - if request.Step == "commit" && len(u.data) == u.request.SizeBytes { + if request.Step == "commit" && len(u.data) == u.request.SizeBytes && time.Now().Before(u.transferDeadline) { if u.request.Action == "finalize" || u.request.Action == "initialize" { apply = true } else { @@ -135,7 +135,10 @@ func (r *Router) finishRuntimePreparationTransferLocked(u *runtimePreparationTra func (r *Router) runRuntimePreparationTransfer(ctx context.Context, u *runtimePreparationTransfer, apply func(context.Context, proto.RuntimePreparePayload, []byte) error) { defer r.shutdownWG.Done() defer u.cancel() + staging := time.NewTimer(time.Until(u.transferDeadline)) + defer staging.Stop() select { + case <-staging.C: case <-u.ready: case <-r.shutdownCh: case <-ctx.Done(): diff --git a/apps/daemon/internal/dispatch/runtime_preparation_test.go b/apps/daemon/internal/dispatch/runtime_preparation_test.go index 7a8a31c70..90a0c418e 100644 --- a/apps/daemon/internal/dispatch/runtime_preparation_test.go +++ b/apps/daemon/internal/dispatch/runtime_preparation_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "testing" + "testing/synctest" "time" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent" @@ -57,7 +58,7 @@ func capabilityEnvelope(t *testing.T, id string, request proto.RuntimePreparePay func capabilityBegin(environment, session string, body []byte) proto.RuntimePreparePayload { digest := sha256.Sum256(body) return proto.RuntimePreparePayload{ - Step: "begin", Action: "skill", EnvironmentID: environment, SessionID: session, + BudgetMS: 300000, Step: "begin", Action: "skill", EnvironmentID: environment, SessionID: session, Skill: &agentskill.Metadata{Type: "inline", Name: "proof", Description: "A proof."}, SizeBytes: len(body), SHA256: hex.EncodeToString(digest[:]), } @@ -247,7 +248,7 @@ func TestRuntimePreparationCancellationKeepsOwnershipUntilApplyStops(t *testing. r, sender, environment, session := capabilitiesTestRouter(t) ctx, cancel := context.WithCancel(context.Background()) id := uuid.NewString() - request := proto.RuntimePreparePayload{Step: "begin", Action: "finalize", EnvironmentID: environment, SessionID: session, Sources: &agentcapabilities.Input{}} + request := proto.RuntimePreparePayload{BudgetMS: 300000, Step: "begin", Action: "finalize", EnvironmentID: environment, SessionID: session, Sources: &agentcapabilities.Input{}} owner := &runtimePreparationTransfer{envelope: capabilityEnvelope(t, id, request), request: request, ready: make(chan struct{}), cancel: cancel, finished: true, apply: true} close(owner.ready) r.runtimePreparation = owner @@ -346,3 +347,79 @@ func TestRuntimePreparationResultCategoriesAndUnknownOwnership(t *testing.T) { t.Fatal("shutdown claimed uncertain mutation settled") } } + +func TestRuntimePreparationStagingDeadlineRejectsWithoutApplying(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + r, sender, environment, session := capabilitiesTestRouter(t) + id := uuid.NewString() + request := capabilityBegin(environment, session, []byte("abc")) + if err := r.Handle(t.Context(), capabilityEnvelope(t, id, request)); err != nil { + t.Fatal(err) + } + capabilitiesReceipt(t, sender, id, "ready") + time.Sleep(121 * time.Second) + synctest.Wait() + capabilitiesReceipt(t, sender, id, "rejected") + r.mu.Lock() + pending := r.runtimePreparation != nil + r.mu.Unlock() + if pending { + t.Fatal("uncommitted staging retained ownership") + } + shutdownCapabilitiesRouter(t, r) + }) +} + +func TestRuntimePreparationApplyOutlivesStagingAndHonorsBudget(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + r, sender, environment, session := capabilitiesTestRouter(t) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute) + id := uuid.NewString() + request := capabilityBegin(environment, session, []byte("abc")) + owner := &runtimePreparationTransfer{envelope: capabilityEnvelope(t, id, request), request: request, ready: make(chan struct{}), cancel: cancel, finished: true, apply: true, transferDeadline: time.Now().Add(2 * time.Minute)} + close(owner.ready) + r.runtimePreparation = owner + r.shutdownWG.Add(1) + stopped := make(chan struct{}) + go r.runRuntimePreparationTransfer(ctx, owner, func(ctx context.Context, _ proto.RuntimePreparePayload, _ []byte) error { + <-ctx.Done() + close(stopped) + return ctx.Err() + }) + time.Sleep(121 * time.Second) + synctest.Wait() + select { + case <-stopped: + t.Fatal("staging timeout killed apply") + default: + } + time.Sleep(180 * time.Second) + synctest.Wait() + <-stopped + capabilitiesReceipt(t, sender, id, "unknown") + // Unknown effects remain retained; this test must not claim safe replay. + r.mu.Lock() + unknown := r.runtimePreparation == owner && owner.uncertain + r.mu.Unlock() + if !unknown { + t.Fatal("unknown apply lost ownership") + } + }) +} + +func TestRuntimePreparationHonorsBeginBudgetDuringStaging(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + r, sender, environment, session := capabilitiesTestRouter(t) + id := uuid.NewString() + request := capabilityBegin(environment, session, []byte("abc")) + request.BudgetMS = 10 + if err := r.Handle(t.Context(), capabilityEnvelope(t, id, request)); err != nil { + t.Fatal(err) + } + capabilitiesReceipt(t, sender, id, "ready") + time.Sleep(11 * time.Millisecond) + synctest.Wait() + capabilitiesReceipt(t, sender, id, "rejected") + shutdownCapabilitiesRouter(t, r) + }) +} diff --git a/apps/daemon/internal/localworkspace/runtime_initialization_process.go b/apps/daemon/internal/localworkspace/runtime_initialization_process.go index 68174485d..a8464ed97 100644 --- a/apps/daemon/internal/localworkspace/runtime_initialization_process.go +++ b/apps/daemon/internal/localworkspace/runtime_initialization_process.go @@ -101,15 +101,13 @@ func initializationEnvironment(configured map[string]string) []string { // The shared process owner settles the leader and descendants. Readers finish // before return; output is discarded with constant memory, never put in errors. func runInitializationProcess(ctx context.Context, binary string, args []string, directory string, env []string) error { - operation, cancel := context.WithTimeout(ctx, 30*time.Minute) - defer cancel() - if operation.Err() != nil { + if ctx.Err() != nil { return ErrInitializationUnconfirmed } if info, err := os.Stat(directory); err != nil || !info.IsDir() { return &InitializationFailure{} } - process, err := clirunner.Start(clirunner.StartOptions{Parent: operation, Binary: binary, Args: args, + process, err := clirunner.Start(clirunner.StartOptions{Parent: ctx, Binary: binary, Args: args, Dir: directory, Env: env, KillTimeout: 250 * time.Millisecond}) if err != nil { return ErrInitializationUnconfirmed @@ -127,7 +125,7 @@ func runInitializationProcess(ctx context.Context, binary string, args []string, } first, second := <-finished, <-finished _ = process.Wait() - if operation.Err() != nil || first != nil || second != nil || process.Cmd.ProcessState == nil { + if ctx.Err() != nil || first != nil || second != nil || process.Cmd.ProcessState == nil { return ErrInitializationUnconfirmed } code := process.Cmd.ProcessState.ExitCode() diff --git a/apps/daemon/internal/localworkspace/runtime_initialization_test.go b/apps/daemon/internal/localworkspace/runtime_initialization_test.go index e16e46787..47896582a 100644 --- a/apps/daemon/internal/localworkspace/runtime_initialization_test.go +++ b/apps/daemon/internal/localworkspace/runtime_initialization_test.go @@ -348,7 +348,7 @@ func TestRuntimePreparationRejectsFilesAfterFinalization(t *testing.T) { t.Fatal(err) } digest := sha256.Sum256(nil) - input := proto.RuntimePreparePayload{Step: "begin", EnvironmentID: b.environment, SessionID: b.capabilityIdentity().SessionID, + input := proto.RuntimePreparePayload{BudgetMS: 300000, Step: "begin", EnvironmentID: b.environment, SessionID: b.capabilityIdentity().SessionID, Action: "file", File: &proto.RuntimeInitialFile{Path: "/workspace/file"}, SHA256: hex.EncodeToString(digest[:])} if err = b.ApplyRuntimePreparation(t.Context(), input, nil); !errors.Is(err, agentcapabilities.ErrInvalid) { t.Fatal("finalized Runtime accepted file", err) @@ -361,3 +361,25 @@ func TestRuntimePreparationRejectsFilesAfterFinalization(t *testing.T) { t.Fatal("finalized Runtime accepted initialize", err) } } + +func TestRuntimeInitializationDeadlineStopsDescendants(t *testing.T) { + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + directory := t.TempDir() + ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond) + defer cancel() + env := append(initializationEnvironment(nil), "OAC_INITIALIZATION_FIXTURE=parent", "OAC_INITIALIZATION_FIXTURE_DIR="+directory) + err = runInitializationProcess(ctx, binary, []string{"-test.run=^TestRuntimeInitializationChild$"}, directory, env) + if !errors.Is(err, ErrInitializationUnconfirmed) || !errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatal("deadline outcome", err, ctx.Err()) + } + if _, err := os.Stat(filepath.Join(directory, "started")); err != nil { + t.Fatal("fixture did not start", err) + } + time.Sleep(1100 * time.Millisecond) + if _, err := os.Stat(filepath.Join(directory, "late")); !os.IsNotExist(err) { + t.Fatal("descendant survived deadline") + } +} diff --git a/apps/daemon/internal/localworkspace/snapshot_marker_test.go b/apps/daemon/internal/localworkspace/snapshot_marker_test.go index 7f610e721..90132e1e5 100644 --- a/apps/daemon/internal/localworkspace/snapshot_marker_test.go +++ b/apps/daemon/internal/localworkspace/snapshot_marker_test.go @@ -186,7 +186,7 @@ func TestSnapshotMarkerRefusesDamagedState(t *testing.T) { func TestCapabilityFinalizeRecordsCompletionAndRejectsLaterImports(t *testing.T) { b, _ := markerBinding(t) identity := b.capabilityIdentity() - finalize := proto.RuntimePreparePayload{Step: "begin", EnvironmentID: identity.EnvironmentID, SessionID: identity.SessionID, Action: "finalize", Sources: &agentcapabilities.Input{}} + finalize := proto.RuntimePreparePayload{BudgetMS: 300000, Step: "begin", EnvironmentID: identity.EnvironmentID, SessionID: identity.SessionID, Action: "finalize", Sources: &agentcapabilities.Input{}} if err := b.ApplyRuntimePreparation(t.Context(), finalize, nil); err != nil { t.Fatal(err) } @@ -197,7 +197,7 @@ func TestCapabilityFinalizeRecordsCompletionAndRejectsLaterImports(t *testing.T) if err := os.RemoveAll(b.capabilityRoot); err != nil { t.Fatal(err) } - skill := proto.RuntimePreparePayload{Step: "begin", EnvironmentID: identity.EnvironmentID, SessionID: identity.SessionID, Action: "skill", Skill: &agentskill.Metadata{Type: "inline", Name: "example", Description: "Example"}, SizeBytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} + skill := proto.RuntimePreparePayload{BudgetMS: 300000, Step: "begin", EnvironmentID: identity.EnvironmentID, SessionID: identity.SessionID, Action: "skill", Skill: &agentskill.Metadata{Type: "inline", Name: "example", Description: "Example"}, SizeBytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"} if err := b.ApplyRuntimePreparation(t.Context(), skill, []byte("x")); err == nil { t.Fatal("completed identity accepted import") } diff --git a/contracts/agents-api/environments.md b/contracts/agents-api/environments.md index 099243c5f..79126cbc7 100644 --- a/contracts/agents-api/environments.md +++ b/contracts/agents-api/environments.md @@ -229,7 +229,7 @@ An Environment's initialization is `pending`, `running`, `complete` or `failed`, - The Worker's initialization scheduler scans 32 Environments at a time, wraps at the end and bounds concurrent preparations by execution concurrency, independently of Provider maintenance. - A missing socket does not consume a pending attempt. An unavailable Harness fails before installation. Each operation rechecks current authority and the original socket; completion rechecks the exact binding. -- Each file transfer, configure, Skill, Plugin, package and setup step has a two-minute budget; the whole initialization has 30 minutes. Initial input keeps its five-minute admission deadline, so large installations should start from an idle Session. +- Each file, Skill or Plugin transfer has a two-minute staging budget. Configure, package installation, setup commands and snapshot finalization share the whole initialization’s 30-minute budget, whose remaining duration Core carries on each operation and the Runtime enforces. Initial input keeps its five-minute admission deadline, so large installations should start from an idle Session. - A running initialization whose owner is lost, including across a Core restart, fails as unconfirmed; nothing is replayed. A completed Environment never reinstalls on reconnect or native recovery, so later user changes survive. - Failure is terminal for the Session but destroys neither compute nor files. diff --git a/contracts/agents-api/zh/environments.md b/contracts/agents-api/zh/environments.md index 570681196..ffed2f938 100644 --- a/contracts/agents-api/zh/environments.md +++ b/contracts/agents-api/zh/environments.md @@ -1,7 +1,7 @@ --- title: "环境与模板" source: contracts/agents-api/environments.md -source_hash: 48b9d8679a983adc5eb9164c9a94e1137bf75800fbb1a0f81a0ae1b3cfccdafb +source_hash: 9007be1099cf50f5ba376d2de4a060d039b6ee0b1c13c78a94665c28c2fc3fc6 --- Environment 是 Session 的执行资源，包括 Harness 运行所在的机器、工作区以及已完成准备的能力。Session 通过其 `environment` 配置创建 Environment；不存在独立的 create 调用。Environment Template 是 Session 创建时解析的可复用准备配置。本契约涵盖这两类资源、两种放置方式、输入接纳、能力准备、Skills、Plugins 和 MCP 连接来源。 @@ -231,7 +231,7 @@ Environment 的初始化状态为 `pending`、`running`、`complete` 或 `failed - Worker 的初始化调度器每次扫描 32 个 Environment，在末尾循环回绕，并依据执行并发度限制并发准备，且独立于 Provider 维护。 - 缺少套接字不会消耗一次 pending 尝试。Harness 不可用时，会在安装前失败。每个操作都会重新检查当前权限和原始套接字；完成时还会重新检查精确绑定。 -- 每个文件传输、configure、Skill、Plugin、软件包和设置步骤都有两分钟的预算；整个初始化过程有 30 分钟。初始输入仍保留其五分钟接纳期限，因此大型安装应从空闲 Session 开始。 +- 每个文件、Skill 或 Plugin 传输的暂存预算为两分钟。configure、软件包安装、设置命令和快照最终确定共享整个初始化过程的 30 分钟预算，Core 在每项操作中携带剩余时长，由 Runtime 执行。初始输入仍保留其五分钟接纳期限，因此大型安装应从空闲 Session 开始。 - 正在运行且所有权丧失的初始化，包括跨 Core 重启丧失所有权，会作为未确认而失败；不会重播任何内容。已完成的 Environment 在重连或原生恢复时绝不会重新安装，因此用户后续更改会保留下来。 - 失败对 Session 而言是终止状态，但不会销毁计算资源或文件。 diff --git a/docs/runtime-protocol.md b/docs/runtime-protocol.md index b99b6b668..e7c818fa8 100644 --- a/docs/runtime-protocol.md +++ b/docs/runtime-protocol.md @@ -110,7 +110,7 @@ Usage frames and the final usage snapshot each carry the cumulative measurement ## Preparation and execution order -Environment initialization uses `runtime_prepare` on every connection, managed or user-owned; the [Environment contract](../contracts/agents-api/environments.md#runtime-capability-preparation) owns what is prepared and when. For a file or archive, send `begin`, wait for `ready`, send ordered chunks and await each matching `received` offset, then send `commit` and await `completed`. Initialization and finalization have typed headers without file data. Validate the expected outcome, offset, size and finite error code with the shared validator. One transfer is allowed per connection. A chunk receipt confirms staged bytes, not installation; a completed commit confirms that operation, not that a later Turn ran. +Environment initialization uses `runtime_prepare` on every connection, managed or user-owned; the [Environment contract](../contracts/agents-api/environments.md#runtime-capability-preparation) owns what is prepared and when. For a file or archive, send `begin`, wait for `ready`, send ordered chunks and await each matching `received` offset, then send `commit` and await `completed`. Initialization and finalization have typed headers without file data. Every `begin` requires `budget_ms`, an integer from 1 through 1800000 carrying the remaining Core-owned initialization budget; chunks and commits omit it. The Runtime enforces that relative budget from receipt of `begin`, and Core waits no longer than its original operation deadline. The complete `begin`–`commit` staging phase has a separate two-minute limit; applying a committed operation and waiting for `completed` use the remaining initialization budget, not the staging limit. Expiry before application rejects the transfer; expiry during application reports `unknown` after local mutations stop and never authorizes replay. Validate the expected outcome, offset, size and finite error code with the shared validator. One transfer is allowed per connection. A chunk receipt confirms staged bytes, not installation; a completed commit confirms that operation, not that a later Turn ran. An execution Turn runs in five steps: diff --git a/docs/zh/runtime-protocol.md b/docs/zh/runtime-protocol.md index 92de66cca..ab1b54296 100644 --- a/docs/zh/runtime-protocol.md +++ b/docs/zh/runtime-protocol.md @@ -1,7 +1,7 @@ --- title: "Core–Runtime 协议" source: docs/runtime-protocol.md -source_hash: 59633fbfa54d9071ed6a5ef510328d9216f4d8572a0cc068de65c93ecabefc59 +source_hash: 142a3d2a3dc5b790bcc80dff2364d094857f835e0e0687f91373251a4f4d9b3b --- 此协议在 Runtime daemon 获取机器凭据后连接 Core 与 daemon，定义 daemon 连接上消息的含义和顺序。wire 类型、限制和验证器仅在 [`internal/agentdaemon/proto`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/internal/agentdaemon/proto) 中定义一次；Core 的 [gateway](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/runtimegateway) 与参考 Runtime 的 [dispatcher](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/apps/daemon/internal/dispatch) 都使用它们，因此无需同步第二套 payload schema。签发凭据和打开连接的 HTTP 路由见[机器连接 API](../../contracts/agents-api/zh/machine-api.md)。 @@ -112,7 +112,7 @@ Usage frame 和最终 usage snapshot 都携带当前执行的累计测量，替 ## 准备与执行顺序 {#preparation-and-execution-order} -无论托管还是用户自有环境，每条连接都通过 `runtime_prepare` 初始化 Environment；[Environment 契约](../../contracts/agents-api/zh/environments.md#runtime-capability-preparation)负责准备内容和时机。传输文件或 archive 时，发送 `begin`，等待 `ready`，发送有序 chunk 并等待每个匹配的 `received` offset，再发送 `commit` 并等待 `completed`。初始化和终结阶段使用不含文件数据的类型化 header。使用共享 validator 验证预期结果、offset、size 和有限错误 code。每条连接允许一个 transfer。chunk 回执确认暂存字节，不确认安装；完成的 commit 确认该操作，不证明后续 Turn 已运行。 +无论托管还是用户自有环境，每条连接都通过 `runtime_prepare` 初始化 Environment；[Environment 契约](../../contracts/agents-api/zh/environments.md#runtime-capability-preparation)负责准备内容和时机。传输文件或 archive 时，发送 `begin`，等待 `ready`，发送有序 chunk 并等待每个匹配的 `received` offset，再发送 `commit` 并等待 `completed`。初始化和终结阶段使用不含文件数据的类型化 header。每个 `begin` 必须携带 `budget_ms`，取值为 1 至 1800000 的整数，表示 Core 所拥有的初始化预算剩余毫秒数；chunk 和 commit 不携带该字段。Runtime 从收到 `begin` 起执行此相对预算，Core 的等待不超过原操作期限。完整的 `begin`–`commit` 暂存阶段另有两分钟限制；应用已提交操作及等待 `completed` 使用剩余初始化预算，不受暂存期限限制。应用前到期会拒绝传输；应用中到期则在本地变更停止后报告 `unknown`，绝不授权重放。使用共享 validator 验证预期结果、offset、size 和有限错误 code。每条连接允许一个 transfer。chunk 回执确认暂存字节，不确认安装；完成的 commit 确认该操作，不证明后续 Turn 已运行。 执行 Turn 分为五步： diff --git a/internal/agentdaemon/proto/runtime_prepare.go b/internal/agentdaemon/proto/runtime_prepare.go index bbb2549c6..576cd8be4 100644 --- a/internal/agentdaemon/proto/runtime_prepare.go +++ b/internal/agentdaemon/proto/runtime_prepare.go @@ -14,11 +14,13 @@ import ( ) const ( - TypeRuntimePrepare = "runtime_prepare" - TypeRuntimePrepareResult = "runtime_prepare_result" - RuntimePrepareMaxBytes = 50 << 20 - RuntimePrepareChunkBytes = WorkspaceWriteChunkBytes - RuntimePrepareMaxFrameBytes = 1 << 20 + TypeRuntimePrepare = "runtime_prepare" + TypeRuntimePrepareResult = "runtime_prepare_result" + RuntimePrepareMaxBudgetMS = 30 * 60 * 1000 + RuntimePrepareTransferBudgetMS = 2 * 60 * 1000 + RuntimePrepareMaxBytes = 50 << 20 + RuntimePrepareChunkBytes = WorkspaceWriteChunkBytes + RuntimePrepareMaxFrameBytes = 1 << 20 ) // RuntimeInitialFile addresses a file within the logical workspace. @@ -38,6 +40,7 @@ type RuntimeInitialization struct { // Runtime resolves logical paths and owns installation destinations. Envelope.ID // identifies one connection-local transfer. type RuntimePreparePayload struct { + BudgetMS int64 `json:"budget_ms,omitempty"` Step string `json:"step"` EnvironmentID string `json:"environment_id,omitempty"` SessionID string `json:"session_id,omitempty"` @@ -64,6 +67,9 @@ type RuntimePrepareResultPayload struct { func ValidRuntimePrepareRequest(p RuntimePreparePayload) bool { if p.Step == "begin" { + if p.BudgetMS <= 0 || p.BudgetMS > RuntimePrepareMaxBudgetMS { + return false + } for _, id := range []string{p.EnvironmentID, p.SessionID} { value, err := uuid.Parse(id) if err != nil || value == uuid.Nil || value.String() != id { @@ -119,7 +125,7 @@ func ValidRuntimePrepareRequest(p RuntimePreparePayload) bool { encoded, err := json.Marshal(p) return err == nil && len(encoded) <= RuntimePrepareMaxFrameBytes } - if p.EnvironmentID != "" || p.SessionID != "" || p.Action != "" || p.Slot != 0 || + if p.BudgetMS != 0 || p.EnvironmentID != "" || p.SessionID != "" || p.Action != "" || p.Slot != 0 || p.Skill != nil || p.Plugin != nil || p.Sources != nil || p.File != nil || p.Initialization != nil || p.SizeBytes != 0 || p.SHA256 != "" { return false } diff --git a/internal/agentdaemon/proto/runtime_prepare_test.go b/internal/agentdaemon/proto/runtime_prepare_test.go index 84519b778..8b63322e9 100644 --- a/internal/agentdaemon/proto/runtime_prepare_test.go +++ b/internal/agentdaemon/proto/runtime_prepare_test.go @@ -13,7 +13,7 @@ import ( ) func capabilityBegin() RuntimePreparePayload { - return RuntimePreparePayload{Step: "begin", EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), + return RuntimePreparePayload{BudgetMS: 300000, Step: "begin", EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "skill", Skill: &agentskill.Metadata{Type: "inline", Name: "example", Description: "Example"}, SizeBytes: 10, SHA256: strings.Repeat("a", 64)} } @@ -23,6 +23,9 @@ func TestCapabilitiesRequestValidation(t *testing.T) { t.Fatal("valid skill refused") } for name, mutate := range map[string]func(*RuntimePreparePayload){ + "missing budget": func(p *RuntimePreparePayload) { p.BudgetMS = 0 }, + "negative budget": func(p *RuntimePreparePayload) { p.BudgetMS = -1 }, + "excessive budget": func(p *RuntimePreparePayload) { p.BudgetMS = RuntimePrepareMaxBudgetMS + 1 }, "noncanonical identity": func(p *RuntimePreparePayload) { p.EnvironmentID = "AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA" }, "zero identity": func(p *RuntimePreparePayload) { p.SessionID = uuid.Nil.String() }, "begin data": func(p *RuntimePreparePayload) { p.Data = []byte("x") }, @@ -91,7 +94,7 @@ func TestCapabilitiesRequestValidation(t *testing.T) { } func TestCapabilitiesFinalizeRequiresExplicitBoundedSelection(t *testing.T) { - p := RuntimePreparePayload{Step: "begin", EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "finalize", Sources: &agentcapabilities.Input{}} + p := RuntimePreparePayload{BudgetMS: 300000, Step: "begin", EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "finalize", Sources: &agentcapabilities.Input{}} if !ValidRuntimePrepareRequest(p) { t.Fatal("explicit empty finalization refused") } @@ -173,7 +176,7 @@ func TestLocalEnvironmentCarriesSelectionButNeverInstalledRoots(t *testing.T) { } func TestRuntimePreparationInitialActions(t *testing.T) { - base := RuntimePreparePayload{Step: "begin", EnvironmentID: uuid.NewString(), SessionID: uuid.NewString()} + base := RuntimePreparePayload{BudgetMS: 300000, Step: "begin", EnvironmentID: uuid.NewString(), SessionID: uuid.NewString()} for _, initialization := range []RuntimeInitialization{ {Action: "configure", Env: map[string]string{"EXAMPLE": "value"}}, {Action: "npm", Packages: []string{"typescript"}}, @@ -263,7 +266,7 @@ func TestRuntimePreparationExitCodes(t *testing.T) { } func TestRuntimePreparationPortableSources(t *testing.T) { - p := RuntimePreparePayload{Step: "begin", EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "finalize", Sources: &agentcapabilities.Input{Directories: []string{`C:\Users\operator\skills`, `\\host\share\skills`}}} + p := RuntimePreparePayload{BudgetMS: 300000, Step: "begin", EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "finalize", Sources: &agentcapabilities.Input{Directories: []string{`C:\Users\operator\skills`, `\\host\share\skills`}}} if !ValidRuntimePrepareRequest(p) { t.Fatal("portable sources rejected by wire") } @@ -271,3 +274,18 @@ func TestRuntimePreparationPortableSources(t *testing.T) { t.Fatal("Windows sources accepted by Linux resolver") } } + +func TestRuntimePreparationBudgetOnlyOnBegin(t *testing.T) { + for _, p := range []RuntimePreparePayload{{Step: "commit", BudgetMS: 1}, {Step: "chunk", Data: []byte("a"), BudgetMS: 1}} { + if ValidRuntimePrepareRequest(p) { + t.Fatal("budget admitted outside begin") + } + } + for _, budget := range []int64{1, RuntimePrepareMaxBudgetMS} { + p := capabilityBegin() + p.BudgetMS = budget + if !ValidRuntimePrepareRequest(p) { + t.Fatal("valid budget boundary rejected", budget) + } + } +} diff --git a/internal/agentdaemon/proto/version.go b/internal/agentdaemon/proto/version.go index 7f6f11783..114299f22 100644 --- a/internal/agentdaemon/proto/version.go +++ b/internal/agentdaemon/proto/version.go @@ -2,7 +2,7 @@ package proto // Version identifies the complete Core–Runtime wire contract. Change it when // removing or changing a payload or its semantics; deploy both endpoints together. -const Version = "0.14.0" +const Version = "0.15.0" // VersionCompatible accepts only this contract. Patch drift, prerelease suffixes // and malformed versions do not select an implicit compatibility path. diff --git a/services/core/internal/execution/runtime_capabilities_test.go b/services/core/internal/execution/runtime_capabilities_test.go index c2c29bca1..63319df84 100644 --- a/services/core/internal/execution/runtime_capabilities_test.go +++ b/services/core/internal/execution/runtime_capabilities_test.go @@ -40,7 +40,7 @@ func TestRuntimeCapabilitiesPreserveRawBundlesAndSetupOrdering(t *testing.T) { owner := agentcapabilities.Identity{EnvironmentID: uuid.NewString(), SessionID: uuid.NewString()} for _, op := range operations { peer := &capabilityFixture{outcome: "completed"} - if err := runRuntimeSetup(t.Context(), peer, owner, op); err != nil { + if err := runRuntimeSetup(initializationTestContext(t), peer, owner, op); err != nil { t.Fatal(err) } if _, err := uuid.Parse(peer.requestID); err != nil { @@ -64,14 +64,14 @@ func TestRuntimeCapabilitiesPreserveRawBundlesAndSetupOrdering(t *testing.T) { func TestRuntimeCapabilitiesConfirmedAndUnknownFailures(t *testing.T) { for _, outcome := range []string{"failed", "rejected", "unknown", "unexpected"} { peer := &capabilityFixture{outcome: outcome} - err := runRuntimeSetup(t.Context(), peer, agentcapabilities.Identity{}, runtimeSetupOperation{Request: proto.RuntimePreparePayload{Action: "finalize"}}) + err := runRuntimeSetup(initializationTestContext(t), peer, agentcapabilities.Identity{}, runtimeSetupOperation{Request: proto.RuntimePreparePayload{Action: "finalize"}}) var confirmed *runtimeStepFailure if err == nil || errors.As(err, &confirmed) != (outcome == "failed" || outcome == "rejected") { t.Fatal(outcome, err) } } peer := &capabilityFixture{outcome: "completed", err: errors.New(setupCanary)} - err := runRuntimeSetup(t.Context(), peer, agentcapabilities.Identity{}, runtimeSetupOperation{Request: proto.RuntimePreparePayload{}}) + err := runRuntimeSetup(initializationTestContext(t), peer, agentcapabilities.Identity{}, runtimeSetupOperation{Request: proto.RuntimePreparePayload{}}) if err == nil || bytes.Contains([]byte(err.Error()), []byte(setupCanary)) { t.Fatal("transport error leaked or succeeded", err) } diff --git a/services/core/internal/execution/runtime_initialization.go b/services/core/internal/execution/runtime_initialization.go index 8c2072578..1c958d8ea 100644 --- a/services/core/internal/execution/runtime_initialization.go +++ b/services/core/internal/execution/runtime_initialization.go @@ -8,6 +8,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentcapabilities" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" @@ -91,7 +92,7 @@ func (w *Worker) runEnvironmentInitializations(ctx context.Context) error { } func (w *Worker) initializeEnvironment(ctx context.Context, owner sessions.EnvironmentInitialization) { - operation, cancel := context.WithTimeout(ctx, 30*time.Minute) + operation, cancel := context.WithTimeout(ctx, time.Duration(proto.RuntimePrepareMaxBudgetMS)*time.Millisecond) defer cancel() failure := sessions.ProvisioningFailure{} err := w.prepareEnvironment(operation, owner, &failure) @@ -140,11 +141,10 @@ func (w *Worker) prepareEnvironment(ctx context.Context, owner sessions.Environm identity := agentcapabilities.Identity{EnvironmentID: owner.EnvironmentID, SessionID: owner.SessionID} operations := setupOperations(setup) for index := 0; index < len(cfg.Files)+len(operations); index++ { - step, stop := context.WithTimeout(ctx, 2*time.Minute) - err = w.lease.CheckOwnership(step) + err = w.lease.CheckOwnership(ctx) if err == nil { var currentPeer = peer - currentPeer, err = w.dispatcher.authorizedPeer(step, owner.DeviceID) + currentPeer, err = w.dispatcher.authorizedPeer(ctx, owner.DeviceID) if err == nil && currentPeer != peer { err = errors.New("Runtime connection changed during initialization") } @@ -153,16 +153,15 @@ func (w *Worker) prepareEnvironment(ctx context.Context, owner sessions.Environm if err == nil && index < len(cfg.Files) { var metadata environmentconfig.InitialFileMetadata var body []byte - metadata, body, err = w.dispatcher.SessionsReader.ReadInitialEnvironmentFile(step, owner.TenantID, owner.SessionID, index) + metadata, body, err = w.dispatcher.SessionsReader.ReadInitialEnvironmentFile(ctx, owner.TenantID, owner.SessionID, index) if err == nil { - err = installInitialFile(step, peer, identity, metadata, body) + err = installInitialFile(ctx, peer, identity, metadata, body) } } else if err == nil { command := operations[index-len(cfg.Files)] candidate = command.provisioningFailure(0) - err = runRuntimeSetup(step, peer, identity, command) + err = runRuntimeSetup(ctx, peer, identity, command) } - stop() if err != nil { var confirmed *runtimeStepFailure if errors.As(err, &confirmed) { diff --git a/services/core/internal/execution/runtime_setup.go b/services/core/internal/execution/runtime_setup.go index ccfcb795f..1f1822177 100644 --- a/services/core/internal/execution/runtime_setup.go +++ b/services/core/internal/execution/runtime_setup.go @@ -3,6 +3,7 @@ package execution import ( "context" "errors" + "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentcapabilities" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" @@ -85,7 +86,13 @@ func runRuntimeSetup(ctx context.Context, peer runtimePreparer, identity agentca if peer == nil { return errors.New("environment initialization request unavailable") } + deadline, bounded := ctx.Deadline() + budget := time.Until(deadline).Milliseconds() + if !bounded || budget <= 0 || budget > proto.RuntimePrepareMaxBudgetMS || ctx.Err() != nil { + return errors.New("environment initialization budget unavailable") + } request := operation.Request + request.BudgetMS = budget request.EnvironmentID, request.SessionID = identity.EnvironmentID, identity.SessionID result, err := peer.PrepareRuntime(ctx, uuid.NewString(), request, operation.Data) if err == nil && result.Outcome == "completed" { diff --git a/services/core/internal/execution/runtime_setup_test.go b/services/core/internal/execution/runtime_setup_test.go index be2a99461..2bd9be1bb 100644 --- a/services/core/internal/execution/runtime_setup_test.go +++ b/services/core/internal/execution/runtime_setup_test.go @@ -5,6 +5,7 @@ import ( "errors" "strings" "testing" + "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentcapabilities" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" @@ -36,7 +37,7 @@ func TestRuntimeSetupReceiptOutcomes(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { peer := &receiptRuntime{result: proto.RuntimePrepareResultPayload{Outcome: test.outcome, ExitCode: test.code}, err: test.err} - err := runRuntimeSetup(t.Context(), peer, agentcapabilities.Identity{}, runtimeSetupOperation{Request: proto.RuntimePreparePayload{Action: "initialize", Initialization: &proto.RuntimeInitialization{Action: "setup", Command: setupCanary}}}) + err := runRuntimeSetup(initializationTestContext(t), peer, agentcapabilities.Identity{}, runtimeSetupOperation{Request: proto.RuntimePreparePayload{Action: "initialize", Initialization: &proto.RuntimeInitialization{Action: "setup", Command: setupCanary}}}) if test.outcome == "completed" && test.err == nil { if err != nil { t.Fatal(err) @@ -73,14 +74,47 @@ func TestInitialFileUsesTypedRuntimeBytes(t *testing.T) { size := int64(len(body)) owner := agentcapabilities.Identity{EnvironmentID: "environment", SessionID: "session"} peer := &receiptRuntime{result: proto.RuntimePrepareResultPayload{Outcome: "completed"}} - if err := installInitialFile(t.Context(), peer, owner, environmentconfig.InitialFileMetadata{Path: "/workspace/a", SizeBytes: &size}, body); err != nil { + if err := installInitialFile(initializationTestContext(t), peer, owner, environmentconfig.InitialFileMetadata{Path: "/workspace/a", SizeBytes: &size}, body); err != nil { t.Fatal(err) } if peer.request.Action != "file" || peer.request.File.Path != "/workspace/a" || peer.request.EnvironmentID != owner.EnvironmentID || peer.request.SessionID != owner.SessionID || string(peer.data) != setupCanary { t.Fatal("file transport changed") } size++ - if err := installInitialFile(t.Context(), peer, owner, environmentconfig.InitialFileMetadata{SizeBytes: &size}, body); err == nil { + if err := installInitialFile(initializationTestContext(t), peer, owner, environmentconfig.InitialFileMetadata{SizeBytes: &size}, body); err == nil { t.Fatal("mismatched source size accepted") } } + +func initializationTestContext(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute) + t.Cleanup(cancel) + return ctx +} + +func TestRuntimeSetupUsesRemainingInitializationBudget(t *testing.T) { + ctx := initializationTestContext(t) + deadline, _ := ctx.Deadline() + peer := &receiptRuntime{result: proto.RuntimePrepareResultPayload{Outcome: "completed"}} + err := runRuntimeSetup(ctx, peer, agentcapabilities.Identity{}, runtimeSetupOperation{}) + if err != nil { + t.Fatal(err) + } + remaining := time.Until(deadline).Milliseconds() + if peer.request.BudgetMS < remaining || peer.request.BudgetMS > remaining+1000 || peer.request.BudgetMS <= 120000 { + t.Fatalf("remaining operation budget lost: %d vs %d", peer.request.BudgetMS, remaining) + } + for name, c := range map[string]context.Context{"unbounded": t.Context(), "expired": func() context.Context { + c, stop := context.WithDeadline(t.Context(), time.Now().Add(-time.Second)) + stop() + return c + }()} { + t.Run(name, func(t *testing.T) { + peer := &receiptRuntime{} + if runRuntimeSetup(c, peer, agentcapabilities.Identity{}, runtimeSetupOperation{}) == nil || peer.request.BudgetMS != 0 { + t.Fatal("invalid budget sent") + } + }) + } +} diff --git a/services/core/internal/runtimegateway/runtime_prepare.go b/services/core/internal/runtimegateway/runtime_prepare.go index 77e6493f9..677928f6a 100644 --- a/services/core/internal/runtimegateway/runtime_prepare.go +++ b/services/core/internal/runtimegateway/runtime_prepare.go @@ -50,8 +50,10 @@ func (s *Session) PrepareRuntime(ctx context.Context, id string, request proto.R s.capabilities = map[string]chan proto.Envelope{id: replies} s.capabilitiesMu.Unlock() defer func() { s.capabilitiesMu.Lock(); delete(s.capabilities, id); s.capabilitiesMu.Unlock() }() - ctx, cancel := context.WithTimeout(ctx, 195*time.Second) + ctx, cancel := context.WithTimeout(ctx, time.Duration(request.BudgetMS)*time.Millisecond) defer cancel() + transfer, stopTransfer := context.WithTimeout(ctx, time.Duration(proto.RuntimePrepareTransferBudgetMS)*time.Millisecond) + defer stopTransfer() exchange := func(payload proto.RuntimePreparePayload, outcome string, offset int) (proto.RuntimePrepareResultPayload, error) { env, err := proto.NewEnvelope(proto.TypeRuntimePrepare, id, payload) if err != nil { @@ -61,7 +63,13 @@ func (s *Session) PrepareRuntime(ctx context.Context, id string, request proto.R if err != nil || len(encoded) > proto.RuntimePrepareMaxFrameBytes { return unknown, errors.New("agentdaemon gateway: invalid Runtime frame") } - reply, err := s.exchangeChunkFrame(ctx, env, replies) + operation := ctx + if outcome != "completed" { + operation = transfer + } else if err := transfer.Err(); err != nil { + return unknown, err + } + reply, err := s.exchangeChunkFrame(operation, env, replies) if err != nil { return unknown, err } diff --git a/services/core/internal/runtimegateway/runtime_prepare_test.go b/services/core/internal/runtimegateway/runtime_prepare_test.go index 02834d87a..f0beea586 100644 --- a/services/core/internal/runtimegateway/runtime_prepare_test.go +++ b/services/core/internal/runtimegateway/runtime_prepare_test.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "testing" + "testing/synctest" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentcapabilities" @@ -18,7 +19,7 @@ import ( ) func skillPreparation() proto.RuntimePreparePayload { - return proto.RuntimePreparePayload{EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "skill", + return proto.RuntimePreparePayload{BudgetMS: 300000, EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "skill", Skill: &agentskill.Metadata{Type: "inline", Name: "example", Description: "Example"}} } @@ -116,7 +117,7 @@ func TestCapabilitiesTransfersMoreThanFrameLimitAndCorrelates(t *testing.T) { func TestCapabilitiesFinalizeTransfersNoArchive(t *testing.T) { s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) defer s.Close("test") - request := proto.RuntimePreparePayload{EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "finalize", Sources: &agentcapabilities.Input{}} + request := proto.RuntimePreparePayload{BudgetMS: 300000, EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "finalize", Sources: &agentcapabilities.Input{}} id := uuid.NewString() done := beginCapabilities(s, t.Context(), id, request, nil) env := nextCapabilityFrame(t, s) @@ -239,7 +240,7 @@ func TestRuntimeInitialFileChunking(t *testing.T) { t.Run(fmt.Sprint(size), func(t *testing.T) { s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) defer s.Close("test") - request := proto.RuntimePreparePayload{EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "file", File: &proto.RuntimeInitialFile{Path: "/workspace/project/file"}} + request := proto.RuntimePreparePayload{BudgetMS: 300000, EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "file", File: &proto.RuntimeInitialFile{Path: "/workspace/project/file"}} data := bytes.Repeat([]byte("z"), size) id := uuid.NewString() done := beginCapabilities(s, t.Context(), id, request, data) @@ -289,7 +290,7 @@ func TestRuntimeInitializationNoDataAndExitReceipt(t *testing.T) { t.Run(fmt.Sprint(exit), func(t *testing.T) { s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) defer s.Close("test") - request := proto.RuntimePreparePayload{EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "initialize", Initialization: &proto.RuntimeInitialization{Action: "setup", Command: "echo test"}} + request := proto.RuntimePreparePayload{BudgetMS: 300000, EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "initialize", Initialization: &proto.RuntimeInitialization{Action: "setup", Command: "echo test"}} if _, err := s.PrepareRuntime(t.Context(), uuid.NewString(), request, []byte("forbidden")); err == nil { t.Fatal("initialization body accepted") } @@ -327,3 +328,72 @@ func TestRuntimeInitializationNoDataAndExitReceipt(t *testing.T) { }) } } + +func TestRuntimePreparationSeparatesTransferAndApplyBudgets(t *testing.T) { + for _, commit := range []bool{false, true} { + t.Run(fmt.Sprint(commit), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) + defer s.Close("test") + request := proto.RuntimePreparePayload{BudgetMS: 300000, EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "initialize", Initialization: &proto.RuntimeInitialization{Action: "configure"}} + id := uuid.NewString() + done := beginCapabilities(s, t.Context(), id, request, nil) + frame := nextCapabilityFrame(t, s) + var sent proto.RuntimePreparePayload + if frame.DecodePayload(&sent) != nil || sent.BudgetMS != request.BudgetMS { + t.Fatal("begin budget lost") + } + if commit { + replyCapabilities(s, id, proto.RuntimePrepareResultPayload{Outcome: "ready"}) + nextCapabilityFrame(t, s) + } + time.Sleep(121 * time.Second) + synctest.Wait() + if !commit { + result := finishCapabilities(t, done) + if !errors.Is(result.err, context.DeadlineExceeded) || result.result.Outcome != "unknown" { + t.Fatal(result) + } + return + } + select { + case result := <-done: + t.Fatal("transfer timer clamped execution", result) + default: + } + time.Sleep(180 * time.Second) + result := finishCapabilities(t, done) + if !errors.Is(result.err, context.DeadlineExceeded) || result.result.Outcome != "unknown" { + t.Fatal(result) + } + }) + }) + } +} + +func TestRuntimePreparationCommittedApplyStopsWaitingOnDisconnectOrCallerDeadline(t *testing.T) { + for _, disconnect := range []bool{false, true} { + t.Run(fmt.Sprint(disconnect), func(t *testing.T) { + s := NewSession(newFakeConn(), "device", "tenant", "test", nil, nil) + defer s.Close("test") + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + request := proto.RuntimePreparePayload{BudgetMS: 300000, EnvironmentID: uuid.NewString(), SessionID: uuid.NewString(), Action: "initialize", Initialization: &proto.RuntimeInitialization{Action: "configure"}} + id := uuid.NewString() + done := beginCapabilities(s, ctx, id, request, nil) + nextCapabilityFrame(t, s) + replyCapabilities(s, id, proto.RuntimePrepareResultPayload{Outcome: "ready"}) + nextCapabilityFrame(t, s) + expected := error(context.DeadlineExceeded) + if disconnect { + s.Close("disconnect during apply") + expected = ErrSessionClosed + } + result := finishCapabilities(t, done) + if !errors.Is(result.err, expected) || result.result.Outcome != "unknown" { + t.Fatal(result) + } + noCapabilityFrame(t, s) + }) + } +} From a4420261577d9c7cb8bbe1e82a992f18b047543b Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 01:22:43 +0800 Subject: [PATCH 13/22] Preserve Runtime PATH in login shells (#649) * Preserve Runtime PATH in login shells * Keep PATH literals aligned in translated image guide --- Makefile | 1 + docs/maintainers.md | 4 +- docs/zh/maintainers.md | 6 ++- scripts/build-claude-runtime.sh | 1 + scripts/build-codex-runtime.sh | 1 + scripts/build-mcode-runtime.sh | 1 + services/core/deploy/claude/Dockerfile | 2 + services/core/deploy/codex/Dockerfile | 2 + services/core/deploy/mcode/Dockerfile | 2 + services/core/deploy/runtime_profile.py | 27 ++++++++++++ services/core/deploy/runtime_profile_test.py | 44 ++++++++++++++++++++ 11 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 services/core/deploy/runtime_profile.py create mode 100644 services/core/deploy/runtime_profile_test.py diff --git a/Makefile b/Makefile index 5a93f513b..888e671c2 100644 --- a/Makefile +++ b/Makefile @@ -183,6 +183,7 @@ check-microsandbox-provider: .PHONY: check-distribution build-core-distribution check-distribution: + PYTHONDONTWRITEBYTECODE=1 python3 services/core/deploy/runtime_profile_test.py node --test scripts/build-native-catalog.test.mjs go test ./services/web ./services/core/cmd/oac -count=1 PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s deploy/node -p 'test_*.py' diff --git a/docs/maintainers.md b/docs/maintainers.md index a410ad2d1..d0154a883 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -59,6 +59,8 @@ The catalog records the commit, the Runtime protocol version, each archive's SHA `make build-core-distribution` builds all of these. Build one on its own to test a Harness image or a helper. Run every command from the repository root; default outputs go under `${OAC_DEV_HOME:-$HOME/.oac}/build`. +The three maintained Runtime images share a build-time adjustment to the pinned Debian login profile: an inherited, exported `PATH` is preserved, while an unset `PATH` receives Debian’s defaults. This keeps the Runtime’s initialized package and user paths available in login and non-login tools without another package-path setting. The combined image inherits the same profile. A changed upstream profile fails the build for review; custom shell startup files can still explicitly change `PATH`. + **Codex Runtime image.** Extract the official npm package `@openai/codex@0.153.4-linux-x64` under `~/.oac` (for example with `npm pack --ignore-scripts` and `tar -xzf`), then: ```sh @@ -67,7 +69,7 @@ make build-codex-runtime docker build --platform linux/amd64 -t oac-runtime:codex "${OAC_DEV_HOME:-$HOME/.oac}/build/codex-runtime" ``` -The script checks the package version, builds `oac-daemon` for Linux amd64 and prepares a context with only the daemon, the unmodified native executable, its resources and `services/core/deploy/codex/Dockerfile`. +The script checks the package version, builds `oac-daemon` for Linux amd64 and prepares a context with only the daemon, the unmodified native executable, its resources, the shared login-profile build step and `services/core/deploy/codex/Dockerfile`. **Claude Code Runtime image.** Node 20 or newer and pnpm are required. diff --git a/docs/zh/maintainers.md b/docs/zh/maintainers.md index 924c3b19f..e61ec78da 100644 --- a/docs/zh/maintainers.md +++ b/docs/zh/maintainers.md @@ -1,7 +1,7 @@ --- title: "构建并发布 OpenAgentCore" source: docs/maintainers.md -source_hash: aaadeb3a5e99b8d926e9f78b7fc41c7aba808e0d2af58969119e67954f9d7a58 +source_hash: 494618f4d605d96bfcb9b1413f8224722324e1037764cc5b38baca102a8f1e90 --- 本指南面向负责构建和发布 OpenAgentCore 的维护者。要安装 Core 和 Web，请使用 [安装指南](getting-started/install.md)。安装器代码遵循的规则见 [部署](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/README.md) 和 [节点安装器](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/node/README.md)；必需检查见 [CONTRIBUTING](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/CONTRIBUTING.md#required-checks)。 @@ -61,6 +61,8 @@ export OAC_NATIVE_INSTALLER_BUILD_DIR=OUTPUT_DIR `make build-core-distribution` 会构建以下全部内容。也可以单独构建其中一项，以测试某个 Harness 镜像或辅助程序。所有命令都必须从仓库根目录运行；默认输出位于 `${OAC_DEV_HOME:-$HOME/.oac}/build` 下。 +三个维护的 Runtime 镜像共用一个构建期调整：固定 Debian 登录 profile 会保留继承且已导出的 `PATH`，未设置 `PATH` 时仍使用 Debian 默认值。这样，登录和非登录工具都能使用 Runtime 初始化的包路径与用户路径，无需另一份包路径设置。组合镜像继承同一 profile。上游 profile 结构变化会使构建失败以便审查；自定义 shell 启动文件仍可以显式更改 `PATH`。 + **Codex Runtime 镜像。** 在 `~/.oac` 下解压官方 npm 包 `@openai/codex@0.153.4-linux-x64`（例如使用 `npm pack --ignore-scripts` 和 `tar -xzf`），然后执行： ```sh @@ -69,7 +71,7 @@ make build-codex-runtime docker build --platform linux/amd64 -t oac-runtime:codex "${OAC_DEV_HOME:-$HOME/.oac}/build/codex-runtime" ``` -该脚本会检查软件包版本，为 Linux amd64 构建 `oac-daemon`，并准备一个仅包含守护进程、未修改的原生可执行文件、相关资源和 `services/core/deploy/codex/Dockerfile` 的上下文。 +该脚本会检查软件包版本，为 Linux amd64 构建 `oac-daemon`，并准备一个仅包含守护进程、未修改的原生可执行文件、相关资源、共用的登录 profile 构建步骤和 `services/core/deploy/codex/Dockerfile` 的上下文。 **Claude Code Runtime 镜像。** 必须使用 Node 20 或更高版本以及 pnpm。 diff --git a/scripts/build-claude-runtime.sh b/scripts/build-claude-runtime.sh index af5b3b587..d5b9c60a8 100755 --- a/scripts/build-claude-runtime.sh +++ b/scripts/build-claude-runtime.sh @@ -22,6 +22,7 @@ node "$repo_root/scripts/check-claude-sdk-runtime.mjs" "$context/claude-sdk" -o "$context/oac-daemon" ./apps/daemon/cmd/oac-daemon ) cp "$repo_root/services/core/deploy/claude/Dockerfile" "$context/Dockerfile" +cp "$repo_root/services/core/deploy/runtime_profile.py" "$context/runtime_profile.py" mkdir -p "$output_dir" cp -R "$context/." "$output_dir/" printf 'Claude Runtime image context: %s\n' "$output_dir" diff --git a/scripts/build-codex-runtime.sh b/scripts/build-codex-runtime.sh index fcae07a81..8269cd07c 100755 --- a/scripts/build-codex-runtime.sh +++ b/scripts/build-codex-runtime.sh @@ -33,6 +33,7 @@ trap 'rm -rf "$context"' EXIT cp "$native_dir/bin/codex" "$context/codex" cp -R "$native_dir/codex-resources" "$context/codex-resources" cp "$repo_root/services/core/deploy/codex/Dockerfile" "$context/Dockerfile" +cp "$repo_root/services/core/deploy/runtime_profile.py" "$context/runtime_profile.py" # Preserve the previous bundle if compilation or validation failed. mkdir -p "$output_dir" cp -R "$context/." "$output_dir/" diff --git a/scripts/build-mcode-runtime.sh b/scripts/build-mcode-runtime.sh index 04a507698..8698a2212 100644 --- a/scripts/build-mcode-runtime.sh +++ b/scripts/build-mcode-runtime.sh @@ -23,6 +23,7 @@ cp -RL "$companion/." "$context/mcode-harness/" -o "$context/oac-daemon" ./apps/daemon/cmd/oac-daemon ) cp "$repo_root/services/core/deploy/mcode/Dockerfile" "$context/Dockerfile" +cp "$repo_root/services/core/deploy/runtime_profile.py" "$context/runtime_profile.py" mkdir -p "$output" cp -R "$context/." "$output/" printf 'MiniMax Code Runtime image context: %s\n' "$output" diff --git a/services/core/deploy/claude/Dockerfile b/services/core/deploy/claude/Dockerfile index b4ce63d03..9862c8582 100644 --- a/services/core/deploy/claude/Dockerfile +++ b/services/core/deploy/claude/Dockerfile @@ -5,6 +5,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates bash git python3 python3-pip ripgrep \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p /environment/workspace /workspace /home/runtime +COPY runtime_profile.py /tmp/oac-runtime-profile.py +RUN python3 /tmp/oac-runtime-profile.py && rm /tmp/oac-runtime-profile.py COPY --chmod=0555 oac-daemon /usr/local/bin/ COPY claude-sdk /opt/claude-sdk ENV HOME=/home/runtime OAC_RUNTIME_HOME=/home/runtime/.oac \ diff --git a/services/core/deploy/codex/Dockerfile b/services/core/deploy/codex/Dockerfile index 4dc401212..53887748d 100644 --- a/services/core/deploy/codex/Dockerfile +++ b/services/core/deploy/codex/Dockerfile @@ -8,6 +8,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates bash git python3 python3-pip ripgrep \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p /environment/workspace /workspace /home/runtime +COPY runtime_profile.py /tmp/oac-runtime-profile.py +RUN python3 /tmp/oac-runtime-profile.py && rm /tmp/oac-runtime-profile.py COPY --chmod=0555 oac-daemon codex /usr/local/bin/ COPY codex-resources /usr/local/codex-resources ENV HOME=/home/runtime OAC_RUNTIME_HOME=/home/runtime/.oac \ diff --git a/services/core/deploy/mcode/Dockerfile b/services/core/deploy/mcode/Dockerfile index 15aa27e74..aa43dbbb4 100644 --- a/services/core/deploy/mcode/Dockerfile +++ b/services/core/deploy/mcode/Dockerfile @@ -5,6 +5,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates bash git python3 python3-pip ripgrep \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p /environment/workspace /workspace /home/runtime +COPY runtime_profile.py /tmp/oac-runtime-profile.py +RUN python3 /tmp/oac-runtime-profile.py && rm /tmp/oac-runtime-profile.py COPY --chmod=0555 oac-daemon /usr/local/bin/ COPY mcode-harness /opt/mcode-harness ENV HOME=/home/runtime OAC_RUNTIME_HOME=/home/runtime/.oac \ diff --git a/services/core/deploy/runtime_profile.py b/services/core/deploy/runtime_profile.py new file mode 100644 index 000000000..cd4558dd7 --- /dev/null +++ b/services/core/deploy/runtime_profile.py @@ -0,0 +1,27 @@ +"""Preserve the Runtime's exported PATH in the pinned Debian login profile.""" +from pathlib import Path + +DEBIAN_PATH = '''if [ "$(id -u)" -eq 0 ]; then + PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +else + PATH="/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games" +fi +export PATH +''' + + +def preserve_runtime_path(profile: str) -> str: + if profile.count(DEBIAN_PATH) != 1: + raise ValueError("Pinned Debian profile PATH block changed") + # Bash creates an unexported PATH when its caller supplied none. Check the + # exported environment so that case still receives Debian's UID defaults. + replacement = ('# Preserve the Runtime tool environment across login shells.\n' + 'if ! /usr/bin/printenv PATH >/dev/null; then\n' + + ''.join(' ' + line for line in DEBIAN_PATH.splitlines(keepends=True)) + + 'fi\n') + return profile.replace(DEBIAN_PATH, replacement) + + +if __name__ == '__main__': + path = Path('/etc/profile') + path.write_text(preserve_runtime_path(path.read_text())) diff --git a/services/core/deploy/runtime_profile_test.py b/services/core/deploy/runtime_profile_test.py new file mode 100644 index 000000000..079f5936d --- /dev/null +++ b/services/core/deploy/runtime_profile_test.py @@ -0,0 +1,44 @@ +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +from runtime_profile import DEBIAN_PATH, preserve_runtime_path + + +class RuntimeProfileTest(unittest.TestCase): + def test_rejects_changed_or_duplicate_upstream_block(self): + for value in ('', DEBIAN_PATH.replace('/usr/games', '/changed'), DEBIAN_PATH * 2, + preserve_runtime_path(DEBIAN_PATH)): + with self.subTest(profile=value): + self.assertRaises(ValueError, preserve_runtime_path, value) + + def test_preserves_other_profile_content(self): + result = preserve_runtime_path('# before\n' + DEBIAN_PATH + '# after\n') + self.assertTrue(result.startswith('# before\n')) + self.assertTrue(result.endswith('# after\n')) + + @unittest.skipUnless(os.name == 'posix' and Path('/bin/bash').exists(), 'Bash profile') + def test_exported_unset_and_empty_path(self): + with tempfile.TemporaryDirectory() as directory: + original, fixed = Path(directory) / 'original', Path(directory) / 'fixed' + original.write_text(DEBIAN_PATH) + fixed.write_text(preserve_runtime_path(DEBIAN_PATH)) + def run(profile, path): + env = dict(os.environ) + env.pop('BASH_ENV', None) + if path is None: + env.pop('PATH', None) + else: + env['PATH'] = path + return subprocess.check_output(['/bin/bash', '--noprofile', '--norc', '-c', + '. "$1"; printf %s "$PATH"', 'profile-test', str(profile)], env=env, text=True) + custom = '/custom package/bin:/usr/bin:/bin' + self.assertEqual(run(fixed, custom), custom) + self.assertEqual(run(fixed, ''), '') + self.assertEqual(run(fixed, None), run(original, None)) + + +if __name__ == '__main__': + unittest.main() From 9c85f25d24c01fb15c80e40005d907a03bb4d266 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 01:26:00 +0800 Subject: [PATCH 14/22] Retain compute transition failure observations (#650) --- .../internal/execution/runtime_compute.go | 20 ++-- .../execution/runtime_compute_wake.go | 7 +- .../internal/execution/runtime_observation.go | 25 +++++ .../execution/runtime_observation_test.go | 93 +++++++++++++++++++ 4 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 services/core/internal/execution/runtime_observation_test.go diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 9512d6765..f558ef275 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -45,7 +45,8 @@ func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner deployment.All } return r.deployment.SetCompute(ctx, owner, phase, raw, until, idleTimeout) } -func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner deployment.Allocation) error { +func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner deployment.Allocation) (err error) { + defer func() { err = withObservationOwner(owner, err) }() p := r.config.Provider if err := providercontract.Require(p, "Initial"); err != nil { return err @@ -65,7 +66,8 @@ func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner deployment.A return err } -func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment.Allocation) error { +func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment.Allocation) (err error) { + defer func() { err = withObservationOwner(owner, err) }() p := r.config.Provider if err := providercontract.Require(p, "Initial"); err != nil { return err @@ -105,7 +107,8 @@ func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment. } } -func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) error { +func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) (err error) { + defer func() { err = withObservationOwner(owner, err) }() compute, err := p.GetCompute(ctx, runtimeReference(owner), state.Current) if err != nil { return err @@ -141,6 +144,7 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro if err != nil { return err } + owner = next result, err := peer.SuspendControl(ctx, proto.TypeEnvironmentQuiesce, proto.EnvironmentSuspendPayload{EnvironmentID: owner.EnvironmentID, SuspendID: state.SuspendID}) if err != nil { return err @@ -175,7 +179,8 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro return r.captureCompute(ctx, p, suspending, state, false) } -func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) error { +func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) (err error) { + defer func() { err = withObservationOwner(owner, err) }() result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Snapshot: state.Snapshot, ObserveOnly: observeOnly}) if err != nil { return err @@ -201,6 +206,7 @@ func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.Sandbox if err != nil { return err } + owner = next if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { return err } @@ -208,7 +214,8 @@ func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.Sandbox return err } -func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) error { +func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) (err error) { + defer func() { err = withObservationOwner(owner, err) }() activity, err := r.reader.Activity(ctx, owner.ID) if err != nil { return err @@ -231,7 +238,8 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San } return r.restoreCompute(ctx, p, next, state, false) } -func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) error { +func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) (err error) { + defer func() { err = withObservationOwner(owner, err) }() if state.Target == nil || state.Snapshot == nil || state.Rollback { return sandbox.ErrOwnership } diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index 315b08c37..2aab604f6 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -13,7 +13,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) error { +func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) (err error) { + defer func() { err = withObservationOwner(owner, err) }() if state.Rollback { if _, err := p.ResumeCompute(ctx, runtimeReference(owner), state.Current); err != nil { return err @@ -65,7 +66,9 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxPro if err != nil { return err } - if err := r.deployment.ClearWake(ctx, next, owner.ComputeActivityAt); err != nil { + observedActivity := owner.ComputeActivityAt + owner = next + if err := r.deployment.ClearWake(ctx, next, observedActivity); err != nil { return err } return r.observeConnection(ctx, next) diff --git a/services/core/internal/execution/runtime_observation.go b/services/core/internal/execution/runtime_observation.go index e4d93e3d0..bc8cf2fab 100644 --- a/services/core/internal/execution/runtime_observation.go +++ b/services/core/internal/execution/runtime_observation.go @@ -8,7 +8,32 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) +// observationFailure retains the receipt held by the operation that failed. +// A later phase must never be queried and relabelled with an earlier failure. +type observationFailure struct { + owner deployment.Allocation + cause error +} + +func (e *observationFailure) Error() string { return e.cause.Error() } +func (e *observationFailure) Unwrap() error { return e.cause } + +func withObservationOwner(owner deployment.Allocation, err error) error { + if err == nil { + return nil + } + var observed *observationFailure + if errors.As(err, &observed) { + return err + } + return &observationFailure{owner: owner, cause: err} +} + func (r *runtimeLifecycle) recordObservation(ctx context.Context, owner deployment.Allocation, observed error) { + var failure *observationFailure + if errors.As(observed, &failure) { + owner = failure.owner + } if owner.NodeID == "" { return } diff --git a/services/core/internal/execution/runtime_observation_test.go b/services/core/internal/execution/runtime_observation_test.go new file mode 100644 index 000000000..8ad10bdae --- /dev/null +++ b/services/core/internal/execution/runtime_observation_test.go @@ -0,0 +1,93 @@ +package execution + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +type observationCaptureProvider struct { + retentionProvider + captureError error +} + +func (p *observationCaptureProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + if p.captureError != nil { + return sandbox.ComputeState{}, p.captureError + } + return sandbox.ComputeState{Compute: q.Source, Snapshot: &sandbox.SnapshotIdentity{ID: "captured"}, Status: "suspended", SourceStopped: true}, nil +} + +func TestComputeFailureObservationUsesCommittedReceipt(t *testing.T) { + for _, stage := range []string{"capture", "source_cleanup"} { + t.Run(stage, func(t *testing.T) { + provider := &observationCaptureProvider{} + if stage == "capture" { + provider.captureError = sandbox.ErrComputeUnconfirmed + } else { + provider.killError = sandbox.ErrComputeUnconfirmed + } + fixture := workspaceSettlementFixtureWithSetup(t, provider, workspaceNodeSetup(t, true)) + r, session := fixture.lifecycle, fixture.session + initial, err := r.provision(t.Context(), session.TenantID, session.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + // Seed the acknowledged idle barrier; subsequent phase commits use the real + // Session-locked lifecycle operations, with real observation CAS writes. + if _, err = fixture.pool.Exec(t.Context(), `UPDATE environments SET initialization='complete',status='disconnected' WHERE id=$1`, initial.EnvironmentID); err != nil { + t.Fatal(err) + } + if _, err = fixture.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='quiescing',compute_revision=2,compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, initial.ID); err != nil { + t.Fatal(err) + } + original, err := r.reader.EnvironmentAllocation(t.Context(), initial.Key()) + if err != nil { + t.Fatal(err) + } + state := runtimeCompute{Current: sandbox.Compute{ID: "source", Name: "source"}, SuspendID: "attempt"} + until := time.Now().Add(time.Hour) + suspending, err := r.saveCompute(t.Context(), original, "suspending", state, &until) + if err != nil { + t.Fatal(err) + } + failure := r.observeCompute(t.Context(), suspending) + if !errors.Is(failure, sandbox.ErrComputeUnconfirmed) { + t.Fatalf("capture failure = %v", failure) + } + // Model the outer reconcile entry retaining its earlier scan receipt. + r.recordObservation(t.Context(), original, failure) + current, err := r.reader.EnvironmentAllocation(t.Context(), initial.Key()) + if err != nil || current.ObservationError != "compute_unconfirmed" { + t.Fatalf("observation after transition = %q, %v", current.ObservationError, err) + } + wantRevision := int64(3) + if stage == "source_cleanup" { + wantRevision++ + } + if current.ComputeRevision != wantRevision { + t.Fatalf("revision = %d, want %d", current.ComputeRevision, wantRevision) + } + // A late result must not borrow a later operation's receipt or erase its + // diagnostic, even when both observations belong to the same allocation. + later, err := r.saveCompute(t.Context(), current, "waking", state, &until) + if err != nil { + t.Fatal(err) + } + r.recordObservation(t.Context(), later, sandbox.ErrOwnership) + r.recordObservation(t.Context(), original, failure) + current, err = r.reader.EnvironmentAllocation(t.Context(), initial.Key()) + if err != nil || current.ObservationError != "ownership_mismatch" { + t.Fatalf("late failure overwrote successor: %q, %v", current.ObservationError, err) + } + r.recordObservation(t.Context(), current, nil) + current, err = r.reader.EnvironmentAllocation(t.Context(), initial.Key()) + if err != nil || current.ObservationError != "" { + t.Fatalf("current successful observation did not clear diagnostic: %q, %v", current.ObservationError, err) + } + }) + } +} From 4a72318da7a9d3580440bd6cb771707abd52b6f2 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 01:49:15 +0800 Subject: [PATCH 15/22] Preserve snapshots until retention expiry (#653) --- docs/sandbox-provider.md | 2 +- docs/zh/sandbox-provider.md | 4 +- .../internal/deployment/retention_pressure.go | 44 ---- .../deployment/suspension_pressure.go | 23 +- .../internal/execution/runtime_compute.go | 3 +- .../execution/runtime_replacement_test.go | 58 +++++ .../deploymentpg/retention_pressure_test.go | 79 ++---- .../deploymentpg/suspension_pressure_test.go | 237 +++++++----------- 8 files changed, 190 insertions(+), 260 deletions(-) delete mode 100644 services/core/internal/deployment/retention_pressure.go diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 8e7780e91..a821f3343 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -196,7 +196,7 @@ Queued work and live Environment file access wake a suspended Environment; histo Snapshot retention bounds compute resources, not the lifetime of an eligible retained Session. After the deadline, Core can clean up the allocation without terminating the Session only when its independent filesystem object is ready, initialization is complete, the Session and Environment remain nonterminal, and the previous authenticated Runtime has a persisted `retained_native_history` declaration. The capability and native recovery obligations are defined by the [Core–Runtime protocol](runtime-protocol.md). Filesystem retention alone is insufficient. -When compatible demand cannot obtain a retained slot, Core may end an idle, stopped allocation’s checkpoint retention early. It rechecks native-history eligibility, pending work and capacity under the Session and deployment locks. The same expiry cleanup must confirm resource deletion before the waiting demand can reserve that slot. The 24-hour retention is an upper bound, not a guarantee of memory preservation under capacity pressure; files and qualified native history remain retained. +Capacity pressure never shortens checkpoint retention. Until its configured deadline, a suspended allocation keeps its snapshot and retained slot; unavailable capacity or an incompatible node does not authorize cold replacement. Pending requests retain their original input deadline. After retention expires, the ordinary expiry cleanup must confirm resource deletion before another allocation can reserve that slot; files and qualified native history remain retained. Core confirms the original creation is settled and all owned compute and snapshots are cleaned up before releasing allocation and placement ownership. The old device's authority is revoked before replacement admission. Until Create, Kill or DeleteSnapshot has a confirmed outcome, it keeps ownership and admits no replacement writer. A recoverable Session and Environment remain disconnected with the same filesystem object and initialization result. Archive, reset and deletion take precedence and cannot be undone by a wake request. diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 24631a0d6..48562ef57 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: 4a7ce15e8fc2c9519b4974a60c746fbd436b24bbce5ab2ef461e442bea3e5f86 +source_hash: 567cd130134a45f483b156d7f2b74d246b191dd5db77b976976c0d710414dd37 --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -198,7 +198,7 @@ Worker lease、Session lock 与 per-node gate 对每个 provider 负责 suspensi 快照保留期限制计算资源，不限制符合条件的保留 Session 的生命周期。到期后，只有独立文件系统对象已就绪、初始化已完成、Session 与 Environment 均未终结，且此前已认证 Runtime 的 `retained_native_history` 声明已持久化时，Core 才能在不终结 Session 的情况下清理 allocation。能力及原生恢复义务由 [Core–Runtime 协议](runtime-protocol.md)定义。仅保留文件系统并不足够。 -兼容需求无法获得 retained slot 时，Core 可以提前结束空闲且已停止 allocation 的检查点保留期。它在 Session 和 deployment 锁内重新检查原生历史资格、pending work 和容量。等待需求预留该 slot 前，同一 expiry cleanup 必须确认资源已删除。24 小时保留期是上限，不保证容量压力下仍保留内存；文件与已验证的原生历史继续保留。 +容量压力不会缩短检查点保留期。在配置的期限到达前，已挂起的 allocation 保留快照并继续占用 retained slot；容量不足或节点不兼容不允许降级为冷替换。等待请求保持原有的输入期限。保留期到期后，常规 expiry cleanup 必须确认资源已删除，其他 allocation 才能预留该 slot；文件与已验证的原生历史继续保留。 Core 在释放 allocation 和 placement 归属前，确认原始创建已结算且所属计算资源与快照均已清理。替代计算资源准入前撤销旧 device 的权限。Create、Kill 或 DeleteSnapshot 的结果尚未确认时，Core 保留归属，不准入替代写入方。可恢复的 Session 与 Environment 保持断连，保留同一文件系统对象和初始化结果。归档、重置和删除优先，唤醒请求不能撤销它们。 diff --git a/services/core/internal/deployment/retention_pressure.go b/services/core/internal/deployment/retention_pressure.go deleted file mode 100644 index aeef016b1..000000000 --- a/services/core/internal/deployment/retention_pressure.go +++ /dev/null @@ -1,44 +0,0 @@ -package deployment - -import ( - "context" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" -) - -// EndRetentionForDemand brings forward the existing retention deadline only -// when a stopped, idle owner can keep its native history and release a needed -// retained slot. The ordinary expiry cleanup still proves every resource gone. -func (e *ExecutionOperations) EndRetentionForDemand(ctx context.Context, owner Allocation) (Allocation, error) { - if owner.NodeID == "" || owner.ComputePhase != "suspended" { - return owner, nil - } - return e.change(ctx, owner, true, func(tx AllocationTx, current Allocation) (Allocation, error) { - if current.State != "running" || current.ComputePhase != "suspended" || current.ComputeRevision != owner.ComputeRevision || current.Expired { - return Allocation{}, ErrAllocationConflict - } - activity, err := tx.LoadActivity(current) - if err != nil || activity.Busy || activity.WakeRequested { - return current, err - } - retained, err := tx.CanRetainEnvironment(current) - if err != nil || !retained { - return current, err - } - pressure, err := tx.LoadSuspensionDemand(current) - if err != nil { - return current, err - } - if pressure.Deployment.Resetting || pressure.Deployment.Mode != string(sandbox.DeploymentNodes) || pressure.Deployment.InstallationID != current.ProviderKey { - return current, nil - } - needed, err := e.placementNeedsCapacity(tx, pressure, current.NodeID, false, true) - if err != nil { - return current, err - } - if needed { - return tx.SetCompute(current, ComputeChange{Phase: "suspended", State: current.ComputeState, RetainedUntil: &activity.ObservedAt}) - } - return current, nil - }) -} diff --git a/services/core/internal/deployment/suspension_pressure.go b/services/core/internal/deployment/suspension_pressure.go index 8d40e7e1f..06c0bf492 100644 --- a/services/core/internal/deployment/suspension_pressure.go +++ b/services/core/internal/deployment/suspension_pressure.go @@ -40,20 +40,16 @@ func (e *ExecutionOperations) canSuspendForDemand(tx AllocationTx, owner Allocat if pressure.RestoreWaiting { return !slices.Contains(pressure.InFlightNodes, owner.NodeID), nil } - reclaimRetained := source.Retained >= int64(source.MaxRetained) - if reclaimRetained { - retained, err := tx.CanRetainEnvironment(owner) - if err != nil || !retained { - return false, err - } + if source.Retained >= int64(source.MaxRetained) { + return false, nil } - return e.placementNeedsCapacity(tx, pressure, owner.NodeID, true, reclaimRetained) + return e.placementNeedsCapacity(tx, pressure, owner.NodeID) } // placementNeedsCapacity scans bounded pages under the caller's deployment -// lock and transaction deadline. Suspension and early retention expiry use the -// same admission decision; an ineligible old receipt never supplies pressure. -func (e *ExecutionOperations) placementNeedsCapacity(tx AllocationTx, pressure SuspensionDemand, nodeID string, freeActive, freeRetained bool) (bool, error) { +// lock and transaction deadline. Suspension must make active capacity usable +// without releasing a retained slot; an ineligible receipt supplies no pressure. +func (e *ExecutionOperations) placementNeedsCapacity(tx AllocationTx, pressure SuspensionDemand, nodeID string) (bool, error) { var after PlacementDemandCursor for { demands, next, err := tx.PlacementDemand(after) @@ -80,12 +76,7 @@ func (e *ExecutionOperations) placementNeedsCapacity(tx AllocationTx, pressure S } for i := range nodes { if nodes[i].ID == nodeID { - if freeActive { - nodes[i].Active-- - } - if freeRetained && nodes[i].Retained >= int64(nodes[i].MaxRetained) { - nodes[i].Retained-- - } + nodes[i].Active-- } } chosen, err := e.service.rules.DecidePlacement(pressure.Deployment, nodes) diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index f558ef275..d6f2f62eb 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -221,8 +221,7 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San return err } if !activity.Busy && !activity.WakeRequested { - _, err := r.deployment.EndRetentionForDemand(ctx, owner) - return err + return nil } if state.Snapshot == nil || state.Target != nil { return sandbox.ErrOwnership diff --git a/services/core/internal/execution/runtime_replacement_test.go b/services/core/internal/execution/runtime_replacement_test.go index 182ddb17f..9a4e2f098 100644 --- a/services/core/internal/execution/runtime_replacement_test.go +++ b/services/core/internal/execution/runtime_replacement_test.go @@ -11,10 +11,13 @@ import ( "testing" "time" + v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -51,6 +54,61 @@ func (p *retentionProvider) DeleteSnapshot(context.Context, sandbox.Reference, s return nil } +func TestCapacityPressurePreservesSuspendedSnapshot(t *testing.T) { + provider := &retentionProvider{} + fixture := newWorkspaceSettlementFixture(t, provider) + r, session := fixture.lifecycle, fixture.session + owner, err := r.provision(t.Context(), session.TenantID, session.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + for _, statement := range []struct { + sql string + arg any + }{ + {`UPDATE runtime_nodes SET max_active=1,max_retained=1 WHERE id=$1`, owner.NodeID}, + {`UPDATE environments SET initialization='complete',status='disconnected' WHERE id=$1`, owner.EnvironmentID}, + {`UPDATE devices SET supported_agent_kinds='[{"kind":"codex","available":true,"capabilities":{"retained_native_history":true}}]' WHERE id=$1`, owner.DeviceID}, + {`UPDATE runtime_allocations SET state='running',compute_phase='suspended',compute_state='{"current":{"id":"old-compute"},"snapshot":{"id":"old-snapshot"}}',compute_retained_until=clock_timestamp()+interval '24 hours' WHERE id=$1`, owner.ID}, + } { + if _, err := fixture.pool.Exec(t.Context(), statement.sql, statement.arg); err != nil { + t.Fatal(err) + } + } + waiting, err := fixture.sessions.CreateSession(t.Context(), session.TenantID, sessions.CreateSession{SupportsRetainedNativeHistory: true, Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted","network":{"access":"disabled"}}}`), ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ExecutionSourceSession}) + if err != nil { + t.Fatal(err) + } + input, err := fixture.sessions.ReserveEnvironmentInput(t.Context(), session.TenantID, waiting.Session.ID, "waiting-for-capacity", []sessions.Input{{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"continue"}]}]}`)}}) + if err != nil { + t.Fatal(err) + } + owner, err = r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + for range 3 { + if err := r.observeCompute(t.Context(), owner); err != nil { + t.Fatal(err) + } + current, err := r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil || current.Expired || current.ComputePhase != "suspended" || current.ComputeRevision != owner.ComputeRevision || current.ComputeRetainedUntil == nil || !current.ComputeRetainedUntil.Equal(*owner.ComputeRetainedUntil) { + t.Fatal("pressure changed retained snapshot", current, err) + } + if _, err := r.deployment.EnsurePlacement(t.Context(), deployment.AllocationKey{TenantID: session.TenantID, EnvironmentID: waiting.Session.Environment.ID}, r.config.InstallationID); !errors.Is(err, placement.ErrNodeUnavailable) { + t.Fatal("released retained capacity without cleanup", err) + } + } + pending, err := r.sessions.GetEnvironmentInputReservation(t.Context(), session.TenantID, waiting.Session.ID, input.ID) + if err != nil || pending.State != sessions.EnvironmentInputPending || !pending.Deadline.Equal(input.Deadline) { + t.Fatal("pressure changed input deadline", pending, err) + } + + if provider.kills != 0 || provider.snapshots != 0 || provider.creates != 1 { + t.Fatal("pressure changed native resources", provider) + } +} + func TestExpiredRetainedComputePreservesSessionAndPendingInput(t *testing.T) { provider := &retentionProvider{killError: sandbox.ErrComputeUnconfirmed} fixture := newWorkspaceSettlementFixture(t, provider) diff --git a/services/core/internal/persistence/postgres/deploymentpg/retention_pressure_test.go b/services/core/internal/persistence/postgres/deploymentpg/retention_pressure_test.go index bbccedac9..ac6864145 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/retention_pressure_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/retention_pressure_test.go @@ -2,10 +2,12 @@ package deploymentpg_test import ( "encoding/json" + "errors" "testing" "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" ) @@ -37,26 +39,36 @@ func TestRetentionPressureKeepsHistoryUntilOrdinaryCleanupSettles(t *testing.T) f := newFixture(t) changes, _ := f.execution(t) installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) - node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 3, MaxRetained: 3}) f.connect(t, node.NodeID) owner := qualifyPressureRetention(t, f, runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)) + for range 2 { + qualifyPressureRetention(t, f, runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)) + } waiting := pendingPressureDemand(t, f) - ended, err := changes.EndRetentionForDemand(t.Context(), owner) - if err != nil { - t.Fatal(err) + for range 6 { + pendingPressureDemand(t, f) } - if ended.ComputeRetainedUntil == nil || !ended.ComputeRetainedUntil.Before(*owner.ComputeRetainedUntil) { - t.Fatal("retention was not ended", ended) + if _, err := changes.EnsurePlacement(t.Context(), waiting, installation); !errors.Is(err, placement.ErrNodeUnavailable) { + t.Fatal("reserved despite retained capacity limit") + } + current, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil || current.Expired || !current.ComputeRetainedUntil.Equal(*owner.ComputeRetainedUntil) { + t.Fatal("pressure shortened retention", current, err) + } + // Advance the fixture clock past the authored deadline, not because of demand. + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID); err != nil { + t.Fatal(err) } // The database deadline is only an intent. It is not a cleanup receipt. nodes, err := f.adapter.Nodes(t.Context()) - if err != nil || nodes[0].Retained != 1 { + if err != nil || nodes[0].Retained != 3 || nodes[0].Active != 0 { t.Fatal("released capacity before cleanup", nodes, err) } - if _, err = changes.EnsurePlacement(t.Context(), waiting, installation); err == nil { + if _, err = changes.EnsurePlacement(t.Context(), waiting, installation); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("reserved before cleanup") } - current, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) + current, err = f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) if err != nil || !current.Expired { t.Fatal("database expiry", current, err) } @@ -78,50 +90,7 @@ func TestRetentionPressureKeepsHistoryUntilOrdinaryCleanupSettles(t *testing.T) } } -func TestRetentionPressureGuards(t *testing.T) { - for _, name := range []string{"no demand", "retained headroom", "wake requested", "missing native history", "free other node", "already ending"} { - t.Run(name, func(t *testing.T) { - f := newFixture(t) - changes, _ := f.execution(t) - installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) - limit := 1 - if name == "retained headroom" || name == "already ending" { - limit = 2 - } - node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: limit}) - f.connect(t, node.NodeID) - owner := qualifyPressureRetention(t, f, runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)) - if name != "no demand" { - pendingPressureDemand(t, f) - } - var err error - switch name { - case "wake requested": - _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_wake_requested=true WHERE id=$1`, owner.ID) - case "missing native history": - _, err = f.pool.Exec(t.Context(), `UPDATE devices SET supported_agent_kinds='[]' WHERE id=$1`, owner.DeviceID) - case "free other node": - other := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) - f.connect(t, other.NodeID) - case "already ending": - other := qualifyPressureRetention(t, f, runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)) - _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, other.ID) - } - if err != nil { - t.Fatal(err) - } - next, err := changes.EndRetentionForDemand(t.Context(), owner) - if err != nil { - t.Fatal(err) - } - if next.ComputeRevision != owner.ComputeRevision || next.ComputeRetainedUntil == nil || !next.ComputeRetainedUntil.Equal(*owner.ComputeRetainedUntil) { - t.Fatal("ended retention despite guard", next) - } - }) - } -} - -func TestPressureSuspendsRetainableOwnerWhenRetainedSlotsAreFull(t *testing.T) { +func TestPressurePreservesRetainableOwnerWhenRetainedSlotsAreFull(t *testing.T) { f := newFixture(t) changes, _ := f.execution(t) installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) @@ -137,7 +106,7 @@ func TestPressureSuspendsRetainableOwnerWhenRetainedSlotsAreFull(t *testing.T) { } pendingPressureDemand(t, f) until := time.Now().Add(24 * time.Hour) - if _, err = changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute); err != nil { - t.Fatal(err) + if _, err = changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute); !errors.Is(err, deployment.ErrNotIdle) { + t.Fatal("suspended without usable capacity", err) } } diff --git a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go index a5e9c6308..4d3b419e0 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go @@ -3,13 +3,13 @@ package deploymentpg_test import ( "encoding/json" "errors" - "github.com/google/uuid" "sync" "testing" "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" ) func runningPressureOwner(t *testing.T, f fixture, changes *deployment.ExecutionOperations, installation, node string, generation uint64) deployment.Allocation { @@ -150,46 +150,27 @@ func TestPressureSuspensionServesRestoreOnItsOriginalNode(t *testing.T) { } func TestPressureDemandScansPastIncompatiblePage(t *testing.T) { - for _, retained := range []bool{false, true} { - t.Run(map[bool]string{false: "active", true: "retained"}[retained], func(t *testing.T) { - f := newFixture(t) - changes, _ := f.execution(t) - installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) - capacity := deployment.Capacity{MaxActive: 1, MaxRetained: 2} - if retained { - capacity.MaxRetained = 1 - } - node := f.enroll(t, view, capacity) - f.connect(t, node.NodeID) - owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) - if retained { - owner = qualifyPressureRetention(t, f, owner) - } - for range 32 { - key := pendingPressureDemand(t, f) - if _, err := f.pool.Exec(t.Context(), `UPDATE sessions SET engine='mcode' WHERE id=(SELECT session_id FROM environments WHERE id=$1)`, key.EnvironmentID); err != nil { - t.Fatal(err) - } - } - pendingPressureDemand(t, f) - until := time.Now().Add(time.Hour) - var result deployment.Allocation - var err error - if retained { - result, err = changes.EndRetentionForDemand(t.Context(), owner) - } else { - result, err = changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute) - } - if err != nil { - t.Fatal(err) - } - if retained && (result.ComputeRetainedUntil == nil || !result.ComputeRetainedUntil.Before(*owner.ComputeRetainedUntil)) { - t.Fatal("later compatible demand did not end retention") - } - if !retained && result.ComputePhase != "quiescing" { - t.Fatal("later compatible demand did not suspend compute") - } - }) + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + capacity := deployment.Capacity{MaxActive: 1, MaxRetained: 2} + node := f.enroll(t, view, capacity) + f.connect(t, node.NodeID) + owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) + for range 32 { + key := pendingPressureDemand(t, f) + if _, err := f.pool.Exec(t.Context(), `UPDATE sessions SET engine='mcode' WHERE id=(SELECT session_id FROM environments WHERE id=$1)`, key.EnvironmentID); err != nil { + t.Fatal(err) + } + } + pendingPressureDemand(t, f) + until := time.Now().Add(time.Hour) + result, err := changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute) + if err != nil { + t.Fatal(err) + } + if result.ComputePhase != "quiescing" { + t.Fatal("later compatible demand did not suspend compute") } } @@ -236,116 +217,92 @@ func TestPressureDemandRejectsUnqualifiedRetainedReceipt(t *testing.T) { func TestPressureReclamationIgnoresUnavailableNodeReceipts(t *testing.T) { for _, obstruction := range []string{"offline expired", "offline quiescing", "offline suspending", "different address", "incompatible generation"} { - for _, retention := range []bool{false, true} { - t.Run(obstruction+map[bool]string{false: "/suspend", true: "/retention"}[retention], func(t *testing.T) { - f := newFixture(t) - changes, _ := f.execution(t) - spec := retainedSpecification() - if obstruction == "incompatible generation" { - spec = testSpecification("microsandbox") - } - installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: spec}) - badNode := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) - f.connect(t, badNode.NodeID) - stuck := runningPressureOwner(t, f, changes, installation, badNode.NodeID, view.Generation) - phase := "quiescing" - if obstruction == "offline suspending" { - phase = "suspending" - } - if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase=$2,compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, stuck.ID, phase); err != nil { + t.Run(obstruction, func(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + spec := retainedSpecification() + if obstruction == "incompatible generation" { + spec = testSpecification("microsandbox") + } + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: spec}) + badNode := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + f.connect(t, badNode.NodeID) + stuck := runningPressureOwner(t, f, changes, installation, badNode.NodeID, view.Generation) + phase := "quiescing" + if obstruction == "offline suspending" { + phase = "suspending" + } + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase=$2,compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, stuck.ID, phase); err != nil { + t.Fatal(err) + } + switch obstruction { + case "offline expired", "offline quiescing", "offline suspending": + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, badNode.NodeID); err != nil { t.Fatal(err) } - switch obstruction { - case "offline expired", "offline quiescing", "offline suspending": - if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, badNode.NodeID); err != nil { - t.Fatal(err) - } - if obstruction == "offline expired" { - if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 minute' WHERE id=$1`, stuck.ID); err != nil { - t.Fatal(err) - } - } - case "different address": - if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET core_url='https://other.example' WHERE id=$1`, badNode.NodeID); err != nil { + if obstruction == "offline expired" { + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 minute' WHERE id=$1`, stuck.ID); err != nil { t.Fatal(err) } - case "incompatible generation": - var err error - view, err = changes.Update(admin(t), installation, sandbox.Selection{Provider: "microsandbox", ExpectedGeneration: view.Generation, DeploymentSpec: retainedSpecification()}) - if err != nil { - t.Fatal(err) - } - } - limit := 2 - if retention { - limit = 1 } - node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: limit}) - f.connect(t, node.NodeID) - var waiting deployment.AllocationKey - if obstruction == "incompatible generation" { - old := releaseRetainedFixture(t, f, changes, runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)) - waiting = old.Key() - } else { - waiting = pendingPressureDemand(t, f) - } - // A real pending reservation supplies demand and retains its original deadline. - if _, err := f.pool.Exec(t.Context(), `INSERT INTO environment_input_reservations(id,session_id,idempotency_key,batch,created_at,deadline) SELECT $1,session_id,'pressure','[{}]',clock_timestamp(),clock_timestamp()+interval '5 minutes' FROM environments WHERE id=$2`, uuid.NewString(), waiting.EnvironmentID); err != nil { + case "different address": + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET core_url='https://other.example' WHERE id=$1`, badNode.NodeID); err != nil { t.Fatal(err) } - owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) - if retention { - owner = qualifyPressureRetention(t, f, owner) - } - until := time.Now().Add(time.Hour) - if retention { - if _, err := changes.EndRetentionForDemand(t.Context(), owner); err != nil { - t.Fatal(err) - } - current, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) - if err != nil || !current.Expired { - t.Fatal("healthy retention did not advance", current, err) - } - pending, err := changes.RequestCleanup(t.Context(), current) - if err != nil { - t.Fatal(err) - } - // Supply the native deletion receipt for the healthy fixture only. - if _, err = changes.ReleaseAllocation(t.Context(), pending); err != nil { - t.Fatal(err) - } - } else { - current, err := changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute) - if err != nil { - t.Fatal("healthy suspension blocked", err) - } - // Supply healthy native quiesce/capture/stop receipts through the - // existing fenced phase transitions; never settle the stuck owner. - for _, next := range []string{"suspending", "suspended"} { - current, err = changes.SetCompute(t.Context(), current, next, json.RawMessage(`{}`), &until, 5*time.Minute) - if err != nil { - t.Fatal(err) - } - } - } - reserved, err := changes.EnsurePlacement(t.Context(), waiting, installation) - if err != nil || reserved.NodeID != node.NodeID { - t.Fatal("waiting input did not advance on healthy node", reserved, err) - } - observed, err := f.adapter.EnvironmentAllocation(t.Context(), stuck.Key()) - if err != nil || observed.ID != stuck.ID || observed.DeviceID != stuck.DeviceID || observed.State != "running" || observed.ComputePhase != phase { - t.Fatal("changed unknown receipt", observed, err) + case "incompatible generation": + var err error + view, err = changes.Update(admin(t), installation, sandbox.Selection{Provider: "microsandbox", ExpectedGeneration: view.Generation, DeploymentSpec: retainedSpecification()}) + if err != nil { + t.Fatal(err) } - nodes, err := f.adapter.Nodes(t.Context()) + } + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + f.connect(t, node.NodeID) + var waiting deployment.AllocationKey + if obstruction == "incompatible generation" { + old := releaseRetainedFixture(t, f, changes, runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)) + waiting = old.Key() + } else { + waiting = pendingPressureDemand(t, f) + } + // A real pending reservation supplies demand and retains its original deadline. + if _, err := f.pool.Exec(t.Context(), `INSERT INTO environment_input_reservations(id,session_id,idempotency_key,batch,created_at,deadline) SELECT $1,session_id,'pressure','[{}]',clock_timestamp(),clock_timestamp()+interval '5 minutes' FROM environments WHERE id=$2`, uuid.NewString(), waiting.EnvironmentID); err != nil { + t.Fatal(err) + } + owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) + + until := time.Now().Add(time.Hour) + + current, err := changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute) + if err != nil { + t.Fatal("healthy suspension blocked", err) + } + // Supply healthy native quiesce/capture/stop receipts through the + // existing fenced phase transitions; never settle the stuck owner. + for _, next := range []string{"suspending", "suspended"} { + current, err = changes.SetCompute(t.Context(), current, next, json.RawMessage(`{}`), &until, 5*time.Minute) if err != nil { t.Fatal(err) } - for _, n := range nodes { - if n.ID == badNode.NodeID && (n.Active != 1 || n.Retained != 1) { - t.Fatal("released unknown capacity", n) - } + } + + reserved, err := changes.EnsurePlacement(t.Context(), waiting, installation) + if err != nil || reserved.NodeID != node.NodeID { + t.Fatal("waiting input did not advance on healthy node", reserved, err) + } + observed, err := f.adapter.EnvironmentAllocation(t.Context(), stuck.Key()) + if err != nil || observed.ID != stuck.ID || observed.DeviceID != stuck.DeviceID || observed.State != "running" || observed.ComputePhase != phase { + t.Fatal("changed unknown receipt", observed, err) + } + nodes, err := f.adapter.Nodes(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, n := range nodes { + if n.ID == badNode.NodeID && (n.Active != 1 || n.Retained != 1) { + t.Fatal("released unknown capacity", n) } - }) - } + } + }) } } From 274ee05db0041c1f98ab90e9624937abfff6f014 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 02:10:42 +0800 Subject: [PATCH 16/22] Preserve resident executors during suspension (#651) --- .../internal/cli/connect_suspend_test.go | 37 +++- apps/daemon/internal/dispatch/executor.go | 4 +- .../daemon/internal/dispatch/executor_test.go | 49 +++++ apps/daemon/internal/dispatch/suspend.go | 26 +-- apps/daemon/internal/dispatch/suspend_test.go | 174 ++++++++++++------ contracts/agents-api/harness-onboarding.md | 2 +- contracts/agents-api/zh/harness-onboarding.md | 4 +- docs/runtime-protocol.md | 2 +- docs/zh/runtime-protocol.md | 4 +- internal/agentdaemon/proto/envelope_test.go | 1 + internal/agentdaemon/proto/suspend.go | 5 +- internal/agentdaemon/proto/version.go | 2 +- 12 files changed, 218 insertions(+), 92 deletions(-) diff --git a/apps/daemon/internal/cli/connect_suspend_test.go b/apps/daemon/internal/cli/connect_suspend_test.go index 52dd2caea..6d92648e2 100644 --- a/apps/daemon/internal/cli/connect_suspend_test.go +++ b/apps/daemon/internal/cli/connect_suspend_test.go @@ -339,7 +339,7 @@ func TestSuspensionReconnectBeforeConfirmation(t *testing.T) { } } -func TestQuiesceCloseFailureDisconnectsAndSettlesBeforeReconnect(t *testing.T) { +func TestQuiesceRetainsExecutorAcrossReconnectAndShutdownConfirmsCleanup(t *testing.T) { environment, session := uuid.NewString(), uuid.NewString() t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) t.Setenv("OAC_RUNTIME_STATE_DIRECTORY", t.TempDir()) @@ -431,13 +431,38 @@ func TestQuiesceCloseFailureDisconnectsAndSettlesBeforeReconnect(t *testing.T) { } readStatus("released") sendLifecycleFrame(t, peer, proto.TypeEnvironmentQuiesce, request) - if got := readLifecycleResult(t, peer, proto.TypeEnvironmentQuiesced); got.Accepted || got.ErrorCode != "resource_busy" { - t.Fatalf("failed Close acknowledged: %+v", got) + if got := readLifecycleResult(t, peer, proto.TypeEnvironmentQuiesced); !got.Accepted { + t.Fatalf("idle owner rejected: %+v", got) + } + if native.closes.Load() != 0 { + t.Fatal("quiesce closed the native owner") + } + control.signal <- syscall.SIGUSR1 + select { + case peer = <-peers: + case <-time.After(3 * time.Second): + t.Fatal("no planned reconnect") } + defer peer.Close() + sendLifecycleFrame(t, peer, proto.TypeEnvironmentResume, request) + if got := readLifecycleResult(t, peer, proto.TypeEnvironmentResumed); !got.Accepted { + t.Fatalf("resume rejected: %+v", got) + } + preparation.ID = "resumed-prepare" + if err := peer.WriteJSON(preparation); err != nil { + t.Fatal(err) + } + resumed := readStatus("ready") + if !resumed.Reused || resumed.ExecutorID != ready.ExecutorID || resumed.Handle == ready.Handle || native.closes.Load() != 0 { + t.Fatalf("resume replaced native owner: before=%+v after=%+v closes=%d", ready, resumed, native.closes.Load()) + } + // Cancelling the daemon still retires resident native resources. Failed + // cleanup must settle before the loop can finish or create another router. + cancel() select { case <-native.retry: case <-time.After(3 * time.Second): - t.Fatal("fenced close failure did not enter Shutdown retry") + t.Fatal("cancelled resident owner did not enter Shutdown retry") } _ = peer.SetReadDeadline(time.Now().Add(time.Second)) for { @@ -462,7 +487,7 @@ func TestQuiesceCloseFailureDisconnectsAndSettlesBeforeReconnect(t *testing.T) { if native.closes.Load() != 2 { t.Fatalf("Close calls=%d", native.closes.Load()) } - if _, err := os.Stat(control.path); !os.IsNotExist(err) { - t.Fatal("failed quiesce armed snapshot control") + if control.identity.SuspendID != "" || control.lastResumed == nil || !control.lastResumed.SameSuspension(request) { + t.Fatal("resumed suspension left snapshot control armed or lost its receipt") } } diff --git a/apps/daemon/internal/dispatch/executor.go b/apps/daemon/internal/dispatch/executor.go index 0e47e8c20..3a07e09a2 100644 --- a/apps/daemon/internal/dispatch/executor.go +++ b/apps/daemon/internal/dispatch/executor.go @@ -253,7 +253,7 @@ func (r *Router) abandonExecutorAdmission(p *preparationState, state, code strin } func (r *Router) scheduleExecutorIdleLocked(owner *executorState) { - if owner.invalid || owner.preparing || owner.run != nil || owner.admission != nil { + if r.closed || r.suspension != nil || owner.invalid || owner.preparing || owner.run != nil || owner.admission != nil { return } if owner.timer != nil { @@ -281,7 +281,7 @@ func (r *Router) scheduleExecutorIdleLocked(owner *executorState) { // expireIdleExecutor closes owner only while lease is its current idle lease. func (r *Router) expireIdleExecutor(owner *executorState, lease uint64) { r.mu.Lock() - if r.executors[owner.sessionID] != owner || owner.idleLease != lease || owner.run != nil || owner.admission != nil || owner.invalid { + if r.closed || r.suspension != nil || r.executors[owner.sessionID] != owner || owner.idleLease != lease || owner.run != nil || owner.admission != nil || owner.invalid { r.mu.Unlock() return } diff --git a/apps/daemon/internal/dispatch/executor_test.go b/apps/daemon/internal/dispatch/executor_test.go index a2324a78c..2cba36047 100644 --- a/apps/daemon/internal/dispatch/executor_test.go +++ b/apps/daemon/internal/dispatch/executor_test.go @@ -342,3 +342,52 @@ func TestExecutorRejectsOutputFromAnotherTurn(t *testing.T) { func (*reusableTurn) SteerWithReceipt(context.Context, proto.PromptSteerPayload, func()) error { return agent.ErrSteeringRejected } + +func TestExecutorResumesSameOwnerAndNextTurnAfterSuspension(t *testing.T) { + owner := &reusableExecutor{starts: make(chan *reusableTurn, 2)} + var factories atomic.Int32 + registry := agent.NewRegistry() + registerExecutorKind(registry, proto.SupportedAgentKind{Kind: "prepared", Available: true, Capabilities: prototest.Capabilities(proto.AgentKindCapabilities{LocalEnvironment: proto.CapabilitySupported, DurableInputReceipts: proto.CapabilitySupported})}, func(context.Context, proto.PromptRequestPayload) (agent.Executor, error) { + factories.Add(1) + return owner, nil + }) + sender := &recSender{} + r, err := dispatch.New(dispatch.Config{Registry: registry, Sender: sender, LocalWorkspace: preparationWorkspace(t)}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := r.Shutdown(context.Background()); err != nil { + t.Error(err) + } + }) + first := executorAdmission(t, r, sender, "first", preparationRequest()) + startExecutorTurn(t, r, sender, "first", "one", first) + (<-owner.starts).finish() + waitFor(t, func() bool { return r.ActiveRuns() == 0 }, "first Turn settled") + request := proto.EnvironmentSuspendPayload{EnvironmentID: preparationEnvironmentID, SuspendID: "suspension"} + if err := r.Quiesce(t.Context(), request); err != nil { + t.Fatal(err) + } + resumedSender := &recSender{} + if err := r.Resume(request, resumedSender); err != nil { + t.Fatal(err) + } + req := preparationRequest() + req.Configuration.AgentSessionID = "native-session" + conflict := req + conflict.Configuration.AgentOptions = map[string]any{"model": "changed"} + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "conflict", conflict)); err == nil { + t.Fatal("suspension relaxed the fixed configuration") + } + second := executorAdmission(t, r, resumedSender, "second", req) + if !second.Reused || first.ExecutorID != second.ExecutorID || first.Handle == second.Handle || factories.Load() != 1 || owner.closes.Load() != 0 { + t.Fatal("suspension replaced the resident native owner") + } + startExecutorTurn(t, r, resumedSender, "second", "two", second) + (<-owner.starts).finish() + waitFor(t, func() bool { return r.ActiveRuns() == 0 }, "resumed Turn settled") + if owner.closes.Load() != 0 { + t.Fatal("resumed Turn retired its healthy native owner") + } +} diff --git a/apps/daemon/internal/dispatch/suspend.go b/apps/daemon/internal/dispatch/suspend.go index 5945c7cdc..b38eb9bdc 100644 --- a/apps/daemon/internal/dispatch/suspend.go +++ b/apps/daemon/internal/dispatch/suspend.go @@ -3,7 +3,6 @@ package dispatch import ( "context" "errors" - "fmt" "strings" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" @@ -14,9 +13,9 @@ var ErrRouterQuiesced = errors.New("dispatch: router quiesced") var ErrRouterBusy = errors.New("dispatch: router has unsettled work") // Quiesce serializes against admission, then drains every admitted output and -// receipt and closes idle Executors before acknowledging suspension. Busy -// rejection leaves admission open; a failed close or drain timeout keeps it -// closed until the caller shuts the connection down. +// receipt while retaining settled idle Executors for suspension. Busy rejection +// leaves admission open; a drain timeout keeps it closed until the caller shuts +// the connection down. func (r *Router) Quiesce(ctx context.Context, request proto.EnvironmentSuspendPayload) error { if strings.TrimSpace(request.EnvironmentID) == "" || strings.TrimSpace(request.SuspendID) == "" || len(request.SuspendID) > 128 { return errors.New("dispatch: invalid suspension identity") @@ -56,27 +55,20 @@ func (r *Router) Quiesce(ctx context.Context, request proto.EnvironmentSuspendPa p.timer.Stop() } } - owners := r.closeIdleExecutorsLocked() - for _, owner := range owners { + for _, owner := range r.executors { + if owner.timer != nil { + owner.timer.Stop() + } + // Stop cannot retract a callback already waiting for the lock. Fence its + // lease before releasing the lock, including after an eventual Resume. owner.idleLease++ - owner.closeReason = "suspend" } r.mu.Unlock() - r.closeIdleExecutors(owners) err := r.shutdownWG.waitContext(ctx) r.mu.Lock() if r.closed { err = ErrRouterClosed - } else if err == nil { - for _, owner := range r.executors { - cause := owner.closeErr - if cause == nil { - cause = errors.New("cleanup has not settled") - } - err = errors.Join(err, fmt.Errorf("dispatch: executor %s: %w", owner.id, cause)) - } } - r.mu.Unlock() if err != nil { return errors.Join(ErrRouterQuiesced, err) diff --git a/apps/daemon/internal/dispatch/suspend_test.go b/apps/daemon/internal/dispatch/suspend_test.go index ddb48f030..36562c3e2 100644 --- a/apps/daemon/internal/dispatch/suspend_test.go +++ b/apps/daemon/internal/dispatch/suspend_test.go @@ -123,7 +123,7 @@ func TestQuiesceDrainsPendingReceiptAndFencesConcurrentAdmission(t *testing.T) { } } -func TestQuiesceClosesIdleExecutorOnceAndRequiresExactResume(t *testing.T) { +func TestQuiesceRetainsIdleExecutorAndRequiresExactResume(t *testing.T) { sender := suspendSender(func(context.Context, proto.Envelope) error { return nil }) r := suspensionRouter(t, sender) native := &suspendedExecutor{} @@ -134,32 +134,78 @@ func TestQuiesceClosesIdleExecutorOnceAndRequiresExactResume(t *testing.T) { oldLease := owner.idleLease r.mu.Unlock() request := proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"} - if err := r.Quiesce(context.Background(), request); err != nil { + if err := r.Quiesce(t.Context(), request); err != nil { t.Fatal(err) } r.expireIdleExecutor(owner, oldLease) - if native.closed.Load() != 1 { - t.Fatal("quiesce and stale timer did not close the owner exactly once") + if native.closed.Load() != 0 { + t.Fatal("quiesce or stale timer closed the resident owner") + } + for _, wrong := range []proto.EnvironmentSuspendPayload{ + {EnvironmentID: "env", SuspendID: "obsolete"}, + {EnvironmentID: "foreign", SuspendID: "attempt"}, + } { + if err := r.Resume(wrong, sender); err == nil { + t.Fatal("wrong suspension reopened admission") + } } - wrong := request - wrong.SuspendID = "obsolete" - if err := r.Resume(wrong, sender); err == nil { - t.Fatal("stale operation reopened admission") + if err := r.Resume(request, nil); err == nil { + t.Fatal("resume accepted no sender") } if err := r.Resume(request, sender); err != nil { t.Fatal(err) } + // A callback queued before Stop may run after Resume. It must not close + // the same owner under its renewed idle lease. + r.expireIdleExecutor(owner, oldLease) + r.mu.Lock() + retained := r.executors[owner.sessionID] == owner && !owner.invalid && owner.native == native && owner.idleLease > oldLease + lease := owner.idleLease + r.mu.Unlock() + if !retained || native.closed.Load() != 0 { + t.Fatal("resume replaced or retired the resident owner") + } + r.expireIdleExecutor(owner, lease) + if native.closed.Load() != 1 { + t.Fatal("resumed idle expiry did not close the exact owner") + } +} + +func TestQuiesceStopsIdleTimerUntilResume(t *testing.T) { + sender := suspendSender(func(context.Context, proto.Envelope) error { return nil }) + r := suspensionRouter(t, sender) + closed := make(chan struct{}) + native := &suspendedExecutor{close: func(context.Context) error { close(closed); return nil }} + owner := &executorState{id: "executor", sessionID: "session", environmentID: "env", native: native, cancel: func() {}} r.mu.Lock() - remaining := len(r.executors) + r.idleTimeout = 40 * time.Millisecond + r.executors[owner.sessionID] = owner + r.scheduleExecutorIdleLocked(owner) r.mu.Unlock() - if remaining != 0 || native.closed.Load() != 1 { - t.Fatal("resume retained or reclosed a retired native owner") + request := proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"} + if err := r.Quiesce(t.Context(), request); err != nil { + t.Fatal(err) + } + select { + case <-closed: + t.Fatal("parked owner's idle timer closed native resources") + case <-time.After(2 * r.idleTimeout): + } + if err := r.Resume(request, sender); err != nil { + t.Fatal(err) + } + select { + case <-closed: + case <-time.After(time.Second): + t.Fatal("resume did not restore idle expiry") } } func TestShutdownDestroysQuiescedOwnerAndCannotResume(t *testing.T) { sender := suspendSender(func(context.Context, proto.Envelope) error { return nil }) r := suspensionRouter(t, sender) + native := &suspendedExecutor{} + r.executors["session"] = &executorState{id: "executor", sessionID: "session", environmentID: "env", native: native, cancel: func() {}} request := proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"} if err := r.Quiesce(context.Background(), request); err != nil { t.Fatal(err) @@ -170,6 +216,9 @@ func TestShutdownDestroysQuiescedOwnerAndCannotResume(t *testing.T) { if !errors.Is(r.Resume(request, sender), ErrRouterClosed) { t.Fatal("closed Router resurrected") } + if native.closed.Load() != 1 || len(r.executors) != 0 { + t.Fatal("shutdown did not retire the retained native owner") + } } func TestQuiesceDrainDeadlineCannotReopenAdmission(t *testing.T) { @@ -191,47 +240,17 @@ func TestQuiesceDrainDeadlineCannotReopenAdmission(t *testing.T) { } } -func TestQuiesceWaitsForIdleExecutorClose(t *testing.T) { - entered, release := make(chan struct{}), make(chan struct{}) - native := &suspendedExecutor{close: func(context.Context) error { close(entered); <-release; return nil }} - r := suspensionRouter(t, suspendSender(func(context.Context, proto.Envelope) error { return nil })) - t.Cleanup(func() { - select { - case <-release: - default: - close(release) - } - }) - owner := &executorState{id: "executor", sessionID: "session", environmentID: "env", native: native, cancel: func() {}} - r.executors[owner.sessionID] = owner - quiet := make(chan error, 1) - go func() { - quiet <- r.Quiesce(t.Context(), proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}) - }() - <-entered - select { - case err := <-quiet: - t.Fatalf("acknowledged before native close: %v", err) - case <-time.After(20 * time.Millisecond): - } - close(release) - if err := <-quiet; err != nil { - t.Fatal(err) - } - if native.closed.Load() != 1 { - t.Fatal("native Close was duplicated") - } -} - -func TestQuiesceCloseFailureRetainsOwnerAndFencesAdmission(t *testing.T) { - failure := errors.New("native history flush failed") +func TestQuiescedShutdownRetainsFailedCloseForRetry(t *testing.T) { + failure := errors.New("native cleanup failed") native := &suspendedExecutor{close: func(context.Context) error { return failure }} r := suspensionRouter(t, suspendSender(func(context.Context, proto.Envelope) error { return nil })) owner := &executorState{id: "executor", sessionID: "session", environmentID: "env", native: native, cancel: func() {}} r.executors[owner.sessionID] = owner - request := proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"} - if err := r.Quiesce(t.Context(), request); !errors.Is(err, failure) || !errors.Is(err, ErrRouterQuiesced) { - t.Fatalf("quiesce lost native close failure: %v", err) + if err := r.Quiesce(t.Context(), proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}); err != nil || native.closed.Load() != 0 { + t.Fatalf("quiesce retired owner: err=%v closes=%d", err, native.closed.Load()) + } + if err := r.Shutdown(t.Context()); !errors.Is(err, failure) { + t.Fatalf("shutdown lost native close failure: %v", err) } r.mu.Lock() retained := r.executors[owner.sessionID] == owner && owner.invalid && owner.closeErr == failure @@ -239,9 +258,6 @@ func TestQuiesceCloseFailureRetainsOwnerAndFencesAdmission(t *testing.T) { if !retained { t.Fatal("failed close abandoned ownership") } - if err := r.Handle(t.Context(), proto.Envelope{Type: proto.TypeExecutionPrepare, ID: "late"}); !errors.Is(err, ErrRouterQuiesced) { - t.Fatalf("failed close reopened admission: %v", err) - } native.close = nil if err := r.Shutdown(t.Context()); err != nil { t.Fatal(err) @@ -251,7 +267,7 @@ func TestQuiesceCloseFailureRetainsOwnerAndFencesAdmission(t *testing.T) { } } -func TestQuiesceDeadlineJoinsNativeCloseDuringShutdown(t *testing.T) { +func TestQuiesceCancelledDrainRetainsOwnerUntilShutdownSettles(t *testing.T) { entered, release := make(chan struct{}), make(chan struct{}) native := &suspendedExecutor{close: func(context.Context) error { close(entered); <-release; return nil }} r := suspensionRouter(t, suspendSender(func(context.Context, proto.Envelope) error { return nil })) @@ -264,21 +280,22 @@ func TestQuiesceDeadlineJoinsNativeCloseDuringShutdown(t *testing.T) { }) owner := &executorState{id: "executor", sessionID: "session", environmentID: "env", native: native, cancel: func() {}} r.executors[owner.sessionID] = owner + r.shutdownWG.Add(1) ctx, cancel := context.WithCancel(t.Context()) - quiet := make(chan error, 1) - go func() { - quiet <- r.Quiesce(ctx, proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}) - }() - <-entered cancel() - if err := <-quiet; !errors.Is(err, context.Canceled) { + if err := r.Quiesce(ctx, proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"}); !errors.Is(err, context.Canceled) { t.Fatalf("quiesce=%v", err) } + if native.closed.Load() != 0 { + t.Fatal("cancelled drain closed resident owner") + } if err := r.Handle(t.Context(), proto.Envelope{Type: proto.TypeExecutionPrepare, ID: "late"}); !errors.Is(err, ErrRouterQuiesced) { t.Fatalf("deadline reopened admission: %v", err) } + r.shutdownWG.Done() stopped := make(chan error, 1) go func() { stopped <- r.Shutdown(t.Context()) }() + <-entered select { case err := <-stopped: t.Fatalf("shutdown abandoned in-flight close: %v", err) @@ -308,3 +325,44 @@ func TestQuiesceRejectsAdmissionInProgress(t *testing.T) { t.Fatal("busy rejection changed suspension") } } + +func TestQuiesceFencesAlreadyRunningTimerAcrossResume(t *testing.T) { + sender := suspendSender(func(context.Context, proto.Envelope) error { return nil }) + r := suspensionRouter(t, sender) + native := &suspendedExecutor{} + owner := &executorState{id: "executor", sessionID: "session", environmentID: "env", native: native, cancel: func() {}} + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + t.Cleanup(func() { + select { + case <-release: + default: + close(release) + } + <-done + }) + r.mu.Lock() + r.executors[owner.sessionID] = owner + r.scheduleExecutorIdleLocked(owner) + owner.timer.Stop() + lease := owner.idleLease + owner.timer = time.AfterFunc(0, func() { + close(entered) + <-release + r.expireIdleExecutor(owner, lease) + close(done) + }) + r.mu.Unlock() + <-entered + request := proto.EnvironmentSuspendPayload{EnvironmentID: "env", SuspendID: "attempt"} + if err := r.Quiesce(t.Context(), request); err != nil { + t.Fatal(err) + } + if err := r.Resume(request, sender); err != nil { + t.Fatal(err) + } + close(release) + <-done + if native.closed.Load() != 0 { + t.Fatal("callback already running before suspension retired resumed owner") + } +} diff --git a/contracts/agents-api/harness-onboarding.md b/contracts/agents-api/harness-onboarding.md index 6032efbe2..d473444ef 100644 --- a/contracts/agents-api/harness-onboarding.md +++ b/contracts/agents-api/harness-onboarding.md @@ -102,7 +102,7 @@ The service profile qualifies public combinations and the Runtime advertises the A Session owns one reusable Executor in its connected Runtime; a Turn owns one input execution, its output stream and its cancellation. `agent.ExecutorFactory` prepares the fixed configuration without model input, and `Executor.StartTurn` creates a new `agent.Turn` without replacing healthy native resources. Normal completion settles only the Turn. `Executor.Close` releases native resources on idle expiry, Environment shutdown or confirmed invalidation; it releases neither the Environment allocation nor the workspace. Core keeps no second Executor cache. The same lifecycle applies to hosted, self-hosted and `none` placements. -**Binding.** The Runtime binds its Executor record to the Session, Environment, connection and immutable execution configuration. Resume identity and prior-Turn recovery flags are continuity assertions, not configuration changes. A supplied native identity must match the retained owner, and when existing history is required, recovery never starts a new root. A configuration conflict is an error, not a hot switch. A lost connection retires its owners and handles; old timers, output and cancellation cannot affect their replacements. +**Binding.** The Runtime binds its Executor record to the Session, Environment, connection and immutable execution configuration. Resume identity and prior-Turn recovery flags are continuity assertions, not configuration changes. A supplied native identity must match the retained owner, and when existing history is required, recovery never starts a new root. A configuration conflict is an error, not a hot switch. An ordinary lost connection retires its owners and handles; [planned suspension](../../docs/runtime-protocol.md#preparation-and-execution-order) retains settled idle owners through exact authenticated resume. Old timers, output and cancellation cannot affect replacement owners. **Per-Turn state.** Each Turn gets a fresh wrapper, output channel and receipt state. Steering and function interfaces belong to that Turn. Native callbacks capture the originating Turn before asynchronous work, so a late event is never attributed to whichever Turn is active. Native processes, query or transport connections, fixed capability configuration and native session identity belong to the Executor. Do not reset completed `sync.Once` values or reuse an old Turn object. diff --git a/contracts/agents-api/zh/harness-onboarding.md b/contracts/agents-api/zh/harness-onboarding.md index 5df702061..d422a7116 100644 --- a/contracts/agents-api/zh/harness-onboarding.md +++ b/contracts/agents-api/zh/harness-onboarding.md @@ -1,7 +1,7 @@ --- title: "添加 Harness" source: contracts/agents-api/harness-onboarding.md -source_hash: 779954060b3be129f858e2f57e62b1af56cfe2f925f98362b48b20d32574f681 +source_hash: 2fe919ec4fd2285cc9e54fed58ef66aebe66d132fd8fb5adc272e26b2e673774 --- **Harness** 是一种运行模型和工具循环的原生代理引擎（Codex、Claude Code、MiniMax Code）。**Harness 适配器**将 Runtime 的 Executor 和 Turn 契约转换到该引擎的 SDK 或协议。本文档定义 Runtime–Harness 协议：适配器接口及其生命周期义务、注册、Core 资格认定和验收。[Harness capabilities](harness-capabilities.md) 记录了当前每个 Harness 支持的功能。 @@ -104,7 +104,7 @@ func (s *Session) SubmitFunctionResult(context.Context, proto.FunctionResultPayl Session 在其已连接的 Runtime 中拥有一个可复用的 Executor；Turn 拥有一次输入执行、其输出流和其取消操作。`agent.ExecutorFactory` 在没有模型输入的情况下准备固定配置，而 `Executor.StartTurn` 创建新的 `agent.Turn`，不替换健康的原生资源。正常完成仅结算 Turn。`Executor.Close` 在空闲过期、Environment 关闭或确认失效时释放原生资源；它既不释放 Environment 分配，也不释放工作区。Core 不保留第二套 Executor 缓存。相同的生命周期适用于托管、自托管和 `none` 放置方式。 -**绑定。** Runtime 将其 Executor 记录绑定到 Session、Environment、连接和不可变执行配置。恢复身份和先前 Turn 恢复标志是连续性断言，而不是配置更改。提供的原生身份必须与保留的所有者匹配；当需要现有历史时，恢复绝不能启动新的根。配置冲突属于错误，而不是热切换。连接丢失会让其所有者和句柄退役；旧计时器、输出和取消操作不能影响替代对象。 +**绑定。** Runtime 将其 Executor 记录绑定到 Session、Environment、连接和不可变执行配置。恢复身份和先前 Turn 恢复标志是连续性断言，而不是配置更改。提供的原生身份必须与保留的所有者匹配；当需要现有历史时，恢复绝不能启动新的根。配置冲突属于错误，而不是热切换。普通连接丢失会让其所有者和句柄退役；[计划性挂起](../../../docs/zh/runtime-protocol.md#preparation-and-execution-order) 则保留已结算的空闲所有者，直至精确匹配且已认证的恢复完成。旧计时器、输出和取消操作不能影响替代所有者。 **每 Turn 状态。** 每个 Turn 都会获得全新的包装器、输出通道和回执状态。引导和函数接口均属于该 Turn。原生回调必须在异步工作开始前捕获来源 Turn，因此迟到事件绝不会被归到当前活动的 Turn 上。原生进程、query 或传输连接、固定能力配置和原生 session 身份均属于 Executor。不要重置已完成的 `sync.Once` 值，也不要复用旧 Turn 对象。 diff --git a/docs/runtime-protocol.md b/docs/runtime-protocol.md index e7c818fa8..f892f27a6 100644 --- a/docs/runtime-protocol.md +++ b/docs/runtime-protocol.md @@ -126,7 +126,7 @@ Preparation and start run outside the receive loop and router lock. An admission Idle expiry of an Executor is a Runtime resource policy, separate from Core's active-Turn concurrency. On shutdown the Runtime closes active and idle Executors, keeps any target whose close failed and allows a later serialized retry. An ordinary disconnection closes the failed transport and keeps the exact router until shutdown succeeds; a wait timeout or failed cleanup never authorizes reconnection, and process shutdown keeps waiting rather than discarding owned native resources. Workspace operations keep their binding and settlement rules across Turn boundaries and Executor closure. -Suspension closes admission, drains admitted work and receipts, and confirms closure of every idle Executor before acknowledging `environment_quiesced`. A failed or unconfirmed native close prevents acknowledgement and retains cleanup ownership. The Sandbox Provider remains responsible for the filesystem flush and compute-stop guarantees of its checkpoint implementation; Runtime quiescence alone does not establish those guarantees. +Suspension closes admission and drains admitted work and receipts while retaining settled idle Executors before acknowledging `environment_quiesced`. Active, preparing, invalid or otherwise unsettled owners prevent suspension. Idle expiry is stopped and its callbacks fenced throughout suspension; exact authenticated `environment_resume` restores a fresh idle interval on the same owners. Only this planned reconnect preserves the Router and native owners across sockets; ordinary disconnect, shutdown and cancellation of the connection lifecycle still require confirmed cleanup. A drain failure keeps admission closed until shutdown. Preparation after resume retains the same native identity and immutable-configuration checks, including credential conflicts; it never silently replaces a conflicting resident Executor. The Sandbox Provider remains responsible for the filesystem flush and compute-stop guarantees of its checkpoint implementation; Runtime quiescence alone proves neither those guarantees nor restoration of native RAM or external network connections. ## Active input receipts diff --git a/docs/zh/runtime-protocol.md b/docs/zh/runtime-protocol.md index ab1b54296..279231ac7 100644 --- a/docs/zh/runtime-protocol.md +++ b/docs/zh/runtime-protocol.md @@ -1,7 +1,7 @@ --- title: "Core–Runtime 协议" source: docs/runtime-protocol.md -source_hash: 142a3d2a3dc5b790bcc80dff2364d094857f835e0e0687f91373251a4f4d9b3b +source_hash: 902f53e5471de13ecac0e914e4d23d7650955a4c50ec1aa01517c13aef7946f7 --- 此协议在 Runtime daemon 获取机器凭据后连接 Core 与 daemon，定义 daemon 连接上消息的含义和顺序。wire 类型、限制和验证器仅在 [`internal/agentdaemon/proto`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/internal/agentdaemon/proto) 中定义一次；Core 的 [gateway](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/runtimegateway) 与参考 Runtime 的 [dispatcher](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/apps/daemon/internal/dispatch) 都使用它们，因此无需同步第二套 payload schema。签发凭据和打开连接的 HTTP 路由见[机器连接 API](../../contracts/agents-api/zh/machine-api.md)。 @@ -129,7 +129,7 @@ preparation 和 start 在 receive loop 与 router lock 之外运行。admission Executor 空闲到期属于 Runtime 资源策略，与 Core 的活动 Turn 并发限制独立。关闭时 Runtime 关闭活动和空闲 Executor，保留关闭失败的目标，并允许稍后串行重试。普通断连会关闭失败的 transport 并保留原 router，直到 shutdown 成功；等待超时或清理失败不授权重连，进程 shutdown 继续等待，不丢弃自己拥有的原生资源。工作区操作在跨 Turn 和 Executor 关闭后仍保留绑定与结算规则。 -挂起先关闭准入、排空已接纳工作与回执，并确认每个空闲 Executor 已关闭，随后才确认 `environment_quiesced`。原生关闭失败或结果未确认时，不得确认挂起，且保留清理责任。Sandbox Provider 仍负责其 checkpoint 实现的文件系统刷新和计算停止保证；Runtime 静止本身不证明这些保证。 +挂起先关闭准入、排空已接纳工作与回执，同时保留已结算的空闲 Executor，随后才确认 `environment_quiesced`。活动、准备中、失效或其他未结算的所有者会阻止挂起。整个挂起期间停止空闲到期计时并隔离其回调；精确匹配且已认证的 `environment_resume` 为同一批所有者恢复一个新的空闲计时间隔。只有这种计划性重连跨 socket 保留 Router 与原生所有者；普通断连、shutdown 和连接生命周期取消仍要求确认清理完成。排空失败后，准入保持关闭直到 shutdown。恢复后的准备仍检查相同的原生身份与不可变配置，包括凭据冲突；绝不静默替换配置冲突的驻留 Executor。Sandbox Provider 仍负责其 checkpoint 实现的文件系统刷新和计算停止保证；Runtime 静止本身既不证明这些保证，也不证明原生 RAM 或外部网络连接已恢复。 ## 活动输入回执 {#active-input-receipts} diff --git a/internal/agentdaemon/proto/envelope_test.go b/internal/agentdaemon/proto/envelope_test.go index d485d8f7d..cec6e5d6d 100644 --- a/internal/agentdaemon/proto/envelope_test.go +++ b/internal/agentdaemon/proto/envelope_test.go @@ -88,6 +88,7 @@ func TestVersionCompatible(t *testing.T) { ok bool }{ {Version, true}, // exact match + {"0.15.0", false}, // closes idle Executors before suspension {"0.8.99", false}, // patch drift NOT OK {"0.6.99", false}, // minor drift NOT OK {"1.0.0", false}, // major drift NOT OK diff --git a/internal/agentdaemon/proto/suspend.go b/internal/agentdaemon/proto/suspend.go index 0e0ad7896..b9c4dede7 100644 --- a/internal/agentdaemon/proto/suspend.go +++ b/internal/agentdaemon/proto/suspend.go @@ -18,8 +18,9 @@ type EnvironmentSuspendPayload struct { } // An accepted quiesce confirms closed admission, drained work and receipts, -// and successful closure of idle native Executors. It does not confirm a -// filesystem checkpoint or that provider-owned compute has stopped. +// and retention of settled idle native Executors with idle expiry fenced until +// exact authenticated resume. It does not confirm a filesystem checkpoint or +// that provider-owned compute has stopped. type EnvironmentSuspendResultPayload struct { EnvironmentID string `json:"environment_id"` SuspendID string `json:"suspend_id"` diff --git a/internal/agentdaemon/proto/version.go b/internal/agentdaemon/proto/version.go index 114299f22..2dcbf7337 100644 --- a/internal/agentdaemon/proto/version.go +++ b/internal/agentdaemon/proto/version.go @@ -2,7 +2,7 @@ package proto // Version identifies the complete Core–Runtime wire contract. Change it when // removing or changing a payload or its semantics; deploy both endpoints together. -const Version = "0.15.0" +const Version = "0.16.0" // VersionCompatible accepts only this contract. Patch drift, prerelease suffixes // and malformed versions do not select an implicit compatibility path. From 4970339c828a81d02d40a336b2bb3d8a1b7ee3af Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 02:12:33 +0800 Subject: [PATCH 17/22] Upgrade microsandbox to the official 0.7.8 release (#652) --- Makefile | 10 +-- docs/maintainers.md | 8 +- docs/zh/maintainers.md | 10 +-- scripts/build-core-distribution.sh | 10 +-- scripts/build-microsandbox-provider.py | 90 +++++++++++++++++++ scripts/build-microsandbox-provider.test.py | 48 ++++++++++ scripts/ci_plan.py | 1 + scripts/ci_plan_test.py | 4 + scripts/core-distribution-manifest.py | 6 +- .../internal/sandbox/microsandbox/types.go | 2 +- .../tools/microsandbox-provider/README.md | 12 +-- .../tools/microsandbox-provider/bootstrap.go | 2 +- .../tools/microsandbox-provider/deployment.go | 2 +- .../core/tools/microsandbox-provider/go.mod | 2 +- .../core/tools/microsandbox-provider/go.sum | 4 +- .../core/tools/microsandbox-provider/main.go | 23 ++++- .../microsandbox-provider/sdk_release_test.go | 28 ++++++ 17 files changed, 223 insertions(+), 39 deletions(-) create mode 100644 scripts/build-microsandbox-provider.py create mode 100644 scripts/build-microsandbox-provider.test.py create mode 100644 services/core/tools/microsandbox-provider/sdk_release_test.go diff --git a/Makefile b/Makefile index 888e671c2..9de40a1a8 100644 --- a/Makefile +++ b/Makefile @@ -166,17 +166,13 @@ build-mcode-runtime: .PHONY: build-microsandbox-provider check-microsandbox-provider build-microsandbox-provider: - @test "$$(go env GOOS)" = linux || { echo 'The microsandbox provider helper requires Linux' >&2; exit 1; } - @set -e; output="$${OAC_DEV_HOME:-$$HOME/.oac}/build/microsandbox-provider"; \ - [[ "$$output" == /* ]] || { echo 'Provider output directory must be absolute' >&2; exit 1; }; \ - mkdir -p "$$output"; \ - cd services/core/tools/microsandbox-provider; \ - GOWORK=off CGO_ENABLED=1 go build -mod=readonly -trimpath -o "$$output/oac-microsandbox-provider" . + python3 scripts/build-microsandbox-provider.py build "$${OAC_DEV_HOME:-$$HOME/.oac}/build/microsandbox-provider/oac-microsandbox-provider" check-microsandbox-provider: + python3 scripts/build-microsandbox-provider.test.py go test -mod=readonly ./services/core/internal/sandbox/microsandbox/... -count=1 @if [[ "$$(go env GOOS)" == linux ]]; then \ - cd services/core/tools/microsandbox-provider && GOWORK=off CGO_ENABLED=1 go test -mod=readonly ./... -count=1; \ + python3 scripts/build-microsandbox-provider.py test; \ else \ printf 'Skipping the Linux-only microsandbox SDK helper tests; the full Linux gate is required before release.\n'; \ fi diff --git a/docs/maintainers.md b/docs/maintainers.md index d0154a883..c3c8f33fc 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -33,7 +33,7 @@ make build-core-distribution | `OAC_NATIVE_INSTALLER_BUILD_DIR` | Native installer catalog directory; see [Native installers](#native-installers) | | `CORE_DISTRIBUTION_BUILD_DIR` | Output directory under `~/.oac`. Default: `~/.oac/build/core-distribution` | | `CORE_DISTRIBUTION_BUILD_NETWORK` | Docker build network: `default`, `host` or `none` | -| `CORE_DISTRIBUTION_MICROSANDBOX_ARCHIVE` | Cached microsandbox release archive. Default: `~/.oac/cache/microsandbox-v0.7.2-linux-x86_64.tar.gz`, downloaded when missing | +| `CORE_DISTRIBUTION_MICROSANDBOX_ARCHIVE` | Cached microsandbox release archive. Default: `~/.oac/cache/microsandbox-v0.7.8-linux-x86_64.tar.gz`, downloaded when missing | | `CORE_DISTRIBUTION_DATABASE_IMAGE` | PostgreSQL 16 image; the default is pinned by its linux/amd64 manifest digest | The build reuses the Core, Web, Runtime, SDK and helper builders. The manifest records the commit and source tree, image config and OCI manifest digests, the Runtime OCI manifest digest, the microsandbox runtime and firmware hashes, and the size and SHA-256 of every Runtime and node artifact; native installers carry only their SHA-256 in the [catalog](#native-installers). Output is the control archive and its `.sha256`, the optional offline archive, and the versioned Runtime, node and native installer assets. Nothing is published. Rebuilding into a directory that already holds this commit's distribution is refused. @@ -103,16 +103,16 @@ make build-e2b-provider Docker builds the Linux helper for `GOARCH=amd64` (default) or `GOARCH=arm64` with the pinned CPython and Debian 12 image. The Python dependency closure, including PyInstaller, is hash-locked in `services/core/tools/e2b-provider/requirements.lock`; no E2B account key is needed. Set `E2B_PROVIDER_BUILD_DIR` for another output directory. The build is a pure function of the helper sources, `LICENSE` and the build script, so it is cached under `~/.oac/cache/e2b-provider/` by their hash and rebuilt only when they change. The output is `oac-e2b-provider-linux-.tar.gz` with its `.sha256`; it extracts to `oac-e2b-provider/` with the executable, `_internal/`, `licenses/`, `requirements.lock` and `manifest.json`. The Core image uses that tree; the host needs a compatible glibc and CA certificates, not Python. -**microsandbox helper.** Linux only, with a C compiler: +**microsandbox helper.** Linux amd64 only, with a C compiler: ```sh make build-microsandbox-provider make check-microsandbox-provider ``` -The helper is written to `~/.oac/build/microsandbox-provider/oac-microsandbox-provider`. Its separate Go module pins the microsandbox Go SDK v0.7.2 and embeds the matching FFI library; never build production with the SDK's `microsandbox_ffi_path` tag. Core itself stays a CGO-disabled build. The helper needs glibc and runs only on nodes. +The helper is written to `~/.oac/build/microsandbox-provider/oac-microsandbox-provider`. Its separate Go module pins the official microsandbox SDK source commit. Both commands and the distribution build use `scripts/build-microsandbox-provider.py`: it stages the module dependencies, verifies the matching official FFI release checksum, fills the SDK's empty release bundle and builds with that FFI embedded. The SDK source is unchanged and no vendored binary is committed. Use this entry point rather than invoking `go build` directly; never build production with the SDK's `microsandbox_ffi_path` tag. Core itself stays a CGO-disabled build. The helper needs glibc and runs only on nodes. -**microsandbox runtime.** The distribution uses the official [v0.7.2 release](https://github.com/superradcompany/microsandbox/releases/tag/v0.7.2) archive `microsandbox-linux-x86_64.tar.gz`, SHA256 `47c223e3ef5298abf05f47ed9f87981106e400d99bb3f1d042d4d6881346b18b` (`RUNTIME_ARCHIVE_SHA256` in `scripts/core-distribution-manifest.py`). The build verifies the checksum before extracting `msb` and `libkrunfw.so.5.6.1` and records both files' hashes. The helper checks those hashes on every call and never installs or upgrades them. +**microsandbox runtime.** The distribution uses the official [v0.7.8 release](https://github.com/superradcompany/microsandbox/releases/tag/v0.7.8) archive `microsandbox-linux-x86_64.tar.gz`, SHA256 `86f9f72dc3e639c7175bc07909b4b63ce412517c1ef8a2e1921171af5682fded` (`RUNTIME_ARCHIVE_SHA256` in `scripts/core-distribution-manifest.py`). The build verifies the checksum before extracting `msb` and `libkrunfw.so.5.6.1` and records both files' hashes. The helper checks those hashes on every call and never installs or upgrades them. ### Standalone Core builds diff --git a/docs/zh/maintainers.md b/docs/zh/maintainers.md index e61ec78da..de88bb348 100644 --- a/docs/zh/maintainers.md +++ b/docs/zh/maintainers.md @@ -1,7 +1,7 @@ --- title: "构建并发布 OpenAgentCore" source: docs/maintainers.md -source_hash: 494618f4d605d96bfcb9b1413f8224722324e1037764cc5b38baca102a8f1e90 +source_hash: f91505a6502e09cc13b5ece7cb4ba578f6d2861621d70a1f836f89dcf9299ea4 --- 本指南面向负责构建和发布 OpenAgentCore 的维护者。要安装 Core 和 Web，请使用 [安装指南](getting-started/install.md)。安装器代码遵循的规则见 [部署](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/README.md) 和 [节点安装器](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/node/README.md)；必需检查见 [CONTRIBUTING](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/CONTRIBUTING.md#required-checks)。 @@ -35,7 +35,7 @@ make build-core-distribution | `OAC_NATIVE_INSTALLER_BUILD_DIR` | 原生安装器目录；请参阅[原生安装器](#native-installers) | | `CORE_DISTRIBUTION_BUILD_DIR` | `~/.oac` 下的输出目录。默认值：`~/.oac/build/core-distribution` | | `CORE_DISTRIBUTION_BUILD_NETWORK` | Docker 构建网络：`default`、`host` 或 `none` | -| `CORE_DISTRIBUTION_MICROSANDBOX_ARCHIVE` | 已缓存的 microsandbox 发布归档。默认值：`~/.oac/cache/microsandbox-v0.7.2-linux-x86_64.tar.gz`，缺失时下载 | +| `CORE_DISTRIBUTION_MICROSANDBOX_ARCHIVE` | 已缓存的 microsandbox 发布归档。默认值：`~/.oac/cache/microsandbox-v0.7.8-linux-x86_64.tar.gz`，缺失时下载 | | `CORE_DISTRIBUTION_DATABASE_IMAGE` | PostgreSQL 16 镜像；默认值通过其 linux/amd64 清单摘要固定 | 构建过程会复用 Core、Web、Runtime、SDK 和辅助程序构建器。清单会记录提交和源代码树、镜像配置及 OCI 清单摘要、Runtime OCI 清单摘要、microsandbox 运行时和固件哈希，以及每个 Runtime 和节点构件的大小与 SHA-256；原生安装器在[目录](#native-installers)中仅记录其 SHA-256。输出包括控制归档及其 `.sha256`、可选的离线归档，以及带版本号的 Runtime、节点和原生安装器资源。此过程不会发布任何内容。如果目标目录中已包含此提交的分发包，重建会拒绝执行。 @@ -105,16 +105,16 @@ make build-e2b-provider Docker 使用固定版本的 CPython 和 Debian 12 镜像按 `GOARCH=amd64`（默认）或 `GOARCH=arm64` 构建 Linux 辅助程序。Python 依赖闭包（including PyInstaller）在 `services/core/tools/e2b-provider/requirements.lock` 中按哈希锁定；不需要 E2B 账户密钥。要使用其他输出目录，请设置 `E2B_PROVIDER_BUILD_DIR`。构建结果完全由辅助程序源代码、`LICENSE` 和构建脚本决定，因此会按它们的哈希缓存在 `~/.oac/cache/e2b-provider/` 下，仅在它们变化时重新构建。输出为 `oac-e2b-provider-linux-.tar.gz` 及其 `.sha256`；解压后会得到 `oac-e2b-provider/`，其中包含可执行文件、`_internal/`、`licenses/`、`requirements.lock` 和 `manifest.json`。Core 镜像使用该目录树；主机需要兼容的 glibc 和 CA 证书，而不需要 Python。 -**microsandbox 辅助程序。** 仅支持 Linux，并且需要 C 编译器： +**microsandbox 辅助程序。** 仅支持 Linux amd64，并且需要 C 编译器： ```sh make build-microsandbox-provider make check-microsandbox-provider ``` -该辅助程序会写入 `~/.oac/build/microsandbox-provider/oac-microsandbox-provider`。其独立的 Go 模块固定 microsandbox Go SDK v0.7.2，并嵌入匹配的 FFI 库；构建生产版本时，绝不能使用该 SDK 的 `microsandbox_ffi_path` 标签。Core 本身仍采用禁用 CGO 的构建。该辅助程序需要 glibc，并且只能在节点上运行。 +该辅助程序会写入 `~/.oac/build/microsandbox-provider/oac-microsandbox-provider`。其独立的 Go 模块固定官方 microsandbox SDK 源码提交。上述两个命令与分发构建均使用 `scripts/build-microsandbox-provider.py`：它暂存模块依赖，验证匹配的官方 FFI 发行文件校验和，填充 SDK 的空发行包，并构建嵌入该 FFI 的程序。SDK 源码保持不变，仓库不提交 vendored 二进制文件。请使用此入口，而非直接调用 `go build`；构建生产版本时，绝不能使用该 SDK 的 `microsandbox_ffi_path` 标签。Core 本身仍采用禁用 CGO 的构建。该辅助程序需要 glibc，并且只能在节点上运行。 -**microsandbox 运行时。** 分发包使用官方的 [v0.7.2 release](https://github.com/superradcompany/microsandbox/releases/tag/v0.7.2) 归档 `microsandbox-linux-x86_64.tar.gz`，SHA256 为 `47c223e3ef5298abf05f47ed9f87981106e400d99bb3f1d042d4d6881346b18b`（即 `scripts/core-distribution-manifest.py` 中的 `RUNTIME_ARCHIVE_SHA256`）。构建过程会先验证校验和，再解压 `msb` 和 `libkrunfw.so.5.6.1`，并记录这两个文件的哈希。辅助程序会在每次调用时检查这些哈希，并且绝不安装或升级它们。 +**microsandbox 运行时。** 分发包使用官方的 [v0.7.8 release](https://github.com/superradcompany/microsandbox/releases/tag/v0.7.8) 归档 `microsandbox-linux-x86_64.tar.gz`，SHA256 为 `86f9f72dc3e639c7175bc07909b4b63ce412517c1ef8a2e1921171af5682fded`（即 `scripts/core-distribution-manifest.py` 中的 `RUNTIME_ARCHIVE_SHA256`）。构建过程会先验证校验和，再解压 `msb` 和 `libkrunfw.so.5.6.1`，并记录这两个文件的哈希。辅助程序会在每次调用时检查这些哈希，并且绝不安装或升级它们。 ### 独立 Core 构建 {#standalone-core-builds} diff --git a/scripts/build-core-distribution.sh b/scripts/build-core-distribution.sh index ac40a609a..37da251f5 100755 --- a/scripts/build-core-distribution.sh +++ b/scripts/build-core-distribution.sh @@ -110,16 +110,12 @@ cp services/core/deploy/codex/seccomp.json "$bundle/runtime/" cp LICENSE "$bundle/" OAC_DEV_BUILD_REVISION="$revision" scripts/build-core-image-context.sh "$stage/core" -( - cd services/core/tools/microsandbox-provider - GOWORK=off CGO_ENABLED=1 go build -mod=readonly -trimpath \ - -o "$stage/core/bin/oac-microsandbox-provider" . -) -msb_archive="${CORE_DISTRIBUTION_MICROSANDBOX_ARCHIVE:-$runtime_root/cache/microsandbox-v0.7.2-linux-x86_64.tar.gz}" +python3 scripts/build-microsandbox-provider.py build "$stage/core/bin/oac-microsandbox-provider" +msb_archive="${CORE_DISTRIBUTION_MICROSANDBOX_ARCHIVE:-$runtime_root/cache/microsandbox-v0.7.8-linux-x86_64.tar.gz}" if [[ ! -f "$msb_archive" ]]; then mkdir -p "$(dirname "$msb_archive")" curl --fail --location --proto '=https' --tlsv1.2 \ - https://github.com/superradcompany/microsandbox/releases/download/v0.7.2/microsandbox-linux-x86_64.tar.gz \ + https://github.com/superradcompany/microsandbox/releases/download/v0.7.8/microsandbox-linux-x86_64.tar.gz \ --output "$stage/microsandbox.download" python3 scripts/core-distribution-manifest.py extract-runtime "$stage/microsandbox.download" "$stage/core/microsandbox" mv "$stage/microsandbox.download" "$msb_archive" diff --git a/scripts/build-microsandbox-provider.py b/scripts/build-microsandbox-provider.py new file mode 100644 index 000000000..2e5e31f90 --- /dev/null +++ b/scripts/build-microsandbox-provider.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Build the provider with the official SDK source and matching embedded FFI.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + +SDK_MODULE = "github.com/superradcompany/microsandbox/sdk/go" +FFI_SHA256 = "bc079888050a92d3652191ae8e18fba96e1ac66dc78bce4e5aaeb7eca9b04e25" +FFI_FILE = "libmicrosandbox_go_ffi-linux-amd64.so" + + +def verify_ffi(path): + with Path(path).open("rb") as stream: + digest = hashlib.sha256() + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + if digest.hexdigest() != FFI_SHA256: + raise ValueError("Official microsandbox FFI checksum mismatch") + + +def install_ffi(source, destination): + verify_ffi(source) + if destination.is_symlink() or not destination.is_file() or destination.stat().st_size != 0: + raise ValueError("Official SDK must contain an empty FFI release sentinel") + shutil.copyfile(source, destination) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("build", "test")) + parser.add_argument("output", nargs="?", type=Path) + args = parser.parse_args() + if (args.operation == "build") != (args.output is not None): + parser.error("build requires an absolute output path; test takes no output") + if args.output is not None and not args.output.is_absolute(): + parser.error("output must be absolute") + root = Path(__file__).resolve().parents[1] + module = root / "services/core/tools/microsandbox-provider" + env = dict(os.environ, GOWORK="off", CGO_ENABLED="1") + def run(*command, cwd=module): + try: + return subprocess.check_output(command, cwd=cwd, env=env, text=True) + except subprocess.CalledProcessError as error: + print(error.output, end="") + raise + if run("go", "env", "GOOS", "GOARCH").splitlines() != ["linux", "amd64"]: + parser.error("the provider distribution requires Linux amd64") + run("go", "mod", "download", SDK_MODULE) + sdk = json.loads(run("go", "list", "-mod=readonly", "-m", "-json", SDK_MODULE)) + if "Replace" in sdk: + raise ValueError("The SDK must be the pinned official module") + versions = re.findall(r'^const sdkVersion = "([0-9.]+)"$', (Path(sdk["Dir"]) / "setup.go").read_text(), re.M) + if len(versions) != 1: + raise ValueError("Official SDK release declaration changed") + version = versions[0] + cache = Path(os.environ.get("OAC_DEV_HOME", str(Path.home() / ".oac"))) / "cache" + if not cache.is_absolute(): + raise ValueError("OAC_DEV_HOME must be absolute") + cache.mkdir(parents=True, exist_ok=True) + ffi = cache / (FFI_SHA256 + ".so") + with tempfile.TemporaryDirectory(prefix="oac-microsandbox-build-", dir=cache) as temporary: + stage = Path(temporary) + if not ffi.exists(): + download = stage / FFI_FILE + run("curl", "--fail", "--location", "--silent", "--show-error", "--retry", "3", "--output", str(download), + f"https://github.com/superradcompany/microsandbox/releases/download/v{version}/{FFI_FILE}") + verify_ffi(download) + os.replace(download, ffi) + verify_ffi(ffi) + for source in [*module.glob("*.go"), module / "go.mod", module / "go.sum"]: + shutil.copyfile(source, stage / source.name) + run("go", "mod", "vendor", "-o", str(stage / "vendor")) + install_ffi(ffi, stage / "vendor" / SDK_MODULE / "internal/bundle/bundles" / FFI_FILE) + if args.operation == "test": + print(run("go", "test", "-mod=vendor", "./...", "-count=1", cwd=stage), end="") + print(run("go", "vet", "-mod=vendor", "./...", cwd=stage), end="") + else: + args.output.parent.mkdir(parents=True, exist_ok=True) + print(run("go", "build", "-mod=vendor", "-trimpath", "-o", str(args.output), ".", cwd=stage), end="") + + +if __name__ == "__main__": + main() diff --git a/scripts/build-microsandbox-provider.test.py b/scripts/build-microsandbox-provider.test.py new file mode 100644 index 000000000..d53f80ccd --- /dev/null +++ b/scripts/build-microsandbox-provider.test.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.util +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("build_provider", Path(__file__).with_name("build-microsandbox-provider.py")) +build = importlib.util.module_from_spec(spec) +spec.loader.exec_module(build) + + +class OfficialBundleTests(unittest.TestCase): + def test_only_verified_payload_replaces_empty_sentinel(self): + with tempfile.TemporaryDirectory() as directory: + source, destination = Path(directory) / "ffi", Path(directory) / "sentinel" + source.write_bytes(b"official ffi") + destination.touch() + with patch.object(build, "FFI_SHA256", hashlib.sha256(source.read_bytes()).hexdigest()): + build.install_ffi(source, destination) + self.assertEqual(destination.read_bytes(), source.read_bytes()) + with self.assertRaisesRegex(ValueError, "empty FFI"): + build.install_ffi(source, destination) + + def test_corrupt_payload_does_not_replace_sentinel(self): + with tempfile.TemporaryDirectory() as directory: + source, destination = Path(directory) / "ffi", Path(directory) / "sentinel" + source.write_bytes(b"wrong release") + destination.touch() + with self.assertRaisesRegex(ValueError, "checksum mismatch"): + build.install_ffi(source, destination) + self.assertEqual(destination.read_bytes(), b"") + + def test_symlink_sentinel_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + source, destination, outside = (Path(directory) / name for name in ("ffi", "sentinel", "outside")) + source.write_bytes(b"official ffi") + outside.touch() + destination.symlink_to(outside) + with patch.object(build, "FFI_SHA256", hashlib.sha256(source.read_bytes()).hexdigest()): + with self.assertRaisesRegex(ValueError, "empty FFI"): + build.install_ffi(source, destination) + self.assertEqual(outside.read_bytes(), b"") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci_plan.py b/scripts/ci_plan.py index 02f0f47f6..19a0fadd2 100644 --- a/scripts/ci_plan.py +++ b/scripts/ci_plan.py @@ -78,6 +78,7 @@ (("scripts/build-native-", "scripts/native-"), SCRIPTS, ("native", "backend", "distribution")), (("scripts/build-core.sh", "scripts/build-core-image-context.sh"), (".sh",), ("backend", "api", "distribution", "native")), (("deploy/distribution/",), ("Dockerfile",), ("backend", "api", "distribution", "native", "compose")), + (("scripts/build-microsandbox-provider.",), (".py",), ("backend", "distribution")), (("scripts/build-e2b-provider.sh",), (".sh",), ("backend", "api", "distribution")), (("scripts/build-claude", "scripts/check-claude", "scripts/build-mcode", "scripts/prepare-release-runtimes.sh"), SCRIPTS, ("harness", "native", "backend", "distribution")), diff --git a/scripts/ci_plan_test.py b/scripts/ci_plan_test.py index d5ee84204..40ec23043 100644 --- a/scripts/ci_plan_test.py +++ b/scripts/ci_plan_test.py @@ -57,6 +57,10 @@ def test_generated_outputs_keep_freshness_checks(self): with self.subTest(path=path): self.assertIn("distribution", self.jobs(path)) + def test_microsandbox_builder_keeps_helper_and_distribution_checks(self): + for path in ("scripts/build-microsandbox-provider.py", "scripts/build-microsandbox-provider.test.py"): + self.assertTrue({"backend", "distribution"} <= self.jobs(path)) + def test_distribution_image_build_keeps_api_acceptance(self): plan = ci.select(["scripts/build-core-distribution.sh"]) self.assertTrue(plan["image"]) diff --git a/scripts/core-distribution-manifest.py b/scripts/core-distribution-manifest.py index 3529fb8fe..cdb7e9843 100644 --- a/scripts/core-distribution-manifest.py +++ b/scripts/core-distribution-manifest.py @@ -17,7 +17,7 @@ import zipapp -RUNTIME_ARCHIVE_SHA256 = "47c223e3ef5298abf05f47ed9f87981106e400d99bb3f1d042d4d6881346b18b" +RUNTIME_ARCHIVE_SHA256 = "86f9f72dc3e639c7175bc07909b4b63ce412517c1ef8a2e1921171af5682fded" DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z") sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "deploy/node")) import provider_assets @@ -204,7 +204,7 @@ def verify_runtime(image, daemon, source): def extract_runtime(archive, destination): if sha256(archive) != RUNTIME_ARCHIVE_SHA256: - raise ValueError("microsandbox v0.7.2 release checksum mismatch") + raise ValueError("microsandbox v0.7.8 release checksum mismatch") destination = pathlib.Path(destination) destination.mkdir(parents=True, exist_ok=True) with tarfile.open(archive, "r:gz") as bundle: @@ -346,7 +346,7 @@ def node_payload(bundle, stage, revision, source_tree, artifact_base_url="", off "image_manifest_digests": {name: identity[1] for name, identity in identities.items()}, "runtime_ref": "oac-runtime@" + digest, "microsandbox": { - "version": "0.7.2", + "version": "0.7.8", "runtime_sha256": sha256(stage / "core/microsandbox/msb"), "firmware_sha256": sha256(stage / "core/microsandbox/libkrunfw.so.5.6.1"), }, diff --git a/services/core/internal/sandbox/microsandbox/types.go b/services/core/internal/sandbox/microsandbox/types.go index bb8d9246d..68eb8b3d3 100644 --- a/services/core/internal/sandbox/microsandbox/types.go +++ b/services/core/internal/sandbox/microsandbox/types.go @@ -10,7 +10,7 @@ import ( ) const ProtocolVersion = 4 -const SDKVersion = "v0.7.2" +const SDKVersion = "v0.7.8" const MaxOutputBytes = 1024 * 1024 const MaxRequestBytes = 72 * 1024 * 1024 const MaxResponseBytes = 16 * 1024 * 1024 diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index ac7e22c2c..243c6d682 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -1,6 +1,6 @@ # microsandbox Sandbox Provider helper -microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 4; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. +microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 4; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.8 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. [Add a Sandbox Provider](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources, Runtime release and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy. @@ -8,7 +8,7 @@ microsandbox runs each hosted Session in its own microVM on a Linux amd64 node w The [maintainer guide](../../../../docs/maintainers.md#runtime-images-and-helpers) builds the helper and packages the checksum-verified `msb` runtime and `libkrunfw` firmware. The node's provider configuration, written by the node installer, supplies the absolute helper, runtime and firmware paths with their SHA-256 values, the runtime home, the Runtime image reference, the saved resources and the host network policy. -Before every operation the helper checks that it was built with the published SDK module v0.7.2 without a replacement, that the runtime and firmware match their hashes, that the runtime's `.msbver` ELF section reports the SDK version and that the SDK resolves exactly those paths with the local backend ([`main.go`](main.go)). It never installs or upgrades these files; keep them unchanged for the lifetime of the provider's backend. Ambient SDK profiles are ignored. +Before every operation the helper checks that it was built with the exact official SDK module declared in its embedded `go.mod` without a replacement, that the runtime and firmware match their hashes, that the runtime's `.msbver` ELF section reports the declared native release, that the SDK source and embedded FFI report that release, and that the SDK resolves exactly those paths with the local backend ([`main.go`](main.go)). It never installs or upgrades these files; keep them unchanged for the lifetime of the provider's backend. Ambient SDK profiles are ignored. The runtime home must be private (mode 0700), short, on local persistent storage and used by no other installation, profile or manual lifecycle tool. microsandbox uses Unix sockets there, so the node installer refuses a home whose path would exceed their limit. The home holds confidential VM disks, memory snapshots and SDK state; preserve it with the node identity and Core's database when recovering a host. Every managed lifecycle change goes through the provider. @@ -16,11 +16,11 @@ The Runtime image is an immutable `repository@sha256:<64 lowercase hex>` referen ## Create and bootstrap -Create names the VM from a hash of the installation and allocation reference plus the compute generation; a name never serves another incarnation. It creates the VM with the saved CPUs and memory as both initial and maximum, a managed root disk of `root_disk_mib`, an owned ext4 disk of `environment_disk_mib` or the explicitly resolved external directory mounted at `/environment`, user 1000:1000, working directory `/` and the node's network policy ([`bootstrap.go`](bootstrap.go)). The resource checks run before bootstrap. +Create names the VM from a hash of the installation and allocation reference plus the compute generation; a name never serves another incarnation. It creates the VM with the saved CPUs and memory as both initial and maximum, a managed root disk of `root_disk_mib`, an owned ext4 disk of `environment_disk_mib` or the explicitly resolved external directory mounted at `/environment`, user 1000:1000, working directory `/` and the node's network policy ([`bootstrap.go`](bootstrap.go)). The resource checks run before bootstrap. Creation and restore retain the SDK's strict hostname policy enforcement; the adapter does not disable it. -Workspace, staging and outputs share the `/environment` filesystem, which keeps the Runtime's cross-device and link checks intact; the layered root filesystem can report different device IDs for a directory and its upper-layer files, so it holds no workspace data. In owned mode, full snapshots and sandbox removal capture, restore and reclaim this disk. In external mode, the filesystem provider owns the directory; sandbox removal never deletes it. The binding and resource rules belong to [Independent workspace attachment](../../../../docs/sandbox-provider.md#independent-workspace-attachment). Private HOME and history remain in the checkpointed root. +Workspace, staging and outputs share the `/environment` filesystem, which keeps the Runtime's cross-device and link checks intact; the layered root filesystem can report different device IDs for a directory and its upper-layer files, so it holds no workspace data. In owned mode, full snapshots and sandbox removal capture, restore and reclaim this disk. In external mode, the filesystem provider owns the directory; sandbox removal never deletes it. The binding and resource rules belong to [Independent workspace attachment](../../../../docs/sandbox-provider.md#independent-workspace-attachment). The [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) owns the Runtime directory layout. -VM creation does not run the image's entry point. The bootstrap runs as root through confidential standard input, with a two-minute limit. It creates the Runtime directories and the private control directory `/run/oac` (mode 0700, owned by UID 1000), writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json`, bind-mounts `/environment/workspace` at `/workspace` and starts `oac-daemon connect --bootstrap-file` in the background as UID/GID 1000. `OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE=/run/oac/daemon-suspend.json` enables the daemon's park and wake control. The `io.oac.bootstrap` label then changes from `pending` to `complete` through the SDK's next-start modification policy, because v0.7.2 cannot update the labels of a running VM. That label confirms only these writes and the launch, not authentication or native readiness. +VM creation does not run the image's entry point. The bootstrap runs as root through confidential standard input, with a two-minute limit. It creates the Runtime directories and the private control directory `/run/oac` (mode 0700, owned by UID 1000), writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json`, bind-mounts `/environment/workspace` at `/workspace` and starts `oac-daemon connect --bootstrap-file` in the background as UID/GID 1000. `OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE=/run/oac/daemon-suspend.json` enables the daemon's park and wake control. The `io.oac.bootstrap` label then changes from `pending` to `complete` through the SDK's next-start modification policy, because v0.7.8 cannot update the labels of a running VM. That label confirms only these writes and the launch, not authentication or native readiness. A helper response carries `CreateSettled` with a configuration rejection only after native Create has completed, the first inspection has verified the exact compute ID and ownership, and the resource check has rejected the VM before bootstrap started. The adapter keeps the original error and validates the compute identity before passing the proof to Core. The private `initial_info` operation used by `GetInfo` and `Renew` can also return an exact initial `State` with `Status="absent"`, no native ID and `CreateSettled=true`, after typed native absence and durable closure of initial Create admission under the allocation lock. It accepts only generation zero without restored ancestry or a native ID. The adapter validates the complete receipt and returns absence without an error. Ordinary `inspect` used by `GetCompute`, uncertain Create outcomes, timeouts and ownership failures never produce this receipt; missing compute alone is not settlement. @@ -57,7 +57,7 @@ The command checks the protected Environment and suspension identities and the p ## Metrics -The read-only metrics operation holds the allocation lock and verifies the exact compute ID through the SDK before and after it runs `msb metrics NAME --format json` with the same pinned runtime binary ([`metrics.go`](metrics.go)). The CLI report keeps the native sample timestamp and fractional-second uptime, so the helper reconstructs one run start consistently across polls and Core restarts, and a new run gets a new start. The Go SDK's projection drops the timestamp and truncates uptime to whole seconds, so it cannot provide this; sandbox creation time is not a run start time. In the pinned source (`v0.7.2`, commit `1c59b8dbf0ad47dda2f807c0214b529aceb81c74`), `crates/metrics/lib/registry.rs` reads `sampled_at_unix_ms` and `started_at_unix_ms` together and subtracts them for uptime, and `crates/cli/lib/commands/metrics.rs` serializes the timestamp and `uptime.as_secs_f64()`. +The read-only metrics operation holds the allocation lock and verifies the exact compute ID through the SDK before and after it runs `msb metrics NAME --format json` with the same pinned runtime binary ([`metrics.go`](metrics.go)). The CLI report keeps the native sample timestamp and fractional-second uptime, so the helper reconstructs one run start consistently across polls and Core restarts, and a new run gets a new start. The Go SDK's projection drops the timestamp and truncates uptime to whole seconds, so it cannot provide this; sandbox creation time is not a run start time. In the pinned source (`v0.7.8`, commit `7b7b9dc89e9a1c77801918f7833c3381d1754579`), `crates/metrics/lib/registry.rs` reads `sampled_at_unix_ms` and `started_at_unix_ms` together and subtracts them for uptime, and `crates/cli/lib/commands/metrics.rs` serializes the timestamp and `uptime.as_secs_f64()`. The helper returns only the native observation time, exact uptime, cumulative vCPU time and guest memory usage and limit. It rejects stale or exited reports and missing or malformed fields, bounds the CLI output and reports failures only as an unavailable code, never native diagnostics. Metrics never connect to the guest, renew activity, resume paused compute or change lifecycle state. [Runtime observability](../../../../contracts/agents-api/runtime-observability.md) owns the mapping to observations. diff --git a/services/core/tools/microsandbox-provider/bootstrap.go b/services/core/tools/microsandbox-provider/bootstrap.go index d1753e4f4..79125a12d 100644 --- a/services/core/tools/microsandbox-provider/bootstrap.go +++ b/services/core/tools/microsandbox-provider/bootstrap.go @@ -107,7 +107,7 @@ func (b backend) create(ctx context.Context) (wire.Response, error) { return wire.Response{}, sandbox.ErrCommandUnconfirmed } // Persist the final bootstrap receipt without restarting the live guest. - // v0.7.2 cannot update active labels; ownership reads persisted config. + // v0.7.8 cannot update active labels; ownership reads persisted config. _, e = live.Modify(ctx, sdk.ModifyOptions{Labels: map[string]string{bootstrapLabel: "complete"}, Policy: sdk.ModificationPolicyNextStart}) if e != nil { return wire.Response{}, e diff --git a/services/core/tools/microsandbox-provider/deployment.go b/services/core/tools/microsandbox-provider/deployment.go index c950588aa..c768e0e59 100644 --- a/services/core/tools/microsandbox-provider/deployment.go +++ b/services/core/tools/microsandbox-provider/deployment.go @@ -31,7 +31,7 @@ func resourceProof(config wire.Config) string { return hex.EncodeToString(digest[:]) } -// The SDK decodes native CPU/memory/rootfs configuration. Its v0.7.2 projection +// The SDK decodes native CPU/memory/rootfs configuration. Its v0.7.8 projection // omits mounts, so the explicitly owned or external inventory is projected here. // Restored roots have no configured size: a verified full snapshot supplies that // proof, retained as a label only after inspecting the restored target. diff --git a/services/core/tools/microsandbox-provider/go.mod b/services/core/tools/microsandbox-provider/go.mod index 25d0423c5..c04558e14 100644 --- a/services/core/tools/microsandbox-provider/go.mod +++ b/services/core/tools/microsandbox-provider/go.mod @@ -4,7 +4,7 @@ go 1.26.8 require ( github.com/MiniMax-AI/OpenAgentCore v0.0.0 - github.com/superradcompany/microsandbox/sdk/go v0.7.2 + github.com/superradcompany/microsandbox/sdk/go v0.0.0-20261009155513-7b7b9dc89e9a ) require ( diff --git a/services/core/tools/microsandbox-provider/go.sum b/services/core/tools/microsandbox-provider/go.sum index 77cb4e04c..a92270ad0 100644 --- a/services/core/tools/microsandbox-provider/go.sum +++ b/services/core/tools/microsandbox-provider/go.sum @@ -1,6 +1,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/superradcompany/microsandbox/sdk/go v0.7.2 h1:aO70srBRgu50rNZTtzQTOO3xxPSthkZLkq3kjWvmsqg= -github.com/superradcompany/microsandbox/sdk/go v0.7.2/go.mod h1:p7Tm/p9zkO7sHO3N8A1fiTVeLB2w/fQoKdYlpN/5neA= +github.com/superradcompany/microsandbox/sdk/go v0.0.0-20261009155513-7b7b9dc89e9a h1:r9mZpz3z3E5tXRWDulZSs0gyM2EEhsTUqUjyZQ6MT4w= +github.com/superradcompany/microsandbox/sdk/go v0.0.0-20261009155513-7b7b9dc89e9a/go.mod h1:p7Tm/p9zkO7sHO3N8A1fiTVeLB2w/fQoKdYlpN/5neA= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= diff --git a/services/core/tools/microsandbox-provider/main.go b/services/core/tools/microsandbox-provider/main.go index c16a7d8da..b7dcb9a5a 100644 --- a/services/core/tools/microsandbox-provider/main.go +++ b/services/core/tools/microsandbox-provider/main.go @@ -8,6 +8,7 @@ import ( "context" "crypto/sha256" "debug/elf" + _ "embed" "encoding/hex" "encoding/json" "errors" @@ -25,6 +26,9 @@ import ( sdk "github.com/superradcompany/microsandbox/sdk/go" ) +//go:embed go.mod +var moduleDefinition string + func main() { // The inherited node generation lease covers this helper's actual lifetime. // Set CLOEXEC before SDK initialization or any possible subprocess spawn so @@ -207,6 +211,16 @@ func allocationLock(q wire.Request) (*allocationGuard, error) { time.Sleep(20 * time.Millisecond) } } +func sdkModuleVersion() string { + for _, line := range strings.Split(moduleDefinition, "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && fields[0] == "github.com/superradcompany/microsandbox/sdk/go" { + return fields[1] + } + } + return "" +} + func checkInstallation(c wire.Config) error { build, ok := debug.ReadBuildInfo() if !ok { @@ -215,7 +229,7 @@ func checkInstallation(c wire.Config) error { matched := false for _, d := range build.Deps { if d.Path == "github.com/superradcompany/microsandbox/sdk/go" { - matched = d.Version == wire.SDKVersion && d.Replace == nil + matched = d.Version == sdkModuleVersion() && d.Replace == nil } } if !matched { @@ -269,6 +283,13 @@ func checkInstallation(c wire.Config) error { if runtime.MSBPath != c.RuntimePath || runtime.LibkrunfwPath != c.FirmwarePath { return sandbox.ErrOwnership } + nativeVersion, e := sdk.RuntimeVersion() + if e != nil { + return e + } + if sdk.SDKVersion() != strings.TrimPrefix(wire.SDKVersion, "v") || nativeVersion != sdk.SDKVersion() { + return sandbox.ErrInvalid + } selected, e := sdk.DefaultBackendInfo() if e != nil { return e diff --git a/services/core/tools/microsandbox-provider/sdk_release_test.go b/services/core/tools/microsandbox-provider/sdk_release_test.go new file mode 100644 index 000000000..7d528949f --- /dev/null +++ b/services/core/tools/microsandbox-provider/sdk_release_test.go @@ -0,0 +1,28 @@ +//go:build linux + +package main + +import ( + "strings" + "testing" + + wire "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" + sdk "github.com/superradcompany/microsandbox/sdk/go" +) + +func TestOfficialSDKReleaseMatchesEmbeddedFFI(t *testing.T) { + t.Setenv("MSB_HOME", t.TempDir()) + if sdkModuleVersion() == "" { + t.Fatal("missing pinned SDK module identity") + } + if sdk.SDKVersion() != strings.TrimPrefix(wire.SDKVersion, "v") { + t.Fatal("SDK source does not match admitted native release") + } + version, err := sdk.RuntimeVersion() + if err != nil { + t.Fatal(err) + } + if version != sdk.SDKVersion() { + t.Fatalf("embedded FFI %q does not match SDK %q", version, sdk.SDKVersion()) + } +} From 2bba238a3efd85390e4e0d4a7d4fdb51eb1e89e4 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 03:22:25 +0800 Subject: [PATCH 18/22] Allow reconnect after settled initialization uncertainty (#658) --- .../dispatch/runtime_preparation_test.go | 35 ++++++++++++++----- apps/daemon/internal/dispatch/shutdown.go | 6 ++-- docs/runtime-protocol.md | 2 +- docs/zh/runtime-protocol.md | 4 +-- .../runtime_initialization_test.go | 18 ++++++++-- 5 files changed, 49 insertions(+), 16 deletions(-) diff --git a/apps/daemon/internal/dispatch/runtime_preparation_test.go b/apps/daemon/internal/dispatch/runtime_preparation_test.go index 90a0c418e..ba8631a86 100644 --- a/apps/daemon/internal/dispatch/runtime_preparation_test.go +++ b/apps/daemon/internal/dispatch/runtime_preparation_test.go @@ -246,6 +246,7 @@ func TestRuntimePreparationUploadBlocksWorkspaceWriteAndSuspension(t *testing.T) func TestRuntimePreparationCancellationKeepsOwnershipUntilApplyStops(t *testing.T) { r, sender, environment, session := capabilitiesTestRouter(t) + sender.frames = make(chan proto.Envelope) ctx, cancel := context.WithCancel(context.Background()) id := uuid.NewString() request := proto.RuntimePreparePayload{BudgetMS: 300000, Step: "begin", Action: "finalize", EnvironmentID: environment, SessionID: session, Sources: &agentcapabilities.Input{}} @@ -263,7 +264,10 @@ func TestRuntimePreparationCancellationKeepsOwnershipUntilApplyStops(t *testing. <-ctx.Done() close(interrupted) <-release - return os.WriteFile(retained, []byte("retained"), 0400) + if err := os.WriteFile(retained, []byte("retained"), 0400); err != nil { + return err + } + return context.Canceled }) <-started wait, stop := context.WithTimeout(context.Background(), 20*time.Millisecond) @@ -280,13 +284,22 @@ func TestRuntimePreparationCancellationKeepsOwnershipUntilApplyStops(t *testing. t.Fatal("cancel released unsettled capability ownership") } close(release) + wait, stop = context.WithTimeout(context.Background(), 20*time.Millisecond) + err = r.Shutdown(wait) + stop() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("shutdown abandoned pending result delivery: %v", err) + } + capabilitiesReceipt(t, sender, id, "unknown") shutdownCapabilitiesRouter(t, r) - capabilitiesReceipt(t, sender, id, "completed") if _, err := os.Stat(retained); err != nil { t.Fatal("shutdown deleted installation result") } - if r.runtimePreparation != nil { - t.Fatal("confirmed completion retained capacity") + if r.runtimePreparation != owner || !owner.uncertain { + t.Fatal("shutdown erased unknown outcome") + } + if err := r.Handle(t.Context(), capabilityEnvelope(t, uuid.NewString(), request)); !errors.Is(err, ErrRouterClosed) { + t.Fatalf("closed router admitted a successor: %v", err) } } @@ -341,10 +354,16 @@ func TestRuntimePreparationResultCategoriesAndUnknownOwnership(t *testing.T) { if !owned { t.Fatal("unknown mutation released its ownership") } - wait, stop := context.WithTimeout(context.Background(), time.Second) - defer stop() - if err := r.Shutdown(wait); err == nil { - t.Fatal("shutdown claimed uncertain mutation settled") + next := uuid.NewString() + if err := r.Handle(t.Context(), capabilityEnvelope(t, next, request)); err != nil { + t.Fatal(err) + } + if got := capabilitiesReceipt(t, sender, next, "rejected"); got.ErrorCode != "runtime_preparation_capacity" { + t.Fatalf("unknown operation lost its admission fence: %+v", got) + } + shutdownCapabilitiesRouter(t, r) + if r.runtimePreparation != owner || !owner.uncertain { + t.Fatal("shutdown erased unknown outcome") } } diff --git a/apps/daemon/internal/dispatch/shutdown.go b/apps/daemon/internal/dispatch/shutdown.go index b09de4c70..ec476d1f1 100644 --- a/apps/daemon/internal/dispatch/shutdown.go +++ b/apps/daemon/internal/dispatch/shutdown.go @@ -69,9 +69,9 @@ func (r *Router) runShutdownAttempt(attempt *shutdownAttempt, victims []sessionC r.shutdownWG.Wait() r.mu.Lock() - if r.runtimePreparation != nil && r.runtimePreparation.uncertain { - attempt.err = errors.Join(attempt.err, errors.New("dispatch: capability preparation remains uncertain")) - } + // Runtime preparation joins only after ApplyRuntimePreparation stops local + // mutations and its receipt send finishes. An unknown result still fences + // this closed Router, but is not outstanding cleanup. Core owns no-replay. if r.workspaceWrite != nil && r.workspaceWrite.uncertain { attempt.err = errors.Join(attempt.err, errors.New("dispatch: local workspace write remains uncertain")) } diff --git a/docs/runtime-protocol.md b/docs/runtime-protocol.md index f892f27a6..d87df0a68 100644 --- a/docs/runtime-protocol.md +++ b/docs/runtime-protocol.md @@ -124,7 +124,7 @@ A preparation reserves a per-Turn admission, not a new Executor. It carries an e Preparation and start run outside the receive loop and router lock. An admission expires five minutes after it is granted, and retries do not extend that deadline; expiry does not remove the Runtime's obligation to settle cleanup. The Runtime bounds active preparation and execution separately from idle retained resources and counts closing or uncertain resources until their cleanup succeeds. A definite `execution_prepare` rejection with `preparation_capacity` leaves the queued Turn unclaimed for the Worker to retry, including when cleanup holds the capacity; any other error or uncertain delivery authorizes no replay. The Runtime retains at most 64 admission records, and an old handle never consumes a replacement's admission. These records are connection-local, not durable input replay. -Idle expiry of an Executor is a Runtime resource policy, separate from Core's active-Turn concurrency. On shutdown the Runtime closes active and idle Executors, keeps any target whose close failed and allows a later serialized retry. An ordinary disconnection closes the failed transport and keeps the exact router until shutdown succeeds; a wait timeout or failed cleanup never authorizes reconnection, and process shutdown keeps waiting rather than discarding owned native resources. Workspace operations keep their binding and settlement rules across Turn boundaries and Executor closure. +Idle expiry of an Executor is a Runtime resource policy, separate from Core's active-Turn concurrency. On shutdown the Runtime closes active and idle Executors, keeps any target whose close failed and allows a later serialized retry. An ordinary disconnection closes the failed transport and keeps the exact router until shutdown succeeds; a wait timeout or failed cleanup never authorizes reconnection, and process shutdown keeps waiting rather than discarding owned native resources. For Runtime initialization, shutdown joins the synchronous apply operation and its receipt delivery before releasing the connection; an `unknown` result remains unknown and blocks further preparation on that Router, but does not prevent reconnection after local mutations have stopped. Core retains the failed initialization and never replays it. Workspace operations keep their binding and settlement rules across Turn boundaries and Executor closure. Suspension closes admission and drains admitted work and receipts while retaining settled idle Executors before acknowledging `environment_quiesced`. Active, preparing, invalid or otherwise unsettled owners prevent suspension. Idle expiry is stopped and its callbacks fenced throughout suspension; exact authenticated `environment_resume` restores a fresh idle interval on the same owners. Only this planned reconnect preserves the Router and native owners across sockets; ordinary disconnect, shutdown and cancellation of the connection lifecycle still require confirmed cleanup. A drain failure keeps admission closed until shutdown. Preparation after resume retains the same native identity and immutable-configuration checks, including credential conflicts; it never silently replaces a conflicting resident Executor. The Sandbox Provider remains responsible for the filesystem flush and compute-stop guarantees of its checkpoint implementation; Runtime quiescence alone proves neither those guarantees nor restoration of native RAM or external network connections. diff --git a/docs/zh/runtime-protocol.md b/docs/zh/runtime-protocol.md index 279231ac7..3500b87df 100644 --- a/docs/zh/runtime-protocol.md +++ b/docs/zh/runtime-protocol.md @@ -1,7 +1,7 @@ --- title: "Core–Runtime 协议" source: docs/runtime-protocol.md -source_hash: 902f53e5471de13ecac0e914e4d23d7650955a4c50ec1aa01517c13aef7946f7 +source_hash: 8349aca995e66669b7941b6aad5f28e6ae3259516d923a0494938a61fb3e711e --- 此协议在 Runtime daemon 获取机器凭据后连接 Core 与 daemon，定义 daemon 连接上消息的含义和顺序。wire 类型、限制和验证器仅在 [`internal/agentdaemon/proto`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/internal/agentdaemon/proto) 中定义一次；Core 的 [gateway](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/runtimegateway) 与参考 Runtime 的 [dispatcher](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/apps/daemon/internal/dispatch) 都使用它们，因此无需同步第二套 payload schema。签发凭据和打开连接的 HTTP 路由见[机器连接 API](../../contracts/agents-api/zh/machine-api.md)。 @@ -126,7 +126,7 @@ preparation 预约每个 Turn 的准入，而不是新 Executor。它携带明 preparation 和 start 在 receive loop 与 router lock 之外运行。admission 在授予五分钟后到期，重试不延长截止时间；到期不解除 Runtime 完成清理结算的义务。Runtime 分别限制活动 preparation、execution 和保留的空闲资源，关闭中或不确定资源持续计入限制，直到清理成功。明确的 `execution_prepare` 拒绝若为 `preparation_capacity`，会让排队 Turn 保持未领取，供 Worker 重试，包括清理占用容量的情况；其他错误或不确定交付都不授权重放。Runtime 最多保留 64 条 admission 记录，旧 handle 不会消耗替代项的 admission。这些记录仅属于连接，不是持久化输入重放。 -Executor 空闲到期属于 Runtime 资源策略，与 Core 的活动 Turn 并发限制独立。关闭时 Runtime 关闭活动和空闲 Executor，保留关闭失败的目标，并允许稍后串行重试。普通断连会关闭失败的 transport 并保留原 router，直到 shutdown 成功；等待超时或清理失败不授权重连，进程 shutdown 继续等待，不丢弃自己拥有的原生资源。工作区操作在跨 Turn 和 Executor 关闭后仍保留绑定与结算规则。 +Executor 空闲到期属于 Runtime 资源策略，与 Core 的活动 Turn 并发限制独立。关闭时 Runtime 关闭活动和空闲 Executor，保留关闭失败的目标，并允许稍后串行重试。普通断连会关闭失败的 transport 并保留原 router，直到 shutdown 成功；等待超时或清理失败不授权重连，进程 shutdown 继续等待，不丢弃自己拥有的原生资源。对于 Runtime 初始化，shutdown 会等待同步 apply 操作和回执发送结束后才释放连接；`unknown` 结果仍是不确定结果，并阻止该 Router 上的后续准备，但本地变更停止后不再阻止重连。Core 保留失败的初始化状态，绝不重放初始化。工作区操作在跨 Turn 和 Executor 关闭后仍保留绑定与结算规则。 挂起先关闭准入、排空已接纳工作与回执，同时保留已结算的空闲 Executor，随后才确认 `environment_quiesced`。活动、准备中、失效或其他未结算的所有者会阻止挂起。整个挂起期间停止空闲到期计时并隔离其回调；精确匹配且已认证的 `environment_resume` 为同一批所有者恢复一个新的空闲计时间隔。只有这种计划性重连跨 socket 保留 Router 与原生所有者；普通断连、shutdown 和连接生命周期取消仍要求确认清理完成。排空失败后，准入保持关闭直到 shutdown。恢复后的准备仍检查相同的原生身份与不可变配置，包括凭据冲突；绝不静默替换配置冲突的驻留 Executor。Sandbox Provider 仍负责其 checkpoint 实现的文件系统刷新和计算停止保证；Runtime 静止本身既不证明这些保证，也不证明原生 RAM 或外部网络连接已恢复。 diff --git a/services/core/tests/integration/runtime_initialization_test.go b/services/core/tests/integration/runtime_initialization_test.go index 636474b84..b50a949c5 100644 --- a/services/core/tests/integration/runtime_initialization_test.go +++ b/services/core/tests/integration/runtime_initialization_test.go @@ -121,8 +121,22 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { if int(p.writes.Load()) != expectedSteps { t.Fatal("completed preparation replayed") } - } else if p.writes.Load() != 1 { - t.Fatal("unknown operation replayed", p.writes.Load()) + } else { + if p.writes.Load() != 1 { + t.Fatal("unknown operation replayed", p.writes.Load()) + } + stop() + _, _ = managedWorkerMode(t, s, key, p, true) + if err := p.connect(sandbox.Bootstrap{DeviceID: owner.DeviceID, Credential: p.credential}); err != nil { + t.Fatal(err) + } + time.Sleep(350 * time.Millisecond) + if p.writes.Load() != 1 || initializationState(t, s, tenant, env.ID) != "failed" { + t.Fatal("reconnect or Worker restart replayed unknown initialization") + } + if _, err := sessionAdapter(s).GetSessionExecutionBinding(t.Context(), tenant, session.ID); !errors.Is(err, sessions.ErrNotFound) { + t.Fatal("unknown initialization admitted execution", err) + } } p.mu.Lock() kills := p.kills From 951b7555e3361b38c476cd57d8faabf6fb172bd5 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 03:25:09 +0800 Subject: [PATCH 19/22] Classify workspace storage exhaustion as unavailable (#657) * Classify workspace storage exhaustion as unavailable * Run storage exhaustion assertions in native platform CI --- .../internal/localworkspace/native_files.go | 17 +++++++++-- .../localworkspace/native_write_error_unix.go | 12 ++++++++ .../native_write_error_unix_test.go | 28 +++++++++++++++++++ .../native_write_error_windows.go | 11 ++++++++ .../native_write_error_windows_test.go | 20 +++++++++++++ contracts/agents-api/environment-files.md | 2 +- contracts/agents-api/zh/environment-files.md | 4 +-- ...onment_file_write_semantics_public_test.go | 16 +++++++++-- 8 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 apps/daemon/internal/localworkspace/native_write_error_unix.go create mode 100644 apps/daemon/internal/localworkspace/native_write_error_unix_test.go create mode 100644 apps/daemon/internal/localworkspace/native_write_error_windows.go create mode 100644 apps/daemon/internal/localworkspace/native_write_error_windows_test.go diff --git a/apps/daemon/internal/localworkspace/native_files.go b/apps/daemon/internal/localworkspace/native_files.go index e669bc66a..7748ca783 100644 --- a/apps/daemon/internal/localworkspace/native_files.go +++ b/apps/daemon/internal/localworkspace/native_files.go @@ -115,12 +115,12 @@ func (b *Binding) writeNativeFile(ctx context.Context, path string, data []byte) } defer root.Close() if err = root.MkdirAll(filepath.Dir(local), 0700); err != nil { - return result, agent.ErrWorkspaceWriteRejected + return result, unpublishedWriteError(err) } temporary := ".oac-write-" + uuid.NewString() file, err := root.OpenFile(temporary, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) if err != nil { - return result, agent.ErrWorkspaceWriteRejected + return result, unpublishedWriteError(err) } defer root.Remove(temporary) _, err = file.Write(data) @@ -129,7 +129,7 @@ func (b *Binding) writeNativeFile(ctx context.Context, path string, data []byte) } closeErr := file.Close() if err != nil || closeErr != nil { - return result, agent.ErrWorkspaceWriteRejected + return result, unpublishedWriteError(errors.Join(err, closeErr)) } // Link publishes complete bytes without replacing an existing destination. if err = root.Link(temporary, local); err != nil { @@ -138,12 +138,23 @@ func (b *Binding) writeNativeFile(ctx context.Context, path string, data []byte) return result, agent.ErrWorkspaceWriteDirectory } return result, agent.ErrWorkspaceWriteUnsafe + } else if errors.Is(e, fs.ErrNotExist) { + return result, unpublishedWriteError(err) } return result, agent.ErrWorkspaceWriteRejected } return agent.WorkspaceWriteResult{SizeBytes: int64(len(data))}, nil } +// unpublishedWriteError applies only before this operation publishes a file. +// Parent directories or a temporary file may already have been created. +func unpublishedWriteError(err error) error { + if storageExhausted(err) { + return agent.ErrWorkspaceWriteUnavailable + } + return agent.ErrWorkspaceWriteRejected +} + const artifactFileBytes int64 = 200 << 20 const artifactBatchBytes int64 = 500 << 20 const artifactEntries = 4096 diff --git a/apps/daemon/internal/localworkspace/native_write_error_unix.go b/apps/daemon/internal/localworkspace/native_write_error_unix.go new file mode 100644 index 000000000..6469b1b69 --- /dev/null +++ b/apps/daemon/internal/localworkspace/native_write_error_unix.go @@ -0,0 +1,12 @@ +//go:build linux || darwin + +package localworkspace + +import ( + "errors" + "syscall" +) + +func storageExhausted(err error) bool { + return errors.Is(err, syscall.ENOSPC) || errors.Is(err, syscall.EDQUOT) +} diff --git a/apps/daemon/internal/localworkspace/native_write_error_unix_test.go b/apps/daemon/internal/localworkspace/native_write_error_unix_test.go new file mode 100644 index 000000000..9c4b62370 --- /dev/null +++ b/apps/daemon/internal/localworkspace/native_write_error_unix_test.go @@ -0,0 +1,28 @@ +//go:build linux || darwin + +package localworkspace + +import ( + "errors" + "fmt" + "os" + "syscall" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent" +) + +func TestNativeUnpublishedWriteStorageFailure(t *testing.T) { + for _, cause := range []error{syscall.ENOSPC, syscall.EDQUOT} { + for _, err := range []error{cause, &os.PathError{Op: "write", Path: "private/path", Err: cause}, fmt.Errorf("sync failed: %w", cause), errors.Join(syscall.EIO, cause)} { + if got := unpublishedWriteError(err); got != agent.ErrWorkspaceWriteUnavailable { + t.Fatalf("storage exhaustion: got %v, want unavailable", got) + } + } + } + for _, cause := range []error{syscall.EIO, syscall.EACCES, syscall.EEXIST, syscall.ENOTDIR} { + if got := unpublishedWriteError(&os.PathError{Op: "write", Path: "private/path", Err: cause}); got != agent.ErrWorkspaceWriteRejected { + t.Fatalf("changed ordinary rejection: %v", got) + } + } +} diff --git a/apps/daemon/internal/localworkspace/native_write_error_windows.go b/apps/daemon/internal/localworkspace/native_write_error_windows.go new file mode 100644 index 000000000..b577072d4 --- /dev/null +++ b/apps/daemon/internal/localworkspace/native_write_error_windows.go @@ -0,0 +1,11 @@ +package localworkspace + +import ( + "errors" + + "golang.org/x/sys/windows" +) + +func storageExhausted(err error) bool { + return errors.Is(err, windows.ERROR_DISK_FULL) || errors.Is(err, windows.ERROR_HANDLE_DISK_FULL) || errors.Is(err, windows.ERROR_DISK_QUOTA_EXCEEDED) +} diff --git a/apps/daemon/internal/localworkspace/native_write_error_windows_test.go b/apps/daemon/internal/localworkspace/native_write_error_windows_test.go new file mode 100644 index 000000000..c250a0cd3 --- /dev/null +++ b/apps/daemon/internal/localworkspace/native_write_error_windows_test.go @@ -0,0 +1,20 @@ +package localworkspace + +import ( + "os" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent" + "golang.org/x/sys/windows" +) + +func TestNativeUnpublishedWriteStorageFailure(t *testing.T) { + for _, cause := range []error{windows.ERROR_DISK_FULL, windows.ERROR_HANDLE_DISK_FULL, windows.ERROR_DISK_QUOTA_EXCEEDED} { + if got := unpublishedWriteError(&os.PathError{Op: "write", Path: "private/path", Err: cause}); got != agent.ErrWorkspaceWriteUnavailable { + t.Fatalf("storage exhaustion: got %v, want unavailable", got) + } + } + if got := unpublishedWriteError(windows.ERROR_ACCESS_DENIED); got != agent.ErrWorkspaceWriteRejected { + t.Fatalf("changed ordinary rejection: %v", got) + } +} diff --git a/contracts/agents-api/environment-files.md b/contracts/agents-api/environment-files.md index 783da51fe..4f00b04f9 100644 --- a/contracts/agents-api/environment-files.md +++ b/contracts/agents-api/environment-files.md @@ -77,7 +77,7 @@ Unless the table names a code, the 400 errors have type and code `invalid_reques - Before sending any bytes, Core records the write under the Session lock. While input is pending, a Turn is running or an earlier write is unsettled, a new write returns 409 `turn_conflict`. An unsettled write also makes new messages to the Session return 409. - The Runtime checks the complete body against its digest before creating anything, so incomplete input creates nothing. A write that fails later can leave newly created empty parent directories. -- Only a definite receipt from the Runtime settles a write, as committed or rejected. A rejected write changes nothing and releases the Session. If the connection drops, the request times out or no receipt arrives, the request returns 503 and the write stays unsettled, across Core restarts. Core never resends it and has no automatic recovery, so the Session accepts no further writes or messages. Reads still work. +- Only a definite receipt from the Runtime settles a write, as committed or rejected. A rejected write publishes no destination file and releases the Session; newly created empty parent directories can remain. Storage or quota exhaustion confirmed before publication returns 503, while malformed input and destination conflicts retain their existing errors. If the connection drops, the request times out or no receipt arrives, the request returns 503 and the write stays unsettled, across Core restarts. Core never resends it and has no automatic recovery, so the Session accepts no further writes or messages. Reads still work. - Deleting the source File after its bytes were read does not affect the copy. ## Artifacts diff --git a/contracts/agents-api/zh/environment-files.md b/contracts/agents-api/zh/environment-files.md index 1977f2b63..8f31c84e2 100644 --- a/contracts/agents-api/zh/environment-files.md +++ b/contracts/agents-api/zh/environment-files.md @@ -1,7 +1,7 @@ --- title: "Environment 文件与 Artifact" source: contracts/agents-api/environment-files.md -source_hash: 1b58aa02aaccddb9675ef41ebfe2506a6fba0bb12139efb67e0da378d879aee7 +source_hash: f87f06138789666b91140c15ffd104cffba4566580921b94b094bf31591d201d --- Session 工作区保存由 agent 及其工具修改的实时文件。`/agents/environments/{environment_id}/files` 列出一个工作区目录，并在其中创建文件。Turn 完成时，Core 将工作区 `outputs/` 目录中的文件复制为不可变 Artifact，通过 `/agents/sessions/{session_id}/artifacts` 读取。Artifact 的生命周期长于 Environment；工作区文件则不是。 @@ -79,7 +79,7 @@ Session 工作区保存由 agent 及其工具修改的实时文件。`/agents/en - 发送任何字节之前，Core 在 Session 锁下记录写入。输入待处理、Turn 运行或较早写入未结算时，新写入返回 409 `turn_conflict`。未结算写入也使 Session 新消息返回 409。 - Runtime 在创建任何内容前根据摘要检查完整正文，因此不完整输入不创建内容。后续失败的写入可能留下新建空父目录。 -- 仅 Runtime 的确定回执将写入结算为已提交或已拒绝。被拒绝写入不改变内容并释放 Session。连接断开、请求超时或无回执时，返回 503，写入持续未结算，Core 重启后仍如此。Core 不重发，也无自动恢复，因此 Session 不再接受写入或消息。读取仍可用。 +- 仅 Runtime 的确定回执将写入结算为已提交或已拒绝。被拒绝写入不会发布目标文件，并释放 Session；新建的空父目录可能保留。确认在发布前发生的存储或配额耗尽返回 503，格式错误的输入和目标冲突仍保持原有错误。连接断开、请求超时或无回执时，返回 503，写入持续未结算，Core 重启后仍如此。Core 不重发，也无自动恢复，因此 Session 不再接受写入或消息。读取仍可用。 - 源 File 字节读取后，删除该 File 不影响副本。 ## Artifact {#artifacts} diff --git a/services/core/tests/integration/environment_file_write_semantics_public_test.go b/services/core/tests/integration/environment_file_write_semantics_public_test.go index eef998402..c8e66c22e 100644 --- a/services/core/tests/integration/environment_file_write_semantics_public_test.go +++ b/services/core/tests/integration/environment_file_write_semantics_public_test.go @@ -59,7 +59,7 @@ func TestEnvironmentFileCreateRejectionsLeaveNoReceiptOrConsumption(t *testing.T {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: h.tenant}, {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "tenant-b", TokenSHA256: runtimedevice.HashCredential(other), TenantID: uuid.NewString()}, }) - handler, err := publicHandler(t, h.s, auth, "codex", workerExecution(t, w)) + handler, err := publicHandler(t, h.s, auth, "codex", workerExecution(t, w), acceptUnavailable(t)) if err != nil { t.Fatal(err) } @@ -144,6 +144,16 @@ func TestEnvironmentFileCreateRejectionsLeaveNoReceiptOrConsumption(t *testing.T t.Fatal("rejection did not settle", intent, err) } } + // Exhaustion rejected before publication is unavailable, not invalid input; + // its exact receipt still settles ownership so another write can proceed. + doneUnavailable := post(token, environment.ID, inline) + unavailableID := serveFileWrite(h, proto.WorkspaceWriteResultPayload{Outcome: "rejected", ErrorCode: "resource_unavailable"}) + if got := await(doneUnavailable); got.status != http.StatusServiceUnavailable { + t.Fatal("resource rejection became invalid input", got) + } + if intent, err := FixtureFileWrite(t.Context(), h.s.pool, h.tenant, environment.ID, unavailableID); err != nil || intent.State != "rejected" { + t.Fatal("known unavailable result retained uncertainty", intent, err) + } // The rejected copy did not consume its Source File. if got, err := fileStore.Get(t.Context(), h.tenant, source.ID); err != nil || got.SizeBytes != 3 { t.Fatal("source file consumed", got, err) @@ -156,7 +166,7 @@ func TestEnvironmentFileCreateRejectionsLeaveNoReceiptOrConsumption(t *testing.T if foreign.status != 404 || foreign != missing { t.Fatal("tenant B reached the Environment", foreign, missing) } - if got := states(); !reflect.DeepEqual(got, map[string]int{"rejected": 3}) { + if got := states(); !reflect.DeepEqual(got, map[string]int{"rejected": 4}) { t.Fatal("rejections left a receipt or blocking intent", got) } // Known rejections release the mutation owner for a successor. @@ -165,7 +175,7 @@ func TestEnvironmentFileCreateRejectionsLeaveNoReceiptOrConsumption(t *testing.T if got := await(done); got.status != 201 { t.Fatal("successor rejected", got) } - if got := states(); !reflect.DeepEqual(got, map[string]int{"rejected": 3, "committed": 1}) { + if got := states(); !reflect.DeepEqual(got, map[string]int{"rejected": 4, "committed": 1}) { t.Fatal("successor receipt", got) } session, err := sessionAdapter(h.s).GetSession(t.Context(), h.tenant, h.session.ID) From 601da24829386c03bd06b3557f3bd1afd09d33ee Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sun, 11 Oct 2026 03:57:47 +0800 Subject: [PATCH 20/22] Restore retained checkpoints across compatible nodes (#655) * Restore retained checkpoints across compatible nodes * Align checkpoint fixtures with explicit restore admission * Wake checkpoint destinations and settle canceled restore preparation * Close expired restore requests before native dispatch * Accept checkpoint qualification in node health JSON * Document checkpoint compatibility during node maintenance --- .../agents-api/node-generation-protocol.md | 4 +- contracts/agents-api/sandbox-deployment.md | 2 +- .../agents-api/zh/node-generation-protocol.md | 6 +- contracts/agents-api/zh/sandbox-deployment.md | 4 +- deploy/node/node_generations.py | 4 + deploy/node/node_install.py | 56 +- deploy/node/test_node_generations.py | 2 +- deploy/node/test_node_install.py | 31 +- docs/getting-started/nodes.md | 9 +- docs/getting-started/operations.md | 2 + docs/sandbox-provider.md | 16 +- docs/zh/getting-started/nodes.md | 11 +- docs/zh/getting-started/operations.md | 4 +- docs/zh/sandbox-provider.md | 18 +- services/core/cmd/sandbox-node/main.go | 8 +- services/core/cmd/server/managed_nodes.go | 2 +- .../internal/db/queries/node_generations.sql | 16 +- .../db/queries/runtime_suspension.sql | 21 + .../queries/runtime_suspension_pressure.sql | 19 +- services/core/internal/db/sqlc/models.go | 1 + .../internal/db/sqlc/node_generations.sql.go | 44 +- .../db/sqlc/runtime_suspension.sql.go | 64 ++ .../sqlc/runtime_suspension_pressure.sql.go | 37 +- .../core/internal/deployment/allocations.go | 81 ++- .../internal/deployment/allocations_test.go | 45 +- services/core/internal/deployment/nodes.go | 22 +- .../deployment/placement/placement.go | 54 +- .../deployment/placement/placement_test.go | 20 +- services/core/internal/deployment/storage.go | 10 +- .../deployment/suspension_pressure.go | 49 +- .../internal/execution/runtime_compute.go | 49 +- .../execution/runtime_compute_wake.go | 19 +- .../internal/execution/runtime_lifecycle.go | 1 + .../internal/execution/runtime_manager.go | 15 +- .../execution/runtime_observation_test.go | 2 +- .../execution/runtime_replacement_test.go | 10 +- .../execution/runtime_restore_attempt_test.go | 238 ++++++++ .../execution/runtime_wake_hint_test.go | 29 + .../postgres/deploymentpg/allocations.go | 27 +- .../deploymentpg/checkpoint_transfer_test.go | 301 ++++++++++ .../postgres/deploymentpg/fixture_test.go | 23 +- .../deploymentpg/host_history_test.go | 12 +- .../postgres/deploymentpg/presence_test.go | 8 +- .../deploymentpg/suspension_pressure.go | 30 +- .../deploymentpg/suspension_pressure_test.go | 7 +- .../persistence/postgres/deploymentpg/tx.go | 9 +- .../postgres/placementpg/placementpg.go | 41 +- services/core/internal/sandbox/docker/node.go | 4 +- services/core/internal/sandbox/generation.go | 9 +- .../sandbox/microsandbox/checkpoint_linux.go | 133 +++++ .../microsandbox/checkpoint_linux_test.go | 88 +++ .../sandbox/microsandbox/checkpoint_other.go | 10 + .../internal/sandbox/microsandbox/identity.go | 11 +- .../internal/sandbox/microsandbox/node.go | 38 +- .../sandbox/microsandbox/node_test.go | 10 +- .../internal/sandbox/microsandbox/provider.go | 7 + .../sandbox/microsandbox/provider_test.go | 4 +- .../microsandbox/restore_settlement_test.go | 41 ++ .../internal/sandbox/microsandbox/types.go | 3 +- .../node/checkpoint_generation_test.go | 98 ++++ .../internal/sandbox/node/generation_json.go | 10 +- .../sandbox/node/generation_json_test.go | 64 +- .../internal/sandbox/node/generation_wire.go | 6 +- .../core/internal/sandbox/node/generations.go | 12 +- .../internal/sandbox/node/generations_test.go | 12 +- services/core/internal/sandbox/node/hub.go | 6 +- services/core/internal/sandbox/node/proxy.go | 6 + services/core/internal/sandbox/node/wire.go | 2 +- .../internal/sandbox/node/workspace_test.go | 2 +- .../sandbox/providers/generation_test.go | 2 +- .../core/internal/sandbox/sandbox_provider.go | 34 +- .../000099_checkpoint_compatibility.sql | 6 + .../integration/deployment_fixture_test.go | 15 + .../runtime_compute_lifecycle_test.go | 10 +- .../runtime_node_generations_test.go | 8 +- .../runtime_node_lifecycle_fixture_test.go | 6 +- .../tests/integration/runtime_nodes_test.go | 20 +- .../integration/runtime_suspension_test.go | 13 +- .../sandbox_deployment_worker_test.go | 2 +- .../session_execution_configuration_test.go | 5 + .../tools/microsandbox-provider/README.md | 20 +- .../tools/microsandbox-provider/archive.go | 494 ++++++++++++++++ .../tools/microsandbox-provider/backend.go | 25 +- .../tools/microsandbox-provider/lock_test.go | 23 +- .../core/tools/microsandbox-provider/main.go | 15 +- .../restore_completion_test.go | 6 +- .../tools/microsandbox-provider/snapshot.go | 169 ++++-- .../snapshot_observation_test.go | 550 ++++++++++++++++-- 88 files changed, 3097 insertions(+), 385 deletions(-) create mode 100644 services/core/internal/execution/runtime_restore_attempt_test.go create mode 100644 services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go create mode 100644 services/core/internal/sandbox/microsandbox/checkpoint_linux.go create mode 100644 services/core/internal/sandbox/microsandbox/checkpoint_linux_test.go create mode 100644 services/core/internal/sandbox/microsandbox/checkpoint_other.go create mode 100644 services/core/internal/sandbox/microsandbox/restore_settlement_test.go create mode 100644 services/core/internal/sandbox/node/checkpoint_generation_test.go create mode 100644 services/core/migrations/000099_checkpoint_compatibility.sql create mode 100644 services/core/tools/microsandbox-provider/archive.go diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index de65b9069..3b19e7fe6 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -73,9 +73,9 @@ Node startup and generation loading validate complete Provider operation declara ## Generation control -A node without generation management serves only its enrolled generation, with fixed configuration, and receives no preparation or retention frames. A generation-managing node prepares the target generation Core announces in `welcome` and `heartbeat_ack` and keeps serving its durable serving generation while it does; target preparation is independent of the serving provider's readiness. +A node without generation management serves only its enrolled generation, with fixed configuration, and receives no preparation or retention frames. It can report that exact generation in `Health.Generations`; a checkpoint-capable provider must do so, including its verified checkpoint qualification. `provider_ready` and the reported state must agree. A generation-managing node prepares the target generation Core announces in `welcome` and `heartbeat_ack` and keeps serving its durable serving generation while it does; target preparation is independent of the serving provider's readiness. -Its `hello` and heartbeats carry at most eight generation observations. Each names a positive signed-64-bit generation, its lowercase SHA-256 specification digest, a `ready`, `preparing` or `failed` state and an optional fixed diagnostic. The target and serving generations come first; other records rotate fairly. Eight bounds one message, not the number of generations a node may keep. An omitted observation never authorizes deletion or implies absence. +Its `hello` and heartbeats carry at most eight generation observations. Each names a positive signed-64-bit generation, its lowercase SHA-256 specification digest, a `ready`, `preparing` or `failed` state and an optional fixed diagnostic. The target and serving generations come first; other records rotate fairly. Eight bounds one message, not the number of generations a node may keep. An omitted observation never authorizes deletion or implies absence. A ready checkpoint-capable generation includes `checkpoint` with nonempty opaque `artifact_domain` and `execution_class` tokens, each at most 256 bytes. Unsupported providers omit it; preparing or failed generations never advertise it. Core persists the declaration with the authenticated connection and generation, and uses it only while that generation is ready on the current online connection. The [Provider contract](../../docs/sandbox-provider.md#checkpoint-transfer) defines archive and execution compatibility. Retention uses its own bounded exchange. A `retention` request names at most eight local `(generation, specification_digest)` references, a UUID, a sequence that increases by one, the current `connection_id` and the owner epoch. The `retention_ack` must match the complete pending request, entry order and identity included, and give an explicit boolean `keep` for every entry. One exchange is pending per connection, and a disconnect discards it. An unsolicited, replayed, stale, partial or mixed acknowledgement deletes nothing. Retention traffic never uses the Provider request queue. diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index b3b79721e..41668d90e 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -142,7 +142,7 @@ A node is online while it is connected under the current owner epoch with a hear Each node adds `rollout: {state, ready_generation, diagnostic?}`, where `ready_generation` is the nullable durable serving pin and `diagnostic` a fixed code for the target generation; allocation items add `deployment_generation`. Poll every five seconds only while `rollout.state` is `preparing` or `reset` is not null; old Sessions and failed, update-required or offline nodes alone do not keep polling active. -Node-backed creation validates the target deployment and commits the Session, pending Environment and any initial input without reserving compute. A full, offline or preparing fleet leaves that accepted work waiting; an unconfigured deployment, reset or unsupported combination still rejects admission. The common scheduler uses bounded rotating scans of unplaced demand, with pages ordered by recorded time and Environment ID and a fixed time boundary for each scan so continuous arrivals cannot prevent it from revisiting older work. It checks online presence, exact serving-generation readiness, address, shared capacity and the selected generation's Harness/filesystem compatibility, then prefers the newest qualifying pin. Placement is immutable once reserved. A bounded scan skips temporarily unavailable demand and resumes after a restart. Existing suspended allocations restore on their original node through the same capacity lock; this is not a global fairness guarantee across hot restores and unplaced work. Input retains its [original five-minute deadline](./environments.md#reservations), including time waiting for capacity. +Node-backed creation validates the target deployment and commits the Session, pending Environment and any initial input without reserving compute. A full, offline or preparing fleet leaves that accepted work waiting; an unconfigured deployment, reset or unsupported combination still rejects admission. The common scheduler uses bounded rotating scans of unplaced demand, with pages ordered by recorded time and Environment ID and a fixed time boundary for each scan so continuous arrivals cannot prevent it from revisiting older work. It checks online presence, exact serving-generation readiness, address, shared capacity and the selected generation's Harness/filesystem compatibility, then prefers the newest qualifying pin. Placement is immutable once reserved until confirmed release or a settled checkpoint transfer. A bounded scan skips temporarily unavailable demand and resumes after a restart. Existing suspended allocations reserve eligible checkpoint capacity through the same lock under the [checkpoint transfer contract](../../docs/sandbox-provider.md#checkpoint-transfer); this is not a global fairness guarantee across hot restores and unplaced work. Input retains its [original five-minute deadline](./environments.md#reservations), including time waiting for capacity. ## Reset diff --git a/contracts/agents-api/zh/node-generation-protocol.md b/contracts/agents-api/zh/node-generation-protocol.md index 481dbc4af..b035ab7ab 100644 --- a/contracts/agents-api/zh/node-generation-protocol.md +++ b/contracts/agents-api/zh/node-generation-protocol.md @@ -1,7 +1,7 @@ --- title: "沙箱节点协议" source: contracts/agents-api/node-generation-protocol.md -source_hash: 3fa1d9257acefa2d144f6e76ac5c2d3ff1d074ebb2b21fb48a2f79ef58ad6073 +source_hash: fb446e91c3df3c9e397e5e9abb794245b0ae11d81176879207718680e8ab4173 --- 沙箱节点在其主机上运行 Docker 或 microsandbox Provider，并通过一个 WebSocket 与 Core 相连。Core 通过该连接发送 Provider 操作；节点针对本地 Provider 执行这些操作，并报告就绪状态、主机测量值及其持有的部署代次。Core 始终是唯一的生命周期所有者：节点绝不重试变更操作或调度工作。帧和校验器位于 [`services/core/internal/sandbox/node`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/sandbox/node)（`wire.go`、`generation_wire.go`）；节点用于注册和读取配置的 HTTP 路由位于[机器连接 API](machine-api.md#node-routes)。 @@ -75,9 +75,9 @@ Core 发送包含以下内容的 `request` 帧： ## 代次控制 {#generation-control} -未启用代次管理的节点只服务其登记的代次，配置固定，并且不会收到准备或保留帧。支持代次管理的节点会准备 Core 在 `welcome` 和 `heartbeat_ack` 中通告的目标代次，并在此期间继续服务其持久化的服务代次；目标代次的准备独立于服务 Provider 的就绪状态。 +未启用代次管理的节点只服务其登记的代次，配置固定，并且不会收到准备或保留帧。它可以在 `Health.Generations` 报告该精确代次；支持检查点的 Provider 必须报告，并包含已验证的检查点资格。`provider_ready` 必须与报告的状态一致。支持代次管理的节点会准备 Core 在 `welcome` 和 `heartbeat_ack` 中通告的目标代次，并在此期间继续服务其持久化的服务代次；目标代次的准备独立于服务 Provider 的就绪状态。 -节点的 `hello` 和心跳最多携带八条代次观察记录。每条记录指定一个正值的有符号 64 位代次编号、其小写 SHA-256 规范摘要、`ready`、`preparing` 或 `failed` 状态，以及可选的固定诊断信息。目标代次和服务代次的记录排在前面，其余记录公平轮换。八条记录限制的是单条消息，而不是节点可保留的代次数量。省略某条观察记录绝不会授权删除，也不会暗示不存在。 +节点的 `hello` 和心跳最多携带八条代次观察记录。每条记录指定一个正值的有符号 64 位代次编号、其小写 SHA-256 规范摘要、`ready`、`preparing` 或 `failed` 状态，以及可选的固定诊断信息。目标代次和服务代次的记录排在前面，其余记录公平轮换。八条记录限制的是单条消息，而不是节点可保留的代次数量。省略某条观察记录绝不会授权删除，也不会暗示不存在。 支持检查点的 ready 代次携带 `checkpoint`，包含非空且不透明的 `artifact_domain` 和 `execution_class` token，各最多 256 字节。不支持的 Provider 省略该字段；preparing 或 failed 代次不声明它。Core 将声明与已认证连接和代次一起持久化，仅在当前在线连接的该代次仍 ready 时使用。[Provider 契约](../../../docs/zh/sandbox-provider.md#checkpoint-transfer) 定义归档与执行兼容性。 保留使用独立且有界的交换。`retention` 请求最多指定八个本地 `(generation, specification_digest)` 引用、一个 UUID、一个每次递增 1 的 `sequence`、当前 `connection_id` 和所有者 epoch。`retention_ack` 必须与完整的待处理请求匹配，包括条目顺序和身份信息，并为每个条目给出显式布尔值 `keep`。每条连接只能有一个交换处于待处理状态，断连会将其丢弃。任何未经请求、重放、过期、不完整或混合的确认都不会删除任何内容。保留流量从不使用 Provider 请求队列。 diff --git a/contracts/agents-api/zh/sandbox-deployment.md b/contracts/agents-api/zh/sandbox-deployment.md index db6d81c6c..05fbe1fc8 100644 --- a/contracts/agents-api/zh/sandbox-deployment.md +++ b/contracts/agents-api/zh/sandbox-deployment.md @@ -1,7 +1,7 @@ --- title: "沙箱部署" source: contracts/agents-api/sandbox-deployment.md -source_hash: 245f1791cf2e42b3a40c91aa0df4f390a44709db5e0ffbcae1e12dc4170344fe +source_hash: 537ae957a77d625b3adb7828156352a661ae09270b57b174406c6db159a957bd --- 沙箱部署为 Core 管理的 `openai_hosted` 执行选择 Sandbox Provider、每个沙箱的资源以及不可变的 Runtime 发行版。PostgreSQL 为每个安装维护一个当前有效选择；Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径，且不能覆盖其资源或 Runtime。该选择独立于 Harness。部署可以保持未配置状态，没有节点；此时它拒绝托管准入。 @@ -145,7 +145,7 @@ POST 会在持久保存候选配置之前对其进行验证，并且不会创建 每个节点会添加 `rollout: {state, ready_generation, diagnostic?}`，其中 `ready_generation` 是可为 null 的持久服务固定状态，`diagnostic` 是目标代次的固定代码；分配项会添加 `deployment_generation`。仅当 `rollout.state` 为 `preparing` 或 `reset` 非 null 时，才每五秒轮询一次；旧 Session 以及失败、需要更新或离线的节点本身均不会使轮询保持活动状态。 -节点模式创建先验证目标部署，提交 Session、pending Environment 及初始输入，但不预留计算资源。节点全部已满、离线或准备中时，已接受的工作继续等待；未配置部署、reset 或不支持的组合仍拒绝准入。共同调度器对尚未放置的需求进行有界循环扫描，分页按需求记录时间和 Environment ID 排序，每轮使用固定时间上界，避免持续的新需求阻止重新访问较早的工作。它检查在线状态、精确服务代次就绪情况、地址、共享容量和所选代次的 Harness/文件系统兼容性，然后优先选择最新的合格固定代次。预留后的 placement 保持不可变。有界扫描跳过暂时不可调度的需求，并在重启后继续。已有 suspended allocation 通过同一容量锁在原节点恢复；这不构成热恢复和未放置工作之间的全局公平性保证。输入保留[原始五分钟期限](./environments.md#reservations)，等待容量的时间也计入其中。 +节点模式创建先验证目标部署，提交 Session、pending Environment 及初始输入，但不预留计算资源。节点全部已满、离线或准备中时，已接受的工作继续等待；未配置部署、reset 或不支持的组合仍拒绝准入。共同调度器对尚未放置的需求进行有界循环扫描，分页按需求记录时间和 Environment ID 排序，每轮使用固定时间上界，避免持续的新需求阻止重新访问较早的工作。它检查在线状态、精确服务代次就绪情况、地址、共享容量和所选代次的 Harness/文件系统兼容性，然后优先选择最新的合格固定代次。预留后的 placement 保持不可变，直到确认释放或已结清的检查点转移。有界扫描跳过暂时不可调度的需求，并在重启后继续。已有 suspended allocation 按[检查点转移契约](../../../docs/zh/sandbox-provider.md#checkpoint-transfer)，通过同一容量锁预留符合条件的检查点容量；这不构成热恢复和未放置工作之间的全局公平性保证。输入保留[原始五分钟期限](./environments.md#reservations)，等待容量的时间也计入其中。 ## 重置 {#reset} diff --git a/deploy/node/node_generations.py b/deploy/node/node_generations.py index 70e61fe0d..d056421f0 100644 --- a/deploy/node/node_generations.py +++ b/deploy/node/node_generations.py @@ -239,6 +239,9 @@ def validate_preparation_plan(root, plan, base, installer): if not any(paths == [release / name for name in installer.MICRO] for release in (root, root / "releases" / source)): raise installer.InstallError("Preparation artifacts are outside their immutable release") expected_home = generation_home(root, plan, base, installer) + if micro["checkpoint_root"] != base["native"]["checkpoint_root"]: + raise installer.InstallError("Preparation checkpoint store differs") + installer.prepare_checkpoint_root(Path(micro["checkpoint_root"]), initialize=False) if Path(micro["runtime_home"]) != expected_home: raise installer.InstallError("Preparation native store differs") for path in paths + [expected_home]: @@ -399,6 +402,7 @@ def prepare(args, installer): raise installer.RuntimeDownloadError("Runtime release provenance differs") from error if args.provider == "microsandbox": args.runtime_home = Path(value["native"]["runtime_home"]) if value else generation_home(root, args.configuration, base, installer) + args.checkpoint_root = Path(base["native"]["checkpoint_root"]) if value is None: value = installer.provider_config(root / "releases" / runtime["source_commit"], args, runtime["image_id"]) if preparation is None: diff --git a/deploy/node/node_install.py b/deploy/node/node_install.py index d563a29cf..5cda1e2fc 100644 --- a/deploy/node/node_install.py +++ b/deploy/node/node_install.py @@ -279,6 +279,49 @@ def micro_home(installation_id): return directory +def checkpoint_root(args): + return getattr(args, "checkpoint_root", None) or Path.home() / ".oac/checkpoints" / hashlib.sha256(args.installation_id.encode()).hexdigest()[:12] + + +def prepare_checkpoint_root(directory, *, initialize=True): + directory = Path(directory) + if not directory.is_absolute() or directory.resolve() != directory: + raise InstallError("Checkpoint root must be a canonical absolute directory") + if not initialize and not directory.is_dir(): + raise InstallError("Retained checkpoint root is missing") + safe_directory(directory) + marker = directory / ".oac-checkpoint-store" + if not existing_file(marker): + if not initialize: + raise InstallError("Retained checkpoint store identity is missing") + if any(directory.iterdir()): + raise InstallError("Checkpoint root contains unowned state") + try: + descriptor = os.open(marker, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + except FileExistsError: + pass + else: + with os.fdopen(descriptor, "w") as output: + output.write(str(uuid.uuid4()) + "\n") + output.flush() + os.fsync(output.fileno()) + existing_file(marker) + value = marker.read_text() + try: + identity = uuid.UUID(value.rstrip("\n")) + except ValueError: + raise InstallError("Invalid checkpoint store identity") from None + if identity.int == 0 or value != str(identity) + "\n": + raise InstallError("Invalid checkpoint store identity") + with marker.open("rb") as source: + os.fsync(source.fileno()) + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + def provider_config(root, args, runtime_image): result = {"installation_id": args.installation_id, "provider": args.provider, "core_url": args.core_url + "/api/v1", "specification": args.configuration["specification"], "generation": args.configuration["generation"]} @@ -294,6 +337,7 @@ def provider_config(root, args, runtime_image): result["native"] = { "helper_path": str(root / MICRO[0]), "runtime_path": str(root / MICRO[1]), "firmware_path": str(root / MICRO[2]), "runtime_home": str(getattr(args, "runtime_home", micro_home(args.installation_id))), + "checkpoint_root": str(checkpoint_root(args)), "network": {"default_egress": "deny", "default_ingress": "deny", "rules": core_rules + [ {"action": "allow", "direction": "egress", "destination": "public"}, {"action": "allow", "direction": "egress", "destination": "host", "protocol": "udp", "port": "53"}, @@ -348,8 +392,17 @@ def configure_node(root, args, token): args.configuration = node_spec.fetch(args, token, retained, open_request, allow_enrollment=not (root / "registered.json").exists()) args.provider = args.configuration["provider"] preflight(args.provider) + if args.provider != "microsandbox" and getattr(args, "checkpoint_root", None) is not None: + raise InstallError("Checkpoint root applies only to microsandbox nodes") if args.provider == "microsandbox": + retained_provider = private_json(root / "provider.json") + if retained_provider: + retained_root = Path(retained_provider["native"]["checkpoint_root"]) + if getattr(args, "checkpoint_root", None) not in (None, retained_root): + raise InstallError("Retained checkpoint root differs; preserve its state") + args.checkpoint_root = retained_root runtime_home = micro_home(args.installation_id) + prepare_checkpoint_root(checkpoint_root(args), initialize=retained_provider is None) safe_directory(runtime_home) owner = runtime_home / "oac-installation.json" if not owner.exists() and any(runtime_home.iterdir()): @@ -1244,6 +1297,7 @@ def main(argv=None): parser.add_argument("--core-url", type=origin) parser.add_argument("--provider", choices=("docker", "microsandbox"), help="Optional assertion; Core owns provider selection") parser.add_argument("--installation-id", required=True) + parser.add_argument("--checkpoint-root", type=Path, help="Private microsandbox checkpoint directory, shared across compatible nodes when configured") parser.add_argument("--enrollment-token-stdin", action="store_true", help="Read the one-time enrollment token from standard input") parser.add_argument("--generation-action", choices=("prepare", "collect"), help=argparse.SUPPRESS) parser.add_argument("--generation", type=int, help=argparse.SUPPRESS) @@ -1272,7 +1326,7 @@ def main(argv=None): if os.geteuid() != 0: raise InstallError("Node installation and removal require root. Run this command with sudo.") if args.uninstall: - if args.source_url or args.bundle or args.core_url or args.provider or args.enrollment_token_stdin: + if args.source_url or args.bundle or args.core_url or args.provider or args.enrollment_token_stdin or args.checkpoint_root: parser.error("--uninstall takes only --installation-id and --force") uninstall_system(args) return diff --git a/deploy/node/test_node_generations.py b/deploy/node/test_node_generations.py index c83ff6f0d..0b2c66311 100644 --- a/deploy/node/test_node_generations.py +++ b/deploy/node/test_node_generations.py @@ -174,7 +174,7 @@ def micro_fixture(self): runtime.chmod(0o700) image = self.value["specification"]["runtime"]["microsandbox_ref"] self.value["specification"]["runtime"]["runtime_sha256"] = hashlib.sha256(runtime.read_bytes()).hexdigest() - self.value["native"] = {"helper_path": str(self.release / "helper"), "runtime_home": str(home), "runtime_path": str(runtime), "firmware_path": str(self.release / "firmware")} + self.value["native"] = {"helper_path": str(self.release / "helper"), "runtime_home": str(home), "checkpoint_root": str(self.root / "checkpoints"), "runtime_path": str(runtime), "firmware_path": str(self.release / "firmware")} self.args.specification_digest = node_spec.digest("microsandbox", self.value["specification"]) node_generations.atomic_json(self.root / "provider.json", self.value) # This fixture changes provider before any helper exists. diff --git a/deploy/node/test_node_install.py b/deploy/node/test_node_install.py index 203db53bf..aada90152 100644 --- a/deploy/node/test_node_install.py +++ b/deploy/node/test_node_install.py @@ -485,7 +485,7 @@ def test_microsandbox_imports_image_and_allows_only_explicit_private_core_endpoi self.install() config = json.loads((self.root / "provider.json").read_text())["native"] # Resources, the image and artifact hashes are read from the specification. - self.assertEqual(set(config), {"helper_path", "runtime_path", "firmware_path", "runtime_home", "network"}) + self.assertEqual(set(config), {"helper_path", "runtime_path", "firmware_path", "runtime_home", "checkpoint_root", "network"}) rules = config["network"]["rules"] self.assertIn({"action": "allow", "direction": "egress", "destination": "172.29.144.1", "protocol": "tcp", "port": "24443"}, rules) self.assertNotIn("private", [rule["destination"] for rule in rules]) @@ -1116,6 +1116,35 @@ def test_microsandbox_short_home_is_stable_and_rejects_long_user_home(self): installer.micro_home("94be54a1-138c-4f30-bc87-b13686272dbe") +class CheckpointRootTests(unittest.TestCase): + def test_identity_is_private_stable_and_not_recreated(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "checkpoints" + installer.prepare_checkpoint_root(root) + marker = root / ".oac-checkpoint-store" + first = marker.read_bytes() + installer.prepare_checkpoint_root(root) + self.assertEqual(first, marker.read_bytes()) + self.assertEqual(stat.S_IMODE(root.stat().st_mode), 0o700) + self.assertEqual(stat.S_IMODE(marker.stat().st_mode), 0o600) + marker.write_text("incomplete") + with self.assertRaises(installer.InstallError): + installer.prepare_checkpoint_root(root) + + def test_foreign_contents_and_symlink_are_not_adopted(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "checkpoints" + root.mkdir() + (root / "unowned").write_bytes(b"data") + with self.assertRaisesRegex(installer.InstallError, "unowned"): + installer.prepare_checkpoint_root(root) + (root / "unowned").unlink() + marker = root / ".oac-checkpoint-store" + marker.symlink_to(Path(temporary) / "missing") + with self.assertRaises(installer.InstallError): + installer.prepare_checkpoint_root(root) + + class UnsupportedNodeUpdateTests(unittest.TestCase): def test_update_refuses_without_host_operations(self): with self.assertRaisesRegex(installer.InstallError, "not supported;.*reinstall"): diff --git a/docs/getting-started/nodes.md b/docs/getting-started/nodes.md index d4890f614..aa02e573c 100644 --- a/docs/getting-started/nodes.md +++ b/docs/getting-started/nodes.md @@ -12,6 +12,10 @@ You add a node by generating a command in Web and running it on the host. The [s - **The sandbox configuration is saved.** Open **System** → **Manage sandbox configuration**, choose **Own machines**, the backend and a sandbox size, and **Save configuration**. To change a saved configuration, choose **Reset deployment** first. Every node of an installation uses that backend. - **The console can serve the node files.** Nodes download their Runtime and provider files from the console, which redirects to the release for files it does not hold, and check each file's size and SHA-256 against the release manifest. Node hosts therefore need access to the release. Without the files, Add node says *This console has no node files for …*. +For microsandbox checkpoint recovery across nodes, add `--checkpoint-root /absolute/private/shared-store` to the installer command on each participating node. Mount the same private store at those paths before installation and grant only the node service account access. The default is a private node-local store and supports recovery on that node. The checkpoint store is separate from workspace storage and must never be guest-accessible; the [checkpoint transfer contract](../sandbox-provider.md#checkpoint-transfer) owns compatibility and retention requirements. + +Microsandbox includes the host kernel and CPU profile in its checkpoint execution compatibility. An OS or CPU-profile change can therefore make retained checkpoints incompatible even on their original node. Before maintenance, keep a node with the matching execution class and the same ready generation available for those checkpoints; otherwise their requests wait under the [checkpoint retention policy](../sandbox-provider.md#checkpoint-transfer). + For microsandbox with independent workspace storage, complete the [NFS mount and service-account setup](../configuration.md#independent-workspace-storage) on this host before enrollment. The node receives the selected immutable filesystem configuration with each binding; do not author a separate node storage setting. The Core host joins like any other host: to run sandboxes on it, add it as a node. @@ -127,7 +131,8 @@ Use manual registration when you manage the node's files and service yourself in 3. Read the node configuration with the token, which does not consume it: `GET /api/v1/sandbox-node/configuration` with `Authorization: Bearer `. 4. Write a private provider file. Copy `provider`, `installation_id`, `core_url`, `generation` and `specification` from the response, and add a `native` object with the host settings of that provider. The adapter reads sandbox size, the Runtime image and artifact hashes from `specification`: - Docker: the [Docker node configuration](../configuration.md#docker-node-configuration) fields, with `host` an explicit Unix socket, `image` the local ID of the imported Runtime image and `seccomp_file` absolute. - - microsandbox: absolute `helper_path`, `runtime_path` and `firmware_path`, a `network` policy, and `runtime_home`: a private directory, which the helper creates with mode `0700` when it is missing. microsandbox places Unix sockets under it, so keep its path within 48 bytes; the installer refuses a longer one for its own nodes. + - microsandbox: absolute `helper_path`, `runtime_path` and `firmware_path`, a `network` policy, an explicit `checkpoint_root` for private checkpoint archives, and `runtime_home`: a private directory, which the helper creates with mode `0700` when it is missing. microsandbox places Unix sockets under it, so keep its path within 48 bytes; the installer refuses a longer one for its own nodes. + Before the first microsandbox registration, initialize the empty `checkpoint_root` as the node service account using the same release's source checkout: `PYTHONPATH=deploy/node python3 -c 'from node_install import prepare_checkpoint_root; prepare_checkpoint_root("/absolute/private/checkpoints")'`. This reuses the installer's exclusive UUID-marker initialization and ownership checks. Initialize a shared store once; other nodes use the existing marker. Never replace the marker or initialize over restored or nonempty unowned storage. 5. Register, then run the node under the host's service supervisor, with real absolute paths: ```sh @@ -148,7 +153,7 @@ The node connects out to Core; Core needs no SSH or Docker TCP access to the hos ## When a node host fails -A restarted node service keeps its identity and finds its existing sandboxes again. Core never replaces a missing sandbox by itself, and never moves a Session to another node: the Session's resources show as **Node disconnected** or **Sandbox resource missing** until the original host and its storage are back, or you archive the Session. A lost node state directory is a recovery incident: restore it from its [backup](./operations.md#back-up) together with the database and the provider storage, rather than registering the host again over existing resources. +A restarted node service keeps its identity and finds its existing sandboxes again. Core does not take over an active or unconfirmed writer merely because its node is unreachable or its sandbox is missing: those resources show as **Node disconnected** or **Sandbox resource missing** while ownership remains unresolved. A settled retained checkpoint can follow the [checkpoint transfer contract](../sandbox-provider.md#checkpoint-transfer), and confirmed release can permit [cold replacement](../sandbox-provider.md#cold-replacement-after-checkpoint-retention). A lost node state directory is a recovery incident: restore it from its [backup](./operations.md#back-up) together with the database and the provider storage, rather than registering the host again over existing resources. ## Troubleshooting diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index 33cadfe6a..c3e5b8ca7 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -130,6 +130,8 @@ A recoverable backup contains one consistent, stopped-write set. A database dump - Every independent workspace namespace selected by current or retained objects, even when it is outside the installation volume. Back up its entire root, including `.oac-storage-root`, object identities, staging, live data, trash and deletion markers. Preserve numeric ownership, modes, links and extended attributes, including every `user.*` attribute. Copying only `live/data` loses lifecycle evidence and can resurrect deleted identities. [Workspace storage](../workspace-provider.md#kernel-nfs-adapter) owns the layout and ownership requirements. - Each node's state directory, `/var/lib/oac-node/.oac/nodes//`, and its provider storage: Docker volumes or microsandbox's store. Include all compute and snapshot resources still referenced by the database. See [when a node host fails](./nodes.md#when-a-node-host-fails). +- Each private `checkpoint_root` in its entirety, including `.oac-checkpoint-store`, archives, operation journals, lock files and tombstones, together with the matching Core database. It is separate from the native SDK store and independent workspace namespace. Preserve its identity and private ownership; the [checkpoint transfer contract](../sandbox-provider.md#checkpoint-transfer) owns its lifecycle. + ### Establish a stopped-write window 1. Block new application input, live file access and administrative mutations at the installation's ingress. Let active execution, initialization, file operations and native cleanup finish. Stop other applications or host processes that can write to the same filesystem. diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index a821f3343..f70b826dc 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -180,7 +180,7 @@ Before releasing the execution lease, the coordinator stops accepting work and c Placement is automatic: Session creation commits durable pending work, and the common scheduler reserves a compatible node before allocation. The [deployment contract](../contracts/agents-api/sandbox-deployment.md#generation-ownership-and-rollout) owns waiting, ordering and generation selection. Callers cannot choose a node, and an active allocation keeps its original node even while it is offline. After confirmed allocation release, eligible retained Sessions can reserve compatible capacity again, including on another node. Node capacity counts pending reservations and unresolved resources, and new placement and a suspended-to-restoring transition share a database lock. Unknown operations keep their reservations, source teardown must be confirmed before active capacity is released, and confirmed cleanup releases placement capacity. Retained ownership needs exact provider evidence: a socket path, a missing instance or an empty listing never proves cleanup or authorizes a replacement. -The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and the snapshot retention bound each node. Cold replacement after confirmed release does not provide node-level drain, cross-node memory restore, failover from an unreachable writer, concurrent writers, multi-active Core, autoscaling or snapshot replication. Node removal is refused while the node holds allocations, snapshots, reservations, unknown results or cleanup, and offline ownership is kept. +The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and the snapshot retention bound each node. Cold replacement after confirmed release does not provide node-level drain, failover from an unreachable writer, concurrent writers, multi-active Core, autoscaling or snapshot replication. Node removal is refused while the node holds allocations, snapshots, reservations, unknown results or cleanup, and offline ownership is kept. ### Suspension @@ -190,7 +190,17 @@ Capacity pressure can suspend an already idle node allocation before the normal The Worker lease, the Session lock and the per-node gates own suspension for every provider. New Turn claims, file-write intents and capture admission serialize under the Session lock and share one compute-phase check; new pending work cancels a capture and wakes the same source. Normal preparation waits for the compute phase to be running, after the authenticated resume handshake, and pending input stays pending when its promotion conflicts with a lifecycle transition. Compute phases and revision-checked receipts live on the allocation. Core persists quiesce, capture and restore intent before the effect, only a fresh receipt performs a capture or restore, and recovery observes the exact attempt without retrying an unknown creation, capture or restore. A consumed snapshot never rolls a running generation back. Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap, and unknown cleanup identities are kept until owned resources are confirmed absent. Consumed artifacts and old compute are deleted, so suspension cycles never build a chain of writable disks. -Queued work and live Environment file access wake a suspended Environment; history and published Artifact reads do not. Live file requests wait for the current Runtime’s credential-authorized connection and Harness declaration before entering the file work queue; a completed compute Create alone is not Runtime readiness. Planned suspension uses an Environment and suspension token on the daemon connection. A PID and start-time fenced local control signal (`RunCommandCompute`) wakes the parked daemon, which authenticates again before admitting work. A transient disconnect before confirmation retries the same armed suspension with bounded attempts and backoff; a permanent authentication or protocol rejection closes it. The retained checkpoint resumes on its original node. Core owns the snapshot's retention deadline, and the daemon has no timer for it. A lost quiesce acknowledgement may thaw the same source through explicit rollback but never authorizes capturing it. +Queued work and live Environment file access wake a suspended Environment; history and published Artifact reads do not. Live file requests wait for the current Runtime’s credential-authorized connection and Harness declaration before entering the file work queue; a completed compute Create alone is not Runtime readiness. Planned suspension uses an Environment and suspension token on the daemon connection. A PID and start-time fenced local control signal (`RunCommandCompute`) wakes the parked daemon, which authenticates again before admitting work. A transient disconnect before confirmation retries the same armed suspension with bounded attempts and backoff; a permanent authentication or protocol rejection closes it. A retained checkpoint resumes on its original node when it is eligible, or on another eligible node through the checkpoint transfer rules below. Core owns the snapshot's retention deadline, and the daemon has no timer for it. A lost quiesce acknowledgement may thaw the same source through explicit rollback but never authorizes capturing it. + +### Checkpoint transfer + +A checkpoint-capable generation reports `CheckpointCompatibility`: opaque `ArtifactDomain` and `ExecutionClass` tokens. Every verified `SnapshotIdentity` carries the same qualification. Core compares these values without interpreting CPU features, filesystem paths or storage implementations. A restore destination must be online and ready for the allocation's immutable deployment generation, match both tokens, and have active capacity plus a retained slot when moving from another node. The allocation, Device and Session identities remain unchanged. The Session and deployment transaction commits the destination route, placement and exact restore intent before target-side native work. No capacity or compatible destination leaves the retained snapshot owned until its configured deadline. The target must already hold that exact generation; Core does not automatically prepare historical generations on new nodes. During upgrades, retain eligible nodes and their generation providers until their checkpoint retention obligations end. + +`Suspend` may report `SourceStopped` only after publishing a verified full archive durably, stopping the exact source compute and settling source-local capture and cleanup obligations. Core persists the snapshot before marking it suspended; native absence alone is insufficient. `Suspend.ObserveOnly` may finish archive publication and source cleanup for an already verified snapshot of the same operation, but cannot capture another snapshot or stop a source that has resumed running. If a previously published archive is missing or damaged, `ObserveOnly` may return its exact durable ownership receipt with `Status: unknown` and `SourceStopped: false` for cleanup; that receipt proves neither current archive usability nor source absence. `Resume` still verifies the archive before execution. After the source-settlement barrier, source-node unavailability does not prevent restoration. Deletion and ordinary expiry can transfer an idle suspended allocation's cleanup route to an online node in the same artifact domain; deletion does not require execution-class compatibility. The cleanup destination must already serve the same immutable generation and have retained capacity. Without such a node, cleanup stays pending and ownership is not released; operators must restore an eligible node or its capacity. An unknown running or restoring writer never moves merely because its node is unreachable. + +Recovery uses `Resume.ObserveOnly` for the persisted target and operation. `RestoreAttemptClosed` is an exact operation ID, not a generic absence flag: the adapter may return it only after durably closing an attempt that never entered native execution and fencing every late request for that ID. An adapter may also close a durably admitted attempt when it can prove that its current invocation failed before dispatching native restoration. The microsandbox adapter does this only when the fresh request's wire deadline expires before native dispatch; archive corruption, qualification errors and opaque SDK failures do not authorize an automatic retry. Core then persists a new operation ID before attempting execution. Once native restoration has been dispatched, an unknown outcome retains the same target, ownership and original retention deadline; a missing native listing, helper death or timeout does not authorize replay. Such a restore can remain unavailable until explicit deletion or ordinary retention expiry, and pending inputs can reach their public deadline while waiting. User deletion still requires exact native settlement before resources and ownership can be released. + +The microsandbox adapter requires an explicit private `checkpoint_root`, outside every guest-accessible filesystem. Its directory is mode `0700`; a private namespace marker identifies the actual archive store. The default installation creates a node-local private store. Cross-node restoration requires operators to mount the same private store on each participating node; mount paths may differ. Runtime state directories and workspace bindings are never used to infer this root. The adapter verifies the archive, external filesystem identity and native execution class again before restoration. Restored RAM and file descriptors do not promise continuity of external TCP connections or replay safety for application side effects. ### Cold replacement after checkpoint retention @@ -252,4 +262,4 @@ The node uses the explicit Unix socket in its [provider configuration](./configu `DeploymentPolicy.Workspace` declares supported attachment requirements; absence means external storage is unsupported. The derived immutable `DeploymentSpec.Workspace` receipt selects the generation mode and capabilities; its [deployment workflow](../contracts/agents-api/sandbox-deployment.md#resources) never copies filesystem configuration into the generation. Microsandbox requires `host_directory` and `user_xattr`; Docker and E2B reject external attachments. `ValidateWorkspacePolicy` uses the filesystem protocol's combination validator: an external filesystem without an enforced quota rejects a positive `EnvironmentDiskMiB`, while zero requests no quota. Root disk capacity is still required. Without an external declaration, the existing owned disk bounds apply. -Microsandbox resolves each Create and Resume binding, including observation of an interrupted restore, and passes only the local directory and immutable ObjectID to its private helper. The helper binds the whole `/environment`; its retained contents and private native history follow the [workspace filesystem scope](workspace-provider.md#lifetime-and-filesystem-scope) and [Runtime resource directories](configuration.md#runtime-resource-directories). Private HOME, credentials and the compute root are not a durable history store. Explicit mode, object and path labels qualify native resources; mount shape alone never selects a mode. Same-node checkpoint restore verifies the snapshot's object, remaps `/environment` through SDK `Volumes`, requires strict external mount policy and rejects every restore warning. Partial restore errors retain the exact target identity for cleanup and never establish readiness. Unknown outcomes remain observations of the original operation. Native Bind checkpoint and guest-local flock continuity evidence does not establish cross-VM fencing or cross-node memory restore. +Microsandbox resolves each Create and Resume binding, including observation of an interrupted restore, and passes only the local directory and immutable ObjectID to its private helper. The helper binds the whole `/environment`; its retained contents and private native history follow the [workspace filesystem scope](workspace-provider.md#lifetime-and-filesystem-scope) and [Runtime resource directories](configuration.md#runtime-resource-directories). Private HOME, credentials and the compute root are not a durable history store. Explicit mode, object and path labels qualify native resources; mount shape alone never selects a mode. Checkpoint restore verifies the snapshot's object, remaps `/environment` through SDK `Volumes`, requires strict external mount policy and rejects every restore warning. Partial restore errors retain the exact target identity for cleanup and never establish readiness. Unknown outcomes remain observations of the original operation. Native Bind checkpoint and guest-local flock continuity do not establish fencing against an unknown writer. diff --git a/docs/zh/getting-started/nodes.md b/docs/zh/getting-started/nodes.md index d7c572de2..9e7d24e05 100644 --- a/docs/zh/getting-started/nodes.md +++ b/docs/zh/getting-started/nodes.md @@ -1,7 +1,7 @@ --- title: "添加和管理节点" source: docs/getting-started/nodes.md -source_hash: 9b75587ccab0088003e8bd12afe477af0e72fe6650045f5283700248f9df327e +source_hash: 0c62518cc740387ed03035ea4870108edeaaa82331a7509b573fb2b775ae3aba --- 节点是一台 Linux 主机，在沙箱后端为 Docker 或 microsandbox 时，为 Core 托管 Session 运行沙箱。Core 将新 Session 分配给有空余容量的节点；节点创建沙箱，沙箱回连 Core。E2B 不需要节点。应用为自己的 Session 连接的机器是[自托管执行器](self-hosted.md)，而不是节点。 @@ -16,6 +16,10 @@ source_hash: 9b75587ccab0088003e8bd12afe477af0e72fe6650045f5283700248f9df327e Core 主机与其他主机一样加入：要在它上面运行沙箱，将它添加为节点。 +microsandbox 跨节点检查点恢复要求在每个参与节点的安装命令加入 `--checkpoint-root /absolute/private/shared-store`。安装前将同一个私有 store 挂载到这些路径，并只允许 node 服务账户访问。默认使用节点本地私有 store，只支持在该节点恢复。检查点 store 独立于 workspace 存储，绝不能允许 guest 访问；[检查点转移契约](../sandbox-provider.md#checkpoint-transfer) 定义兼容性和保留要求。 + +Microsandbox 的检查点执行兼容性包含主机内核和 CPU profile。因此，操作系统或 CPU profile 变化可能使保留的检查点不再兼容，甚至无法在原节点恢复。维护前，应为这些检查点保留具有匹配 execution class 且已就绪同一 generation 的节点；否则，相关请求会按[检查点保留策略](../sandbox-provider.md#checkpoint-transfer)等待。 + 对于使用独立工作区存储的 microsandbox，应在注册前完成此主机上的 [NFS 挂载和服务账户配置](../configuration.md#independent-workspace-storage)。节点随每次 binding 接收所选不可变文件系统配置；不要单独编写节点存储设置。 ## 添加节点 {#add-a-node} @@ -129,7 +133,8 @@ root 只准备账号、组和服务单元；其他操作（包括 Docker 网络 3. 使用令牌读取节点配置，不会消耗令牌：`GET /api/v1/sandbox-node/configuration`，带 `Authorization: Bearer `。 4. 写入私有提供商文件。从响应复制 `provider`、`installation_id`、`core_url`、`generation` 和 `specification`，并添加 `native` 对象，写入该提供商的主机设置。适配器从 `specification` 读取沙箱规格、Runtime 镜像和产物哈希： - Docker：[Docker 节点配置](../configuration.md#docker-node-configuration)中的字段，其中 `host` 是显式 Unix 套接字，`image` 是导入的 Runtime 镜像的本地 ID，`seccomp_file` 是绝对路径。 - - microsandbox：绝对路径 `helper_path`、`runtime_path` 和 `firmware_path`；`network` 策略；以及 `runtime_home` 私有目录。目录缺失时辅助程序以 `0700` 创建。microsandbox 在其中放置 Unix 套接字，因此路径不要超过 48 字节；安装程序对自管节点拒绝更长路径。 + - microsandbox：绝对路径 `helper_path`、`runtime_path` 和 `firmware_path`；`network` 策略；私有检查点归档的显式 `checkpoint_root`；以及 `runtime_home` 私有目录。目录缺失时辅助程序以 `0700` 创建。microsandbox 在其中放置 Unix 套接字，因此路径不要超过 48 字节；安装程序对自管节点拒绝更长路径。 + 首次注册 microsandbox 前，以 node 服务账户在相同发行版本的源码 checkout 中初始化空的 `checkpoint_root`：`PYTHONPATH=deploy/node python3 -c 'from node_install import prepare_checkpoint_root; prepare_checkpoint_root("/absolute/private/checkpoints")'`。它复用安装程序的 UUID marker 排他初始化与所有权检查。共享 store 只初始化一次，其他 node 使用已有 marker。不得替换 marker，也不得在恢复后的存储或非空且无所有权证明的目录上重新初始化。 5. 使用真实绝对路径注册，然后通过主机服务管理器运行节点： ```sh @@ -150,7 +155,7 @@ root 只准备账号、组和服务单元；其他操作（包括 Docker 网络 ## 节点主机故障时 {#when-a-node-host-fails} -重启节点服务会保留身份并重新发现已有沙箱。Core 不会自行替换缺失沙箱，也不会将 Session 移到其他节点：Session 资源显示 **Node disconnected** 或 **Sandbox resource missing**，直到原主机及存储恢复，或你归档 Session。丢失节点状态目录属于恢复事件：从[备份](operations.md#back-up)恢复，并同时恢复数据库和提供商存储；不要在已有资源上重新注册主机。 +重启节点服务会保留身份并重新发现已有沙箱。Core 不会仅因节点不可达或沙箱缺失就接管活动或未确认的 writer：所有权未结清时，这些资源显示 **Node disconnected** 或 **Sandbox resource missing**。已结清的保留检查点可以遵循[检查点转移契约](../sandbox-provider.md#checkpoint-transfer)，确认释放后可以按条件进行[冷替换](../sandbox-provider.md#cold-replacement-after-checkpoint-retention)。丢失节点状态目录属于恢复事件：从[备份](operations.md#back-up)恢复，并同时恢复数据库和提供商存储；不要在已有资源上重新注册主机。 ## 问题排查 {#troubleshooting} diff --git a/docs/zh/getting-started/operations.md b/docs/zh/getting-started/operations.md index 8ab281776..4316ac34d 100644 --- a/docs/zh/getting-started/operations.md +++ b/docs/zh/getting-started/operations.md @@ -1,7 +1,7 @@ --- title: "运维" source: docs/getting-started/operations.md -source_hash: acd6e73b0bdf43f3bb5adfc375c8938d1b8169a326620076242c844ab8b4766b +source_hash: 591196d61ed9946435d07afcbbcc2f53d64b9db0cf6b4806ddfb0f85a16974ba --- 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。 @@ -132,6 +132,8 @@ Core 记录每次公开资源写入所使用的密钥；历史保留策略为 [` - 当前或保留对象所使用的每个独立工作区命名空间，即使其位于安装卷之外。备份整个根目录，包括 `.oac-storage-root`、对象标识、staging、live 数据、trash 和删除标记。保留数值所有者、权限、链接及扩展属性，包括全部 `user.*` 属性。仅复制 `live/data` 会丢失生命周期证据，并可能使已删除标识复活。[工作区存储](../workspace-provider.md#kernel-nfs-adapter)负责定义布局与所有权要求。 - 各节点状态目录 `/var/lib/oac-node/.oac/nodes//` 及其 Provider 存储：Docker 卷或 microsandbox 存储。包含数据库仍然引用的全部计算资源和快照。参见[节点主机故障时](./nodes.md#when-a-node-host-fails)。 +- 每个私有 `checkpoint_root` 的全部内容，包括 `.oac-checkpoint-store`、归档、operation journal、lock 文件与 tombstone，并与匹配的 Core 数据库一起保留。它独立于原生 SDK store 和工作区 namespace。保留其身份和私有所有权；[检查点转移契约](../sandbox-provider.md#checkpoint-transfer)定义其生命周期。 + ### 建立停止写入窗口 {#establish-a-stopped-write-window} 1. 在安装入口阻止新应用 input、实时文件访问和管理变更。等待活动执行、初始化、文件操作和原生清理完成。停止其他能够写入同一文件系统的应用或主机进程。 diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 48562ef57..ff6501786 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: 567cd130134a45f483b156d7f2b74d246b191dd5db77b976976c0d710414dd37 +source_hash: 63ad2e39e7536caee9bd43f9b643073d32b32147f1440d188d6d38752628adc7 --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -182,7 +182,7 @@ allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan jo placement 自动完成：Session 创建提交持久的 pending 工作，共同调度器在 allocation 之前预留兼容节点。[部署契约](../../contracts/agents-api/zh/sandbox-deployment.md#generation-ownership-and-rollout) 定义等待、排序与代次选择。调用方不能选择 node；活动 allocation 即使在 node 离线时也保留原 node。确认 allocation 释放后，符合条件的保留 Session 可以重新预约兼容容量，包括其他 node。Node capacity 计入 pending reservation 和未决资源，新 placement 与 suspended-to-restoring 转移共享数据库锁。未知操作保留预约，source teardown 必须确认后才能释放 active capacity，确认 cleanup 后释放 placement capacity。保留所有权需要精确 provider evidence：socket path、缺失 instance 或空列表都不证明 cleanup，也不授权 replacement。 -部署的 CPU、memory、disk 设置、`max_active`、`max_retained` 和 snapshot retention 限制每个 node。确认释放后的冷替换不提供 node-level drain、跨 node 内存恢复、不可达写入方的故障转移、并发写入方、multi-active Core、autoscaling 或 snapshot replication。node 持有 allocation、snapshot、reservation、unknown result 或 cleanup 时拒绝移除 node，离线 ownership 保留。 +部署的 CPU、memory、disk 设置、`max_active`、`max_retained` 和 snapshot retention 限制每个 node。确认释放后的冷替换不提供 node-level drain、不可达写入方的故障转移、并发写入方、multi-active Core、autoscaling 或 snapshot replication。node 持有 allocation、snapshot、reservation、unknown result 或 cleanup 时拒绝移除 node，离线 ownership 保留。 ### 暂停 {#suspension} @@ -192,7 +192,17 @@ Core 用同一个共享 policy 暂停每个声明 checkpoint 支持的 provider Worker lease、Session lock 与 per-node gate 对每个 provider 负责 suspension。新 Turn claim、file-write intent 和 capture admission 在 Session lock 下串行化，共享一个 compute-phase 检查；新 pending work 取消 capture 并唤醒同一 source。正常 preparation 在经过认证的 resume handshake 后等待 compute phase 为 running；pending input 的 promotion 与 lifecycle transition 冲突时保持 pending。compute phase 和 revision-checked receipt 位于 allocation。Core 在 effect 前持久化 quiesce、capture 和 restore intent，仅新 receipt 执行 capture 或 restore，恢复观察精确 attempt，不重试未知 creation、capture 或 restore。已消费 snapshot 不让 running generation 回滚。删除、撤销和 retention expiry 优先于 wake，一直持续到最终数据库 compare-and-swap；未知 cleanup identity 保留，直到确认所属资源不存在。已消费 artifact 和旧 compute 被删除，因此暂停循环不累积可写磁盘链。 -排队工作和实时 Environment file access 唤醒 suspended Environment；history 和已发布 Artifact read 不唤醒。实时文件请求在进入文件工作队列前，等待当前 Runtime 通过 credential 授权的连接及 Harness 声明；compute Create 完成并不代表 Runtime 已就绪。计划暂停在 daemon 连接上使用 Environment 和 suspension token。受 PID 与 start-time fencing 的本地 control signal（`RunCommandCompute`）唤醒 parked daemon，daemon 在准入工作前重新认证。确认前临时断连通过有界 attempt 和 backoff 重试同一已 armed suspension；永久认证或协议拒绝则关闭。保留的检查点在原 node 恢复。Core 负责 snapshot retention deadline，daemon 没有相应 timer。quiesce 确认丢失时可以通过明确 rollback 解冻同一 source，但不授权 capture。 +排队工作和实时 Environment file access 唤醒 suspended Environment；history 和已发布 Artifact read 不唤醒。实时文件请求在进入文件工作队列前，等待当前 Runtime 通过 credential 授权的连接及 Harness 声明；compute Create 完成并不代表 Runtime 已就绪。计划暂停在 daemon 连接上使用 Environment 和 suspension token。受 PID 与 start-time fencing 的本地 control signal（`RunCommandCompute`）唤醒 parked daemon，daemon 在准入工作前重新认证。确认前临时断连通过有界 attempt 和 backoff 重试同一已 armed suspension；永久认证或协议拒绝则关闭。保留的检查点在原 node 仍具资格时优先恢复，也可以按以下检查点转移规则在其他符合条件的 node 恢复。Core 负责 snapshot retention deadline，daemon 没有相应 timer。quiesce 确认丢失时可以通过明确 rollback 解冻同一 source，但不授权 capture。 + +### 检查点转移 {#checkpoint-transfer} + +支持检查点的 generation 报告 `CheckpointCompatibility`，包含不透明的 `ArtifactDomain` 与 `ExecutionClass` token。每个已验证 `SnapshotIdentity` 携带同样的资格。Core 只比较这些值，不解释 CPU 特性、文件系统路径或存储实现。恢复目标必须在线、对 allocation 的不可变 deployment generation 已就绪、两个 token 均匹配，并有 active 容量；跨 node 时还需 retained slot。allocation、Device 和 Session 身份保持不变。Session 与 deployment 事务先提交目标路由、placement 和精确 restore intent，再执行目标侧原生操作。没有容量或兼容目标时，保留快照的所有权一直持续到配置期限。目标必须已持有该精确 generation；Core 不会在新节点自动准备历史 generation。升级期间应保留符合条件的节点及其 generation provider，直到检查点保留义务结束。 + +`Suspend` 只有在持久发布已验证的完整归档、停止精确 source compute，并结清 source 本地 capture 与 cleanup 义务后，才能报告 `SourceStopped`。Core 先持久化快照，再标记 suspended；仅原生资源不存在并不足够。`Suspend.ObserveOnly` 可以为同一 operation 已验证的快照完成归档发布和源清理，但不能重新捕获快照，也不能停止已经恢复 running 的源。此前已发布的归档缺失或损坏时，`ObserveOnly` 可以返回其精确持久 ownership receipt，标记 `Status: unknown` 和 `SourceStopped: false`，供清理使用；该 receipt 不证明归档当前可用，也不证明源不存在。`Resume` 仍在执行前验证归档。完成源结清屏障后，source node 不可用不再阻止恢复。删除和正常到期可以将空闲 suspended allocation 的清理路由转给同一 artifact domain 内的在线 node；删除不要求 execution class 匹配。清理目标必须已提供同一不可变 generation，且有 retained 容量。没有这样的 node 时，清理保持 pending，不释放所有权；运维需恢复符合条件的 node 或其容量。未知 running 或 restoring writer 不会仅因 node 不可达而迁移。 + +恢复用 `Resume.ObserveOnly` 观察已持久化的 target 与 operation。`RestoreAttemptClosed` 是精确 operation ID，不是通用 absence 标志：adapter 只有在持久关闭从未进入原生执行的 attempt，并隔离该 ID 的所有迟到请求后才能返回。如果 adapter 能证明当前调用在派发原生恢复之前失败，也可以关闭已持久 admitted 的 attempt。microsandbox adapter 仅在 fresh 请求的 wire deadline 于原生派发之前到期时执行这种关闭；归档损坏、资格错误和不透明 SDK 失败都不授权自动重试。Core 随后先持久化新的 operation ID，再尝试执行。一旦已派发原生恢复，结果未知的 attempt 保留相同 target、所有权与原始 retention deadline；原生列表缺失、helper 死亡或超时都不授权重放。这种恢复可能一直不可用，直到显式删除或正常保留期到期；等待中的 input 可能达到其公共 deadline。用户删除仍需精确原生 settlement，之后才能释放资源与所有权。 + +microsandbox adapter 要求显式的私有 `checkpoint_root`，位于所有 guest 可访问文件系统之外。目录权限为 `0700`，私有 namespace marker 标识真实归档存储。默认安装创建 node 本地私有 store。跨 node 恢复要求运维为参与 node 挂载同一个私有 store，挂载路径可以不同。Runtime state 目录与 workspace binding 都不用于推导该 root。adapter 在恢复前再次验证归档、外部文件系统身份和原生 execution class。恢复 RAM 与文件描述符不保证外部 TCP 连接连续，也不保证应用副作用可安全重放。 ### 检查点保留期后的冷替换 {#cold-replacement-after-checkpoint-retention} @@ -254,4 +264,4 @@ node 使用 [provider 配置](configuration.md#docker-node-configuration)中的 `DeploymentPolicy.Workspace` 声明挂载要求；未声明表示不支持外部存储。派生的不可变 `DeploymentSpec.Workspace` 回执选择代次的模式和能力；[部署流程](../../contracts/agents-api/zh/sandbox-deployment.md#resources)不会将文件系统配置复制进代次。Microsandbox 要求 `host_directory` 和 `user_xattr`；Docker 和 E2B 拒绝外部挂载。`ValidateWorkspacePolicy` 使用文件系统协议的组合校验器：不强制容量配额的外部文件系统拒绝正值 `EnvironmentDiskMiB`，零表示不请求配额。根磁盘容量仍为必需。未提供外部声明时，继续使用自有磁盘的容量边界。 -Microsandbox 在每次 Create 和 Resume（包括观察中断的恢复）时解析绑定，仅向私有 helper 传递本地目录和不可变 ObjectID。Helper 挂载整个 `/environment`；保留内容与私有原生历史遵循[工作区文件系统范围](workspace-provider.md#lifetime-and-filesystem-scope)和 [Runtime 资源目录](configuration.md#runtime-resource-directories)。私有 HOME、凭据和计算根磁盘不是持久历史存储。原生资源通过明确的模式、对象和路径标签校验，不根据挂载形状推断模式。同 node 检查点恢复校验快照对象，通过 SDK `Volumes` 重映射 `/environment`，要求严格外部挂载策略，并拒绝任何恢复警告。恢复部分失败时保留精确目标身份用于清理，不证明就绪。未知结果继续观察原操作。原生 Bind 检查点及 guest 内 flock 连续性证据不证明跨 VM 隔离或跨 node 内存恢复。 +Microsandbox 在每次 Create 和 Resume（包括观察中断的恢复）时解析绑定，仅向私有 helper 传递本地目录和不可变 ObjectID。Helper 挂载整个 `/environment`；保留内容与私有原生历史遵循[工作区文件系统范围](workspace-provider.md#lifetime-and-filesystem-scope)和 [Runtime 资源目录](configuration.md#runtime-resource-directories)。私有 HOME、凭据和计算根磁盘不是持久历史存储。原生资源通过明确的模式、对象和路径标签校验，不根据挂载形状推断模式。检查点恢复校验快照对象，通过 SDK `Volumes` 重映射 `/environment`，要求严格外部挂载策略，并拒绝任何恢复警告。恢复部分失败时保留精确目标身份用于清理，不证明就绪。未知结果继续观察原操作。原生 Bind 检查点及 guest 内 flock 连续性不证明对未知 writer 的隔离。 diff --git a/services/core/cmd/sandbox-node/main.go b/services/core/cmd/sandbox-node/main.go index 2a6d3b032..4e955be99 100644 --- a/services/core/cmd/sandbox-node/main.go +++ b/services/core/cmd/sandbox-node/main.go @@ -93,8 +93,12 @@ func run(ctx context.Context, args []string) error { } expected := node.Identity{SpecificationDigest: built.SpecificationDigest, DeploymentGeneration: config.Generation, InstallationID: built.InstallationID, Provider: config.Provider, BackendFingerprint: built.BackendFingerprint} probe := func(ctx context.Context) (node.Health, error) { - err := built.Probe(ctx) - return node.Health{ProviderReady: err == nil}, err + checkpoint, err := built.Probe(ctx) + status := sandbox.GenerationStatus{Generation: config.Generation, SpecificationDigest: built.SpecificationDigest, State: "ready", Checkpoint: checkpoint} + if err != nil { + status.State, status.Diagnostic, status.Checkpoint = "failed", sandbox.NodeDiagnostic(err), nil + } + return node.Health{ProviderReady: err == nil, Generations: []sandbox.GenerationStatus{status}}, err } if args[0] == "register" { if !filepath.IsAbs(*tokenFile) { diff --git a/services/core/cmd/server/managed_nodes.go b/services/core/cmd/server/managed_nodes.go index b908bbab6..c514c7e3d 100644 --- a/services/core/cmd/server/managed_nodes.go +++ b/services/core/cmd/server/managed_nodes.go @@ -67,7 +67,7 @@ func configureManagedNodes(nodes *deployment.Service, reader deployment.Reader, if err := owner(ctx); err != nil { return err } - return nodes.Heartbeat(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health)) + return nodes.Heartbeat(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health), health.Generations) }, }) result.setup = &managedSetup{processPaths: config.ProviderPaths, registry: registry, deployment: nodes, allocations: reader, hub: result.hub, installationID: config.InstallationID, runtimeAPI: config.PublicOrigin.RuntimeAPI()} diff --git a/services/core/internal/db/queries/node_generations.sql b/services/core/internal/db/queries/node_generations.sql index 7b917e529..436268027 100644 --- a/services/core/internal/db/queries/node_generations.sql +++ b/services/core/internal/db/queries/node_generations.sql @@ -10,10 +10,10 @@ UNION ALL SELECT g.provider_kind,g.specification FROM runtime_deployment_generat LIMIT 1; -- name: UpsertNodeGenerationStatus :exec -INSERT INTO runtime_node_generation_status(node_id,generation,specification_digest,connection_id,owner_epoch,state,diagnostic) -VALUES($1,$2,$3,$4,$5,$6,$7) +INSERT INTO runtime_node_generation_status(node_id,generation,specification_digest,connection_id,owner_epoch,state,diagnostic,checkpoint) +VALUES($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT(node_id,generation) DO UPDATE SET specification_digest=EXCLUDED.specification_digest,connection_id=EXCLUDED.connection_id, - owner_epoch=EXCLUDED.owner_epoch,state=EXCLUDED.state,diagnostic=EXCLUDED.diagnostic,observed_at=clock_timestamp(); + owner_epoch=EXCLUDED.owner_epoch,state=EXCLUDED.state,diagnostic=EXCLUDED.diagnostic,checkpoint=EXCLUDED.checkpoint,observed_at=clock_timestamp(); -- name: PromoteNodeServingGeneration :exec UPDATE runtime_nodes SET ready_generation=$2 WHERE id=$1 AND removed_at IS NULL @@ -33,3 +33,13 @@ SELECT EXISTS(SELECT 1 FROM runtime_node_generation_status g JOIN runtime_nodes -- name: DeleteNodeGenerationStatus :exec DELETE FROM runtime_node_generation_status WHERE node_id=$1 AND generation=$2; + +-- name: ListCheckpointGenerationNodes :many +SELECT n.id, g.checkpoint +FROM runtime_nodes n +JOIN runtime_node_generation_status g ON g.node_id=n.id +CROSS JOIN runtime_deployment d +WHERE g.generation=$1 AND n.installation_id=d.installation_id AND n.removed_at IS NULL +AND n.connection_id=g.connection_id AND n.connected_epoch=g.owner_epoch AND g.owner_epoch=d.owner_epoch +AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND g.state='ready' +ORDER BY n.id; diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index 5bb294a8d..1bd60f42d 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -53,3 +53,24 @@ SELECT EXISTS ( SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE e.session_id = $1 AND a.state <> 'released' AND a.node_id IS NOT NULL )::boolean; + +-- name: MoveSuspendedRuntimeCompute :one +WITH moved AS ( + UPDATE runtime_placements p SET node_id=sqlc.arg(destination)::uuid + FROM runtime_allocations a + WHERE a.id=sqlc.arg(id) AND a.environment_id=p.environment_id + AND p.node_id=sqlc.arg(source)::uuid AND p.released_at IS NULL + AND p.deployment_generation=a.deployment_generation + AND a.node_id=p.node_id AND a.compute_revision=sqlc.arg(revision) + AND a.compute_phase='suspended' + AND ((a.state='running' AND sqlc.arg(phase)::text='restoring' + AND a.compute_retained_until>clock_timestamp() + AND EXISTS(SELECT 1 FROM environments e WHERE e.id=a.environment_id AND e.initialization='complete')) + OR (a.state='cleanup_pending' AND sqlc.arg(phase)::text='suspended')) + RETURNING a.id +) +UPDATE runtime_allocations a +SET node_id=sqlc.arg(destination)::uuid, compute_phase=sqlc.arg(phase), + compute_state=sqlc.arg(state)::jsonb, compute_revision=a.compute_revision+1, + compute_phase_changed_at=CASE WHEN a.compute_phase=sqlc.arg(phase)::text THEN a.compute_phase_changed_at ELSE clock_timestamp() END +FROM moved WHERE a.id=moved.id RETURNING a.*; diff --git a/services/core/internal/db/queries/runtime_suspension_pressure.sql b/services/core/internal/db/queries/runtime_suspension_pressure.sql index e46b2fd1f..580c00813 100644 --- a/services/core/internal/db/queries/runtime_suspension_pressure.sql +++ b/services/core/internal/db/queries/runtime_suspension_pressure.sql @@ -5,15 +5,10 @@ WHERE a.node_id IS NOT NULL AND a.state <> 'released' AND (a.compute_phase IN ('quiescing', 'suspending') OR (a.compute_retained_until <= clock_timestamp() AND a.compute_phase <> 'disabled')); --- name: ListWaitingRuntimeRestoreGenerations :many -SELECT DISTINCT a.deployment_generation -FROM runtime_allocations a -JOIN environments e ON e.id = a.environment_id -JOIN sessions s ON s.id = e.session_id -WHERE a.node_id = $1 AND a.state = 'running' AND a.compute_phase = 'suspended' - AND a.compute_retained_until > clock_timestamp() - AND s.deleted_at IS NULL AND e.status NOT IN ('failed', 'expired') - AND (a.compute_wake_requested OR EXISTS ( - SELECT 1 FROM environment_input_reservations r - WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp() - )); +-- name: ListWaitingCheckpointRestores :many +SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'snapshot'->'Compatibility')::jsonb AS checkpoint +FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id +WHERE a.node_id IS NOT NULL AND a.state='running' AND a.compute_phase='suspended' + AND a.compute_retained_until>clock_timestamp() AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND a.compute_state->'snapshot'->'Compatibility' IS NOT NULL + AND (a.compute_wake_requested OR EXISTS(SELECT 1 FROM environment_input_reservations r WHERE r.session_id=s.id AND r.state='pending' AND r.deadline>clock_timestamp())); diff --git a/services/core/internal/db/sqlc/models.go b/services/core/internal/db/sqlc/models.go index 8fd13ca9b..d011c8983 100644 --- a/services/core/internal/db/sqlc/models.go +++ b/services/core/internal/db/sqlc/models.go @@ -342,6 +342,7 @@ type RuntimeNodeGenerationStatus struct { State string `json:"state"` Diagnostic string `json:"diagnostic"` ObservedAt pgtype.Timestamptz `json:"observed_at"` + Checkpoint []byte `json:"checkpoint"` } type RuntimePlacement struct { diff --git a/services/core/internal/db/sqlc/node_generations.sql.go b/services/core/internal/db/sqlc/node_generations.sql.go index ec512ca60..f52041660 100644 --- a/services/core/internal/db/sqlc/node_generations.sql.go +++ b/services/core/internal/db/sqlc/node_generations.sql.go @@ -43,6 +43,42 @@ func (q *Queries) GetNodeGenerationSpecification(ctx context.Context, generation return i, err } +const listCheckpointGenerationNodes = `-- name: ListCheckpointGenerationNodes :many +SELECT n.id, g.checkpoint +FROM runtime_nodes n +JOIN runtime_node_generation_status g ON g.node_id=n.id +CROSS JOIN runtime_deployment d +WHERE g.generation=$1 AND n.installation_id=d.installation_id AND n.removed_at IS NULL +AND n.connection_id=g.connection_id AND n.connected_epoch=g.owner_epoch AND g.owner_epoch=d.owner_epoch +AND n.last_seen_at>clock_timestamp()-interval '45 seconds' AND g.state='ready' +ORDER BY n.id +` + +type ListCheckpointGenerationNodesRow struct { + ID pgtype.UUID `json:"id"` + Checkpoint []byte `json:"checkpoint"` +} + +func (q *Queries) ListCheckpointGenerationNodes(ctx context.Context, generation int64) ([]ListCheckpointGenerationNodesRow, error) { + rows, err := q.db.Query(ctx, listCheckpointGenerationNodes, generation) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListCheckpointGenerationNodesRow{} + for rows.Next() { + var i ListCheckpointGenerationNodesRow + if err := rows.Scan(&i.ID, &i.Checkpoint); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const nodeGenerationKept = `-- name: NodeGenerationKept :one SELECT (EXISTS(SELECT 1 FROM runtime_deployment d WHERE d.generation=$1) OR EXISTS(SELECT 1 FROM runtime_nodes n WHERE n.id=$2 AND n.removed_at IS NULL AND n.ready_generation=$1) @@ -114,10 +150,10 @@ func (q *Queries) RefreshNodeServingReadiness(ctx context.Context, arg RefreshNo } const upsertNodeGenerationStatus = `-- name: UpsertNodeGenerationStatus :exec -INSERT INTO runtime_node_generation_status(node_id,generation,specification_digest,connection_id,owner_epoch,state,diagnostic) -VALUES($1,$2,$3,$4,$5,$6,$7) +INSERT INTO runtime_node_generation_status(node_id,generation,specification_digest,connection_id,owner_epoch,state,diagnostic,checkpoint) +VALUES($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT(node_id,generation) DO UPDATE SET specification_digest=EXCLUDED.specification_digest,connection_id=EXCLUDED.connection_id, - owner_epoch=EXCLUDED.owner_epoch,state=EXCLUDED.state,diagnostic=EXCLUDED.diagnostic,observed_at=clock_timestamp() + owner_epoch=EXCLUDED.owner_epoch,state=EXCLUDED.state,diagnostic=EXCLUDED.diagnostic,checkpoint=EXCLUDED.checkpoint,observed_at=clock_timestamp() ` type UpsertNodeGenerationStatusParams struct { @@ -128,6 +164,7 @@ type UpsertNodeGenerationStatusParams struct { OwnerEpoch int64 `json:"owner_epoch"` State string `json:"state"` Diagnostic string `json:"diagnostic"` + Checkpoint []byte `json:"checkpoint"` } func (q *Queries) UpsertNodeGenerationStatus(ctx context.Context, arg UpsertNodeGenerationStatusParams) error { @@ -139,6 +176,7 @@ func (q *Queries) UpsertNodeGenerationStatus(ctx context.Context, arg UpsertNode arg.OwnerEpoch, arg.State, arg.Diagnostic, + arg.Checkpoint, ) return err } diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index 64fba0e08..9596b297e 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -61,6 +61,70 @@ func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRu return i, err } +const moveSuspendedRuntimeCompute = `-- name: MoveSuspendedRuntimeCompute :one +WITH moved AS ( + UPDATE runtime_placements p SET node_id=$1::uuid + FROM runtime_allocations a + WHERE a.id=$4 AND a.environment_id=p.environment_id + AND p.node_id=$5::uuid AND p.released_at IS NULL + AND p.deployment_generation=a.deployment_generation + AND a.node_id=p.node_id AND a.compute_revision=$6 + AND a.compute_phase='suspended' + AND ((a.state='running' AND $2::text='restoring' + AND a.compute_retained_until>clock_timestamp() + AND EXISTS(SELECT 1 FROM environments e WHERE e.id=a.environment_id AND e.initialization='complete')) + OR (a.state='cleanup_pending' AND $2::text='suspended')) + RETURNING a.id +) +UPDATE runtime_allocations a +SET node_id=$1::uuid, compute_phase=$2, + compute_state=$3::jsonb, compute_revision=a.compute_revision+1, + compute_phase_changed_at=CASE WHEN a.compute_phase=$2::text THEN a.compute_phase_changed_at ELSE clock_timestamp() END +FROM moved WHERE a.id=moved.id RETURNING a.id, a.environment_id, a.device_id, a.provider_key, a.state, a.create_settled, a.created_at, a.released_at, a.compute_phase, a.compute_revision, a.compute_state, a.compute_activity_at, a.compute_wake_requested, a.compute_retained_until, a.node_id, a.observation_error, a.compute_phase_changed_at, a.deployment_generation +` + +type MoveSuspendedRuntimeComputeParams struct { + Destination pgtype.UUID `json:"destination"` + Phase string `json:"phase"` + State []byte `json:"state"` + ID pgtype.UUID `json:"id"` + Source pgtype.UUID `json:"source"` + Revision int64 `json:"revision"` +} + +func (q *Queries) MoveSuspendedRuntimeCompute(ctx context.Context, arg MoveSuspendedRuntimeComputeParams) (RuntimeAllocation, error) { + row := q.db.QueryRow(ctx, moveSuspendedRuntimeCompute, + arg.Destination, + arg.Phase, + arg.State, + arg.ID, + arg.Source, + arg.Revision, + ) + var i RuntimeAllocation + err := row.Scan( + &i.ID, + &i.EnvironmentID, + &i.DeviceID, + &i.ProviderKey, + &i.State, + &i.CreateSettled, + &i.CreatedAt, + &i.ReleasedAt, + &i.ComputePhase, + &i.ComputeRevision, + &i.ComputeState, + &i.ComputeActivityAt, + &i.ComputeWakeRequested, + &i.ComputeRetainedUntil, + &i.NodeID, + &i.ObservationError, + &i.ComputePhaseChangedAt, + &i.DeploymentGeneration, + ) + return i, err +} + const recordRuntimeTerminalActivity = `-- name: RecordRuntimeTerminalActivity :exec UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e diff --git a/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go index da19d0c89..e192c879e 100644 --- a/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go @@ -39,33 +39,34 @@ func (q *Queries) ListRuntimeSuspensionInFlightNodes(ctx context.Context) ([]pgt return items, nil } -const listWaitingRuntimeRestoreGenerations = `-- name: ListWaitingRuntimeRestoreGenerations :many -SELECT DISTINCT a.deployment_generation -FROM runtime_allocations a -JOIN environments e ON e.id = a.environment_id -JOIN sessions s ON s.id = e.session_id -WHERE a.node_id = $1 AND a.state = 'running' AND a.compute_phase = 'suspended' - AND a.compute_retained_until > clock_timestamp() - AND s.deleted_at IS NULL AND e.status NOT IN ('failed', 'expired') - AND (a.compute_wake_requested OR EXISTS ( - SELECT 1 FROM environment_input_reservations r - WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp() - )) +const listWaitingCheckpointRestores = `-- name: ListWaitingCheckpointRestores :many +SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'snapshot'->'Compatibility')::jsonb AS checkpoint +FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id +WHERE a.node_id IS NOT NULL AND a.state='running' AND a.compute_phase='suspended' + AND a.compute_retained_until>clock_timestamp() AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') + AND a.compute_state->'snapshot'->'Compatibility' IS NOT NULL + AND (a.compute_wake_requested OR EXISTS(SELECT 1 FROM environment_input_reservations r WHERE r.session_id=s.id AND r.state='pending' AND r.deadline>clock_timestamp())) ` -func (q *Queries) ListWaitingRuntimeRestoreGenerations(ctx context.Context, nodeID pgtype.UUID) ([]pgtype.Int8, error) { - rows, err := q.db.Query(ctx, listWaitingRuntimeRestoreGenerations, nodeID) +type ListWaitingCheckpointRestoresRow struct { + NodeID pgtype.UUID `json:"node_id"` + DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` + Checkpoint []byte `json:"checkpoint"` +} + +func (q *Queries) ListWaitingCheckpointRestores(ctx context.Context) ([]ListWaitingCheckpointRestoresRow, error) { + rows, err := q.db.Query(ctx, listWaitingCheckpointRestores) if err != nil { return nil, err } defer rows.Close() - items := []pgtype.Int8{} + items := []ListWaitingCheckpointRestoresRow{} for rows.Next() { - var deployment_generation pgtype.Int8 - if err := rows.Scan(&deployment_generation); err != nil { + var i ListWaitingCheckpointRestoresRow + if err := rows.Scan(&i.NodeID, &i.DeploymentGeneration, &i.Checkpoint); err != nil { return nil, err } - items = append(items, deployment_generation) + items = append(items, i) } if err := rows.Err(); err != nil { return nil, err diff --git a/services/core/internal/deployment/allocations.go b/services/core/internal/deployment/allocations.go index 8a1aa92ae..24560832a 100644 --- a/services/core/internal/deployment/allocations.go +++ b/services/core/internal/deployment/allocations.go @@ -226,7 +226,7 @@ func (e *ExecutionOperations) cleanup(ctx context.Context, owner Allocation, abs // running compute to quiescing requires idle compute and either the idle // timeout or capacity pressure, rechecked in the transaction. func (e *ExecutionOperations) SetCompute(ctx context.Context, owner Allocation, phase string, state json.RawMessage, retainedUntil *time.Time, idleTimeout time.Duration) (Allocation, error) { - if !computeTransition(owner.ComputePhase, phase) || !json.Valid(state) || (owner.ComputePhase == "running" && phase == "quiescing" && idleTimeout <= 0) { + if owner.ComputePhase == "suspended" && phase == "restoring" || !computeTransition(owner.ComputePhase, phase) || !json.Valid(state) || (owner.ComputePhase == "running" && phase == "quiescing" && idleTimeout <= 0) { return Allocation{}, ErrInvalidInput } if phase != "running" && (retainedUntil == nil || retainedUntil.IsZero()) { @@ -267,20 +267,79 @@ func (e *ExecutionOperations) SetCompute(ctx context.Context, owner Allocation, } } } - // A node-backed restore reserves capacity on its original node. - if current.ComputePhase == "suspended" && phase == "restoring" && current.NodeID != "" { - restore, err := tx.LoadRestore(current) - if err != nil { - return Allocation{}, err - } - if err := placement.CheckRestore(restore); err != nil { - return Allocation{}, err - } - } return tx.SetCompute(current, ComputeChange{Phase: phase, State: state, RetainedUntil: retainedUntil}) }) } +// BeginRestore reserves a compatible destination before any native restore. +// The caller supplies a fresh restore intent without a target; the destination +// adapter derives the target after this commit. Snapshot portability proves +// that publication and source-local cleanup settled before suspension. +func (e *ExecutionOperations) BeginRestore(ctx context.Context, owner Allocation, compatibility sandbox.CheckpointCompatibility, state json.RawMessage) (Allocation, error) { + var object map[string]json.RawMessage + if compatibility.Validate() != nil || json.Unmarshal(state, &object) != nil || object == nil { + return Allocation{}, ErrInvalidInput + } + return e.change(ctx, owner, true, func(tx AllocationTx, current Allocation) (Allocation, error) { + if current.State != "running" || current.ComputePhase != "suspended" || current.ComputeRevision != owner.ComputeRevision || current.Expired || current.ComputeRetainedUntil == nil { + return Allocation{}, ErrAllocationConflict + } + activity, err := tx.LoadActivity(current) + if err != nil { + return Allocation{}, err + } + if !activity.Busy && !activity.WakeRequested { + return Allocation{}, ErrAllocationConflict + } + if current.NodeID == "" { + return tx.SetCompute(current, ComputeChange{Phase: "restoring", State: state, RetainedUntil: current.ComputeRetainedUntil}) + } + d, nodes, err := tx.LoadCheckpointPlacement(current) + if err != nil { + return Allocation{}, err + } + if d.Resetting { + return Allocation{}, placement.ErrResetAdmission + } + if d.InstallationID != current.ProviderKey || d.Mode != string(sandbox.DeploymentNodes) { + return Allocation{}, placement.ErrNodeUnavailable + } + node, err := e.service.rules.ChooseCheckpoint(nodes, current.NodeID, compatibility, false) + if err != nil { + return Allocation{}, err + } + return tx.MoveSuspended(current, node, ComputeChange{Phase: "restoring", State: state, RetainedUntil: current.ComputeRetainedUntil}) + }) +} + +// RelocateCleanup durably assigns a portable suspended artifact to a node that +// can delete it. An unknown restore remains pinned to its existing target. +func (e *ExecutionOperations) RelocateCleanup(ctx context.Context, owner Allocation, compatibility sandbox.CheckpointCompatibility) (Allocation, error) { + if owner.NodeID == "" || compatibility.Validate() != nil { + return Allocation{}, ErrInvalidInput + } + return e.change(ctx, owner, false, func(tx AllocationTx, current Allocation) (Allocation, error) { + if current.State != "cleanup_pending" || current.ComputePhase != "suspended" || current.ComputeRevision != owner.ComputeRevision { + return Allocation{}, ErrAllocationConflict + } + d, nodes, err := tx.LoadCheckpointPlacement(current) + if err != nil { + return Allocation{}, err + } + if d.InstallationID != current.ProviderKey || d.Mode != string(sandbox.DeploymentNodes) { + return Allocation{}, placement.ErrNodeUnavailable + } + node, err := e.service.rules.ChooseCheckpoint(nodes, current.NodeID, compatibility, true) + if err != nil { + return Allocation{}, err + } + if node == current.NodeID { + return current, nil + } + return tx.MoveSuspended(current, node, ComputeChange{Phase: current.ComputePhase, State: current.ComputeState, RetainedUntil: current.ComputeRetainedUntil}) + }) +} + // RecordObservation records a diagnostic for the observed compute revision // and state of a node-backed allocation. It never releases ownership, changes // public readiness or authorizes replacement. diff --git a/services/core/internal/deployment/allocations_test.go b/services/core/internal/deployment/allocations_test.go index 4de111df1..22d110171 100644 --- a/services/core/internal/deployment/allocations_test.go +++ b/services/core/internal/deployment/allocations_test.go @@ -15,6 +15,7 @@ import ( "github.com/google/uuid" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -141,9 +142,13 @@ func (f *fakeAllocationTx) LoadGenerationSpecification(uint64) (GenerationSpecif return GenerationSpecification{}, nil } -func (f *fakeAllocationTx) LoadRestore(Allocation) (placement.Restore, error) { - unexpected(f.t, "LoadRestore") - return placement.Restore{}, nil +func (f *fakeAllocationTx) LoadCheckpointPlacement(Allocation) (placement.Deployment, []placement.CheckpointNode, error) { + unexpected(f.t, "LoadCheckpointPlacement") + return placement.Deployment{}, nil, nil +} +func (f *fakeAllocationTx) MoveSuspended(Allocation, string, ComputeChange) (Allocation, error) { + unexpected(f.t, "MoveSuspended") + return Allocation{}, nil } func (f *fakeAllocationTx) ObserveRunning(Allocation) (Allocation, error) { @@ -630,7 +635,13 @@ func TestSetComputeRechecksIdlePressureAndActivity(t *testing.T) { }, loadSuspensionDemand: func(Allocation) (SuspensionDemand, error) { pressureChecks++ - return SuspensionDemand{Deployment: placement.Deployment{InstallationID: owner.ProviderKey, Mode: "nodes"}, Nodes: []placement.Node{{ID: owner.NodeID, Online: true, Active: 1, MaxActive: 1}}, RestoreWaiting: test.pressure}, nil + node := placement.Node{ID: owner.NodeID, Online: true, Active: 1, MaxActive: 1, Retained: 1, MaxRetained: 1, CoreURL: testPublicURL} + demand := SuspensionDemand{Deployment: placement.Deployment{InstallationID: owner.ProviderKey, Mode: "nodes"}, Nodes: []placement.Node{node}} + if test.pressure { + compatibility := sandbox.CheckpointCompatibility{ArtifactDomain: "fixture", ExecutionClass: "fixture"} + demand.CheckpointRestores = []CheckpointRestoreDemand{{SourceNode: owner.NodeID, Compatibility: compatibility, Nodes: []placement.CheckpointNode{{Node: node, Checkpoint: &compatibility}}}} + } + return demand, nil }, setCompute: func(a Allocation, c ComputeChange) (Allocation, error) { writes++ @@ -661,3 +672,29 @@ func (f *fakeAllocationTx) CanRetainEnvironment(current Allocation) (bool, error } return f.canRetainEnvironment(current) } + +func TestBeginRestoreDirectKeepsAllocationAndRetention(t *testing.T) { + until := time.Now().Add(time.Hour) + owner := Allocation{ID: uuid.NewString(), DeviceID: uuid.NewString(), TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), ProviderKey: uuid.NewString(), State: "running", ComputePhase: "suspended", ComputeRetainedUntil: &until} + tx := &fakeAllocationTx{t: t, loadAllocation: func() (Allocation, error) { return owner, nil }, loadSessionDevice: func() (SessionDevice, bool, error) { + return SessionDevice{ID: owner.DeviceID, EnvironmentID: owner.EnvironmentID}, true, nil + }, loadActivity: func(Allocation) (Activity, error) { return Activity{WakeRequested: true}, nil }, setCompute: func(current Allocation, change ComputeChange) (Allocation, error) { + if change.Phase != "restoring" || change.RetainedUntil != owner.ComputeRetainedUntil { + t.Fatal("direct restore lost intent", change) + } + current.ComputePhase = change.Phase + return current, nil + }} + storage := &fakeExecutionStorage{t: t, withAllocation: func(_ context.Context, _ AllocationKey, apply func(AllocationTx) error) error { return apply(tx) }} + operations, err := NewExecutionOperations(newService(t, &fakeStorage{t: t}, &fakeReader{t: t}, testPublicURL), storage, engine.Catalog{}) + if err != nil { + t.Fatal(err) + } + restored, err := operations.BeginRestore(t.Context(), owner, sandbox.CheckpointCompatibility{ArtifactDomain: "fixture", ExecutionClass: "fixture"}, json.RawMessage(`{"restore_id":"attempt"}`)) + if err != nil || restored.ID != owner.ID || restored.NodeID != "" || restored.ComputePhase != "restoring" { + t.Fatal("direct restore changed ownership", restored, err) + } + if _, err := operations.SetCompute(t.Context(), owner, "restoring", json.RawMessage(`{}`), &until, time.Minute); !errors.Is(err, ErrInvalidInput) { + t.Fatal("restore bypassed capacity admission", err) + } +} diff --git a/services/core/internal/deployment/nodes.go b/services/core/internal/deployment/nodes.go index f166a12b8..94c281f5b 100644 --- a/services/core/internal/deployment/nodes.go +++ b/services/core/internal/deployment/nodes.go @@ -11,6 +11,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" ) @@ -516,8 +517,8 @@ func (s *Service) NodeRetention(ctx context.Context, nodeID, connectionID string // Heartbeat records a protocol 1 node's health, which reports readiness of its // enrollment generation only. -func (s *Service) Heartbeat(ctx context.Context, nodeID, connectionID string, epoch uint64, health NodeHealth) error { - return s.heartbeat(ctx, nodeID, connectionID, epoch, health, nil, 1) +func (s *Service) Heartbeat(ctx context.Context, nodeID, connectionID string, epoch uint64, health NodeHealth, statuses []sandbox.GenerationStatus) error { + return s.heartbeat(ctx, nodeID, connectionID, epoch, health, statuses, 1) } // HeartbeatGenerations records a node's health and the preparation state of @@ -558,7 +559,12 @@ func (s *Service) heartbeat(ctx context.Context, nodeID, connectionID string, ep if !current { return ErrNodeCredential } - if protocol == 1 { + if protocol == 1 && len(statuses) > 0 { + if len(statuses) != 1 || statuses[0].Generation != n.DeploymentGeneration || statuses[0].SpecificationDigest != n.SpecificationDigest || (statuses[0].State == "ready") != health.ProviderReady { + return ErrInvalidInput + } + } + if protocol == 1 && len(statuses) == 0 { state := "failed" if health.ProviderReady { state = "ready" @@ -570,8 +576,16 @@ func (s *Service) heartbeat(ctx context.Context, nodeID, connectionID string, ep } func (s *Service) recordGenerations(tx NodeTx, d Record, n StoredNode, statuses []sandbox.GenerationStatus, protocol int) error { + adapter, err := s.registry.Lookup(d.Provider) + if err != nil { + return err + } + checkpointSupported := adapter.Operations()["Initial"].State == providercontract.Supported seen := map[uint64]bool{} for _, status := range statuses { + if status.Checkpoint != nil && (status.State != "ready" || !checkpointSupported || status.Checkpoint.Validate() != nil) || status.State == "ready" && checkpointSupported && status.Checkpoint == nil { + return ErrInvalidInput + } if !validGeneration(status.Generation) || !validDigest(status.SpecificationDigest) || seen[status.Generation] || (status.State != "ready" && status.State != "preparing" && status.State != "failed") || status.State == "ready" && status.Diagnostic != "" { return ErrInvalidInput } @@ -592,7 +606,7 @@ func (s *Service) recordGenerations(tx NodeTx, d Record, n StoredNode, statuses if spec.Digest(d.Provider) != status.SpecificationDigest { return ErrSpecificationMismatch } - if err := tx.UpsertGenerationStatus(GenerationStatusRecord{NodeID: n.ID, ConnectionID: n.ConnectionID, Generation: status.Generation, SpecificationDigest: status.SpecificationDigest, OwnerEpoch: d.OwnerEpoch, State: status.State, Diagnostic: sandbox.NormalizeNodeDiagnostic(status.Diagnostic)}); err != nil { + if err := tx.UpsertGenerationStatus(GenerationStatusRecord{NodeID: n.ID, ConnectionID: n.ConnectionID, Generation: status.Generation, SpecificationDigest: status.SpecificationDigest, OwnerEpoch: d.OwnerEpoch, State: status.State, Diagnostic: sandbox.NormalizeNodeDiagnostic(status.Diagnostic), Checkpoint: status.Checkpoint}); err != nil { return err } if status.State == "ready" && status.Generation == d.Generation { diff --git a/services/core/internal/deployment/placement/placement.go b/services/core/internal/deployment/placement/placement.go index d68b8e153..151deea6a 100644 --- a/services/core/internal/deployment/placement/placement.go +++ b/services/core/internal/deployment/placement/placement.go @@ -109,14 +109,41 @@ type Reserved struct { Available bool } -// Restore is what restoring suspended compute on its node reads: the node, -// nil when the installation no longer has it, and whether the node is ready -// for the allocation's generation. -type Restore struct { - Node *Node - // Generation is zero when the allocation has none. - Generation uint64 - GenerationReady bool +// CheckpointNode is a fresh readiness declaration for one immutable generation. +// Capacity is read under the same deployment lock as the allocation transfer. +type CheckpointNode struct { + Node Node + Checkpoint *sandbox.CheckpointCompatibility +} + +// ChooseCheckpoint selects a compatible node without interpreting adapter tokens. +// A retained allocation already owns its source slot. Cleanup needs no active +// slot and compares only the artifact domain, but still reserves a retained slot. +func (r *Rules) ChooseCheckpoint(nodes []CheckpointNode, source string, compatibility sandbox.CheckpointCompatibility, cleanup bool) (string, error) { + var chosen *Node + for i := range nodes { + candidate := &nodes[i] + n := &candidate.Node + if !n.Online || candidate.Checkpoint == nil || candidate.Checkpoint.ArtifactDomain != compatibility.ArtifactDomain { + continue + } + if !cleanup && (candidate.Checkpoint.ExecutionClass != compatibility.ExecutionClass || n.CoreURL != r.publicURL || n.Active >= int64(n.MaxActive)) { + continue + } + if n.ID != source && n.Retained >= int64(n.MaxRetained) { + continue + } + if n.ID == source { + return n.ID, nil + } + if chosen == nil || n.Active < chosen.Active || n.Active == chosen.Active && n.Retained < chosen.Retained { + chosen = n + } + } + if chosen == nil { + return "", ErrNodeUnavailable + } + return chosen.ID, nil } // CheckPublicOrigin rejects a provider that requires a reachable public @@ -205,17 +232,6 @@ func CheckReserved(reserved Reserved) error { return nil } -// CheckRestore admits restoring suspended compute on its original node: the -// node must be online, ready for the allocation's generation and below its -// active capacity. Restore never selects another node. -func CheckRestore(restore Restore) error { - n := restore.Node - if n == nil || restore.Generation == 0 || !n.Online || !restore.GenerationReady || n.Active >= int64(n.MaxActive) { - return ErrNodeUnavailable - } - return nil -} - // LoopbackOrigin reports whether a validated origin names a loopback host, // which nothing outside the Core host can reach. func LoopbackOrigin(value string) bool { diff --git a/services/core/internal/deployment/placement/placement_test.go b/services/core/internal/deployment/placement/placement_test.go index 473c343bb..ae9c7442f 100644 --- a/services/core/internal/deployment/placement/placement_test.go +++ b/services/core/internal/deployment/placement/placement_test.go @@ -158,7 +158,7 @@ func TestDecidePlacement(t *testing.T) { } } -func TestCheckReservedAndRestore(t *testing.T) { +func TestCheckReserved(t *testing.T) { for name, test := range map[string]struct { reserved Reserved want error @@ -171,24 +171,6 @@ func TestCheckReservedAndRestore(t *testing.T) { t.Errorf("%s: CheckReserved = %v, want %v", name, err, test.want) } } - node := func(online bool, active int64) *Node { - return &Node{ID: "a", Online: online, Active: active, MaxActive: 2} - } - for name, test := range map[string]struct { - restore Restore - want error - }{ - "ready": {Restore{Node: node(true, 1), Generation: 3, GenerationReady: true}, nil}, - "missing node": {Restore{Generation: 3, GenerationReady: true}, ErrNodeUnavailable}, - "no generation": {Restore{Node: node(true, 1), GenerationReady: true}, ErrNodeUnavailable}, - "offline": {Restore{Node: node(false, 1), Generation: 3, GenerationReady: true}, ErrNodeUnavailable}, - "generation unready": {Restore{Node: node(true, 1), Generation: 3}, ErrNodeUnavailable}, - "at capacity": {Restore{Node: node(true, 2), Generation: 3, GenerationReady: true}, ErrNodeUnavailable}, - } { - if err := CheckRestore(test.restore); !errors.Is(err, test.want) || (test.want == nil) != (err == nil) { - t.Errorf("%s: CheckRestore = %v, want %v", name, err, test.want) - } - } } func TestLoopbackOrigin(t *testing.T) { diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index d25af71cf..5277a9c3a 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -128,9 +128,12 @@ type AllocationTx interface { PlacementDemand(after PlacementDemandCursor) ([]PlacementDemand, PlacementDemandCursor, error) // LoadGenerationSpecification reads immutable generation requirements. LoadGenerationSpecification(generation uint64) (GenerationSpecification, error) - // LoadRestore locks the deployment and returns what restoring the - // allocation's suspended compute on its node reads. - LoadRestore(current Allocation) (placement.Restore, error) + // LoadCheckpointPlacement locks the deployment and loads fresh readiness + // for the allocation's exact immutable generation, including capacity. + LoadCheckpointPlacement(current Allocation) (placement.Deployment, []placement.CheckpointNode, error) + // MoveSuspended changes node ownership, placement and compute intent in one + // transaction. Only a settled suspended allocation can change nodes. + MoveSuspended(current Allocation, nodeID string, change ComputeChange) (Allocation, error) // ObserveRunning records the allocation running with its creation settled. ObserveRunning(current Allocation) (Allocation, error) // SettleCreation records that the original Create can no longer change @@ -506,4 +509,5 @@ type GenerationStatusRecord struct { SpecificationDigest string OwnerEpoch uint64 State, Diagnostic string + Checkpoint *sandbox.CheckpointCompatibility } diff --git a/services/core/internal/deployment/suspension_pressure.go b/services/core/internal/deployment/suspension_pressure.go index 06c0bf492..d8ae4c173 100644 --- a/services/core/internal/deployment/suspension_pressure.go +++ b/services/core/internal/deployment/suspension_pressure.go @@ -11,10 +11,17 @@ import ( // SuspensionDemand is a transaction-bound capacity observation. It is not a // reservation: only a confirmed suspension frees an active slot. type SuspensionDemand struct { - Deployment placement.Deployment - Nodes []placement.Node - InFlightNodes []string - RestoreWaiting bool + Deployment placement.Deployment + Nodes []placement.Node + InFlightNodes []string + CheckpointRestores []CheckpointRestoreDemand +} + +// CheckpointRestoreDemand carries one stopped owner and exact-generation target evidence. +type CheckpointRestoreDemand struct { + SourceNode string + Compatibility sandbox.CheckpointCompatibility + Nodes []placement.CheckpointNode } func (e *ExecutionOperations) canSuspendForDemand(tx AllocationTx, owner Allocation) (bool, error) { @@ -35,10 +42,36 @@ func (e *ExecutionOperations) canSuspendForDemand(tx AllocationTx, owner Allocat if source == nil || !source.Online || source.Active < int64(source.MaxActive) { return false, nil } - // A retained restore already owns its retained slot and may use an older - // prepared generation than the node's serving generation. - if pressure.RestoreWaiting { - return !slices.Contains(pressure.InFlightNodes, owner.NodeID), nil + for _, demand := range pressure.CheckpointRestores { + if _, err := e.service.rules.ChooseCheckpoint(demand.Nodes, demand.SourceNode, demand.Compatibility, false); err == nil { + continue + } else if !errors.Is(err, placement.ErrNodeUnavailable) { + return false, err + } + candidates := slices.Clone(demand.Nodes) + for i := range candidates { + if candidates[i].Node.ID == owner.NodeID { + candidates[i].Node.Active-- + } + } + chosen, err := e.service.rules.ChooseCheckpoint(candidates, demand.SourceNode, demand.Compatibility, false) + if err == nil && chosen == owner.NodeID { + eligible := make([]placement.Node, 0, len(candidates)) + for _, candidate := range candidates { + if candidate.Checkpoint != nil && *candidate.Checkpoint == demand.Compatibility { + eligible = append(eligible, candidate.Node) + } + } + inFlight, err := e.reclamationInFlight(pressure, eligible) + if err != nil { + return false, err + } + if !inFlight { + return true, nil + } + } else if err != nil && !errors.Is(err, placement.ErrNodeUnavailable) { + return false, err + } } if source.Retained >= int64(source.MaxRetained) { return false, nil diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index d6f2f62eb..71567378d 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -199,6 +199,9 @@ func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.Sandbox } return r.wakeCompute(ctx, p, next, state) } + if result.Snapshot.Compatibility.Validate() != nil { + return sandbox.ErrOwnership + } state.Snapshot = result.Snapshot // Store the verified artifact before any recovery-path kill. Snapshot failure // or an unknown result cannot silently fall back to a cold Environment. @@ -207,8 +210,9 @@ func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.Sandbox return err } owner = next - if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { - return err + if !result.SourceStopped { + // Native absence alone does not settle capture or publish the archive. + return sandbox.ErrComputeUnconfirmed } _, err = r.saveCompute(ctx, next, "suspended", state, next.ComputeRetainedUntil) return err @@ -226,22 +230,43 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San if state.Snapshot == nil || state.Target != nil { return sandbox.ErrOwnership } - target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Snapshot) + state.RestoreID = uuid.NewString() + raw, err := json.Marshal(state) if err != nil { return err } - state.Target, state.RestoreID = &target, uuid.NewString() - next, err := r.saveCompute(ctx, owner, "restoring", state, owner.ComputeRetainedUntil) + next, err := r.deployment.BeginRestore(ctx, owner, state.Snapshot.Compatibility, raw) if err != nil { return err } + if next.NodeID != r.nodeID { + // The durable route and reservation belong to the target node's lane. + if r.hintDestination != nil { + r.hintDestination(next.NodeID) + } + return nil + } return r.restoreCompute(ctx, p, next, state, false) } func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) (err error) { defer func() { err = withObservationOwner(owner, err) }() - if state.Target == nil || state.Snapshot == nil || state.Rollback { + if state.Snapshot == nil || state.Snapshot.Compatibility.Validate() != nil || state.Rollback || state.RestoreID == "" { return sandbox.ErrOwnership } + if state.Target == nil { + // NewCompute derives identity without starting native execution. A crash + // before this commit can safely repeat derivation on the reserved target. + target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Snapshot) + if err != nil { + return err + } + state.Target = &target + next, err := r.saveCompute(ctx, owner, "restoring", state, owner.ComputeRetainedUntil) + if err != nil { + return err + } + owner, observeOnly = next, false + } spec, err := r.workspaceSpecification(ctx, owner.Key(), owner.ID) if err != nil { return err @@ -260,6 +285,18 @@ func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.Sandbox if err != nil { return err } + if result.RestoreAttemptClosed != "" { + if !result.ClosesRestoreAttempt(sandbox.ResumeRequest{OperationID: state.RestoreID, Snapshot: *state.Snapshot, Target: *state.Target, ObserveOnly: observeOnly}) { + return sandbox.ErrOwnership + } + // Only an exact, durably closed attempt with no native dispatch can be replaced. + state.RestoreID = uuid.NewString() + next, err := r.saveCompute(ctx, owner, "restoring", state, owner.ComputeRetainedUntil) + if err != nil { + return err + } + return r.restoreCompute(ctx, p, next, state, false) + } if result.Status != "running" || result.Compute.ID == "" { return sandbox.ErrComputeUnconfirmed } diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index 2aab604f6..4f836dc6a 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -78,6 +78,19 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Sandbox if err := r.lease.CheckOwnership(ctx); err != nil { return err } + if owner.ComputePhase == "suspended" && state.Snapshot != nil && owner.NodeID != "" { + next, err := r.deployment.RelocateCleanup(ctx, owner, state.Snapshot.Compatibility) + if err != nil { + return err + } + if next.NodeID != r.nodeID { + if r.hintDestination != nil { + r.hintDestination(next.NodeID) + } + return nil + } + owner = next + } // An uncommitted artifact is found by its persisted attempt, never a directory // glob. The helper's allocation lock also waits for an earlier unknown call. if owner.ComputePhase == "suspending" && state.Snapshot == nil { @@ -94,8 +107,10 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Sandbox return err } } - if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { - return err + if owner.ComputePhase != "suspended" && owner.ComputePhase != "restoring" { + if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { + return err + } } if state.Snapshot != nil { if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 68e678d6e..a4a14cc11 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -63,6 +63,7 @@ type runtimeLifecycle struct { pendingCursor string connections map[string]*runtimeConnection wakeHints chan struct{} + hintDestination func(string) } func newRuntimeManager(owner Owner, deployments *deployment.Service, deploymentReader deployment.Reader, sessionReader sessions.Reader, registry *runtimegateway.Registry, config *RuntimeProvider) (*runtimeManager, error) { diff --git a/services/core/internal/execution/runtime_manager.go b/services/core/internal/execution/runtime_manager.go index 080a04547..5c3544e96 100644 --- a/services/core/internal/execution/runtime_manager.go +++ b/services/core/internal/execution/runtime_manager.go @@ -98,7 +98,7 @@ func (m *runtimeManager) node(id string) (*runtimeNode, error) { deployment: m.deployment, deployments: m.deploymentService, reader: m.deploymentReader, lease: m.lease, registry: m.registry, config: m.config, nodeID: id, gate: make(chan struct{}, 1), ctx: ctx, stop: stop, - connections: make(map[string]*runtimeConnection), wakeHints: make(chan struct{}, 1), + connections: make(map[string]*runtimeConnection), wakeHints: make(chan struct{}, 1), hintDestination: m.hintDestination, }} m.nodes[id] = n } @@ -118,6 +118,19 @@ func (m *runtimeManager) node(id string) (*runtimeNode, error) { return n, nil } +// A committed handoff can introduce a node before the next inventory scan. +// Reuse its ordinary lane and coalesced wake; polling recovers missed hints. +func (m *runtimeManager) hintDestination(id string) { + n, err := m.node(id) + if err != nil { + return + } + select { + case n.lifecycle.wakeHints <- struct{}{}: + default: + } +} + func (m *runtimeManager) hints(id string) chan<- struct{} { m.mu.Lock() defer m.mu.Unlock() diff --git a/services/core/internal/execution/runtime_observation_test.go b/services/core/internal/execution/runtime_observation_test.go index 8ad10bdae..cc13c2f0c 100644 --- a/services/core/internal/execution/runtime_observation_test.go +++ b/services/core/internal/execution/runtime_observation_test.go @@ -18,7 +18,7 @@ func (p *observationCaptureProvider) Suspend(_ context.Context, q sandbox.Suspen if p.captureError != nil { return sandbox.ComputeState{}, p.captureError } - return sandbox.ComputeState{Compute: q.Source, Snapshot: &sandbox.SnapshotIdentity{ID: "captured"}, Status: "suspended", SourceStopped: true}, nil + return sandbox.ComputeState{Compute: q.Source, Snapshot: &sandbox.SnapshotIdentity{ID: "captured", Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}, Status: "suspended", SourceStopped: p.killError == nil}, nil } func TestComputeFailureObservationUsesCommittedReceipt(t *testing.T) { diff --git a/services/core/internal/execution/runtime_replacement_test.go b/services/core/internal/execution/runtime_replacement_test.go index 9a4e2f098..1d8c2c31f 100644 --- a/services/core/internal/execution/runtime_replacement_test.go +++ b/services/core/internal/execution/runtime_replacement_test.go @@ -132,9 +132,9 @@ func TestExpiredRetainedComputePreservesSessionAndPendingInput(t *testing.T) { if _, err = fixture.pool.Exec(t.Context(), `UPDATE session_devices SET native_session_id='retained-native-session' WHERE device_id=$1`, owner.DeviceID); err != nil { t.Fatal(err) } - compute := runtimeCompute{Current: sandbox.Compute{ID: "old-compute"}, Snapshot: &sandbox.SnapshotIdentity{ID: "old-snapshot"}} + compute := runtimeCompute{Current: sandbox.Compute{ID: "old-compute"}, Snapshot: &sandbox.SnapshotIdentity{Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}, ID: "old-snapshot"}} raw, _ := json.Marshal(compute) - if _, err = fixture.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state=$2,compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID, raw); err != nil { + if _, err = fixture.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspending',compute_state=$2,compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID, raw); err != nil { t.Fatal(err) } input, err := fixture.sessions.ReserveEnvironmentInput(t.Context(), session.TenantID, session.ID, "after-retention", []sessions.Input{{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"resume"}]}]}`)}}) @@ -213,7 +213,7 @@ func TestRetainedEnvironmentDemandRecreatesCompute(t *testing.T) { {`UPDATE environments SET initialization='complete',status='disconnected' WHERE id=$1`, owner.EnvironmentID}, {`UPDATE devices SET supported_agent_kinds='[{"kind":"codex","available":true,"capabilities":{"retained_native_history":true}}]' WHERE id=$1`, owner.DeviceID}, {`UPDATE session_devices SET native_session_id='retained-native-session' WHERE device_id=$1`, owner.DeviceID}, - {`UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"current":{"id":"old-compute"},"snapshot":{"id":"old-snapshot"}}',compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID}, + {`UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"current":{"id":"old-compute"},"snapshot":{"id":"old-snapshot","Compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID}, } { if _, err = fixture.pool.Exec(t.Context(), statement.q, statement.arg); err != nil { t.Fatal(err) @@ -354,7 +354,7 @@ func workspaceNodeSetup(t *testing.T, external bool) func(Owner, *deployment.Ser if err = service.ConnectNode(t.Context(), node.NodeID, connection, epoch); err != nil { t.Fatal(err) } - if err = service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + if err = service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}}); err != nil { t.Fatal(err) } return installation, node.NodeID @@ -462,7 +462,7 @@ func TestRestoreUsesAllocationGenerationStorageInsteadOfCachedLane(t *testing.T) } r.config.Workspace = target.Workspace r.config.Resources = target.Resources - state := runtimeCompute{Target: &sandbox.Compute{ID: "replacement-compute", Generation: 2}, Snapshot: &sandbox.SnapshotIdentity{ID: "snapshot"}, RestoreID: uuid.NewString()} + state := runtimeCompute{Target: &sandbox.Compute{ID: "replacement-compute", Generation: 2}, Snapshot: &sandbox.SnapshotIdentity{Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}, ID: "snapshot"}, RestoreID: uuid.NewString()} if err = r.restoreCompute(t.Context(), provider, owner, state, false); !errors.Is(err, stop) { t.Fatal(err) } diff --git a/services/core/internal/execution/runtime_restore_attempt_test.go b/services/core/internal/execution/runtime_restore_attempt_test.go new file mode 100644 index 000000000..c18833796 --- /dev/null +++ b/services/core/internal/execution/runtime_restore_attempt_test.go @@ -0,0 +1,238 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" +) + +type restoreAttemptProvider struct { + retentionProvider + outcome string + requests []sandbox.ResumeRequest + derived int + stop error +} + +func (p *restoreAttemptProvider) NewCompute(_ context.Context, _ sandbox.Reference, generation uint64, snapshot *sandbox.SnapshotIdentity) (sandbox.Compute, error) { + p.derived++ + return sandbox.Compute{Generation: generation, Name: "target", RestoredFrom: snapshot}, nil +} +func (p *restoreAttemptProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + p.requests = append(p.requests, q) + if !q.ObserveOnly { + return sandbox.ComputeState{}, p.stop + } + if p.outcome == "unknown" { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + closed := q.OperationID + if p.outcome == "wrong_operation" { + closed = uuid.NewString() + } + result := sandbox.ComputeState{Compute: q.Target, Status: "absent", RestoreAttemptClosed: closed} + if p.outcome == "wrong_target" { + result.Compute.Name = "another-target" + } + return result, nil +} + +func TestRestoreAttemptRecoveryUsesExactClosedEvidence(t *testing.T) { + for _, outcome := range []string{"unsent_identity", "closed", "unknown", "wrong_operation", "wrong_target"} { + t.Run(outcome, func(t *testing.T) { + stop := errors.New("native request captured") + provider := &restoreAttemptProvider{outcome: outcome, stop: stop} + f := workspaceSettlementFixtureWithSetup(t, provider, workspaceNodeSetup(t, true)) + r, session := f.lifecycle, f.session + owner, err := r.provision(t.Context(), session.TenantID, session.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + snapshot := &sandbox.SnapshotIdentity{ID: "snapshot", Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}} + state := runtimeCompute{Current: sandbox.Compute{ID: "source", Name: "source", Generation: 1}, Snapshot: snapshot, RestoreID: uuid.NewString()} + if outcome != "unsent_identity" { + state.Target = &sandbox.Compute{Name: "target", Generation: 2, RestoredFrom: snapshot} + } + raw, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='restoring',compute_revision=4,compute_retained_until=clock_timestamp()+interval '1 hour',compute_state=$2 WHERE id=$1`, owner.ID, raw); err != nil { + t.Fatal(err) + } + owner, err = r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + err = r.restoreCompute(t.Context(), provider, owner, state, true) + current, readErr := r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if readErr != nil { + t.Fatal(readErr) + } + var persisted runtimeCompute + if json.Unmarshal(current.ComputeState, &persisted) != nil { + t.Fatal("invalid persisted receipt") + } + switch outcome { + case "unsent_identity": + if !errors.Is(err, stop) || provider.derived != 1 || len(provider.requests) != 1 || provider.requests[0].ObserveOnly || persisted.Target == nil || persisted.RestoreID != state.RestoreID { + t.Fatalf("unsent intent not safely derived: %v %#v", err, provider.requests) + } + case "closed": + if !errors.Is(err, stop) || len(provider.requests) != 2 || !provider.requests[0].ObserveOnly || provider.requests[1].ObserveOnly || persisted.RestoreID == state.RestoreID || provider.requests[1].OperationID != persisted.RestoreID { + t.Fatalf("closed attempt not replaced durably: %v %#v", err, provider.requests) + } + default: + want := sandbox.ErrOwnership + if outcome == "unknown" { + want = sandbox.ErrComputeUnconfirmed + } + if !errors.Is(err, want) || len(provider.requests) != 1 || persisted.RestoreID != state.RestoreID || current.ComputeRevision != owner.ComputeRevision { + t.Fatalf("unproven attempt was replayed: %v %#v", err, provider.requests) + } + } + }) + } +} + +func TestCheckpointTransferRoutesBeforeTargetIO(t *testing.T) { + for _, operation := range []string{"wake", "expiry"} { + t.Run(operation, func(t *testing.T) { + stop := errors.New("target Resume captured") + provider := &restoreAttemptProvider{stop: stop} + f := workspaceSettlementFixtureWithSetup(t, provider, workspaceNodeSetup(t, true)) + r, session := f.lifecycle, f.session + owner, err := r.provision(t.Context(), session.TenantID, session.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + view, err := r.deployments.View(t.Context()) + if err != nil { + t.Fatal(err) + } + token, err := r.deployments.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + if err != nil { + t.Fatal(err) + } + targetNode := uuid.NewString() + node := deployment.Enrollment{NodeID: targetNode, Name: "checkpoint target", Credential: strings.Repeat("t", 64), Provider: view.Provider, BackendFingerprint: strings.Repeat("c", 64), DeploymentGeneration: view.Generation, SpecificationDigest: view.SpecificationDigest, CoreURL: fixturePublicURL} + if _, err = r.deployments.Enroll(t.Context(), token.Token, node); err != nil { + t.Fatal(err) + } + connection := uuid.NewString() + if err = r.deployments.ConnectNode(t.Context(), targetNode, connection, view.OwnerEpoch); err != nil { + t.Fatal(err) + } + compat := sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"} + if err = r.deployments.Heartbeat(t.Context(), targetNode, connection, view.OwnerEpoch, deployment.NodeHealth{ProviderReady: true}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &compat}}); err != nil { + t.Fatal(err) + } + state := runtimeCompute{Current: sandbox.Compute{ID: "source", Name: "source", Generation: 1}, Snapshot: &sandbox.SnapshotIdentity{ID: "snapshot", Compatibility: compat}} + raw, _ := json.Marshal(state) + if _, err = f.pool.Exec(t.Context(), `UPDATE environments SET initialization='complete',status='disconnected' WHERE id=$1`, owner.EnvironmentID); err != nil { + t.Fatal(err) + } + if _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state=$2,compute_revision=4,compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, owner.ID, raw); err != nil { + t.Fatal(err) + } + if _, err = f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET last_seen_at=clock_timestamp()-interval '1 minute' WHERE id=$1`, owner.NodeID); err != nil { + t.Fatal(err) + } + if operation == "wake" { + err = r.deployments.TouchActivity(t.Context(), owner.TenantID, owner.EnvironmentID) + } else { + _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID) + } + if err != nil { + t.Fatal(err) + } + source, err := r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + hints := 0 + r.hintDestination = func(id string) { + hints++ + committed, err := r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil || id != targetNode || committed.NodeID != id || committed.ID != owner.ID { + t.Fatal("hint preceded durable route or targeted wrong node", committed, err) + } + } + if err = r.observe(t.Context(), source); err != nil { + t.Fatal(err) + } + moved, err := r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + if moved.NodeID != targetNode || moved.ID != owner.ID || moved.DeviceID != owner.DeviceID || provider.derived != 0 || len(provider.requests) != 0 || provider.kills != 0 || provider.snapshots != 0 { + t.Fatal("source lane executed target effects or lost identity", moved) + } + if hints != 1 { + t.Fatalf("expected one committed handoff hint, got %d", hints) + } + // Simulate the independently scheduled target lane after the durable handoff. + r.nodeID = targetNode + err = r.observe(t.Context(), moved) + if operation == "wake" { + if !errors.Is(err, stop) || provider.derived != 1 || len(provider.requests) != 1 || provider.requests[0].ObserveOnly { + t.Fatal("target failed to use committed unsent restore", err) + } + } else { + if err != nil { + t.Fatal(err) + } + released, err := r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil || released.State != "released" || provider.kills != 0 || provider.snapshots != 1 { + t.Fatal("offline source cleanup did not settle at archive domain", released, err) + } + } + }) + } +} + +// Archive bytes may be unusable while the exact durable ownership receipt is +// still valid for deletion. Observing it must not authorize a new capture. +type cleanupReceiptProvider struct { + retentionProvider + observed bool +} + +func (p *cleanupReceiptProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + p.observed = q.ObserveOnly + if !q.ObserveOnly { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return sandbox.ComputeState{Compute: q.Source, Status: "unknown", Snapshot: &sandbox.SnapshotIdentity{ID: "owned-corrupt-archive", OperationID: q.OperationID, Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}}, nil +} +func TestLostCaptureReceiptCanStillBeDeleted(t *testing.T) { + p := &cleanupReceiptProvider{} + f := workspaceSettlementFixtureWithSetup(t, p, workspaceNodeSetup(t, true)) + r, s := f.lifecycle, f.session + owner, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + state := runtimeCompute{Current: sandbox.Compute{ID: "source", Name: "source"}, SuspendID: uuid.NewString()} + raw, _ := json.Marshal(state) + if _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspending',compute_state=$2,compute_revision=4,compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID, raw); err != nil { + t.Fatal(err) + } + owner, err = r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + if err = r.observe(t.Context(), owner); err != nil { + t.Fatal(err) + } + current, err := r.reader.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil || current.State != "released" || !p.observed || p.kills != 1 || p.snapshots != 1 { + t.Fatal("exact cleanup receipt was discarded", current, err, p.observed, p.kills, p.snapshots) + } +} diff --git a/services/core/internal/execution/runtime_wake_hint_test.go b/services/core/internal/execution/runtime_wake_hint_test.go index 955afc33c..006871999 100644 --- a/services/core/internal/execution/runtime_wake_hint_test.go +++ b/services/core/internal/execution/runtime_wake_hint_test.go @@ -8,6 +8,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) type maintenanceTestLoop struct { @@ -355,3 +357,30 @@ func TestRuntimeMaintenanceCancellationAfterSuccessfulScanWinsReadyTriggers(t *t t.Fatalf("ready triggers bypassed cancellation: calls=%d, error=%v", calls, err) } } + +func TestDestinationHintCreatesAndReusesOrdinaryLane(t *testing.T) { + m := &runtimeManager{ctx: t.Context(), config: RuntimeProvider{Mode: string(sandbox.DeploymentNodes), Provider: &retentionProvider{}}, nodes: map[string]*runtimeNode{}} + m.hintDestination("target") + n := m.nodes["target"] + if n == nil || n.lifecycle.hintDestination == nil || len(n.lifecycle.wakeHints) != 1 { + t.Fatal("missing destination lane or hint") + } + for range 10 { + m.hintDestination("target") + } + if m.nodes["target"] != n || len(n.lifecycle.wakeHints) != 1 { + t.Fatal("hint duplicated lane or failed to coalesce") + } + m.switching = true + m.hintDestination("during-transition") + if m.nodes["during-transition"] != nil { + t.Fatal("hint bypassed configuration transition") + } + m.switching = false + m.closed = true + m.hintDestination("after-close") + if m.nodes["after-close"] != nil { + t.Fatal("hint bypassed closed manager") + } + n.lifecycle.stop() +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index 8572502c5..7d67fff4b 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -334,12 +334,31 @@ func (t *allocationTx) LoadActivity(current deployment.Allocation) (deployment.A return loadActivity(t.ctx, t.q, id) } -func (t *allocationTx) LoadRestore(current deployment.Allocation) (placement.Restore, error) { - node, err := parseID(current.NodeID) +func (t *allocationTx) LoadCheckpointPlacement(current deployment.Allocation) (placement.Deployment, []placement.CheckpointNode, error) { + d, err := placementpg.LockDeployment(t.ctx, t.q) if err != nil { - return placement.Restore{}, err + return d, nil, err } - return placementpg.LoadRestore(t.ctx, t.q, node, current.DeploymentGeneration) + nodes, err := placementpg.LoadNodes(t.ctx, t.q) + if err != nil { + return d, nil, err + } + candidates, err := placementpg.LoadCheckpointNodes(t.ctx, t.q, current.DeploymentGeneration, nodes) + return d, candidates, err +} + +func (t *allocationTx) MoveSuspended(current deployment.Allocation, nodeID string, change deployment.ComputeChange) (deployment.Allocation, error) { + source, err := parseID(current.NodeID) + if err != nil { + return deployment.Allocation{}, err + } + destination, err := parseID(nodeID) + if err != nil { + return deployment.Allocation{}, err + } + return t.change(current, func(ctx context.Context, id pgtype.UUID) (sqlc.RuntimeAllocation, error) { + return t.q.MoveSuspendedRuntimeCompute(ctx, sqlc.MoveSuspendedRuntimeComputeParams{ID: id, Source: source, Destination: destination, Revision: current.ComputeRevision, Phase: change.Phase, State: change.State}) + }) } func (t *allocationTx) ObserveRunning(current deployment.Allocation) (deployment.Allocation, error) { diff --git a/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go b/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go new file mode 100644 index 000000000..919f2dac6 --- /dev/null +++ b/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go @@ -0,0 +1,301 @@ +package deploymentpg_test + +import ( + "encoding/json" + "errors" + "github.com/google/uuid" + "sync" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" +) + +var checkpointClass = sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-archive", ExecutionClass: "fixture-machine"} + +func readyCheckpointNode(t *testing.T, f fixture, node deployment.Enrollment, view deployment.View, compatibility sandbox.CheckpointCompatibility) { + t.Helper() + connection := uuid.NewString() + epoch, err := f.adapter.OwnerEpoch(t.Context()) + if err != nil { + t.Fatal(err) + } + if err := f.service.ConnectNode(t.Context(), node.NodeID, connection, epoch); err != nil { + t.Fatal(err) + } + if err := f.service.HeartbeatGenerations(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &compatibility}}); err != nil { + t.Fatal(err) + } +} + +func suspendedCheckpointOwner(t *testing.T, f fixture, changes *deployment.ExecutionOperations, installation string, node deployment.Enrollment, view deployment.View) deployment.Allocation { + t.Helper() + owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"snapshot":{"ID":"snapshot","Compatibility":{"artifact_domain":"fixture-archive","execution_class":"fixture-machine"}}}',compute_retained_until=clock_timestamp()+interval '24 hours',compute_wake_requested=true WHERE id=$1`, owner.ID); err != nil { + t.Fatal(err) + } + owner, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil { + t.Fatal(err) + } + return owner +} + +func TestCheckpointTransferPreservesAllocationAndFencesOldOwner(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + source := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + readyCheckpointNode(t, f, source, view, checkpointClass) + target := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + readyCheckpointNode(t, f, target, view, checkpointClass) + owner := suspendedCheckpointOwner(t, f, changes, installation, source, view) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, source.NodeID); err != nil { + t.Fatal(err) + } + intent := json.RawMessage(`{"snapshot":{"ID":"snapshot"},"restore_id":"attempt"}`) + moved, err := changes.BeginRestore(t.Context(), owner, checkpointClass, intent) + if err != nil { + t.Fatal(err) + } + if moved.ID != owner.ID || moved.DeviceID != owner.DeviceID || moved.DeploymentGeneration != owner.DeploymentGeneration || moved.NodeID != target.NodeID || moved.ComputePhase != "restoring" || moved.ComputeRevision != owner.ComputeRevision+1 || !moved.ComputeRetainedUntil.Equal(*owner.ComputeRetainedUntil) { + t.Fatal("lost restore ownership", moved) + } + var placementNode, credentialNode string + if err := f.pool.QueryRow(t.Context(), `SELECT p.node_id::text,a.node_id::text FROM runtime_placements p JOIN runtime_allocations a ON a.environment_id=p.environment_id JOIN devices d ON d.id=a.device_id WHERE a.id=$1 AND d.revoked_at IS NULL`, owner.ID).Scan(&placementNode, &credentialNode); err != nil || placementNode != target.NodeID || credentialNode != target.NodeID { + t.Fatal("placement/device routing split", placementNode, credentialNode, err) + } + nodes, err := f.adapter.Nodes(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, n := range nodes { + if n.ID == source.NodeID && (n.Active != 0 || n.Retained != 0) || n.ID == target.NodeID && (n.Active != 1 || n.Retained != 1) { + t.Fatal("incorrect capacity", n) + } + } + if _, err := changes.BeginRestore(t.Context(), owner, checkpointClass, intent); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal("stale source moved again", err) + } + if _, err := changes.BeginRestore(t.Context(), moved, checkpointClass, intent); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal("unknown restore moved again", err) + } + if _, err := changes.RelocateCleanup(t.Context(), moved, checkpointClass); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal("unknown restore reassigned cleanup", err) + } + +} + +func TestCheckpointTransferRequiresCompatibleGenerationAndCapacity(t *testing.T) { + for _, obstruction := range []string{"archive", "execution", "generation", "connection", "offline", "retained full", "active full", "expired", "quiescing", "no demand", "reset", "deleted", "epoch"} { + t.Run(obstruction, func(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + source := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + readyCheckpointNode(t, f, source, view, checkpointClass) + target := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + readyCheckpointNode(t, f, target, view, checkpointClass) + owner := suspendedCheckpointOwner(t, f, changes, installation, source, view) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, source.NodeID); err != nil { + t.Fatal(err) + } + var statement string + arg := target.NodeID + want := placement.ErrNodeUnavailable + switch obstruction { + case "archive": + statement = `UPDATE runtime_node_generation_status SET checkpoint='{"artifact_domain":"other","execution_class":"fixture-machine"}' WHERE node_id=$1` + case "execution": + statement = `UPDATE runtime_node_generation_status SET checkpoint='{"artifact_domain":"fixture-archive","execution_class":"other"}' WHERE node_id=$1` + case "generation": + statement = `UPDATE runtime_node_generation_status SET generation=generation+1 WHERE node_id=$1` + case "connection": + statement = `UPDATE runtime_node_generation_status SET connection_id=gen_random_uuid() WHERE node_id=$1` + case "offline": + statement = `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1` + case "retained full": + suspendedCheckpointOwner(t, f, changes, installation, target, view) + case "active full": + runningPressureOwner(t, f, changes, installation, target.NodeID, view.Generation) + case "expired": + statement = `UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1` + arg = owner.ID + want = deployment.ErrAllocationConflict + case "quiescing": + statement = `UPDATE runtime_allocations SET compute_phase='quiescing' WHERE id=$1` + arg = owner.ID + want = deployment.ErrAllocationConflict + case "no demand": + statement = `UPDATE runtime_allocations SET compute_wake_requested=false WHERE id=$1` + arg = owner.ID + want = deployment.ErrAllocationConflict + case "deleted": + statement = `UPDATE sessions SET deleted_at=clock_timestamp() WHERE id=(SELECT session_id FROM environments WHERE id=$1)` + arg = owner.EnvironmentID + want = sessions.ErrNotFound + case "epoch": + statement = `UPDATE runtime_node_generation_status SET owner_epoch=owner_epoch+1 WHERE node_id=$1` + case "reset": + statement = `UPDATE runtime_deployment SET reset_clear='force',reset_requested_at=clock_timestamp(),reset_forced_at=clock_timestamp(),reset_audit='{}' WHERE installation_id=$1` + arg = installation + want = placement.ErrResetAdmission + } + if statement != "" { + if _, err := f.pool.Exec(t.Context(), statement, arg); err != nil { + t.Fatal(err) + } + } + if _, err := changes.BeginRestore(t.Context(), owner, checkpointClass, json.RawMessage(`{"restore_id":"attempt"}`)); !errors.Is(err, want) { + t.Fatal("invalid destination admitted", err, want) + } + current, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) + if err != nil || current.NodeID != source.NodeID || current.ComputeRevision != owner.ComputeRevision { + t.Fatal("rejection changed ownership", current, err) + } + }) + } +} + +func TestCheckpointTransfersSerializeDestinationCapacity(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + source := f.enroll(t, view, deployment.Capacity{MaxActive: 2, MaxRetained: 2}) + readyCheckpointNode(t, f, source, view, checkpointClass) + target := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + readyCheckpointNode(t, f, target, view, checkpointClass) + owners := []deployment.Allocation{suspendedCheckpointOwner(t, f, changes, installation, source, view), suspendedCheckpointOwner(t, f, changes, installation, source, view)} + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, source.NodeID); err != nil { + t.Fatal(err) + } + errs := make([]error, len(owners)) + var wg sync.WaitGroup + for i, owner := range owners { + wg.Go(func() { + _, errs[i] = changes.BeginRestore(t.Context(), owner, checkpointClass, json.RawMessage(`{"restore_id":"attempt"}`)) + }) + } + wg.Wait() + success := 0 + for _, err := range errs { + if err == nil { + success++ + } else if !errors.Is(err, placement.ErrNodeUnavailable) { + t.Fatal(err) + } + } + if success != 1 { + t.Fatal("destination overbooked", errs) + } +} + +func TestCheckpointCleanupRelocatesArtifactWithoutExecutionCompatibility(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + source := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + readyCheckpointNode(t, f, source, view, checkpointClass) + target := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + other := checkpointClass + other.ExecutionClass = "other-machine" + readyCheckpointNode(t, f, target, view, other) + runningPressureOwner(t, f, changes, installation, target.NodeID, view.Generation) + owner := suspendedCheckpointOwner(t, f, changes, installation, source, view) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, source.NodeID); err != nil { + t.Fatal(err) + } + if _, err := changes.RelocateCleanup(t.Context(), owner, checkpointClass); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal("moved live allocation for cleanup", err) + } + cleanup, err := changes.RequestCleanup(t.Context(), owner) + if err != nil { + t.Fatal(err) + } + moved, err := changes.RelocateCleanup(t.Context(), cleanup, checkpointClass) + if err != nil { + t.Fatal(err) + } + if moved.NodeID != target.NodeID || moved.State != "cleanup_pending" || moved.ComputePhase != "suspended" || moved.ID != owner.ID || !moved.ComputeRetainedUntil.Equal(*owner.ComputeRetainedUntil) { + t.Fatal("cleanup recreated compute", moved) + } + nodes, err := f.adapter.Nodes(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, n := range nodes { + if n.ID == target.NodeID && (n.Active != 1 || n.Retained != 2) { + t.Fatal("cleanup released retained slot", n) + } + } + if _, err := changes.ReleaseAllocation(t.Context(), moved); err != nil { + t.Fatal(err) + } +} + +func TestCheckpointPressureSuspendsOnlyUsableDestination(t *testing.T) { + for _, scenario := range []string{"waiting restore", "retained full", "incompatible", "other free"} { + t.Run(scenario, func(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + source := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + readyCheckpointNode(t, f, source, view, checkpointClass) + suspendedCheckpointOwner(t, f, changes, installation, source, view) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, source.NodeID); err != nil { + t.Fatal(err) + } + retained := 2 + if scenario == "retained full" { + retained = 1 + } + target := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: retained}) + compatibility := checkpointClass + if scenario == "incompatible" { + compatibility.ExecutionClass = "other" + } + readyCheckpointNode(t, f, target, view, compatibility) + active := runningPressureOwner(t, f, changes, installation, target.NodeID, view.Generation) + if scenario == "other free" { + other := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + readyCheckpointNode(t, f, other, view, checkpointClass) + } + until := time.Now().Add(24 * time.Hour) + _, err := changes.SetCompute(t.Context(), active, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute) + if scenario == "waiting restore" { + if err != nil { + t.Fatal("compatible wake cannot trigger pressure suspension", err) + } + } else if !errors.Is(err, deployment.ErrNotIdle) { + t.Fatal("suspended without usable restore capacity", err) + } + }) + } +} + +func TestCheckpointRestoreUsesRetainedSlotAndImmutableGeneration(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: retainedSpecification()}) + source := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + readyCheckpointNode(t, f, source, view, checkpointClass) + owner := suspendedCheckpointOwner(t, f, changes, installation, source, view) + // A rollout must not substitute the serving generation for this snapshot. + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET ready_generation=ready_generation+1 WHERE id=$1`, source.NodeID); err != nil { + t.Fatal(err) + } + restored, err := changes.BeginRestore(t.Context(), owner, checkpointClass, json.RawMessage(`{"restore_id":"attempt"}`)) + if err != nil { + t.Fatal(err) + } + if restored.NodeID != source.NodeID || restored.DeploymentGeneration != owner.DeploymentGeneration { + t.Fatal("changed retained generation", restored) + } + nodes, err := f.adapter.Nodes(t.Context()) + if err != nil || len(nodes) != 1 || nodes[0].Active != 1 || nodes[0].Retained != 1 { + t.Fatal("restore reserved a second retained slot", nodes, err) + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go b/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go index 8c49f92f0..8032f8288 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go @@ -3,22 +3,21 @@ package deploymentpg_test import ( "bytes" "context" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "strings" "testing" - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgxpool" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" ) const fixturePublicURL = "https://core.example" @@ -146,8 +145,22 @@ func (f fixture) connect(t *testing.T, nodeID string) string { if err := f.service.ConnectNode(t.Context(), nodeID, connection, epoch); err != nil { t.Fatal(err) } - if err := f.service.Heartbeat(t.Context(), nodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + if err := f.heartbeat(t.Context(), nodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}, nil); err != nil { t.Fatal(err) } return connection } + +// heartbeat reports the fixture's verified generation alongside static readiness. +func (f fixture) heartbeat(ctx context.Context, nodeID, connection string, epoch uint64, health deployment.NodeHealth, _ []sandbox.GenerationStatus) error { + var generation uint64 + var digest, provider string + if err := f.pool.QueryRow(ctx, "SELECT n.deployment_generation, n.specification_digest, d.provider_kind FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.id=$1", nodeID).Scan(&generation, &digest, &provider); err != nil { + return err + } + var statuses []sandbox.GenerationStatus + if health.ProviderReady && provider == "microsandbox" { + statuses = []sandbox.GenerationStatus{{Generation: generation, SpecificationDigest: digest, State: "ready", Checkpoint: &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}} + } + return f.service.Heartbeat(ctx, nodeID, connection, epoch, health, statuses) +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/host_history_test.go b/services/core/internal/persistence/postgres/deploymentpg/host_history_test.go index ee83493e4..4b7c09e2a 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/host_history_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/host_history_test.go @@ -36,7 +36,7 @@ func TestNodeHostHistorySamplingAndDetail(t *testing.T) { now := time.Now().UTC() host := &deployment.NodeHost{EffectiveCPUCores: hostHistoryPtr(2.0), CPUUtilization: hostHistoryPtr(0.35), TotalMemoryBytes: hostHistoryPtr(int64(4096)), AvailableMemoryBytes: hostHistoryPtr(int64(1024)), AvailableDiskBytes: hostHistoryPtr(int64(8192)), ObservedAt: &now} health := deployment.NodeHealth{ProviderReady: true, Host: host} - if err := f.service.Heartbeat(t.Context(), node, connection, epoch, health); err != nil { + if err := f.heartbeat(t.Context(), node, connection, epoch, health, nil); err != nil { t.Fatal(err) } for i, want := range []int64{1, 0} { @@ -113,7 +113,7 @@ func TestNodeHostHistorySamplingAndDetail(t *testing.T) { // A disconnected node cannot create another history row, even with a fresh last observation. next := now.Add(time.Millisecond) host.ObservedAt = &next - if err := f.service.Heartbeat(t.Context(), node, connection, epoch, health); err != nil { + if err := f.heartbeat(t.Context(), node, connection, epoch, health, nil); err != nil { t.Fatal(err) } if err := f.service.DisconnectNode(t.Context(), node, connection, epoch); err != nil { @@ -136,7 +136,7 @@ func TestNodeHostHistorySamplingAndDetail(t *testing.T) { func TestNodeHostHistoryFencingAndUnknown(t *testing.T) { f, node, conn, epoch := hostHistoryNode(t) for _, at := range []time.Time{time.Now().Add(-time.Minute), time.Now().Add(time.Hour)} { - if err := f.service.Heartbeat(t.Context(), node, conn, epoch, deployment.NodeHealth{Host: &deployment.NodeHost{ObservedAt: &at}}); err != nil { + if err := f.heartbeat(t.Context(), node, conn, epoch, deployment.NodeHealth{Host: &deployment.NodeHost{ObservedAt: &at}}, nil); err != nil { t.Fatal(err) } if n, err := f.adapter.SampleHostHistory(t.Context()); err != nil || n != 0 { @@ -145,10 +145,10 @@ func TestNodeHostHistoryFencingAndUnknown(t *testing.T) { } now := time.Now().UTC() health := deployment.NodeHealth{Host: &deployment.NodeHost{ObservedAt: &now}} - if err := f.service.Heartbeat(t.Context(), node, uuid.NewString(), epoch, health); !errors.Is(err, deployment.ErrNodeCredential) { + if err := f.heartbeat(t.Context(), node, uuid.NewString(), epoch, health, nil); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal(err) } - if err := f.service.Heartbeat(t.Context(), node, conn, epoch, health); err != nil { + if err := f.heartbeat(t.Context(), node, conn, epoch, health, nil); err != nil { t.Fatal(err) } if n, err := f.adapter.SampleHostHistory(t.Context()); err != nil || n != 1 { @@ -164,7 +164,7 @@ func TestNodeHostHistoryFencingAndUnknown(t *testing.T) { {ObservedAt: &now, TotalMemoryBytes: hostHistoryPtr(int64(10)), AvailableMemoryBytes: hostHistoryPtr(int64(11))}, {ObservedAt: &now, AvailableDiskBytes: hostHistoryPtr(int64(-1))}, {ObservedAt: &now, EffectiveCPUCores: hostHistoryPtr(0.0)}, {}, } { - if err := f.service.Heartbeat(t.Context(), node, conn, epoch, deployment.NodeHealth{Host: host}); !errors.Is(err, deployment.ErrInvalidInput) { + if err := f.heartbeat(t.Context(), node, conn, epoch, deployment.NodeHealth{Host: host}, nil); !errors.Is(err, deployment.ErrInvalidInput) { t.Fatal(host, err) } } diff --git a/services/core/internal/persistence/postgres/deploymentpg/presence_test.go b/services/core/internal/persistence/postgres/deploymentpg/presence_test.go index 52b788997..83957fb31 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/presence_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/presence_test.go @@ -246,7 +246,7 @@ func TestNodeStaleEpochCannotReplaceCurrentConnection(t *testing.T) { if err := f.service.DisconnectNode(t.Context(), node.NodeID, connection, epoch); err != nil { t.Fatal(err) } - if err := f.service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: false}); !errors.Is(err, deployment.ErrNodeCredential) { + if err := f.heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: false}, nil); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("old Core rewrote health", err) } nodes, err := f.service.ListNodes(t.Context()) @@ -277,7 +277,7 @@ func TestNodeDiagnosticReachesListAndDetail(t *testing.T) { {reported: "dial unix /var/run/docker.sock: permission denied", want: "provider_unavailable"}, {reported: "artifacts_unavailable", want: "", ready: true}, } { - if err := f.service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: tc.ready, Diagnostic: sandbox.NodeDiagnosticCode(tc.reported)}); err != nil { + if err := f.heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: tc.ready, Diagnostic: sandbox.NodeDiagnosticCode(tc.reported)}, nil); err != nil { t.Fatal(tc.reported, err) } list, err := f.service.ListNodes(t.Context()) @@ -321,11 +321,11 @@ func TestNodeStatusUsesAuthenticatedFreshPresence(t *testing.T) { if err != nil { t.Fatal(err) } - if err := f.service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: false}); err != nil { + if err := f.heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: false}, nil); err != nil { t.Fatal(err) } status(true, false) - if err := f.service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + if err := f.heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}, nil); err != nil { t.Fatal(err) } status(true, true) diff --git a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure.go b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure.go index 1f51fb9f8..aa1d7f0fd 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure.go +++ b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure.go @@ -2,12 +2,15 @@ package deploymentpg import ( "context" + "encoding/json" "github.com/jackc/pgx/v5/pgtype" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/placementpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) func (t *allocationTx) LoadGenerationSpecification(generation uint64) (deployment.GenerationSpecification, error) { @@ -32,23 +35,28 @@ func (t *allocationTx) LoadSuspensionDemand(current deployment.Allocation) (depl if err != nil { return result, err } - node, err := parseID(current.NodeID) + restores, err := t.q.ListWaitingCheckpointRestores(t.ctx) if err != nil { return result, err } - generations, err := t.q.ListWaitingRuntimeRestoreGenerations(t.ctx, node) - if err != nil { - return result, err - } - for _, generation := range generations { - ready, err := t.q.NodeGenerationReady(t.ctx, sqlc.NodeGenerationReadyParams{NodeID: node, Generation: generation.Int64}) - if err != nil { + byGeneration := map[int64][]placement.CheckpointNode{} + for _, restore := range restores { + var compatibility sandbox.CheckpointCompatibility + if err := json.Unmarshal(restore.Checkpoint, &compatibility); err != nil { return result, err } - if ready { - result.RestoreWaiting = true - return result, nil + if compatibility.Validate() != nil { + continue + } + nodes, ok := byGeneration[restore.DeploymentGeneration.Int64] + if !ok { + nodes, err = placementpg.LoadCheckpointNodes(t.ctx, t.q, uint64(restore.DeploymentGeneration.Int64), result.Nodes) + if err != nil { + return result, err + } + byGeneration[restore.DeploymentGeneration.Int64] = nodes } + result.CheckpointRestores = append(result.CheckpointRestores, deployment.CheckpointRestoreDemand{SourceNode: uuidString(restore.NodeID), Compatibility: compatibility, Nodes: nodes}) } return result, nil } diff --git a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go index 4d3b419e0..77fed6a34 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go @@ -132,13 +132,16 @@ func TestPressureSuspensionServesRestoreOnItsOriginalNode(t *testing.T) { node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) f.connect(t, node.NodeID) waiting := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) - if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_retained_until=clock_timestamp()+interval '1 hour',compute_wake_requested=true WHERE id=$1`, waiting.ID); err != nil { + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"snapshot":{"Compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()+interval '1 hour',compute_wake_requested=true WHERE id=$1`, waiting.ID); err != nil { t.Fatal(err) } owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) - // Free capacity on another node cannot restore this retained compute. + // Free capacity in another artifact domain cannot restore this checkpoint. other := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) f.connect(t, other.NodeID) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_node_generation_status SET checkpoint='{"artifact_domain":"other","execution_class":"fixture-runtime"}' WHERE node_id=$1`, other.NodeID); err != nil { + t.Fatal(err) + } until := time.Now().Add(time.Hour) if _, err := changes.SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 5*time.Minute); err != nil { t.Fatal(err) diff --git a/services/core/internal/persistence/postgres/deploymentpg/tx.go b/services/core/internal/persistence/postgres/deploymentpg/tx.go index 28f35e1c5..4441dd7d0 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/tx.go +++ b/services/core/internal/persistence/postgres/deploymentpg/tx.go @@ -237,7 +237,14 @@ func (t *nodeTx) UpsertGenerationStatus(status deployment.GenerationStatusRecord if err != nil { return err } - return t.q.UpsertNodeGenerationStatus(t.ctx, sqlc.UpsertNodeGenerationStatusParams{NodeID: id, Generation: int64(status.Generation), SpecificationDigest: status.SpecificationDigest, ConnectionID: connection, OwnerEpoch: int64(status.OwnerEpoch), State: status.State, Diagnostic: status.Diagnostic}) + var checkpoint []byte + if status.Checkpoint != nil { + checkpoint, err = json.Marshal(status.Checkpoint) + if err != nil { + return err + } + } + return t.q.UpsertNodeGenerationStatus(t.ctx, sqlc.UpsertNodeGenerationStatusParams{NodeID: id, Generation: int64(status.Generation), SpecificationDigest: status.SpecificationDigest, ConnectionID: connection, OwnerEpoch: int64(status.OwnerEpoch), State: status.State, Diagnostic: status.Diagnostic, Checkpoint: checkpoint}) } func (t *nodeTx) PromoteServingGeneration(nodeID string, generation uint64) error { diff --git a/services/core/internal/persistence/postgres/placementpg/placementpg.go b/services/core/internal/persistence/postgres/placementpg/placementpg.go index 73fc46b35..ad5abde8a 100644 --- a/services/core/internal/persistence/postgres/placementpg/placementpg.go +++ b/services/core/internal/persistence/postgres/placementpg/placementpg.go @@ -7,6 +7,7 @@ package placementpg import ( "context" + "encoding/json" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" @@ -14,6 +15,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) // LockDeployment locks the deployment and returns it as placement reads it. @@ -65,21 +67,34 @@ func ReleasePlacement(ctx context.Context, q *sqlc.Queries, environment pgtype.U return q.ReleaseRuntimePlacement(ctx, environment) } -// LoadRestore locks the deployment and returns what restoring suspended -// compute of the generation on the node reads. -func LoadRestore(ctx context.Context, q *sqlc.Queries, nodeID pgtype.UUID, generation uint64) (placement.Restore, error) { - if _, err := q.LockRuntimeDeployment(ctx); err != nil { - return placement.Restore{}, err +// LoadCheckpointNodes joins current presence/capacity with exact-generation +// readiness. The caller holds the deployment lock for both observations. +func LoadCheckpointNodes(ctx context.Context, q *sqlc.Queries, generation uint64, nodes []placement.Node) ([]placement.CheckpointNode, error) { + ready, err := q.ListCheckpointGenerationNodes(ctx, int64(generation)) + if err != nil { + return nil, err + } + byID := make(map[string]placement.Node, len(nodes)) + for _, node := range nodes { + byID[node.ID] = node } - restore := placement.Restore{Generation: generation} - rows, err := q.ListRuntimeNodes(ctx, nodeID) - if err != nil || len(rows) == 0 { - return restore, err + candidates := make([]placement.CheckpointNode, 0, len(ready)) + for _, row := range ready { + if len(row.Checkpoint) == 0 { + continue + } + var compatibility sandbox.CheckpointCompatibility + if err := json.Unmarshal(row.Checkpoint, &compatibility); err != nil { + return nil, err + } + if err := compatibility.Validate(); err != nil { + return nil, err + } + if node, ok := byID[uuidString(row.ID)]; ok { + candidates = append(candidates, placement.CheckpointNode{Node: node, Checkpoint: &compatibility}) + } } - n := node(rows[0]) - restore.Node = &n - restore.GenerationReady, err = q.NodeGenerationReady(ctx, sqlc.NodeGenerationReadyParams{NodeID: nodeID, Generation: int64(generation)}) - return restore, err + return candidates, nil } // ComputeBlocksAdmission reports whether the Session's managed compute is in diff --git a/services/core/internal/sandbox/docker/node.go b/services/core/internal/sandbox/docker/node.go index 115d6bf0c..b8ddc04fb 100644 --- a/services/core/internal/sandbox/docker/node.go +++ b/services/core/internal/sandbox/docker/node.go @@ -68,7 +68,9 @@ func BuildNode(config sandbox.NodeConfig, _ sandbox.LocalOptions, result *sandbo return func() {}, errors.New("invalid managed Docker provider configuration") } result.Provider = provider - result.Probe = dockerProbe(c, entry.Image, config.Specification.Resources) + result.Probe = func(ctx context.Context) (*sandbox.CheckpointCompatibility, error) { + return nil, dockerProbe(c, entry.Image, config.Specification.Resources)(ctx) + } result.BackendFingerprint = sandbox.BackendFingerprint(config.Provider, entry.Host) return closeProvider, nil } diff --git a/services/core/internal/sandbox/generation.go b/services/core/internal/sandbox/generation.go index f191d2859..66bbbda8d 100644 --- a/services/core/internal/sandbox/generation.go +++ b/services/core/internal/sandbox/generation.go @@ -10,10 +10,11 @@ import ( // GenerationStatus is one sparse observation, not a complete provider inventory. // A serving pin is owned by Core and is independent of target preparation. type GenerationStatus struct { - Generation uint64 `json:"generation"` - SpecificationDigest string `json:"specification_digest"` - State string `json:"state"` - Diagnostic string `json:"diagnostic,omitempty"` + Generation uint64 `json:"generation"` + SpecificationDigest string `json:"specification_digest"` + State string `json:"state"` + Diagnostic string `json:"diagnostic,omitempty"` + Checkpoint *CheckpointCompatibility `json:"checkpoint,omitempty"` } type GenerationReference struct { diff --git a/services/core/internal/sandbox/microsandbox/checkpoint_linux.go b/services/core/internal/sandbox/microsandbox/checkpoint_linux.go new file mode 100644 index 000000000..e28018709 --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/checkpoint_linux.go @@ -0,0 +1,133 @@ +//go:build linux + +package microsandbox + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "path/filepath" + "sort" + "strings" + "syscall" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// CheckpointStore opens only an explicitly configured, private archive root. +// Its marker is an operator-owned identity, never inferred from workspace paths. +func CheckpointStore(c Config) (string, error) { + if !filepath.IsAbs(c.CheckpointRoot) || filepath.Clean(c.CheckpointRoot) != c.CheckpointRoot { + return "", sandbox.ErrInvalid + } + real, err := filepath.EvalSymlinks(c.CheckpointRoot) + if err != nil { + return "", err + } + if real != c.CheckpointRoot { + return "", sandbox.ErrOwnership + } + fd, err := syscall.Open(c.CheckpointRoot, syscall.O_RDONLY|syscall.O_DIRECTORY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0) + if err != nil { + return "", err + } + f := os.NewFile(uintptr(fd), c.CheckpointRoot) + defer f.Close() + st, err := f.Stat() + if err != nil { + return "", err + } + if st.Mode().Perm() != 0700 || st.Sys().(*syscall.Stat_t).Uid != uint32(os.Geteuid()) { + return "", sandbox.ErrOwnership + } + marker, err := syscall.Openat(fd, ".oac-checkpoint-store", syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0) + if err != nil { + return "", err + } + m := os.NewFile(uintptr(marker), "checkpoint marker") + defer m.Close() + st, err = m.Stat() + if err != nil { + return "", err + } + if !st.Mode().IsRegular() || st.Mode().Perm() != 0600 || st.Sys().(*syscall.Stat_t).Uid != uint32(os.Geteuid()) { + return "", sandbox.ErrOwnership + } + data, err := io.ReadAll(io.LimitReader(m, 38)) + if err != nil { + return "", err + } + id := strings.TrimSuffix(string(data), "\n") + if !validID(id) || string(data) != id+"\n" { + return "", sandbox.ErrOwnership + } + h := sha256.Sum256([]byte(c.InstallationID + ":" + id)) + return hex.EncodeToString(h[:]), nil +} + +// CheckpointClass deliberately admits only identical native and host execution +// profiles. Paths, hostname and local SDK cache identities are not compatibility. +func CheckpointClass(c Config) (sandbox.CheckpointCompatibility, error) { + domain, err := CheckpointStore(c) + if err != nil { + return sandbox.CheckpointCompatibility{}, err + } + cpu, err := os.ReadFile("/proc/cpuinfo") + if err != nil { + return sandbox.CheckpointCompatibility{}, err + } + stable := checkpointCPUProfiles(string(cpu)) + if len(stable) == 0 { + return sandbox.CheckpointCompatibility{}, sandbox.ErrInvalid + } + kernel, err := os.ReadFile("/proc/sys/kernel/osrelease") + if err != nil { + return sandbox.CheckpointCompatibility{}, err + } + raw, _ := json.Marshal(struct { + SDK, Runtime, Firmware, Image, Kernel string + CPUs uint8 + Memory, Root, Environment uint32 + CPU []string + }{SDKVersion, c.RuntimeSHA256, c.FirmwareSHA256, c.Image, strings.TrimSpace(string(kernel)), c.CPUs, c.MemoryMiB, c.RootDiskMiB, c.EnvironmentDiskMiB, stable}) + h := sha256.Sum256(raw) + return sandbox.CheckpointCompatibility{ArtifactDomain: domain, ExecutionClass: hex.EncodeToString(h[:])}, nil +} + +// Preserve each processor's feature/model association. Multiplicity and logical +// processor indexes are irrelevant; distinct heterogeneous profiles are not. +func checkpointCPUProfiles(cpuinfo string) []string { + profiles := map[string]bool{} + for _, block := range strings.Split(strings.TrimSpace(cpuinfo), "\n\n") { + fields := []string{} + for _, line := range strings.Split(block, "\n") { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + switch key { + case "vendor_id", "cpu family", "model", "stepping", "flags", "Features", "CPU implementer", "CPU architecture", "CPU part", "CPU revision": + value = strings.TrimSpace(value) + if key == "flags" || key == "Features" { + features := strings.Fields(value) + sort.Strings(features) + value = strings.Join(features, " ") + } + fields = append(fields, key+":"+value) + } + } + if len(fields) > 0 { + sort.Strings(fields) + profiles[strings.Join(fields, "\n")] = true + } + } + result := make([]string, 0, len(profiles)) + for profile := range profiles { + result = append(result, profile) + } + sort.Strings(result) + return result +} diff --git a/services/core/internal/sandbox/microsandbox/checkpoint_linux_test.go b/services/core/internal/sandbox/microsandbox/checkpoint_linux_test.go new file mode 100644 index 000000000..0d296776a --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/checkpoint_linux_test.go @@ -0,0 +1,88 @@ +//go:build linux + +package microsandbox + +import ( + "errors" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +func privateCheckpointRoot(t *testing.T, marker string) string { + t.Helper() + root := t.TempDir() + if err := os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, ".oac-checkpoint-store"), []byte(marker+"\n"), 0600); err != nil { + t.Fatal(err) + } + return root +} +func TestCheckpointCompatibilityUsesStoreIdentityNotMountOrCachePath(t *testing.T) { + c := testConfig() + c.CheckpointRoot = privateCheckpointRoot(t, "11111111-1111-4111-8111-111111111111") + a, err := CheckpointClass(c) + if err != nil { + t.Fatal(err) + } + c.CheckpointRoot = privateCheckpointRoot(t, "11111111-1111-4111-8111-111111111111") + c.RuntimeHome = "/different/cache" + c.RuntimePath = "/different/msb" + c.FirmwarePath = "/different/firmware" + b, err := CheckpointClass(c) + if err != nil || a != b { + t.Fatal("host paths changed compatibility", a, b, err) + } + c.MemoryMiB++ + b, err = CheckpointClass(c) + if err != nil || a.ExecutionClass == b.ExecutionClass || a.ArtifactDomain != b.ArtifactDomain { + t.Fatal("geometry not fenced", a, b, err) + } + c.CheckpointRoot = privateCheckpointRoot(t, "22222222-2222-4222-8222-222222222222") + b, err = CheckpointClass(c) + if err != nil || a.ArtifactDomain == b.ArtifactDomain { + t.Fatal("foreign store matched", a, b, err) + } +} +func TestCheckpointStoreRequiresPrivateCanonicalIdentity(t *testing.T) { + c := testConfig() + c.CheckpointRoot = privateCheckpointRoot(t, "11111111-1111-4111-8111-111111111111") + if err := os.Chmod(c.CheckpointRoot, 0755); err != nil { + t.Fatal(err) + } + if _, err := CheckpointStore(c); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal(err) + } + if err := os.Chmod(c.CheckpointRoot, 0700); err != nil { + t.Fatal(err) + } + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(c.CheckpointRoot, link); err != nil { + t.Fatal(err) + } + c.CheckpointRoot = link + if _, err := CheckpointStore(c); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal(err) + } + c.CheckpointRoot = privateCheckpointRoot(t, "00000000-0000-0000-0000-000000000000") + if _, err := CheckpointStore(c); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal(err) + } +} + +func TestCPUCompatibilityPreservesHeterogeneousProcessorProfiles(t *testing.T) { + first := "processor:0\nmodel:1\nflags:a b\n\nprocessor:1\nmodel:2\nflags:c d\n" + reordered := "processor:9\nflags:d c\nmodel:2\n\nprocessor:4\nflags:b a\nmodel:1\n\nprocessor:5\nflags:a b\nmodel:1\n" + crossed := "processor:0\nmodel:1\nflags:c d\n\nprocessor:1\nmodel:2\nflags:a b\n" + if !reflect.DeepEqual(checkpointCPUProfiles(first), checkpointCPUProfiles(reordered)) { + t.Fatal("processor order/count affected class") + } + if reflect.DeepEqual(checkpointCPUProfiles(first), checkpointCPUProfiles(crossed)) { + t.Fatal("heterogeneous CPU associations collapsed") + } +} diff --git a/services/core/internal/sandbox/microsandbox/checkpoint_other.go b/services/core/internal/sandbox/microsandbox/checkpoint_other.go new file mode 100644 index 000000000..dc92ab8d7 --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/checkpoint_other.go @@ -0,0 +1,10 @@ +//go:build !linux + +package microsandbox + +import "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + +func CheckpointStore(Config) (string, error) { return "", sandbox.ErrHostUnsupported } +func CheckpointClass(Config) (sandbox.CheckpointCompatibility, error) { + return sandbox.CheckpointCompatibility{}, sandbox.ErrHostUnsupported +} diff --git a/services/core/internal/sandbox/microsandbox/identity.go b/services/core/internal/sandbox/microsandbox/identity.go index b831c016d..80ddc2840 100644 --- a/services/core/internal/sandbox/microsandbox/identity.go +++ b/services/core/internal/sandbox/microsandbox/identity.go @@ -24,6 +24,9 @@ func validHash(s string) bool { return e == nil && len(b) == 32 && strings.ToLower(s) == s } func (c Config) Validate() error { + if !filepath.IsAbs(c.CheckpointRoot) || filepath.Clean(c.CheckpointRoot) != c.CheckpointRoot { + return sandbox.ErrInvalid + } if !validID(c.InstallationID) || !filepath.IsAbs(c.HelperPath) || !filepath.IsAbs(c.RuntimeHome) || !filepath.IsAbs(c.RuntimePath) || !filepath.IsAbs(c.FirmwarePath) || !validHash(c.RuntimeSHA256) || !validHash(c.FirmwareSHA256) || c.MemoryMiB == 0 || c.CPUs == 0 || c.RootDiskMiB == 0 || (!c.ExternalWorkspace && c.EnvironmentDiskMiB == 0) { return sandbox.ErrInvalid } @@ -70,7 +73,7 @@ func ValidateCompute(c Config, r sandbox.Reference, v Compute) error { return nil } func ValidateSnapshot(c Config, r sandbox.Reference, s SnapshotIdentity) error { - if !ValidReference(r) || !validID(s.OperationID) || s.Reference != SnapshotReference(c, r, s.OperationID) || s.SourceName != Name(c, r, s.SourceGeneration) || s.SourceID == "" || s.ID == "" || s.Digest == "" || s.CheckpointID == "" || s.CheckpointRoot == "" { + if s.Compatibility.Validate() != nil || !ValidReference(r) || !validID(s.OperationID) || s.Reference != SnapshotReference(c, r, s.OperationID) || s.SourceName != Name(c, r, s.SourceGeneration) || s.SourceID == "" || s.ID == "" || s.Digest == "" || s.CheckpointID == "" || s.CheckpointRoot == "" { return sandbox.ErrInvalid } return nil @@ -99,6 +102,12 @@ func ValidateRequest(q Request) error { if q.Workspace != nil && ((q.Operation != "create" && q.Operation != "resume") || !validID(q.Workspace.ObjectID) || !filepath.IsAbs(q.Workspace.Path) || filepath.Clean(q.Workspace.Path) != q.Workspace.Path || strings.ContainsRune(q.Workspace.Path, 0)) { return sandbox.ErrInvalid } + if q.Workspace != nil { + relative, err := filepath.Rel(q.Workspace.Path, q.Config.CheckpointRoot) + if err != nil || relative == "." || (relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))) { + return sandbox.ErrInvalid + } + } switch q.Operation { case "create": if q.Config.ExternalWorkspace != (q.Workspace != nil) { diff --git a/services/core/internal/sandbox/microsandbox/node.go b/services/core/internal/sandbox/microsandbox/node.go index 8f2789053..2b68552dc 100644 --- a/services/core/internal/sandbox/microsandbox/node.go +++ b/services/core/internal/sandbox/microsandbox/node.go @@ -1,6 +1,7 @@ package microsandbox import ( + "context" "errors" "os" "path/filepath" @@ -23,11 +24,12 @@ var NodeArtifacts = []providerassets.Artifact{ // The helper owns local paths; no ambient backend is selected. Resources, the // image and the artifact hashes come from the deployment specification. type Native struct { - HelperPath string `json:"helper_path"` - RuntimeHome string `json:"runtime_home"` - RuntimePath string `json:"runtime_path"` - FirmwarePath string `json:"firmware_path"` - Network Network `json:"network"` + HelperPath string `json:"helper_path"` + RuntimeHome string `json:"runtime_home"` + CheckpointRoot string `json:"checkpoint_root"` + RuntimePath string `json:"runtime_path"` + FirmwarePath string `json:"firmware_path"` + Network Network `json:"network"` } type Network struct { @@ -55,7 +57,7 @@ func configureMicrosandbox(entry Native, spec sandbox.DeploymentSpec, caller *Pr release, resources := spec.Runtime, spec.Resources config := Config{ ExternalWorkspace: spec.Workspace != nil, - InstallationID: result.InstallationID, HelperPath: entry.HelperPath, RuntimeHome: entry.RuntimeHome, RuntimePath: entry.RuntimePath, FirmwarePath: entry.FirmwarePath, + InstallationID: result.InstallationID, HelperPath: entry.HelperPath, RuntimeHome: entry.RuntimeHome, CheckpointRoot: entry.CheckpointRoot, RuntimePath: entry.RuntimePath, FirmwarePath: entry.FirmwarePath, RuntimeSHA256: release.RuntimeSHA256, FirmwareSHA256: release.FirmwareSHA256, Image: release.MicrosandboxRef, MemoryMiB: resources.MemoryMiB, CPUs: uint8(resources.CPUs), RootDiskMiB: resources.RootDiskMiB, EnvironmentDiskMiB: resources.EnvironmentDiskMiB, Network: network, } @@ -65,7 +67,17 @@ func configureMicrosandbox(entry Native, spec sandbox.DeploymentSpec, caller *Pr } provider.workspace = options.Workspace result.Provider = provider - result.Probe = microsandboxProbe(config, resources) + baseProbe := microsandboxProbe(config, resources) + result.Probe = func(ctx context.Context) (*sandbox.CheckpointCompatibility, error) { + if err := baseProbe(ctx); err != nil { + return nil, err + } + c, err := CheckpointClass(config) + if err != nil { + return nil, err + } + return &c, nil + } result.Quiescent = caller.Quiescent result.BackendFingerprint = sandbox.BackendFingerprint("microsandbox", entry.RuntimeHome) return config, nil @@ -75,7 +87,7 @@ func configureMicrosandbox(entry Native, spec sandbox.DeploymentSpec, caller *Pr func BuildNode(c sandbox.NodeConfig, options sandbox.LocalOptions, result *sandbox.Built) (func(), error) { closeProvider := func() {} var entry Native - if sandbox.DecodeConfigurationObject(c.Native, &entry, "helper_path", "runtime_home", "runtime_path", "firmware_path", "network") != nil { + if sandbox.DecodeConfigurationObject(c.Native, &entry, "helper_path", "runtime_home", "runtime_path", "firmware_path", "network", "checkpoint_root") != nil { return closeProvider, errors.New("invalid managed microsandbox node configuration") } caller := &ProcessCaller{} @@ -96,7 +108,15 @@ func BuildNode(c sandbox.NodeConfig, options sandbox.LocalOptions, result *sandb return closeProvider, err } if options.GenerationStateDirectory != "" { - result.Probe = microsandboxGenerationProbe(config, result.Probe) + baseProbe := result.Probe + result.Probe = func(ctx context.Context) (*sandbox.CheckpointCompatibility, error) { + var compatibility *sandbox.CheckpointCompatibility + imageProbe := microsandboxGenerationProbe(config, func(ctx context.Context) error { var err error; compatibility, err = baseProbe(ctx); return err }) + if err := imageProbe(ctx); err != nil { + return nil, err + } + return compatibility, nil + } } return closeProvider, nil } diff --git a/services/core/internal/sandbox/microsandbox/node_test.go b/services/core/internal/sandbox/microsandbox/node_test.go index 082bfd5a7..8e542fda8 100644 --- a/services/core/internal/sandbox/microsandbox/node_test.go +++ b/services/core/internal/sandbox/microsandbox/node_test.go @@ -21,7 +21,13 @@ func TestMicrosandboxConstructionSelectsGenerationReadiness(t *testing.T) { } useKVM(t, true) dir := t.TempDir() - native := Native{HelperPath: filepath.Join(dir, "helper"), RuntimeHome: dir, RuntimePath: filepath.Join(dir, "msb"), FirmwarePath: filepath.Join(dir, "firmware"), Network: Network{DefaultEgress: "allow", DefaultIngress: "deny"}} + native := Native{HelperPath: filepath.Join(dir, "helper"), RuntimeHome: dir, CheckpointRoot: dir, RuntimePath: filepath.Join(dir, "msb"), FirmwarePath: filepath.Join(dir, "firmware"), Network: Network{DefaultEgress: "allow", DefaultIngress: "deny"}} + if err := os.Chmod(dir, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".oac-checkpoint-store"), []byte(uuid.NewString()+"\n"), 0600); err != nil { + t.Fatal(err) + } raw, err := json.Marshal(native) if err != nil { t.Fatal(err) @@ -44,7 +50,7 @@ func TestMicrosandboxConstructionSelectsGenerationReadiness(t *testing.T) { if err != nil { t.Fatal(err) } - err = built.Probe(t.Context()) + _, err = built.Probe(t.Context()) closeProvider() if options.Standalone && err != nil || !options.Standalone && !errors.Is(err, sandbox.ErrRuntimeImageUnavailable) { t.Fatalf("readiness for %+v: %v", options, err) diff --git a/services/core/internal/sandbox/microsandbox/provider.go b/services/core/internal/sandbox/microsandbox/provider.go index 0b9f62800..abf5acec7 100644 --- a/services/core/internal/sandbox/microsandbox/provider.go +++ b/services/core/internal/sandbox/microsandbox/provider.go @@ -88,6 +88,13 @@ func (p *Provider) state(ctx context.Context, q Request) (State, error) { func (p *Provider) responseState(ctx context.Context, q Request, out Response) (State, error) { var e error + if out.State != nil && out.State.RestoreAttemptClosed != "" { + if q.Operation != "resume" || q.Resume == nil || !out.State.ClosesRestoreAttempt(*q.Resume) { + return State{}, ErrUnconfirmed + } + return *out.State, nil + } + if out.State == nil || ValidateCompute(p.config, q.Reference, out.State.Compute) != nil || out.State.Compute.ID == "" { return State{}, ErrUnconfirmed } diff --git a/services/core/internal/sandbox/microsandbox/provider_test.go b/services/core/internal/sandbox/microsandbox/provider_test.go index 94bfa9881..db67511af 100644 --- a/services/core/internal/sandbox/microsandbox/provider_test.go +++ b/services/core/internal/sandbox/microsandbox/provider_test.go @@ -14,7 +14,7 @@ type callerFunc func(context.Context, Request) (Response, error) func (f callerFunc) Call(c context.Context, q Request) (Response, error) { return f(c, q) } func testConfig() Config { - return Config{ + return Config{CheckpointRoot: "/private-checkpoints", InstallationID: "11111111-1111-4111-8111-111111111111", HelperPath: "/helper", RuntimeHome: "/private/msb", RuntimePath: "/private/bin/msb", FirmwarePath: "/private/lib/libkrunfw.so", RuntimeSHA256: strings.Repeat("a", 64), FirmwareSHA256: strings.Repeat("b", 64), Image: "registry/runtime@sha256:" + strings.Repeat("c", 64), MemoryMiB: 2048, CPUs: 2, RootDiskMiB: 4096, EnvironmentDiskMiB: 2048, Network: NetworkPolicy{DefaultEgress: "deny", DefaultIngress: "deny", Rules: []NetworkRule{{Action: "allow", Direction: "egress", Destination: "host"}}}, @@ -26,7 +26,7 @@ func testRef() sandbox.Reference { func testSnapshot() SnapshotIdentity { c, r := testConfig(), testRef() op := "55555555-5555-4555-8555-555555555555" - return SnapshotIdentity{Reference: SnapshotReference(c, r, op), ID: "snap_exact", Digest: "digest", CheckpointID: "checkpoint", CheckpointRoot: "root", OperationID: op, SourceName: Name(c, r, 0), SourceID: "local:4"} + return SnapshotIdentity{Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "domain", ExecutionClass: "class"}, Reference: SnapshotReference(c, r, op), ID: "snap_exact", Digest: "digest", CheckpointID: "checkpoint", CheckpointRoot: "root", OperationID: op, SourceName: Name(c, r, 0), SourceID: "local:4"} } func deadline(t *testing.T) context.Context { t.Helper() diff --git a/services/core/internal/sandbox/microsandbox/restore_settlement_test.go b/services/core/internal/sandbox/microsandbox/restore_settlement_test.go new file mode 100644 index 000000000..bbf00779b --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/restore_settlement_test.go @@ -0,0 +1,41 @@ +package microsandbox + +import ( + "testing" +) + +func TestRestoreClosedReceiptRequiresExactObserveOnlyOperation(t *testing.T) { + p := &Provider{config: testConfig()} + snapshot := testSnapshot() + target := Compute{Name: Name(testConfig(), testRef(), 1), Generation: 1, RestoredFrom: &snapshot} + q := Request{Operation: "resume", Reference: testRef(), Resume: &ResumeRequest{OperationID: "66666666-6666-4666-8666-666666666666", Target: target, Snapshot: snapshot, ObserveOnly: true}} + valid := State{Compute: target, Status: "absent", RestoreAttemptClosed: q.Resume.OperationID} + for _, kind := range []string{"exact", "wrong operation", "native ID", "foreign target", "bootstrap", "snapshot", "source stopped", "not observation"} { + t.Run(kind, func(t *testing.T) { + request := q + resume := *q.Resume + request.Resume = &resume + state := valid + switch kind { + case "wrong operation": + state.RestoreAttemptClosed = "wrong" + case "native ID": + state.Compute.ID = "local:1" + case "foreign target": + state.Compute.Name = "foreign" + case "bootstrap": + state.BootstrapComplete = true + case "snapshot": + state.Snapshot = &snapshot + case "source stopped": + state.SourceStopped = true + case "not observation": + request.Resume.ObserveOnly = false + } + _, err := p.responseState(t.Context(), request, Response{State: &state}) + if (kind == "exact") != (err == nil) { + t.Fatal(kind, err) + } + }) + } +} diff --git a/services/core/internal/sandbox/microsandbox/types.go b/services/core/internal/sandbox/microsandbox/types.go index 68eb8b3d3..43442c229 100644 --- a/services/core/internal/sandbox/microsandbox/types.go +++ b/services/core/internal/sandbox/microsandbox/types.go @@ -9,7 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -const ProtocolVersion = 4 +const ProtocolVersion = 5 const SDKVersion = "v0.7.8" const MaxOutputBytes = 1024 * 1024 const MaxRequestBytes = 72 * 1024 * 1024 @@ -23,6 +23,7 @@ type Config struct { InstallationID string HelperPath string RuntimeHome string + CheckpointRoot string RuntimePath string FirmwarePath string RuntimeSHA256 string diff --git a/services/core/internal/sandbox/node/checkpoint_generation_test.go b/services/core/internal/sandbox/node/checkpoint_generation_test.go new file mode 100644 index 000000000..5f5bd6251 --- /dev/null +++ b/services/core/internal/sandbox/node/checkpoint_generation_test.go @@ -0,0 +1,98 @@ +package node + +import ( + "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" + "github.com/google/uuid" + "net/http/httptest" + "strings" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +func TestStaticNodeReportsExactCheckpointGeneration(t *testing.T) { + id := identity() + compatibility := &sandbox.CheckpointCompatibility{ArtifactDomain: "private-store", ExecutionClass: "native-class"} + status := sandbox.GenerationStatus{Generation: id.DeploymentGeneration, SpecificationDigest: id.SpecificationDigest, State: "ready", Checkpoint: compatibility} + seen := false + hub := NewHub(HubOptions{Heartbeat: func(_ context.Context, _ Identity, _ string, _ uint64, health Health) error { + seen = true + if len(health.Generations) != 1 || *health.Generations[0].Checkpoint != *compatibility { + t.Fatal("qualification lost") + } + return nil + }}) + p := &peer{identity: id} + health := Health{ProviderReady: true, Generations: []sandbox.GenerationStatus{status}} + if err := hub.recordHealth(t.Context(), p, health); err != nil || !seen { + t.Fatal(err, seen) + } + for _, mutation := range []string{"generation", "digest", "readiness", "extra"} { + t.Run(mutation, func(t *testing.T) { + changed := health + changed.Generations = append([]sandbox.GenerationStatus(nil), health.Generations...) + switch mutation { + case "generation": + changed.Generations[0].Generation++ + case "digest": + changed.Generations[0].SpecificationDigest = strings.Repeat("f", 64) + case "readiness": + changed.ProviderReady = false + case "extra": + changed.Generations = append(changed.Generations, status) + } + seen = false + if err := hub.recordHealth(t.Context(), p, changed); err == nil || seen { + t.Fatal("invalid static qualification reached persistence") + } + }) + } +} + +type closedRestoreProvider struct{ fakeProvider } + +func (*closedRestoreProvider) ProviderOperations() providercontract.Operations { + return microsandbox.Operations() +} +func (*closedRestoreProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + return sandbox.ComputeState{Compute: q.Target, Status: "absent", RestoreAttemptClosed: q.OperationID}, nil +} +func TestClosedRestoreEvidenceSurvivesNodeTransport(t *testing.T) { + id := identity() + var credential string + hub := NewHub(HubOptions{Authenticate: func(_ context.Context, node, token string) (Identity, error) { + if node != id.NodeID || token != credential { + return Identity{}, ErrAuthentication + } + return id, nil + }, OwnerEpoch: func(context.Context) (uint64, error) { return 7, nil }}) + defer hub.Close() + server := httptest.NewServer(hub) + defer server.Close() + directory := stateDir(t) + stored, err := InitIdentity(directory, server.URL, id) + if err != nil { + t.Fatal(err) + } + credential = stored.Credential + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { + done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: directory, Identity: id, Credential: credential, Provider: &closedRestoreProvider{}, Probe: probe}) + }() + defer func() { + cancel() + if err := <-done; err != nil { + t.Error(err) + } + }() + wait(t, func() bool { return hub.Online(id.NodeID) }) + snapshot := sandbox.SnapshotIdentity{ID: "snapshot", Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "store", ExecutionClass: "class"}} + q := sandbox.ResumeRequest{Reference: reference(), OperationID: uuid.NewString(), Snapshot: snapshot, Target: sandbox.Compute{Generation: 1, Name: "exact-target", RestoredFrom: &snapshot}, ObserveOnly: true} + result, err := hub.Proxy(id.NodeID, microsandbox.Operations(), id.DeploymentGeneration).Resume(t.Context(), q) + if err != nil || !result.ClosesRestoreAttempt(q) { + t.Fatalf("exact closed proof lost in proxy: %#v %v", result, err) + } +} diff --git a/services/core/internal/sandbox/node/generation_json.go b/services/core/internal/sandbox/node/generation_json.go index 5336a320f..06ae10fff 100644 --- a/services/core/internal/sandbox/node/generation_json.go +++ b/services/core/internal/sandbox/node/generation_json.go @@ -62,9 +62,15 @@ func generationRecords(raw []byte, required, optional string) error { return sandbox.ErrInvalid } for _, entry := range entries { - if _, err := generationObject(entry, required, optional, ""); err != nil { + fields, err := generationObject(entry, required, optional, "") + if err != nil { return err } + if checkpoint := fields["checkpoint"]; checkpoint != nil { + if _, err := generationObject(checkpoint, "artifact_domain execution_class", "", ""); err != nil { + return err + } + } } return nil } @@ -110,7 +116,7 @@ func validateGenerationJSON(raw []byte, kind string) error { if err != nil { return err } - if err := generationRecords(health["generations"], "generation specification_digest state", "diagnostic"); err != nil { + if err := generationRecords(health["generations"], "generation specification_digest state", "diagnostic checkpoint"); err != nil { return err } } diff --git a/services/core/internal/sandbox/node/generation_json_test.go b/services/core/internal/sandbox/node/generation_json_test.go index b6fef3a10..2f0f7e069 100644 --- a/services/core/internal/sandbox/node/generation_json_test.go +++ b/services/core/internal/sandbox/node/generation_json_test.go @@ -2,6 +2,7 @@ package node import ( "encoding/json" + "fmt" "strings" "testing" "time" @@ -63,7 +64,7 @@ func TestGenerationHealthRejectsUnboundedOrAmbiguousNumbers(t *testing.T) { } func TestNodeProtocolRejectsHistoricalVersions(t *testing.T) { - for _, version := range []int{1, 2, 3, 4} { + for _, version := range []int{1, 2, 3, 4, 5} { raw, _ := json.Marshal(frame{Version: version, Type: "hello", Identity: new(Identity), Health: &Health{ObservedAt: time.Now().UTC()}}) if _, err := decodeFrame(raw); err == nil { t.Fatalf("accepted historical protocol %d", version) @@ -84,9 +85,66 @@ func TestNodeGenerationManagementIsAnExplicitCurrentCapability(t *testing.T) { if managed { f.GenerationManagement = false raw, _ = json.Marshal(f) - if _, err := decodeFrame(raw); err == nil { - t.Fatal("generation management accepted without declaration") + decoded, err := decodeFrame(raw) + if err != nil || decoded.GenerationManagement || len(decoded.Health.Generations) != 1 { + t.Fatal("static qualification implicitly enabled generation management", err) } } } } + +func TestCheckpointGenerationHealthJSONRoundTrip(t *testing.T) { + for _, managed := range []bool{false, true} { + for _, kind := range []string{"hello", "heartbeat"} { + t.Run(fmt.Sprintf("managed=%t/%s", managed, kind), func(t *testing.T) { + compatibility := sandbox.CheckpointCompatibility{ArtifactDomain: "private-store", ExecutionClass: "native-class"} + f := frame{Version: ProtocolVersion, Type: kind, Health: &Health{ProviderReady: true, ObservedAt: time.Now().UTC(), Generations: []sandbox.GenerationStatus{{Generation: 5, SpecificationDigest: strings.Repeat("a", 64), State: "ready", Checkpoint: &compatibility}}}} + if kind == "hello" { + f.Identity = new(Identity) + f.GenerationManagement = managed + } else { + f.ConnectionID = uuid.NewString() + f.OwnerEpoch = 1 + } + raw, err := json.Marshal(f) + if err != nil { + t.Fatal(err) + } + if err := validateVersionFrame(f, len(raw)); err != nil { + t.Fatal("writer rejected valid status", err) + } + decoded, err := decodeFrame(raw) + if err != nil { + t.Fatal("decoder rejected valid status", err) + } + if decoded.Health == nil || len(decoded.Health.Generations) != 1 || decoded.Health.Generations[0].Checkpoint == nil || *decoded.Health.Generations[0].Checkpoint != compatibility { + t.Fatal("checkpoint qualification lost", decoded) + } + original := string(raw) + for name, replacement := range map[string]string{ + "unknown nested field": strings.Replace(original, `"artifact_domain":"private-store"`, `"artifact_domain":"private-store","extra":"value"`, 1), + "unknown generation field": strings.Replace(original, `"checkpoint":`, `"extra":true,"checkpoint":`, 1), + "duplicate token": strings.Replace(original, `"artifact_domain":"private-store"`, `"artifact_domain":"private-store","artifact_domain":"other"`, 1), + "case alias": strings.Replace(original, `"artifact_domain":`, `"Artifact_Domain":`, 1), + "missing token": strings.Replace(original, `"artifact_domain":"private-store",`, "", 1), + "null token": strings.Replace(original, `"artifact_domain":"private-store"`, `"artifact_domain":null`, 1), + "null checkpoint": strings.Replace(original, `{"artifact_domain":"private-store","execution_class":"native-class"}`, `null`, 1), + "empty domain": strings.Replace(original, `"private-store"`, `""`, 1), + "empty execution": strings.Replace(original, `"native-class"`, `""`, 1), + "oversized domain": strings.Replace(original, `"private-store"`, `"`+strings.Repeat("x", 257)+`"`, 1), + "oversized execution": strings.Replace(original, `"native-class"`, `"`+strings.Repeat("x", 257)+`"`, 1), + "nonready checkpoint": strings.Replace(original, `"state":"ready"`, `"state":"preparing"`, 1), + } { + t.Run(name, func(t *testing.T) { + if replacement == original { + t.Fatal("fixture unchanged") + } + if _, err := decodeFrame([]byte(replacement)); err == nil { + t.Fatal("invalid checkpoint accepted") + } + }) + } + }) + } + } +} diff --git a/services/core/internal/sandbox/node/generation_wire.go b/services/core/internal/sandbox/node/generation_wire.go index 11884877b..d8c83327b 100644 --- a/services/core/internal/sandbox/node/generation_wire.go +++ b/services/core/internal/sandbox/node/generation_wire.go @@ -46,6 +46,9 @@ func validateVersionFrame(f frame, size int) error { if !validGeneration(g.Generation) || !validSpecificationDigest(g.SpecificationDigest) || seen[g.Generation] || (g.State != "ready" && g.State != "preparing" && g.State != "failed") || g.State == "ready" && g.Diagnostic != "" || len(g.Diagnostic) > 64 || sandbox.NormalizeNodeDiagnostic(g.Diagnostic) != g.Diagnostic { return sandbox.ErrInvalid } + if g.Checkpoint != nil && (g.State != "ready" || g.Checkpoint.Validate() != nil) { + return sandbox.ErrInvalid + } seen[g.Generation] = true } } @@ -72,9 +75,6 @@ func validateVersionFrame(f frame, size int) error { } switch f.Type { case "hello": - if !f.GenerationManagement && f.Health != nil && f.Health.Generations != nil { - return sandbox.ErrInvalid - } if f.Identity == nil || f.Health == nil || f.Request != nil || f.Response != nil || f.Control != nil || f.Deployment != nil { return sandbox.ErrInvalid } diff --git a/services/core/internal/sandbox/node/generations.go b/services/core/internal/sandbox/node/generations.go index 83354a61a..9e44ef959 100644 --- a/services/core/internal/sandbox/node/generations.go +++ b/services/core/internal/sandbox/node/generations.go @@ -15,7 +15,7 @@ type GenerationProvider struct { Generation uint64 SpecificationDigest string Provider sandbox.SandboxProvider - Probe func(context.Context) error + Probe func(context.Context) (*sandbox.CheckpointCompatibility, error) // Quiescent reports that no helper outlived its canceled caller; nil // means always quiescent. Quiescent func() bool @@ -31,6 +31,7 @@ type GenerationManagerOptions struct { } type localGeneration struct { + checkpoint *sandbox.CheckpointCompatibility value GenerationProvider state, diagnostic string refs int @@ -164,7 +165,7 @@ func (m *GenerationManager) Statuses() []sandbox.GenerationStatus { continue } seen[key] = true - result = append(result, sandbox.GenerationStatus{Generation: key, SpecificationDigest: g.value.SpecificationDigest, State: g.state, Diagnostic: g.diagnostic}) + result = append(result, sandbox.GenerationStatus{Generation: key, SpecificationDigest: g.value.SpecificationDigest, State: g.state, Diagnostic: g.diagnostic, Checkpoint: g.checkpoint}) if key != m.target.Generation && (m.target.ServingGeneration == nil || key != *m.target.ServingGeneration) { m.statusCursor = key } @@ -360,7 +361,10 @@ func (m *GenerationManager) probeLoop() { continue } ctx, cancel := context.WithTimeout(m.ctx, 5*time.Second) - err := g.value.Probe(ctx) + checkpoint, err := g.value.Probe(ctx) + if err == nil && ((sandbox.SupportsCheckpoint(g.value.Provider) && (checkpoint == nil || checkpoint.Validate() != nil)) || (!sandbox.SupportsCheckpoint(g.value.Provider) && checkpoint != nil)) { + err = sandbox.ErrInvalid + } cancel() m.mu.Lock() g.refs-- @@ -368,9 +372,11 @@ func (m *GenerationManager) probeLoop() { if !errors.Is(err, context.Canceled) { state, diagnostic := g.state, g.diagnostic g.state = "ready" + g.checkpoint = checkpoint g.diagnostic = "" if err != nil { g.state = "failed" + g.checkpoint = nil g.diagnostic = sandbox.NodeDiagnostic(err) if errors.Is(err, sandbox.ErrRuntimeImageUnavailable) || errors.Is(err, sandbox.ErrArtifactsUnavailable) { g.repairing = true diff --git a/services/core/internal/sandbox/node/generations_test.go b/services/core/internal/sandbox/node/generations_test.go index bce4d88cd..f81a5e38f 100644 --- a/services/core/internal/sandbox/node/generations_test.go +++ b/services/core/internal/sandbox/node/generations_test.go @@ -207,7 +207,7 @@ func TestRestartRecoversExactOlderGenerationBeforeAdvertisingReadiness(t *testin probe := make(chan struct{}, 1) qualify := make(chan struct{}) m, err := NewGenerationManager(t.Context(), GenerationManagerOptions{ - Initial: []GenerationProvider{{Generation: 2, SpecificationDigest: digest, Provider: &fakeProvider{}, Probe: func(context.Context) error { return nil }}}, + Initial: []GenerationProvider{{Generation: 2, SpecificationDigest: digest, Provider: &fakeProvider{}, Probe: func(context.Context) (*sandbox.CheckpointCompatibility, error) { return nil, nil }}}, Recover: []sandbox.GenerationReference{{Generation: 1, SpecificationDigest: digest}}, Prepare: func(ctx context.Context, generation uint64, got string) (GenerationProvider, error) { if got != digest { @@ -219,16 +219,16 @@ func TestRestartRecoversExactOlderGenerationBeforeAdvertisingReadiness(t *testin case <-ctx.Done(): return GenerationProvider{}, ctx.Err() } - return GenerationProvider{Generation: generation, SpecificationDigest: got, Provider: &fakeProvider{}, Probe: func(ctx context.Context) error { + return GenerationProvider{Generation: generation, SpecificationDigest: got, Provider: &fakeProvider{}, Probe: func(ctx context.Context) (*sandbox.CheckpointCompatibility, error) { select { case probe <- struct{}{}: default: } select { case <-qualify: - return nil + return nil, nil case <-ctx.Done(): - return ctx.Err() + return nil, ctx.Err() } }}, nil }, @@ -296,12 +296,12 @@ func TestInterruptedCollectionNeverPreparesOrServesAfterRestart(t *testing.T) { probed := make(chan struct{}, 1) removes := 0 manager, err := NewGenerationManager(t.Context(), GenerationManagerOptions{ - Initial: []GenerationProvider{{Generation: 2, SpecificationDigest: digest, Provider: &fakeProvider{}, Probe: func(context.Context) error { + Initial: []GenerationProvider{{Generation: 2, SpecificationDigest: digest, Provider: &fakeProvider{}, Probe: func(context.Context) (*sandbox.CheckpointCompatibility, error) { select { case probed <- struct{}{}: default: } - return nil + return nil, nil }}}, Collect: []sandbox.GenerationReference{ref}, Prepare: func(context.Context, uint64, string) (GenerationProvider, error) { diff --git a/services/core/internal/sandbox/node/hub.go b/services/core/internal/sandbox/node/hub.go index bda1da1c4..4feb0e2f6 100644 --- a/services/core/internal/sandbox/node/hub.go +++ b/services/core/internal/sandbox/node/hub.go @@ -391,8 +391,10 @@ func (h *Hub) call(ctx context.Context, id string, q request) (response, error) } func (h *Hub) recordHealth(ctx context.Context, p *peer, health Health) error { - if !p.generationManagement && health.Generations != nil { - return sandbox.ErrInvalid + if !p.generationManagement && len(health.Generations) > 0 { + if len(health.Generations) != 1 || health.Generations[0].Generation != p.identity.DeploymentGeneration || health.Generations[0].SpecificationDigest != p.identity.SpecificationDigest || (health.Generations[0].State == "ready") != health.ProviderReady { + return sandbox.ErrInvalid + } } if p.generationManagement { if h.options.Generations == nil { diff --git a/services/core/internal/sandbox/node/proxy.go b/services/core/internal/sandbox/node/proxy.go index e064f5b38..4949c3795 100644 --- a/services/core/internal/sandbox/node/proxy.go +++ b/services/core/internal/sandbox/node/proxy.go @@ -123,6 +123,12 @@ func (p *provider) state(ctx context.Context, q request) (sandbox.ComputeState, } return sandbox.ComputeState{}, e } + if r.State != nil && r.State.RestoreAttemptClosed != "" { + if q.Operation == "resume" && q.Resume != nil && r.State.ClosesRestoreAttempt(*q.Resume) { + return *r.State, nil + } + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } if r.State == nil || r.State.Compute.ID == "" { return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed } diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index d4ab402ab..e2fe64b2c 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -18,7 +18,7 @@ import ( "github.com/gorilla/websocket" ) -const ProtocolVersion = 5 +const ProtocolVersion = 6 const MaxControlFrameBytes = 32 * 1024 const MaxFrameBytes = 72 * 1024 * 1024 const maxPending = 32 diff --git a/services/core/internal/sandbox/node/workspace_test.go b/services/core/internal/sandbox/node/workspace_test.go index 35ab50569..cfcfe34e4 100644 --- a/services/core/internal/sandbox/node/workspace_test.go +++ b/services/core/internal/sandbox/node/workspace_test.go @@ -68,7 +68,7 @@ func (c *workspaceRefusalCaller) Call(context.Context, microsandbox.Request) (mi func TestActualWorkspaceRefusalKeepsSettledAbsenceOnWire(t *testing.T) { ref := reference() caller := new(workspaceRefusalCaller) - config := microsandbox.Config{ExternalWorkspace: true, InstallationID: "11111111-1111-4111-8111-111111111111", HelperPath: "/helper", RuntimeHome: "/runtime", RuntimePath: "/runtime/msb", FirmwarePath: "/runtime/firmware", RuntimeSHA256: strings.Repeat("a", 64), FirmwareSHA256: strings.Repeat("b", 64), Image: "image@sha256:" + strings.Repeat("c", 64), CPUs: 2, MemoryMiB: 2048, RootDiskMiB: 4096, Network: microsandbox.NetworkPolicy{DefaultEgress: "deny", DefaultIngress: "deny"}} + config := microsandbox.Config{ExternalWorkspace: true, InstallationID: "11111111-1111-4111-8111-111111111111", HelperPath: "/helper", RuntimeHome: "/runtime", CheckpointRoot: "/checkpoints", RuntimePath: "/runtime/msb", FirmwarePath: "/runtime/firmware", RuntimeSHA256: strings.Repeat("a", 64), FirmwareSHA256: strings.Repeat("b", 64), Image: "image@sha256:" + strings.Repeat("c", 64), CPUs: 2, MemoryMiB: 2048, RootDiskMiB: 4096, Network: microsandbox.NetworkPolicy{DefaultEgress: "deny", DefaultIngress: "deny"}} provider, err := microsandbox.NewWithCaller(config, caller) if err != nil { t.Fatal(err) diff --git a/services/core/internal/sandbox/providers/generation_test.go b/services/core/internal/sandbox/providers/generation_test.go index 0bacbe40d..60b42d297 100644 --- a/services/core/internal/sandbox/providers/generation_test.go +++ b/services/core/internal/sandbox/providers/generation_test.go @@ -21,7 +21,7 @@ func generationConfig(t *testing.T) (sandbox.NodeConfig, sandboxmicro.Native) { dir := t.TempDir() spec := validRegistrationSpec() spec.Resources.RootDiskMiB, spec.Resources.EnvironmentDiskMiB = 8192, 8192 - native := sandboxmicro.Native{HelperPath: filepath.Join(dir, "helper"), RuntimeHome: dir, RuntimePath: filepath.Join(dir, "msb"), FirmwarePath: filepath.Join(dir, "firmware"), + native := sandboxmicro.Native{HelperPath: filepath.Join(dir, "helper"), RuntimeHome: dir, CheckpointRoot: t.TempDir(), RuntimePath: filepath.Join(dir, "msb"), FirmwarePath: filepath.Join(dir, "firmware"), Network: sandboxmicro.Network{DefaultEgress: "allow", DefaultIngress: "deny"}} raw, err := json.Marshal(native) if err != nil { diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index 8d311697f..568c16b07 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -139,10 +139,25 @@ type Compute struct { RestoredFrom *SnapshotIdentity } +// CheckpointCompatibility identifies an adapter-verified private archive domain +// and execution compatibility class. Core compares these opaque tokens only. +type CheckpointCompatibility struct { + ArtifactDomain string `json:"artifact_domain"` + ExecutionClass string `json:"execution_class"` +} + +func (c CheckpointCompatibility) Validate() error { + if c.ArtifactDomain == "" || c.ExecutionClass == "" || len(c.ArtifactDomain) > 256 || len(c.ExecutionClass) > 256 { + return ErrInvalid + } + return nil +} + // SnapshotIdentity is provider evidence from a verified full snapshot. Core // persists it unchanged and records consumption separately; it never invents // paths, checksums, native checkpoint fields, or source identity. type SnapshotIdentity struct { + Compatibility CheckpointCompatibility Reference string ID string Digest string @@ -159,8 +174,23 @@ type ComputeState struct { Status string BootstrapComplete bool Snapshot *SnapshotIdentity - SourceStopped bool + // SourceStopped proves the archive is durable and all source-local compute + // and capture obligations have settled, so another node can own restoration. + SourceStopped bool + // RestoreAttemptClosed echoes the exact Resume OperationID only after a + // never-dispatched attempt is durably closed against late native execution. + RestoreAttemptClosed string +} + +// ClosesRestoreAttempt validates a never-executed closure against the exact +// persisted request. Absence without this operation-bound evidence is unknown. +func (s ComputeState) ClosesRestoreAttempt(q ResumeRequest) bool { + return q.ObserveOnly && q.OperationID != "" && s.RestoreAttemptClosed == q.OperationID && s.Status == "absent" && + s.Compute.ID == "" && q.Target.ID == "" && s.Compute.Name == q.Target.Name && s.Compute.Generation == q.Target.Generation && + s.Compute.RestoredFrom != nil && q.Target.RestoredFrom != nil && *s.Compute.RestoredFrom == q.Snapshot && *q.Target.RestoredFrom == q.Snapshot && + !s.BootstrapComplete && !s.SourceStopped && s.Snapshot == nil } + type SuspendRequest struct { Reference Reference OperationID string @@ -335,7 +365,7 @@ type Built struct { SpecificationDigest string Provider SandboxProvider InstallationID, BackendFingerprint string - Probe func(context.Context) error + Probe func(context.Context) (*CheckpointCompatibility, error) // Quiescent is nil when no helper can outlive its caller. Quiescent func() bool } diff --git a/services/core/migrations/000099_checkpoint_compatibility.sql b/services/core/migrations/000099_checkpoint_compatibility.sql new file mode 100644 index 000000000..0e9b0b490 --- /dev/null +++ b/services/core/migrations/000099_checkpoint_compatibility.sql @@ -0,0 +1,6 @@ +-- +goose Up +ALTER TABLE runtime_node_generation_status ADD COLUMN checkpoint jsonb + CHECK (checkpoint IS NULL OR jsonb_typeof(checkpoint) = 'object'); + +-- +goose Down +ALTER TABLE runtime_node_generation_status DROP COLUMN checkpoint; diff --git a/services/core/tests/integration/deployment_fixture_test.go b/services/core/tests/integration/deployment_fixture_test.go index 25018ae00..2073a5b60 100644 --- a/services/core/tests/integration/deployment_fixture_test.go +++ b/services/core/tests/integration/deployment_fixture_test.go @@ -1,12 +1,14 @@ package integration import ( + "context" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/providers" ) @@ -50,3 +52,16 @@ func deploymentExecution(t testing.TB, w *Store) *deployment.ExecutionOperations } return operations } + +func fixtureNodeHeartbeat(ctx context.Context, store *Store, service *deployment.Service, nodeID, connection string, epoch uint64, health deployment.NodeHealth) error { + var generation uint64 + var digest, provider string + if err := store.pool.QueryRow(ctx, "SELECT n.deployment_generation,n.specification_digest,d.provider_kind FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.id=$1", nodeID).Scan(&generation, &digest, &provider); err != nil { + return err + } + var statuses []sandbox.GenerationStatus + if health.ProviderReady && provider == "microsandbox" { + statuses = []sandbox.GenerationStatus{{Generation: generation, SpecificationDigest: digest, State: "ready", Checkpoint: &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}} + } + return service.Heartbeat(ctx, nodeID, connection, epoch, health, statuses) +} diff --git a/services/core/tests/integration/runtime_compute_lifecycle_test.go b/services/core/tests/integration/runtime_compute_lifecycle_test.go index 84b7e6af0..4c27a8e40 100644 --- a/services/core/tests/integration/runtime_compute_lifecycle_test.go +++ b/services/core/tests/integration/runtime_compute_lifecycle_test.go @@ -76,6 +76,10 @@ func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendReq defer p.mu.Unlock() state, ok := p.computes[q.Source.Name] if !ok { + if snapshot, found := p.snapshots[q.OperationID]; found && q.ObserveOnly { + p.captureObservations++ + return sandbox.ComputeState{Compute: q.Source, Snapshot: &snapshot, SourceStopped: true, Status: "suspended"}, nil + } return sandbox.ComputeState{}, sandbox.ErrNotFound } if state.Compute.ID != q.Source.ID { @@ -88,7 +92,7 @@ func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendReq if _, exists := p.snapshots[q.OperationID]; exists { return sandbox.ComputeState{}, errors.New("capture replayed") } - p.snapshots[q.OperationID] = sandbox.SnapshotIdentity{Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Digest: "verified", CheckpointID: "checkpoint", CheckpointRoot: "private", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} + p.snapshots[q.OperationID] = sandbox.SnapshotIdentity{Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}, Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Digest: "verified", CheckpointID: "checkpoint", CheckpointRoot: "private", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} state.Status = "paused" p.computes[q.Source.Name] = state if p.loseCapture { @@ -98,6 +102,10 @@ func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendReq } if snapshot, exists := p.snapshots[q.OperationID]; exists { state.Snapshot = &snapshot + state.SourceStopped = true + state.Status = "stopped" + p.computeKills++ + delete(p.computes, q.Source.Name) } return state, nil } diff --git a/services/core/tests/integration/runtime_node_generations_test.go b/services/core/tests/integration/runtime_node_generations_test.go index 8af333c9c..3197a00c7 100644 --- a/services/core/tests/integration/runtime_node_generations_test.go +++ b/services/core/tests/integration/runtime_node_generations_test.go @@ -29,7 +29,11 @@ func changeNodeTarget(t *testing.T, w *Store, view deployment.View, input sandbo func generationHeartbeat(t *testing.T, s *Store, node deployment.Enrollment, connection string, view deployment.View, state string) { t.Helper() - err := deploymentService(t, s).HeartbeatGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: state}}) + status := sandbox.GenerationStatus{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: state} + if state == "ready" && view.Provider == "microsandbox" { + status.Checkpoint = &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"} + } + err := deploymentService(t, s).HeartbeatGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{status}) if err != nil { t.Fatal(err) } @@ -173,7 +177,7 @@ func TestNodeGenerationsReconnectAndV1Fallback(t *testing.T) { if err := reserveSessionPlacement(t, s, w, queued); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("unconfirmed connection admitted", err) } - if err = service.Heartbeat(t.Context(), node.NodeID, connection, first.OwnerEpoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + if err = service.Heartbeat(t.Context(), node.NodeID, connection, first.OwnerEpoch, deployment.NodeHealth{ProviderReady: true}, nil); err != nil { t.Fatal(err) } if _, err = s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); err != nil { diff --git a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go index f6803bf48..ca2573598 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go @@ -148,7 +148,7 @@ func (f *nodeIsolationFixture) online(id string) { if err := f.nodes.ConnectNode(f.t.Context(), id, connection, f.epoch); err != nil { f.t.Fatal(err) } - if err := f.nodes.Heartbeat(f.t.Context(), id, connection, f.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + if err := fixtureNodeHeartbeat(f.t.Context(), f.store, f.nodes, id, connection, f.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { f.t.Fatal(err) } } @@ -183,7 +183,7 @@ func (f *nodeIsolationFixture) session(node string, initialize bool) (string, se f.t.Fatal(err) } for _, value := range others { - if err := f.nodes.Heartbeat(f.t.Context(), value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: false}); err != nil { + if err := fixtureNodeHeartbeat(f.t.Context(), f.store, f.nodes, value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: false}); err != nil { f.t.Fatal(err) } } @@ -192,7 +192,7 @@ func (f *nodeIsolationFixture) session(node string, initialize bool) (string, se _, err = f.placement.EnsurePlacement(f.t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, f.key) } for _, value := range others { - if err := f.nodes.Heartbeat(context.WithoutCancel(f.t.Context()), value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + if err := fixtureNodeHeartbeat(context.WithoutCancel(f.t.Context()), f.store, f.nodes, value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { f.t.Fatal(err) } } diff --git a/services/core/tests/integration/runtime_nodes_test.go b/services/core/tests/integration/runtime_nodes_test.go index 4150a0681..2d83b955e 100644 --- a/services/core/tests/integration/runtime_nodes_test.go +++ b/services/core/tests/integration/runtime_nodes_test.go @@ -7,11 +7,11 @@ import ( "strings" "sync" "testing" - "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -34,7 +34,7 @@ func onlineManagerNode(t *testing.T, s *Store, id string) string { if err := nodes.ConnectNode(t.Context(), id, connection, managerEpoch(t, s)); err != nil { t.Fatal(err) } - if err := nodes.Heartbeat(t.Context(), id, connection, managerEpoch(t, s), deployment.NodeHealth{ProviderReady: true}); err != nil { + if err := fixtureNodeHeartbeat(t.Context(), s, nodes, id, connection, managerEpoch(t, s), deployment.NodeHealth{ProviderReady: true}); err != nil { t.Fatal(err) } return connection @@ -72,13 +72,13 @@ func createSessionOnNode(t *testing.T, s, w *Store, tenant string, input session } nodes := deploymentService(t, s) for _, value := range others { - if err := nodes.Heartbeat(t.Context(), value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: false}); err != nil { + if err := fixtureNodeHeartbeat(t.Context(), s, nodes, value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: false}); err != nil { t.Fatal(err) } } defer func() { for _, value := range others { - if err := nodes.Heartbeat(context.WithoutCancel(t.Context()), value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + if err := fixtureNodeHeartbeat(context.WithoutCancel(t.Context()), s, nodes, value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { t.Fatal(err) } } @@ -268,7 +268,7 @@ func TestRuntimeNodesEnrollmentAndEpoch(t *testing.T) { if next := managerEpoch(t, s); next != epoch+1 { t.Fatal(next) } - if err := nodes.Heartbeat(t.Context(), input.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); !errors.Is(err, deployment.ErrNodeCredential) { + if err := fixtureNodeHeartbeat(t.Context(), s, nodes, input.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("old epoch heartbeat revived node", err) } queued, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput("stale")) @@ -333,7 +333,9 @@ func TestRuntimeNodesRetention(t *testing.T) { } } func TestRuntimeNodesRestoreAndCreationShareCapacity(t *testing.T) { - s, w, d := managerFixture(t, 1, 4) + s, w, view, _ := webSpecificationFixture(t, "microsandbox") + node := enrollNode(t, s, view, deployment.Capacity{MaxActive: 1, MaxRetained: 4}) + d := managerNode{InstallationID: view.InstallationID, NodeID: node.NodeID} tenant := uuid.NewString() session, err := s.CreateSession(t.Context(), tenant, managerSessionInput("first")) if err != nil { @@ -353,7 +355,9 @@ func TestRuntimeNodesRestoreAndCreationShareCapacity(t *testing.T) { if err != nil { t.Fatal(err) } - until := time.Now().Add(time.Hour) + if err := deploymentService(t, s).TouchActivity(t.Context(), allocation.TenantID, allocation.EnvironmentID); err != nil { + t.Fatal(err) + } second, err := s.CreateSession(t.Context(), tenant, managerSessionInput("second")) if err != nil { t.Fatal(err) @@ -362,7 +366,7 @@ func TestRuntimeNodesRestoreAndCreationShareCapacity(t *testing.T) { results := make(chan error, 2) go func() { <-start - _, err := deploymentExecution(t, w).SetCompute(t.Context(), allocation, "restoring", json.RawMessage(`{"target":{"id":"restore"}}`), &until, 0) + _, err := deploymentExecution(t, w).BeginRestore(t.Context(), allocation, sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}, json.RawMessage(`{"target":{"id":"restore"}}`)) results <- err }() go func() { diff --git a/services/core/tests/integration/runtime_suspension_test.go b/services/core/tests/integration/runtime_suspension_test.go index 2f76e9ffc..e32df1a8c 100644 --- a/services/core/tests/integration/runtime_suspension_test.go +++ b/services/core/tests/integration/runtime_suspension_test.go @@ -10,6 +10,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" @@ -57,7 +58,13 @@ func runtimeSuspensionStep(t *testing.T, w *Store, owner deployment.Allocation, if owner.ComputePhase == "running" && phase == "quiescing" { idleTimeout = time.Nanosecond } - next, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, phase, json.RawMessage(`{"instance":"original","snapshot":"qualified"}`), until, idleTimeout) + var next deployment.Allocation + var err error + if phase == "restoring" { + next, err = deploymentExecution(t, w).BeginRestore(t.Context(), owner, sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}, json.RawMessage(`{"instance":"original","snapshot":"qualified"}`)) + } else { + next, err = deploymentExecution(t, w).SetCompute(t.Context(), owner, phase, json.RawMessage(`{"instance":"original","snapshot":"qualified"}`), until, idleTimeout) + } if err != nil { t.Fatalf("%s -> %s: %v", owner.ComputePhase, phase, err) } @@ -245,7 +252,7 @@ func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { t.Fatal("snapshot retention expiry not observed", expired, err) } // Use the earlier unexpired observation to exercise expiry at the database CAS. - if _, err := deploymentExecution(t, w).SetCompute(t.Context(), retained, "restoring", json.RawMessage(`{}`), &until, 0); !errors.Is(err, deployment.ErrAllocationConflict) { + if _, err := deploymentExecution(t, w).BeginRestore(t.Context(), retained, sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}, json.RawMessage(`{}`)); !errors.Is(err, deployment.ErrAllocationConflict) { t.Fatal("expired snapshot restored from stale observation", err) } if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: owner.TenantID, SessionID: owner.SessionID}); err != nil { @@ -258,7 +265,7 @@ func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { if err != nil || !deleted.SessionDeleted || deleted.ComputeWakeRequested { t.Fatal("deleted session was woken", deleted, err) } - if _, err := deploymentExecution(t, w).SetCompute(t.Context(), deleted, "restoring", json.RawMessage(`{}`), &until, 0); !errors.Is(err, sessions.ErrNotFound) { + if _, err := deploymentExecution(t, w).BeginRestore(t.Context(), deleted, sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}, json.RawMessage(`{}`)); !errors.Is(err, sessions.ErrNotFound) { t.Fatal("deleted session restored", err) } } diff --git a/services/core/tests/integration/sandbox_deployment_worker_test.go b/services/core/tests/integration/sandbox_deployment_worker_test.go index b2143dc1e..528a351c1 100644 --- a/services/core/tests/integration/sandbox_deployment_worker_test.go +++ b/services/core/tests/integration/sandbox_deployment_worker_test.go @@ -72,7 +72,7 @@ func TestSandboxDeploymentWorkerActivatesWithoutRestart(t *testing.T) { if err := deployments.ConnectNode(t.Context(), nodeID, connection, epoch); err != nil { t.Fatal(err) } - if err := deployments.Heartbeat(t.Context(), nodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + if err := deployments.Heartbeat(t.Context(), nodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}, nil); err != nil { t.Fatal(err) } } diff --git a/services/core/tests/integration/session_execution_configuration_test.go b/services/core/tests/integration/session_execution_configuration_test.go index e4a1c168c..3a047f2f8 100644 --- a/services/core/tests/integration/session_execution_configuration_test.go +++ b/services/core/tests/integration/session_execution_configuration_test.go @@ -272,6 +272,11 @@ func TestSessionExecutionConfigurationSurvivesSuspendResume(t *testing.T) { if phase == "running" { retained = nil } + if phase == "restoring" { + if err := deploymentService(t, s).TouchActivity(t.Context(), tenant, environment.ID); err != nil { + t.Fatal(err) + } + } owner = runtimeSuspensionStep(t, w, owner, phase, retained) before, err := deploymentStore(w).Activity(t.Context(), owner.ID) if err != nil { diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index 243c6d682..f30db5987 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -1,12 +1,12 @@ # microsandbox Sandbox Provider helper -microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 4; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.8 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. +microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 5; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.8 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. [Add a Sandbox Provider](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources, Runtime release and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy. ## Installation checks -The [maintainer guide](../../../../docs/maintainers.md#runtime-images-and-helpers) builds the helper and packages the checksum-verified `msb` runtime and `libkrunfw` firmware. The node's provider configuration, written by the node installer, supplies the absolute helper, runtime and firmware paths with their SHA-256 values, the runtime home, the Runtime image reference, the saved resources and the host network policy. +The [maintainer guide](../../../../docs/maintainers.md#runtime-images-and-helpers) builds the helper and packages the checksum-verified `msb` runtime and `libkrunfw` firmware. The node's provider configuration, written by the node installer, supplies the absolute helper, runtime and firmware paths with their SHA-256 values, the runtime home, the private checkpoint root, the Runtime image reference, the saved resources and the host network policy. Before every operation the helper checks that it was built with the exact official SDK module declared in its embedded `go.mod` without a replacement, that the runtime and firmware match their hashes, that the runtime's `.msbver` ELF section reports the declared native release, that the SDK source and embedded FFI report that release, and that the SDK resolves exactly those paths with the local backend ([`main.go`](main.go)). It never installs or upgrades these files; keep them unchanged for the lifetime of the provider's backend. Ambient SDK profiles are ignored. @@ -14,6 +14,8 @@ The runtime home must be private (mode 0700), short, on local persistent storage The Runtime image is an immutable `repository@sha256:<64 lowercase hex>` reference that matches the saved Runtime release; bare image IDs and mutable tags are rejected. The node installer imports the distribution's image under that reference. To load an image by hand, `msb image load --tag repository@sha256:` must register the digest reference explicitly, with the manifest digest from `image inspect`, not the Docker image config ID. The image carries the daemon, Python 3, the native Harnesses and the shared Runtime helpers. The provider installs no registry credentials. +The checkpoint root is a separate adapter-owned directory, mode 0700, with a private `.oac-checkpoint-store` UUID marker. It is never derived from a workspace binding and must not be reachable through any guest mount. Archives contain confidential RAM and root-disk state. The marker and installation identity define the artifact domain; native release and binary hashes, Runtime image, kernel, CPU features and guest geometry define the conservative execution class. Filesystem paths, hostnames and local SDK homes do not define compatibility. Nodes can share this private root at different mount paths; the backing store must preserve durable writes, atomic rename and cross-host file locks. The installer's local default only advertises its own domain. + ## Create and bootstrap Create names the VM from a hash of the installation and allocation reference plus the compute generation; a name never serves another incarnation. It creates the VM with the saved CPUs and memory as both initial and maximum, a managed root disk of `root_disk_mib`, an owned ext4 disk of `environment_disk_mib` or the explicitly resolved external directory mounted at `/environment`, user 1000:1000, working directory `/` and the node's network policy ([`bootstrap.go`](bootstrap.go)). The resource checks run before bootstrap. Creation and restore retain the SDK's strict hostname policy enforcement; the adapter does not disable it. @@ -28,21 +30,23 @@ A helper response carries `CreateSettled` with a configuration rejection only af Core persists operation IDs, source and target generations, exact identities and snapshot evidence before it depends on them. `Initial` and `NewCompute` only construct references and allocate nothing. -- **Suspend** pauses the exact VM, captures a full snapshot under the persisted operation's derived group and member, verifies the complete checkpoint closure, then force-stops the source. Pausing alone does not release memory. A completed matching artifact is inspected instead of captured again. A full snapshot records a resource proof only after the source's limits match. -- **KillCompute** checks the precise incarnation before it stops the VM and removes its writable disks. Core calls it after it has persisted the verified snapshot, even when Suspend already stopped the source, so no chain of old writable disks grows across suspension cycles. -- **Restore** verifies the exact artifact and creates the precommitted target name. An existing target is adopted only when its immutable ID, if known, and its persisted `snapshot_parent` agree. Upstream restore defaults to public networking, so restore passes the same explicit host policy as creation, and no undeclared host resource or mount is inherited. +- **Suspend** pauses the exact VM, captures and verifies the full checkpoint closure, then exports a standalone SDK archive with its image into the private checkpoint root. The adapter synchronizes the archive and its digest, size and ownership receipt before force-stopping the exact paused source, removing its writable disks and deleting its local SDK snapshot. Only after that cleanup is synchronized does `SourceStopped` authorize transfer. A matching artifact is completed instead of captured again; a running source is never killed to finish an old capture. +- **KillCompute** fences future admission for the exact target and checks its incarnation before stopping and removing it. Under that dispatch fence the SDK serializes against native launch, waits for the runtime lifecycle lock on Kill, and rechecks the terminal incarnation under that lock on Remove. A final typed absence permits a durable cleanup receipt, including for an admitted restore whose launcher died. This proves the writer is gone, never that it did not execute. +- **Restore** requires the source-settled archive receipt, matching artifact domain and execution class, and the complete archive hash before importing through the SDK and creating the precommitted target name. An existing target is adopted only when its immutable ID, if known, and its persisted `snapshot_parent` agree. Upstream restore defaults to public networking, so restore passes the same explicit host policy as creation, and no undeclared host resource or mount is inherited. - Native restore leaves the managed root size unset because the target inherits the verified full snapshot. The helper accepts that only with a matching snapshot resource proof and the exact source and target identities, and it checks the target's CPU, memory and Environment disk before keeping the inherited proof. A missing root size never counts as unlimited capacity, and retained state is never resized. - Fresh restore and retry share one completion: verify the original artifact and resource proof, inspect the running target's resources and ancestry, persist its missing derived resource-proof label, then strictly reread the same native ID ([`restore_completion.go`](restore_completion.go)). A conflicting proof is an error. Native restore does not copy the source's ownership labels; ancestry supplies that evidence. There is no ordinary Start, replacement, disk-only restore or cold boot. - **ResumeCompute** thaws the same resident source after an aborted suspension. The pinned SDK handle method is name-based; the allocation lock and ID checks before and after the call fence every managed replacement. Manual lifecycle changes in the managed namespace are unsupported. -- **DeleteSnapshot** accepts only the derived operation selector and the matching full artifact identity, never an arbitrary path. Core owns retention, consumed snapshot generations and cleanup order. A checkpoint never rolls back work admitted after its first restore. +- **DeleteSnapshot** accepts only the derived operation selector and matching artifact identity. It fences delayed publication and restoration with a durable deletion receipt before deleting the local SDK snapshot and portable archive. An unsettled admitted restore blocks deletion. The small ownership tombstone remains; Core owns retention and cleanup order. + +After a lost response Core uses `ObserveOnly`. It never starts a capture or restore. For an existing verified capture it may complete archive publication and exact paused-source cleanup, but never captures again or kills a running source. Source termination is recorded before local removal so a lost cleanup response can be settled from the archive receipt. A valid ownership receipt remains discoverable when archive bytes are missing or corrupt: observation returns its snapshot identity with unknown source state and `SourceStopped=false`, so ordinary explicit cleanup can proceed. Restore still requires complete archive verification. An interrupted restore may finish the derived proof on the exact running target, but never restarts a stopped target or restores again. -After a lost response Core uses `ObserveOnly`. It never starts a capture or restore, and observing a suspend operation never kills its source. Observation checks artifact integrity and source ownership independently of resource checks, so resource drift cannot hide a retained artifact from cleanup; Core can persist recovered snapshot evidence before KillCompute. If the artifact is absent but the exact source is still running or paused with a settled bootstrap, observation returns the source without a snapshot and Core can abort the suspension; thawing and further execution still require the resource checks. For an interrupted restore, `ObserveOnly` may finish the missing resource proof on the exact target but never restarts a stopped target, changes resources or restores again. Missing state never authorizes a replay. +Before native import or restore, the helper synchronizes an admission record bound to the request operation, snapshot and target. Observation of a never-admitted operation writes a permanent closed record and returns that exact `RestoreAttemptClosed`; late dispatch is rejected. A fresh dispatch checks its existing request deadline after admission and immediately before calling native Restore. If the deadline has elapsed, it durably closes its exact attempt while still holding the allocation lock and returns without invoking Restore. Native FFI waits remain uncancelled. Import or qualification errors keep the admission open, avoiding repeated attempts against an invalid archive. This proof is never inferred from a previous admission or a native Restore error. After admission, helper death or native absence alone never establishes no effect. Observation checks an admitted target before reading the archive; typed absence or a non-running target remains unconfirmed without hashing or importing the archive. A running target still requires complete archive verification. Successful resume requires an observable exact running target. A failed or uncertain admitted restore retains its artifact and owner until explicit deletion or the original retention deadline; the normal cleanup operation uses the native lifecycle fence and records completion. It never shortens retention or substitutes a cold restart. GetCompute, commands, cleanup and the next suspension verify restored provenance from the persisted VM configuration after the consumed artifact is deleted. ## Locks and commands -A helper holds a per-allocation lock, under `oac-locks/` in the runtime home, until its SDK call actually settles. Core's response deadline neither kills the helper nor cancels its FFI wait, because cancelling the wait does not prove that the native mutation stopped. On a timeout Core keeps an unknown operation and observes it; a later helper cannot pass the surviving lock holder. A stuck owner needs operator investigation, not lock deletion or another Create. +A helper holds a per-allocation lock in the explicitly configured checkpoint root, until its SDK call actually settles. Core's response deadline neither kills the helper nor cancels its FFI wait, because cancelling the wait does not prove that the native mutation stopped. On a timeout Core keeps an unknown operation and observes it; a later helper cannot pass the surviving lock holder. A stuck owner needs operator investigation, not lock deletion or another Create. The same lock file fences a Create helper that has not yet acquired its lock, including across a node restart. An empty file permits initial Create admission; `closed\n` permanently forbids it, and any other nonempty content fails closed. Only `initial_info` observing typed native absence writes this marker while holding the lock, then synchronizes the file and its directories before returning the receipt. Every Create checks the marker under the same lock before invoking the SDK. Read, write or synchronization errors never establish settlement. The marker is never removed, and neither missing compute nor this receipt permits retrying Create. diff --git a/services/core/tools/microsandbox-provider/archive.go b/services/core/tools/microsandbox-provider/archive.go new file mode 100644 index 000000000..608a5bdf7 --- /dev/null +++ b/services/core/tools/microsandbox-provider/archive.go @@ -0,0 +1,494 @@ +//go:build linux + +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + wire "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" + sdk "github.com/superradcompany/microsandbox/sdk/go" +) + +// This adapter-private namespace is never a guest mount. The allocation lock +// serializes archive publication, admission and deletion across all its nodes. +type archiveReceipt struct { + Snapshot wire.SnapshotIdentity `json:"snapshot"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + SourceTerminated bool `json:"source_terminated"` + SourceStopped bool `json:"source_stopped"` + Deleted bool `json:"deleted"` +} + +func (b backend) storeDirectory() string { + return filepath.Join(b.q.Config.CheckpointRoot, wire.Name(b.q.Config, b.q.Reference, 0)) +} +func (b backend) archiveDirectory(operation string) string { + return filepath.Join(b.storeDirectory(), "s-"+operation) +} +func privateDirectory(path string) error { + if err := os.Mkdir(path, 0700); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + st, err := os.Lstat(path) + if err != nil { + return err + } + if !st.IsDir() || st.Mode().Perm() != 0700 { + return sandbox.ErrOwnership + } + return nil +} +func syncDirectory(path string) error { + f, e := os.Open(path) + if e != nil { + return e + } + defer f.Close() + return f.Sync() +} +func durableJSON(path string, value any) error { + data, e := json.Marshal(value) + if e != nil { + return e + } + f, e := os.CreateTemp(filepath.Dir(path), ".publish-") + if e != nil { + return e + } + defer os.Remove(f.Name()) + if _, e = f.Write(data); e == nil { + e = f.Sync() + } + closeErr := f.Close() + if e != nil { + return e + } + if closeErr != nil { + return closeErr + } + if e = os.Rename(f.Name(), path); e != nil { + return e + } + return syncDirectory(filepath.Dir(path)) +} +func readPrivateJSON(path string, value any) error { + st, e := os.Lstat(path) + if e != nil { + return e + } + if !st.Mode().IsRegular() || st.Mode().Perm() != 0600 { + return sandbox.ErrOwnership + } + real, e := filepath.EvalSymlinks(path) + if e != nil { + return e + } + if real != path { + return sandbox.ErrOwnership + } + f, e := os.Open(path) + if e != nil { + return e + } + defer f.Close() + d := json.NewDecoder(io.LimitReader(f, 65537)) + d.DisallowUnknownFields() + if e = d.Decode(value); e != nil { + return e + } + var extra any + if d.Decode(&extra) != io.EOF { + return sandbox.ErrOwnership + } + return nil +} +func (b backend) receipt(operation string) (archiveReceipt, error) { + var r archiveReceipt + e := readPrivateJSON(filepath.Join(b.archiveDirectory(operation), "receipt.json"), &r) + if e == nil && (r.Snapshot.OperationID != operation || wire.ValidateSnapshot(b.q.Config, b.q.Reference, r.Snapshot) != nil) { + e = sandbox.ErrOwnership + } + if e == nil && !r.Deleted { + decoded, err := hex.DecodeString(r.SHA256) + if err != nil || len(decoded) != 32 || r.Size <= 0 || (r.SourceStopped && !r.SourceTerminated) { + e = sandbox.ErrOwnership + } + } + return r, e +} +func archiveDigest(path string) (string, int64, error) { + real, e := filepath.EvalSymlinks(path) + if e != nil { + return "", 0, e + } + if real != path { + return "", 0, sandbox.ErrOwnership + } + st, e := os.Lstat(path) + if e != nil { + return "", 0, e + } + if !st.Mode().IsRegular() || st.Mode().Perm() != 0600 { + return "", 0, sandbox.ErrOwnership + } + f, e := os.Open(path) + if e != nil { + return "", 0, e + } + defer f.Close() + h := sha256.New() + n, e := io.Copy(h, f) + return hex.EncodeToString(h.Sum(nil)), n, e +} +func (b backend) publishArchive(ctx context.Context, a *sdk.SnapshotArtifact, s wire.SnapshotIdentity) (archiveReceipt, error) { + prior, e := b.receipt(s.OperationID) + if e == nil { + if prior.Snapshot != s || prior.Deleted { + return prior, sandbox.ErrOwnership + } + return prior, nil + } + if !errors.Is(e, os.ErrNotExist) { + return prior, e + } + dir := b.archiveDirectory(s.OperationID) + if e = privateDirectory(dir); e != nil { + return prior, e + } + if e = syncDirectory(b.storeDirectory()); e != nil { + return prior, e + } + archive := filepath.Join(dir, "archive.msb") + // A publication interrupted before its receipt can be rebuilt from the exact + // verified native artifact. The shared allocation lock excludes consumers. + if e = os.Remove(archive); e != nil && !errors.Is(e, os.ErrNotExist) { + return prior, e + } + if e = a.SaveTo(ctx, archive, sdk.SnapshotSaveOptions{WithImage: true}); e != nil { + return prior, e + } + if e = os.Chmod(archive, 0600); e != nil { + return prior, e + } + f, e := os.Open(archive) + if e != nil { + return prior, e + } + e = f.Sync() + f.Close() + if e != nil { + return prior, e + } + digest, size, e := archiveDigest(archive) + if e != nil { + return prior, e + } + r := archiveReceipt{Snapshot: s, SHA256: digest, Size: size} + return r, durableJSON(filepath.Join(dir, "receipt.json"), r) +} +func (b backend) importArchive(ctx context.Context, want wire.SnapshotIdentity) (*sdk.SnapshotArtifact, error) { + r, e := b.receipt(want.OperationID) + if e != nil { + return nil, e + } + if r.Deleted || !r.SourceStopped || r.Snapshot != want { + return nil, sandbox.ErrOwnership + } + class, e := wire.CheckpointClass(b.q.Config) + if e != nil { + return nil, e + } + if class != want.Compatibility { + return nil, sandbox.ErrInvalid + } + path := filepath.Join(b.archiveDirectory(want.OperationID), "archive.msb") + hash, size, e := archiveDigest(path) + if e != nil { + return nil, e + } + if hash != r.SHA256 || size != r.Size { + return nil, sandbox.ErrOwnership + } + artifact, e := b.verifiedSnapshot(ctx, want) + if sdk.IsKind(e, sdk.ErrSnapshotNotFound) { + if _, e = sdk.Snapshot.LoadWithOptions(ctx, path, sdk.SnapshotLoadOptions{Group: wire.Name(b.q.Config, b.q.Reference, 0)}); e != nil { + return nil, e + } + return b.verifiedSnapshot(ctx, want) + } + return artifact, e +} +func (b backend) deleteArchive(ctx context.Context, want wire.SnapshotIdentity) error { + return b.deleteArchiveWithNative(want, func() error { + _, err := b.verifiedSnapshot(ctx, want) + if err == nil { + err = sdk.Snapshot.Remove(ctx, want.Reference, false) + } + if sdk.IsKind(err, sdk.ErrSnapshotNotFound) { + return nil + } + return err + }) +} + +// Native artifact removal must settle before the private archive is unlinked. +func (b backend) deleteArchiveWithNative(want wire.SnapshotIdentity, removeNative func() error) error { + if e := b.requireSettledRestores(nil, &want); e != nil { + return e + } + r, e := b.receipt(want.OperationID) + if errors.Is(e, os.ErrNotExist) { + r = archiveReceipt{Snapshot: want} + if e = privateDirectory(b.archiveDirectory(want.OperationID)); e != nil { + return e + } + if e = syncDirectory(b.storeDirectory()); e != nil { + return e + } + } else if e != nil { + return e + } + if r.Snapshot != want { + return sandbox.ErrOwnership + } + // Fence consumers before removing either local or portable artifact. Keep the + // small tombstone, including its ownership identity, against delayed helpers. + r.Deleted = true + if e = durableJSON(filepath.Join(b.archiveDirectory(want.OperationID), "receipt.json"), r); e != nil { + return e + } + if e = removeNative(); e != nil { + return e + } + if e = os.Remove(filepath.Join(b.archiveDirectory(want.OperationID), "archive.msb")); e != nil && !errors.Is(e, os.ErrNotExist) { + return e + } + return syncDirectory(b.archiveDirectory(want.OperationID)) +} + +type restoreAdmission struct { + OperationID string + Snapshot wire.SnapshotIdentity + Target wire.Compute + Closed bool + CompletedID string + Cleaned bool +} + +func (b backend) admitRestore(q wire.ResumeRequest) (bool, error) { + if _, e := os.Lstat(filepath.Join(b.storeDirectory(), "k-"+q.Target.Name+".json")); e == nil { + return false, wire.ErrUnconfirmed + } else if !errors.Is(e, os.ErrNotExist) { + return false, e + } + path := filepath.Join(b.storeDirectory(), "r-"+q.OperationID+".json") + want := restoreAdmission{OperationID: q.OperationID, Snapshot: q.Snapshot, Target: q.Target} + want.Target.ID = "" + var prior restoreAdmission + e := readPrivateJSON(path, &prior) + if e == nil { + closed := prior.Closed + prior.Closed = false + prior.CompletedID = "" + cleaned := prior.Cleaned + prior.Cleaned = false + if !reflect.DeepEqual(prior, want) { + return false, sandbox.ErrOwnership + } + if closed { + return true, nil + } + if cleaned { + return false, wire.ErrUnconfirmed + } + if !q.ObserveOnly { + return false, wire.ErrUnconfirmed + } + return false, nil + } + if !errors.Is(e, os.ErrNotExist) { + return false, e + } + if q.Target.ID != "" { + return false, sandbox.ErrOwnership + } + want.Closed = q.ObserveOnly + if e = durableJSON(path, want); e != nil { + return false, e + } + return want.Closed, nil +} + +// Only the current fresh dispatch, under the allocation lock and before its +// first RestoreSandbox call, may record this proof. A recovered open admission +// or native restore error cannot establish that no guest execution occurred. +func (b backend) closeExpiredRestore(q wire.ResumeRequest) error { + if time.Now().Before(b.q.Deadline) { + return nil + } + if q.ObserveOnly || q.Target.ID != "" { + return sandbox.ErrOwnership + } + path := filepath.Join(b.storeDirectory(), "r-"+q.OperationID+".json") + var prior restoreAdmission + if err := readPrivateJSON(path, &prior); err != nil { + return err + } + want := restoreAdmission{OperationID: q.OperationID, Snapshot: q.Snapshot, Target: q.Target} + if !reflect.DeepEqual(prior, want) { + return sandbox.ErrOwnership + } + want.Closed = true + if err := durableJSON(path, want); err != nil { + return err + } + return context.DeadlineExceeded +} + +// Completion records the incarnation whose native restore has become observable +// and whose full provenance and running state have been verified. This is +// execution evidence; cleanup separately fences dispatch and uses native locks. +func (b backend) completeRestore(ctx context.Context, q wire.ResumeRequest, target wire.Compute) (wire.State, error) { + state, e := b.finishRestore(ctx, target) + if e != nil { + return state, e + } + path := filepath.Join(b.storeDirectory(), "r-"+q.OperationID+".json") + var record restoreAdmission + if e = readPrivateJSON(path, &record); e != nil { + return state, e + } + if record.Closed || record.Snapshot != q.Snapshot || record.Target.Name != target.Name || record.Target.Generation != target.Generation { + return state, sandbox.ErrOwnership + } + if record.CompletedID != "" && record.CompletedID != state.Compute.ID { + return state, sandbox.ErrOwnership + } + record.CompletedID = state.Compute.ID + return state, durableJSON(path, record) +} +func (b backend) requireSettledRestores(target *wire.Compute, snapshot *wire.SnapshotIdentity) error { + files, e := os.ReadDir(b.storeDirectory()) + if e != nil { + return e + } + for _, entry := range files { + if !strings.HasPrefix(entry.Name(), "r-") || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + var r restoreAdmission + if e = readPrivateJSON(filepath.Join(b.storeDirectory(), entry.Name()), &r); e != nil { + return e + } + if entry.Name() != "r-"+r.OperationID+".json" { + return sandbox.ErrOwnership + } + if r.Closed || r.Cleaned { + continue + } + if target != nil && (r.Target.Name != target.Name || r.Target.Generation != target.Generation) { + continue + } + if snapshot != nil && r.Snapshot != *snapshot { + continue + } + if r.CompletedID == "" { + return wire.ErrUnconfirmed + } + if target != nil && target.ID == "" { + target.ID = r.CompletedID + } + if target != nil && target.ID != "" && target.ID != r.CompletedID { + return sandbox.ErrOwnership + } + } + return nil +} + +func (b backend) verifyArchive(r archiveReceipt) error { + hash, size, e := archiveDigest(filepath.Join(b.archiveDirectory(r.Snapshot.OperationID), "archive.msb")) + if e != nil { + return e + } + if hash != r.SHA256 || size != r.Size { + return sandbox.ErrOwnership + } + return nil +} + +// The native transition/lifecycle locks fence an admitted launcher and its +// detached child. This receipt means cleanup completed, never "did not run". +func (b backend) recordRestoreCleanup(target wire.Compute) error { + files, e := os.ReadDir(b.storeDirectory()) + if e != nil { + return e + } + for _, entry := range files { + if !strings.HasPrefix(entry.Name(), "r-") || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + path := filepath.Join(b.storeDirectory(), entry.Name()) + var r restoreAdmission + if e = readPrivateJSON(path, &r); e != nil { + return e + } + if entry.Name() != "r-"+r.OperationID+".json" { + return sandbox.ErrOwnership + } + if r.Target.Name != target.Name || r.Target.Generation != target.Generation || r.Closed || r.Cleaned { + continue + } + if target.ID != "" && r.CompletedID != "" && target.ID != r.CompletedID { + return sandbox.ErrOwnership + } + r.Cleaned = true + if e = durableJSON(path, r); e != nil { + return e + } + } + return nil +} + +func (b backend) pinRestoreCleanup(target *wire.Compute) error { + files, e := os.ReadDir(b.storeDirectory()) + if e != nil { + return e + } + for _, entry := range files { + if !strings.HasPrefix(entry.Name(), "r-") || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + var r restoreAdmission + if e = readPrivateJSON(filepath.Join(b.storeDirectory(), entry.Name()), &r); e != nil { + return e + } + if entry.Name() != "r-"+r.OperationID+".json" { + return sandbox.ErrOwnership + } + if r.Target.Name != target.Name || r.Target.Generation != target.Generation || r.Closed { + continue + } + if target.RestoredFrom == nil || *target.RestoredFrom != r.Snapshot { + return sandbox.ErrOwnership + } + if r.CompletedID != "" { + if target.ID != "" && target.ID != r.CompletedID { + return sandbox.ErrOwnership + } + target.ID = r.CompletedID + } + } + return nil +} diff --git a/services/core/tools/microsandbox-provider/backend.go b/services/core/tools/microsandbox-provider/backend.go index a9e6d9d55..cc27005c6 100644 --- a/services/core/tools/microsandbox-provider/backend.go +++ b/services/core/tools/microsandbox-provider/backend.go @@ -5,6 +5,7 @@ package main import ( "context" "errors" + "path/filepath" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" wire "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" @@ -51,14 +52,7 @@ func (b backend) run(ctx context.Context) (wire.Response, error) { s, e := b.resume(ctx, *b.q.Resume) return wire.Response{State: &s}, e case "delete_snapshot": - _, e := b.verifiedSnapshot(ctx, *b.q.Snapshot) - if sdk.IsKind(e, sdk.ErrSnapshotNotFound) { - return wire.Response{}, nil - } - if e != nil { - return wire.Response{}, e - } - return wire.Response{}, sdk.Snapshot.Remove(ctx, b.q.Snapshot.Reference, false) + return wire.Response{}, b.deleteArchive(ctx, *b.q.Snapshot) } return wire.Response{}, sandbox.ErrInvalid } @@ -81,13 +75,23 @@ func (b backend) inspectOwned(ctx context.Context, c wire.Compute) (*sdk.Sandbox return h, state, e } func (b backend) kill(ctx context.Context, c wire.Compute) error { + // Close dispatch before consulting native state. An admitted helper has either + // released this allocation flock or died; the SDK's transition and inherited + // lifecycle locks fence any surviving native launcher/runtime. + if e := durableJSON(filepath.Join(b.storeDirectory(), "k-"+c.Name+".json"), c); e != nil { + return e + } + if e := b.pinRestoreCleanup(&c); e != nil { + return e + } h, _, e := b.inspectOwned(ctx, c) if sdk.IsKind(e, sdk.ErrSandboxNotFound) { - return nil + return b.recordRestoreCleanup(c) } if e != nil { return e } + c.ID = h.ID() if e = h.Kill(ctx); e != nil { return e } @@ -96,13 +100,14 @@ func (b backend) kill(ctx context.Context, c wire.Compute) error { } _, e = sdk.GetSandbox(ctx, c.Name) if sdk.IsKind(e, sdk.ErrSandboxNotFound) { - return nil + return b.recordRestoreCleanup(c) } if e == nil { return errors.New("compute removal unconfirmed") } return e } + func (b backend) network() *sdk.NetworkConfig { n := b.q.Config.Network result := &sdk.NetworkConfig{DefaultEgress: sdk.PolicyAction(n.DefaultEgress), DefaultIngress: sdk.PolicyAction(n.DefaultIngress)} diff --git a/services/core/tools/microsandbox-provider/lock_test.go b/services/core/tools/microsandbox-provider/lock_test.go index 46c93595f..02e0ab5eb 100644 --- a/services/core/tools/microsandbox-provider/lock_test.go +++ b/services/core/tools/microsandbox-provider/lock_test.go @@ -16,8 +16,8 @@ import ( ) func TestAllocationLockSurvivesCallerDeadlineUntilExplicitSettlement(t *testing.T) { - home := t.TempDir() - q := wire.Request{Config: wire.Config{RuntimeHome: home}, Deadline: time.Now().Add(time.Second)} + home := checkpointFixture(t) + q := wire.Request{Config: wire.Config{RuntimeHome: home, CheckpointRoot: home}, Deadline: time.Now().Add(time.Second)} release, e := allocationLock(q) if e != nil { t.Fatal(e) @@ -39,12 +39,12 @@ func TestAllocationLockSurvivesCallerDeadlineUntilExplicitSettlement(t *testing. release.Close() } func TestLockDirectoryCannotRedirectIntoAnotherHome(t *testing.T) { - home := t.TempDir() + home := checkpointFixture(t) foreign := t.TempDir() - if e := os.Symlink(foreign, filepath.Join(home, "oac-locks")); e != nil { + if e := os.Symlink(foreign, filepath.Join(home, wire.Name(wire.Config{}, sandbox.Reference{}, 0))); e != nil { t.Fatal(e) } - _, e := allocationLock(wire.Request{Config: wire.Config{RuntimeHome: home}, Deadline: time.Now().Add(time.Second)}) + _, e := allocationLock(wire.Request{Config: wire.Config{RuntimeHome: home, CheckpointRoot: home}, Deadline: time.Now().Add(time.Second)}) if e == nil { t.Fatal("symlink lock directory accepted") } @@ -56,6 +56,7 @@ func initialInfoRequest(t *testing.T) wire.Request { config.InstallationID = "11111111-1111-4111-8111-111111111111" config.HelperPath, config.RuntimePath, config.FirmwarePath = "/helper", "/runtime", "/firmware" config.RuntimeHome = t.TempDir() + config.CheckpointRoot = checkpointFixture(t) config.RuntimeSHA256, config.FirmwareSHA256 = strings.Repeat("a", 64), strings.Repeat("b", 64) config.Network = wire.NetworkPolicy{DefaultEgress: "deny", DefaultIngress: "deny"} ref := sandbox.Reference{TenantID: "22222222-2222-4222-8222-222222222222", EnvironmentID: "33333333-3333-4333-8333-333333333333", AllocationID: "44444444-4444-4444-8444-444444444444"} @@ -193,3 +194,15 @@ func TestInvalidOrUnwritableAdmissionStateFailsClosed(t *testing.T) { } }) } + +func checkpointFixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + if e := os.Chmod(root, 0700); e != nil { + t.Fatal(e) + } + if e := os.WriteFile(filepath.Join(root, ".oac-checkpoint-store"), []byte("55555555-5555-4555-8555-555555555555\n"), 0600); e != nil { + t.Fatal(e) + } + return root +} diff --git a/services/core/tools/microsandbox-provider/main.go b/services/core/tools/microsandbox-provider/main.go index b7dcb9a5a..f6220baae 100644 --- a/services/core/tools/microsandbox-provider/main.go +++ b/services/core/tools/microsandbox-provider/main.go @@ -58,6 +58,13 @@ func serve(input io.Reader) wire.Response { out.ErrorCode = "unconfirmed" return out } + if q.Workspace != nil { + real, err := filepath.EvalSymlinks(q.Workspace.Path) + if err != nil || real != q.Workspace.Path { + out.ErrorCode = "ownership" + return out + } + } lock, err := allocationLock(q) if err != nil { out.ErrorCode = "unconfirmed" @@ -173,7 +180,10 @@ func (g *allocationGuard) settleInitialAbsence(q wire.Request, nativeErr error) } func allocationLock(q wire.Request) (*allocationGuard, error) { - dir := filepath.Join(q.Config.RuntimeHome, "oac-locks") + if _, e := wire.CheckpointStore(q.Config); e != nil { + return nil, e + } + dir := filepath.Join(q.Config.CheckpointRoot, wire.Name(q.Config, q.Reference, 0)) if e := os.MkdirAll(dir, 0700); e != nil { return nil, e } @@ -184,6 +194,9 @@ func allocationLock(q wire.Request) (*allocationGuard, error) { if !st.IsDir() || st.Mode().Perm() != 0700 { return nil, sandbox.ErrOwnership } + if e := syncDirectory(q.Config.CheckpointRoot); e != nil { + return nil, e + } name := filepath.Join(dir, wire.Name(q.Config, q.Reference, 0)+".lock") fd, e := syscall.Open(name, syscall.O_CREAT|syscall.O_RDWR|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0600) if e != nil { diff --git a/services/core/tools/microsandbox-provider/restore_completion_test.go b/services/core/tools/microsandbox-provider/restore_completion_test.go index 2dbd5ec55..5f0b13a14 100644 --- a/services/core/tools/microsandbox-provider/restore_completion_test.go +++ b/services/core/tools/microsandbox-provider/restore_completion_test.go @@ -61,7 +61,7 @@ func TestRestoreCompletionRecoversOnlyDerivedProof(t *testing.T) { } func TestRestoreCompletionRejectsUnverifiedTargets(t *testing.T) { - for _, fault := range []string{"cpu", "memory", "environment", "image", "root", "foreign parent", "foreign ID", "foreign proof", "stopped", "unfinished", "replacement after write", "proof not persisted"} { + for _, fault := range []string{"cpu", "memory", "environment", "image", "root", "foreign parent", "foreign ID", "foreign proof", "stopped", "starting", "paused", "crashed", "unfinished", "replacement after write", "proof not persisted"} { t.Run(fault, func(t *testing.T) { config, actual := deploymentFixture() actual["image"].(map[string]any)["Oci"].(map[string]any)["root_disk"].(map[string]any)["size_mib"] = nil @@ -87,8 +87,8 @@ func TestRestoreCompletionRejectsUnverifiedTargets(t *testing.T) { actualID = "local:foreign" case "foreign proof": labels[resourceProofLabel] = "foreign" - case "stopped": - status = "stopped" + case "stopped", "starting", "paused", "crashed": + status = fault case "unfinished": actual["checkpoint_restore"] = map[string]string{"checkpoint_id": "pending"} } diff --git a/services/core/tools/microsandbox-provider/snapshot.go b/services/core/tools/microsandbox-provider/snapshot.go index 6cb6ee539..01475786b 100644 --- a/services/core/tools/microsandbox-provider/snapshot.go +++ b/services/core/tools/microsandbox-provider/snapshot.go @@ -5,7 +5,10 @@ package main import ( "context" "encoding/json" + "errors" "fmt" + "os" + "path/filepath" "strconv" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -49,7 +52,8 @@ func (b backend) inspectSnapshot(ctx context.Context, operation string, source w if report.Digest != artifact.Digest() || report.Checkpoint == nil || report.Checkpoint.Root == "" || report.Checkpoint.Kind == "" || state.Checkpoint == nil || state.Checkpoint.CheckpointID == "" { return nil, wire.SnapshotIdentity{}, sandbox.ErrOwnership } - identity := wire.SnapshotIdentity{Reference: selector, ID: artifact.ID(), Digest: artifact.Digest(), CheckpointID: state.Checkpoint.CheckpointID, CheckpointRoot: report.Checkpoint.Root, OperationID: operation, SourceName: source.Name, SourceID: source.ID, SourceGeneration: source.Generation} + compatibility := sandbox.CheckpointCompatibility{ArtifactDomain: artifact.Labels()["io.oac.artifact_domain"], ExecutionClass: artifact.Labels()["io.oac.execution_class"]} + identity := wire.SnapshotIdentity{Compatibility: compatibility, Reference: selector, ID: artifact.ID(), Digest: artifact.Digest(), CheckpointID: state.Checkpoint.CheckpointID, CheckpointRoot: report.Checkpoint.Root, OperationID: operation, SourceName: source.Name, SourceID: source.ID, SourceGeneration: source.Generation} if wire.ValidateSnapshot(b.q.Config, b.q.Reference, identity) != nil { return nil, wire.SnapshotIdentity{}, sandbox.ErrOwnership } @@ -66,6 +70,16 @@ func (b backend) verifiedSnapshot(ctx context.Context, want wire.SnapshotIdentit return a, nil } func (b backend) suspend(ctx context.Context, q wire.SuspendRequest) (wire.State, error) { + receipt, receiptErr := b.receipt(q.OperationID) + if receiptErr == nil { + if receipt.Deleted || receipt.Snapshot.SourceName != q.Source.Name || receipt.Snapshot.SourceID != q.Source.ID || receipt.Snapshot.SourceGeneration != q.Source.Generation || (q.Snapshot != nil && *q.Snapshot != receipt.Snapshot) { + return wire.State{}, sandbox.ErrOwnership + } + return b.settleSource(ctx, q, receipt) + } else if !errors.Is(receiptErr, os.ErrNotExist) { + return wire.State{}, receiptErr + } + // The host allocation flock spans pause, capture, verification and source kill. // A surviving completed artifact is observed, never overwritten or recaptured. artifact, snap, e := b.inspectSnapshot(ctx, q.OperationID, q.Source) @@ -97,6 +111,12 @@ func (b backend) suspend(ctx context.Context, q wire.SuspendRequest) (wire.State return wire.State{}, err } labels := b.snapshotLabels(q.OperationID, q.Source) + compatibility, err := wire.CheckpointClass(b.q.Config) + if err != nil { + return wire.State{}, err + } + labels["io.oac.artifact_domain"] = compatibility.ArtifactDomain + labels["io.oac.execution_class"] = compatibility.ExecutionClass var captured struct { Labels map[string]string `json:"labels"` } @@ -120,23 +140,37 @@ func (b backend) suspend(ctx context.Context, q wire.SuspendRequest) (wire.State if q.Snapshot != nil && snap != *q.Snapshot { return wire.State{}, sandbox.ErrOwnership } - if q.ObserveOnly { - // A completed operation receipt remains observable for cleanup even if - // its source or resource proof is no longer qualified for execution. - return observeCapturedSnapshot(q.Source, snap, func(source wire.Compute) (wire.State, error) { - _, state, err := b.inspectOwned(ctx, source) - return state, err - }) - } if e = qualifySnapshotResources(b.q.Config, artifact.Labels()); e != nil { return wire.State{}, e } - h, _, e := b.inspect(ctx, q.Source) + receipt, e = b.publishArchive(ctx, artifact, snap) + if e != nil { + return wire.State{}, e + } + return b.settleSource(ctx, q, receipt) +} + +func (b backend) settleSource(ctx context.Context, q wire.SuspendRequest, r archiveReceipt) (wire.State, error) { + if e := b.verifyArchive(r); e != nil { + if q.ObserveOnly { + // Ownership comes from the durable receipt, independently of archive + // usability. Expose cleanup identity without authorizing source transfer, + // claiming its current state, or weakening the restore integrity check. + return wire.State{Compute: q.Source, Status: "unknown", Snapshot: &r.Snapshot}, nil + } + return wire.State{}, e + } + if r.SourceStopped { + return wire.State{Compute: q.Source, Status: "suspended", BootstrapComplete: true, Snapshot: &r.Snapshot, SourceStopped: true}, nil + } + h, state, e := b.inspectOwned(ctx, q.Source) if e != nil && !sdk.IsKind(e, sdk.ErrSandboxNotFound) { return wire.State{}, e } - if e == nil { - // Graceful stop could run the captured source after the checkpoint. + if e == nil && !terminal(sdk.SandboxStatus(state.Status)) { + if state.Status != "paused" { + return wire.State{}, wire.ErrUnconfirmed + } if e = h.Kill(ctx); e != nil { return wire.State{}, e } @@ -147,12 +181,84 @@ func (b backend) suspend(ctx context.Context, q wire.SuspendRequest) (wire.State if !terminal(observed.Status()) { return wire.State{}, fmt.Errorf("source termination unconfirmed") } + } else if sdk.IsKind(e, sdk.ErrSandboxNotFound) && !r.SourceTerminated { + return wire.State{}, wire.ErrUnconfirmed + } + if !r.SourceTerminated { + r.SourceTerminated = true + if e = durableJSON(filepath.Join(b.archiveDirectory(q.OperationID), "receipt.json"), r); e != nil { + return wire.State{}, e + } + } + if h != nil { + if e = h.Remove(ctx); e != nil { + return wire.State{}, e + } + } + if _, e = sdk.GetSandbox(ctx, q.Source.Name); !sdk.IsKind(e, sdk.ErrSandboxNotFound) { + return wire.State{}, wire.ErrUnconfirmed + } + if e = sdk.Snapshot.Remove(ctx, r.Snapshot.Reference, false); e != nil && !sdk.IsKind(e, sdk.ErrSnapshotNotFound) { + return wire.State{}, e } - return wire.State{Compute: q.Source, Status: "suspended", BootstrapComplete: true, Snapshot: &snap, SourceStopped: true}, nil + r.SourceStopped = true + if e = durableJSON(filepath.Join(b.archiveDirectory(q.OperationID), "receipt.json"), r); e != nil { + return wire.State{}, e + } + return wire.State{Compute: q.Source, Status: "suspended", BootstrapComplete: true, Snapshot: &r.Snapshot, SourceStopped: true}, nil } + func (b backend) resume(ctx context.Context, q wire.ResumeRequest) (wire.State, error) { - // Verify the exact full artifact before either adopting or creating a target. - artifact, e := b.verifiedSnapshot(ctx, q.Snapshot) + // No managed target may predate its exact admission journal. + if _, err := os.Lstat(filepath.Join(b.storeDirectory(), "r-"+q.OperationID+".json")); errors.Is(err, os.ErrNotExist) { + _, err = sdk.GetSandbox(ctx, q.Target.Name) + if err == nil { + return wire.State{}, sandbox.ErrOwnership + } + if !sdk.IsKind(err, sdk.ErrSandboxNotFound) { + return wire.State{}, err + } + } else if err != nil { + return wire.State{}, err + } + + closed, e := b.admitRestore(q) + if e != nil { + return wire.State{}, e + } + if closed { + if !q.ObserveOnly { + return wire.State{}, wire.ErrUnconfirmed + } + target := q.Target + target.ID = "" + return wire.State{Compute: target, Status: "absent", RestoreAttemptClosed: q.OperationID}, nil + } + // An existing admitted observation may cheaply remain unknown. Native absence + // cannot close it: a detached launcher could still make the target visible. + if q.ObserveOnly { + _, observed, err := b.inspectOwned(ctx, q.Target) + if sdk.IsKind(err, sdk.ErrSandboxNotFound) { + return wire.State{}, wire.ErrUnconfirmed + } + if err != nil { + return wire.State{}, err + } + if observed.Status != "running" { + return wire.State{}, wire.ErrUnconfirmed + } + } + // Only a fresh dispatch may close its attempt before native execution. + // Lifecycle FFI calls use an uncancelled context; the request deadline is + // checked between settled operations, never used to cancel a native wait. + if !q.ObserveOnly { + if err := b.closeExpiredRestore(q); err != nil { + return wire.State{}, err + } + } + // Admission is durable before any native import/restore operation. A prior + // admitted operation with no visible target remains unknown, never replayed. + artifact, e := b.importArchive(ctx, q.Snapshot) if e != nil { return wire.State{}, e } @@ -164,7 +270,7 @@ func (b backend) resume(ctx context.Context, q wire.ResumeRequest) (wire.State, } _, _, e = b.inspectOwned(ctx, q.Target) if e == nil { - return b.finishRestore(ctx, q.Target) + return b.completeRestore(ctx, q, q.Target) } if !sdk.IsKind(e, sdk.ErrSandboxNotFound) { return wire.State{}, e @@ -172,21 +278,13 @@ func (b backend) resume(ctx context.Context, q wire.ResumeRequest) (wire.State, if q.ObserveOnly || q.Target.ID != "" { return wire.State{}, wire.ErrUnconfirmed } - source, err := sdk.GetSandbox(ctx, q.Snapshot.SourceName) - if err == nil { - if source.ID() != q.Snapshot.SourceID { - return wire.State{}, sandbox.ErrOwnership - } - if !terminal(source.Status()) { - return wire.State{}, wire.ErrUnconfirmed - } - } else if !sdk.IsKind(err, sdk.ErrSandboxNotFound) { - return wire.State{}, err - } restore := sdk.RestoreConfig{NetworkPolicy: b.network(), ExternalMountPolicy: sdk.ExternalMountStrict} if b.q.Workspace != nil { restore.Volumes = map[string]sdk.MountConfig{"/environment": sdk.Mount.Bind(b.q.Workspace.Path, sdk.MountOptions{})} } + if err := b.closeExpiredRestore(q); err != nil { + return wire.State{}, err + } live, e := sdk.RestoreSandbox(ctx, artifact, q.Target.Name, sdk.WithRestoreConfig(restore)) if e != nil { return wire.State{}, e @@ -195,7 +293,7 @@ func (b backend) resume(ctx context.Context, q wire.ResumeRequest) (wire.State, target.ID = live.ID() // The same completion path handles a fresh target and a previous Restore // whose response or derived proof write was interrupted. - state, err := b.finishRestore(ctx, target) + state, err := b.completeRestore(ctx, q, target) detachErr := live.Detach(context.Background()) if err != nil { return state, err @@ -206,21 +304,6 @@ func (b backend) resume(ctx context.Context, q wire.ResumeRequest) (wire.State, return state, nil } -// Observation consumes an already verified artifact identity and never changes -// source state. Execution qualification belongs to capture and subsequent use. -func observeCapturedSnapshot(source wire.Compute, snapshot wire.SnapshotIdentity, readOwned func(wire.Compute) (wire.State, error)) (wire.State, error) { - state, err := readOwned(source) - if err != nil && !sdk.IsKind(err, sdk.ErrSandboxNotFound) { - return wire.State{}, err - } - stopped := sdk.IsKind(err, sdk.ErrSandboxNotFound) || terminal(sdk.SandboxStatus(state.Status)) - status := "suspended" - if !stopped { - status = state.Status - } - return wire.State{Compute: source, Status: status, BootstrapComplete: true, Snapshot: &snapshot, SourceStopped: stopped}, nil -} - func terminal(s sdk.SandboxStatus) bool { return s == sdk.SandboxStatusStopped || s == sdk.SandboxStatusCrashed } diff --git a/services/core/tools/microsandbox-provider/snapshot_observation_test.go b/services/core/tools/microsandbox-provider/snapshot_observation_test.go index cf2a80b07..74810cc67 100644 --- a/services/core/tools/microsandbox-provider/snapshot_observation_test.go +++ b/services/core/tools/microsandbox-provider/snapshot_observation_test.go @@ -3,83 +3,511 @@ package main import ( - "encoding/json" + "context" "errors" + "fmt" + "os" + "os/exec" + "path/filepath" "testing" + "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" wire "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" - sdk "github.com/superradcompany/microsandbox/sdk/go" ) -func TestCapturedSnapshotObservationDoesNotRequireExecutionQualification(t *testing.T) { - for _, drift := range []string{"cpu", "image", "missing snapshot proof", "foreign snapshot proof"} { - t.Run(drift, func(t *testing.T) { - config, actual := deploymentFixture() - ref := sandbox.Reference{TenantID: "tenant", EnvironmentID: "environment", AllocationID: "allocation"} - source := wire.Compute{Name: "original", ID: "local:original"} - labels := wire.Labels(config, ref) - labels[workspaceModeLabel] = "owned" - labels[bootstrapLabel] = "complete" - actual["labels"] = labels - snapshotLabels := map[string]string{workspaceModeLabel: "owned", resourceProofLabel: resourceProof(config)} - switch drift { - case "cpu": - actual["resources"].(map[string]any)["cpus"] = 1 - case "image": - actual["image"] = "runtime:drifted" - case "missing snapshot proof": - delete(snapshotLabels, resourceProofLabel) - case "foreign snapshot proof": - snapshotLabels[resourceProofLabel] = "foreign" - } - raw, _ := json.Marshal(actual) - if qualifyConfiguration(config, source, string(raw), false) == nil && qualifySnapshotResources(config, snapshotLabels) == nil { - t.Fatal("fixture did not disqualify execution") - } - // inspectSnapshot has already verified the exact artifact and closure. - snapshot := wire.SnapshotIdentity{ID: "verified", SourceID: source.ID, SourceName: source.Name} - reads := 0 - state, err := observeCapturedSnapshot(source, snapshot, func(want wire.Compute) (wire.State, error) { - reads++ - return qualifyCompute(config, ref, want, source.ID, "paused", string(raw)) - }) - if err != nil || state.Snapshot == nil || *state.Snapshot != snapshot || state.Compute != source || state.SourceStopped || reads != 1 { - t.Fatal("owned receipt blocked by execution drift", state, err, reads) +func TestRestoreObservationClosesOnlyNeverAdmittedOperation(t *testing.T) { + request := initialInfoRequest(t) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + b := backend{q: request} + q := wire.ResumeRequest{OperationID: "66666666-6666-4666-8666-666666666666", Target: wire.Compute{Name: "exact-target", Generation: 1}, ObserveOnly: true} + closed, err := b.admitRestore(q) + if err != nil || !closed { + t.Fatal(closed, err) + } + q.ObserveOnly = false + // A late helper cannot replace the durable closed record with admission. + closed, err = b.admitRestore(q) + if err != nil || !closed { + t.Fatal("late restore admitted", closed, err) + } + q.OperationID = "77777777-7777-4777-8777-777777777777" + closed, err = b.admitRestore(q) + if err != nil || closed { + t.Fatal(closed, err) + } + // A crash after admission remains unknown, including when no target is yet + // visible. The helper never manufactures no-effect evidence from absence. + q.ObserveOnly = true + closed, err = b.admitRestore(q) + if err != nil || closed { + t.Fatal("admitted operation closed", closed, err) + } + q.ObserveOnly = false + if _, err = b.admitRestore(q); !errors.Is(err, wire.ErrUnconfirmed) { + t.Fatal("admitted operation replayed", err) + } + q.ObserveOnly = true + q.Target.Name = "foreign" + if _, err = b.admitRestore(q); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal("foreign target admitted", err) + } +} +func TestRestoreJournalRejectsPartialAndSymlinkRecords(t *testing.T) { + for _, kind := range []string{"partial", "symlink"} { + t.Run(kind, func(t *testing.T) { + request := initialInfoRequest(t) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + b := backend{q: request} + q := wire.ResumeRequest{OperationID: "66666666-6666-4666-8666-666666666666", ObserveOnly: true} + path := filepath.Join(b.storeDirectory(), "r-"+q.OperationID+".json") + if kind == "partial" { + err = os.WriteFile(path, []byte("{"), 0600) + } else { + err = os.Symlink(filepath.Join(t.TempDir(), "missing"), path) + } + if err != nil { + t.Fatal(err) + } + if _, err = b.admitRestore(q); err == nil { + t.Fatal("unsafe record accepted") + } + }) + } +} +func TestArchiveDigestRejectsGuestReadableAndSymlinkArtifacts(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "archive") + if err := os.WriteFile(path, []byte("private RAM"), 0600); err != nil { + t.Fatal(err) + } + hash, size, err := archiveDigest(path) + if err != nil || len(hash) != 64 || size != 11 { + t.Fatal(hash, size, err) + } + if err = os.Chmod(path, 0644); err != nil { + t.Fatal(err) + } + if _, _, err = archiveDigest(path); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal(err) + } + link := filepath.Join(dir, "link") + if err = os.Symlink(path, link); err != nil { + t.Fatal(err) + } + if _, _, err = archiveDigest(link); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal(err) + } +} + +func TestAdmittedRestoreBlocksCleanupUntilExactCompletion(t *testing.T) { + request := initialInfoRequest(t) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + b := backend{q: request} + q := wire.ResumeRequest{OperationID: "66666666-6666-4666-8666-666666666666", Target: wire.Compute{Name: "exact-target", Generation: 1}} + if _, err = b.admitRestore(q); err != nil { + t.Fatal(err) + } + if err = b.requireSettledRestores(&q.Target, nil); !errors.Is(err, wire.ErrUnconfirmed) { + t.Fatal("unknown target was releasable", err) + } + if err = b.requireSettledRestores(nil, &q.Snapshot); !errors.Is(err, wire.ErrUnconfirmed) { + t.Fatal("unknown artifact was releasable", err) + } + other := wire.Compute{Name: "other-source"} + if err = b.requireSettledRestores(&other, nil); err != nil { + t.Fatal("unrelated source cleanup blocked", err) + } + record := restoreAdmission{OperationID: q.OperationID, Target: q.Target, Snapshot: q.Snapshot, CompletedID: "local:exact"} + if err = durableJSON(filepath.Join(b.storeDirectory(), "r-"+q.OperationID+".json"), record); err != nil { + t.Fatal(err) + } + if err = b.requireSettledRestores(&q.Target, nil); err != nil || q.Target.ID != "local:exact" { + t.Fatal("cleanup did not pin completed incarnation", q.Target, err) + } + q.Target.ID = "local:foreign" + if err = b.requireSettledRestores(&q.Target, nil); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal("foreign cleanup accepted", err) + } +} + +func TestTargetCleanupFenceSurvivesReopenBeforeAndAfterAdmission(t *testing.T) { + for _, admitted := range []bool{false, true} { + t.Run(fmt.Sprint(admitted), func(t *testing.T) { + request := initialInfoRequest(t) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + b := backend{q: request} + q := wire.ResumeRequest{OperationID: "66666666-6666-4666-8666-666666666666", Target: wire.Compute{Name: "exact-target", Generation: 1}} + if admitted { + if _, err = b.admitRestore(q); err != nil { + t.Fatal(err) + } + } + if err = durableJSON(filepath.Join(b.storeDirectory(), "k-"+q.Target.Name+".json"), q.Target); err != nil { + t.Fatal(err) + } + guard.Close() + guard, err = allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + if _, err = b.admitRestore(q); !errors.Is(err, wire.ErrUnconfirmed) { + t.Fatal("late restore crossed cleanup fence", err) + } + if admitted { + // Only the caller which has completed native Get/Kill/Remove/Get invokes + // this journal update. The live crash probe qualifies the native fence. + if err = b.recordRestoreCleanup(q.Target); err != nil { + t.Fatal(err) + } + if err = b.requireSettledRestores(nil, &q.Snapshot); err != nil { + t.Fatal("cleaned obligation retained", err) + } } }) } } -func TestCapturedSnapshotObservationRetainsSourceOwnershipAndState(t *testing.T) { - source := wire.Compute{Name: "original", ID: "local:original"} - snapshot := wire.SnapshotIdentity{ID: "verified", SourceID: source.ID} - for _, test := range []struct { - name, status string - err error - stopped bool - }{ - {name: "running", status: "running"}, - {name: "unknown", status: "unknown"}, - {name: "stopped", status: "stopped", stopped: true}, - {name: "crashed", status: "crashed", stopped: true}, - {name: "absent", err: &sdk.Error{Kind: sdk.ErrSandboxNotFound}, stopped: true}, - {name: "foreign", err: sandbox.ErrOwnership}, - {name: "unavailable", err: wire.ErrUnconfirmed}, - } { - t.Run(test.name, func(t *testing.T) { - state, err := observeCapturedSnapshot(source, snapshot, func(wire.Compute) (wire.State, error) { - return wire.State{Compute: source, Status: test.status}, test.err +func TestLostCaptureResponseStillExposesOwnedIdentityWhenArchiveIsDamaged(t *testing.T) { + for _, settled := range []bool{false, true} { + for _, damage := range []string{"corrupt", "missing"} { + t.Run(fmt.Sprintf("settled=%t/%s", settled, damage), func(t *testing.T) { + request := initialInfoRequest(t) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + b := backend{q: request} + source := request.Compute + source.ID = "local:known-source" + operation := "66666666-6666-4666-8666-666666666666" + compatibility, err := wire.CheckpointClass(request.Config) + if err != nil { + t.Fatal(err) + } + snapshot := wire.SnapshotIdentity{Compatibility: compatibility, Reference: wire.SnapshotReference(request.Config, request.Reference, operation), ID: "exact-snapshot", Digest: "digest", CheckpointID: "checkpoint", CheckpointRoot: "root", OperationID: operation, SourceName: source.Name, SourceID: source.ID} + dir := b.archiveDirectory(operation) + if err = privateDirectory(dir); err != nil { + t.Fatal(err) + } + archive := filepath.Join(dir, "archive.msb") + if err = os.WriteFile(archive, []byte("verified full archive"), 0600); err != nil { + t.Fatal(err) + } + hash, size, err := archiveDigest(archive) + if err != nil { + t.Fatal(err) + } + receipt := archiveReceipt{Snapshot: snapshot, SHA256: hash, Size: size, SourceTerminated: settled, SourceStopped: settled} + if err = durableJSON(filepath.Join(dir, "receipt.json"), receipt); err != nil { + t.Fatal(err) + } + if damage == "corrupt" { + err = os.WriteFile(archive, []byte("damaged"), 0600) + } else { + err = os.Remove(archive) + } + if err != nil { + t.Fatal(err) + } + // Core lost the capture response and therefore has no Snapshot yet. It + // needs this ownership identity to reach ordinary Kill/DeleteSnapshot. + state, err := b.suspend(t.Context(), wire.SuspendRequest{Reference: request.Reference, OperationID: operation, Source: source, ObserveOnly: true}) + if err != nil || state.Snapshot == nil || *state.Snapshot != snapshot || state.Compute != source || state.SourceStopped || state.Status != "unknown" || state.BootstrapComplete { + t.Fatalf("cleanup identity hidden or execution authorized: %+v %v", state, err) + } + if artifact, err := b.importArchive(t.Context(), snapshot); err == nil || artifact != nil { + t.Fatal("damaged archive was usable for restore", err) + } + if _, err = b.suspend(t.Context(), wire.SuspendRequest{Reference: request.Reference, OperationID: operation, Source: source}); err == nil { + t.Fatal("non-observation accepted unusable archive") + } + foreign := source + foreign.ID = "local:foreign" + denied, err := b.suspend(t.Context(), wire.SuspendRequest{Reference: request.Reference, OperationID: operation, Source: foreign, ObserveOnly: true}) + if !errors.Is(err, sandbox.ErrOwnership) || denied.Snapshot != nil { + t.Fatal("foreign request recovered owned identity", denied, err) + } + nativeRemoved := false + if err = b.deleteArchiveWithNative(*state.Snapshot, func() error { nativeRemoved = true; return nil }); err != nil { + t.Fatal("owned damaged archive could not be deleted", err) + } + if !nativeRemoved { + t.Fatal("native cleanup was skipped") + } + if _, err = os.Lstat(archive); !errors.Is(err, os.ErrNotExist) { + t.Fatal("archive remains", err) + } + deleted, err := b.receipt(operation) + if err != nil || !deleted.Deleted || deleted.Snapshot != snapshot { + t.Fatal("exact deletion fence missing", deleted, err) + } }) - if test.err != nil && !test.stopped { - if !errors.Is(err, test.err) || state.Snapshot != nil { - t.Fatal("unverified source returned a successful observation", state, err) + } + } +} + +// These requests use the real SDK's empty local registry, without starting a VM. +func TestPermanentRestoreImportFailureKeepsOneAdmission(t *testing.T) { + if !isolatedSDKTest(t) { + return + } + home := t.TempDir() + t.Setenv("MSB_HOME", home) + for _, damage := range []string{"missing", "corrupt"} { + t.Run(damage, func(t *testing.T) { + request := initialInfoRequest(t) + request.Config.RuntimeHome = home + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + b := backend{q: request} + q := wire.ResumeRequest{OperationID: "66666666-6666-4666-8666-666666666666", Target: wire.Compute{Name: wire.Name(request.Config, request.Reference, 1), Generation: 1}} + if damage == "corrupt" { + compatibility, err := wire.CheckpointClass(request.Config) + if err != nil { + t.Fatal(err) + } + q.Snapshot = wire.SnapshotIdentity{Compatibility: compatibility, Reference: wire.SnapshotReference(request.Config, request.Reference, q.OperationID), ID: "id", Digest: "digest", CheckpointID: "checkpoint", CheckpointRoot: "root", OperationID: q.OperationID, SourceName: wire.Name(request.Config, request.Reference, 0), SourceID: "local:source"} + dir := b.archiveDirectory(q.OperationID) + if err = privateDirectory(dir); err != nil { + t.Fatal(err) + } + archive := filepath.Join(dir, "archive.msb") + if err = os.WriteFile(archive, []byte("original"), 0600); err != nil { + t.Fatal(err) + } + hash, size, err := archiveDigest(archive) + if err != nil { + t.Fatal(err) + } + if err = durableJSON(filepath.Join(dir, "receipt.json"), archiveReceipt{Snapshot: q.Snapshot, SHA256: hash, Size: size, SourceStopped: true, SourceTerminated: true}); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(archive, []byte("corrupt!"), 0600); err != nil { + t.Fatal(err) + } + } + if _, err = b.resume(t.Context(), q); err == nil { + t.Fatal("missing archive accepted") + } + path := filepath.Join(b.storeDirectory(), "r-"+q.OperationID+".json") + var record restoreAdmission + if err = readPrivateJSON(path, &record); err != nil || record.Closed { + t.Fatal("permanent failure closed", record, err) + } + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + files, err := os.ReadDir(b.storeDirectory()) + if err != nil { + t.Fatal(err) + } + q.ObserveOnly = true + for i := 0; i < 3; i++ { + state, err := b.resume(t.Context(), q) + if !errors.Is(err, wire.ErrUnconfirmed) || state.RestoreAttemptClosed != "" { + t.Fatal(state, err) + } + } + after, err := os.ReadFile(path) + if err != nil || string(before) != string(after) { + t.Fatal("journal changed", err) + } + afterFiles, err := os.ReadDir(b.storeDirectory()) + if err != nil || len(afterFiles) != len(files) { + t.Fatal("journal grew", err) + } + }) + } +} + +func TestFreshRestoreRequestDeadlineClosesBeforeNativeExecution(t *testing.T) { + if !isolatedSDKTest(t) { + return + } + request := initialInfoRequest(t) + t.Setenv("MSB_HOME", request.Config.RuntimeHome) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + // The real request acquired its lock before its deadline. Time can expire + // while prior SDK work settles; the lifecycle context deliberately stays live. + request.Deadline = time.Now().Add(-time.Second) + q := wire.ResumeRequest{OperationID: "66666666-6666-4666-8666-666666666666", Target: wire.Compute{Name: wire.Name(request.Config, request.Reference, 1), Generation: 1}} + request.Operation = "resume" + request.Resume = &q + b := backend{q: request} + _, err = b.run(context.Background()) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatal("request deadline not enforced", err) + } + q.ObserveOnly = true + state, err := b.resume(context.Background(), q) + if err != nil || state.RestoreAttemptClosed != q.OperationID || state.Status != "absent" { + t.Fatal("closed proof missing", state, err) + } + q.ObserveOnly = false + if _, err = b.resume(context.Background(), q); !errors.Is(err, wire.ErrUnconfirmed) { + t.Fatal("late request replayed", err) + } +} + +// The official SDK freezes its default backend on first use. Match production's +// one-request helper process rather than changing MSB_HOME under a cached pool. +func isolatedSDKTest(t *testing.T) bool { + t.Helper() + if os.Getenv("OAC_NATIVE_TEST_CHILD") == t.Name() { + return true + } + cmd := exec.Command(os.Args[0], "-test.run=^"+t.Name()+"$", "-test.count=1") + cmd.Env = append(os.Environ(), "OAC_NATIVE_TEST_CHILD="+t.Name()) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("isolated SDK test: %v\n%s", err, output) + } + return false +} + +func TestAdmittedAbsentObservationSkipsArchiveAndStaysUnknown(t *testing.T) { + if !isolatedSDKTest(t) { + return + } + request := initialInfoRequest(t) + t.Setenv("MSB_HOME", request.Config.RuntimeHome) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + b := backend{q: request} + q := wire.ResumeRequest{OperationID: "66666666-6666-4666-8666-666666666666", Target: wire.Compute{Name: wire.Name(request.Config, request.Reference, 1), Generation: 1}} + if _, err = b.admitRestore(q); err != nil { + t.Fatal(err) + } + // Even a missing archive must not be consulted: its ENOENT would differ from + // the typed unknown result returned for this existing admitted target. + path := filepath.Join(b.storeDirectory(), "r-"+q.OperationID+".json") + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + b.q.Deadline = time.Now().Add(-time.Second) + q.ObserveOnly = true + state, err := b.resume(t.Context(), q) + if !errors.Is(err, wire.ErrUnconfirmed) || state.RestoreAttemptClosed != "" { + t.Fatal(state, err) + } + after, err := os.ReadFile(path) + if err != nil || string(before) != string(after) { + t.Fatal("observation mutated admission", err) + } + q.ObserveOnly = false + if _, err = b.resume(t.Context(), q); !errors.Is(err, wire.ErrUnconfirmed) { + t.Fatal("old admission replayed", err) + } +} + +func TestPreDispatchClosureRejectsConflictingJournal(t *testing.T) { + for _, kind := range []string{"observe", "completed", "cleaned", "foreign", "missing"} { + t.Run(kind, func(t *testing.T) { + request := initialInfoRequest(t) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + request.Deadline = time.Now().Add(-time.Second) + b := backend{q: request} + q := wire.ResumeRequest{OperationID: "66666666-6666-4666-8666-666666666666", Target: wire.Compute{Name: "target", Generation: 1}} + r := restoreAdmission{OperationID: q.OperationID, Target: q.Target, Snapshot: q.Snapshot} + switch kind { + case "observe": + q.ObserveOnly = true + case "completed": + r.CompletedID = "local:1" + case "cleaned": + r.Cleaned = true + case "foreign": + r.Target.Name = "foreign" + } + path := filepath.Join(b.storeDirectory(), "r-"+q.OperationID+".json") + if kind != "missing" { + if err = durableJSON(path, r); err != nil { + t.Fatal(err) + } + } + before, readErr := os.ReadFile(path) + wantErr := sandbox.ErrOwnership + if kind == "missing" { + wantErr = os.ErrNotExist + if !errors.Is(readErr, os.ErrNotExist) { + t.Fatal("expected missing journal", readErr) } - return + } else if readErr != nil { + t.Fatal(readErr) } - if err != nil || state.Snapshot == nil || state.SourceStopped != test.stopped { - t.Fatal("observation changed source termination evidence", state, err) + if err = b.closeExpiredRestore(q); !errors.Is(err, wantErr) { + t.Fatalf("unsafe closure result: got %v, want %v", err, wantErr) + } + after, readErr := os.ReadFile(path) + if kind == "missing" { + if !errors.Is(readErr, os.ErrNotExist) { + t.Fatal("missing journal was created", readErr) + } + } else if readErr != nil || string(after) != string(before) { + t.Fatal("rejected closure changed journal", readErr) } }) } } + +func TestFreshAdmissionClosesOnlyAfterRequestDeadline(t *testing.T) { + request := initialInfoRequest(t) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + b := backend{q: request} + q := wire.ResumeRequest{OperationID: "66666666-6666-4666-8666-666666666666", Target: wire.Compute{Name: "target", Generation: 1}} + if _, err = b.admitRestore(q); err != nil { + t.Fatal(err) + } + if err = b.closeExpiredRestore(q); err != nil { + t.Fatal("live request was closed", err) + } + var r restoreAdmission + path := filepath.Join(b.storeDirectory(), "r-"+q.OperationID+".json") + if err = readPrivateJSON(path, &r); err != nil || r.Closed { + t.Fatal(r, err) + } + b.q.Deadline = time.Now().Add(-time.Second) + if err = b.closeExpiredRestore(q); !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) + } + if err = readPrivateJSON(path, &r); err != nil || !r.Closed { + t.Fatal("expired dispatch not fenced", r, err) + } +} From a166c8223bacd9e65949b42718abf50d46f15820 Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 11 Oct 2026 10:25:23 +0800 Subject: [PATCH 21/22] Fix retained checkpoint demand and node accounting queries --- .../internal/db/queries/runtime_nodes.sql | 2 +- .../queries/runtime_suspension_pressure.sql | 4 +- .../internal/db/sqlc/runtime_nodes.sql.go | 2 +- .../sqlc/runtime_suspension_pressure.sql.go | 4 +- .../runtime_retained_queries_test.go | 84 +++++++++++++++++++ .../deploymentpg/checkpoint_transfer_test.go | 2 +- .../deploymentpg/suspension_pressure_test.go | 2 +- 7 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 services/core/internal/execution/runtime_retained_queries_test.go diff --git a/services/core/internal/db/queries/runtime_nodes.sql b/services/core/internal/db/queries/runtime_nodes.sql index 6a24dbb20..44f68e596 100644 --- a/services/core/internal/db/queries/runtime_nodes.sql +++ b/services/core/internal/db/queries/runtime_nodes.sql @@ -19,7 +19,7 @@ SELECT n.*, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id AND a.state<>'released'))::bigint AS reserved, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='cleanup_pending')::bigint AS cleanup_pending, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='running' AND a.compute_phase IN('running','disabled'))::bigint AS running, - (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'snapshot' IS NOT NULL AND a.compute_state->'snapshot'<>'null'::jsonb)::bigint AS snapshots + (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'retained' IS NOT NULL AND a.compute_state->'retained'<>'null'::jsonb)::bigint AS snapshots FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND (sqlc.narg(node_id)::uuid IS NULL OR n.id=sqlc.narg(node_id)::uuid) ORDER BY n.id; diff --git a/services/core/internal/db/queries/runtime_suspension_pressure.sql b/services/core/internal/db/queries/runtime_suspension_pressure.sql index 580c00813..4c47d4caa 100644 --- a/services/core/internal/db/queries/runtime_suspension_pressure.sql +++ b/services/core/internal/db/queries/runtime_suspension_pressure.sql @@ -6,9 +6,9 @@ WHERE a.node_id IS NOT NULL AND a.state <> 'released' OR (a.compute_retained_until <= clock_timestamp() AND a.compute_phase <> 'disabled')); -- name: ListWaitingCheckpointRestores :many -SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'snapshot'->'Compatibility')::jsonb AS checkpoint +SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'retained'->'Compatibility')::jsonb AS checkpoint FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id WHERE a.node_id IS NOT NULL AND a.state='running' AND a.compute_phase='suspended' AND a.compute_retained_until>clock_timestamp() AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') - AND a.compute_state->'snapshot'->'Compatibility' IS NOT NULL + AND a.compute_state->'retained'->'Compatibility' IS NOT NULL AND (a.compute_wake_requested OR EXISTS(SELECT 1 FROM environment_input_reservations r WHERE r.session_id=s.id AND r.state='pending' AND r.deadline>clock_timestamp())); diff --git a/services/core/internal/db/sqlc/runtime_nodes.sql.go b/services/core/internal/db/sqlc/runtime_nodes.sql.go index e28353bcf..d664a022d 100644 --- a/services/core/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_nodes.sql.go @@ -388,7 +388,7 @@ SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha2 (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id AND a.state<>'released'))::bigint AS reserved, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='cleanup_pending')::bigint AS cleanup_pending, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='running' AND a.compute_phase IN('running','disabled'))::bigint AS running, - (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'snapshot' IS NOT NULL AND a.compute_state->'snapshot'<>'null'::jsonb)::bigint AS snapshots + (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'retained' IS NOT NULL AND a.compute_state->'retained'<>'null'::jsonb)::bigint AS snapshots FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND ($1::uuid IS NULL OR n.id=$1::uuid) ORDER BY n.id diff --git a/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go index e192c879e..86b051015 100644 --- a/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go @@ -40,11 +40,11 @@ func (q *Queries) ListRuntimeSuspensionInFlightNodes(ctx context.Context) ([]pgt } const listWaitingCheckpointRestores = `-- name: ListWaitingCheckpointRestores :many -SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'snapshot'->'Compatibility')::jsonb AS checkpoint +SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'retained'->'Compatibility')::jsonb AS checkpoint FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id WHERE a.node_id IS NOT NULL AND a.state='running' AND a.compute_phase='suspended' AND a.compute_retained_until>clock_timestamp() AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') - AND a.compute_state->'snapshot'->'Compatibility' IS NOT NULL + AND a.compute_state->'retained'->'Compatibility' IS NOT NULL AND (a.compute_wake_requested OR EXISTS(SELECT 1 FROM environment_input_reservations r WHERE r.session_id=s.id AND r.state='pending' AND r.deadline>clock_timestamp())) ` diff --git a/services/core/internal/execution/runtime_retained_queries_test.go b/services/core/internal/execution/runtime_retained_queries_test.go new file mode 100644 index 000000000..e2430706b --- /dev/null +++ b/services/core/internal/execution/runtime_retained_queries_test.go @@ -0,0 +1,84 @@ +package execution + +import ( + "encoding/json" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +// Exercise the SQL projections with the same serialized envelope Core persists. +// Adapter-private data deliberately contains legacy-looking keys: queries must +// read only the shared retained envelope, never the opaque payload. +func TestRuntimeRetainedStateFeedsCheckpointDemandAndNodeCounts(t *testing.T) { + f := newWorkspaceSettlementFixture(t, &retentionProvider{}) + r, session := f.lifecycle, f.session + owner, err := r.provision(t.Context(), session.TenantID, session.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + compatibility := sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"} + source := sandbox.Compute{Generation: 1, Name: "source", ID: "native-source"} + retained := sandbox.RetainedState{Reference: "fixture/retained", ID: "native-retained", Data: `{"snapshot":{"Compatibility":{"artifact_domain":"private","execution_class":"private"}}}`, OperationID: "suspend-operation", SourceGeneration: source.Generation, SourceName: source.Name, SourceID: source.ID, Compatibility: compatibility} + for _, tc := range []struct { + name string + wake, expired, released, noCompatibility, legacy bool + demand, count int + }{ + {name: "waiting", wake: true, demand: 1, count: 1}, + {name: "idle", count: 1}, + {name: "expired", wake: true, expired: true, count: 1}, + {name: "released", wake: true, released: true}, + {name: "no_compatibility", wake: true, noCompatibility: true, count: 1}, + {name: "legacy_shape", wake: true, legacy: true}, + } { + t.Run(tc.name, func(t *testing.T) { + state := runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: source, Retained: &retained, SuspendID: retained.OperationID} + copied := retained + if tc.noCompatibility { + copied.Compatibility = sandbox.CheckpointCompatibility{} + state.Retained = &copied + } + raw, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if tc.legacy { + var envelope map[string]json.RawMessage + if err := json.Unmarshal(raw, &envelope); err != nil { + t.Fatal(err) + } + envelope["snapshot"] = envelope["retained"] + delete(envelope, "retained") + raw, err = json.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + } + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET state=CASE WHEN $3 THEN 'released' ELSE 'running' END,released_at=CASE WHEN $3 THEN clock_timestamp() END,compute_phase='suspended',compute_state=$2,compute_retained_until=clock_timestamp()+CASE WHEN $4 THEN interval '-1 hour' ELSE interval '1 hour' END,compute_wake_requested=$5 WHERE id=$1`, owner.ID, raw, tc.released, tc.expired, tc.wake); err != nil { + t.Fatal(err) + } + q := sqlc.New(f.pool) + demands, err := q.ListWaitingCheckpointRestores(t.Context()) + if err != nil || len(demands) != tc.demand { + t.Fatalf("checkpoint demand=%d want=%d: %v", len(demands), tc.demand, err) + } + if len(demands) == 1 { + var got sandbox.CheckpointCompatibility + if err := json.Unmarshal(demands[0].Checkpoint, &got); err != nil || got != compatibility { + t.Fatalf("shared compatibility lost: %+v %v", got, err) + } + } + nodes, err := q.ListRuntimeNodes(t.Context(), pgtype.UUID{Bytes: uuid.MustParse(owner.NodeID), Valid: true}) + if err != nil || len(nodes) != 1 { + t.Fatalf("node projection: %d %v", len(nodes), err) + } + if nodes[0].Snapshots != int64(tc.count) { + t.Fatalf("retained node count=%d want=%d", nodes[0].Snapshots, tc.count) + } + }) + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go b/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go index 919f2dac6..63f150c89 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go @@ -34,7 +34,7 @@ func readyCheckpointNode(t *testing.T, f fixture, node deployment.Enrollment, vi func suspendedCheckpointOwner(t *testing.T, f fixture, changes *deployment.ExecutionOperations, installation string, node deployment.Enrollment, view deployment.View) deployment.Allocation { t.Helper() owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) - if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"snapshot":{"ID":"snapshot","Compatibility":{"artifact_domain":"fixture-archive","execution_class":"fixture-machine"}}}',compute_retained_until=clock_timestamp()+interval '24 hours',compute_wake_requested=true WHERE id=$1`, owner.ID); err != nil { + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"protocol_version":"1","retained":{"ID":"snapshot","Compatibility":{"artifact_domain":"fixture-archive","execution_class":"fixture-machine"}}}',compute_retained_until=clock_timestamp()+interval '24 hours',compute_wake_requested=true WHERE id=$1`, owner.ID); err != nil { t.Fatal(err) } owner, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) diff --git a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go index 77fed6a34..67acb835e 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go @@ -132,7 +132,7 @@ func TestPressureSuspensionServesRestoreOnItsOriginalNode(t *testing.T) { node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) f.connect(t, node.NodeID) waiting := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) - if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"snapshot":{"Compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()+interval '1 hour',compute_wake_requested=true WHERE id=$1`, waiting.ID); err != nil { + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"protocol_version":"1","retained":{"Compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()+interval '1 hour',compute_wake_requested=true WHERE id=$1`, waiting.ID); err != nil { t.Fatal(err) } owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) From 7edd85ae90636c42d5bf2cfeba71cfade4e675d2 Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 11 Oct 2026 10:28:47 +0800 Subject: [PATCH 22/22] Test beta upgrade through checkpoint compatibility schema --- services/core/migrations/beta_upgrade_test.go | 97 ++++++++++++++++++- 1 file changed, 94 insertions(+), 3 deletions(-) diff --git a/services/core/migrations/beta_upgrade_test.go b/services/core/migrations/beta_upgrade_test.go index 940ac9901..44a044e57 100644 --- a/services/core/migrations/beta_upgrade_test.go +++ b/services/core/migrations/beta_upgrade_test.go @@ -3,6 +3,7 @@ package migrations import ( "context" "database/sql" + "errors" "fmt" "os" "strings" @@ -10,11 +11,12 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" -"github.com/jackc/pgx/v5/stdlib" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/stdlib" "github.com/pressly/goose/v3" ) -// A dedicated disposable database exercises the real 91-to-97 SQL, never a +// A dedicated disposable database exercises the real 91-to-99 SQL, never a // product database. No provider or native sandbox is contacted by this test. func TestBeta91UpgradePreservesRetainedAllocations(t *testing.T) { dsn := os.Getenv("OAC_TEST_DATABASE_URL") @@ -94,7 +96,7 @@ func TestBeta91UpgradePreservesRetainedAllocations(t *testing.T) { t.Fatal("migration changed retained state, ownership or deadline") } var version, live, workspaces int - if err = db.QueryRowContext(t.Context(), `SELECT max(version_id) FROM agents_api_schema_version WHERE is_applied`).Scan(&version); err != nil || version != 97 { + if err = db.QueryRowContext(t.Context(), `SELECT max(version_id) FROM agents_api_schema_version WHERE is_applied`).Scan(&version); err != nil || version != 99 { t.Fatal(version, err) } if err = db.QueryRowContext(t.Context(), `SELECT count(*) FROM runtime_allocations WHERE state<>'released' AND compute_phase='suspended' AND compute_state->>'protocol_version'='1'`).Scan(&live); err != nil || live != 17 { @@ -103,6 +105,11 @@ func TestBeta91UpgradePreservesRetainedAllocations(t *testing.T) { if err = db.QueryRowContext(t.Context(), `SELECT count(*) FROM environment_workspaces`).Scan(&workspaces); err != nil || workspaces != 0 { t.Fatal(workspaces, err) } + checkBeta99CheckpointConstraint(t, db, installation) + checkBeta98HistoricalAllocationDowngrade(t, db, provider) + if snapshot() != before { + t.Fatal("constraint checks changed retained ownership") + } // Empty workspace ownership permits Down, but it does not restore removed // values. Verify the documented E2B-policy rollback incompatibility. if _, err = provider.DownTo(t.Context(), 91); err != nil { @@ -116,3 +123,87 @@ func TestBeta91UpgradePreservesRetainedAllocations(t *testing.T) { t.Fatal("down migration changed retained ownership") } } + +// Migration 99 accepts only structured compatibility evidence. It does not +// manufacture a token for existing nodes or reinterpret adapter-private state. +func checkBeta99CheckpointConstraint(t *testing.T, db *sql.DB, installation string) { + t.Helper() + node, connection := uuid.NewString(), uuid.NewString() + if _, err := db.ExecContext(t.Context(), `INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained) VALUES($1,$2,'migration-fixture',repeat('c',64),repeat('d',64),1,2)`, node, installation); err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(t.Context(), `INSERT INTO runtime_node_generation_status(node_id,generation,specification_digest,connection_id,owner_epoch,state) VALUES($1,1,repeat('e',64),$2,1,'ready')`, node, connection); err != nil { + t.Fatal(err) + } + var absent bool + if err := db.QueryRowContext(t.Context(), `SELECT checkpoint IS NULL FROM runtime_node_generation_status WHERE node_id=$1`, node).Scan(&absent); err != nil || !absent { + t.Fatal("checkpoint default", absent, err) + } + if _, err := db.ExecContext(t.Context(), `UPDATE runtime_node_generation_status SET checkpoint='{"artifact_domain":"fixture","execution_class":"fixture"}' WHERE node_id=$1`, node); err != nil { + t.Fatal(err) + } + for _, invalid := range []string{`[]`, `"opaque"`, `1`, `null`} { + _, err := db.ExecContext(t.Context(), `UPDATE runtime_node_generation_status SET checkpoint=$2 WHERE node_id=$1`, node, invalid) + var pgerr *pgconn.PgError + if !errors.As(err, &pgerr) || pgerr.Code != "23514" { + t.Fatalf("checkpoint shape %s: %v", invalid, err) + } + } + if _, err := db.ExecContext(t.Context(), `DELETE FROM runtime_node_generation_status WHERE node_id=$1`, node); err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(t.Context(), `DELETE FROM runtime_nodes WHERE id=$1`, node); err != nil { + t.Fatal(err) + } +} + +// Migration 98 keeps multiple historical receipts but one active allocation +// and one live device authority. Down must fail rather than erase history. +func checkBeta98HistoricalAllocationDowngrade(t *testing.T, db *sql.DB, provider *goose.Provider) { + t.Helper() + device, allocation := uuid.NewString(), uuid.NewString() + var environment, tenant, installation string + if err := db.QueryRowContext(t.Context(), `SELECT a.environment_id,s.tenant_id,a.provider_key FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id WHERE a.state='running' ORDER BY a.id LIMIT 1`).Scan(&environment, &tenant, &installation); err != nil { + t.Fatal(err) + } + _, err := db.ExecContext(t.Context(), `INSERT INTO devices(id,tenant_id,name,credential_hash,environment_id) VALUES($1,$2,'duplicate-authority',repeat('f',64),$3)`, device, tenant, environment) + requireBetaUniqueViolation(t, err, "devices_environment_authority") + if _, err := db.ExecContext(t.Context(), `INSERT INTO devices(id,tenant_id,name,credential_hash,environment_id,revoked_at) VALUES($1,$2,'historical-device',repeat('f',64),$3,clock_timestamp())`, device, tenant, environment); err != nil { + t.Fatal(err) + } + _, err = db.ExecContext(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,deployment_generation) VALUES($1,$2,$3,$4,'running',true,4)`, allocation, environment, device, installation) + requireBetaUniqueViolation(t, err, "runtime_allocations_current_environment") + if _, err := db.ExecContext(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,released_at,deployment_generation) VALUES($1,$2,$3,$4,'released',true,clock_timestamp(),4)`, allocation, environment, device, installation); err != nil { + t.Fatal(err) + } + _, err = provider.DownTo(t.Context(), 97) + requireBetaUniqueViolation(t, err, "devices_environment_id_key") + // Isolate the allocation downgrade guard after proving device history also + // prevents Down. This fixture never grants the unbound old device authority. + if _, err := db.ExecContext(t.Context(), `UPDATE devices SET environment_id=NULL WHERE id=$1`, device); err != nil { + t.Fatal(err) + } + _, err = provider.DownTo(t.Context(), 97) + requireBetaUniqueViolation(t, err, "runtime_allocations_environment_id_key") + var count int + if err := db.QueryRowContext(t.Context(), `SELECT count(*) FROM runtime_allocations WHERE environment_id=$1`, environment).Scan(&count); err != nil || count != 2 { + t.Fatal("failed downgrade lost history", count, err) + } + if _, err := db.ExecContext(t.Context(), `DELETE FROM runtime_allocations WHERE id=$1`, allocation); err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(t.Context(), `DELETE FROM devices WHERE id=$1`, device); err != nil { + t.Fatal(err) + } + if _, err := provider.Up(t.Context()); err != nil { + t.Fatal(err) + } +} + +func requireBetaUniqueViolation(t *testing.T, err error, constraint string) { + t.Helper() + var pgerr *pgconn.PgError + if !errors.As(err, &pgerr) || pgerr.Code != "23505" || pgerr.ConstraintName != constraint { + t.Fatalf("expected %s uniqueness fence: %v", constraint, err) + } +}