diff --git a/apps/daemon/internal/cli/connect.go b/apps/daemon/internal/cli/connect.go index dc87bb4a9..0ec595e66 100644 --- a/apps/daemon/internal/cli/connect.go +++ b/apps/daemon/internal/cli/connect.go @@ -70,7 +70,7 @@ func runConnect(ctx *runContext, args []string) error { if *remote != "" || *environment != "" || *credentialFile != "" || fs.NArg() != 0 { return errors.New("connect: bootstrap input cannot be combined with enrollment options") } - bootstrapped, err = bootstrapProfile(*bootstrapFile) + bootstrapped, err = bootstrapProfile(*bootstrapFile, ctx) if err != nil { return err } @@ -98,13 +98,6 @@ func runConnect(ctx *runContext, args []string) error { return spawnBackground(context.Background(), ctx, *profile, os.Args) } - // Self-check before loading credentials so a machine with no - // supported agent CLI fails fast. - agentCLIs, err := preflightAgentCLIs(context.Background(), ctx, *profile) - if err != nil { - return err - } - var prof auth.Profile if bootstrapped != nil { prof = *bootstrapped @@ -115,7 +108,7 @@ func runConnect(ctx *runContext, args []string) error { } } - return mainLoop(ctx, *profile, prof, agentCLIs) + return mainLoop(ctx, *profile, prof) } // spawnBackground forks the daemon into the background. Parent @@ -186,11 +179,16 @@ func spawnBackground(ctx context.Context, rc *runContext, profile string, argv [ // background process. SIGINT / SIGTERM cancels the root context, which // unblocks the read pump and any in-flight Send so the daemon exits // without orphaning agent subprocesses. -func mainLoop(rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery) error { - return mainLoopRemote(context.Background(), rc, profile, prof, agentCLIs, "") +func mainLoop(rc *runContext, profile string, prof auth.Profile) error { + return mainLoopRemote(context.Background(), rc, profile, prof, "") } -func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery, remote string) error { +func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof auth.Profile, remote string) error { + return mainLoopRemoteWithDiscovery(parent, rc, profile, prof, remote, func(ctx context.Context) (agentCLIDiscovery, error) { + return preflightAgentCLIs(ctx, rc, profile) + }) +} +func mainLoopRemoteWithDiscovery(parent context.Context, rc *runContext, profile string, prof auth.Profile, remote string, discover func(context.Context) (agentCLIDiscovery, error)) error { // Route through obs/log so daemon log lines pick up the same // trace_id / span_id auto-injection as the server side — when the // daemon adopts an envelope's trace, every log call under that ctx @@ -205,18 +203,18 @@ func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof rootCtx, cancel := daemonize.NotifyContext(parent) defer cancel() - bootCtx, bootCancel := context.WithTimeout(rootCtx, bootstrapTimeout) - var boot *transport.BootstrapResponse - var err error - if remote == "" { - boot, err = transport.Bootstrap(bootCtx, prof.ServerURL, prof.RuntimeID, prof.RunnerCredential, Version) - } else { - boot, err = environmentBootstrap(bootCtx, prof, remote) - } - bootCancel() + boot, agentCLIs, err := prepareConnection(rootCtx, discover, func(ctx context.Context) (*transport.BootstrapResponse, error) { + bootCtx, stop := context.WithTimeout(ctx, bootstrapTimeout) + defer stop() + if remote != "" { + return environmentBootstrap(bootCtx, prof, remote) + } + return transport.Bootstrap(bootCtx, prof.ServerURL, prof.RuntimeID, prof.RunnerCredential, Version) + }) if err != nil { - return fmt.Errorf("connect: bootstrap: %w", err) + return err } + wsURL, err := transport.DeriveWSURL(*boot, prof.ServerURL) if err != nil { return fmt.Errorf("connect: derive ws url: %w", err) diff --git a/apps/daemon/internal/cli/connect_bootstrap.go b/apps/daemon/internal/cli/connect_bootstrap.go index 1e7848808..27705a1c8 100644 --- a/apps/daemon/internal/cli/connect_bootstrap.go +++ b/apps/daemon/internal/cli/connect_bootstrap.go @@ -9,7 +9,7 @@ import ( // The launch file is the sole credential source for this connection. Reopening // it on process restart never reads or overwrites an auth profile. -func bootstrapProfile(path string) (*auth.Profile, error) { +func bootstrapProfile(path string, rc *runContext) (*auth.Profile, error) { raw, err := runtimefs.ReadPrivatePath(path, runtimebootstrap.MaxBytes) if err != nil { return nil, errors.New("connect: Runtime bootstrap file unavailable") @@ -18,5 +18,6 @@ func bootstrapProfile(path string) (*auth.Profile, error) { if err != nil { return nil, err } + rc.installedKinds = map[string]bool{input.Harness: true} return &auth.Profile{ServerURL: input.CoreURL, RuntimeID: input.DeviceID, RunnerCredential: input.Credential}, nil } diff --git a/apps/daemon/internal/cli/connect_bootstrap_test.go b/apps/daemon/internal/cli/connect_bootstrap_test.go index 7f510d699..fbfb704ab 100644 --- a/apps/daemon/internal/cli/connect_bootstrap_test.go +++ b/apps/daemon/internal/cli/connect_bootstrap_test.go @@ -31,7 +31,7 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { if err = os.WriteFile(filepath.Join(profileDir, "auth.json"), priorRaw, 0600); err != nil { t.Fatal(err) } - input := runtimebootstrap.Connection{Version: runtimebootstrap.Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "bootstrap-secret"} + input := runtimebootstrap.Connection{Version: runtimebootstrap.Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "bootstrap-secret", Harness: "codex"} raw, err := input.Marshal() if err != nil { t.Fatal(err) @@ -41,7 +41,11 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { t.Fatal(err) } for range 2 { - p, err := bootstrapProfile(path) + rc := &runContext{} + p, err := bootstrapProfile(path, rc) + if !rc.installedKinds["codex"] || len(rc.installedKinds) != 1 { + t.Fatal("bootstrap did not scope discovery") + } if err != nil || p.ServerURL != input.CoreURL || p.RuntimeID != input.DeviceID || p.RunnerCredential != input.Credential { t.Fatal("failed bootstrap/restart", err) } @@ -53,13 +57,13 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { if err = os.WriteFile(path, []byte("bootstrap-secret"), 0600); err != nil { t.Fatal(err) } - if _, err = bootstrapProfile(path); err == nil || strings.Contains(err.Error(), input.Credential) { + if _, err = bootstrapProfile(path, &runContext{}); err == nil || strings.Contains(err.Error(), input.Credential) { t.Fatal("invalid input fell back or leaked") } if err = os.Remove(path); err != nil { t.Fatal(err) } - if _, err = bootstrapProfile(path); err == nil { + if _, err = bootstrapProfile(path, &runContext{}); err == nil { t.Fatal("missing input fell back to private auth") } } diff --git a/apps/daemon/internal/cli/connect_environment.go b/apps/daemon/internal/cli/connect_environment.go index 0b1c3b4b1..4e9628e9e 100644 --- a/apps/daemon/internal/cli/connect_environment.go +++ b/apps/daemon/internal/cli/connect_environment.go @@ -163,13 +163,8 @@ func runEnvironmentConnect(parent context.Context, rc *runContext, profile strin if background && !daemonize.IsBackgroundChild() { return spawnBackground(parent, rc, profile, os.Args) } - // Discovery consumes the immutable Runtime binding; it must follow enrollment. - discovery, err := preflightAgentCLIs(parent, rc, profile) - if err != nil { - return err - } prof := auth.Profile{ServerURL: base, RuntimeID: bound.DeviceID, RunnerCredential: credential} - return rejected(mainLoopRemote(parent, rc, profile, prof, discovery, remote)) + return rejected(mainLoopRemote(parent, rc, profile, prof, remote)) } var ( diff --git a/apps/daemon/internal/cli/connect_startup.go b/apps/daemon/internal/cli/connect_startup.go new file mode 100644 index 000000000..68258f297 --- /dev/null +++ b/apps/daemon/internal/cli/connect_startup.go @@ -0,0 +1,50 @@ +package cli + +import ( + "context" + "errors" + "fmt" + + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/transport" +) + +// prepareConnection overlaps independent preflight work after local credentials +// and enrollment are resolved. Both workers are joined before returning, including +// on failure. Their temporary context never owns the live connection or executor. +func prepareConnection(parent context.Context, discover func(context.Context) (agentCLIDiscovery, error), bootstrap func(context.Context) (*transport.BootstrapResponse, error)) (*transport.BootstrapResponse, agentCLIDiscovery, error) { + if err := parent.Err(); err != nil { + return nil, nil, err + } + ctx, cancel := context.WithCancel(parent) + defer cancel() + var agents agentCLIDiscovery + var boot *transport.BootstrapResponse + done := make(chan error, 2) + go func() { + var err error + agents, err = discover(ctx) + done <- err + }() + go func() { + var err error + boot, err = bootstrap(ctx) + if err != nil { + err = fmt.Errorf("connect: bootstrap: %w", err) + } + done <- err + }() + var result error + for range 2 { + if err := <-done; err != nil { + result = errors.Join(result, err) + cancel() + } + } + if result != nil { + return nil, nil, result + } + if err := parent.Err(); err != nil { + return nil, nil, err + } + return boot, agents, nil +} diff --git a/apps/daemon/internal/cli/connect_startup_test.go b/apps/daemon/internal/cli/connect_startup_test.go new file mode 100644 index 000000000..6b5bca1b6 --- /dev/null +++ b/apps/daemon/internal/cli/connect_startup_test.go @@ -0,0 +1,266 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent" + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/auth" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/transport" +) + +func awaitStartup(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatal("startup worker did not reach barrier") + } +} + +func TestConnectionPreflightOverlapsAndJoins(t *testing.T) { + for _, first := range []string{"bootstrap", "discovery"} { + t.Run(first, func(t *testing.T) { + discoveryStarted, bootstrapStarted := make(chan struct{}), make(chan struct{}) + releaseDiscovery, releaseBootstrap := make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + boot := &transport.BootstrapResponse{DeviceID: "device"} + go func() { + result, _, err := prepareConnection(t.Context(), func(context.Context) (agentCLIDiscovery, error) { + close(discoveryStarted) + <-releaseDiscovery + return agentCLIDiscovery{}, nil + }, func(context.Context) (*transport.BootstrapResponse, error) { + close(bootstrapStarted) + <-releaseBootstrap + return boot, nil + }) + if err == nil && result != boot { + err = errors.New("bootstrap result lost") + } + done <- err + }() + awaitStartup(t, discoveryStarted) + awaitStartup(t, bootstrapStarted) + if first == "bootstrap" { + close(releaseBootstrap) + } else { + close(releaseDiscovery) + } + select { + case <-done: + t.Fatal("returned before both prerequisites completed") + default: + } + if first == "bootstrap" { + close(releaseDiscovery) + } else { + close(releaseBootstrap) + } + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("join blocked") + } + }) + } +} + +func TestConnectionPreflightFailureCancelsAndJoinsSibling(t *testing.T) { + for _, failing := range []string{"bootstrap", "discovery"} { + t.Run(failing, func(t *testing.T) { + cause := errors.New("rejected") + started, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + wait := func(ctx context.Context) error { + close(started) + <-ctx.Done() + close(canceled) + <-release + return ctx.Err() + } + fail := func() error { <-started; return cause } + go func() { + boot, agents, err := prepareConnection(t.Context(), func(ctx context.Context) (agentCLIDiscovery, error) { + if failing == "discovery" { + return nil, fail() + } + return nil, wait(ctx) + }, func(ctx context.Context) (*transport.BootstrapResponse, error) { + if failing == "bootstrap" { + return nil, fail() + } + return nil, wait(ctx) + }) + if boot != nil || agents != nil { + err = errors.New("partial successful result escaped") + } + done <- err + }() + awaitStartup(t, canceled) + select { + case <-done: + t.Fatal("returned before sibling cleanup") + default: + } + close(release) + select { + case err := <-done: + if !errors.Is(err, cause) { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("join blocked") + } + }) + } +} + +func TestConnectionPreflightParentCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + started := make(chan struct{}, 2) + done := make(chan error, 1) + go func() { + _, _, err := prepareConnection(ctx, func(ctx context.Context) (agentCLIDiscovery, error) { + started <- struct{}{} + <-ctx.Done() + return nil, ctx.Err() + }, func(ctx context.Context) (*transport.BootstrapResponse, error) { + started <- struct{}{} + <-ctx.Done() + return nil, ctx.Err() + }) + done <- err + }() + awaitStartup(t, started) + awaitStartup(t, started) + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("cancellation blocked") + } + _, _, err := prepareConnection(ctx, func(context.Context) (agentCLIDiscovery, error) { + t.Error("discovery started after cancellation") + return nil, nil + }, func(context.Context) (*transport.BootstrapResponse, error) { + t.Error("bootstrap started after cancellation") + return nil, nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +// Controlled waits model independent work, not production latency. Compare the +// same two operations to demonstrate the removed dependency edge. +func BenchmarkConnectionPreflight(b *testing.B) { + for _, parallel := range []bool{false, true} { + name := "serial" + if parallel { + name = "overlap" + } + b.Run(name, func(b *testing.B) { + discover := func(context.Context) (agentCLIDiscovery, error) { + time.Sleep(5 * time.Millisecond) + return agentCLIDiscovery{}, nil + } + bootstrap := func(context.Context) (*transport.BootstrapResponse, error) { + time.Sleep(5 * time.Millisecond) + return &transport.BootstrapResponse{}, nil + } + for b.Loop() { + if parallel { + _, _, _ = prepareConnection(b.Context(), discover, bootstrap) + } else { + _, _ = discover(b.Context()) + _, _ = bootstrap(b.Context()) + } + } + }) + } +} + +func TestMainLoopOverlapsPreflightWithoutEarlyRegistration(t *testing.T) { + for _, failure := range []string{"", "discovery", "bootstrap"} { + t.Run(failure, func(t *testing.T) { + t.Setenv("OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE", "") + original := harnessDeclarations + defer func() { harnessDeclarations = original }() + started, release, completed := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var probes, boots, dials atomic.Int32 + declaration := original[0] + declaration.Discover = func(ctx context.Context, _ agent.DiscoveryOptions, info proto.SupportedAgentKind) *agent.Runtime { + probes.Add(1) + close(started) + select { + case <-release: + case <-ctx.Done(): + } + defer close(completed) + info.Available = failure != "discovery" && ctx.Err() == nil + return &agent.Runtime{Info: info} + } + harnessDeclarations = []agent.Declaration{declaration} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/agent-daemon/bootstrap": + boots.Add(1) + select { + case <-started: + case <-r.Context().Done(): + return + } + if dials.Load() != 0 { + t.Error("dial before discovery completed") + } + close(release) + if failure == "bootstrap" { + http.Error(w, "rejected", http.StatusUnauthorized) + return + } + _ = json.NewEncoder(w).Encode(transport.BootstrapResponse{DeviceID: "device"}) + case "/agent-daemon/ws": + select { + case <-completed: + default: + t.Error("registration before discovery joined") + } + dials.Add(1) + http.Error(w, "end test", http.StatusUnauthorized) + default: + t.Errorf("unexpected route %s", r.URL.Path) + http.NotFound(w, r) + } + })) + defer server.Close() + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + err := mainLoopRemote(ctx, &runContext{stdout: io.Discard, stderr: io.Discard}, "default", auth.Profile{ServerURL: server.URL, RuntimeID: "device", RunnerCredential: "fixture"}, "") + if err == nil || ctx.Err() != nil { + t.Fatalf("unexpected completion: %v, context %v", err, ctx.Err()) + } + expected := int32(0) + if failure == "" { + expected = 1 + } + if probes.Load() != 1 || boots.Load() != 1 || dials.Load() != expected { + t.Fatalf("calls discovery=%d bootstrap=%d dial=%d", probes.Load(), boots.Load(), dials.Load()) + } + }) + } +} diff --git a/apps/daemon/internal/cli/native_discovery_test.go b/apps/daemon/internal/cli/native_discovery_test.go index fb230b5c3..e1dd6f225 100644 --- a/apps/daemon/internal/cli/native_discovery_test.go +++ b/apps/daemon/internal/cli/native_discovery_test.go @@ -79,3 +79,23 @@ func TestDiscoveryUnavailableAndCancelled(t *testing.T) { t.Fatal("unconfigured adapter retained") } } + +func TestSelectedHarnessUnavailableDoesNotProbeOthers(t *testing.T) { + declarations := append([]agent.Declaration(nil), harnessDeclarations...) + for i := range declarations { + declarations[i].Discover = func(_ context.Context, _ agent.DiscoveryOptions, info proto.SupportedAgentKind) *agent.Runtime { + if info.Kind != "codex" { + t.Fatalf("unselected Harness was probed: %s", info.Kind) + } + return &agent.Runtime{Info: info} + } + } + rc := &runContext{stdout: io.Discard, stderr: io.Discard, installedKinds: map[string]bool{"codex": true}} + if _, err := discoverAgentCLIs(t.Context(), rc, "default", declarations); err == nil { + t.Fatal("unavailable selection was accepted") + } + rc.installedKinds = map[string]bool{"unknown": true} + if _, err := discoverAgentCLIs(t.Context(), rc, "default", declarations); err == nil { + t.Fatal("unknown selection fell back") + } +} diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index 3b19e7fe6..74e338367 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -44,20 +44,21 @@ Each operation carries its own arguments and returns the following result on suc | `command` | `RunCommand` | `command` | `command` | | `observe` | `Observe` | `observation` | `sample` | | `initial` | `Initial` | None | `compute` | -| `new_compute` | `NewCompute` | Positive compute `generation` and optional `snapshot` | `compute` | +| `new_compute` | `NewCompute` | Positive `generation` and optional `retained` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | Exact current `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | None | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` and `command` | `command` | | `suspend` | `Suspend` | `suspend` | `state` | | `resume` | `Resume` | `resume` | `state` | -| `delete_snapshot` | `DeleteSnapshot` | `snapshot` | None | +| `delete_retained` | `DeleteRetained` | `retained` | None | -A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. A node without generation management accepts only its enrolled `deployment_generation`; a generation-managing node runs the request on that generation's provider and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not observe-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. +A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. A node without generation management accepts only its enrolled `deployment_generation`; a generation-managing node runs the request on that generation's provider and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not reconciliation-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. The budget is relative: the node anchors `timeout_ms` to its own clock on receipt and consumes it while the request waits in its queue, so the hosts' clocks need not agree. Core still bounds its own wait. A full node queue closes the connection. -The `response` frame carries `id` and `connection_id`. A successful response carries the result named in the operation table, with no result field for `kill`, `kill_compute` or `delete_snapshot`. A failed response carries an `error_code`: +The `response` frame carries `id` and `connection_id`. A successful response carries the result named in the operation table, with no result field for `kill`, `kill_compute` or `delete_retained`. A failed response carries an `error_code`: | `error_code` | Meaning | | --- | --- | @@ -67,7 +68,7 @@ The `response` frame carries `id` and `connection_id`. A successful response car | `unsupported` | The operation is declared unsupported; see below | | `unconfirmed`, or any other value | The outcome is unknown | -A failed response carries no result, except an `info` that is an exact-reference `CreateSettled` receipt: a confirmed native Create that failed a later check can still prove that the attempt settled. A timeout, a lost response or a disconnect is unavailable or uncertain, never evidence of absence, and Core never replays a mutation after one; it observes the original operation instead. The [Sandbox Provider guide](../../docs/sandbox-provider.md#operation-outcomes-and-retries) defines each outcome. +A failed response carries no result except an exact-provenance Resume target for cleanup (as specified below), or an `info` that is an exact-reference `CreateSettled` receipt: a confirmed native Create that failed a later check can still prove that the attempt settled. A timeout, a lost response or a disconnect is unavailable or uncertain, never evidence of absence, and Core never replays a mutation after one; it observes the original operation instead. The [Sandbox Provider guide](../../docs/sandbox-provider.md#operation-outcomes-and-retries) defines each outcome. Node startup and generation loading validate complete Provider operation declarations before accepting work, and the Core proxy uses the same registered declaration, so an unsupported operation rejects before node resolution or native I/O. The [operation contract](../../docs/sandbox-provider.md#explicit-operation-contracts) owns the inventory. An `unsupported` response carries an `unsupported` object with the exact method `operation` and an authored safe `reason`; the proxy checks both against the request. Missing, malformed or mismatched evidence is an unconfirmed result, never proof that a mutation was rejected. Unsupported stays distinct from observation unavailability and unknown compute or command results, and it neither settles resource ownership nor authorizes a replay. @@ -123,4 +124,6 @@ The readiness classes, one exported error and one code each, are authored in `se Preparation diagnostics keep fixed typed causes. Only artifact transfer, checksum or release-provenance failures report `runtime_download_failed`; the private preparer signals that class through its exit category, without Core or the node parsing stderr. Provider, ownership, cancellation and unclassified failures keep their typed code or `provider_unavailable`. No raw provider text crosses the protocol. -The current wire version is 5. Create bootstrap and Resume requests may carry an optional workspace filesystem binding. The node validates its tenant and Environment against the allocation reference, its immutable ObjectID, and its attachment configuration ID against the supplied configuration before forwarding it. The filesystem resolver validates adapter-native ownership. A missing binding selects owned storage; a present binding cannot fall back. Error responses may retain a Resume target only when its nonempty native ID, name, generation and snapshot provenance match the requested target; this is cleanup evidence, never successful restore. +Creation carries the Session-selected `Bootstrap.Harness` into Runtime bootstrap version 2. Core and nodes use protocol version 8 and require a coordinated upgrade. `Bootstrap.Harness` is required. All ten suspension operations belong to the same `SandboxProvider` and are declared supported or unsupported together; dispatch uses no optional interface. `Observe` reads one allocation per request. + +The current wire version is 8. Create bootstrap and Resume requests may carry an optional workspace filesystem binding. The node validates its tenant and Environment against the allocation reference, its immutable ObjectID, and its attachment configuration ID against the supplied configuration before forwarding it. The filesystem resolver validates adapter-native ownership. A missing binding selects owned storage; a present binding cannot fall back. Error responses may retain a Resume target only when its nonempty native ID, name, generation and retained-state provenance match the requested target; this is cleanup evidence, never successful restore. diff --git a/contracts/agents-api/zh/node-generation-protocol.md b/contracts/agents-api/zh/node-generation-protocol.md index b035ab7ab..b147594c5 100644 --- a/contracts/agents-api/zh/node-generation-protocol.md +++ b/contracts/agents-api/zh/node-generation-protocol.md @@ -1,7 +1,7 @@ --- title: "沙箱节点协议" source: contracts/agents-api/node-generation-protocol.md -source_hash: fb446e91c3df3c9e397e5e9abb794245b0ae11d81176879207718680e8ab4173 +source_hash: 074df236614044bf7e2c7a3049a6acbfad4bc84f29fe248600cfbcd8c3e8bf09 --- 沙箱节点在其主机上运行 Docker 或 microsandbox Provider，并通过一个 WebSocket 与 Core 相连。Core 通过该连接发送 Provider 操作；节点针对本地 Provider 执行这些操作，并报告就绪状态、主机测量值及其持有的部署代次。Core 始终是唯一的生命周期所有者：节点绝不重试变更操作或调度工作。帧和校验器位于 [`services/core/internal/sandbox/node`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/sandbox/node)（`wire.go`、`generation_wire.go`）；节点用于注册和读取配置的 HTTP 路由位于[机器连接 API](machine-api.md#node-routes)。 @@ -46,20 +46,21 @@ Core 发送包含以下内容的 `request` 帧： | `command` | `RunCommand` | `command` | `command` | | `observe` | `Observe` | `observation` | `sample` | | `initial` | `Initial` | 无 | `compute` | -| `new_compute` | `NewCompute` | 大于零的计算 `generation` 和可选的 `snapshot` | `compute` | +| `new_compute` | `NewCompute` | 大于零的计算 `generation` 和可选的 `retained` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | 精确的当前 `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | 无 | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` 和 `command` | `command` | | `suspend` | `Suspend` | `suspend` | `state` | | `resume` | `Resume` | `resume` | `state` | -| `delete_snapshot` | `DeleteSnapshot` | `snapshot` | 无 | +| `delete_retained` | `DeleteRetained` | `retained` | 无 | -只要 `connection_id`、`owner_epoch` 或 `sequence` 中任一值不匹配，请求就会关闭连接。格式错误的请求会得到 `invalid` 响应。未启用代次管理的节点仅接受其登记的 `deployment_generation`；支持代次管理的节点在对应代次的 Provider 上运行请求，无法运行时回复 `unconfirmed`。Core 仅向节点上已就绪的代次发送 `create` 和非 observe-only 的 `resume`，并且每条连接最多保留 32 个待处理请求。 +只要 `connection_id`、`owner_epoch` 或 `sequence` 中任一值不匹配，请求就会关闭连接。格式错误的请求会得到 `invalid` 响应。未启用代次管理的节点仅接受其登记的 `deployment_generation`；支持代次管理的节点在对应代次的 Provider 上运行请求，无法运行时回复 `unconfirmed`。Core 仅向节点上已就绪的代次发送 `create` 和非 reconciliation-only 的 `resume`，并且每条连接最多保留 32 个待处理请求。 预算采用相对计时：节点收到请求时以自己的时钟为基准锚定 `timeout_ms`，并在请求排队等待期间持续消耗该预算，因此各主机的时钟无需保持一致。Core 仍会限制自身等待时长。节点队列已满时会关闭连接。 -`response` 帧包含 `id` 和 `connection_id`。成功响应携带操作表中指定的结果；对于 `kill`、`kill_compute` 或 `delete_snapshot`，响应不含结果字段。失败响应携带一个 `error_code`： +`response` 帧包含 `id` 和 `connection_id`。成功响应携带操作表中指定的结果；对于 `kill`、`kill_compute` 或 `delete_retained`，响应不含结果字段。失败响应携带一个 `error_code`： | `error_code` | 含义 | | --- | --- | @@ -69,7 +70,7 @@ Core 发送包含以下内容的 `request` 帧： | `unsupported` | 该操作被声明为不支持；见下文 | | `unconfirmed` 或任何其他值 | 结果未知 | -失败响应不携带结果，唯一的例外是作为精确引用 `CreateSettled` 回执的 `info` 结果：即便已确认的原生 Create 在后续检查中失败，仍可证明该尝试已有确定结果。超时、响应丢失或断连属于不可用或不确定情况，绝不能证明资源不存在；发生这些情况后，Core 绝不重放变更操作，而是改为观察原始操作。[Sandbox Provider 指南](../../../docs/zh/sandbox-provider.md#operation-outcomes-and-retries) 定义了每种结果。 +失败响应不携带结果；例外包括下文规定的精确来源 Resume 目标清理证据，以及作为精确引用 `CreateSettled` 回执的 `info` 结果：即便已确认的原生 Create 在后续检查中失败，仍可证明该尝试已有确定结果。超时、响应丢失或断连属于不可用或不确定情况，绝不能证明资源不存在；发生这些情况后，Core 绝不重放变更操作，而是改为观察原始操作。[Sandbox Provider 指南](../../../docs/zh/sandbox-provider.md#operation-outcomes-and-retries) 定义了每种结果。 节点启动和代次加载会在接受工作前验证完整的 Provider 操作声明，Core 代理使用同一份已注册声明，因此不支持的操作会在节点解析或原生 I/O 之前被拒绝。操作清单由[操作契约](../../../docs/zh/sandbox-provider.md#explicit-operation-contracts)维护。`unsupported` 响应包含一个 `unsupported` 对象，其中有精确的方法 `operation` 和经作者编写且安全的 `reason`；代理会将两者与请求进行核对。证据缺失、格式错误或不匹配会得到 `unconfirmed` 结果，而绝不会证明变更操作被拒绝。`unsupported` 始终不同于观察不可用，也不同于计算或命令结果未知；它既不确定资源所有权，也不授权重放。 @@ -125,4 +126,6 @@ Runtime 字节缺失时，绝不将固定的放置实例迁移到当前 Runtime 准备诊断使用固定的类型化原因。只有制品传输、校验和或版本来源验证失败才会报告 `runtime_download_failed`；私有准备器通过退出类别指示这一类失败，Core 和节点都不解析 stderr。Provider 故障、所有权故障、取消和未分类故障保留其类型化代码，或使用 `provider_unavailable`。协议中不会传输任何 Provider 原始文本。 -当前线协议版本为 5。Create 引导和 Resume 请求可携带可选的工作区文件系统绑定。节点在转发前校验其租户及 Environment 与分配引用一致、ObjectID 不可变且有效，以及挂载配置 ID 与所传配置一致。文件系统解析器负责适配器原生所有权校验。未提供绑定时选择自有存储；已提供绑定时不得回退。错误响应仅可保留原生 ID 非空且名称、代次、快照来源均匹配请求的 Resume 目标；这仅为清理证据，不代表恢复成功。 +创建操作通过 `Bootstrap.Harness` 将会话选择传递到 Runtime 启动协议版本 2。Core 和节点使用协议版本 8，需协调升级配套组件。`Bootstrap.Harness` 是必填字段。十项暂停操作均属于同一个 `SandboxProvider`，必须完整声明支持或不支持；不通过可选接口分派。`Observe` 每次只观测一个 allocation。 + +当前线协议版本为 7。Create 引导和 Resume 请求可携带可选的工作区文件系统绑定。节点在转发前校验其租户及 Environment 与分配引用一致、ObjectID 不可变且有效，以及挂载配置 ID 与所传配置一致。文件系统解析器负责适配器原生所有权校验。未提供绑定时选择自有存储；已提供绑定时不得回退。错误响应仅可保留原生 ID 非空且名称、代次、保留状态来源均匹配请求的 Resume 目标；这仅为清理证据，不代表恢复成功。 diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml index 70471dc31..d91f0d8cd 100644 --- a/deploy/compose/compose.yaml +++ b/deploy/compose/compose.yaml @@ -59,6 +59,8 @@ services: OAC_CREDENTIAL_KEY_FILE: /run/oac/credential.key OAC_CORE_KEY_DIGESTS_FILE: /run/oac/core-key-digests.json OAC_EXECUTION_CONCURRENCY: ${OAC_EXECUTION_CONCURRENCY:-} + OAC_SANDBOX_MAX_ACTIVE: ${OAC_SANDBOX_MAX_ACTIVE:-} + OAC_SANDBOX_MAX_RETAINED: ${OAC_SANDBOX_MAX_RETAINED:-} OAC_DEFAULT_HARNESS: ${OAC_DEFAULT_HARNESS:-} OAC_HARNESSES: ${OAC_HARNESSES:-} OAC_WRITE_AUDIT_RETENTION: ${OAC_WRITE_AUDIT_RETENTION:-} diff --git a/deploy/compose/test_compose.py b/deploy/compose/test_compose.py index 4f14ef726..7a4fcc6b3 100644 --- a/deploy/compose/test_compose.py +++ b/deploy/compose/test_compose.py @@ -71,9 +71,16 @@ def test_compose_uses_private_services_and_ordered_initialization(self): self.assertEqual(services['web']['healthcheck']['test'], ['CMD', '/usr/local/bin/oac-web', 'healthcheck']) self.assertNotIn('python3', json.dumps(self.compose)) self.assertEqual(services['init']['environment']['OAC_REVISION'], 'd' * 40) - for name in ('OAC_EXECUTION_CONCURRENCY', 'OAC_DEFAULT_HARNESS', 'OAC_HARNESSES', 'OAC_WRITE_AUDIT_RETENTION', 'OAC_LOG_LEVEL'): + for name in ('OAC_SANDBOX_MAX_ACTIVE', 'OAC_SANDBOX_MAX_RETAINED', 'OAC_EXECUTION_CONCURRENCY', 'OAC_DEFAULT_HARNESS', 'OAC_HARNESSES', 'OAC_WRITE_AUDIT_RETENTION', 'OAC_LOG_LEVEL'): self.assertEqual(services['core']['environment'][name], '', name) + def test_direct_sandbox_capacity_reaches_core(self): + from unittest.mock import patch + with patch.dict(os.environ, {'OAC_SANDBOX_MAX_ACTIVE': '17', 'OAC_SANDBOX_MAX_RETAINED': '61'}): + configured = self.render()['services']['core']['environment'] + self.assertEqual(configured['OAC_SANDBOX_MAX_ACTIVE'], '17') + self.assertEqual(configured['OAC_SANDBOX_MAX_RETAINED'], '61') + def test_public_url_can_be_configured_after_initial_startup(self): for value in (None, '', 'https://oac.example.test', 'http://localhost:9080'): with self.subTest(public_url=value): diff --git a/docs/configuration.md b/docs/configuration.md index fb83fb72c..248af0ba8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -50,6 +50,8 @@ Model providers are not process settings; see [Default models](#default-models). | `OAC_LOG_FORMAT` | `auto` | `auto`, `text` or `json` | | `OAC_LOG_ADD_SOURCE` | unset | `1` adds source locations | | `OAC_EXECUTION_CONCURRENCY` | `4` | Concurrent execution work, from 1 to 1024 | +| `OAC_SANDBOX_MAX_ACTIVE` | `100` | Active sandbox limit for direct Providers with suspension, from 1 to 100000. Independent of execution concurrency and node capacity | +| `OAC_SANDBOX_MAX_RETAINED` | `400` | Retained sandbox limit for direct Providers, from 1 to 100000, including active, suspended and unconfirmed cleanup. Must be at least the active limit | | `OAC_DEFAULT_HARNESS` | `codex` | Harness used when a request does not name one | | `OAC_HARNESSES` | Every registered Harness | Comma-separated Harnesses to enable besides the default one. Unknown names stop startup | | `OAC_WRITE_AUDIT_RETENTION` | `2160h` | Minimum `1h` | @@ -101,6 +103,10 @@ Runtime settings live in Core's database. Change them in Web; scripts use the sa Which harnesses are enabled, and the default one, are process settings (`core.harnesses`, `core.default_harness`); System shows them read-only. The [Core administration API](../contracts/agents-api/admin-api.md) lists every Core API route, and the [deployment contract](../contracts/agents-api/sandbox-deployment.md) defines the sandbox fields, limits and change rules. +### Direct Provider capacity + +Set `OAC_SANDBOX_MAX_ACTIVE` and `OAC_SANDBOX_MAX_RETAINED` in `.env`, then run `oac apply`. Core uses these limits for direct Providers with suspension enabled. Every unreleased allocation consumes retained capacity. Lowering a limit stops no existing sandbox; new allocations wait until usage falls below both limits. These settings are independent of `OAC_EXECUTION_CONCURRENCY` and enrolled node capacity. + ### Independent workspace storage The initial supported independent filesystem combination is microsandbox with the [kernel NFS adapter](./workspace-provider.md#kernel-nfs-adapter). For a new installation, prepare storage and service accounts first, expose the mount to Core, select storage, configure microsandbox, then enroll nodes. Mount the same NFSv4.2 export on the Linux Core host and every participating node before starting their services, at the same absolute path, for example `/srv/oac-workspaces`. The operator manages the export, mount availability and service startup ordering. Use a trusted-client AUTH_SYS export with `root_squash`; do not enable `no_root_squash` or broaden permissions to make a check pass. Prepare the namespace and service principals according to the adapter's [ownership requirements](./workspace-provider.md#kernel-nfs-adapter). @@ -207,7 +213,7 @@ Core reads its process environment. Compose interpolates `.env` into it and moun | `OAC_CREDENTIAL_KEY_FILE` | Required. `/run/oac/credential.key`: a base64-encoded random 32-byte key. Core seals stored credentials with it | | `OAC_CORE_KEY_DIGESTS_FILE` | Required. `/run/oac/core-key-digests.json`: a JSON array with the SHA-256 of the Core key | | `OAC_INSTALLATION_ID_FILE` | Required. `/run/oac/installation.id`: the installation ID, a canonical UUID. Core refuses an ID other than the one its database recorded | -| `OAC_EXECUTION_CONCURRENCY`, `OAC_DEFAULT_HARNESS`, `OAC_HARNESSES`, `OAC_WRITE_AUDIT_RETENTION`, `OAC_OAUTH_TRUSTED_ORIGINS`, `OAC_HISTORY_SETTINGS_FILE`, `OAC_LOG_LEVEL`, `OAC_LOG_FORMAT`, `OAC_LOG_ADD_SOURCE` | The matching [process settings](#settings). Web reads the three log settings too | +| `OAC_SANDBOX_MAX_ACTIVE`, `OAC_SANDBOX_MAX_RETAINED`, `OAC_EXECUTION_CONCURRENCY`, `OAC_DEFAULT_HARNESS`, `OAC_HARNESSES`, `OAC_WRITE_AUDIT_RETENTION`, `OAC_OAUTH_TRUSTED_ORIGINS`, `OAC_HISTORY_SETTINGS_FILE`, `OAC_LOG_LEVEL`, `OAC_LOG_FORMAT`, `OAC_LOG_ADD_SOURCE` | The matching [process settings](#settings). Web reads the three log settings too | | `OAC_PROVIDER_ROOT` | Absolute adapter artifact root. The Core image sets `/opt/oac`. Each adapter owns its helper paths beneath this root. Core serves self-hosted daemon installers from its `native-installers/` directory when that holds a `catalog.json`, after checking the catalog against its own release. Adapter state lives at `/state`, the data volume's [`state/`](#compose-installations) | Core logs the history file path it loads, never environment values or file contents. diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index c3e5b8ca7..2f52a1a99 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -24,6 +24,9 @@ docker compose -f ~/.oac/core/compose.yaml ps The examples use the default installation directory. On Windows, invoke the management command with `& "$HOME/.oac/core/oac.exe"` followed by the same arguments. For a custom installation directory, replace the path in each command. +After local credentials and enrollment are resolved, Runtime Harness discovery and the authenticated bootstrap HTTP request run concurrently. Both must succeed before the Runtime opens its connection or publishes capabilities. Failure cancels the sibling operation and waits for cleanup. The executor remains owned by the connection lifetime. Runtime startup changes require a rebuilt, qualified Runtime template. + + ## Service health Use these observations for different questions: diff --git a/docs/runtime-bootstrap.md b/docs/runtime-bootstrap.md index 7397e07fa..5a788ddf5 100644 --- a/docs/runtime-bootstrap.md +++ b/docs/runtime-bootstrap.md @@ -17,6 +17,7 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json | `version` | The exact bootstrap version, `runtimebootstrap.Version` | | `core_url` | HTTP(S) machine API base ending in `/api/v1`, without credentials, query or fragment | | `device_id` | Canonical nonzero UUID of the daemon identity Core issued | +| `harness` | The immutable Session Harness identifier; only this Harness is probed and registered by a managed Runtime | | `credential` | Nonempty daemon credential Core issued, without whitespace or NUL | The decoder rejects unknown, duplicate, missing and case-aliased fields, other versions and documents larger than `runtimebootstrap.MaxBytes` (16 KiB). Errors never include submitted values. A missing or malformed file fails before the daemon connects. @@ -27,10 +28,16 @@ The file is the only authentication input for this launch: the daemon refuses to The provider creates the account, mounts and workspace, delivers this file, sets the Runtime's resource and Environment binding settings, and starts the daemon as the unprivileged Runtime account. Docker writes the file into the Runtime's owned home volume; microsandbox and E2B deliver it before launching the same command. +Core derives `harness` from the Session that owns the allocation. A combined image can contain several Harnesses, but its managed Runtime discovers only this selection; an unknown, missing or unavailable selection fails without probing another Harness. The bootstrap version is 2. Upgrade Core, its provider helpers and newly launched Runtime images together; retained allocations keep their existing bootstrap and Runtime. Self-hosted installations continue to discover their installed Harness set. + The Runtime validates the input and owns authentication and connection. A successful launch proves only the handoff: an authenticated connection, prepared capabilities and execution readiness are separate observations under the [Core–Runtime protocol](./runtime-protocol.md), and the [Sandbox Provider guide](./sandbox-provider.md#four-distinct-readiness-facts) lists what each one proves. Self-hosted executors and operator-provisioned devices get their daemon identity in other ways; the [machine connection API](../contracts/agents-api/machine-api.md#credentials) lists every credential source. All of them enter the same Runtime execution loop. +## Hosted suspension control + +The private hosted park/wake control-file path is authored as `SuspendControlFile` in `internal/runtimebootstrap/bootstrap.go`. Core recovery and native Go adapters read that value; the E2B helper contract generator projects it into the template builder. Managed startup prepares its private directory and supplies `OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE` to enable Runtime suspension. This is a packaged protocol setting. The shared Sandbox Provider registration owns idle and retention defaults; the adapter owns its native lease timeout. + ## Verification `go test ./internal/runtimebootstrap ./apps/daemon/internal/cli` covers the input contract, the exclusivity of credential sources and restart behavior. Provider tests verify delivery and file permissions without relying on the Runtime's private storage. diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index f70b826dc..d96ce60ac 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -25,7 +25,7 @@ Core owns durable Environment, allocation, placement and cleanup state; the Prov ## Implement the interface -`sandbox_provider.go` holds the Core–Sandbox Provider protocol: the `SandboxProvider` interface for allocation, checkpoint and observation, its request and result types, and the setup-time `ConfigurationAdapter` with its typed errors. Value types that Core also uses beyond this boundary, such as `DeploymentSpec` and `CallFence`, live in their own files of the same package. Every method is required at compile time, and `ProviderOperations()` declares which ones the Provider supports. Five operations are always supported: +`sandbox_provider.go` holds the Core–Sandbox Provider protocol: the `SandboxProvider` interface for allocation, suspension and observation, its request and result types, and the setup-time `ConfigurationAdapter` with its typed errors. Value types that Core also uses beyond this boundary, such as `DeploymentSpec` and `CallFence`, live in their own files of the same package. Every method is required at compile time, and `ProviderOperations()` declares which ones the Provider supports. Five operations are always supported: | Operation | Purpose | | --- | --- | @@ -45,7 +45,7 @@ Every provider returns a complete `ProviderOperations()` declaration with one en | --- | --- | --- | | `Create`, `GetInfo`, `Renew`, `Kill`, `RunCommand` | Supported | Allocation lifecycle and bounded commands | | `Observe` | Explicit decision | Ownership-checked read-only observation of one allocation | -| `Initial`, `NewCompute`, `GetCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute` | The same decision for every checkpoint method; supported only by a `nodes` registration | Exact compute incarnations, capture and restore, retained-source resume and cleanup | +| `Initial`, `NewCompute`, `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteRetained`, `RunCommandCompute` | The same decision for all ten suspension methods in `nodes` or `direct` mode | Exact compute incarnations, capture and restore, retained-source resume and cleanup | `Initial` and `NewCompute` construct compute references without allocating, `ResumeCompute` thaws only the same resident instance after an aborted pause, and `RunCommandCompute` runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)). @@ -89,7 +89,7 @@ Every call receives a bounded context. Expiry or cancellation ends the caller's `ErrInvalid`, `ErrOwnership`, `ErrExists`, `ErrNotFound`, `ErrComputeUnconfirmed` and `ErrCommandUnconfirmed` keep their defined meanings. An unclassified native or transport error is unknown, never permission to retry a mutation. Core never reads provider diagnostics as lifecycle truth or exposes native error text or credentials; the node transport maps errors to fixed codes, and direct SDK details stay private. -Checkpoint support adds `Compute` generation, name and ID and `SnapshotIdentity`; persist operation IDs and the provider's snapshot provenance unchanged. `ObserveOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and snapshot, not whatever instance now has the same name. Read [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) and its failure tests before declaring checkpoint support. +Suspension support adds `Compute` generation, name and ID and `RetainedState`; persist operation IDs and the provider's retained-state provenance unchanged. `ReconcileOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and retained state, not whatever instance now has the same name. Read [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) and its failure tests before declaring suspension support. ### Four distinct readiness facts @@ -112,7 +112,7 @@ A new provider takes these steps: 2. Add its specification and resource validators. 3. Implement `sandbox.ConfigurationAdapter` over a typed native configuration. `DecodeInput` strictly parses the separate public `configuration` and write-only `credential` objects of a request. `Encode` produces whitelisted public selectors, read-only observations and separate secret bytes, and never passes request JSON through. `Decode` restores stored selectors, and keeps access to owned resources, without remote admission or new template validation. `Normalize` copies its input before changing it. `ResolveChange`, `Equal` and `WithCredential` own inheritance, identity and credential composition. `Requirements` declares whether a credential and a public Core origin are required, and which setup operations are supported: `Discovery` for `DiscoverConfiguration`, `SelectionDiscovery` for `DiscoverSelection` and `CredentialVerification` for `VerifyCredential`. `DiscoverConfiguration` validates the query and returns a safe catalog, never a mutation or an admission decision, while Core keeps authorization, input limits and deadlines. `DiscoverSelection` resolves a candidate's omitted native values before commit, and `VerifyCredential` verifies a credential's access to owned resources without mutation. Both receive the candidate's `sandbox.DirectConfig` and build any native client for that call only. A node provider accepts only an empty public object, rejects credentials and returns Unsupported for every setup operation and for credential replacement. 4. For a node adapter, export from its package the `BuildLocal` constructor, the typed `native` object it decodes and the native files it adds to the shared node artifacts. Node-local settings, such as host paths, live only in that object; resources and the Runtime release are read from the node configuration's `specification`. -5. Register its constructor, policies, configuration adapter and operation declaration in `providers/registry.go`. Its key is the provider kind, which also labels the Provider's observations, and checkpoint support reads this entry. The generated projections combine each registered mode and deployment policy with the shared field bounds in `sandbox/deployment_contract.go`: the installer reads them from `deploy/node/node_spec.py`, and the TypeScript client and Web from `packages/agents-client/src/deployment-contract.ts`, so Web reads these declarations instead of comparing provider kinds. Regenerate both with `go run ./services/core/cmd/specification-contract -write`. +5. Register its constructor, policies, configuration adapter and operation declaration in `providers/registry.go`. Its key is the provider kind, which also labels the Provider's observations, and suspension support reads this entry. The generated projections combine each registered mode and deployment policy with the shared field bounds in `sandbox/deployment_contract.go`: the installer reads them from `deploy/node/node_spec.py`, and the TypeScript client and Web from `packages/agents-client/src/deployment-contract.ts`, so Web reads these declarations instead of comparing provider kinds. Regenerate both with `go run ./services/core/cmd/specification-contract -write`. 6. Supply the distribution artifacts for the adapter and its helper, and offer the provider to operators through the registered configuration contract. **Known design gap:** Web still names providers in the setup wizard's backend choice, the Docker confirmation and E2B's configuration fields wherever Web shows or parses them (the setup step with its service presets, the deployment summary and the client's deployment projection), because the protocol declares no configuration fields yet. Exposing another provider through that surface currently requires a shared Web edit. This coupling does not meet [Complexity stays in the adapter](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#complexity-stays-in-the-adapter); new integrations must express their configuration through the protocol and keep vendor-specific behavior in the adapter. Never add a Session or Turn scheduling path, a vendor column or API field, or a vendor switch in the store. @@ -126,7 +126,7 @@ A node configuration, `sandbox.NodeConfig`, holds `provider`, `generation`, `ins - A `nodes` registration has only `BuildLocal`, and a `direct` registration only `BuildDirect`; missing, mixed or unknown modes are rejected. - The specification and resource validators, the configuration adapter and a complete operation declaration are mandatory, so an incomplete registration cannot publish a partial projection. A declared default size must pass the adapter's resource validator. - The Runtime input policy either accepts the pinned Runtime or gives the adapter's fixed reason for rejecting it, never both. -- Checkpoint is admitted only for a `nodes` registration, because the common lifecycle suspends only node allocations; registration rejects a `direct` Provider that declares it. A registration carries no suspension values: Core applies its one [suspension policy](#suspension) to every Provider that declares checkpoint support. +- Suspension is admitted in both `nodes` and `direct` modes when the complete lifecycle is declared supported. A registration carries no suspension values: Core applies its one [suspension policy](#suspension) to every supported Provider. The configuration adapter must be non-nil, including its concrete value. Every `ConfigurationRequirements` field needs an explicit valid decision: `Credential` and `PublicOrigin` are `Required` or `NotRequired`, and `Discovery`, `SelectionDiscovery` and `CredentialVerification` use the shared supported or unsupported declaration with a safe reason. A new requirement field needs an explicit validation update and never inherits an existing decision. Requiring a credential does not promise the `VerifyCredential` operation. These checks establish complete registration, not correct native SDK behavior; constructor and adapter contract tests still apply. @@ -136,7 +136,7 @@ Preview and persistence use `providers.Normalize` and `providers.Describe`. `pro A direct adapter with a credential verifies all retained generations and allocation references before a key is replaced. The common `sandbox.CallFence` excludes native calls and waits for helper completion, including calls whose callers timed out; execution invokes the prepared verification and fencing callbacks without branching on a vendor. -Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `sandbox.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and a `Quiescent` check when a helper can outlive its caller; generation collection waits for it. The factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes checkpoint operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through the checkpoint declaration, never through a provider name. +Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `sandbox.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and a `Quiescent` check when a helper can outlive its caller; generation collection waits for it. The factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes suspension operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through the suspension declaration, never through a provider name. The backend fingerprint identifies a native resource namespace, not capacity. Core keeps deployment generations so that owned allocations keep resolving to their original backend; never repoint retained allocations at a replacement backend. @@ -160,16 +160,20 @@ Node readiness binds to the exact generation, the current connection and the own ### Allocation lifecycle +Core includes the owning Session’s immutable Harness in `Bootstrap.Harness`; every provider projects it into the [Runtime bootstrap](./runtime-bootstrap.md) without choosing an implementation. + The allocation, its dedicated daemon credential digest and the exact Session binding commit atomically before `Create`, under the execution lease and the Session lock. Only a fresh allocation receipt permits `Create`; retries and a Core restart observe the same reference without replaying it or rotating the credential. An allocation is private compute ownership, separate from public Environment connection and native readiness; adapters qualify bootstrap completion, and Core never infers it from an engine or provider name. With a configured provider, the Worker scans committed pending hosted Environments that have no allocation, which covers idle Session creation and recovery after an interruption between commit and bootstrap; an existing allocation never re-enters that path. The scan is bounded and serialized by the lifecycle owner and needs no caller action. An initial reservation without a Turn leaves its Session idle, and a daemon connection is never treated as native readiness. The same scan publishes authenticated connection observations with durable generations, after verifying the exact Session and device binding and a settled bootstrap. -Between Turns, Core checks that connected, observed compute is still its Session's running allocation; the check changes nothing and never revives a cleanup request. Running compute never expires: explicit lifecycle cleanup and snapshot retention govern reclamation. A stopped or missing container never authorizes discarding retained workspace or history or selecting replacement compute. Disabling the provider stops new hosted admission and bootstrap but never blocks cancellation, function results or input retry outcomes of existing Sessions. +Between Turns, Core checks that connected, observed compute is still its Session's running allocation; the check changes nothing and never revives a cleanup request. Running compute never expires: explicit lifecycle cleanup and retained-state retention govern reclamation. A stopped or missing container never authorizes discarding retained workspace or history or selecting replacement compute. Disabling the provider stops new hosted admission and bootstrap but never blocks cancellation, function results or input retry outcomes of existing Sessions. Terminal cleanup atomically revokes the device's authority, records the Environment's failure or expiry, settles pending input and requests cancellation, and only then calls `Kill`; original input deadlines and retry outcomes are kept. Temporary provider outages, unknown Create results and stopped compute never prove a permanent failure. After public Session deletion Core keeps the allocation and marks it released only after owned compute and volume cleanup and proof that the original Create settled; an unknown creation keeps cleanup ownership even after an absence observation, and bounded scans continue to catch late resources without another `Create`. ### Per-node lifecycle workers +After a hosted Environment commits, Core sends a bounded hint to the lifecycle worker for its placement. Committed pending input also sends a hint to recover a missed creation notification. Hints reuse the serial gate, execution lease, capacity checks and one-shot allocation receipt; admission handlers never create a sandbox directly. Each normal maintenance period admits at most one extra hinted scan, and periodic scans recover missed or coalesced hints. Existing allocations are observed before pending provisioning, so a slow observation on the same node can still delay a fresh Environment. + Each registered node has one serial lifecycle worker that owns its gate, allocation and pending cursors, connections and wake hints; E2B allocations share one serial lifecycle without a node. A thin coordinator discovers nodes and shuts workers down, and never holds its map mutex during database, provider or wait operations. Workers advance independently, so a stuck provider on one online node never stalls another: lifecycle concurrency is one operation per node and grows with the node count. Offline workers stay, so their retained resources remain observable after reconnection. Allocation scans filter by node before their 32-row page limit, and pending scans join the unreleased committed placement. Each node advances its own cursor, including past failed observations, and wraps once at the end. Direct provisioning resolves the tenant-scoped placement before entering that node's gate, and an active allocation must agree with it. Only confirmed release permits a new placement; a disconnect never moves an active allocation. @@ -180,35 +184,49 @@ Before releasing the execution lease, the coordinator stops accepting work and c Placement is automatic: Session creation commits durable pending work, and the common scheduler reserves a compatible node before allocation. The [deployment contract](../contracts/agents-api/sandbox-deployment.md#generation-ownership-and-rollout) owns waiting, ordering and generation selection. Callers cannot choose a node, and an active allocation keeps its original node even while it is offline. After confirmed allocation release, eligible retained Sessions can reserve compatible capacity again, including on another node. Node capacity counts pending reservations and unresolved resources, and new placement and a suspended-to-restoring transition share a database lock. Unknown operations keep their reservations, source teardown must be confirmed before active capacity is released, and confirmed cleanup releases placement capacity. Retained ownership needs exact provider evidence: a socket path, a missing instance or an empty listing never proves cleanup or authorizes a replacement. -The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and the snapshot retention bound each node. Cold replacement after confirmed release does not provide node-level drain, failover from an unreachable writer, concurrent writers, multi-active Core, autoscaling or snapshot replication. Node removal is refused while the node holds allocations, snapshots, reservations, unknown results or cleanup, and offline ownership is kept. +The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and the retained-state retention bound each node. Cold replacement after confirmed release does not provide node-level drain, failover from an unreachable writer, concurrent writers, multi-active Core, autoscaling or snapshot replication. Node removal is refused while the node holds allocations, retained states, reservations, unknown results or cleanup, and offline ownership is kept. ### Suspension -Core suspends the idle work of every provider that declares checkpoint support, with one shared policy: normally it suspends work idle for 5 minutes (300 seconds) and keeps the snapshot for 24 hours (86400 seconds). The deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) reports these values. Core suspends after initialization completes, when no root or Subagent Turn is queued, in progress or waiting, no input or file operation is pending, and real activity has been idle for that time. A Session need not have run a Turn. The idle clock starts no earlier than the allocation entering the running compute phase, including after a wake. For node allocations Core records the first root or child terminal transition with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial snapshot retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. +The single `SandboxProvider` contract declares `Initial`, `NewCompute`, `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `KillCompute`, `DeleteRetained`, `RunCommandCompute` and `ResumeCompute` together: all are supported or all are unsupported. No optional suspension interface or provider-name branch selects this lifecycle. + +`RetainedState` is an opaque adapter-owned envelope with `Reference`, `ID`, `Data`, `OperationID`, `SourceGeneration`, `SourceName`, `SourceID` and optional `Compatibility`. `Data` is nonempty and at most 64 KiB. Core preserves the envelope unchanged and validates operation and source ownership; the immutable allocation provider/generation binding selects the only adapter allowed to interpret it. A retained state need not be an independent snapshot. Microsandbox retains its complete native snapshot identity in `Data`; E2B retains its versioned native pause receipt and cannot promise a separate disk image. Each adapter validates its native payload and ownership before any effect. + +`Compute.RestoredFrom`, `ComputeState.Retained`, `ResourcesReleased`, `SuspendSettled` and recovery's `ReconcileOnly` are authoritative settlement evidence. A missing resource is not proof that an unknown suspension settled. `RenewCompute` renews only the exact incarnation; an uncertain result never authorizes a second create, capture or restore. Durable `runtime_compute` uses `protocol_version: "1"`; startup rejects missing or unknown versions. Version-1 envelopes matching the current retained-state field shape remain readable without a rewrite. Snapshot-only durable shapes are rejected even when labeled version 1; equal version numbers do not establish compatibility. Upgrade performs no migration, synthetic replay or retained-state discard. + +`ResumeRequest.Workspace` carries the allocation's independently owned filesystem binding. Core obtains a ready binding before restore. Suspension must prove the source released active execution before a target can become a writer. Adapters validate the binding before native effects; an adapter without independent filesystem support rejects a non-null binding. `DeleteRetained` deletes only adapter-owned retained compute state, never the workspace. Explicit Session deletion first settles compute cleanup and then deletes independent storage, preserving a single active writer throughout recovery. + +Core suspends the idle work of every provider that declares suspension support, with one shared policy: normally it suspends work idle for 5 minutes (300 seconds) and keeps retained state for 24 hours (86400 seconds). The deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) reports these values. Core suspends initialized Environments, including those with no Turn yet, when no root or Subagent Turn is queued, in progress or waiting, no input, file operation or initialization is pending, and real activity has been idle for that time. The idle clock starts no earlier than the allocation entering the running compute phase, including after a wake. For allocations Core records initialization completion and root or child terminal transitions with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial retained-state retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. Capacity pressure can suspend an already idle node allocation before the normal timeout, after a 15-second inactivity grace. The Session-locked decision rechecks pending work, wake requests and activity, then takes the shared deployment capacity lock and confirms compatible waiting demand. It never interrupts an active Turn or releases capacity before the ordinary quiesce, capture and source-stop proofs complete. Pressure-triggered reclamation is conservative: an in-flight reclamation delays another only when its node is currently eligible to serve the same demand. Unavailable or incompatible nodes keep their ownership receipts without blocking healthy capacity. If suspension cannot make capacity usable, the waiting request keeps its original deadline. -The Worker lease, the Session lock and the per-node gates own suspension for every provider. New Turn claims, file-write intents and capture admission serialize under the Session lock and share one compute-phase check; new pending work cancels a capture and wakes the same source. Normal preparation waits for the compute phase to be running, after the authenticated resume handshake, and pending input stays pending when its promotion conflicts with a lifecycle transition. Compute phases and revision-checked receipts live on the allocation. Core persists quiesce, capture and restore intent before the effect, only a fresh receipt performs a capture or restore, and recovery observes the exact attempt without retrying an unknown creation, capture or restore. A consumed snapshot never rolls a running generation back. Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap, and unknown cleanup identities are kept until owned resources are confirmed absent. Consumed artifacts and old compute are deleted, so suspension cycles never build a chain of writable disks. +The Worker lease, the Session lock and the per-node gates own suspension for every provider. New Turn claims, file-write intents and capture admission serialize under the Session lock and share one compute-phase check; new pending work cancels a capture and wakes the same source. Normal preparation waits for the compute phase to be running, after the authenticated resume handshake, and pending input stays pending when its promotion conflicts with a lifecycle transition. Compute phases and revision-checked receipts live on the allocation. Core persists quiesce, capture and restore intent before the effect, only a fresh receipt performs a capture or restore, and recovery observes the exact attempt without retrying an unknown creation, capture or restore. Consumed retained state never rolls a running generation back. Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap, and unknown cleanup identities are kept until owned resources are confirmed absent. Consumed artifacts and old compute are deleted, so suspension cycles never build a chain of writable disks. -Queued work and live Environment file access wake a suspended Environment; history and published Artifact reads do not. Live file requests wait for the current Runtime’s credential-authorized connection and Harness declaration before entering the file work queue; a completed compute Create alone is not Runtime readiness. Planned suspension uses an Environment and suspension token on the daemon connection. A PID and start-time fenced local control signal (`RunCommandCompute`) wakes the parked daemon, which authenticates again before admitting work. A transient disconnect before confirmation retries the same armed suspension with bounded attempts and backoff; a permanent authentication or protocol rejection closes it. A retained checkpoint resumes on its original node when it is eligible, or on another eligible node through the checkpoint transfer rules below. Core owns the snapshot's retention deadline, and the daemon has no timer for it. A lost quiesce acknowledgement may thaw the same source through explicit rollback but never authorizes capturing it. +`Suspend` owns native resource release. Core releases active capacity only after receiving a bound retained handle, suspended state, `ResourcesReleased` and `SuspendSettled`. `ReconcileOnly` prohibits replaying the original capture or pause but permits adapter cleanup justified by durable retained evidence. An outcome without retained state permits rollback only with `SuspendSettled` and an exact source that can resume; other uncertainty retains ownership and closes admission. Core never unconditionally destroys the source after `Suspend`. Without retained state, `SuspendSettled` also proves durable closure of that exact source and operation against every late native dispatch. An in-process call fence, an allocation lock or current native absence alone cannot establish this proof. Both E2B and microsandbox keep a bounded adapter-private admission journal; rollback cannot erase same-generation closures. Only a verified later source generation can advance its fence. + +`Resume` consumes retained state once into the precommitted target. Recovery observes that same attempt. Core persists `waking`, authenticates and resumes the daemon, deletes consumed retained resources, then commits `running` and admits work. `DeleteRetained` is idempotent cleanup that preserves running compute. Cleanup failure keeps `waking` and cannot trigger another restore. `KillCompute` remains destructive; old-generation cleanup must not kill a newer active incarnation sharing its native ID. Every unreleased allocation, including a running one, consumes `max_retained`. Every reservation records the shared compute protocol version, including allocations whose compute phase is `disabled`. Activation rejects an unreleased allocation with a missing or different version; the old version must complete its normal cleanup before upgrade. Session history remains. + +### Coordinated protocol upgrade + +Upgrade Core and nodes together with node wire version 8, the E2B helper with private wire version 4 and its rebuilt template containing hosted suspension control, and the microsandbox helper with private wire version 5. All Providers deliver Runtime bootstrap version 2. Each boundary validates its own contract; version numbers are not interchangeable across boundaries. Before activation, use the previous compatible release to complete normal cleanup of incompatible unreleased allocations. The startup fence preserves their stored receipts and Session history and makes no Provider calls to migrate them. ### Checkpoint transfer -A checkpoint-capable generation reports `CheckpointCompatibility`: opaque `ArtifactDomain` and `ExecutionClass` tokens. Every verified `SnapshotIdentity` carries the same qualification. Core compares these values without interpreting CPU features, filesystem paths or storage implementations. A restore destination must be online and ready for the allocation's immutable deployment generation, match both tokens, and have active capacity plus a retained slot when moving from another node. The allocation, Device and Session identities remain unchanged. The Session and deployment transaction commits the destination route, placement and exact restore intent before target-side native work. No capacity or compatible destination leaves the retained snapshot owned until its configured deadline. The target must already hold that exact generation; Core does not automatically prepare historical generations on new nodes. During upgrades, retain eligible nodes and their generation providers until their checkpoint retention obligations end. +A checkpoint-capable generation reports `CheckpointCompatibility`: opaque `ArtifactDomain` and `ExecutionClass` tokens. A transferable `RetainedState` carries both tokens in `Compatibility`. A direct retained state may omit them; omission never qualifies cross-node restoration or cleanup. E2B native pause does not declare checkpoint portability. Core compares these values without interpreting CPU features, filesystem paths or storage implementations. A restore destination must be online and ready for the allocation's immutable deployment generation, match both tokens, and have active capacity plus a retained slot when moving from another node. The allocation, Device and Session identities remain unchanged. The Session and deployment transaction commits the destination route, placement and exact restore intent before target-side native work. No capacity or compatible destination leaves the retained snapshot owned until its configured deadline. The target must already hold that exact generation; Core does not automatically prepare historical generations on new nodes. During upgrades, retain eligible nodes and their generation providers until their checkpoint retention obligations end. -`Suspend` may report `SourceStopped` only after publishing a verified full archive durably, stopping the exact source compute and settling source-local capture and cleanup obligations. Core persists the snapshot before marking it suspended; native absence alone is insufficient. `Suspend.ObserveOnly` may finish archive publication and source cleanup for an already verified snapshot of the same operation, but cannot capture another snapshot or stop a source that has resumed running. If a previously published archive is missing or damaged, `ObserveOnly` may return its exact durable ownership receipt with `Status: unknown` and `SourceStopped: false` for cleanup; that receipt proves neither current archive usability nor source absence. `Resume` still verifies the archive before execution. After the source-settlement barrier, source-node unavailability does not prevent restoration. Deletion and ordinary expiry can transfer an idle suspended allocation's cleanup route to an online node in the same artifact domain; deletion does not require execution-class compatibility. The cleanup destination must already serve the same immutable generation and have retained capacity. Without such a node, cleanup stays pending and ownership is not released; operators must restore an eligible node or its capacity. An unknown running or restoring writer never moves merely because its node is unreachable. +For transferable checkpoints, `Suspend` may report `ResourcesReleased` and `SuspendSettled` only after publishing a verified full archive durably, stopping the exact source compute and settling source-local capture and cleanup obligations. Core persists the bound retained state before marking it suspended; native absence alone is insufficient. `Suspend.ReconcileOnly` may finish archive publication and source cleanup for an already verified snapshot of the same operation, but cannot capture another snapshot or stop a source that has resumed running. If a previously published archive is missing or damaged, `Suspend.ReconcileOnly` may return its exact durable ownership receipt with `Status: unknown` and `ResourcesReleased: false` for cleanup; that receipt proves neither current archive usability nor source absence. `Resume` still verifies the archive before execution. After the source-settlement barrier, source-node unavailability does not prevent restoration. Deletion and ordinary expiry can transfer an idle suspended allocation's cleanup route to an online node in the same artifact domain; deletion does not require execution-class compatibility. The cleanup destination must already serve the same immutable generation and have retained capacity. Without such a node, cleanup stays pending and ownership is not released; operators must restore an eligible node or its capacity. An unknown running or restoring writer never moves merely because its node is unreachable. -Recovery uses `Resume.ObserveOnly` for the persisted target and operation. `RestoreAttemptClosed` is an exact operation ID, not a generic absence flag: the adapter may return it only after durably closing an attempt that never entered native execution and fencing every late request for that ID. An adapter may also close a durably admitted attempt when it can prove that its current invocation failed before dispatching native restoration. The microsandbox adapter does this only when the fresh request's wire deadline expires before native dispatch; archive corruption, qualification errors and opaque SDK failures do not authorize an automatic retry. Core then persists a new operation ID before attempting execution. Once native restoration has been dispatched, an unknown outcome retains the same target, ownership and original retention deadline; a missing native listing, helper death or timeout does not authorize replay. Such a restore can remain unavailable until explicit deletion or ordinary retention expiry, and pending inputs can reach their public deadline while waiting. User deletion still requires exact native settlement before resources and ownership can be released. +Recovery uses `Resume.ReconcileOnly` for the persisted target and operation. `RestoreAttemptClosed` is an exact operation ID, not a generic absence flag: the adapter may return it only after durably closing an attempt that never entered native execution and fencing every late request for that ID. An adapter may also close a durably admitted attempt when it can prove that its current invocation failed before dispatching native restoration. The microsandbox adapter does this only when the fresh request's wire deadline expires before native dispatch; archive corruption, qualification errors and opaque SDK failures do not authorize an automatic retry. Core then persists a new operation ID before attempting execution. Once native restoration has been dispatched, an unknown outcome retains the same target, ownership and original retention deadline; a missing native listing, helper death or timeout does not authorize replay. Such a restore can remain unavailable until explicit deletion or ordinary retention expiry, and pending inputs can reach their public deadline while waiting. User deletion still requires exact native settlement before resources and ownership can be released. An in-place adapter may retain a nonempty native ID: the closure must equal the requested ID, logical generation, name and retained provenance. The closure proves that this restore attempt was never dispatched, not that its underlying paused source is absent. The microsandbox adapter requires an explicit private `checkpoint_root`, outside every guest-accessible filesystem. Its directory is mode `0700`; a private namespace marker identifies the actual archive store. The default installation creates a node-local private store. Cross-node restoration requires operators to mount the same private store on each participating node; mount paths may differ. Runtime state directories and workspace bindings are never used to infer this root. The adapter verifies the archive, external filesystem identity and native execution class again before restoration. Restored RAM and file descriptors do not promise continuity of external TCP connections or replay safety for application side effects. ### Cold replacement after checkpoint retention -Snapshot retention bounds compute resources, not the lifetime of an eligible retained Session. After the deadline, Core can clean up the allocation without terminating the Session only when its independent filesystem object is ready, initialization is complete, the Session and Environment remain nonterminal, and the previous authenticated Runtime has a persisted `retained_native_history` declaration. The capability and native recovery obligations are defined by the [Core–Runtime protocol](runtime-protocol.md). Filesystem retention alone is insufficient. +Retained-state retention bounds compute resources, not the lifetime of an eligible retained Session. After the deadline, Core can clean up the allocation without terminating the Session only when its independent filesystem object is ready, initialization is complete, the Session and Environment remain nonterminal, and the previous authenticated Runtime has a persisted `retained_native_history` declaration. The capability and native recovery obligations are defined by the [Core–Runtime protocol](runtime-protocol.md). Filesystem retention alone is insufficient. -Capacity pressure never shortens checkpoint retention. Until its configured deadline, a suspended allocation keeps its snapshot and retained slot; unavailable capacity or an incompatible node does not authorize cold replacement. Pending requests retain their original input deadline. After retention expires, the ordinary expiry cleanup must confirm resource deletion before another allocation can reserve that slot; files and qualified native history remain retained. +Core confirms the original creation is settled and all owned compute and retained states are cleaned up before releasing allocation and placement ownership. The old device's authority is revoked before replacement admission. Until Create, Kill or DeleteRetained has a confirmed outcome, it keeps ownership and admits no replacement writer. A recoverable Session and Environment remain disconnected with the same filesystem object and initialization result. Archive, reset and deletion take precedence and cannot be undone by a wake request. -Core confirms the original creation is settled and all owned compute and snapshots are cleaned up before releasing allocation and placement ownership. The old device's authority is revoked before replacement admission. Until Create, Kill or DeleteSnapshot has a confirmed outcome, it keeps ownership and admits no replacement writer. A recoverable Session and Environment remain disconnected with the same filesystem object and initialization result. Archive, reset and deletion take precedence and cannot be undone by a wake request. +Capacity pressure never shortens checkpoint retention. Until its configured deadline, a suspended allocation keeps its snapshot and retained slot; unavailable capacity or an incompatible node does not authorize cold replacement. Pending requests retain their original input deadline. After retention expires, the ordinary expiry cleanup must confirm resource deletion before another allocation can reserve that slot; files and qualified native history remain retained. New input or live Environment file access reserves a new allocation on a compatible node and bootstraps a fresh Runtime device. Preparation reuses the same filesystem and completed initialization rather than replaying initialization. Execution resumes the same native Session from retained history after validating the replacement Runtime's declaration. Missing or invalid native history fails explicitly; Core never substitutes a new native Session or replays completed input. History and published Artifact reads do not reserve compute. This path does not preserve VM memory, background processes or open connections, and it does not qualify an undeclared Runtime, Harness or filesystem combination. @@ -218,7 +236,7 @@ A [reset](../contracts/agents-api/sandbox-deployment.md#reset) is durable execut One snapshot and timestamp partition the held resources. Offline ownership comes from the allocation's or active placement's node, with the same 45-second connection and owner-epoch predicate as online presence, independently of provider readiness; no cleanup failure, offline state or empty read authorizes a synthetic release. At zero held resources, Core drains outside database transactions, rechecks under the deployment lock and clears the deployment in one transaction, then publishes a generation-bearing empty provider without fallible work. If the final write or drain fails, it restores the committed provider with a bounded owner context before releasing the mutation gate; if that recovery fails, admission stays fenced and the owner stops. -An administrator [Session archive](../contracts/agents-api/admin-api.md#session-archive) keeps its Project scope and hosted eligibility checks in a Session-first transaction, together with Environment expiry, cancellation, Runtime authority revocation and audit; a reset's background archive reconstructs the actual Project scope from trusted records and keeps the requester's provenance. The ordinary provider lifecycle releases compute and snapshots. Archive cancellation keeps a healthy receipt path until the terminal commit: only the archive that first revokes a device records its exact `archive_cancel_turn_id` (ordinary revocation clears it, and repeated cleanup keeps it), and the existing authenticated delivery may drain that cancellation for at most 20 seconds from the Turn's original `cancel_requested_at`. Core tracks the delivery through `done`, the cancellation acknowledgement and the terminal commit, independently of subscription removal, and grants no new connection, input, file or MCP authority or lease renewal. No transaction or lifecycle gate waits for the receipt, and a lost peer, expiry or restart falls back to ordinary failure and cleanup, never a fabricated cancelled outcome. +An administrator [Session archive](../contracts/agents-api/admin-api.md#session-archive) keeps its Project scope and hosted eligibility checks in a Session-first transaction, together with Environment expiry, cancellation, Runtime authority revocation and audit; a reset's background archive reconstructs the actual Project scope from trusted records and keeps the requester's provenance. The ordinary provider lifecycle releases compute and retained states. Archive cancellation keeps a healthy receipt path until the terminal commit: only the archive that first revokes a device records its exact `archive_cancel_turn_id` (ordinary revocation clears it, and repeated cleanup keeps it), and the existing authenticated delivery may drain that cancellation for at most 20 seconds from the Turn's original `cancel_requested_at`. Core tracks the delivery through `done`, the cancellation acknowledgement and the terminal commit, independently of subscription removal, and grants no new connection, input, file or MCP authority or lease renewal. No transaction or lifecycle gate waits for the receipt, and a lost peer, expiry or restart falls back to ordinary failure and cleanup, never a fabricated cancelled outcome. ## Validate the integration diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 7cf303515..b46b33e0a 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: 6bfcaa201c332aa1907f6ab7b0e0ffde4c8f635b363dbea2d0633947615e7264 +source_hash: 9b8fbf0b986794b5f56474668dfdee67fb39b28708c3ca0a0290b60a7dec496a --- Core 安装的每项设置都恰好只有一个归属位置，分属以下三类： @@ -54,6 +54,8 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 | `OAC_LOG_FORMAT` | `auto` | `auto`, `text` or `json` | | `OAC_LOG_ADD_SOURCE` | unset | `1` adds source locations | | `OAC_EXECUTION_CONCURRENCY` | `4` | Concurrent execution work, from 1 to 1024 | +| `OAC_SANDBOX_MAX_ACTIVE` | `100` | 启用暂停能力的直接 Provider 的活跃沙箱上限，范围 1–100000；独立于执行并发度和节点容量 | +| `OAC_SANDBOX_MAX_RETAINED` | `400` | 直接 Provider 的保留沙箱上限，范围 1–100000，包括活跃、已暂停以及尚未确认清理完成的分配；必须不小于活跃上限 | | `OAC_DEFAULT_HARNESS` | `codex` | Harness used when a request does not name one | | `OAC_HARNESSES` | Every registered Harness | Comma-separated Harnesses to enable besides the default one. Unknown names stop startup | | `OAC_WRITE_AUDIT_RETENTION` | `2160h` | Minimum `1h` | @@ -105,6 +107,10 @@ Runtime 在发现 Harness 适配器之前解析资源目录。`` 哪些 Harness 已启用以及默认 Harness 属于进程设置（`core.harnesses`、`core.default_harness`）；System 会以只读方式显示它们。[Core 管理 API](../../contracts/agents-api/zh/admin-api.md) 列出了所有 Core API 路由，[部署契约](../../contracts/agents-api/zh/sandbox-deployment.md) 定义了沙箱字段、限制和更改规则。 +### 直接 Provider 容量 {#direct-provider-capacity} + +在 `.env` 中设置 `OAC_SANDBOX_MAX_ACTIVE` 和 `OAC_SANDBOX_MAX_RETAINED`，然后运行 `oac apply`。Core 将这些限制用于启用暂停能力的直接 Provider。每个尚未释放的分配都占用保留容量。降低上限不会终止已有沙箱；新分配会等待使用量低于两项上限。这些设置独立于 `OAC_EXECUTION_CONCURRENCY` 和已注册节点的容量。 + ### 独立工作区存储 {#independent-workspace-storage} 首个支持的独立文件系统组合是 microsandbox 与[内核 NFS 适配器](./workspace-provider.md#kernel-nfs-adapter)。新安装应先准备存储和服务账户，再向 Core 暴露挂载、选择存储、配置 microsandbox，最后注册节点。启动服务前，在 Linux Core 主机和所有参与节点上将同一个 NFSv4.2 导出挂载到相同的绝对路径，例如 `/srv/oac-workspaces`。运维人员负责导出、挂载可用性和服务启动顺序。使用带有 `root_squash` 的受信任客户端 AUTH_SYS 导出；不要启用 `no_root_squash` 或放宽权限来使检查通过。按照适配器的[所有权要求](./workspace-provider.md#kernel-nfs-adapter)准备命名空间和服务身份。 @@ -211,7 +217,7 @@ Core 读取进程环境。Compose 将 `.env` 插值到环境中，并把机密 | `OAC_CREDENTIAL_KEY_FILE` | 必填。`/run/oac/credential.key`：Base64 编码的 32 字节随机密钥。Core 用它加密存储的凭据 | | `OAC_CORE_KEY_DIGESTS_FILE` | 必填。`/run/oac/core-key-digests.json`：一个包含 Core 密钥 SHA-256 的 JSON 数组 | | `OAC_INSTALLATION_ID_FILE` | 必填。`/run/oac/installation.id`：安装 ID，采用规范 UUID 格式。如果 ID 与数据库记录的 ID 不一致，Core 会拒绝它 | -| `OAC_EXECUTION_CONCURRENCY`、`OAC_DEFAULT_HARNESS`、`OAC_HARNESSES`、`OAC_WRITE_AUDIT_RETENTION`、`OAC_OAUTH_TRUSTED_ORIGINS`、`OAC_HISTORY_SETTINGS_FILE`、`OAC_LOG_LEVEL`、`OAC_LOG_FORMAT`、`OAC_LOG_ADD_SOURCE` | 对应的[进程设置](#settings)。Web 也读取三个日志设置 | +| `OAC_SANDBOX_MAX_ACTIVE`、`OAC_SANDBOX_MAX_RETAINED`、`OAC_EXECUTION_CONCURRENCY`、`OAC_DEFAULT_HARNESS`、`OAC_HARNESSES`、`OAC_WRITE_AUDIT_RETENTION`、`OAC_OAUTH_TRUSTED_ORIGINS`、`OAC_HISTORY_SETTINGS_FILE`、`OAC_LOG_LEVEL`、`OAC_LOG_FORMAT`、`OAC_LOG_ADD_SOURCE` | 对应的[进程设置](#settings)。Web 也读取三个日志设置 | | `OAC_PROVIDER_ROOT` | 适配器构件的绝对根目录。Core 镜像设置为 `/opt/oac`。每个适配器都拥有此根目录下的辅助路径。当其中的 `native-installers/` 目录包含 `catalog.json` 时，Core 在核对该目录清单与自身发行版后提供自托管守护进程安装程序。适配器状态位于 `/state`，即数据卷的 [`state/`](#compose-installations) | Core 会记录所加载的历史文件路径，但绝不记录环境变量的值或文件内容。 diff --git a/docs/zh/getting-started/operations.md b/docs/zh/getting-started/operations.md index 4316ac34d..e67413ba6 100644 --- a/docs/zh/getting-started/operations.md +++ b/docs/zh/getting-started/operations.md @@ -1,7 +1,7 @@ --- title: "运维" source: docs/getting-started/operations.md -source_hash: 591196d61ed9946435d07afcbbcc2f53d64b9db0cf6b4806ddfb0f85a16974ba +source_hash: 17c585589882683cb80bacd2513b372ff7c0164a3395aeb917f486d57d94db5b --- 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。 @@ -26,6 +26,9 @@ docker compose -f ~/.oac/core/compose.yaml ps 示例使用默认安装目录。Windows 上使用 `& "$HOME/.oac/core/oac.exe"` 调用管理命令，后接相同参数。使用自定义安装目录时，替换各命令中的路径。 +本地凭据和注册解析完成后，Runtime 的 Harness 发现与已认证的引导 HTTP 请求并发执行。两者都成功后才建立连接并发布能力。失败会取消另一项操作并等待其清理。执行器仍由连接生命周期持有。Runtime 启动变更需要重新构建并验证 Runtime 模板。 + + ## 服务健康状态 {#service-health} 根据不同问题使用这些观察： diff --git a/docs/zh/runtime-bootstrap.md b/docs/zh/runtime-bootstrap.md index d3338fa61..aeecb495e 100644 --- a/docs/zh/runtime-bootstrap.md +++ b/docs/zh/runtime-bootstrap.md @@ -1,7 +1,7 @@ --- title: "Runtime 引导" source: docs/runtime-bootstrap.md -source_hash: f7e9275feee79ac1fb1299227e148b85c26efc8b8af5ca0e8f498126c563d182 +source_hash: 537197f388a57bb983c0e7e0b8a3bfe66b1e8e0ef7f4f1fef572ad7dcf724cb3 --- Sandbox Provider 通过交付一个引导文件来启动托管 Runtime。本文负责 Provider 到 Runtime 的启动输入。类型与验证器位于 [`internal/runtimebootstrap`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/runtimebootstrap/bootstrap.go)；Go provider 使用 [`runtime_bootstrap.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/runtime_bootstrap.go) 中的 `sandbox.Bootstrap.RuntimeConnection()` 构造输入，SDK helper 原样转发序列化对象。provider 不读取或写入 Runtime 的私有认证存储。 @@ -19,6 +19,7 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json | `version` | 精确的引导版本 `runtimebootstrap.Version` | | `core_url` | 以 `/api/v1` 结尾的 HTTP(S) 机器 API 基址，不含凭据、查询或片段 | | `device_id` | Core 签发的 daemon 身份的规范非零 UUID | +| `harness` | 会话固定选择的 Harness 标识；托管 Runtime 只探测并注册此 Harness | | `credential` | Core 签发的非空 daemon 凭据，不含空白或 NUL | 解码器拒绝未知、重复、缺失和大小写别名字段，拒绝其他版本及超过 `runtimebootstrap.MaxBytes`（16 KiB）的文档。错误不包含提交的值。文件缺失或格式错误时，daemon 在连接前失败。 @@ -29,10 +30,16 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json provider 创建账户、挂载和工作区，交付该文件，设置 Runtime 的资源与 Environment 绑定配置，然后以无特权 Runtime 账户启动 daemon。Docker 将文件写入 Runtime 拥有的 home volume；microsandbox 和 E2B 在启动同一命令之前交付文件。 +Core 从分配所属会话派生 `harness`。组合镜像可以包含多个 Harness，但托管 Runtime 只探测所选 Harness；未知、缺失或不可用的选择会失败，不探测其他 Harness。启动协议版本为 2，Core、提供商辅助程序和新启动的 Runtime 镜像需配套升级；已有分配保留原启动文件和 Runtime。自托管安装仍探测已安装的 Harness 集合。 + Runtime 验证输入，并负责认证与连接。启动成功仅证明交付完成：经过认证的连接、已准备的能力和执行就绪是 [Core–Runtime 协议](runtime-protocol.md) 下的独立观测；[Sandbox Provider 指南](sandbox-provider.md#four-distinct-readiness-facts) 列出各自证明的事实。 自托管 executor 和运维人员供应的设备通过其他方式获取 daemon 身份；[机器连接 API](../../contracts/agents-api/zh/machine-api.md#credentials) 列出所有凭据来源。它们都进入同一 Runtime 执行循环。 +## 托管暂停控制 {#hosted-suspension-control} + +私有托管暂停/唤醒控制文件的路径在 `internal/runtimebootstrap/bootstrap.go` 中以 `SuspendControlFile` 定义。Core 恢复逻辑和原生 Go adapter 读取该值；E2B helper 契约生成器将其投射到模板构建器。托管启动准备私有目录并提供 `OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE`，以启用 Runtime 暂停。这是随软件包发布的协议设置。共享 Sandbox Provider 注册负责空闲和保留默认值；adapter 负责自身原生租约超时。 + ## 验证 {#verification} `go test ./internal/runtimebootstrap ./apps/daemon/internal/cli` 覆盖输入契约、凭据来源互斥规则和重启行为。Provider 测试验证交付与文件权限，不依赖 Runtime 的私有存储。 diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index ff6501786..462fc10f6 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: 63ad2e39e7536caee9bd43f9b643073d32b32147f1440d188d6d38752628adc7 +source_hash: fc14797b2af53ed0e2bd64828c83c19e711e5bf30bb1cdfa24574be522b1c13c --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -27,7 +27,7 @@ Core 拥有持久 Environment、allocation、placement 和 cleanup 状态；Prov ## 实现接口 {#implement-the-interface} -`sandbox_provider.go` 包含 Core–Sandbox Provider 协议：负责 allocation、checkpoint 和观测的 `SandboxProvider` 接口及其请求与结果类型，以及 setup 阶段的 `ConfigurationAdapter` 及其类型化错误。Core 在此边界之外也使用的值类型，例如 `DeploymentSpec` 和 `CallFence`，位于同一 package 的独立文件中。每个方法在编译期都必须实现，`ProviderOperations()` 声明 Provider 支持哪些方法。以下五项操作始终支持： +`sandbox_provider.go` 包含 Core–Sandbox Provider 协议：负责 allocation、暂停和观测的 `SandboxProvider` 接口及其请求与结果类型，以及 setup 阶段的 `ConfigurationAdapter` 及其类型化错误。Core 在此边界之外也使用的值类型，例如 `DeploymentSpec` 和 `CallFence`，位于同一 package 的独立文件中。每个方法在编译期都必须实现，`ProviderOperations()` 声明 Provider 支持哪些方法。以下五项操作始终支持： | 操作 | 用途 | | --- | --- | @@ -47,7 +47,7 @@ Docker 等没有原生可续期租约的 backend 仍遵守 Core 的 hosted expir | --- | --- | --- | | `Create`、`GetInfo`、`Renew`、`Kill`、`RunCommand` | 支持 | Allocation 生命周期与有界命令 | | `Observe` | 明确决定 | 检查所有权、只读地观测一个 allocation | -| `Initial`、`NewCompute`、`GetCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute` | 所有 checkpoint 方法决定一致；仅 `nodes` 注册可以支持 | 精确计算实例、捕获与恢复、保留源恢复和清理 | +| `Initial`、`NewCompute`、`GetCompute`、`RenewCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteRetained`、`RunCommandCompute` | 全部十项暂停方法决定一致；`nodes` 与 `direct` 模式均可支持 | 精确计算实例、捕获与恢复、保留源恢复和清理 | `Initial` 和 `NewCompute` 构造 compute reference，不分配资源；`ResumeCompute` 在暂停中止后仅解冻同一驻留实例；`RunCommandCompute` 在一个精确 compute incarnation 中运行有界命令。Core 使用 `RunCommandCompute` 在恢复后唤醒 parked daemon（[`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)）。 @@ -91,7 +91,7 @@ Microsandbox 在返回创建不存在回执前永久关闭初始 Create 准入 `ErrInvalid`、`ErrOwnership`、`ErrExists`、`ErrNotFound`、`ErrComputeUnconfirmed` 和 `ErrCommandUnconfirmed` 保持其定义含义。未分类原生或 transport error 表示未知，不授权重试 mutation。Core 不将 provider diagnostics 读作生命周期事实，也不暴露原生错误文本或凭据；node transport 将错误映射为固定 code，直接 SDK 细节保持私有。 -Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity`；原样持久化 operation ID 与 provider snapshot provenance。suspend 或 resume 的 `ObserveOnly` 仅观察上次尝试，不启动另一 capture 或 restore。`ResumeCompute` 仅解冻保留源，不冷启动已停止源。清理针对精确 compute incarnation 和 snapshot，不针对当前同名实例。声明 checkpoint 支持前阅读 [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) 及其失败测试。 +暂停支持增加精确的 `Compute` 代次、名称和 ID，以及不透明的 `RetainedState` 收据。原样持久化操作 ID 和 Provider 保留状态的来源。暂停或恢复中的 `ReconcileOnly` 观察原始尝试，不会开始另一次捕获或恢复。`ResumeCompute` 仅解冻保留的源实例，绝不冷启动已停止的源实例。清理针对精确计算实例和保留状态，而非当前恰好同名的实例。声明支持前请阅读 [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) 及其失败测试。 ### 四个独立就绪事实 {#four-distinct-readiness-facts} @@ -114,7 +114,7 @@ Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity` 2. 添加 specification 和 resource validator。 3. 基于类型化原生配置实现 `sandbox.ConfigurationAdapter`。`DecodeInput` 严格解析请求中独立的公开 `configuration` 与只写 `credential` 对象。`Encode` 生成白名单公开 selector、只读观测和独立 secret bytes，不透传请求 JSON。`Decode` 恢复已存储 selector 并保留对所属资源的访问，不做远程 admission 或新模板验证。`Normalize` 修改前复制输入。`ResolveChange`、`Equal` 和 `WithCredential` 负责继承、身份与凭据组合。`Requirements` 声明是否需要凭据和公开 Core origin，以及支持哪些 setup 操作：`Discovery` 对应 `DiscoverConfiguration`，`SelectionDiscovery` 对应 `DiscoverSelection`，`CredentialVerification` 对应 `VerifyCredential`。`DiscoverConfiguration` 验证 query 并返回安全 catalog，不做 mutation 或 admission decision；Core 保留授权、输入限制与 deadline。`DiscoverSelection` 在提交前解析候选项省略的原生值，`VerifyCredential` 验证凭据对所属资源的访问，不修改资源。两者都接收候选项的 `sandbox.DirectConfig`，原生 client 只为该次调用构造。node provider 仅接受空公开对象，拒绝凭据，对每项 setup 操作和 credential replacement 返回 Unsupported。 4. node adapter 从自己的包中导出 `BuildLocal` constructor、它解码的类型化 `native` 对象，以及它在共享 node artifact 之外添加的原生文件。node-local 设置（如主机路径）只存放在该对象中；resources 和 Runtime release 从 node 配置的 `specification` 读取。 -5. 在 `providers/registry.go` 中注册 constructor、policy、configuration adapter 和 operation 声明。其键即 provider kind，也用于标记该 Provider 的观测；checkpoint 支持读取此项。生成的投影组合每个已注册的部署模式和 deployment policy 与 `sandbox/deployment_contract.go` 中的共享 field bound：installer 从 `deploy/node/node_spec.py` 读取，TypeScript 客户端和 Web 从 `packages/agents-client/src/deployment-contract.ts` 读取，因此 Web 读取这些声明，而不比较 provider kind。通过 `go run ./services/core/cmd/specification-contract -write` 重新生成两者。 +5. 在 `providers/registry.go` 中注册 constructor、policy、configuration adapter 和 operation 声明。其键即 provider kind，也用于标记该 Provider 的观测；暂停支持读取此项。生成的投影组合每个已注册的部署模式和 deployment policy 与 `sandbox/deployment_contract.go` 中的共享 field bound：installer 从 `deploy/node/node_spec.py` 读取，TypeScript 客户端和 Web 从 `packages/agents-client/src/deployment-contract.ts` 读取，因此 Web 读取这些声明，而不比较 provider kind。通过 `go run ./services/core/cmd/specification-contract -write` 重新生成两者。 6. 提供 adapter 和 helper 的发行产物，通过已注册 configuration 契约向运维人员提供 provider。 **已知设计缺口：** Web 仍在 setup 向导的后端选择、Docker 确认，以及所有显示或解析 E2B 配置字段的地方（带服务预设的 setup 步骤、部署摘要和客户端的部署投影）中指名 provider，因为协议尚未声明配置字段。通过该界面提供另一 provider 目前需要修改共享的 Web。此耦合不符合[复杂性留在 adapter 内](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#complexity-stays-in-the-adapter)；新集成必须通过协议表达配置，把厂商专有行为留在 adapter。不得添加 Session 或 Turn 调度路径、厂商专有 column 或 API field，或 store 中的厂商 switch。 @@ -128,7 +128,7 @@ node 配置 `sandbox.NodeConfig` 包含 `provider`、`generation`、`installatio - `nodes` 注册仅有 `BuildLocal`，`direct` 注册仅有 `BuildDirect`；缺失、混合或未知 mode 被拒绝。 - specification 和 resource validator、configuration adapter 与完整 operation 声明都是必需项，因此不完整注册不能发布部分投影。声明的默认大小必须通过 adapter 的 resource validator。 - Runtime input policy 要么接受固定 Runtime，要么给出 adapter 拒绝它的固定原因，不能两者兼有。 -- Checkpoint 仅准入 `nodes` 注册，因为公共 lifecycle 只暂停 node allocation；声明 checkpoint 的 `direct` Provider 会被注册拒绝。注册不携带 suspension 数值：Core 对每个声明 checkpoint 支持的 Provider 应用同一个 [suspension policy](#suspension)。 +- 完整生命周期声明支持时，`nodes` 与 `direct` 模式都准入暂停。注册不携带 suspension 数值；Core 对所有支持的 Provider 应用同一个 [暂停策略](#suspension)。 configuration adapter 必须非 nil，包括其具体值。每个 `ConfigurationRequirements` 字段都需要明确有效的决定：`Credential` 和 `PublicOrigin` 为 `Required` 或 `NotRequired`，`Discovery`、`SelectionDiscovery` 和 `CredentialVerification` 使用共享 supported 或 unsupported 声明并携带安全 reason。新增 requirement field 需要明确更新验证，不继承已有决定。要求凭据不承诺支持 `VerifyCredential` 操作。这些检查证明注册完整，不证明原生 SDK 行为正确；constructor 和 adapter 契约测试仍然适用。 @@ -138,7 +138,7 @@ configuration adapter 必须非 nil，包括其具体值。每个 `Configuration 具有凭据的 direct adapter 在替换 key 前验证全部保留 generation 与 allocation reference。公共 `sandbox.CallFence` 排除原生调用并等待 helper 完成，包括调用方已超时的调用；execution 调用已准备的 verification 和 fencing callback，不按厂商分支。 -厂商部署验证和 SDK setup 留在构造边界，构造不创建 Environment。node-local adapter 的 `sandbox.Built` 返回 provider、probe、installation identity、backend fingerprint 和 specification digest；helper 可能比调用方存活更久时，还返回 `Quiescent` 检查，generation 回收会等待它。factory 还返回 close 函数。`execution.RuntimeProvider` 将 adapter 绑定到 kind、installation ID、backend fingerprint、generation、mode 和 node ownership；选择由数据库负责，内存副本不构成另一权限来源。Docker 与 microsandbox 在 node 上运行，E2B 直接构造。node proxy 仅对注册声明支持 checkpoint 的 backend 暴露 checkpoint 操作，公共 lifecycle 通过 checkpoint 声明准入 suspension，不通过 provider name。 +厂商部署验证和 SDK setup 留在构造边界，构造不创建 Environment。node-local adapter 的 `sandbox.Built` 返回 provider、probe、installation identity、backend fingerprint 和 specification digest；helper 可能比调用方存活更久时，还返回 `Quiescent` 检查，generation 回收会等待它。factory 还返回 close 函数。`execution.RuntimeProvider` 将 adapter 绑定到 kind、installation ID、backend fingerprint、generation、mode 和 node ownership；选择由数据库负责，内存副本不构成另一权限来源。Docker 与 microsandbox 在 node 上运行，E2B 直接构造。node proxy 仅对注册声明支持 暂停的 backend 暴露暂停操作，公共 lifecycle 通过暂停声明准入 suspension，不通过 provider name。 backend fingerprint 标识原生资源命名空间，不表示容量。Core 保留部署 generation，使所属 allocation 继续解析到原 backend；不要将保留 allocation 重新指向替代 backend。 @@ -162,16 +162,20 @@ Node readiness 绑定到精确 generation、当前连接和 owner epoch。持久 ### Allocation 生命周期 {#allocation-lifecycle} +Core 在 `Bootstrap.Harness` 中传递所属会话固定选择的 Harness；所有提供商都将其投影到 [Runtime 启动输入](./runtime-bootstrap.md)，不自行选择实现。 + allocation、专用 daemon credential digest 和精确 Session binding 在 `Create` 前、execution lease 与 Session lock 下原子提交。只有新 allocation receipt 允许 `Create`；重试和 Core 重启观察同一 reference，不重放或轮换凭据。allocation 是私有计算资源所有权，与公开 Environment connection 和原生 readiness 独立；adapter 验证 bootstrap completion，Core 不从 engine 或 provider name 推断。 配置 provider 后，Worker 扫描已提交且没有 allocation 的 pending hosted Environment，涵盖空闲 Session 创建以及 commit 与 bootstrap 之间中断后的恢复；已有 allocation 不重新进入此路径。scan 有界，由 lifecycle owner 串行化，不需要调用方操作。没有 Turn 的初始预约让 Session 保持空闲，daemon 连接不被当作原生 readiness。同一 scan 在验证精确 Session、device binding 和已结算 bootstrap 后，发布带持久 generation 的认证连接观测。 -Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 正在运行的 allocation；该检查不做任何修改，也不复活 cleanup 请求。正在运行的计算资源不会到期：显式生命周期清理和快照保留策略管理资源回收。停止或缺失 container 不授权丢弃保留工作区或历史，也不授权选择替代计算资源。禁用 provider 停止新 hosted admission 与 bootstrap，但不阻止现有 Session 的取消、function result 或 input retry outcome。 +Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 正在运行的 allocation；该检查不做任何修改，也不复活 cleanup 请求。正在运行的计算资源不会到期：显式生命周期清理和保留状态期限策略管理资源回收。停止或缺失 container 不授权丢弃保留工作区或历史，也不授权选择替代计算资源。禁用 provider 停止新 hosted admission 与 bootstrap，但不阻止现有 Session 的取消、function result 或 input retry outcome。 终结清理原子撤销 device authority、记录 Environment 失败或到期、结算 pending input 并请求取消，然后才调用 `Kill`；原 input deadline 与 retry outcome 保留。临时 provider outage、未知 Create result 和停止的计算资源不证明永久失败。公开 Session 删除后 Core 保留 allocation，仅在所属 compute 与 volume 清理完成且原 Create 已结算的证明成立后标记 released；未知创建即使观察到不存在也保留 cleanup ownership，有界 scan 继续捕捉延迟资源，不再调用 `Create`。 ### 每节点生命周期 worker {#per-node-lifecycle-workers} +托管 Environment 提交后，Core 会向其放置节点的生命周期 worker 发送有界提示。已提交的待处理输入也会发送提示，以恢复遗漏的创建通知。提示复用现有串行门控、执行租约、容量检查和一次性分配凭据，不会在准入处理器内直接创建沙箱。每个正常维护周期最多允许一次额外提示扫描；周期扫描负责恢复遗漏或合并的提示。已有分配的观察仍先于待创建资源处理，因此同一节点上的慢观察仍可能延迟新 Environment。 + 每个注册 node 有一个串行 lifecycle worker，负责 gate、allocation 与 pending cursor、connection 和 wake hint；E2B allocation 共享一个没有 node 的串行 lifecycle。薄 coordinator 发现 node 并关闭 worker，数据库、provider 或等待操作期间不持有 map mutex。worker 独立推进，因此一个在线 node 的 provider 卡住不会阻塞其他 node：lifecycle 并发为每 node 一项操作，随 node 数量增长。离线 worker 保留，因此其保留资源在重连后仍可观察。 allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan join 尚未释放的已提交 placement。每个 node 推进自己的 cursor，包括越过失败观测，并在末尾回绕一次。direct provisioning 在进入该 node gate 前解析 tenant 范围内 placement，活动 allocation 必须与其一致。只有确认释放后才能建立新的 placement；断连不会迁移活动 allocation。 @@ -182,35 +186,53 @@ allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan jo placement 自动完成：Session 创建提交持久的 pending 工作，共同调度器在 allocation 之前预留兼容节点。[部署契约](../../contracts/agents-api/zh/sandbox-deployment.md#generation-ownership-and-rollout) 定义等待、排序与代次选择。调用方不能选择 node；活动 allocation 即使在 node 离线时也保留原 node。确认 allocation 释放后，符合条件的保留 Session 可以重新预约兼容容量，包括其他 node。Node capacity 计入 pending reservation 和未决资源，新 placement 与 suspended-to-restoring 转移共享数据库锁。未知操作保留预约，source teardown 必须确认后才能释放 active capacity，确认 cleanup 后释放 placement capacity。保留所有权需要精确 provider evidence：socket path、缺失 instance 或空列表都不证明 cleanup，也不授权 replacement。 -部署的 CPU、memory、disk 设置、`max_active`、`max_retained` 和 snapshot retention 限制每个 node。确认释放后的冷替换不提供 node-level drain、不可达写入方的故障转移、并发写入方、multi-active Core、autoscaling 或 snapshot replication。node 持有 allocation、snapshot、reservation、unknown result 或 cleanup 时拒绝移除 node，离线 ownership 保留。 +部署的 CPU、memory、disk 设置、`max_active`、`max_retained` 和 retained-state retention 限制每个 node。确认释放后的冷替换不提供 node-level drain、不可达写入方的故障转移、并发写入方、multi-active Core、autoscaling 或 snapshot replication。node 持有 allocation、保留状态、reservation、unknown result 或 cleanup 时拒绝移除 node，离线 ownership 保留。 ### 暂停 {#suspension} -Core 用同一个共享 policy 暂停每个声明 checkpoint 支持的 provider 的空闲工作：通常在工作空闲 5 分钟（300 秒）后暂停，snapshot 保留 24 小时（86400 秒）。部署的 [`suspension`](../../contracts/agents-api/zh/sandbox-deployment.md#safe-response) 报告这两个值。Core 在 initialization 完成、没有 root 或 Subagent Turn 排队、进行中或等待、没有 pending input 或 file operation，且真实 activity 已空闲达到该时间后暂停。Session 无需已运行过 Turn。idle clock 不早于 allocation 进入 running compute phase 的时刻开始，包括 wake 后重新进入该 phase。对于 node allocation，Core 在同一事务中用数据库时钟记录首个 root 或 child terminal transition。candidate filter 和 Session-locked recheck 比较数据库已过时间与 idle duration，初始 snapshot retention deadline 也锚定同一数据库观测，因此 Core 与数据库主机时钟无需一致。原生 completion timestamp 在公开历史中保持不变，但不驱动 idle admission，heartbeat 不重置 activity。确认计划暂停前，daemon 关闭 admission 并排空 native cleanup、output receipt 和 file work。 +单一 `SandboxProvider` 一并声明 `Initial`、`NewCompute`、`GetCompute`、`RenewCompute`、`Suspend`、`Resume`、`KillCompute`、`DeleteRetained`、`RunCommandCompute` 和 `ResumeCompute`：十项全部支持或全部不支持。生命周期不通过可选暂停接口或 Provider 名称分支选择。 + +`RetainedState` 是 adapter 拥有的不透明信封，包含 `Reference`、`ID`、`Data`、`OperationID`、`SourceGeneration`、`SourceName`、`SourceID` 和可选的 `Compatibility`。`Data` 必须非空且不超过 64 KiB。Core 原样保留信封并校验操作与源实例归属；不可变的 allocation Provider/代次绑定决定唯一有权解释它的 adapter。保留状态不必是独立快照：microsandbox 将完整原生快照身份放入 `Data`，E2B 保留带版本的原生暂停收据，不承诺独立磁盘镜像。adapter 在任何副作用前校验原生内容及归属。 + +`Compute.RestoredFrom`、`ComputeState.Retained`、`ResourcesReleased`、`SuspendSettled` 和恢复的 `ReconcileOnly` 是结算证据。资源缺失不证明未知暂停已结束。`RenewCompute` 只续租精确实例；不确定结果不授权第二次创建、捕获或恢复。持久 `runtime_compute` 使用 `protocol_version: "1"`；启动拒绝缺失或未知版本。仅匹配当前 retained-state 字段形状的版本 1 信封可直接读取，无需改写。旧 snapshot-only 形状即使标记相同的版本 1 也会被拒绝；数字相同不证明互通。升级不做迁移、合成重放或丢弃保留状态。 + +`ResumeRequest.Workspace` 携带 allocation 独立拥有的文件系统绑定；Core 在恢复前获取 ready 绑定。源实例必须已释放活跃执行能力，目标才可成为写者。adapter 在原生副作用前校验绑定；不支持独立文件系统的 adapter 拒绝非空绑定。`DeleteRetained` 仅删除 adapter 的计算保留状态，绝不删除 workspace。显式 Session 删除先完成 compute 清理，再删除独立存储，恢复期间保持单一活跃写者。 + +Core 对所有声明暂停支持的 Provider 应用统一共享策略：通常在工作空闲 5 分钟（300 秒）后暂停，保留状态期限为 24 小时（86400 秒）；部署的 [`suspension`](../../contracts/agents-api/zh/sandbox-deployment.md#safe-response) 返回这些值。Core 暂停已经初始化的 Environment，包括尚未执行 Turn 的 Environment，前提是没有 root 或 Subagent Turn 排队、执行或等待，没有输入、文件操作或初始化待处理，且真实活动已空闲达到该时长。空闲时钟不早于 allocation 进入 running compute phase 的时刻，包括唤醒后。Core 在同一事务中使用数据库时钟记录 allocation 的初始化完成和 root 或 child 终结转换。候选筛选和持有 Session 锁的复查比较数据库经过时间与空闲时长，初始保留状态期限也锚定同一数据库观测，因此 Core 与数据库主机的时钟无需一致。公开历史中的原生完成时间戳保持不变，但不驱动空闲准入，心跳也不重置活动时间。确认计划暂停前，daemon 关闭准入并排空原生清理、输出收据和文件工作。 容量压力可以让已经空闲的 node allocation 在通常超时之前暂停，但仍需经过 15 秒无活动宽限期。Session 锁内的决策重新检查 pending work、wake request 和 activity，再获取共享 deployment 容量锁并确认兼容的等待需求。它不会中断活动 Turn，也不会在普通 quiesce、capture 和 source-stop 证明完成前释放容量。压力触发的回收采用保守策略：只有进行中回收所属的节点当前有资格服务同一需求时，它才会推迟另一次回收。不可用或不兼容的节点保留其 ownership receipt，但不会阻挡健康节点的容量。如果暂停无法使容量可用，等待请求仍保留原始期限。 -Worker lease、Session lock 与 per-node gate 对每个 provider 负责 suspension。新 Turn claim、file-write intent 和 capture admission 在 Session lock 下串行化，共享一个 compute-phase 检查；新 pending work 取消 capture 并唤醒同一 source。正常 preparation 在经过认证的 resume handshake 后等待 compute phase 为 running；pending input 的 promotion 与 lifecycle transition 冲突时保持 pending。compute phase 和 revision-checked receipt 位于 allocation。Core 在 effect 前持久化 quiesce、capture 和 restore intent，仅新 receipt 执行 capture 或 restore，恢复观察精确 attempt，不重试未知 creation、capture 或 restore。已消费 snapshot 不让 running generation 回滚。删除、撤销和 retention expiry 优先于 wake，一直持续到最终数据库 compare-and-swap；未知 cleanup identity 保留，直到确认所属资源不存在。已消费 artifact 和旧 compute 被删除，因此暂停循环不累积可写磁盘链。 +Worker 租约、Session 锁和每个 node 的 gate 负责所有 Provider 的暂停。新 Turn claim、文件写入意图和捕获准入在 Session 锁下串行化，共享计算阶段检查；新待处理工作取消捕获并唤醒同一源实例。正常准备在经过认证的恢复握手后等待计算阶段变为 running；待处理输入的提升与生命周期转换冲突时，输入保持 pending。计算阶段和经 revision 校验的收据存储在 allocation 中。Core 在副作用前持久化静止、捕获和恢复意图，仅新收据执行捕获或恢复；恢复流程观察精确尝试，不重试结果未知的创建、捕获或恢复。已消费的保留状态不会使运行中的代次回滚。删除、撤销和保留期到期始终优先于唤醒，直至最后的数据库 compare-and-swap；未知清理身份会一直保留，直到确认所属资源已不存在。已消费产物和旧计算实例会被删除，因此暂停循环不会累积可写磁盘链。 + +排队工作和实时 Environment 文件访问会唤醒暂停的 Environment；历史和已发布 Artifact 的读取不会唤醒。实时文件请求在进入文件工作队列前，等待当前 Runtime 通过 credential 授权的连接及 Harness 声明；compute Create 完成并不代表 Runtime 已就绪。direct 保留状态通过 allocation 对应的 adapter 恢复；node 检查点遵循[检查点转移](#checkpoint-transfer)中的兼容性与 placement 规则。计划暂停在 daemon 连接上使用 Environment 与 suspension token。由 PID 和启动时间隔离的本地控制信号（`RunCommandCompute`）唤醒 parked daemon，daemon 在准入工作前重新认证。确认前的临时断连通过有界尝试和退避重试同一已准备好的暂停；永久认证或协议拒绝会将其关闭。Core 负责保留状态的到期期限，daemon 没有相应定时器。静止确认丢失时可以通过明确回滚解冻同一源实例，但不授权捕获。 -排队工作和实时 Environment file access 唤醒 suspended Environment；history 和已发布 Artifact read 不唤醒。实时文件请求在进入文件工作队列前，等待当前 Runtime 通过 credential 授权的连接及 Harness 声明；compute Create 完成并不代表 Runtime 已就绪。计划暂停在 daemon 连接上使用 Environment 和 suspension token。受 PID 与 start-time fencing 的本地 control signal（`RunCommandCompute`）唤醒 parked daemon，daemon 在准入工作前重新认证。确认前临时断连通过有界 attempt 和 backoff 重试同一已 armed suspension；永久认证或协议拒绝则关闭。保留的检查点在原 node 仍具资格时优先恢复，也可以按以下检查点转移规则在其他符合条件的 node 恢复。Core 负责 snapshot retention deadline，daemon 没有相应 timer。quiesce 确认丢失时可以通过明确 rollback 解冻同一 source，但不授权 capture。 +`Suspend` 负责原生资源释放，返回绑定的保留句柄、suspended 状态、`ResourcesReleased` 和 `SuspendSettled` 后，Core 才释放活跃容量。`ReconcileOnly` 禁止重放原始捕获或暂停，但允许完成由持久保留产物证明安全的 adapter 清理。无保留状态的结果只有在带有 `SuspendSettled` 且源实例处于可恢复的运行或暂停状态时才允许回滚。其他所有不确定结果均保留所有权并关闭准入。Core 从不在 `Suspend` 后无条件销毁源实例。 没有保留状态时，`SuspendSettled` 还证明已持久关闭该精确源实例与操作的所有迟到原生派发。进程内调用栅栏、分配锁或当前原生资源不存在，都不能单独证明这一点。E2B 与 microsandbox 均维护有界的 adapter 私有准入日志；回滚不能清除同代次关闭记录，只有已验证的更高源代次可以推进栅栏。 + +`Resume` 将保留状态恰好消费一次并恢复到预先提交的目标。恢复逻辑观察同一次尝试。Core 持久化 `waking`、认证并恢复 daemon、删除已消费的保留资源，然后提交 `running` 并准入工作。`DeleteRetained` 是幂等产物清理，会保留运行中的计算资源。清理失败会保持 `waking` 阶段，不能触发再次恢复。`KillCompute` 仍然是破坏性操作；清理旧代次时不得终止共享原生 ID 的较新活跃实例。 + +现有 Session 锁、生命周期租约、空闲规则、容量查询和清理顺序继续作为权威。每个尚未释放的分配都占用 `max_retained`，包括运行中的分配。每个分配在预留时都写入共享计算协议版本，包括暂停阶段为 `disabled` 的分配。激活会拒绝任何协议版本缺失或不同的未释放分配；升级前必须由旧版本完成普通清理。Session 历史保留。 + +### 协调协议升级 {#coordinated-protocol-upgrade} + +Core 与 node 一同升级到 node wire version 8；E2B helper 使用 private wire version 4，并重建包含托管暂停控制的模板；microsandbox helper 使用 private wire version 5。所有 Provider 交付 Runtime bootstrap version 2。每个边界独立验证自身契约，不同边界的版本号不能互换。激活前，使用此前兼容的版本对不兼容的未释放 allocation 完成正常清理。启动栅栏保留其持久收据及 Session 历史，不通过 Provider 调用迁移它们。 ### 检查点转移 {#checkpoint-transfer} -支持检查点的 generation 报告 `CheckpointCompatibility`，包含不透明的 `ArtifactDomain` 与 `ExecutionClass` token。每个已验证 `SnapshotIdentity` 携带同样的资格。Core 只比较这些值，不解释 CPU 特性、文件系统路径或存储实现。恢复目标必须在线、对 allocation 的不可变 deployment generation 已就绪、两个 token 均匹配，并有 active 容量；跨 node 时还需 retained slot。allocation、Device 和 Session 身份保持不变。Session 与 deployment 事务先提交目标路由、placement 和精确 restore intent，再执行目标侧原生操作。没有容量或兼容目标时，保留快照的所有权一直持续到配置期限。目标必须已持有该精确 generation；Core 不会在新节点自动准备历史 generation。升级期间应保留符合条件的节点及其 generation provider，直到检查点保留义务结束。 +支持检查点的 generation 报告 `CheckpointCompatibility`，包含不透明的 `ArtifactDomain` 与 `ExecutionClass` token。可转移的 `RetainedState` 在 `Compatibility` 中携带两个 token。direct 保留状态可省略它们，但省略不提供跨 node 恢复或清理资格。E2B 原生暂停不声明检查点可迁移性。Core 只比较这些值，不解释 CPU 特性、文件系统路径或存储实现。恢复目标必须在线、对 allocation 的不可变 deployment generation 已就绪、两个 token 均匹配，并有 active 容量；跨 node 时还需 retained slot。allocation、Device 和 Session 身份保持不变。Session 与 deployment 事务先提交目标路由、placement 和精确 restore intent，再执行目标侧原生操作。没有容量或兼容目标时，保留快照的所有权一直持续到配置期限。目标必须已持有该精确 generation；Core 不会在新节点自动准备历史 generation。升级期间应保留符合条件的节点及其 generation provider，直到检查点保留义务结束。 -`Suspend` 只有在持久发布已验证的完整归档、停止精确 source compute，并结清 source 本地 capture 与 cleanup 义务后，才能报告 `SourceStopped`。Core 先持久化快照，再标记 suspended；仅原生资源不存在并不足够。`Suspend.ObserveOnly` 可以为同一 operation 已验证的快照完成归档发布和源清理，但不能重新捕获快照，也不能停止已经恢复 running 的源。此前已发布的归档缺失或损坏时，`ObserveOnly` 可以返回其精确持久 ownership receipt，标记 `Status: unknown` 和 `SourceStopped: false`，供清理使用；该 receipt 不证明归档当前可用，也不证明源不存在。`Resume` 仍在执行前验证归档。完成源结清屏障后，source node 不可用不再阻止恢复。删除和正常到期可以将空闲 suspended allocation 的清理路由转给同一 artifact domain 内的在线 node；删除不要求 execution class 匹配。清理目标必须已提供同一不可变 generation，且有 retained 容量。没有这样的 node 时，清理保持 pending，不释放所有权；运维需恢复符合条件的 node 或其容量。未知 running 或 restoring writer 不会仅因 node 不可达而迁移。 +对于可转移检查点，`Suspend` 只有在持久发布已验证的完整归档、停止精确 source compute，并结清 source 本地 capture 与 cleanup 义务后，才能报告 `ResourcesReleased` 和 `SuspendSettled`。Core 先持久化绑定的保留状态，再标记 suspended；仅原生资源不存在并不足够。`Suspend.ReconcileOnly` 可以为同一 operation 已验证的快照完成归档发布和源清理，但不能重新捕获快照，也不能停止已经恢复 running 的源。此前已发布的归档缺失或损坏时，`Suspend.ReconcileOnly` 可以返回其精确持久 ownership receipt，标记 `Status: unknown` 和 `ResourcesReleased: false`，供清理使用；该 receipt 不证明归档当前可用，也不证明源不存在。`Resume` 仍在执行前验证归档。完成源结清屏障后，source node 不可用不再阻止恢复。删除和正常到期可以将空闲 suspended allocation 的清理路由转给同一 artifact domain 内的在线 node；删除不要求 execution class 匹配。清理目标必须已提供同一不可变 generation，且有 retained 容量。没有这样的 node 时，清理保持 pending，不释放所有权；运维需恢复符合条件的 node 或其容量。未知 running 或 restoring writer 不会仅因 node 不可达而迁移。 -恢复用 `Resume.ObserveOnly` 观察已持久化的 target 与 operation。`RestoreAttemptClosed` 是精确 operation ID，不是通用 absence 标志：adapter 只有在持久关闭从未进入原生执行的 attempt，并隔离该 ID 的所有迟到请求后才能返回。如果 adapter 能证明当前调用在派发原生恢复之前失败，也可以关闭已持久 admitted 的 attempt。microsandbox adapter 仅在 fresh 请求的 wire deadline 于原生派发之前到期时执行这种关闭；归档损坏、资格错误和不透明 SDK 失败都不授权自动重试。Core 随后先持久化新的 operation ID，再尝试执行。一旦已派发原生恢复，结果未知的 attempt 保留相同 target、所有权与原始 retention deadline；原生列表缺失、helper 死亡或超时都不授权重放。这种恢复可能一直不可用，直到显式删除或正常保留期到期；等待中的 input 可能达到其公共 deadline。用户删除仍需精确原生 settlement，之后才能释放资源与所有权。 +恢复用 `Resume.ReconcileOnly` 观察已持久化的 target 与 operation。`RestoreAttemptClosed` 是精确 operation ID，不是通用 absence 标志：adapter 只有在持久关闭从未进入原生执行的 attempt，并隔离该 ID 的所有迟到请求后才能返回。如果 adapter 能证明当前调用在派发原生恢复之前失败，也可以关闭已持久 admitted 的 attempt。microsandbox adapter 仅在 fresh 请求的 wire deadline 于原生派发之前到期时执行这种关闭；归档损坏、资格错误和不透明 SDK 失败都不授权自动重试。Core 随后先持久化新的 operation ID，再尝试执行。一旦已派发原生恢复，结果未知的 attempt 保留相同 target、所有权与原始 retention deadline；原生列表缺失、helper 死亡或超时都不授权重放。这种恢复可能一直不可用，直到显式删除或正常保留期到期；等待中的 input 可能达到其公共 deadline。用户删除仍需精确原生 settlement，之后才能释放资源与所有权。 原地恢复的 adapter 可以保留非空 native ID：关闭证明必须与请求的 ID、逻辑代次、名称和保留状态来源完全一致。该证明说明本次恢复未派发，不表示底层已暂停源实例不存在。 microsandbox adapter 要求显式的私有 `checkpoint_root`，位于所有 guest 可访问文件系统之外。目录权限为 `0700`，私有 namespace marker 标识真实归档存储。默认安装创建 node 本地私有 store。跨 node 恢复要求运维为参与 node 挂载同一个私有 store，挂载路径可以不同。Runtime state 目录与 workspace binding 都不用于推导该 root。adapter 在恢复前再次验证归档、外部文件系统身份和原生 execution class。恢复 RAM 与文件描述符不保证外部 TCP 连接连续，也不保证应用副作用可安全重放。 ### 检查点保留期后的冷替换 {#cold-replacement-after-checkpoint-retention} -快照保留期限制计算资源，不限制符合条件的保留 Session 的生命周期。到期后，只有独立文件系统对象已就绪、初始化已完成、Session 与 Environment 均未终结，且此前已认证 Runtime 的 `retained_native_history` 声明已持久化时，Core 才能在不终结 Session 的情况下清理 allocation。能力及原生恢复义务由 [Core–Runtime 协议](runtime-protocol.md)定义。仅保留文件系统并不足够。 +保留状态期限限制计算资源，不限制符合条件的保留 Session 的生命周期。到期后，只有独立文件系统对象已就绪、初始化已完成、Session 与 Environment 均未终结，且此前已认证 Runtime 的 `retained_native_history` 声明已持久化时，Core 才能在不终结 Session 的情况下清理 allocation。能力及原生恢复义务由 [Core–Runtime 协议](runtime-protocol.md)定义。仅保留文件系统并不足够。 -容量压力不会缩短检查点保留期。在配置的期限到达前，已挂起的 allocation 保留快照并继续占用 retained slot；容量不足或节点不兼容不允许降级为冷替换。等待请求保持原有的输入期限。保留期到期后，常规 expiry cleanup 必须确认资源已删除，其他 allocation 才能预留该 slot；文件与已验证的原生历史继续保留。 +Core 在释放 allocation 和 placement 归属前，确认原始创建已结算且所属计算资源与保留状态均已清理。替代计算资源准入前撤销旧 device 的权限。Create、Kill 或 DeleteRetained 的结果尚未确认时，Core 保留归属，不准入替代写入方。可恢复的 Session 与 Environment 保持断连，保留同一文件系统对象和初始化结果。归档、重置和删除优先，唤醒请求不能撤销它们。 -Core 在释放 allocation 和 placement 归属前，确认原始创建已结算且所属计算资源与快照均已清理。替代计算资源准入前撤销旧 device 的权限。Create、Kill 或 DeleteSnapshot 的结果尚未确认时，Core 保留归属，不准入替代写入方。可恢复的 Session 与 Environment 保持断连，保留同一文件系统对象和初始化结果。归档、重置和删除优先，唤醒请求不能撤销它们。 +容量压力不会缩短检查点保留期。在配置的期限到达前，已挂起的 allocation 保留快照并继续占用 retained slot；容量不足或节点不兼容不允许降级为冷替换。等待请求保持原有的输入期限。保留期到期后，常规 expiry cleanup 必须确认资源已删除，其他 allocation 才能预留该 slot；文件与已验证的原生历史继续保留。 新 input 或实时 Environment 文件访问会在兼容 node 上预约新 allocation，并引导新的 Runtime device。准备流程复用同一文件系统和已完成的初始化，不重放初始化。验证替代 Runtime 的声明后，执行从保留历史续接同一原生 Session。原生历史缺失或无效时明确失败；Core 不替换为新的原生 Session，也不重放已完成的 input。读取历史和已发布 Artifact 不预约计算资源。此路径不保留 VM 内存、后台进程或开放连接，也不为未声明能力的 Runtime、Harness 或文件系统组合提供资格。 @@ -220,7 +242,7 @@ Core 在释放 allocation 和 placement 归属前，确认原始创建已结算 一个 snapshot 和 timestamp 对持有资源分区。离线 ownership 来自 allocation 或 active placement 的 node，与 online presence 使用同一 45 秒 connection 和 owner-epoch predicate，独立于 provider readiness；cleanup failure、offline state 或空 read 都不授权合成 release。held resource 为零时，Core 在数据库事务外 drain，在 deployment lock 下复查，并在一个事务中清空 deployment，再发布携带 generation 的空 provider，没有可失败工作。最终 write 或 drain 失败时，在释放 mutation gate 前使用有界 owner context 恢复已提交 provider；恢复失败时 admission 保持 fenced，owner 停止。 -管理员 [Session archive](../../contracts/agents-api/zh/admin-api.md#session-archive) 在 Session-first transaction 中保留 Project scope 与 hosted eligibility 检查，并一起处理 Environment expiry、cancellation、Runtime authority revocation 和 audit；reset 的后台 archive 从可信记录重建实际 Project scope，保留请求方 provenance。普通 provider lifecycle 释放 compute 和 snapshot。archive cancellation 在 terminal commit 前保留健康 receipt path：仅首次撤销 device 的 archive 记录精确 `archive_cancel_turn_id`（普通 revocation 清空它，重复 cleanup 保留它），现有已认证 delivery 可从 Turn 原 `cancel_requested_at` 起最多 20 秒排空该 cancellation。Core 独立于 subscription removal，通过 `done`、cancellation acknowledgement 和 terminal commit 跟踪 delivery，不授予新 connection、input、file 或 MCP authority，也不续期 lease。事务或 lifecycle gate 不等待 receipt，peer 丢失、到期或重启回到普通 failure 与 cleanup，不虚构 cancelled outcome。 +管理员 [Session archive](../../contracts/agents-api/zh/admin-api.md#session-archive) 在 Session-first transaction 中保留 Project scope 与 hosted eligibility 检查，并一起处理 Environment expiry、cancellation、Runtime authority revocation 和 audit；reset 的后台 archive 从可信记录重建实际 Project scope，保留请求方 provenance。普通 provider lifecycle 释放 compute 和保留状态。archive cancellation 在 terminal commit 前保留健康 receipt path：仅首次撤销 device 的 archive 记录精确 `archive_cancel_turn_id`（普通 revocation 清空它，重复 cleanup 保留它），现有已认证 delivery 可从 Turn 原 `cancel_requested_at` 起最多 20 秒排空该 cancellation。Core 独立于 subscription removal，通过 `done`、cancellation acknowledgement 和 terminal commit 跟踪 delivery，不授予新 connection、input、file 或 MCP authority，也不续期 lease。事务或 lifecycle gate 不等待 receipt，peer 丢失、到期或重启回到普通 failure 与 cleanup，不虚构 cancelled outcome。 ## 验证集成 {#validate-the-integration} diff --git a/internal/runtimebootstrap/bootstrap.go b/internal/runtimebootstrap/bootstrap.go index bab5e7b16..183f5b054 100644 --- a/internal/runtimebootstrap/bootstrap.go +++ b/internal/runtimebootstrap/bootstrap.go @@ -14,7 +14,10 @@ import ( "github.com/google/uuid" ) -const Version = 1 +// SuspendControlFile is the packaged private hosted Runtime park/wake location. +const SuspendControlFile = "/run/oac/daemon-suspend.json" + +const Version = 2 const MaxBytes = 16 * 1024 var ErrInvalid = errors.New("invalid Runtime bootstrap input") @@ -27,12 +30,13 @@ type Connection struct { CoreURL string `json:"core_url"` DeviceID string `json:"device_id"` Credential string `json:"credential"` + Harness string `json:"harness"` } func (c Connection) Validate() error { u, err := url.Parse(c.CoreURL) id, idErr := uuid.Parse(c.DeviceID) - if c.Version != Version || err != nil || (u.Scheme != "https" && u.Scheme != "http") || + if c.Version != Version || !ValidHarness(c.Harness) || err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.RawPath != "" || u.Path != "/api/v1" || strings.ContainsAny(c.CoreURL, "?#") || strings.ContainsFunc(c.CoreURL, unicode.IsSpace) || @@ -83,6 +87,8 @@ func Decode(raw []byte) (Connection, error) { err = d.Decode(&c.DeviceID) case "credential": err = d.Decode(&c.Credential) + case "harness": + err = d.Decode(&c.Harness) default: return Connection{}, ErrInvalid } @@ -93,8 +99,21 @@ func Decode(raw []byte) (Connection, error) { if token, err = d.Token(); err != nil || token != json.Delim('}') { return Connection{}, ErrInvalid } - if len(seen) != 4 || d.Decode(new(any)) != io.EOF || c.Validate() != nil { + if len(seen) != 5 || d.Decode(new(any)) != io.EOF || c.Validate() != nil { return Connection{}, ErrInvalid } return c, nil } + +// ValidHarness checks the identifier syntax shared by startup projections. +func ValidHarness(kind string) bool { + if len(kind) == 0 || len(kind) > 64 || kind[0] < 'a' || kind[0] > 'z' { + return false + } + for _, c := range kind { + if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '_') { + return false + } + } + return true +} diff --git a/internal/runtimebootstrap/bootstrap_test.go b/internal/runtimebootstrap/bootstrap_test.go index 132e5659a..7fb194495 100644 --- a/internal/runtimebootstrap/bootstrap_test.go +++ b/internal/runtimebootstrap/bootstrap_test.go @@ -7,7 +7,7 @@ import ( ) func TestConnectionRoundTrip(t *testing.T) { - input := Connection{Version: Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-credential"} + input := Connection{Version: Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-credential", Harness: "codex"} raw, err := input.Marshal() if err != nil { t.Fatal(err) @@ -19,11 +19,11 @@ func TestConnectionRoundTrip(t *testing.T) { } func TestConnectionRejectsAmbiguousOrForeignInput(t *testing.T) { - good := `{"version":1,"core_url":"https://core.example/api/v1","device_id":"da912024-1543-4242-a2c1-5f4f7ebbc6c7","credential":"test-secret"}` + good := `{"version":2,"harness":"codex","core_url":"https://core.example/api/v1","device_id":"da912024-1543-4242-a2c1-5f4f7ebbc6c7","credential":"test-secret"}` for name, raw := range map[string]string{ "null": "null", "array": "[]", "trailing": good + "{}", - "duplicate": strings.Replace(good, `"version":1`, `"version":1,"version":1`, 1), - "unknown": strings.Replace(good, `"version":1`, `"extra":1,"version":1`, 1), + "duplicate": strings.Replace(good, `"version":2`, `"version":2,"version":2`, 1), + "unknown": strings.Replace(good, `"version":2`, `"extra":1,"version":2`, 1), "case": strings.Replace(good, "credential", "Credential", 1), "old auth": `{"server_url":"https://core.example","runtime_id":"old","runner_credential":"test-secret"}`, "oversized": strings.Repeat(" ", MaxBytes) + good, @@ -35,9 +35,10 @@ func TestConnectionRejectsAmbiguousOrForeignInput(t *testing.T) { }) } for field, values := range map[string][]any{ - "version": {nil, 0, 2, "1"}, + "version": {nil, 0, 1, 3, "2"}, "core_url": {"http://user:pass@core.example/api/v1", "https://core.example/api/v1?", "https://core.example/api/v1#", "https://core.example", "https://core.example/api/v1/", "https://core.example/api/v1?q=1", ""}, "device_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, + "harness": {nil, "", "Codex", "unknown-kind", "../codex", strings.Repeat("x", 65)}, "credential": {nil, "", "test\nsecret", "test\x00secret"}, } { for _, value := range values { diff --git a/services/core/cmd/server/managed_generation_operations.go b/services/core/cmd/server/managed_generation_operations.go index 2f7c99b65..aba23a97c 100644 --- a/services/core/cmd/server/managed_generation_operations.go +++ b/services/core/cmd/server/managed_generation_operations.go @@ -17,7 +17,7 @@ func (p *generationRouter) Initial(ctx context.Context, r sandbox.Reference) (sa defer done() return v.Initial(ctx, r) } -func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, snapshot *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, snapshot *sandbox.RetainedState) (sandbox.Compute, error) { if err := providercontract.Require(p, "NewCompute"); err != nil { return sandbox.Compute{}, err } @@ -72,8 +72,8 @@ func (p *generationRouter) KillCompute(ctx context.Context, r sandbox.Reference, defer done() return v.KillCompute(ctx, r, c) } -func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Reference, snapshot sandbox.SnapshotIdentity) error { - if err := providercontract.Require(p, "DeleteSnapshot"); err != nil { +func (p *generationRouter) DeleteRetained(ctx context.Context, r sandbox.Reference, snapshot sandbox.RetainedState) error { + if err := providercontract.Require(p, "DeleteRetained"); err != nil { return err } v, done, err := p.route(ctx, r) @@ -81,7 +81,7 @@ func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Referen return err } defer done() - return v.DeleteSnapshot(ctx, r, snapshot) + return v.DeleteRetained(ctx, r, snapshot) } func (p *generationRouter) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { if err := providercontract.Require(p, "RunCommandCompute"); err != nil { @@ -105,3 +105,15 @@ func (p *generationRouter) ResumeCompute(ctx context.Context, r sandbox.Referenc defer done() return v.ResumeCompute(ctx, r, c) } + +func (p *generationRouter) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + if err := providercontract.Require(p, "RenewCompute"); err != nil { + return sandbox.ComputeState{}, err + } + v, done, err := p.route(ctx, r) + if err != nil { + return sandbox.ComputeState{}, err + } + defer done() + return v.RenewCompute(ctx, r, c) +} diff --git a/services/core/cmd/server/managed_generations_test.go b/services/core/cmd/server/managed_generations_test.go index 7e5855f6b..341d27cd5 100644 --- a/services/core/cmd/server/managed_generations_test.go +++ b/services/core/cmd/server/managed_generations_test.go @@ -29,7 +29,7 @@ q=json.load(sys.stdin) with (pathlib.Path(q['Config']['StateDir'])/'requests').open('a') as f: f.write(json.dumps(q)+'\n') info=dict(q['Reference'],State='running',ProviderID='owned',CreateSettled=True) if q['Operation']=='kill': info['State']='absent' -print(json.dumps({'Version':1,'Info':info})) +print(json.dumps({'Version':q['Version'],'Info':info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -54,7 +54,7 @@ print(json.dumps({'Version':1,'Info':info})) value.Configuration = &key return value, nil } - setup := &managedSetup{processPaths: paths, registry: providers.Builtin(), deployment: &fakeDeploymentSetups{t: t, allocationSetup: allocation}, installationID: id} + setup := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), deployment: &fakeDeploymentSetups{t: t, allocationSetup: allocation}, installationID: id} // A facade retained by a generation-one lifecycle still reads current credentials. router := &generationRouter{setup: setup} ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) @@ -132,7 +132,7 @@ code = '' if k == 'revoked': code = 'unauthorized' elif k == 'unconfirmed': code = 'unconfirmed' elif not (k.startswith('team-a') and template in ('owned-a', 'new-a') or k == 'team-b' and template == 'public-b'): code = 'team_mismatch' -result = {'Version': 1, 'ErrorCode': code} +result = {'Version': q['Version'], 'ErrorCode': code} if not code: result['DeploymentValid'] = True if q['Operation'] == 'validate_deployment': @@ -151,7 +151,7 @@ print(json.dumps(result)) setups := &fakeDeploymentSetups{t: t, setup: committedSetup(&committed), withCredential: credentialService(t), generationPage: func(context.Context, int64) ([]deployment.Setup, error) { return nil, nil }} allocations := &fakeGenerationAllocations{t: t, credentialAllocations: func(context.Context, string) ([]deployment.Allocation, error) { return nil, nil }} - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: setups, allocations: allocations} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: setups, allocations: allocations} loaded, err := s.load(t.Context()) if err != nil { t.Fatal(err) diff --git a/services/core/cmd/server/managed_nodes.go b/services/core/cmd/server/managed_nodes.go index c514c7e3d..402a1912c 100644 --- a/services/core/cmd/server/managed_nodes.go +++ b/services/core/cmd/server/managed_nodes.go @@ -70,7 +70,7 @@ func configureManagedNodes(nodes *deployment.Service, reader deployment.Reader, return nodes.Heartbeat(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health), health.Generations) }, }) - result.setup = &managedSetup{processPaths: config.ProviderPaths, registry: registry, deployment: nodes, allocations: reader, hub: result.hub, installationID: config.InstallationID, runtimeAPI: config.PublicOrigin.RuntimeAPI()} + result.setup = &managedSetup{capacity: config.SandboxCapacity, processPaths: config.ProviderPaths, registry: registry, deployment: nodes, allocations: reader, hub: result.hub, installationID: config.InstallationID, runtimeAPI: config.PublicOrigin.RuntimeAPI()} result.runtime = execution.NewDeferredRuntimeProvider(config.InstallationID, result.setup.load, result.setup.prepare) result.runtime.PublishUnconfigured = result.setup.publishUnconfigured return result diff --git a/services/core/cmd/server/managed_setup.go b/services/core/cmd/server/managed_setup.go index 758ff5db0..f913b508c 100644 --- a/services/core/cmd/server/managed_setup.go +++ b/services/core/cmd/server/managed_setup.go @@ -11,6 +11,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/processconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -22,6 +23,7 @@ import ( // observation. The database owns the selection; this cache is never a writer. type managedSetup struct { processPaths sandbox.ProcessPaths + capacity processconfig.SandboxLimits // registry builds the selected direct provider and discovers configuration; // the deployment setup reports what the registration declares. registry *providers.Registry @@ -159,7 +161,7 @@ func (s *managedSetup) configuration(setup deployment.Setup) (execution.Prepared Resources: setup.Specification.Resources, WorkspaceRequirements: adapter.Policy.Workspace, Workspace: setup.Specification.Workspace} if setup.Suspension != nil { selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.Suspension.IdleSeconds) * time.Second, - Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second} + Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second, MaxActive: s.capacity.MaxActive, MaxRetained: s.capacity.MaxRetained} } return execution.PreparedRuntimeDeployment{Config: selected, Publish: s.publish}, nil } diff --git a/services/core/cmd/server/managed_setup_preflight_test.go b/services/core/cmd/server/managed_setup_preflight_test.go index dc977e41a..959fb49fd 100644 --- a/services/core/cmd/server/managed_setup_preflight_test.go +++ b/services/core/cmd/server/managed_setup_preflight_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -19,7 +20,7 @@ import ( func TestE2BRejectedSpecificationHasSafeActionableDiagnostic(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(e2b.ProtocolVersion)+",\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") @@ -28,7 +29,7 @@ func TestE2BRejectedSpecificationHasSafeActionableDiagnostic(t *testing.T) { } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id} selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 3, MemoryMiB: 3072}}, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} _, err := s.prepare(t.Context(), selection) if !errors.Is(err, sandbox.ErrConfigurationSelection) || strings.Contains(err.Error(), "synthetic-private-key") || s.selected.Load() != nil { @@ -49,12 +50,12 @@ op = q['Operation'] with (pathlib.Path(q['Config']['StateDir']) / 'operations').open('a') as log: log.write(op + '\n') if op == 'validate_deployment': - print(json.dumps({'Version': 1, 'ErrorCode': 'invalid'})) + print(json.dumps({'Version': q['Version'], 'ErrorCode': 'invalid'})) else: info = dict(q['Reference'], ProviderID='owned-compute', State='stopped', CreateSettled=True) if op == 'kill': info.update(ProviderID='', State='absent') - print(json.dumps({'Version': 1, 'Info': info})) + print(json.dumps({'Version': q['Version'], 'Info': info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -69,7 +70,7 @@ else: Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} allocation := func(context.Context, sandbox.Reference) (deployment.Setup, error) { return selection, nil } - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: committedSetup(&selection), allocationSetup: allocation}} if _, err := s.prepare(t.Context(), selection); err == nil || s.selected.Load() != nil { t.Fatal("invalid new template selection was published", err) @@ -101,13 +102,13 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { } helper := filepath.Join(t.TempDir(), "provider") requests := filepath.Join(state, "requests") - script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" + script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":" + strconv.Itoa(e2b.ProtocolVersion) + ",\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} candidate, err := s.prepare(t.Context(), selection) disk := int32(24063) @@ -133,7 +134,7 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(e2b.ProtocolVersion)+",\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") @@ -142,7 +143,7 @@ func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-team-a", Template: "public-team-b:" + uuid.NewString()}} if _, err := s.prepare(t.Context(), selection); !errors.Is(err, sandbox.ErrCredentialOwnership) || s.selected.Load() != nil { t.Fatal("public readability accepted as team ownership", err) @@ -150,7 +151,7 @@ func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { } func TestManagedSetupRoutesProviderWithoutCredentialRequirement(t *testing.T) { - s := &managedSetup{} + s := &managedSetup{capacity: testSandboxCapacity(t)} config := &execution.RuntimeProvider{ProviderKind: "docker"} candidate, err := s.routeGenerations( execution.PreparedRuntimeDeployment{Config: config}, diff --git a/services/core/cmd/server/managed_setup_test.go b/services/core/cmd/server/managed_setup_test.go index aaaf9cd02..d2f95a334 100644 --- a/services/core/cmd/server/managed_setup_test.go +++ b/services/core/cmd/server/managed_setup_test.go @@ -122,7 +122,7 @@ func TestManagedSetupNeverReusesAnotherGenerationOrUnverifiedState(t *testing.T) value := deployment.Setup{InstallationID: "installation", Provider: "docker", Mode: "nodes", Generation: 1} var loadErr error setups := &fakeDeploymentSetups{t: t, setup: func(context.Context) (deployment.Setup, error) { return value, loadErr }} - s := &managedSetup{registry: providers.Builtin(), deployment: setups, allocations: &fakeGenerationAllocations{t: t}, installationID: "installation"} + s := &managedSetup{capacity: testSandboxCapacity(t), registry: providers.Builtin(), deployment: setups, allocations: &fakeGenerationAllocations{t: t}, installationID: "installation"} cached := &execution.RuntimeProvider{InstallationID: "installation", ProviderKind: "docker", Generation: 1} s.publish(cached) if got, err := s.load(t.Context()); err != nil || got != cached { @@ -153,7 +153,7 @@ func TestMissingE2BHelperReportsProviderUnavailable(t *testing.T) { id := uuid.NewString() committed := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-key", Template: "runtime:" + uuid.NewString()}} - s := &managedSetup{processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id, + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: committedSetup(&committed)}} if _, err := s.load(t.Context()); !errors.Is(err, execution.ErrExecutionUnavailable) { t.Fatal("missing local helper must leave administrative recovery available", err) @@ -164,10 +164,12 @@ func TestMissingE2BHelperReportsProviderUnavailable(t *testing.T) { } func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "7") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "31") id := uuid.NewString() hub := node.NewHub(node.HubOptions{}) defer hub.Close() - s := &managedSetup{registry: providers.Builtin(), installationID: id, hub: hub, deployment: &fakeDeploymentSetups{t: t}, allocations: &fakeGenerationAllocations{t: t}, runtimeAPI: "https://core.example/api/v1"} + s := &managedSetup{capacity: testSandboxCapacity(t), registry: providers.Builtin(), installationID: id, hub: hub, deployment: &fakeDeploymentSetups{t: t}, allocations: &fakeGenerationAllocations{t: t}, runtimeAPI: "https://core.example/api/v1"} previous := &execution.RuntimeProvider{InstallationID: id, Generation: 1, ProviderKind: "docker"} s.publish(previous) candidate, err := s.prepare(t.Context(), deployment.Setup{InstallationID: id, Provider: "microsandbox", Mode: "nodes", Operations: microsandbox.Operations(), Suspension: &deployment.Suspension{IdleSeconds: 300, RetentionSeconds: 86400}}) @@ -177,6 +179,9 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { if s.selected.Load().Config != previous || candidate.Config.ProviderKind != "microsandbox" || candidate.Config.Suspension == nil || candidate.Config.CoreURL != "https://core.example/api/v1" { t.Fatal("preparation published or lost candidate configuration") } + if candidate.Config.Suspension.MaxActive != 7 || candidate.Config.Suspension.MaxRetained != 31 { + t.Fatal("configured capacity was not propagated") + } committed := *candidate.Config committed.Generation = 2 candidate.Publish(&committed) @@ -187,7 +192,7 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { func TestManagedSetupRejectedCandidateRetainsSelection(t *testing.T) { id := uuid.NewString() - s := &managedSetup{processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id} previous := &execution.RuntimeProvider{InstallationID: id, Generation: 1, ProviderKind: "docker"} s.publish(previous) _, err := s.prepare(t.Context(), deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, @@ -210,7 +215,7 @@ func TestManagedSetupResetTombstoneRejectsDelayedProviderLoad(t *testing.T) { return deployment.Setup{}, ctx.Err() } } - s := &managedSetup{registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: delayed}} + s := &managedSetup{capacity: testSandboxCapacity(t), registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: delayed}} done := make(chan error, 1) go func() { provider, err := s.load(t.Context()) diff --git a/services/core/cmd/server/sandbox_capacity_test.go b/services/core/cmd/server/sandbox_capacity_test.go new file mode 100644 index 000000000..46ad1cc0d --- /dev/null +++ b/services/core/cmd/server/sandbox_capacity_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/processconfig" +) + +func testSandboxCapacity(t testing.TB) processconfig.SandboxLimits { + t.Helper() + c, err := processconfig.SandboxCapacity() + if err != nil { + t.Fatal(err) + } + return c +} diff --git a/services/core/deploy/e2b/build-template.py b/services/core/deploy/e2b/build-template.py index d78926213..30616a94d 100644 --- a/services/core/deploy/e2b/build-template.py +++ b/services/core/deploy/e2b/build-template.py @@ -10,6 +10,7 @@ import tempfile from e2b import Template +from helper_contract_generated import SUSPEND_CONTROL_FILE BASE = 'node:22.23.1-bookworm-slim@sha256:8607a9064d4a571140998ae9e52a3b3fcf9cff361d04642d5971e6cd76d39e27' parser = argparse.ArgumentParser() @@ -27,6 +28,7 @@ if value.startswith(('HOME=', 'OAC_'))) if environment.get('OAC_RUNTIME_WORKSPACE') != '/environment/workspace': parser.error('Image does not use the colocated Runtime layout') +environment['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'] = SUSPEND_CONTROL_FILE dev_home = Path(os.environ.get('OAC_DEV_HOME') or Path.home() / '.oac') if not dev_home.is_absolute(): parser.error('OAC_DEV_HOME must be absolute') diff --git a/services/core/deploy/e2b/build_template_test.py b/services/core/deploy/e2b/build_template_test.py index 9a3b3034f..3e4f90508 100644 --- a/services/core/deploy/e2b/build_template_test.py +++ b/services/core/deploy/e2b/build_template_test.py @@ -64,6 +64,9 @@ def build(instance, **kwargs): self.assertIs(instance, template) context = Path(factory.call_args.kwargs['file_context_path']) self.assertEqual(stat.S_IMODE(context.stat().st_mode), 0o700) + from helper_contract_generated import SUSPEND_CONTROL_FILE + runtime_env = json.loads((context / 'runtime-env.json').read_text()) + self.assertEqual(runtime_env['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'], SUSPEND_CONTROL_FILE) with tarfile.open(context / 'runtime.tar.gz') as archive: modes = {m.name: stat.S_IMODE(m.mode) for m in archive.getmembers()} for parent in ('usr', 'usr/local', 'etc'): diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index 77d1869fd..d454cdd8c 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -1,5 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" +COMPUTE_FIELDS = ["Generation","Name","ID","RestoredFrom"] ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] @@ -10,9 +11,13 @@ MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 1 +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute_info","compute_renew","suspend","resume","compute_kill","delete_retained","compute_command","resume_compute"] +PROTOCOL_VERSION = 4 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Retained","Deadline"] +RESPONSE_FIELDS = ["Version","State","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +RESUME_FIELDS = ["Workspace","Reference","OperationID","Retained","Target","ReconcileOnly"] +RETAINED_FIELDS = ["Compatibility","Reference","ID","Data","OperationID","SourceGeneration","SourceName","SourceID"] SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" +SUSPEND_FIELDS = ["Reference","OperationID","Source","Retained","ReconcileOnly"] diff --git a/services/core/deploy/e2b/init.py b/services/core/deploy/e2b/init.py index 75d8e1f07..fa8171c29 100644 --- a/services/core/deploy/e2b/init.py +++ b/services/core/deploy/e2b/init.py @@ -106,6 +106,8 @@ def initialize(): # Claim before any side effect. An interrupted attempt must never start twice. write_private(ROOT / 'launch.json', identity) environment = prepare_runtime() + # Application-owned startup does not participate in Core-managed suspension. + environment.pop('OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE', None) credential = PROFILE.parent / 'executor-key.json' write_private(credential, payload['executor_key'], owner=1000) source.unlink() diff --git a/services/core/deploy/e2b/init_test.py b/services/core/deploy/e2b/init_test.py index 3c416c13f..31a1489d3 100644 --- a/services/core/deploy/e2b/init_test.py +++ b/services/core/deploy/e2b/init_test.py @@ -37,7 +37,8 @@ def test_launch_handoff_is_private_and_cannot_replay(self): profile = Path(temporary) / 'private/default' environment_file = Path(temporary) / 'image.json' environment_file.write_text(json.dumps({'HOME': '/home/runtime', - 'OAC_RUNTIME_HOME': '/home/runtime/.oac', 'OAC_RUNTIME_WORKSPACE': '/environment/workspace'})) + 'OAC_RUNTIME_HOME': '/home/runtime/.oac', 'OAC_RUNTIME_WORKSPACE': '/environment/workspace', + 'OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE': '/private/control/fixture.json'})) (root / 'bootstrap.json').write_text(json.dumps(PAYLOAD)) real_chmod = Path.chmod @@ -57,6 +58,7 @@ def chmod(path, mode): self.assertEqual(argv[argv.index('--remote') + 1], PAYLOAD['remote_url']) self.assertNotIn('test-private-key', repr(popen.call_args)) self.assertNotIn('OAC_RUNTIME_SESSION_ID', options['env']) + self.assertNotIn('OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE', options['env']) self.assertEqual((options['user'], options['group'], options['extra_groups']), (1000, 1000, [])) self.assertEqual(options['umask'], 0o077) key = profile.parent / 'executor-key.json' diff --git a/services/core/deploy/e2b/managed_init.py b/services/core/deploy/e2b/managed_init.py index 588f69d80..8c004102e 100644 --- a/services/core/deploy/e2b/managed_init.py +++ b/services/core/deploy/e2b/managed_init.py @@ -49,6 +49,13 @@ def initialize(): binding = identity(payload) shared.write_private(root / 'managed-launch.json', binding) environment = shared.prepare_runtime() + control_file = Path(environment['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE']) + if not control_file.is_absolute() or '..' in control_file.parts: + raise ValueError('Absolute private suspend control file required') + control_directory = control_file.parent + control_directory.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chown(control_directory, 1000, 1000) + control_directory.chmod(0o700) environment.update(OAC_RUNTIME_ENVIRONMENT_ID=payload['EnvironmentID'], OAC_RUNTIME_SESSION_ID=payload['SessionID'], OAC_RUNTIME_NETWORK_ACCESS=payload['NetworkAccess'], diff --git a/services/core/deploy/e2b/managed_init_test.py b/services/core/deploy/e2b/managed_init_test.py index d9017ba82..c6d6a957e 100644 --- a/services/core/deploy/e2b/managed_init_test.py +++ b/services/core/deploy/e2b/managed_init_test.py @@ -16,7 +16,7 @@ def payload(): value = {key: str(uuid4()) for key in ['InstallationID', 'TenantID', 'EnvironmentID', 'AllocationID', 'SessionID', 'DeviceID']} - return dict(value, RuntimeBootstrap={'version': 1, 'core_url': 'https://core.example/api/v1', + return dict(value, RuntimeBootstrap={'version': 2, 'harness': 'codex', 'core_url': 'https://core.example/api/v1', 'device_id': value['DeviceID'], 'credential': 'private-managed-token'}, NetworkAccess='restricted', AllowedDomains=['example.com']) @@ -63,9 +63,11 @@ def exercise(self, failed=False): if failed: process.side_effect = RuntimeError('private process diagnostic') image_env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'OAC_RUNTIME_HOME': str(Path(temporary) / '.oac'), - 'OAC_RUNTIME_WORKSPACE': '/environment/workspace'} + 'OAC_RUNTIME_WORKSPACE': '/environment/workspace', + 'OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE': str(Path(temporary) / 'control' / 'custom-suspend.json')} with patch.object(managed_init.shared, 'ROOT', root), patch.object(managed_init.shared, 'PROFILE', profile), \ patch.object(managed_init.shared, 'prepare_runtime', return_value=image_env), \ + patch.object(managed_init.os, 'chown') as chown, \ patch.object(managed_init.os, 'fchown'), patch.object(managed_init.subprocess, 'Popen', process): if failed: with self.assertRaises(RuntimeError): @@ -83,6 +85,10 @@ def exercise(self, failed=False): self.assertNotIn(data['RuntimeBootstrap']['credential'], json.dumps(process.call_args.kwargs['env'])) self.assertEqual(process.call_args.kwargs['env']['OAC_RUNTIME_ENVIRONMENT_ID'], data['EnvironmentID']) self.assertEqual(process.call_args.kwargs['user'], 1000) + control = Path(temporary) / 'control' + self.assertEqual(control.stat().st_mode & 0o777, 0o700) + chown.assert_called_once_with(control, 1000, 1000) + self.assertEqual(process.call_args.kwargs['env']['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'], str(control / 'custom-suspend.json')) if failed: self.assertFalse((root / 'managed-ready.json').exists()) else: diff --git a/services/core/internal/db/queries/runtime_allocations.sql b/services/core/internal/db/queries/runtime_allocations.sql index a33ccc532..f724ca29d 100644 --- a/services/core/internal/db/queries/runtime_allocations.sql +++ b/services/core/internal/db/queries/runtime_allocations.sql @@ -1,6 +1,6 @@ -- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) -VALUES ($1, $2, $3, $4, $5, $6) RETURNING *; +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, compute_state) +VALUES ($1, $2, $3, $4, $5, $6, jsonb_build_object('protocol_version', sqlc.arg(protocol_version)::text)) RETURNING *; -- name: GetLatestRuntimeAllocation :one SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired diff --git a/services/core/internal/db/queries/runtime_nodes.sql b/services/core/internal/db/queries/runtime_nodes.sql index 6a24dbb20..44f68e596 100644 --- a/services/core/internal/db/queries/runtime_nodes.sql +++ b/services/core/internal/db/queries/runtime_nodes.sql @@ -19,7 +19,7 @@ SELECT n.*, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id AND a.state<>'released'))::bigint AS reserved, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='cleanup_pending')::bigint AS cleanup_pending, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='running' AND a.compute_phase IN('running','disabled'))::bigint AS running, - (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'snapshot' IS NOT NULL AND a.compute_state->'snapshot'<>'null'::jsonb)::bigint AS snapshots + (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'retained' IS NOT NULL AND a.compute_state->'retained'<>'null'::jsonb)::bigint AS snapshots FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND (sqlc.narg(node_id)::uuid IS NULL OR n.id=sqlc.narg(node_id)::uuid) ORDER BY n.id; diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index 1bd60f42d..54a5523af 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -25,8 +25,6 @@ WHERE id = $1 AND compute_phase = 'running' AND compute_activity_at <= $2; -- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, a.compute_phase_changed_at, - CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, - CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) @@ -46,7 +44,7 @@ SELECT EXISTS ( UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e WHERE a.environment_id = e.id AND e.session_id = $1 - AND a.node_id IS NOT NULL AND a.state = 'running'; + AND a.state = 'running'; -- name: SessionHasRuntimeNode :one SELECT EXISTS ( @@ -54,6 +52,23 @@ SELECT EXISTS ( WHERE e.session_id = $1 AND a.state <> 'released' AND a.node_id IS NOT NULL )::boolean; +-- name: HasIncompatibleRuntimeComputeState :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations + WHERE state <> 'released' + AND ((compute_state->>'protocol_version') IS DISTINCT FROM sqlc.arg(protocol_version)::text + OR compute_state ? 'snapshot' + OR (compute_state #> '{current,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot'] + OR (compute_state #> '{target,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot']) +)::boolean; + +-- name: CountRuntimeComputeReservations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended'; + +-- name: CountRuntimeRetainedAllocations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released'; -- name: MoveSuspendedRuntimeCompute :one WITH moved AS ( UPDATE runtime_placements p SET node_id=sqlc.arg(destination)::uuid diff --git a/services/core/internal/db/queries/runtime_suspension_pressure.sql b/services/core/internal/db/queries/runtime_suspension_pressure.sql index 580c00813..4c47d4caa 100644 --- a/services/core/internal/db/queries/runtime_suspension_pressure.sql +++ b/services/core/internal/db/queries/runtime_suspension_pressure.sql @@ -6,9 +6,9 @@ WHERE a.node_id IS NOT NULL AND a.state <> 'released' OR (a.compute_retained_until <= clock_timestamp() AND a.compute_phase <> 'disabled')); -- name: ListWaitingCheckpointRestores :many -SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'snapshot'->'Compatibility')::jsonb AS checkpoint +SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'retained'->'Compatibility')::jsonb AS checkpoint FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id WHERE a.node_id IS NOT NULL AND a.state='running' AND a.compute_phase='suspended' AND a.compute_retained_until>clock_timestamp() AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') - AND a.compute_state->'snapshot'->'Compatibility' IS NOT NULL + AND a.compute_state->'retained'->'Compatibility' IS NOT NULL AND (a.compute_wake_requested OR EXISTS(SELECT 1 FROM environment_input_reservations r WHERE r.session_id=s.id AND r.state='pending' AND r.deadline>clock_timestamp())); diff --git a/services/core/internal/db/sqlc/runtime_allocations.sql.go b/services/core/internal/db/sqlc/runtime_allocations.sql.go index 3c85fc718..d3ae628ad 100644 --- a/services/core/internal/db/sqlc/runtime_allocations.sql.go +++ b/services/core/internal/db/sqlc/runtime_allocations.sql.go @@ -51,8 +51,8 @@ func (q *Queries) CanRetainRuntimeEnvironment(ctx context.Context, id pgtype.UUI } const createRuntimeAllocation = `-- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) -VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, compute_state) +VALUES ($1, $2, $3, $4, $5, $6, jsonb_build_object('protocol_version', $7::text)) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` type CreateRuntimeAllocationParams struct { @@ -62,6 +62,7 @@ type CreateRuntimeAllocationParams struct { ProviderKey pgtype.UUID `json:"provider_key"` NodeID pgtype.UUID `json:"node_id"` DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` + ProtocolVersion string `json:"protocol_version"` } func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntimeAllocationParams) (RuntimeAllocation, error) { @@ -72,6 +73,7 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime arg.ProviderKey, arg.NodeID, arg.DeploymentGeneration, + arg.ProtocolVersion, ) var i RuntimeAllocation err := row.Scan( diff --git a/services/core/internal/db/sqlc/runtime_nodes.sql.go b/services/core/internal/db/sqlc/runtime_nodes.sql.go index e28353bcf..d664a022d 100644 --- a/services/core/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_nodes.sql.go @@ -388,7 +388,7 @@ SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha2 (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id AND a.state<>'released'))::bigint AS reserved, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='cleanup_pending')::bigint AS cleanup_pending, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='running' AND a.compute_phase IN('running','disabled'))::bigint AS running, - (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'snapshot' IS NOT NULL AND a.compute_state->'snapshot'<>'null'::jsonb)::bigint AS snapshots + (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'retained' IS NOT NULL AND a.compute_state->'retained'<>'null'::jsonb)::bigint AS snapshots FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND ($1::uuid IS NULL OR n.id=$1::uuid) ORDER BY n.id diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index 9596b297e..44949ca26 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -27,11 +27,33 @@ func (q *Queries) ClearRuntimeWake(ctx context.Context, arg ClearRuntimeWakePara return err } +const countRuntimeComputeReservations = `-- name: CountRuntimeComputeReservations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended' +` + +func (q *Queries) CountRuntimeComputeReservations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { + row := q.db.QueryRow(ctx, countRuntimeComputeReservations, providerKey) + var count int64 + err := row.Scan(&count) + return count, err +} + +const countRuntimeRetainedAllocations = `-- name: CountRuntimeRetainedAllocations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' +` + +func (q *Queries) CountRuntimeRetainedAllocations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { + row := q.db.QueryRow(ctx, countRuntimeRetainedAllocations, providerKey) + var count int64 + err := row.Scan(&count) + return count, err +} + const getRuntimeActivity = `-- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, a.compute_phase_changed_at, - CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, - CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) @@ -61,6 +83,24 @@ func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRu return i, err } +const hasIncompatibleRuntimeComputeState = `-- name: HasIncompatibleRuntimeComputeState :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations + WHERE state <> 'released' + AND ((compute_state->>'protocol_version') IS DISTINCT FROM $1::text + OR compute_state ? 'snapshot' + OR (compute_state #> '{current,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot'] + OR (compute_state #> '{target,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot']) +)::boolean +` + +func (q *Queries) HasIncompatibleRuntimeComputeState(ctx context.Context, protocolVersion string) (bool, error) { + row := q.db.QueryRow(ctx, hasIncompatibleRuntimeComputeState, protocolVersion) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + const moveSuspendedRuntimeCompute = `-- name: MoveSuspendedRuntimeCompute :one WITH moved AS ( UPDATE runtime_placements p SET node_id=$1::uuid @@ -129,7 +169,7 @@ const recordRuntimeTerminalActivity = `-- name: RecordRuntimeTerminalActivity :e UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e WHERE a.environment_id = e.id AND e.session_id = $1 - AND a.node_id IS NOT NULL AND a.state = 'running' + AND a.state = 'running' ` func (q *Queries) RecordRuntimeTerminalActivity(ctx context.Context, sessionID pgtype.UUID) error { diff --git a/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go index e192c879e..86b051015 100644 --- a/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go @@ -40,11 +40,11 @@ func (q *Queries) ListRuntimeSuspensionInFlightNodes(ctx context.Context) ([]pgt } const listWaitingCheckpointRestores = `-- name: ListWaitingCheckpointRestores :many -SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'snapshot'->'Compatibility')::jsonb AS checkpoint +SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'retained'->'Compatibility')::jsonb AS checkpoint FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id WHERE a.node_id IS NOT NULL AND a.state='running' AND a.compute_phase='suspended' AND a.compute_retained_until>clock_timestamp() AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') - AND a.compute_state->'snapshot'->'Compatibility' IS NOT NULL + AND a.compute_state->'retained'->'Compatibility' IS NOT NULL AND (a.compute_wake_requested OR EXISTS(SELECT 1 FROM environment_input_reservations r WHERE r.session_id=s.id AND r.state='pending' AND r.deadline>clock_timestamp())) ` diff --git a/services/core/internal/deployment/allocations.go b/services/core/internal/deployment/allocations.go index 24560832a..e26fcd634 100644 --- a/services/core/internal/deployment/allocations.go +++ b/services/core/internal/deployment/allocations.go @@ -277,7 +277,7 @@ func (e *ExecutionOperations) SetCompute(ctx context.Context, owner Allocation, // that publication and source-local cleanup settled before suspension. func (e *ExecutionOperations) BeginRestore(ctx context.Context, owner Allocation, compatibility sandbox.CheckpointCompatibility, state json.RawMessage) (Allocation, error) { var object map[string]json.RawMessage - if compatibility.Validate() != nil || json.Unmarshal(state, &object) != nil || object == nil { + if (owner.NodeID != "" && compatibility.Validate() != nil) || json.Unmarshal(state, &object) != nil || object == nil { return Allocation{}, ErrInvalidInput } return e.change(ctx, owner, true, func(tx AllocationTx, current Allocation) (Allocation, error) { diff --git a/services/core/internal/deployment/allocations_test.go b/services/core/internal/deployment/allocations_test.go index 22d110171..22d6ea3db 100644 --- a/services/core/internal/deployment/allocations_test.go +++ b/services/core/internal/deployment/allocations_test.go @@ -690,7 +690,7 @@ func TestBeginRestoreDirectKeepsAllocationAndRetention(t *testing.T) { if err != nil { t.Fatal(err) } - restored, err := operations.BeginRestore(t.Context(), owner, sandbox.CheckpointCompatibility{ArtifactDomain: "fixture", ExecutionClass: "fixture"}, json.RawMessage(`{"restore_id":"attempt"}`)) + restored, err := operations.BeginRestore(t.Context(), owner, sandbox.CheckpointCompatibility{}, json.RawMessage(`{"restore_id":"attempt"}`)) if err != nil || restored.ID != owner.ID || restored.NodeID != "" || restored.ComputePhase != "restoring" { t.Fatal("direct restore changed ownership", restored, err) } diff --git a/services/core/internal/deployment/execution.go b/services/core/internal/deployment/execution.go index 7390deacd..40d420e45 100644 --- a/services/core/internal/deployment/execution.go +++ b/services/core/internal/deployment/execution.go @@ -291,3 +291,17 @@ func (e *ExecutionOperations) recordMetadata(tx DeploymentTx, input sandbox.Sele } return tx.RecordConfigurationMetadata(record.Metadata) } + +// CheckRuntimeComputeProtocol refuses incompatible allocation receipts before activation. +func (e *ExecutionOperations) CheckRuntimeComputeProtocol(ctx context.Context, version string) error { + return e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { + incompatible, err := tx.HasIncompatibleComputeState(version) + if err != nil { + return err + } + if incompatible { + return errors.New("incompatible retained runtime state: use the previous release to archive allocations before upgrading; history is preserved") + } + return nil + }) +} diff --git a/services/core/internal/deployment/fakes_test.go b/services/core/internal/deployment/fakes_test.go index 5b3e69dfa..4ab3c354b 100644 --- a/services/core/internal/deployment/fakes_test.go +++ b/services/core/internal/deployment/fakes_test.go @@ -76,27 +76,29 @@ func (f *fakeExecutionStorage) WithDeployment(ctx context.Context, apply func(De } type fakeReader struct { - t testing.TB - deployment func(context.Context) (Record, error) - snapshot func(context.Context) (Snapshot, error) - ownerEpoch func(context.Context) (uint64, error) - allocation func(context.Context, sandbox.Reference) (AllocationRecord, error) - generations func(context.Context, int64) ([]GenerationRecord, error) - nodes func(context.Context) ([]NodeRecord, error) - nodeHistory func(context.Context, string, coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) - readNodes func(context.Context, func(NodeReads) error) error - resetSessions func(context.Context, string, bool) ([]ResetSession, error) - addressBindings func(context.Context, string) (AddressBindings, error) - environmentAllocation func(context.Context, AllocationKey) (Allocation, error) - credentialAllocations func(context.Context, string) ([]Allocation, error) - observationSessions func(context.Context, string, int) (ObservationSessionPage, error) - nodeAllocations func(context.Context, string) ([]NodeAllocation, error) - nodeOnline func(context.Context, string) (bool, error) - lifecycleNodes func(context.Context) ([]string, error) - lifecycleAllocations func(context.Context, string, string) ([]Allocation, error) - unallocatedEnvironments func(context.Context, string, string) ([]UnallocatedEnvironment, error) - lifecyclePlacement func(context.Context, AllocationKey) (LifecyclePlacement, error) - activity func(context.Context, string) (Activity, error) + countRetainedAllocations func(context.Context, string) (int64, error) + countComputeReservations func(context.Context, string) (int64, error) + t testing.TB + deployment func(context.Context) (Record, error) + snapshot func(context.Context) (Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (AllocationRecord, error) + generations func(context.Context, int64) ([]GenerationRecord, error) + nodes func(context.Context) ([]NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) + readNodes func(context.Context, func(NodeReads) error) error + resetSessions func(context.Context, string, bool) ([]ResetSession, error) + addressBindings func(context.Context, string) (AddressBindings, error) + environmentAllocation func(context.Context, AllocationKey) (Allocation, error) + credentialAllocations func(context.Context, string) ([]Allocation, error) + observationSessions func(context.Context, string, int) (ObservationSessionPage, error) + nodeAllocations func(context.Context, string) ([]NodeAllocation, error) + nodeOnline func(context.Context, string) (bool, error) + lifecycleNodes func(context.Context) ([]string, error) + lifecycleAllocations func(context.Context, string, string) ([]Allocation, error) + unallocatedEnvironments func(context.Context, string, string) ([]UnallocatedEnvironment, error) + lifecyclePlacement func(context.Context, AllocationKey) (LifecyclePlacement, error) + activity func(context.Context, string) (Activity, error) } func (f *fakeReader) Deployment(ctx context.Context) (Record, error) { @@ -346,6 +348,7 @@ func (f *fakeNodeTx) RefreshServingReadiness(nodeID string, protocol int) error } type fakeDeploymentTx struct { + hasIncompatibleComputeState func(string) (bool, error) t testing.TB loadDeployment func() (Record, error) loadSnapshot func() (Snapshot, error) @@ -638,6 +641,28 @@ func (f *fakeSessionReader) GetManagedSessionArchive(context.Context, string, st return sessions.ManagedArchive{}, nil } +func (f *fakeDeploymentTx) HasIncompatibleComputeState(version string) (bool, error) { + if f.hasIncompatibleComputeState == nil { + f.t.Fatal("unexpected HasIncompatibleComputeState") + return false, nil + } + return f.hasIncompatibleComputeState(version) +} + +func (f *fakeReader) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { + if f.countComputeReservations == nil { + unexpected(f.t, "CountComputeReservations") + } + return f.countComputeReservations(ctx, installationID) +} + +func (f *fakeReader) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { + if f.countRetainedAllocations == nil { + unexpected(f.t, "CountRetainedAllocations") + } + return f.countRetainedAllocations(ctx, installationID) +} + func (f *fakeReader) PlacementDemand(context.Context, PlacementDemandCursor) ([]PlacementDemand, PlacementDemandCursor, error) { panic("unexpected PlacementDemand") } diff --git a/services/core/internal/deployment/service.go b/services/core/internal/deployment/service.go index 022a0d359..582b67960 100644 --- a/services/core/internal/deployment/service.go +++ b/services/core/internal/deployment/service.go @@ -253,7 +253,7 @@ func (s *Service) SetupForSelection(installationID string, input sandbox.Selecti // suspension returns Core's idle suspension policy for a provider that // declares checkpoint support, and nil for any other provider. func (s *Service) suspension(provider string) (*Suspension, error) { - checkpoint, err := s.registry.SupportsCheckpoint(provider) + checkpoint, err := s.registry.SupportsSuspension(provider) if err != nil || !checkpoint { return nil, err } diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index 5277a9c3a..c15b778ca 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -273,6 +273,12 @@ type Reader interface { // Activity returns the allocation's activity; a missing allocation is // ErrNotFound. Activity(ctx context.Context, allocationID string) (Activity, error) + // CountComputeReservations counts the installation's allocations that + // reserve active compute. + CountComputeReservations(ctx context.Context, installationID string) (int64, error) + // CountRetainedAllocations counts the installation's allocations that + // retain resources. + CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) } // NodeReads loads node authentication facts. @@ -318,6 +324,8 @@ type NodeTx interface { // DeploymentTx is one leased deployment change. type DeploymentTx interface { + // HasIncompatibleComputeState checks all unreleased allocation protocol receipts. + HasIncompatibleComputeState(version string) (bool, error) LoadDeployment() (Record, error) LoadSnapshot() (Snapshot, error) CountResources() (Resources, error) diff --git a/services/core/internal/execution/archive_cancellation_cleanup_test.go b/services/core/internal/execution/archive_cancellation_cleanup_test.go index 3ac684477..0823c6cc0 100644 --- a/services/core/internal/execution/archive_cancellation_cleanup_test.go +++ b/services/core/internal/execution/archive_cancellation_cleanup_test.go @@ -107,7 +107,7 @@ func TestArchiveWaitingCleanupReceiptBarrier(t *testing.T) { } currentCompute := sandbox.Compute{ID: uuid.NewString(), Name: owner.ID + "-g0"} if checkpoint { - state, _ := json.Marshal(runtimeCompute{Current: currentCompute}) + state, _ := json.Marshal(runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: currentCompute}) owner, err = leased.Deployment.SetCompute(t.Context(), owner, "running", state, nil, 0) if err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/deployment_fixture_test.go b/services/core/internal/execution/deployment_fixture_test.go index 9132a9292..0aa389d13 100644 --- a/services/core/internal/execution/deployment_fixture_test.go +++ b/services/core/internal/execution/deployment_fixture_test.go @@ -141,29 +141,31 @@ func (s *strictExecutionStorage) WithDeployment(ctx context.Context, apply func( // strictDeploymentReader runs each set func; any other call fails the test. type strictDeploymentReader struct { - placementDemand func(context.Context, deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) - retainedNativeHistory func(context.Context, deployment.AllocationKey) (bool, error) - t *testing.T - deployment func(context.Context) (deployment.Record, error) - snapshot func(context.Context) (deployment.Snapshot, error) - ownerEpoch func(context.Context) (uint64, error) - allocation func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) - generations func(context.Context, int64) ([]deployment.GenerationRecord, error) - nodes func(context.Context) ([]deployment.NodeRecord, error) - nodeHistory func(context.Context, string, coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) - readNodes func(context.Context, func(deployment.NodeReads) error) error - resetSessions func(context.Context, string, bool) ([]deployment.ResetSession, error) - addressBindings func(context.Context, string) (deployment.AddressBindings, error) - environmentAllocation func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) - credentialAllocations func(context.Context, string) ([]deployment.Allocation, error) - observationSessions func(context.Context, string, int) (deployment.ObservationSessionPage, error) - nodeAllocations func(context.Context, string) ([]deployment.NodeAllocation, error) - nodeOnline func(context.Context, string) (bool, error) - lifecycleNodes func(context.Context) ([]string, error) - lifecycleAllocations func(context.Context, string, string) ([]deployment.Allocation, error) - unallocatedEnvironments func(context.Context, string, string) ([]deployment.UnallocatedEnvironment, error) - lifecyclePlacement func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) - activity func(context.Context, string) (deployment.Activity, error) + countRetainedAllocations func(context.Context, string) (int64, error) + countComputeReservations func(context.Context, string) (int64, error) + placementDemand func(context.Context, deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) + retainedNativeHistory func(context.Context, deployment.AllocationKey) (bool, error) + t *testing.T + deployment func(context.Context) (deployment.Record, error) + snapshot func(context.Context) (deployment.Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) + generations func(context.Context, int64) ([]deployment.GenerationRecord, error) + nodes func(context.Context) ([]deployment.NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) + readNodes func(context.Context, func(deployment.NodeReads) error) error + resetSessions func(context.Context, string, bool) ([]deployment.ResetSession, error) + addressBindings func(context.Context, string) (deployment.AddressBindings, error) + environmentAllocation func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) + credentialAllocations func(context.Context, string) ([]deployment.Allocation, error) + observationSessions func(context.Context, string, int) (deployment.ObservationSessionPage, error) + nodeAllocations func(context.Context, string) ([]deployment.NodeAllocation, error) + nodeOnline func(context.Context, string) (bool, error) + lifecycleNodes func(context.Context) ([]string, error) + lifecycleAllocations func(context.Context, string, string) ([]deployment.Allocation, error) + unallocatedEnvironments func(context.Context, string, string) ([]deployment.UnallocatedEnvironment, error) + lifecyclePlacement func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) + activity func(context.Context, string) (deployment.Activity, error) } func (r *strictDeploymentReader) Deployment(ctx context.Context) (deployment.Record, error) { @@ -345,6 +347,20 @@ func (r *strictDeploymentReader) Activity(ctx context.Context, allocationID stri return r.activity(ctx, allocationID) } +func (r *strictDeploymentReader) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { + if r.countComputeReservations == nil { + return 0, unexpectedDeploymentCall(r.t, "CountComputeReservations") + } + return r.countComputeReservations(ctx, installationID) +} + +func (r *strictDeploymentReader) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { + if r.countRetainedAllocations == nil { + return 0, unexpectedDeploymentCall(r.t, "CountRetainedAllocations") + } + return r.countRetainedAllocations(ctx, installationID) +} + func (r *strictDeploymentReader) PlacementDemand(ctx context.Context, after deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) { if r.placementDemand == nil { return nil, deployment.PlacementDemandCursor{}, unexpectedDeploymentCall(r.t, "PlacementDemand") diff --git a/services/core/internal/execution/environment_admission.go b/services/core/internal/execution/environment_admission.go index 0f34d9f47..18ec95ac5 100644 --- a/services/core/internal/execution/environment_admission.go +++ b/services/core/internal/execution/environment_admission.go @@ -99,7 +99,7 @@ func (w *Worker) submitEnvironmentInputs(ctx context.Context, session sessions.S return nil, err } w.wakeScheduler() - if reservation.State == sessions.EnvironmentInputPending && !reservation.IsInitial { + if reservation.State == sessions.EnvironmentInputPending { w.hintRuntimeWake(ctx, session) } ticker := time.NewTicker(250 * time.Millisecond) diff --git a/services/core/internal/execution/provider_operations_fixture_test.go b/services/core/internal/execution/provider_operations_fixture_test.go index 98e951c04..c1a132c1a 100644 --- a/services/core/internal/execution/provider_operations_fixture_test.go +++ b/services/core/internal/execution/provider_operations_fixture_test.go @@ -17,11 +17,12 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -30,7 +31,7 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { func (*lifecycleOnlySandbox) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*lifecycleOnlySandbox) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*lifecycleOnlySandbox) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*lifecycleOnlySandbox) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (*lifecycleOnlySandbox) Resume(context.Context, sandbox.ResumeRequest) (san func (*lifecycleOnlySandbox) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*lifecycleOnlySandbox) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*lifecycleOnlySandbox) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*lifecycleOnlySandbox) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -57,3 +58,7 @@ func (*lifecycleOnlySandbox) ResumeCompute(context.Context, sandbox.Reference, s func (*lifecycleOnlySandbox) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } + +func (*lifecycleOnlySandbox) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 71567378d..08125b5ab 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -1,9 +1,11 @@ package execution import ( + "bytes" "context" "encoding/json" "errors" + "io" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" @@ -17,20 +19,24 @@ import ( // RuntimeSuspensionPolicy is the idle suspension policy of a provider that // suspends sandboxes. type RuntimeSuspensionPolicy struct { + MaxActive int + MaxRetained int IdleTimeout time.Duration Retention time.Duration } type runtimeCompute struct { - Current sandbox.Compute `json:"current"` - Target *sandbox.Compute `json:"target,omitempty"` - Snapshot *sandbox.SnapshotIdentity `json:"snapshot,omitempty"` - SuspendID string `json:"suspend_id,omitempty"` - RestoreID string `json:"restore_id,omitempty"` - Rollback bool `json:"rollback,omitempty"` + Version string `json:"protocol_version"` + Current sandbox.Compute `json:"current"` + Target *sandbox.Compute `json:"target,omitempty"` + Retained *sandbox.RetainedState `json:"retained,omitempty"` + SuspendID string `json:"suspend_id,omitempty"` + RestoreID string `json:"restore_id,omitempty"` + Rollback bool `json:"rollback,omitempty"` } func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner deployment.Allocation, phase string, state runtimeCompute, until *time.Time) (deployment.Allocation, error) { + state.Version = sandbox.SuspensionStateVersion raw, err := json.Marshal(state) if err != nil { return owner, err @@ -73,7 +79,7 @@ func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment. return err } var state runtimeCompute - if json.Unmarshal(owner.ComputeState, &state) != nil || state.Current.ID == "" { + if decodeRuntimeCompute(owner.ComputeState, &state) != nil || state.Current.ID == "" { return sandbox.ErrOwnership } if owner.SessionDeleted || owner.Expired || owner.State == "cleanup_pending" { @@ -116,6 +122,13 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro if compute.Status != "running" || !compute.BootstrapComplete { return sandbox.ErrComputeUnconfirmed } + renewed, err := p.RenewCompute(ctx, runtimeReference(owner), state.Current) + if err != nil { + return err + } + if sandbox.ValidateComputeResult(state.Current, renewed.Compute) != nil || renewed.Status != "running" || !renewed.BootstrapComplete { + return sandbox.ErrComputeUnconfirmed + } peer, err := authorizedRuntimePeer(ctx, r.sessions, r.registry, owner.DeviceID) if err != nil { return err @@ -181,15 +194,16 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) (err error) { defer func() { err = withObservationOwner(owner, err) }() - result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Snapshot: state.Snapshot, ObserveOnly: observeOnly}) + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Retained: state.Retained, ReconcileOnly: observeOnly}) if err != nil { return err } - if result.Compute.ID != state.Current.ID { - return sandbox.ErrOwnership + validationErr := sandbox.ValidateSuspendResult(sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Retained: state.Retained, ReconcileOnly: observeOnly}, result) + if validationErr != nil && (result.Retained == nil || !errors.Is(validationErr, sandbox.ErrComputeUnconfirmed)) { + return validationErr } - if result.Snapshot == nil { - if !observeOnly || result.SourceStopped || (result.Status != "running" && result.Status != "paused") { + if result.Retained == nil { + if !observeOnly || !result.SuspendSettled || result.ResourcesReleased || (result.Status != "running" && result.Status != "paused") { return sandbox.ErrComputeUnconfirmed } state.Rollback = true @@ -199,18 +213,18 @@ func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.Sandbox } return r.wakeCompute(ctx, p, next, state) } - if result.Snapshot.Compatibility.Validate() != nil { + if owner.NodeID != "" && result.Retained.Compatibility.Validate() != nil { return sandbox.ErrOwnership } - state.Snapshot = result.Snapshot - // Store the verified artifact before any recovery-path kill. Snapshot failure + state.Retained = result.Retained + // Store the verified artifact before any recovery-path kill. Retained failure // or an unknown result cannot silently fall back to a cold Environment. next, err := r.saveCompute(ctx, owner, "suspending", state, owner.ComputeRetainedUntil) if err != nil { return err } owner = next - if !result.SourceStopped { + if validationErr != nil { // Native absence alone does not settle capture or publish the archive. return sandbox.ErrComputeUnconfirmed } @@ -227,7 +241,10 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San if !activity.Busy && !activity.WakeRequested { return nil } - if state.Snapshot == nil || state.Target != nil { + if err := r.computeCapacityForAllocation(ctx, owner); err != nil { + return err + } + if state.Retained == nil || state.Target != nil { return sandbox.ErrOwnership } state.RestoreID = uuid.NewString() @@ -235,7 +252,7 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San if err != nil { return err } - next, err := r.deployment.BeginRestore(ctx, owner, state.Snapshot.Compatibility, raw) + next, err := r.deployment.BeginRestore(ctx, owner, state.Retained.Compatibility, raw) if err != nil { return err } @@ -250,16 +267,19 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San } func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) (err error) { defer func() { err = withObservationOwner(owner, err) }() - if state.Snapshot == nil || state.Snapshot.Compatibility.Validate() != nil || state.Rollback || state.RestoreID == "" { + if state.Retained == nil || state.Rollback || state.RestoreID == "" { return sandbox.ErrOwnership } if state.Target == nil { // NewCompute derives identity without starting native execution. A crash // before this commit can safely repeat derivation on the reserved target. - target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Snapshot) + target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Retained) if err != nil { return err } + if target.Name == "" || target.Generation != state.Current.Generation+1 || target.RestoredFrom == nil || *target.RestoredFrom != *state.Retained { + return sandbox.ErrOwnership + } state.Target = &target next, err := r.saveCompute(ctx, owner, "restoring", state, owner.ComputeRetainedUntil) if err != nil { @@ -267,26 +287,16 @@ func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.Sandbox } owner, observeOnly = next, false } - spec, err := r.workspaceSpecification(ctx, owner.Key(), owner.ID) + workspace, err := r.restoreWorkspace(ctx, owner) if err != nil { return err } - var workspace *workspacefs.Binding - if spec.Workspace != nil { - workspace, err = r.workspaces.GetReady(ctx, owner.TenantID, owner.EnvironmentID, r.config.WorkspaceRequirements, spec.Resources.EnvironmentDiskMiB) - if err == nil && workspace == nil { - err = workspacefs.ErrNotFound - } - if err != nil { - return err - } - } - result, err := p.Resume(ctx, sandbox.ResumeRequest{Workspace: workspace, Reference: runtimeReference(owner), OperationID: state.RestoreID, Snapshot: *state.Snapshot, Target: *state.Target, ObserveOnly: observeOnly}) + result, err := p.Resume(ctx, sandbox.ResumeRequest{Workspace: workspace, Reference: runtimeReference(owner), OperationID: state.RestoreID, Retained: *state.Retained, Target: *state.Target, ReconcileOnly: observeOnly}) if err != nil { return err } if result.RestoreAttemptClosed != "" { - if !result.ClosesRestoreAttempt(sandbox.ResumeRequest{OperationID: state.RestoreID, Snapshot: *state.Snapshot, Target: *state.Target, ObserveOnly: observeOnly}) { + if !result.ClosesRestoreAttempt(sandbox.ResumeRequest{OperationID: state.RestoreID, Retained: *state.Retained, Target: *state.Target, ReconcileOnly: observeOnly}) { return sandbox.ErrOwnership } // Only an exact, durably closed attempt with no native dispatch can be replaced. @@ -297,7 +307,7 @@ func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.Sandbox } return r.restoreCompute(ctx, p, next, state, false) } - if result.Status != "running" || result.Compute.ID == "" { + if result.Status != "running" || !result.BootstrapComplete || sandbox.ValidateComputeResult(*state.Target, result.Compute) != nil { return sandbox.ErrComputeUnconfirmed } state.Current, state.Target = result.Compute, nil @@ -316,3 +326,61 @@ func ignoreComputeAbsent(err error) error { } return err } + +// Node-backed restores reserve capacity atomically in SetCompute. +func (r *runtimeLifecycle) computeCapacityForAllocation(ctx context.Context, owner deployment.Allocation) error { + if owner.NodeID != "" { + return nil + } + return r.computeCapacity(ctx, owner.ProviderKey) +} + +func decodeRuntimeCompute(raw []byte, state *runtimeCompute) error { + d := json.NewDecoder(bytes.NewReader(raw)) + d.DisallowUnknownFields() + if err := d.Decode(state); err != nil { + return err + } + if d.Decode(new(any)) != io.EOF || state.Version != sandbox.SuspensionStateVersion { + return sandbox.ErrOwnership + } + return nil +} + +func (r *runtimeLifecycle) computeCapacity(ctx context.Context, key string) error { + policy := r.config.Suspension + if key != r.config.InstallationID { + return sandbox.ErrOwnership + } + if policy == nil { + return nil + } + count, err := r.reader.CountComputeReservations(ctx, key) + if err != nil { + return err + } + if count >= int64(policy.MaxActive) { + return ErrExecutionUnavailable + } + return nil +} + +// restoreWorkspace follows the allocation's immutable generation, never the +// current deployment's filesystem selection or sizing. +func (r *runtimeLifecycle) restoreWorkspace(ctx context.Context, owner deployment.Allocation) (*workspacefs.Binding, error) { + spec, err := r.workspaceSpecification(ctx, owner.Key(), owner.ID) + if err != nil { + return nil, err + } + if spec.Workspace == nil { + return nil, nil + } + if r.workspaces == nil { + return nil, workspacefs.ErrUnavailable + } + binding, err := r.workspaces.GetReady(ctx, owner.TenantID, owner.EnvironmentID, r.config.WorkspaceRequirements, spec.Resources.EnvironmentDiskMiB) + if err == nil && binding == nil { + return nil, workspacefs.ErrNotFound + } + return binding, err +} diff --git a/services/core/internal/execution/runtime_compute_compatibility_test.go b/services/core/internal/execution/runtime_compute_compatibility_test.go new file mode 100644 index 000000000..06a1e0156 --- /dev/null +++ b/services/core/internal/execution/runtime_compute_compatibility_test.go @@ -0,0 +1,54 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "reflect" + "testing" +) + +// This literal freezes the deployed beta v1 shape independently of current types. +func TestBetaRetainedV1RoundTripPreservesEveryField(t *testing.T) { + const raw = `{"protocol_version":"1","current":{"Generation":4,"Name":"owned-source","ID":"native-source","RestoredFrom":{"Reference":"owned/reference","ID":"native-parent","Data":"{\"version\":1,\"sandbox_id\":\"native-source\"}","OperationID":"prior-operation","SourceGeneration":3,"SourceName":"prior-source","SourceID":"native-parent"}},"target":{"Generation":5,"Name":"owned-target","ID":"","RestoredFrom":{"Reference":"owned/reference","ID":"native-source","Data":"{\"version\":1,\"sandbox_id\":\"native-source\"}","OperationID":"pause-operation","SourceGeneration":4,"SourceName":"owned-source","SourceID":"native-source"}},"retained":{"Reference":"owned/reference","ID":"native-source","Data":"{\"version\":1,\"sandbox_id\":\"native-source\"}","OperationID":"pause-operation","SourceGeneration":4,"SourceName":"owned-source","SourceID":"native-source"},"suspend_id":"pause-operation","restore_id":"resume-operation","rollback":true}` + var state runtimeCompute + if err := decodeRuntimeCompute([]byte(raw), &state); err != nil { + t.Fatal(err) + } + out, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + var before, after any + if json.Unmarshal([]byte(raw), &before) != nil || json.Unmarshal(out, &after) != nil || !reflect.DeepEqual(before, after) { + t.Fatal("beta v1 field loss or rewrite") + } + for _, invalid := range []string{`{}`, `{"protocol_version":"2"}`, `{"protocol_version":"1","snapshot":{}}`, `{"protocol_version":"1","current":{"RestoredFrom":{"Digest":"legacy"}}}`, `{"protocol_version":"1","target":{"RestoredFrom":{"CheckpointRoot":"legacy"}}}`, raw + `{}`} { + if decodeRuntimeCompute([]byte(invalid), new(runtimeCompute)) == nil { + t.Fatal("incompatible receipt accepted") + } + } +} + +func TestReleasedDirectAllocationStillChecksCapacity(t *testing.T) { + for _, limit := range []string{"active", "retained"} { + t.Run(limit, func(t *testing.T) { + r := runtimeLifecycle{sessions: workspaceEnvironmentReader{}, config: RuntimeProvider{Mode: "direct", InstallationID: "fixture", Generation: 1, Suspension: &RuntimeSuspensionPolicy{MaxActive: 1, MaxRetained: 2}}, reader: &strictDeploymentReader{t: t, + environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + return deployment.Allocation{State: "released"}, nil + }, + countComputeReservations: func(context.Context, string) (int64, error) { + if limit == "active" { + return 1, nil + } + return 0, nil + }, + countRetainedAllocations: func(context.Context, string) (int64, error) { return 2, nil }, + }} + if _, err := r.provision(t.Context(), "tenant", "environment", "fixture"); !errors.Is(err, ErrExecutionUnavailable) { + t.Fatal("released owner bypassed direct capacity", err) + } + }) + } +} diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index 4f836dc6a..d27af6fd3 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -6,6 +6,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" @@ -27,7 +28,7 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxPro } // This idempotent control signal is fenced by guest PID/start time and the // suspension token. It cannot execute or replay an agent request. - result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", "/run/oac/daemon-suspend.json", "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) + result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", runtimebootstrap.SuspendControlFile, "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) if err != nil { return err } @@ -57,8 +58,8 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxPro } // The artifact has been consumed. Never restore it after this generation // admits work, even if garbage collection or the final database commit fails. - if state.Snapshot != nil { - if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + if state.Retained != nil { + if err := ignoreComputeAbsent(p.DeleteRetained(ctx, runtimeReference(owner), *state.Retained)); err != nil { return err } } @@ -78,8 +79,8 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Sandbox if err := r.lease.CheckOwnership(ctx); err != nil { return err } - if owner.ComputePhase == "suspended" && state.Snapshot != nil && owner.NodeID != "" { - next, err := r.deployment.RelocateCleanup(ctx, owner, state.Snapshot.Compatibility) + if owner.ComputePhase == "suspended" && state.Retained != nil && owner.NodeID != "" { + next, err := r.deployment.RelocateCleanup(ctx, owner, state.Retained.Compatibility) if err != nil { return err } @@ -93,13 +94,13 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Sandbox } // An uncommitted artifact is found by its persisted attempt, never a directory // glob. The helper's allocation lock also waits for an earlier unknown call. - if owner.ComputePhase == "suspending" && state.Snapshot == nil { - result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, ObserveOnly: true}) + if owner.ComputePhase == "suspending" && state.Retained == nil { + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, ReconcileOnly: true}) if err != nil && !errors.Is(err, sandbox.ErrNotFound) { return err } if err == nil { - state.Snapshot = result.Snapshot + state.Retained = result.Retained } } if state.Target != nil { @@ -112,8 +113,8 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Sandbox return err } } - if state.Snapshot != nil { - if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + if state.Retained != nil { + if err := ignoreComputeAbsent(p.DeleteRetained(ctx, runtimeReference(owner), *state.Retained)); err != nil { return err } } diff --git a/services/core/internal/execution/runtime_fresh_hint_test.go b/services/core/internal/execution/runtime_fresh_hint_test.go new file mode 100644 index 000000000..fec4b3817 --- /dev/null +++ b/services/core/internal/execution/runtime_fresh_hint_test.go @@ -0,0 +1,194 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" + "github.com/google/uuid" +) + +type freshHintProvider struct { + sandbox.SandboxProvider + creates atomic.Int32 + created chan struct{} +} + +type hintSessionReader struct { + sessions.Reader + environment sessions.Environment +} + +func (r hintSessionReader) GetSessionEnvironment(ctx context.Context, _, _ string) (sessions.Environment, error) { + return r.environment, ctx.Err() +} + +func TestFreshHintRoutesAndPreservesLifecycleGuards(t *testing.T) { + for _, scenario := range []string{"direct", "node", "closed", "switching", "cancelled", "self_hosted", "released_placement", "lookup_failed", "suspended"} { + t.Run(scenario, func(t *testing.T) { + m := testRuntimeManager(t) + m.setupGate = make(chan struct{}, 1) + m.config = RuntimeProvider{InstallationID: uuid.NewString(), ProviderKind: "e2b", Mode: "direct", Provider: &freshHintProvider{}} + nodeID := "" + if scenario == "node" || scenario == "released_placement" { + m.config.ProviderKind, m.config.Mode = "docker", "nodes" + nodeID = uuid.NewString() + } + node, err := m.node(nodeID) + if err != nil { + t.Fatal(err) + } + environment := sessions.Environment{ID: uuid.NewString(), Configuration: json.RawMessage(`{"type":"openai_hosted"}`)} + if scenario == "self_hosted" { + environment.Configuration = json.RawMessage(`{"type":"self_hosted"}`) + } + if scenario == "suspended" { + environment.Initialization = "complete" + m.config.Suspension = &RuntimeSuspensionPolicy{} + } + reader := &strictDeploymentReader{t: t, + environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + if scenario == "lookup_failed" { + return deployment.Allocation{}, errors.New("database unavailable") + } + if scenario == "suspended" { + return deployment.Allocation{ProviderKey: m.config.InstallationID, State: "running", CreateSettled: true, ComputePhase: "suspended"}, nil + } + return deployment.Allocation{}, deployment.ErrNotFound + }, + lifecyclePlacement: func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) { + return deployment.LifecyclePlacement{Provider: m.config.ProviderKind, Mode: m.config.Mode, PlacementNodeID: nodeID, PlacementReleased: scenario == "released_placement"}, nil + }, + } + m.deploymentService, _ = deploymentOperations(t, &strictDeploymentStorage{t: t}, reader, &strictExecutionStorage{t: t}) + worker := &Worker{runtimes: m, dispatcher: &Dispatcher{SessionsReader: hintSessionReader{environment: environment}, DeploymentReader: reader}} + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + switch scenario { + case "closed": + m.closed = true + case "switching": + m.switching = true + case "cancelled": + cancel() + } + worker.hintRuntimeWake(ctx, sessions.Session{TenantID: uuid.NewString(), ID: uuid.NewString()}) + want := 0 + if scenario == "direct" || scenario == "node" || scenario == "suspended" { + want = 1 + } + if got := len(node.lifecycle.wakeHints); got != want { + t.Fatalf("hints = %d, want %d", got, want) + } + }) + } +} + +func (p *freshHintProvider) Create(_ context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { + p.creates.Add(1) + select { + case p.created <- struct{}{}: + default: + } + return sandbox.Info{Reference: b.Reference, ProviderID: b.AllocationID, State: "running", BootstrapComplete: true, CreateSettled: true}, nil +} + +func TestFreshEnvironmentHintProvisionsWithoutMaintenanceTick(t *testing.T) { + for _, mode := range []string{"create", "create_initial", "recovered_input", "recovered_initial"} { + t.Run(mode, func(t *testing.T) { + owner, deployments, reader, pool := resetManagerDB(t, nil) + installation := initializeE2BDeployment(t, owner) + sessionReader, sessionService := testSessions(t, pool, pgtest.CredentialKey(t)) + provider := &freshHintProvider{created: make(chan struct{}, 1)} + m := testRuntimeManager(t) + m.setupGate = make(chan struct{}, 1) + m.sessions, m.sessionExecution = sessionReader, owner.Sessions + m.deployment, m.deploymentService, m.deploymentReader = owner.Deployment, deployments, reader + m.lease, m.registry = owner.Lease, runtimegateway.NewRegistry() + m.config = RuntimeProvider{InstallationID: installation, Generation: 1, ProviderKind: "e2b", Mode: "direct", CoreURL: fixturePublicURL + "/api/v1", Provider: provider} + worker := &Worker{lease: owner.Lease, runtimes: m, dispatcher: &Dispatcher{Sessions: sessionService, SessionsReader: sessionReader, DeploymentReader: reader, notifications: &executionNotifications{}}, scheduleWake: make(chan struct{}, 1)} + node, err := m.node("") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + done, scans := make(chan error, 1), make(chan struct{}, 4) + // No value is ever sent to ticks: provisioning must come from a hint. + ticks := make(chan time.Time) + go func() { + done <- runRuntimeMaintenance(ctx, ticks, node.lifecycle.wakeHints, func(ctx context.Context) error { + err := node.lifecycle.reconcile(ctx) + scans <- struct{}{} + return err + }) + }() + t.Cleanup(func() { cancel(); <-done }) + <-scans // Startup scan finishes before any Session is committed. + tenant := uuid.NewString() + input := sessions.CreateSession{Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted"}}`), ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ExecutionSourceSession} + messages := []sessions.Input{{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"fixture"}]}]}`)}} + if mode == "recovered_initial" || mode == "create_initial" { + input.InitialInputs = messages + } + var creation sessions.Creation + switch mode { + case "create", "create_initial": + // Ordinary and streaming HTTP creation share this Worker entrypoint. + creation, err = worker.CreateSession(ctx, tenant, input) + case "recovered_input", "recovered_initial": + creation, err = sessionService.CreateSession(ctx, tenant, input) + } + session := creation.Session + if err != nil { + t.Fatal(err) + } + if mode == "recovered_input" || mode == "recovered_initial" { + key := uuid.NewString() + if mode == "recovered_initial" { + if err := pool.QueryRow(ctx, "SELECT idempotency_key FROM environment_input_reservations WHERE session_id=$1 AND is_initial", session.ID).Scan(&key); err != nil { + t.Fatal(err) + } + } + inputDone := make(chan error, 20) + inputCtx, stopInput := context.WithCancel(ctx) + for range 20 { + go func() { + _, inputErr := worker.submitEnvironmentInputs(inputCtx, session, key, messages) + inputDone <- inputErr + }() + } + t.Cleanup(func() { + stopInput() + for range 20 { + <-inputDone + } + }) + } + select { + case <-provider.created: + case <-time.After(2 * time.Second): + t.Fatal("committed Environment waited for a maintenance tick") + } + var requests sync.WaitGroup + for range 20 { + requests.Go(func() { worker.hintRuntimeWake(ctx, session) }) + } + requests.Wait() + <-scans // Wait for the hinted scan to settle its durable allocation. + if got := provider.creates.Load(); got != 1 { + t.Fatalf("concurrent hints issued %d Creates", got) + } + }) + } +} diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index a4a14cc11..c19c7f21c 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -103,12 +103,9 @@ func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway. if copied.ProviderKind == "" || (copied.Mode != string(sandbox.DeploymentNodes) && copied.Mode != string(sandbox.DeploymentDirect)) { return RuntimeProvider{}, sandbox.ErrInvalid } - if copied.Mode == string(sandbox.DeploymentDirect) && copied.Suspension != nil { - return RuntimeProvider{}, sandbox.ErrInvalid - } if config.Suspension != nil { policy := *config.Suspension - if !sandbox.SupportsCheckpoint(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second { + if !sandbox.SupportsSuspension(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second || (copied.Mode == string(sandbox.DeploymentDirect) && (policy.MaxActive < 1 || policy.MaxRetained < policy.MaxActive)) { return RuntimeProvider{}, sandbox.ErrInvalid } copied.Suspension = &policy @@ -182,6 +179,10 @@ func (w *Worker) ProvisionEnvironment(ctx context.Context, tenant, environment, // provision runs under the lifecycle gate and uses the durable one-shot receipt. func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, providerKey string) (deployment.Allocation, error) { provider := r.config.Provider + // The lifecycle lane, not provider configuration, owns node identity. + if (r.nodeID == "") != (r.config.Mode == string(sandbox.DeploymentDirect)) { + return deployment.Allocation{}, sandbox.ErrOwnership + } if providerKey != r.config.InstallationID { return deployment.Allocation{}, sandbox.ErrInvalid } @@ -193,6 +194,27 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if err != nil || placement.Type != "openai_hosted" { return deployment.Allocation{}, sandbox.ErrInvalid } + key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} + existing, lookupErr := r.reader.EnvironmentAllocation(ctx, key) + if errors.Is(lookupErr, deployment.ErrNotFound) || (lookupErr == nil && existing.State == "released") { + if r.config.Generation == 0 { + return deployment.Allocation{}, ErrExecutionUnavailable + } + if err := r.computeFreshCapacity(ctx, providerKey); err != nil { + return deployment.Allocation{}, err + } + if policy := r.config.Suspension; policy != nil && (r.config.ProviderKind == "" || r.config.Mode == "direct") { + count, err := r.reader.CountRetainedAllocations(ctx, providerKey) + if err != nil { + return deployment.Allocation{}, err + } + if count >= int64(policy.MaxRetained) { + return deployment.Allocation{}, ErrExecutionUnavailable + } + } + } else if lookupErr != nil { + return deployment.Allocation{}, lookupErr + } spec, err := r.workspaceSpecification(ctx, deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment}, "") if err != nil { return deployment.Allocation{}, err @@ -226,7 +248,6 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p return existing, sandbox.ErrOwnership } } - key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} secret := make([]byte, 32) if _, err := rand.Read(secret); err != nil { return deployment.Allocation{}, err @@ -245,9 +266,13 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if err := r.lease.CheckOwnership(ctx); err != nil { return owner, err } + session, err := r.sessions.GetSession(ctx, tenant, environmentValue.SessionID) + if err != nil { + return deployment.Allocation{}, err + } info, err := provider.Create(ctx, sandbox.Bootstrap{ Reference: runtimeReference(owner), SessionID: owner.SessionID, DeviceID: owner.DeviceID, - Workspace: workspace, CoreURL: r.config.CoreURL, Credential: token, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, + Workspace: workspace, Harness: session.Engine, CoreURL: r.config.CoreURL, Credential: token, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, }) if info.Reference == runtimeReference(owner) && info.CreateSettled && info.State == "absent" { record, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) @@ -454,6 +479,14 @@ func (w *Worker) runManagedRuntimes(ctx context.Context) error { return w.runtimes.run(ctx) } +// Manager deployments reserve capacity with Session placement before provisioning. +func (r *runtimeLifecycle) computeFreshCapacity(ctx context.Context, key string) error { + if r.config.ProviderKind != "" && r.config.Mode != "direct" { + return nil + } + return r.computeCapacity(ctx, key) +} + // workspaceSpecification reads the immutable generation selected for this // Environment. A restore also fences the exact allocation before native I/O. func (r *runtimeLifecycle) workspaceSpecification(ctx context.Context, key deployment.AllocationKey, allocationID string) (sandbox.DeploymentSpec, error) { diff --git a/services/core/internal/execution/runtime_observation_test.go b/services/core/internal/execution/runtime_observation_test.go index cc13c2f0c..aa5aecced 100644 --- a/services/core/internal/execution/runtime_observation_test.go +++ b/services/core/internal/execution/runtime_observation_test.go @@ -18,7 +18,7 @@ func (p *observationCaptureProvider) Suspend(_ context.Context, q sandbox.Suspen if p.captureError != nil { return sandbox.ComputeState{}, p.captureError } - return sandbox.ComputeState{Compute: q.Source, Snapshot: &sandbox.SnapshotIdentity{ID: "captured", Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}, Status: "suspended", SourceStopped: p.killError == nil}, nil + return sandbox.ComputeState{Compute: q.Source, Retained: &sandbox.RetainedState{Reference: "capture", Data: "opaque", OperationID: q.OperationID, SourceID: q.Source.ID, SourceName: q.Source.Name, SourceGeneration: q.Source.Generation, ID: "captured", Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}, Status: "suspended", BootstrapComplete: true, SuspendSettled: true, ResourcesReleased: p.killError == nil}, nil } func TestComputeFailureObservationUsesCommittedReceipt(t *testing.T) { @@ -48,7 +48,7 @@ func TestComputeFailureObservationUsesCommittedReceipt(t *testing.T) { if err != nil { t.Fatal(err) } - state := runtimeCompute{Current: sandbox.Compute{ID: "source", Name: "source"}, SuspendID: "attempt"} + state := runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: sandbox.Compute{ID: "source", Name: "source"}, SuspendID: "attempt"} until := time.Now().Add(time.Hour) suspending, err := r.saveCompute(t.Context(), original, "suspending", state, &until) if err != nil { diff --git a/services/core/internal/execution/runtime_replacement_test.go b/services/core/internal/execution/runtime_replacement_test.go index 1d8c2c31f..6e5aa3212 100644 --- a/services/core/internal/execution/runtime_replacement_test.go +++ b/services/core/internal/execution/runtime_replacement_test.go @@ -49,7 +49,7 @@ func (p *retentionProvider) KillCompute(context.Context, sandbox.Reference, sand p.kills++ return p.killError } -func (p *retentionProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { +func (p *retentionProvider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { p.snapshots++ return nil } @@ -69,7 +69,7 @@ func TestCapacityPressurePreservesSuspendedSnapshot(t *testing.T) { {`UPDATE runtime_nodes SET max_active=1,max_retained=1 WHERE id=$1`, owner.NodeID}, {`UPDATE environments SET initialization='complete',status='disconnected' WHERE id=$1`, owner.EnvironmentID}, {`UPDATE devices SET supported_agent_kinds='[{"kind":"codex","available":true,"capabilities":{"retained_native_history":true}}]' WHERE id=$1`, owner.DeviceID}, - {`UPDATE runtime_allocations SET state='running',compute_phase='suspended',compute_state='{"current":{"id":"old-compute"},"snapshot":{"id":"old-snapshot"}}',compute_retained_until=clock_timestamp()+interval '24 hours' WHERE id=$1`, owner.ID}, + {`UPDATE runtime_allocations SET state='running',compute_phase='suspended',compute_state='{"protocol_version":"1","current":{"ID":"old-compute"},"retained":{"ID":"old-snapshot","Data":"opaque","Compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()+interval '24 hours' WHERE id=$1`, owner.ID}, } { if _, err := fixture.pool.Exec(t.Context(), statement.sql, statement.arg); err != nil { t.Fatal(err) @@ -132,7 +132,7 @@ func TestExpiredRetainedComputePreservesSessionAndPendingInput(t *testing.T) { if _, err = fixture.pool.Exec(t.Context(), `UPDATE session_devices SET native_session_id='retained-native-session' WHERE device_id=$1`, owner.DeviceID); err != nil { t.Fatal(err) } - compute := runtimeCompute{Current: sandbox.Compute{ID: "old-compute"}, Snapshot: &sandbox.SnapshotIdentity{Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}, ID: "old-snapshot"}} + compute := runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: sandbox.Compute{ID: "old-compute"}, Retained: &sandbox.RetainedState{ID: "old-snapshot", Data: "native-state"}} raw, _ := json.Marshal(compute) if _, err = fixture.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspending',compute_state=$2,compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID, raw); err != nil { t.Fatal(err) @@ -213,7 +213,7 @@ func TestRetainedEnvironmentDemandRecreatesCompute(t *testing.T) { {`UPDATE environments SET initialization='complete',status='disconnected' WHERE id=$1`, owner.EnvironmentID}, {`UPDATE devices SET supported_agent_kinds='[{"kind":"codex","available":true,"capabilities":{"retained_native_history":true}}]' WHERE id=$1`, owner.DeviceID}, {`UPDATE session_devices SET native_session_id='retained-native-session' WHERE device_id=$1`, owner.DeviceID}, - {`UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"current":{"id":"old-compute"},"snapshot":{"id":"old-snapshot","Compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID}, + {`UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"protocol_version":"1","current":{"ID":"old-compute"},"retained":{"ID":"old-snapshot","Data":"native-state","Compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID}, } { if _, err = fixture.pool.Exec(t.Context(), statement.q, statement.arg); err != nil { t.Fatal(err) @@ -462,7 +462,7 @@ func TestRestoreUsesAllocationGenerationStorageInsteadOfCachedLane(t *testing.T) } r.config.Workspace = target.Workspace r.config.Resources = target.Resources - state := runtimeCompute{Target: &sandbox.Compute{ID: "replacement-compute", Generation: 2}, Snapshot: &sandbox.SnapshotIdentity{Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}, ID: "snapshot"}, RestoreID: uuid.NewString()} + state := runtimeCompute{Target: &sandbox.Compute{ID: "replacement-compute", Generation: 2}, Retained: &sandbox.RetainedState{ID: "snapshot", Data: "native-state"}, RestoreID: uuid.NewString()} if err = r.restoreCompute(t.Context(), provider, owner, state, false); !errors.Is(err, stop) { t.Fatal(err) } @@ -484,3 +484,27 @@ func TestRestoreUsesAllocationGenerationStorageInsteadOfCachedLane(t *testing.T) }) } } + +func TestSuspendedAllocationDemandDoesNotColdReplaceWriter(t *testing.T) { + for _, external := range []bool{false, true} { + t.Run(map[bool]string{false: "owned", true: "external"}[external], func(t *testing.T) { + provider := &retentionProvider{} + f := workspaceSettlementFixtureWithSetup(t, provider, workspaceNodeSetup(t, external)) + r, s := f.lifecycle, f.session + owner, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + if _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, owner.ID); err != nil { + t.Fatal(err) + } + if _, err = f.sessions.ReserveEnvironmentInput(t.Context(), s.TenantID, s.ID, "owned-pause", []sessions.Input{{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"resume"}]}]}`)}}); err != nil { + t.Fatal(err) + } + replay, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if err != nil || replay.ID != owner.ID || replay.DeviceID != owner.DeviceID || !replay.Replayed || replay.ComputePhase != "suspended" || provider.creates != 1 || provider.kills != 0 { + t.Fatal("owned suspended writer was cold replaced", replay, err) + } + }) + } +} diff --git a/services/core/internal/execution/runtime_restore_attempt_test.go b/services/core/internal/execution/runtime_restore_attempt_test.go index c18833796..86feedce4 100644 --- a/services/core/internal/execution/runtime_restore_attempt_test.go +++ b/services/core/internal/execution/runtime_restore_attempt_test.go @@ -20,13 +20,13 @@ type restoreAttemptProvider struct { stop error } -func (p *restoreAttemptProvider) NewCompute(_ context.Context, _ sandbox.Reference, generation uint64, snapshot *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *restoreAttemptProvider) NewCompute(_ context.Context, _ sandbox.Reference, generation uint64, snapshot *sandbox.RetainedState) (sandbox.Compute, error) { p.derived++ return sandbox.Compute{Generation: generation, Name: "target", RestoredFrom: snapshot}, nil } func (p *restoreAttemptProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { p.requests = append(p.requests, q) - if !q.ObserveOnly { + if !q.ReconcileOnly { return sandbox.ComputeState{}, p.stop } if p.outcome == "unknown" { @@ -54,8 +54,8 @@ func TestRestoreAttemptRecoveryUsesExactClosedEvidence(t *testing.T) { if err != nil { t.Fatal(err) } - snapshot := &sandbox.SnapshotIdentity{ID: "snapshot", Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}} - state := runtimeCompute{Current: sandbox.Compute{ID: "source", Name: "source", Generation: 1}, Snapshot: snapshot, RestoreID: uuid.NewString()} + snapshot := &sandbox.RetainedState{ID: "snapshot", Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}} + state := runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: sandbox.Compute{ID: "source", Name: "source", Generation: 1}, Retained: snapshot, RestoreID: uuid.NewString()} if outcome != "unsent_identity" { state.Target = &sandbox.Compute{Name: "target", Generation: 2, RestoredFrom: snapshot} } @@ -81,11 +81,11 @@ func TestRestoreAttemptRecoveryUsesExactClosedEvidence(t *testing.T) { } switch outcome { case "unsent_identity": - if !errors.Is(err, stop) || provider.derived != 1 || len(provider.requests) != 1 || provider.requests[0].ObserveOnly || persisted.Target == nil || persisted.RestoreID != state.RestoreID { + if !errors.Is(err, stop) || provider.derived != 1 || len(provider.requests) != 1 || provider.requests[0].ReconcileOnly || persisted.Target == nil || persisted.RestoreID != state.RestoreID { t.Fatalf("unsent intent not safely derived: %v %#v", err, provider.requests) } case "closed": - if !errors.Is(err, stop) || len(provider.requests) != 2 || !provider.requests[0].ObserveOnly || provider.requests[1].ObserveOnly || persisted.RestoreID == state.RestoreID || provider.requests[1].OperationID != persisted.RestoreID { + if !errors.Is(err, stop) || len(provider.requests) != 2 || !provider.requests[0].ReconcileOnly || provider.requests[1].ReconcileOnly || persisted.RestoreID == state.RestoreID || provider.requests[1].OperationID != persisted.RestoreID { t.Fatalf("closed attempt not replaced durably: %v %#v", err, provider.requests) } default: @@ -133,7 +133,7 @@ func TestCheckpointTransferRoutesBeforeTargetIO(t *testing.T) { if err = r.deployments.Heartbeat(t.Context(), targetNode, connection, view.OwnerEpoch, deployment.NodeHealth{ProviderReady: true}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &compat}}); err != nil { t.Fatal(err) } - state := runtimeCompute{Current: sandbox.Compute{ID: "source", Name: "source", Generation: 1}, Snapshot: &sandbox.SnapshotIdentity{ID: "snapshot", Compatibility: compat}} + state := runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: sandbox.Compute{ID: "source", Name: "source", Generation: 1}, Retained: &sandbox.RetainedState{ID: "snapshot", Compatibility: compat}} raw, _ := json.Marshal(state) if _, err = f.pool.Exec(t.Context(), `UPDATE environments SET initialization='complete',status='disconnected' WHERE id=$1`, owner.EnvironmentID); err != nil { t.Fatal(err) @@ -181,7 +181,7 @@ func TestCheckpointTransferRoutesBeforeTargetIO(t *testing.T) { r.nodeID = targetNode err = r.observe(t.Context(), moved) if operation == "wake" { - if !errors.Is(err, stop) || provider.derived != 1 || len(provider.requests) != 1 || provider.requests[0].ObserveOnly { + if !errors.Is(err, stop) || provider.derived != 1 || len(provider.requests) != 1 || provider.requests[0].ReconcileOnly { t.Fatal("target failed to use committed unsent restore", err) } } else { @@ -205,11 +205,11 @@ type cleanupReceiptProvider struct { } func (p *cleanupReceiptProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { - p.observed = q.ObserveOnly - if !q.ObserveOnly { + p.observed = q.ReconcileOnly + if !q.ReconcileOnly { return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed } - return sandbox.ComputeState{Compute: q.Source, Status: "unknown", Snapshot: &sandbox.SnapshotIdentity{ID: "owned-corrupt-archive", OperationID: q.OperationID, Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}}, nil + return sandbox.ComputeState{Compute: q.Source, Status: "unknown", Retained: &sandbox.RetainedState{ID: "owned-corrupt-archive", OperationID: q.OperationID, Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}}, nil } func TestLostCaptureReceiptCanStillBeDeleted(t *testing.T) { p := &cleanupReceiptProvider{} @@ -219,7 +219,7 @@ func TestLostCaptureReceiptCanStillBeDeleted(t *testing.T) { if err != nil { t.Fatal(err) } - state := runtimeCompute{Current: sandbox.Compute{ID: "source", Name: "source"}, SuspendID: uuid.NewString()} + state := runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: sandbox.Compute{ID: "source", Name: "source"}, SuspendID: uuid.NewString()} raw, _ := json.Marshal(state) if _, err = f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspending',compute_state=$2,compute_revision=4,compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID, raw); err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/runtime_retained_queries_test.go b/services/core/internal/execution/runtime_retained_queries_test.go new file mode 100644 index 000000000..e2430706b --- /dev/null +++ b/services/core/internal/execution/runtime_retained_queries_test.go @@ -0,0 +1,84 @@ +package execution + +import ( + "encoding/json" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +// Exercise the SQL projections with the same serialized envelope Core persists. +// Adapter-private data deliberately contains legacy-looking keys: queries must +// read only the shared retained envelope, never the opaque payload. +func TestRuntimeRetainedStateFeedsCheckpointDemandAndNodeCounts(t *testing.T) { + f := newWorkspaceSettlementFixture(t, &retentionProvider{}) + r, session := f.lifecycle, f.session + owner, err := r.provision(t.Context(), session.TenantID, session.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + compatibility := sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"} + source := sandbox.Compute{Generation: 1, Name: "source", ID: "native-source"} + retained := sandbox.RetainedState{Reference: "fixture/retained", ID: "native-retained", Data: `{"snapshot":{"Compatibility":{"artifact_domain":"private","execution_class":"private"}}}`, OperationID: "suspend-operation", SourceGeneration: source.Generation, SourceName: source.Name, SourceID: source.ID, Compatibility: compatibility} + for _, tc := range []struct { + name string + wake, expired, released, noCompatibility, legacy bool + demand, count int + }{ + {name: "waiting", wake: true, demand: 1, count: 1}, + {name: "idle", count: 1}, + {name: "expired", wake: true, expired: true, count: 1}, + {name: "released", wake: true, released: true}, + {name: "no_compatibility", wake: true, noCompatibility: true, count: 1}, + {name: "legacy_shape", wake: true, legacy: true}, + } { + t.Run(tc.name, func(t *testing.T) { + state := runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: source, Retained: &retained, SuspendID: retained.OperationID} + copied := retained + if tc.noCompatibility { + copied.Compatibility = sandbox.CheckpointCompatibility{} + state.Retained = &copied + } + raw, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if tc.legacy { + var envelope map[string]json.RawMessage + if err := json.Unmarshal(raw, &envelope); err != nil { + t.Fatal(err) + } + envelope["snapshot"] = envelope["retained"] + delete(envelope, "retained") + raw, err = json.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + } + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET state=CASE WHEN $3 THEN 'released' ELSE 'running' END,released_at=CASE WHEN $3 THEN clock_timestamp() END,compute_phase='suspended',compute_state=$2,compute_retained_until=clock_timestamp()+CASE WHEN $4 THEN interval '-1 hour' ELSE interval '1 hour' END,compute_wake_requested=$5 WHERE id=$1`, owner.ID, raw, tc.released, tc.expired, tc.wake); err != nil { + t.Fatal(err) + } + q := sqlc.New(f.pool) + demands, err := q.ListWaitingCheckpointRestores(t.Context()) + if err != nil || len(demands) != tc.demand { + t.Fatalf("checkpoint demand=%d want=%d: %v", len(demands), tc.demand, err) + } + if len(demands) == 1 { + var got sandbox.CheckpointCompatibility + if err := json.Unmarshal(demands[0].Checkpoint, &got); err != nil || got != compatibility { + t.Fatalf("shared compatibility lost: %+v %v", got, err) + } + } + nodes, err := q.ListRuntimeNodes(t.Context(), pgtype.UUID{Bytes: uuid.MustParse(owner.NodeID), Valid: true}) + if err != nil || len(nodes) != 1 { + t.Fatalf("node projection: %d %v", len(nodes), err) + } + if nodes[0].Snapshots != int64(tc.count) { + t.Fatalf("retained node count=%d want=%d", nodes[0].Snapshots, tc.count) + } + }) + } +} diff --git a/services/core/internal/execution/runtime_wake_hint.go b/services/core/internal/execution/runtime_wake_hint.go index ed7c9f1d6..b25fb4d03 100644 --- a/services/core/internal/execution/runtime_wake_hint.go +++ b/services/core/internal/execution/runtime_wake_hint.go @@ -2,14 +2,15 @@ package execution import ( "context" + "errors" "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -// A hint only accelerates observation of an already committed input. Lookup or -// delivery failure leaves that input for the normal maintenance scan. +// Hints only accelerate lifecycle work for committed Environments and inputs. +// Lookup or delivery failure leaves that work for the normal maintenance scan. func (w *Worker) hintRuntimeWake(ctx context.Context, session sessions.Session) { r := w.runtimes if r == nil { @@ -19,16 +20,40 @@ func (w *Worker) hintRuntimeWake(ctx context.Context, session sessions.Session) config := r.config available := !r.closed && !r.switching r.mu.Unlock() - if !available || config.Suspension == nil || r.ctx.Err() != nil { + if !available || r.ctx.Err() != nil { return } lookup, cancel := context.WithTimeout(ctx, time.Second) defer cancel() environment, err := w.dispatcher.SessionsReader.GetSessionEnvironment(lookup, session.TenantID, session.ID) - if err != nil || environment.Initialization != "complete" { + if err != nil { + return + } + placement, err := parseEnvironmentPlacement(environment.Configuration) + if err != nil || placement.Type != "openai_hosted" { return } owner, err := w.dispatcher.DeploymentReader.EnvironmentAllocation(lookup, deployment.AllocationKey{TenantID: session.TenantID, EnvironmentID: environment.ID}) + if errors.Is(err, deployment.ErrNotFound) { + // Placement exists before allocation. Route through its lifecycle owner; + // the scan still enforces lease, capacity and one-shot Create receipts. + nodeID, err := r.deploymentService.LifecycleNode(lookup, session.TenantID, environment.ID) + if err != nil || lookup.Err() != nil { + return + } + node, err := r.node(nodeID) + if err != nil { + return + } + select { + case node.lifecycle.wakeHints <- struct{}{}: + default: + } + return + } + if config.Suspension == nil || environment.Initialization != "complete" { + return + } if err != nil || owner.ProviderKey != config.InstallationID || owner.State != "running" || !owner.CreateSettled || owner.SessionDeleted || owner.Expired { return diff --git a/services/core/internal/execution/runtime_workspace_settlement_test.go b/services/core/internal/execution/runtime_workspace_settlement_test.go index 6207c92df..5eef8ee81 100644 --- a/services/core/internal/execution/runtime_workspace_settlement_test.go +++ b/services/core/internal/execution/runtime_workspace_settlement_test.go @@ -123,7 +123,7 @@ func workspaceSettlementFixtureWithSetup(t *testing.T, provider sandbox.SandboxP control := &settlementWorkspaceControl{configuration: configuration} filesystems := workspaces.NewExecution(storage, writer, settlementWorkspaceControls{control}, lease) declaration := control.Declaration() - lifecycle := &runtimeLifecycle{nodeID: nodeID, registry: runtimegateway.NewRegistry(), sessions: sessionReader, sessionExecution: owner.Sessions, deployment: operations, deployments: deployments, reader: reader, lease: lease, workspaces: filesystems, config: RuntimeProvider{InstallationID: installation, Provider: provider, Workspace: &declaration, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} + lifecycle := &runtimeLifecycle{nodeID: nodeID, registry: runtimegateway.NewRegistry(), sessions: sessionReader, sessionExecution: owner.Sessions, deployment: operations, deployments: deployments, reader: reader, lease: lease, workspaces: filesystems, config: RuntimeProvider{InstallationID: installation, Mode: "nodes", Generation: 1, Provider: provider, Workspace: &declaration, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} return workspaceSettlementFixture{pool: pool, lifecycle: lifecycle, sessions: service, storage: storage, control: control, session: session} } diff --git a/services/core/internal/execution/runtime_workspace_test.go b/services/core/internal/execution/runtime_workspace_test.go index cd5d1cc1a..70210a6f6 100644 --- a/services/core/internal/execution/runtime_workspace_test.go +++ b/services/core/internal/execution/runtime_workspace_test.go @@ -97,7 +97,7 @@ func TestRuntimeWorkspaceConvergesBeforeComputeReservation(t *testing.T) { return deployment.LifecyclePlacement{Specification: []byte(`{"workspace":{"attachment":"host_directory"}}`)}, nil }, environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { return deployment.Allocation{}, deployment.ErrNotFound - }}, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{InstallationID: uuid.NewString(), Workspace: &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory}, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} + }}, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{Mode: "direct", Generation: 1, InstallationID: uuid.NewString(), Workspace: &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory}, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} _, err := r.provision(t.Context(), f.record.Reference.TenantID, f.record.Reference.EnvironmentID, r.config.InstallationID) if fail { if !errors.Is(err, workspacefs.ErrUnavailable) || reserved { @@ -153,9 +153,11 @@ func TestOwnedGenerationDoesNotAdoptLaterFilesystemSelection(t *testing.T) { _, operations := deploymentOperations(t, &strictDeploymentStorage{t: t}, &strictDeploymentReader{t: t}, &strictExecutionStorage{t: t, withReservation: func(context.Context, deployment.AllocationKey, func(sessions.LockedSession, deployment.ReservationTx) error) error { return stop }}) - r := &runtimeLifecycle{reader: &strictDeploymentReader{t: t, lifecyclePlacement: func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) { + r := &runtimeLifecycle{sessions: workspaceEnvironmentReader{}, deployment: operations, reader: &strictDeploymentReader{t: t, lifecyclePlacement: func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) { return deployment.LifecyclePlacement{Specification: []byte(`{}`)}, nil - }}, sessions: workspaceEnvironmentReader{}, deployment: operations, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{InstallationID: uuid.NewString()}} + }, environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + return deployment.Allocation{}, deployment.ErrNotFound + }}, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{Mode: "direct", Generation: 1, InstallationID: uuid.NewString()}} _, err := r.provision(t.Context(), f.record.Reference.TenantID, f.record.Reference.EnvironmentID, r.config.InstallationID) if !errors.Is(err, stop) || f.binds != 0 || f.creates != 0 { t.Fatal("owned generation retroactively attached filesystem", err) diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index bf9c77434..9ef486de9 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -75,6 +76,9 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher, owner Owner) (_ *W if owner.Deployment == nil { return nil, errors.New("execution worker requires the deployment execution operations") } + if err := owner.Deployment.CheckRuntimeComputeProtocol(ctx, sandbox.SuspensionStateVersion); err != nil { + return nil, err + } owned.notifications = &executionNotifications{} worker := &Worker{concurrency: dispatcher.MaxConcurrentExecutions, dispatcher: owned, lease: owner.Lease, directoryReads: make(chan directoryReadRequest), fileWrites: make(chan fileWriteRequest), stopped: make(chan struct{}), scheduleWake: make(chan struct{}, 1), enrolledConnections: make(map[string]*runtimeConnection)} worker.runtimes, err = newRuntimeManager(owner, owned.Deployment, owned.DeploymentReader, owned.SessionsReader, owned.Registry, owned.ManagedRuntimes) @@ -140,6 +144,7 @@ func (w *Worker) CreateSession(ctx context.Context, tenant string, input session input.SupportsRetainedNativeHistory = profile.RetainedNativeHistory.IsSupported() creation, err := w.dispatcher.Sessions.CreateSession(ctx, tenant, input) if err == nil { + w.hintRuntimeWake(ctx, creation.Session) w.wakeScheduler() } return creation, err @@ -269,6 +274,8 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { rescanOnCompletion = false case <-w.scheduleWake: rescanOnCompletion = true + case <-w.dispatcher.Registry.CapabilityHints(): + rescanOnCompletion = true case <-ticker.C: maintenance = true } @@ -298,10 +305,7 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { w.observeSchedulerPoll(0, nil) continue } - if !maintenance { - schedule.nextEnvironmentScan = time.Time{} - } - work, err := schedule.selectWork(ctx, w, devices, active) + work, err := schedule.selectWork(ctx, w, devices, active, !maintenance) w.observeSlots(len(active)) if err != nil { w.observeSchedulerPoll(0, err) diff --git a/services/core/internal/execution/worker_capability_test.go b/services/core/internal/execution/worker_capability_test.go new file mode 100644 index 000000000..d4d76c5bd --- /dev/null +++ b/services/core/internal/execution/worker_capability_test.go @@ -0,0 +1,63 @@ +package execution + +import ( + "context" + "encoding/json" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" +) + +type candidateQueryObserver struct { + sessions.Reader + scans atomic.Int32 +} + +func (o *candidateQueryObserver) ListExecutionWork(context.Context, string, []string, []string) ([]sessions.ExecutionWork, error) { + return nil, nil +} +func (o *candidateQueryObserver) ListEnvironmentInputWork(context.Context, string, []string) ([]sessions.EnvironmentInputWork, error) { + o.scans.Add(1) + return nil, nil +} + +func TestSchedulingHintScansBeforeDeadlineAndRestoresPollingDelay(t *testing.T) { + observer := &candidateQueryObserver{} + worker := &Worker{dispatcher: &Dispatcher{SessionsReader: observer}, concurrency: 1} + schedule := workerSchedule{nextEnvironmentScan: time.Now().Add(time.Hour)} + for _, hinted := range []bool{false, true, false} { + if _, err := schedule.selectWork(t.Context(), worker, nil, map[string]bool{}, hinted); err != nil { + t.Fatal(err) + } + } + if observer.scans.Load() != 1 { + t.Fatalf("candidate scans = %d; hint must bypass the deadline once", observer.scans.Load()) + } +} + +type phaseSessionReader struct{ sessions.Reader } + +func (phaseSessionReader) GetSessionEnvironment(context.Context, string, string) (sessions.Environment, error) { + return sessions.Environment{ID: "environment", Configuration: json.RawMessage(`{"type":"openai_hosted"}`)}, nil +} +func (phaseSessionReader) GetSessionDevice(context.Context, string, string) (sessions.ExecutionDevice, error) { + panic("non-running compute must not reach device eligibility") +} +func TestCapabilityHintDoesNotBypassComputePhase(t *testing.T) { + for _, phase := range []string{"quiescing", "suspending", "suspended", "restoring", "waking"} { + t.Run(phase, func(t *testing.T) { + reader := &strictDeploymentReader{t: t, environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + return deployment.Allocation{ComputePhase: phase}, nil + }} + worker := &Worker{dispatcher: &Dispatcher{SessionsReader: phaseSessionReader{}, DeploymentReader: reader}} + session := sessions.Session{Configuration: json.RawMessage(`{"environment":{"type":"openai_hosted"}}`)} + ready, err := worker.bindSessionDevice(t.Context(), session, func(string) bool { t.Fatal("phase bypassed"); return true }) + if err != nil || ready { + t.Fatal("non-running compute selected", ready, err) + } + }) + } +} diff --git a/services/core/internal/execution/worker_schedule.go b/services/core/internal/execution/worker_schedule.go index bf3a821cc..c38c07077 100644 --- a/services/core/internal/execution/worker_schedule.go +++ b/services/core/internal/execution/worker_schedule.go @@ -20,7 +20,10 @@ type scheduledWork struct { reservationID string } -func (s *workerSchedule) selectWork(ctx context.Context, w *Worker, devices []string, active map[string]bool) ([]scheduledWork, error) { +func (s *workerSchedule) selectWork(ctx context.Context, w *Worker, devices []string, active map[string]bool, hinted bool) ([]scheduledWork, error) { + if hinted { + s.nextEnvironmentScan = time.Time{} + } turns, err := w.dispatcher.SessionsReader.ListExecutionWork(ctx, s.turnCursor, []string{sessions.TurnQueued}, devices) if err != nil { return nil, err diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index 7d67fff4b..bcbd6ab4c 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -14,6 +14,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/placementpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/sessionpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -292,6 +293,7 @@ func (t *reservationTx) InsertAllocation(a deployment.NewAllocation) (deployment row, err := t.q.CreateRuntimeAllocation(t.ctx, sqlc.CreateRuntimeAllocationParams{ ID: id, EnvironmentID: t.environment, DeviceID: device, ProviderKey: provider, NodeID: node, DeploymentGeneration: pgtype.Int8{Int64: int64(a.Generation), Valid: true}, + ProtocolVersion: sandbox.SuspensionStateVersion, }) if err != nil { return deployment.Allocation{}, err @@ -657,6 +659,22 @@ func (s *Store) Activity(ctx context.Context, allocationID string) (deployment.A return loadActivity(ctx, s.pool.Queries(), id) } +func (s *Store) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { + id, err := parseID(installationID) + if err != nil { + return 0, err + } + return s.pool.Queries().CountRuntimeComputeReservations(ctx, id) +} + +func (s *Store) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { + id, err := parseID(installationID) + if err != nil { + return 0, err + } + return s.pool.Queries().CountRuntimeRetainedAllocations(ctx, id) +} + func (t *reservationTx) LoadGenerationSpecification(generation uint64) (deployment.GenerationSpecification, error) { return (unit{ctx: t.ctx, q: t.q}).LoadGenerationSpecification(generation) } diff --git a/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go b/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go index 919f2dac6..63f150c89 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go @@ -34,7 +34,7 @@ func readyCheckpointNode(t *testing.T, f fixture, node deployment.Enrollment, vi func suspendedCheckpointOwner(t *testing.T, f fixture, changes *deployment.ExecutionOperations, installation string, node deployment.Enrollment, view deployment.View) deployment.Allocation { t.Helper() owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) - if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"snapshot":{"ID":"snapshot","Compatibility":{"artifact_domain":"fixture-archive","execution_class":"fixture-machine"}}}',compute_retained_until=clock_timestamp()+interval '24 hours',compute_wake_requested=true WHERE id=$1`, owner.ID); err != nil { + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"protocol_version":"1","retained":{"ID":"snapshot","Compatibility":{"artifact_domain":"fixture-archive","execution_class":"fixture-machine"}}}',compute_retained_until=clock_timestamp()+interval '24 hours',compute_wake_requested=true WHERE id=$1`, owner.ID); err != nil { t.Fatal(err) } owner, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) diff --git a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go index 77fed6a34..67acb835e 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go @@ -132,7 +132,7 @@ func TestPressureSuspensionServesRestoreOnItsOriginalNode(t *testing.T) { node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) f.connect(t, node.NodeID) waiting := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) - if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"snapshot":{"Compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()+interval '1 hour',compute_wake_requested=true WHERE id=$1`, waiting.ID); err != nil { + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"protocol_version":"1","retained":{"Compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()+interval '1 hour',compute_wake_requested=true WHERE id=$1`, waiting.ID); err != nil { t.Fatal(err) } owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) diff --git a/services/core/internal/persistence/postgres/deploymentpg/tx.go b/services/core/internal/persistence/postgres/deploymentpg/tx.go index 4441dd7d0..9a2fc2dc0 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/tx.go +++ b/services/core/internal/persistence/postgres/deploymentpg/tx.go @@ -361,3 +361,7 @@ func (t *deploymentTx) RecordAudit(action, installationID string) error { func (t *deploymentTx) RecordAuditAs(source adminaudit.Source, action, installationID string) error { return auditpg.RecordDeploymentMutation(adminaudit.WithSource(t.ctx, source), t.q, action, "sandbox_deployment", installationID) } + +func (t *deploymentTx) HasIncompatibleComputeState(version string) (bool, error) { + return t.q.HasIncompatibleRuntimeComputeState(t.ctx, version) +} diff --git a/services/core/internal/processconfig/config.go b/services/core/internal/processconfig/config.go index 6a5b75edf..6bc5c9b72 100644 --- a/services/core/internal/processconfig/config.go +++ b/services/core/internal/processconfig/config.go @@ -41,6 +41,7 @@ const ( // Config is Core's process configuration. type Config struct { + SandboxCapacity SandboxLimits // Addr is OAC_ADDR, the listener address. Addr string // PublicOrigin is OAC_PUBLIC_URL. @@ -102,6 +103,9 @@ type runtimeHistoryFile struct { func Load() (Config, error) { var c Config var err error + if c.SandboxCapacity, err = SandboxCapacity(); err != nil { + return Config{}, err + } if c.Log, err = log.LoadConfig(); err != nil { return Config{}, err } @@ -170,6 +174,8 @@ func (c Config) Settings() []api.InstallationSetting { setting("log.level", strings.ToLower(c.Log.Level.String()), "info", []api.InstallationService{api.InstallationCore, api.InstallationWeb}), setting("log.format", format, "auto", []api.InstallationService{api.InstallationCore, api.InstallationWeb}), setting("log.add_source", c.Log.AddSource, false, []api.InstallationService{api.InstallationCore, api.InstallationWeb}), + setting("core.sandbox_capacity.max_active", c.SandboxCapacity.MaxActive, defaultSandboxMaxActive, []api.InstallationService{api.InstallationCore}), + setting("core.sandbox_capacity.max_retained", c.SandboxCapacity.MaxRetained, defaultSandboxMaxRetained, []api.InstallationService{api.InstallationCore}), setting("core.execution_concurrency", c.ExecutionConcurrency, execution.DefaultExecutionConcurrency, []api.InstallationService{api.InstallationCore}), setting("core.harnesses", c.Harnesses, (engine.Catalog{}).Kinds(), []api.InstallationService{api.InstallationCore}), setting("core.default_harness", c.DefaultHarness, defaultHarness, []api.InstallationService{api.InstallationCore}), diff --git a/services/core/internal/processconfig/sandbox_capacity.go b/services/core/internal/processconfig/sandbox_capacity.go new file mode 100644 index 000000000..742b057ff --- /dev/null +++ b/services/core/internal/processconfig/sandbox_capacity.go @@ -0,0 +1,41 @@ +package processconfig + +import ( + "os" + "strconv" +) + +const defaultSandboxMaxActive = 100 +const defaultSandboxMaxRetained = 400 + +// SandboxLimits bounds direct Providers with suspension independently of +// execution concurrency and each enrolled node's capacity. +type SandboxLimits struct { + MaxActive int + MaxRetained int +} + +// SandboxCapacity reads the process-owned active and retained limits. Unset or +// empty variables select the same defaults as the other process settings. +func SandboxCapacity() (SandboxLimits, error) { + capacity := SandboxLimits{MaxActive: defaultSandboxMaxActive, MaxRetained: defaultSandboxMaxRetained} + for _, setting := range []struct { + name string + target *int + }{ + {"OAC_SANDBOX_MAX_ACTIVE", &capacity.MaxActive}, + {"OAC_SANDBOX_MAX_RETAINED", &capacity.MaxRetained}, + } { + if raw := os.Getenv(setting.name); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100000 { + return SandboxLimits{}, configError(setting.name + " must be an integer between 1 and 100000") + } + *setting.target = value + } + } + if capacity.MaxRetained < capacity.MaxActive { + return SandboxLimits{}, configError("OAC_SANDBOX_MAX_RETAINED must be at least OAC_SANDBOX_MAX_ACTIVE") + } + return capacity, nil +} diff --git a/services/core/internal/processconfig/sandbox_capacity_test.go b/services/core/internal/processconfig/sandbox_capacity_test.go new file mode 100644 index 000000000..6c06fd7f0 --- /dev/null +++ b/services/core/internal/processconfig/sandbox_capacity_test.go @@ -0,0 +1,53 @@ +package processconfig + +import ( + "strings" + "testing" +) + +func TestSandboxCapacityDefaultsAndSettings(t *testing.T) { + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "") + capacity, err := SandboxCapacity() + if err != nil || capacity.MaxActive != 100 || capacity.MaxRetained != 400 { + t.Fatal("empty Compose settings lost defaults", capacity, err) + } + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "7") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "31") + capacity, err = SandboxCapacity() + settings := (Config{SandboxCapacity: capacity}).Settings() + if err != nil { + t.Fatal(err) + } + found := map[string]any{} + for _, setting := range settings { + found[setting.Key] = setting.Value + } + if found["core.sandbox_capacity.max_active"] != 7 || found["core.sandbox_capacity.max_retained"] != 31 { + t.Fatal("installation omitted effective direct capacity", found) + } +} + +func TestCheckValidatesSandboxCapacity(t *testing.T) { + for _, key := range []string{"OAC_SANDBOX_MAX_ACTIVE", "OAC_SANDBOX_MAX_RETAINED"} { + for _, tc := range []struct { + value string + valid bool + }{{"0", false}, {"-1", false}, {"1.5", false}, {"synthetic-secret-value", false}, {"100001", false}, {"100000", true}, {"1", true}, {"", true}} { + t.Run(key+"/"+tc.value, func(t *testing.T) { + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "1") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "100000") + t.Setenv(key, tc.value) + _, err := SandboxCapacity() + if (err == nil) != tc.valid || err != nil && (!strings.Contains(err.Error(), key) || strings.Contains(err.Error(), "synthetic-secret-value")) { + t.Fatal("wrong process validation", err) + } + }) + } + } + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "100") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "99") + if _, err := SandboxCapacity(); err == nil { + t.Fatal("accepted retained limit below active limit") + } +} diff --git a/services/core/internal/runtimegateway/capability_hint_test.go b/services/core/internal/runtimegateway/capability_hint_test.go new file mode 100644 index 000000000..ba4fbd9e3 --- /dev/null +++ b/services/core/internal/runtimegateway/capability_hint_test.go @@ -0,0 +1,138 @@ +package runtimegateway + +import ( + "context" + "encoding/json" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "testing" + "time" +) + +func expectCapabilityHint(t *testing.T, reg *Registry, want bool) { + t.Helper() + select { + case <-reg.CapabilityHints(): + if !want { + t.Fatal("unexpected capability hint") + } + default: + if want { + t.Fatal("missing capability hint") + } + } +} + +func TestCapabilityHintsRequireCurrentAvailableSnapshot(t *testing.T) { + reg := NewRegistry() + old := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(old) + expectCapabilityHint(t, reg, false) // Transport alone is insufficient. + publishCapabilityHeartbeat(t, old, []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: false}}) + expectCapabilityHint(t, reg, false) + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: true}} + publishCapabilityHeartbeat(t, old, kinds) + expectCapabilityHint(t, reg, true) + publishCapabilityHeartbeat(t, old, kinds) + expectCapabilityHint(t, reg, false) // Ordinary heartbeats do not rescan. + newer := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(newer) + kinds[0].Version = "changed" + publishCapabilityHeartbeat(t, old, kinds) + expectCapabilityHint(t, reg, false) // Superseded socket cannot accelerate work. + publishCapabilityHeartbeat(t, newer, kinds) + expectCapabilityHint(t, reg, true) +} + +func TestCapabilityHintsCoalesceAndIgnoreDisconnectedPeers(t *testing.T) { + reg := NewRegistry() + s := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(s) + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: true}} + publishCapabilityHeartbeat(t, s, kinds) + kinds[0].Capabilities.Streaming = true + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, true) + expectCapabilityHint(t, reg, false) + reg.Deregister(s) + kinds[0].Capabilities.Steering = true + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, false) +} + +func TestInvalidHeartbeatCannotWakeScheduler(t *testing.T) { + reg := NewRegistry() + s := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(s) + s.handleHeartbeat(proto.Envelope{Type: proto.TypeHeartbeat, Payload: json.RawMessage(`{"ts":1,"active_requests":0,"supported_agent_kinds":[{"kind":"fake","available":true,"capabilities":{"streaming":"invented"}}]}`)}) + expectCapabilityHint(t, reg, false) + if !s.IsClosed() { + t.Fatal("malformed capabilities did not close transport") + } +} + +func publishCapabilityHeartbeat(t *testing.T, s *Session, kinds []runtimedevice.SupportedAgentKind) { + t.Helper() + advertised := make([]proto.SupportedAgentKind, len(kinds)) + for i, k := range kinds { + advertised[i] = proto.SupportedAgentKind{Kind: k.Kind, Available: k.Available, Version: k.Version, + Capabilities: prototest.Capabilities(proto.AgentKindCapabilities{ + Streaming: proto.CapabilityFromBool(k.Capabilities.Streaming), + Steering: proto.CapabilityFromBool(k.Capabilities.Steering), + })} + } + env, err := proto.NewEnvelope(proto.TypeHeartbeat, "", proto.HeartbeatPayload{SupportedAgentKinds: advertised}) + if err != nil { + t.Fatal(err) + } + s.handleHeartbeat(env) +} + +type capabilityAuthorityStore struct { + HeartbeatTouch + deleted bool + err error +} + +func (a *capabilityAuthorityStore) TouchAgentDaemonHeartbeat(context.Context, runtimedevice.Heartbeat) (runtimedevice.HeartbeatStatus, error) { + return runtimedevice.HeartbeatStatus{Deleted: a.deleted}, a.err +} +func TestCapabilityConfirmationRetriesWithoutObservingRejectedAuthority(t *testing.T) { + for _, mode := range []string{"error", "deleted", "draining"} { + t.Run(mode, func(t *testing.T) { + reg := NewRegistry() + s := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(s) + t.Cleanup(func() { s.Close("test finished") }) + authority := &capabilityAuthorityStore{deleted: mode != "error"} + if mode == "error" { + authority.err = errors.New("private credential must not be observed") + } + s.heartbeat = authority + if mode == "draining" { + s.credentialHash = "original-hash" + s.archivedCancellations = &receiptStore{receipt: runtimedevice.ArchivedCancellationReceipt{RunID: "run", Deadline: time.Now().Add(time.Minute)}} + release, err := s.TrackExecutionDelivery("run") + if err != nil { + t.Fatal(err) + } + t.Cleanup(release) + } + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: true}} + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, false) + if mode == "draining" && s.IsClosed() { + t.Fatal("existing receipt drain was interrupted") + } + if mode == "error" { + authority.err = nil + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, true) + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, false) + } + }) + } +} diff --git a/services/core/internal/runtimegateway/registry.go b/services/core/internal/runtimegateway/registry.go index b5984821e..c30084280 100644 --- a/services/core/internal/runtimegateway/registry.go +++ b/services/core/internal/runtimegateway/registry.go @@ -12,6 +12,8 @@ import ( "errors" "sync" "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" ) // ErrDeviceNotRegistered is returned by Registry lookups when a caller @@ -36,16 +38,18 @@ type Registry struct { // is inserted into byDevice. Buffered(1) so a Register that // happens between WaitForDevice registering and selecting on the // chan still wakes the waiter. - waiters map[string][]chan *Session + waiters map[string][]chan *Session + capabilityHints chan struct{} } // NewRegistry returns an empty registry. The zero value would also // work but the constructor avoids accidental nil-map panics. func NewRegistry() *Registry { return &Registry{ - byDevice: map[string]*Session{}, - byRun: map[string]*Session{}, - waiters: map[string][]chan *Session{}, + byDevice: map[string]*Session{}, + byRun: map[string]*Session{}, + waiters: map[string][]chan *Session{}, + capabilityHints: make(chan struct{}, 1), } } @@ -237,3 +241,29 @@ func (r *Registry) removeWaiter(deviceID string, ch chan *Session) { r.waiters[deviceID] = filtered } } + +// CapabilityHints coalesces capability changes for the single execution Worker. +// A hint grants no execution authority; the Worker rechecks its normal gates. +func (r *Registry) CapabilityHints() <-chan struct{} { return r.capabilityHints } + +func (r *Registry) observeCapabilitySnapshot(sess *Session, kinds []runtimedevice.SupportedAgentKind) bool { + r.mu.RLock() + defer r.mu.RUnlock() + if r.byDevice[sess.DeviceID] != sess || sess.IsClosed() { + return false + } + available := 0 + for _, kind := range kinds { + if kind.Available { + available++ + } + } + if available == 0 { + return true + } + select { + case r.capabilityHints <- struct{}{}: + default: + } + return true +} diff --git a/services/core/internal/runtimegateway/session.go b/services/core/internal/runtimegateway/session.go index 42191c76e..38440c853 100644 --- a/services/core/internal/runtimegateway/session.go +++ b/services/core/internal/runtimegateway/session.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "strings" "sync" "time" @@ -100,9 +101,10 @@ type Session struct { // supportedKinds is the latest daemon-advertised agent_kind snapshot, // updated from heartbeat frames and read by the connector before // sending execution_prepare so unsupported engines fail on the server. - kindsMu sync.RWMutex - kindsSeen bool - supportedKinds []runtimedevice.SupportedAgentKind + kindsMu sync.RWMutex + kindsSeen bool + supportedKinds []runtimedevice.SupportedAgentKind + capabilityObservationPending bool // Subscribers keyed by runID. The read loop only sends on these // channels; Unsubscribe is the only place that closes them. @@ -222,11 +224,25 @@ func (s *Session) setSupportedAgentKinds(kinds []runtimedevice.SupportedAgentKin copyKinds := make([]runtimedevice.SupportedAgentKind, len(kinds)) copy(copyKinds, kinds) s.kindsMu.Lock() + changed := !s.kindsSeen || !reflect.DeepEqual(s.supportedKinds, copyKinds) s.kindsSeen = true s.supportedKinds = copyKinds + if changed { + s.capabilityObservationPending = true + } s.kindsMu.Unlock() } +// A failed authorization or persistence check retains the pending hint +// for the next confirmed heartbeat without changing capability publication. +func (s *Session) observeConfirmedCapabilities(kinds []runtimedevice.SupportedAgentKind) { + s.kindsMu.Lock() + defer s.kindsMu.Unlock() + if s.capabilityObservationPending && s.reg.observeCapabilitySnapshot(s, kinds) { + s.capabilityObservationPending = false + } +} + // Close closes the transport and subscriptions with ErrSessionClosed, then // releases connection ownership. It establishes no execution outcome. Idempotent. func (s *Session) Close(reason string) { @@ -453,6 +469,7 @@ func (s *Session) handleHeartbeat(env proto.Envelope) { kinds := deviceKindsFromHeartbeat(p) s.setSupportedAgentKinds(kinds) if s.heartbeat == nil { + s.observeConfirmedCapabilities(kinds) return } @@ -481,7 +498,9 @@ func (s *Session) handleHeartbeat(env proto.Envelope) { // actual admin action; the daemon only sees it's no longer // the current owner. s.CloseWithCode(CloseRuntimeDeleted, "runtime retired") + return } + s.observeConfirmedCapabilities(kinds) } func deviceKindsFromHeartbeat(p proto.HeartbeatPayload) []runtimedevice.SupportedAgentKind { diff --git a/services/core/internal/sandbox/docker/bootstrap_test.go b/services/core/internal/sandbox/docker/bootstrap_test.go index 31d2845ff..5d0b0f088 100644 --- a/services/core/internal/sandbox/docker/bootstrap_test.go +++ b/services/core/internal/sandbox/docker/bootstrap_test.go @@ -14,7 +14,7 @@ import ( ) func TestBootstrapDeliversOnlyPublicConnectionInput(t *testing.T) { - b := sandbox.Bootstrap{CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-secret"} + b := sandbox.Bootstrap{CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-secret", Harness: "codex"} found := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != "PUT" || !strings.HasSuffix(r.URL.Path, "/containers/test/archive") { diff --git a/services/core/internal/sandbox/docker/contract_test.go b/services/core/internal/sandbox/docker/contract_test.go index 09db3fef8..8105d6d40 100644 --- a/services/core/internal/sandbox/docker/contract_test.go +++ b/services/core/internal/sandbox/docker/contract_test.go @@ -26,7 +26,7 @@ type contractStep struct { func dockerContractFixture(t *testing.T, cancel context.CancelFunc, script func(*Provider, sandbox.Reference) []contractStep) contracttest.Fixture { t.Helper() - b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} var mu sync.Mutex var calls []string var steps []contractStep diff --git a/services/core/internal/sandbox/docker/deployment_test.go b/services/core/internal/sandbox/docker/deployment_test.go index 9df0727d2..7dfa1283c 100644 --- a/services/core/internal/sandbox/docker/deployment_test.go +++ b/services/core/internal/sandbox/docker/deployment_test.go @@ -87,7 +87,7 @@ func TestManagedDriftRejectsBeforeBootstrapAndRetainsCleanup(t *testing.T) { t.Fatal(err) } resources.CPUs = 9 - b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "secret", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "secret", Harness: "codex", NetworkAccess: "enabled"} info, err := p.Create(t.Context(), b) if !errors.Is(err, sandbox.ErrInvalid) { t.Fatal("drift accepted", err) diff --git a/services/core/internal/sandbox/docker/operations.go b/services/core/internal/sandbox/docker/operations.go index 2722da0dd..0d479f3f1 100644 --- a/services/core/internal/sandbox/docker/operations.go +++ b/services/core/internal/sandbox/docker/operations.go @@ -16,11 +16,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "NewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "GetCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Suspend": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Resume": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "KillCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Observe": {State: providercontract.Supported}, @@ -30,7 +31,7 @@ func (*Provider) ProviderOperations() providercontract.Operations { return Opera func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} } -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} } func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Compu func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} } -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} +func (p *Provider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: Operations()["DeleteRetained"].Reason} } func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} @@ -54,3 +55,7 @@ func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} } + +func (p *Provider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: Operations()["RenewCompute"].Reason} +} diff --git a/services/core/internal/sandbox/docker/provider_test.go b/services/core/internal/sandbox/docker/provider_test.go index a7f71289b..0df733084 100644 --- a/services/core/internal/sandbox/docker/provider_test.go +++ b/services/core/internal/sandbox/docker/provider_test.go @@ -76,7 +76,7 @@ func TestDockerProviderLifecycle(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() bootstrap := func() sandbox.Bootstrap { - return sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-test-credential", NetworkAccess: "enabled"} + return sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-test-credential", Harness: "codex", NetworkAccess: "enabled"} } b := bootstrap() b.NetworkAccess, b.AllowedDomains = "restricted", []string{"Example.com", "api.example.com"} diff --git a/services/core/internal/sandbox/docker/recovery_test.go b/services/core/internal/sandbox/docker/recovery_test.go index 06ce2e8ae..9cb813598 100644 --- a/services/core/internal/sandbox/docker/recovery_test.go +++ b/services/core/internal/sandbox/docker/recovery_test.go @@ -87,7 +87,7 @@ func TestDockerProviderRecoveryObservations(t *testing.T) { if e != nil { t.Fatal(e) } - b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-recovery-token", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-recovery-token", Harness: "codex", NetworkAccess: "enabled"} direct, e := client.New(client.WithHost("unix:///var/run/docker.sock")) if e != nil { t.Fatal(e) diff --git a/services/core/internal/sandbox/e2b/contract_test.go b/services/core/internal/sandbox/e2b/contract_test.go index b4d64227b..0216cfc23 100644 --- a/services/core/internal/sandbox/e2b/contract_test.go +++ b/services/core/internal/sandbox/e2b/contract_test.go @@ -16,7 +16,7 @@ func (f contractCaller) Call(ctx context.Context, q Request) (Response, error) { func TestProviderContract(t *testing.T) { contracttest.RunFailures(t, func(t *testing.T, s contracttest.Scenario, cancel context.CancelFunc) contracttest.Fixture { p, _, r := fixture(t) - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} var calls []string p.caller = contractCaller(func(ctx context.Context, q Request) (Response, error) { calls = append(calls, q.Operation) @@ -47,7 +47,7 @@ func TestProviderContract(t *testing.T) { func TestProviderContractObservation(t *testing.T) { p, f, r := fixture(t) f.response.Info = &sandbox.Info{Reference: r, ProviderID: "native-owned", State: "running", CreateSettled: true, BootstrapComplete: true} - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} got, err := p.Create(bounded(t), b) contracttest.AssertObservation(t, got, err, r, "native-owned", "running") got, err = p.GetInfo(bounded(t), r) diff --git a/services/core/internal/sandbox/e2b/helper_contract.go b/services/core/internal/sandbox/e2b/helper_contract.go index f4cf30ffb..517357622 100644 --- a/services/core/internal/sandbox/e2b/helper_contract.go +++ b/services/core/internal/sandbox/e2b/helper_contract.go @@ -11,7 +11,7 @@ import ( //go:generate go run ./internal/contractgen // This adapter-private boundary is documented in tools/e2b-provider/README.md. -const ProtocolVersion = 1 +const ProtocolVersion = 4 const MaxOutputBytes = 1024 * 1024 const MaxRequestBytes = 72 * 1024 * 1024 const MaxResponseBytes = 16 * 1024 * 1024 @@ -20,7 +20,7 @@ const MaxCommandInputBytes = sandbox.MaxCommandInputBytes // HelperOperations declares the complete set of one-shot helper operations. func HelperOperations() []string { - return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential"} + return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential", "compute_info", "compute_renew", "suspend", "resume", "compute_kill", "delete_retained", "compute_command", "resume_compute"} } // HelperErrors are sanitized wire outcomes; an empty code denotes success. @@ -56,6 +56,28 @@ func (q Request) Validate() error { return sandbox.ErrInvalid } } + + switch q.Operation { + case "compute_info", "compute_renew", "compute_kill", "compute_command", "resume_compute": + if q.Compute == nil || q.Compute.Name != q.Reference.AllocationID || (q.Operation != "compute_info" && q.Compute.ID == "") { + return sandbox.ErrInvalid + } + if q.Operation == "compute_command" && q.Command == nil { + return sandbox.ErrInvalid + } + case "suspend": + if q.Suspend == nil || q.Suspend.Reference != q.Reference || !validID(q.Suspend.OperationID) || q.Suspend.Source.Name != q.Reference.AllocationID || q.Suspend.Source.ID == "" { + return sandbox.ErrInvalid + } + case "resume": + if q.Resume == nil || q.Resume.Workspace != nil || q.Resume.Reference != q.Reference || !validID(q.Resume.OperationID) || sandbox.ValidateRetained(q.Resume.Retained) != nil || q.Resume.Retained.Reference != q.Reference.AllocationID { + return sandbox.ErrInvalid + } + case "delete_retained": + if q.Retained == nil || sandbox.ValidateRetained(*q.Retained) != nil || q.Retained.Reference != q.Reference.AllocationID { + return sandbox.ErrInvalid + } + } return nil } @@ -69,10 +91,15 @@ type Request struct { Bootstrap *sandbox.Bootstrap `json:",omitempty"` RuntimeBootstrap *runtimebootstrap.Connection `json:",omitempty"` Command *sandbox.Command `json:",omitempty"` + Compute *sandbox.Compute `json:",omitempty"` + Suspend *sandbox.SuspendRequest `json:",omitempty"` + Resume *sandbox.ResumeRequest `json:",omitempty"` + Retained *sandbox.RetainedState `json:",omitempty"` Deadline time.Time } type Response struct { Version int + State *sandbox.ComputeState `json:",omitempty"` Info *sandbox.Info `json:",omitempty"` Command *sandbox.CommandResult `json:",omitempty"` ErrorCode string diff --git a/services/core/internal/sandbox/e2b/helper_contract_test.go b/services/core/internal/sandbox/e2b/helper_contract_test.go index 686321cb1..de319aa09 100644 --- a/services/core/internal/sandbox/e2b/helper_contract_test.go +++ b/services/core/internal/sandbox/e2b/helper_contract_test.go @@ -67,6 +67,7 @@ func validManagedExchange(data []byte) bool { _ = json.Unmarshal(bootstrapBytes, &expected) delete(expected, "CoreURL") delete(expected, "Credential") + delete(expected, "Harness") expected["InstallationID"] = nil expected["RuntimeBootstrap"] = nil if workspace, present := fields["Workspace"]; present { diff --git a/services/core/internal/sandbox/e2b/installed_paths_test.go b/services/core/internal/sandbox/e2b/installed_paths_test.go index 5012e0d97..35a813ef9 100644 --- a/services/core/internal/sandbox/e2b/installed_paths_test.go +++ b/services/core/internal/sandbox/e2b/installed_paths_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "strconv" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -31,7 +32,7 @@ func TestConfigurationDiscoveryUsesSuppliedProcessPaths(t *testing.T) { if err := os.MkdirAll(filepath.Dir(binary), 0700); err != nil { t.Fatal(err) } - if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"Templates\":[]}'\n"), 0700); err != nil { + if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(ProtocolVersion)+",\"Templates\":[]}'\n"), 0700); err != nil { t.Fatal(err) } input := sandbox.ConfigurationDiscoveryInput{Credential: json.RawMessage(`{"api_key":"synthetic-key"}`)} diff --git a/services/core/internal/sandbox/e2b/internal/contractgen/main.go b/services/core/internal/sandbox/e2b/internal/contractgen/main.go index 118fe1c3f..de3685b1e 100644 --- a/services/core/internal/sandbox/e2b/internal/contractgen/main.go +++ b/services/core/internal/sandbox/e2b/internal/contractgen/main.go @@ -71,9 +71,9 @@ func main() { write("services/core/internal/sandbox/e2b/helper_sdk_generated.go", goCode) bootstrap := fields(reflect.TypeFor[sandbox.Bootstrap]()) requiredBootstrap := fieldNames(reflect.TypeFor[sandbox.Bootstrap](), true) - requiredBootstrap = slices.DeleteFunc(requiredBootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" }) + requiredBootstrap = slices.DeleteFunc(requiredBootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" || s == "Harness" }) requiredBootstrap = append(requiredBootstrap, "InstallationID", "RuntimeBootstrap") - bootstrap = slices.DeleteFunc(bootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" }) + bootstrap = slices.DeleteFunc(bootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" || s == "Harness" }) bootstrap = append(bootstrap, "InstallationID", "RuntimeBootstrap") identity := fields(reflect.TypeFor[sandbox.Reference]()) for _, name := range []string{"SessionID", "DeviceID"} { @@ -85,11 +85,17 @@ func main() { identity = append(identity, "InstallationID") values := map[string]any{ "PROTOCOL_VERSION": e2b.ProtocolVersion, "SDK_VERSION": sdk, - "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, - "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, - "OPERATIONS": e2b.HelperOperations(), "ERROR_CODES": e2b.HelperErrors(), + "SUSPEND_CONTROL_FILE": runtimebootstrap.SuspendControlFile, + "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, + "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, + "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, + "OPERATIONS": e2b.HelperOperations(), "ERROR_CODES": e2b.HelperErrors(), "REQUEST_FIELDS": fields(reflect.TypeFor[e2b.Request]()), "RESPONSE_FIELDS": fields(reflect.TypeFor[e2b.Response]()), "REFERENCE_FIELDS": fields(reflect.TypeFor[sandbox.Reference]()), + "COMPUTE_FIELDS": fields(reflect.TypeFor[sandbox.Compute]()), + "RETAINED_FIELDS": fields(reflect.TypeFor[sandbox.RetainedState]()), + "SUSPEND_FIELDS": fields(reflect.TypeFor[sandbox.SuspendRequest]()), + "RESUME_FIELDS": fields(reflect.TypeFor[sandbox.ResumeRequest]()), "MANAGED_BOOTSTRAP_FIELDS": bootstrap, "MANAGED_BOOTSTRAP_REQUIRED_FIELDS": requiredBootstrap, "MANAGED_IDENTITY_FIELDS": identity, "NETWORK_ACCESS": networkValues(filepath.Join(root, "internal/agentnetwork/policy.go")), } @@ -115,7 +121,7 @@ func main() { // envelopes and managed bootstrap inputs in both implementations. func fixtures() []byte { r := sandbox.Reference{TenantID: "11111111-1111-4111-8111-111111111111", EnvironmentID: "22222222-2222-4222-8222-222222222222", AllocationID: "33333333-3333-4333-8333-333333333333"} - b := sandbox.Bootstrap{Reference: r, SessionID: "44444444-4444-4444-8444-444444444444", DeviceID: "55555555-5555-4555-8555-555555555555", CoreURL: "https://core.example/api/v1", Credential: "fixture-only", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: "44444444-4444-4444-8444-444444444444", DeviceID: "55555555-5555-4555-8555-555555555555", CoreURL: "https://core.example/api/v1", Credential: "fixture-only", Harness: "codex", NetworkAccess: "enabled"} installation := "66666666-6666-4666-8666-666666666666" connection := b.RuntimeConnection() q := e2b.Request{Version: e2b.ProtocolVersion, Operation: "create", Config: e2b.Config{InstallationID: installation}, Reference: r, Bootstrap: &b, RuntimeBootstrap: &connection, Deadline: time.Date(2099, 1, 1, 0, 0, 0, 0, time.UTC)} @@ -132,7 +138,24 @@ func fixtures() []byte { } for _, operation := range e2b.HelperOperations() { copy := q + copy.Operation = operation + c := sandbox.Compute{Name: r.AllocationID, ID: "native-fixture"} + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: installation, OperationID: installation, SourceName: c.Name, SourceID: c.ID, Data: "opaque"} + switch operation { + case "compute_info", "compute_renew", "compute_kill", "compute_command", "resume_compute": + copy.Compute = &c + if operation == "compute_command" { + copy.Command = &sandbox.Command{Args: []string{"true"}} + } + case "suspend": + copy.Suspend = &sandbox.SuspendRequest{Reference: r, OperationID: installation, Source: c} + case "resume": + target := sandbox.Compute{Generation: 1, Name: c.Name, ID: c.ID, RestoredFrom: &retained} + copy.Resume = &sandbox.ResumeRequest{Reference: r, OperationID: installation, Retained: retained, Target: target} + case "delete_retained": + copy.Retained = &retained + } if operation == "verify_credential" { copy.References = []sandbox.Reference{r} } @@ -157,6 +180,17 @@ func fixtures() []byte { value["Bootstrap"] = bootstrapValue add("request", fmt.Sprintf("workspace-%v", workspace), workspace == nil, value) } + for _, workspace := range []any{nil, map[string]any{}} { + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: installation, OperationID: installation, SourceName: r.AllocationID, SourceID: "native-fixture", Data: "opaque"} + copy := q + copy.Operation = "resume" + copy.Resume = &sandbox.ResumeRequest{Reference: r, OperationID: installation, Retained: retained, Target: sandbox.Compute{Generation: 1, Name: r.AllocationID, ID: "native-fixture", RestoredFrom: &retained}} + value := object(copy) + resume := object(copy.Resume) + resume["Workspace"] = workspace + value["Resume"] = resume + add("request", fmt.Sprintf("resume-workspace-%v", workspace), workspace == nil, value) + } for _, count := range []int{0, e2b.MaxCredentialReferences, e2b.MaxCredentialReferences + 1} { copy := q copy.Operation = "verify_credential" @@ -187,6 +221,7 @@ func fixtures() []byte { managed := object(b) delete(managed, "CoreURL") delete(managed, "Credential") + delete(managed, "Harness") managed["InstallationID"] = installation managed["RuntimeBootstrap"] = runtimebootstrap.Connection(connection) for _, policy := range []agentnetwork.Policy{{Access: "enabled"}, {Access: "disabled"}, {Access: "restricted", AllowedDomains: []string{"example.com"}}} { diff --git a/services/core/internal/sandbox/e2b/operations.go b/services/core/internal/sandbox/e2b/operations.go index a3da4688e..7bb90e068 100644 --- a/services/core/internal/sandbox/e2b/operations.go +++ b/services/core/internal/sandbox/e2b/operations.go @@ -1,11 +1,12 @@ package e2b import ( - "context" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) +var _ sandbox.SandboxProvider = (*Provider)(nil) + // Operations is this adapter's complete authored resource contract. func Operations() providercontract.Operations { return providercontract.Operations{ @@ -14,43 +15,17 @@ func Operations() providercontract.Operations { "Renew": {State: providercontract.Supported}, "Kill": {State: providercontract.Supported}, "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Suspend": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Resume": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "Initial": {State: providercontract.Supported}, + "NewCompute": {State: providercontract.Supported}, + "GetCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, + "Suspend": {State: providercontract.Supported}, + "Resume": {State: providercontract.Supported}, + "KillCompute": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, + "RunCommandCompute": {State: providercontract.Supported}, + "ResumeCompute": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, } } func (*Provider) ProviderOperations() providercontract.Operations { return Operations() } -func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} -} -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} -} -func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "GetCompute", Reason: Operations()["GetCompute"].Reason} -} -func (p *Provider) Suspend(context.Context, sandbox.SuspendRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Suspend", Reason: Operations()["Suspend"].Reason} -} -func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Resume", Reason: Operations()["Resume"].Reason} -} -func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { - return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} -} -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} -} -func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { - return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} -} -func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} -} diff --git a/services/core/internal/sandbox/e2b/process_test.go b/services/core/internal/sandbox/e2b/process_test.go index 2d15997f4..695333885 100644 --- a/services/core/internal/sandbox/e2b/process_test.go +++ b/services/core/internal/sandbox/e2b/process_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -29,7 +30,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { marker := filepath.Join(root, "settled") // The private test path contains no shell syntax; the real adapter never puts // credentials or request contents in argv or inherited environment. - script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -50,7 +51,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { func TestEnvironmentDropsProviderSelectorsAndCredentials(t *testing.T) { root := t.TempDir() binary := filepath.Join(root, "helper") - script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -67,7 +68,7 @@ func TestCredentialFenceWaitsForActualHelperExitAfterCancellation(t *testing.T) binary := filepath.Join(root, "helper") marker := filepath.Join(root, "started") finish := filepath.Join(root, "finish") - script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } diff --git a/services/core/internal/sandbox/e2b/provider.go b/services/core/internal/sandbox/e2b/provider.go index 81e74cdba..4c022070e 100644 --- a/services/core/internal/sandbox/e2b/provider.go +++ b/services/core/internal/sandbox/e2b/provider.go @@ -195,12 +195,20 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen } connection = &value } - out, err := p.caller.Call(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) + return p.callRequest(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) +} + +func (p *Provider) callRequest(ctx context.Context, q Request) (Response, error) { + operation, r := q.Operation, q.Reference + if q.Validate() != nil { + return Response{}, sandbox.ErrInvalid + } + out, err := p.caller.Call(ctx, q) if errors.Is(err, errHelperNotStarted) { return unstarted(operation, r), sandbox.ErrComputeUnconfirmed } if err != nil || out.Version != ProtocolVersion { - if operation == "command" { + if operation == "command" || operation == "compute_command" { return Response{}, sandbox.ErrCommandUnconfirmed } return Response{}, sandbox.ErrComputeUnconfirmed diff --git a/services/core/internal/sandbox/e2b/provider_test.go b/services/core/internal/sandbox/e2b/provider_test.go index 406160671..6263ede3e 100644 --- a/services/core/internal/sandbox/e2b/provider_test.go +++ b/services/core/internal/sandbox/e2b/provider_test.go @@ -103,7 +103,7 @@ func TestKillRequiresTerminalProof(t *testing.T) { } func TestCreateAndCommandUseOnlyPrivateRequest(t *testing.T) { p, f, r := fixture(t) - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private-bootstrap", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private-bootstrap", Harness: "codex", NetworkAccess: "enabled"} f.response.Info = &sandbox.Info{Reference: r, State: "running", ProviderID: "native-id", CreateSettled: true, BootstrapComplete: true} if _, err := p.Create(bounded(t), b); err != nil { t.Fatal(err) @@ -143,7 +143,7 @@ func TestDeadlineAndCommandAdmission(t *testing.T) { func TestDefinitePreHelperCreateFailureCarriesAbsenceProof(t *testing.T) { p, f, r := fixture(t) - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private", Harness: "codex", NetworkAccess: "enabled"} for _, err := range []error{errHelperNotStarted, context.DeadlineExceeded} { f.err = err info, gotErr := p.Create(bounded(t), b) diff --git a/services/core/internal/sandbox/e2b/suspension.go b/services/core/internal/sandbox/e2b/suspension.go new file mode 100644 index 000000000..0b55260e0 --- /dev/null +++ b/services/core/internal/sandbox/e2b/suspension.go @@ -0,0 +1,121 @@ +package e2b + +import ( + "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +func (p *Provider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { + if !validReference(r) { + return sandbox.Compute{}, sandbox.ErrInvalid + } + return sandbox.Compute{Name: r.AllocationID}, nil +} +func (p *Provider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, retained *sandbox.RetainedState) (sandbox.Compute, error) { + if !validReference(r) || retained == nil || sandbox.ValidateRetained(*retained) != nil || retained.Reference != r.AllocationID || retained.SourceName != r.AllocationID || generation == 0 || generation != retained.SourceGeneration+1 { + return sandbox.Compute{}, sandbox.ErrInvalid + } + value := *retained + return sandbox.Compute{Generation: generation, Name: r.AllocationID, ID: retained.SourceID, RestoredFrom: &value}, nil +} +func (p *Provider) computeCall(ctx context.Context, q Request) (Response, error) { + deadline, ok := ctx.Deadline() + if !ok || ctx.Err() != nil { + return Response{}, sandbox.ErrInvalid + } + q.Version, q.Config, q.Deadline = ProtocolVersion, p.config, deadline + return p.callRequest(ctx, q) +} +func (p *Provider) computeState(ctx context.Context, operation string, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + if c.Name != r.AllocationID { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + out, err := p.computeCall(ctx, Request{Operation: operation, Reference: r, Compute: &c}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State == nil || sandbox.ValidateComputeResult(c, out.State.Compute) != nil { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return *out.State, nil +} +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "compute_info", r, c) +} +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "compute_renew", r, c) +} +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "resume_compute", r, c) +} +func (p *Provider) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + if !validID(q.OperationID) || q.Source.Name != q.Reference.AllocationID || q.Source.ID == "" { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State == nil { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + if err = sandbox.ValidateSuspendResult(q, *out.State); err != nil { + return sandbox.ComputeState{}, err + } + if out.State.Retained != nil && out.State.Retained.Reference != q.Reference.AllocationID { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + return *out.State, nil +} +func (p *Provider) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + if q.Workspace != nil { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Resume", Reason: "external_workspace_unsupported"} + } + planned, err := p.NewCompute(ctx, q.Reference, q.Target.Generation, &q.Retained) + if err != nil || sandbox.ValidateComputeResult(planned, q.Target) != nil || !validID(q.OperationID) { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State != nil && out.State.RestoreAttemptClosed != "" { + if !out.State.ClosesRestoreAttempt(q) { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + return *out.State, nil + } + if out.State == nil || sandbox.ValidateComputeResult(q.Target, out.State.Compute) != nil || out.State.Status != "running" || !out.State.BootstrapComplete { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return *out.State, nil +} +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { + if c.Name != r.AllocationID || c.ID == "" { + return sandbox.ErrInvalid + } + _, err := p.computeCall(ctx, Request{Operation: "compute_kill", Reference: r, Compute: &c}) + return err +} +func (p *Provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + if sandbox.ValidateRetained(s) != nil || s.Reference != r.AllocationID { + return sandbox.ErrInvalid + } + _, err := p.computeCall(ctx, Request{Operation: "delete_retained", Reference: r, Retained: &s}) + return err +} +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { + if c.Name != r.AllocationID || c.ID == "" { + return sandbox.CommandResult{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "compute_command", Reference: r, Compute: &c, Command: &command}) + if err != nil { + return sandbox.CommandResult{}, err + } + if out.Command == nil { + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + return *out.Command, nil +} diff --git a/services/core/internal/sandbox/e2b/suspension_test.go b/services/core/internal/sandbox/e2b/suspension_test.go new file mode 100644 index 000000000..5955c2223 --- /dev/null +++ b/services/core/internal/sandbox/e2b/suspension_test.go @@ -0,0 +1,67 @@ +package e2b + +import ( + "errors" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" +) + +func TestSuspensionPlansSameNativeIDWithoutCallingHelper(t *testing.T) { + p, f, r := fixture(t) + initial, err := p.Initial(bounded(t), r) + if err != nil || initial.Name != r.AllocationID || initial.ID != "" { + t.Fatal(initial, err) + } + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: uuid.NewString(), OperationID: uuid.NewString(), SourceName: r.AllocationID, SourceID: "native-id", Data: "opaque"} + target, err := p.NewCompute(bounded(t), r, 1, &retained) + if err != nil || target.ID != retained.SourceID || target.Generation != 1 || *target.RestoredFrom != retained || len(f.requests) != 0 { + t.Fatal(target, err) + } + if _, err = p.NewCompute(bounded(t), r, 2, &retained); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("generation jump accepted", err) + } + retained.Reference = uuid.NewString() + if _, err = p.NewCompute(bounded(t), r, 1, &retained); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("foreign allocation accepted", err) + } +} +func TestSuspensionRejectsUnsettledAndForeignHelperOutcomes(t *testing.T) { + p, f, r := fixture(t) + current := sandbox.Compute{Name: r.AllocationID, ID: "native-id"} + q := sandbox.SuspendRequest{Reference: r, Source: current, OperationID: uuid.NewString()} + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: q.OperationID, OperationID: q.OperationID, SourceName: current.Name, SourceID: current.ID, Data: "opaque"} + f.response.State = &sandbox.ComputeState{Compute: current, Status: "suspended", BootstrapComplete: true, Retained: &retained, ResourcesReleased: true, SuspendSettled: true} + if _, err := p.Suspend(bounded(t), q); err != nil { + t.Fatal(err) + } + f.response.State.SuspendSettled = false + if _, err := p.Suspend(bounded(t), q); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { + t.Fatal(err) + } + f.response.State.SuspendSettled = true + f.response.State.Compute.Generation = 1 + if _, err := p.Suspend(bounded(t), q); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal(err) + } +} + +func TestResumeClosedAttemptPreservesInPlaceNativeIdentity(t *testing.T) { + p, f, r := fixture(t) + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: uuid.NewString(), OperationID: uuid.NewString(), SourceName: r.AllocationID, SourceID: "native-id", Data: "opaque"} + target, err := p.NewCompute(bounded(t), r, 1, &retained) + if err != nil { + t.Fatal(err) + } + q := sandbox.ResumeRequest{Reference: r, OperationID: uuid.NewString(), Retained: retained, Target: target, ReconcileOnly: true} + f.response.State = &sandbox.ComputeState{Compute: target, Status: "absent", RestoreAttemptClosed: q.OperationID} + got, err := p.Resume(bounded(t), q) + if err != nil || !got.ClosesRestoreAttempt(q) { + t.Fatal("in-place closure lost", got, err) + } + f.response.State.Compute.ID = "foreign-id" + if _, err := p.Resume(bounded(t), q); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal("foreign native closure accepted", err) + } +} diff --git a/services/core/internal/sandbox/e2b/workspace_test.go b/services/core/internal/sandbox/e2b/workspace_test.go index 0f6ed46d2..5acc16786 100644 --- a/services/core/internal/sandbox/e2b/workspace_test.go +++ b/services/core/internal/sandbox/e2b/workspace_test.go @@ -15,3 +15,11 @@ func TestCreateExplicitlyRejectsExternalWorkspace(t *testing.T) { t.Fatal("external workspace not rejected before native access", err) } } + +func TestResumeExplicitlyRejectsExternalWorkspace(t *testing.T) { + provider := new(Provider) + _, err := provider.Resume(context.Background(), sandbox.ResumeRequest{Workspace: &workspacefs.Binding{}}) + if reason, ok := providercontract.UnsupportedReason(err, "Resume"); !ok || reason != "external_workspace_unsupported" { + t.Fatal("external workspace not rejected before native access", err) + } +} diff --git a/services/core/internal/sandbox/microsandbox/contract_test.go b/services/core/internal/sandbox/microsandbox/contract_test.go index 0733824a9..7ecfc5da8 100644 --- a/services/core/internal/sandbox/microsandbox/contract_test.go +++ b/services/core/internal/sandbox/microsandbox/contract_test.go @@ -7,12 +7,13 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/contracttest" "github.com/google/uuid" "testing" + "time" ) func TestProviderContract(t *testing.T) { contracttest.RunFailures(t, func(t *testing.T, s contracttest.Scenario, cancel context.CancelFunc) contracttest.Fixture { c, r := testConfig(), testRef() - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} var calls []string p, err := NewWithCaller(c, callerFunc(func(ctx context.Context, q Request) (Response, error) { calls = append(calls, q.Operation) @@ -62,7 +63,7 @@ func TestProviderContractObservation(t *testing.T) { if err != nil { t.Fatal(err) } - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} got, err := p.Create(deadline(t), b) contracttest.AssertObservation(t, got, err, r, "native-owned", "running") got, err = p.GetInfo(deadline(t), r) @@ -70,3 +71,21 @@ func TestProviderContractObservation(t *testing.T) { got, err = p.Renew(deadline(t), r) contracttest.AssertObservation(t, got, err, r, "native-owned", "stopped") } + +func TestBootstrapWireRejectsSupersededVersion(t *testing.T) { + c, r := testConfig(), testRef() + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "enabled"} + q := Request{Version: ProtocolVersion, Operation: "create", Config: c, Reference: r, Bootstrap: &b, Deadline: time.Now().Add(time.Minute)} + if err := ValidateRequest(q); err != nil { + t.Fatal(err) + } + q.Version = 2 + if err := ValidateRequest(q); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("superseded bootstrap wire accepted", err) + } + q.Version = ProtocolVersion + b.Harness = "" + if err := ValidateRequest(q); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("missing selected Harness accepted", err) + } +} diff --git a/services/core/internal/sandbox/microsandbox/creation_settlement_test.go b/services/core/internal/sandbox/microsandbox/creation_settlement_test.go index db52bcfae..f088e8308 100644 --- a/services/core/internal/sandbox/microsandbox/creation_settlement_test.go +++ b/services/core/internal/sandbox/microsandbox/creation_settlement_test.go @@ -11,7 +11,7 @@ import ( func TestCreateConfigurationRejectionSettlement(t *testing.T) { config, ref := testConfig(), testRef() bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, - CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled"} + CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "disabled"} for _, test := range []struct { name string change func(*Response) @@ -150,7 +150,7 @@ func TestOrdinaryComputeInspectionDoesNotRequestInitialSettlement(t *testing.T) if err != nil { t.Fatal(err) } - _, err = provider.GetCompute(deadline(t), ref, Compute{Name: Name(config, ref, 0)}) + _, err = provider.nativeGetCompute(deadline(t), ref, Compute{Name: Name(config, ref, 0)}) if !errors.Is(err, sandbox.ErrNotFound) { t.Fatal(err) } diff --git a/services/core/internal/sandbox/microsandbox/operations.go b/services/core/internal/sandbox/microsandbox/operations.go index 13e81ff96..e3742747e 100644 --- a/services/core/internal/sandbox/microsandbox/operations.go +++ b/services/core/internal/sandbox/microsandbox/operations.go @@ -12,11 +12,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Supported}, "NewCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "GetCompute": {State: providercontract.Supported}, "Suspend": {State: providercontract.Supported}, "Resume": {State: providercontract.Supported}, "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, diff --git a/services/core/internal/sandbox/microsandbox/process_test.go b/services/core/internal/sandbox/microsandbox/process_test.go index 68021cd2d..894568e83 100644 --- a/services/core/internal/sandbox/microsandbox/process_test.go +++ b/services/core/internal/sandbox/microsandbox/process_test.go @@ -335,7 +335,11 @@ identity={"installation_id":"test-installation","generation":1,"specification_di with g.collection_lease(Path(sys.argv[2]),1,i,identity,initialize=sys.argv[3]=="init"): pass ` run := func(mode string) error { - return exec.Command("python3", "-c", script, installer, root, mode, testLeaseIdentity().SpecificationDigest).Run() + output, err := exec.Command("python3", "-c", script, installer, root, mode, testLeaseIdentity().SpecificationDigest).CombinedOutput() + if err != nil { + t.Log(string(output)) + } + return err } if err := run("init"); err != nil { t.Fatal("Python initialization", err) diff --git a/services/core/internal/sandbox/microsandbox/provider.go b/services/core/internal/sandbox/microsandbox/provider.go index abf5acec7..20b093e15 100644 --- a/services/core/internal/sandbox/microsandbox/provider.go +++ b/services/core/internal/sandbox/microsandbox/provider.go @@ -26,7 +26,7 @@ func NewWithCaller(c Config, caller Caller) (*Provider, error) { c.Network.Rules = append([]NetworkRule(nil), c.Network.Rules...) return &Provider{config: c, caller: caller}, nil } -func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (Compute, error) { +func (p *Provider) nativeInitial(ctx context.Context, r sandbox.Reference) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err } @@ -100,7 +100,7 @@ func (p *Provider) responseState(ctx context.Context, q Request, out Response) ( } want := q.Compute if q.Operation == "create" { - want, e = p.Initial(ctx, q.Reference) + want, e = p.nativeInitial(ctx, q.Reference) if e != nil { return State{}, e } @@ -173,7 +173,7 @@ func (p *Provider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Inf return result, err } func (p *Provider) GetInfo(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { - c, e := p.Initial(ctx, r) + c, e := p.nativeInitial(ctx, r) if e != nil { return sandbox.Info{}, e } @@ -192,34 +192,37 @@ func (p *Provider) Renew(ctx context.Context, r sandbox.Reference) (sandbox.Info return p.GetInfo(ctx, r) } func (p *Provider) Kill(ctx context.Context, r sandbox.Reference) error { - c, e := p.Initial(ctx, r) + c, e := p.nativeInitial(ctx, r) if e != nil { return e } - return p.KillCompute(ctx, r, c) + return p.nativeKillCompute(ctx, r, c) } func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { - compute, e := p.Initial(ctx, r) + compute, e := p.nativeInitial(ctx, r) if e != nil { return sandbox.CommandResult{}, e } - return p.RunCommandCompute(ctx, r, compute, c) + return p.nativeRunCommandCompute(ctx, r, compute, c) } -func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { +func (p *Provider) nativeGetCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { return p.state(ctx, Request{Operation: "inspect", Reference: r, Compute: c}) } -func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c Compute) error { +func (p *Provider) nativeKillCompute(ctx context.Context, r sandbox.Reference, c Compute) error { _, e := p.call(ctx, Request{Operation: "kill", Reference: r, Compute: c}) return e } -func (p *Provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s SnapshotIdentity) error { +func (p *Provider) nativeDeleteSnapshot(ctx context.Context, r sandbox.Reference, s SnapshotIdentity) error { _, e := p.call(ctx, Request{Operation: "delete_snapshot", Reference: r, Snapshot: &s}) return e } -func (p *Provider) Suspend(ctx context.Context, q SuspendRequest) (State, error) { +func (p *Provider) nativeSuspend(ctx context.Context, q SuspendRequest) (State, error) { return p.state(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}) } -func (p *Provider) Resume(ctx context.Context, q ResumeRequest) (State, error) { +func (p *Provider) nativeResume(ctx context.Context, q ResumeRequest) (State, error) { + if p.config.ExternalWorkspace != (q.Workspace != nil) { + return State{}, sandbox.ErrInvalid + } workspace, err := p.resolveWorkspace(ctx, q.Reference, q.Workspace) if err != nil { return State{}, err @@ -227,7 +230,7 @@ func (p *Provider) Resume(ctx context.Context, q ResumeRequest) (State, error) { q.Workspace = nil return p.state(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q, Workspace: workspace}) } -func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c Compute, command sandbox.Command) (sandbox.CommandResult, error) { +func (p *Provider) nativeRunCommandCompute(ctx context.Context, r sandbox.Reference, c Compute, command sandbox.Command) (sandbox.CommandResult, error) { out, e := p.call(ctx, Request{Operation: "command", Reference: r, Compute: c, Command: &command}) if e != nil { return sandbox.CommandResult{}, e @@ -240,11 +243,11 @@ func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c // ResumeCompute thaws the exact resident source after an aborted suspension. // It never starts stopped compute or restores a checkpoint. -func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { +func (p *Provider) nativeResumeCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { return p.state(ctx, Request{Operation: "resume_compute", Reference: r, Compute: c}) } -func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { +func (p *Provider) nativeNewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err } diff --git a/services/core/internal/sandbox/microsandbox/provider_test.go b/services/core/internal/sandbox/microsandbox/provider_test.go index db67511af..8b5f1153e 100644 --- a/services/core/internal/sandbox/microsandbox/provider_test.go +++ b/services/core/internal/sandbox/microsandbox/provider_test.go @@ -44,7 +44,7 @@ func TestRejectsChangedComputeIdentity(t *testing.T) { if e != nil { t.Fatal(e) } - _, e = p.GetCompute(deadline(t), r, Compute{Name: Name(c, r, 0), ID: "local:4"}) + _, e = p.nativeGetCompute(deadline(t), r, Compute{Name: Name(c, r, 0), ID: "local:4"}) if !errors.Is(e, sandbox.ErrOwnership) { t.Fatalf("foreign identity accepted: %v", e) } @@ -60,7 +60,7 @@ func TestRestoreMustRetainExactProvenance(t *testing.T) { got.RestoredFrom = &v return Response{Version: ProtocolVersion, State: &State{Compute: got, Status: "running", BootstrapComplete: true}}, nil })) - _, e := p.Resume(deadline(t), ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: target}) + _, e := p.nativeResume(deadline(t), ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: target}) if !errors.Is(e, sandbox.ErrOwnership) { t.Fatalf("different snapshot accepted: %v", e) } @@ -70,7 +70,7 @@ func TestRejectsSnapshotPathBeforeHelper(t *testing.T) { s.Reference = "/foreign/checkpoint" calls := 0 p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) - if e := p.DeleteSnapshot(deadline(t), r, s); !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { + if e := p.nativeDeleteSnapshot(deadline(t), r, s); !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { t.Fatalf("e=%v calls=%d", e, calls) } } @@ -83,7 +83,7 @@ func TestObserveOnlyPreservesCapturedButResidentState(t *testing.T) { } return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: "paused", Snapshot: &s}}, nil })) - got, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + got, e := p.nativeSuspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) if e != nil || got.SourceStopped || got.Status != "paused" { t.Fatalf("state=%+v error=%v", got, e) } @@ -117,11 +117,11 @@ func TestNoDeadlineOrForeignAllocationNeverCallsHelper(t *testing.T) { r := testRef() foreign := r foreign.EnvironmentID = "77777777-7777-4777-8777-777777777777" - initial, initialErr := p.Initial(deadline(t), r) + initial, initialErr := p.nativeInitial(deadline(t), r) if initialErr != nil { t.Fatal(initialErr) } - _, e = p.GetCompute(deadline(t), foreign, initial) + _, e = p.nativeGetCompute(deadline(t), foreign, initial) if !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { t.Fatalf("e=%v calls=%d", e, calls) } @@ -146,7 +146,7 @@ func TestMissingSnapshotObservationAllowsOnlyIntactSourceRollback(t *testing.T) p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: status, BootstrapComplete: true}}, nil })) - _, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + _, e := p.nativeSuspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) if status == "running" || status == "paused" { if e != nil { t.Fatal(e) diff --git a/services/core/internal/sandbox/microsandbox/suspension.go b/services/core/internal/sandbox/microsandbox/suspension.go new file mode 100644 index 000000000..16dd58719 --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/suspension.go @@ -0,0 +1,137 @@ +package microsandbox + +import ( + "context" + "encoding/json" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// retained translates a verified native snapshot without exposing its schema to Core. +func retained(s SnapshotIdentity) sandbox.RetainedState { + raw, _ := json.Marshal(s) + return sandbox.RetainedState{Compatibility: s.Compatibility, Reference: s.Reference, ID: s.ID, OperationID: s.OperationID, SourceGeneration: s.SourceGeneration, SourceName: s.SourceName, SourceID: s.SourceID, Data: string(raw)} +} +func (p *Provider) snapshot(r sandbox.Reference, s sandbox.RetainedState) (SnapshotIdentity, error) { + var native SnapshotIdentity + if sandbox.ValidateRetained(s) != nil || json.Unmarshal([]byte(s.Data), &native) != nil || retained(native) != s || ValidateSnapshot(p.config, r, native) != nil { + return native, sandbox.ErrOwnership + } + return native, nil +} +func compute(c Compute) sandbox.Compute { + out := sandbox.Compute{Generation: c.Generation, Name: c.Name, ID: c.ID} + if c.RestoredFrom != nil { + s := retained(*c.RestoredFrom) + out.RestoredFrom = &s + } + return out +} +func (p *Provider) nativeCompute(r sandbox.Reference, c sandbox.Compute) (Compute, error) { + out := Compute{Generation: c.Generation, Name: c.Name, ID: c.ID} + if c.RestoredFrom != nil { + s, e := p.snapshot(r, *c.RestoredFrom) + if e != nil { + return out, e + } + out.RestoredFrom = &s + } + if ValidateCompute(p.config, r, out) != nil { + return out, sandbox.ErrOwnership + } + return out, nil +} +func state(s State) sandbox.ComputeState { + out := sandbox.ComputeState{Compute: compute(s.Compute), Status: s.Status, BootstrapComplete: s.BootstrapComplete, ResourcesReleased: s.SourceStopped, RestoreAttemptClosed: s.RestoreAttemptClosed} + if s.Snapshot != nil { + v := retained(*s.Snapshot) + out.Retained = &v + } + return out +} +func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Compute, error) { + c, e := p.nativeInitial(ctx, r) + return compute(c), e +} +func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.RetainedState) (sandbox.Compute, error) { + var snap *SnapshotIdentity + if s != nil { + v, e := p.snapshot(r, *s) + if e != nil { + return sandbox.Compute{}, e + } + snap = &v + } + c, e := p.nativeNewCompute(ctx, r, g, snap) + return compute(c), e +} +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.ComputeState{}, e + } + s, e := p.nativeGetCompute(ctx, r, n) + return state(s), e +} +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.GetCompute(ctx, r, c) +} +func (p *Provider) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + c, e := p.nativeCompute(q.Reference, q.Source) + if e != nil { + return sandbox.ComputeState{}, e + } + n := SuspendRequest{Reference: q.Reference, OperationID: q.OperationID, Source: c, ObserveOnly: q.ReconcileOnly} + if q.Retained != nil { + s, e := p.snapshot(q.Reference, *q.Retained) + if e != nil { + return sandbox.ComputeState{}, e + } + n.Snapshot = &s + } + s, e := p.nativeSuspend(ctx, n) + out := state(s) + // The helper holds the allocation lock until the original native work settles. + out.SuspendSettled = e == nil + return out, e +} +func (p *Provider) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + c, e := p.nativeCompute(q.Reference, q.Target) + if e != nil { + return sandbox.ComputeState{}, e + } + s, e := p.snapshot(q.Reference, q.Retained) + if e != nil { + return sandbox.ComputeState{}, e + } + v, e := p.nativeResume(ctx, ResumeRequest{Workspace: q.Workspace, Reference: q.Reference, OperationID: q.OperationID, Snapshot: s, Target: c, ObserveOnly: q.ReconcileOnly}) + return state(v), e +} +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { + n, e := p.nativeCompute(r, c) + if e != nil { + return e + } + return p.nativeKillCompute(ctx, r, n) +} +func (p *Provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + n, e := p.snapshot(r, s) + if e != nil { + return e + } + return p.nativeDeleteSnapshot(ctx, r, n) +} +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, q sandbox.Command) (sandbox.CommandResult, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.CommandResult{}, e + } + return p.nativeRunCommandCompute(ctx, r, n, q) +} +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.ComputeState{}, e + } + v, e := p.nativeResumeCompute(ctx, r, n) + return state(v), e +} diff --git a/services/core/internal/sandbox/microsandbox/suspension_contract_test.go b/services/core/internal/sandbox/microsandbox/suspension_contract_test.go new file mode 100644 index 000000000..ce0bb688a --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/suspension_contract_test.go @@ -0,0 +1,52 @@ +package microsandbox + +import ( + "context" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "testing" +) + +func TestSharedSuspendSettlesExactSourceCleanupInsideAdapter(t *testing.T) { + for _, reconcile := range []bool{false, true} { + for _, cleanupFails := range []bool{false, true} { + c, r, snap := testConfig(), testRef(), testSnapshot() + source := Compute{Name: snap.SourceName, ID: snap.SourceID} + calls := []string{} + p, e := NewWithCaller(c, callerFunc(func(_ context.Context, q Request) (Response, error) { + calls = append(calls, q.Operation) + if q.Operation == "suspend" { + if q.Suspend.ObserveOnly != reconcile { + t.Fatal("reconcile intent changed") + } + return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: "suspended", BootstrapComplete: true, Snapshot: &snap, SourceStopped: !cleanupFails}}, nil + } + t.Fatal("unexpected extra native operation", q.Operation) + return Response{}, nil + })) + if e != nil { + t.Fatal(e) + } + q := sandbox.SuspendRequest{Reference: r, OperationID: snap.OperationID, Source: compute(source), ReconcileOnly: reconcile} + got, e := p.Suspend(deadline(t), q) + if len(calls) != 1 || calls[0] != "suspend" { + t.Fatal(calls) + } + if cleanupFails { + if got.ResourcesReleased || (e == nil && sandbox.ValidateSuspendResult(q, got) == nil) { + t.Fatal("failed cleanup released capacity") + } + continue + } + if e != nil { + t.Fatal(e) + } + if e = sandbox.ValidateSuspendResult(q, got); e != nil { + t.Fatal(e) + } + if got.Retained == nil || got.Retained.Data == "" { + t.Fatal("missing native proof") + } + } + } +} diff --git a/services/core/internal/sandbox/microsandbox/types.go b/services/core/internal/sandbox/microsandbox/types.go index 43442c229..012a9f56a 100644 --- a/services/core/internal/sandbox/microsandbox/types.go +++ b/services/core/internal/sandbox/microsandbox/types.go @@ -4,6 +4,7 @@ package microsandbox import ( "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -43,12 +44,55 @@ type NetworkPolicy struct { } type NetworkRule struct{ Action, Direction, Destination, Protocol, Port string } -// These aliases keep the helper wire private while Core uses provider-neutral types. -type Compute = sandbox.Compute -type SnapshotIdentity = sandbox.SnapshotIdentity -type State = sandbox.ComputeState -type SuspendRequest = sandbox.SuspendRequest -type ResumeRequest = sandbox.ResumeRequest +// Native snapshot wire types belong to this adapter. +type Compute struct { + Generation uint64 + Name string + ID string + RestoredFrom *SnapshotIdentity +} + +// SnapshotIdentity is provider evidence from a verified full snapshot. Core +// persists it unchanged and records consumption separately; it never invents +// paths, checksums, native checkpoint fields, or source identity. +type SnapshotIdentity struct { + Compatibility sandbox.CheckpointCompatibility + Reference string + ID string + Digest string + CheckpointID string + CheckpointRoot string + OperationID string + SourceGeneration uint64 + SourceName string + SourceID string +} + +type State struct { + RestoreAttemptClosed string + Compute Compute + Status string + BootstrapComplete bool + Snapshot *SnapshotIdentity + SourceStopped bool +} +type SuspendRequest struct { + Reference sandbox.Reference + OperationID string + Source Compute + Snapshot *SnapshotIdentity + // Recovery observes the previous attempt and never starts a new capture. + ObserveOnly bool +} +type ResumeRequest struct { + Workspace *workspacefs.Binding `json:",omitempty"` + Reference sandbox.Reference + OperationID string + Snapshot SnapshotIdentity + Target Compute + // Recovery observes the previous target and never starts a new restore. + ObserveOnly bool +} // Request and Response are the finite, private helper boundary. Confidential // Bootstrap and Command bytes travel only through stdin and are never logged. @@ -97,3 +141,11 @@ type Metrics struct { type Caller interface { Call(context.Context, Request) (Response, error) } + +// ClosesRestoreAttempt verifies the exact never-dispatched private helper attempt. +func (s State) ClosesRestoreAttempt(q ResumeRequest) bool { + return q.ObserveOnly && q.OperationID != "" && s.RestoreAttemptClosed == q.OperationID && s.Status == "absent" && + s.Compute.ID == "" && q.Target.ID == "" && s.Compute.Name == q.Target.Name && s.Compute.Generation == q.Target.Generation && + s.Compute.RestoredFrom != nil && q.Target.RestoredFrom != nil && *s.Compute.RestoredFrom == q.Snapshot && *q.Target.RestoredFrom == q.Snapshot && + !s.BootstrapComplete && !s.SourceStopped && s.Snapshot == nil +} diff --git a/services/core/internal/sandbox/microsandbox/workspace_test.go b/services/core/internal/sandbox/microsandbox/workspace_test.go index 64ba57717..bc74b2aec 100644 --- a/services/core/internal/sandbox/microsandbox/workspace_test.go +++ b/services/core/internal/sandbox/microsandbox/workspace_test.go @@ -42,17 +42,17 @@ func TestWorkspaceResolveBeforeRestoreAndRetainPartialTarget(t *testing.T) { return workspacefs.Directory{Path: "/resolved/environment"}, nil }) q := ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: Compute{Generation: 1, Name: Name(c, r, 1), RestoredFrom: &s}, Workspace: workspaceBinding()} - state, err := p.Resume(deadline(t), q) + state, err := p.nativeResume(deadline(t), q) if !errors.Is(err, ErrUnconfirmed) || state.Compute.ID != "local:partial" || resolves != 1 || calls != 1 { t.Fatal(state, err, resolves, calls) } q.ObserveOnly = true - _, _ = p.Resume(deadline(t), q) + _, _ = p.nativeResume(deadline(t), q) if resolves != 2 { t.Fatal("restore observation did not resolve original binding") } q.Workspace.Attachment.Reference.EnvironmentID = r.TenantID - _, err = p.Resume(deadline(t), q) + _, err = p.nativeResume(deadline(t), q) if !errors.Is(err, workspacefs.ErrOwnership) || calls != 2 { t.Fatal("foreign binding reached helper", err, calls) } @@ -62,7 +62,7 @@ func TestWorkspaceDoesNotFallbackWithoutResolver(t *testing.T) { config := testConfig() config.ExternalWorkspace = true p, _ := NewWithCaller(config, callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) - _, err := p.Create(deadline(t), sandbox.Bootstrap{Reference: testRef(), SessionID: testRef().TenantID, DeviceID: testRef().EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()}) + _, err := p.Create(deadline(t), sandbox.Bootstrap{Reference: testRef(), SessionID: testRef().TenantID, DeviceID: testRef().EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "disabled", Workspace: workspaceBinding()}) if !errors.Is(err, workspacefs.ErrUnsupported) || calls != 0 { t.Fatal(err, calls) } @@ -81,7 +81,7 @@ func TestPreNativeWorkspaceFailureSettlesAbsentCreation(t *testing.T) { provider.workspace = workspaceResolverFunc(func(context.Context, workspacefs.Binding) (workspacefs.Directory, error) { return workspacefs.Directory{}, workspacefs.ErrUnavailable }) - bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()} + bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "disabled", Workspace: workspaceBinding()} ctx := deadline(t) switch failure { case "mode": diff --git a/services/core/internal/sandbox/node/checkpoint_generation_test.go b/services/core/internal/sandbox/node/checkpoint_generation_test.go index 5f5bd6251..b54341383 100644 --- a/services/core/internal/sandbox/node/checkpoint_generation_test.go +++ b/services/core/internal/sandbox/node/checkpoint_generation_test.go @@ -89,8 +89,8 @@ func TestClosedRestoreEvidenceSurvivesNodeTransport(t *testing.T) { } }() wait(t, func() bool { return hub.Online(id.NodeID) }) - snapshot := sandbox.SnapshotIdentity{ID: "snapshot", Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "store", ExecutionClass: "class"}} - q := sandbox.ResumeRequest{Reference: reference(), OperationID: uuid.NewString(), Snapshot: snapshot, Target: sandbox.Compute{Generation: 1, Name: "exact-target", RestoredFrom: &snapshot}, ObserveOnly: true} + snapshot := sandbox.RetainedState{ID: "snapshot", Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "store", ExecutionClass: "class"}} + q := sandbox.ResumeRequest{Reference: reference(), OperationID: uuid.NewString(), Retained: snapshot, Target: sandbox.Compute{Generation: 1, Name: "exact-target", RestoredFrom: &snapshot}, ReconcileOnly: true} result, err := hub.Proxy(id.NodeID, microsandbox.Operations(), id.DeploymentGeneration).Resume(t.Context(), q) if err != nil || !result.ClosesRestoreAttempt(q) { t.Fatalf("exact closed proof lost in proxy: %#v %v", result, err) diff --git a/services/core/internal/sandbox/node/connection_test.go b/services/core/internal/sandbox/node/connection_test.go index abee501b3..931d1e78e 100644 --- a/services/core/internal/sandbox/node/connection_test.go +++ b/services/core/internal/sandbox/node/connection_test.go @@ -179,7 +179,7 @@ func TestCopiedIdentityCannotReplaceNodeWithInflightCreate(t *testing.T) { proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) r := reference() created := make(chan error, 1) - go func() { _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: r}); created <- err }() + go func() { _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: r, Harness: "codex"}); created <- err }() <-original.started stopDuplicate, duplicateDone := start(duplicateDir, duplicate) defer func() { diff --git a/services/core/internal/sandbox/node/creation_settlement_test.go b/services/core/internal/sandbox/node/creation_settlement_test.go index 09d8daec5..cc0048506 100644 --- a/services/core/internal/sandbox/node/creation_settlement_test.go +++ b/services/core/internal/sandbox/node/creation_settlement_test.go @@ -76,7 +76,7 @@ func TestNodeCarriesCreationSettlementWithoutConvertingFailureToSuccess(t *testi proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) for _, operation := range []func(context.Context) (sandbox.Info, error){ func(ctx context.Context) (sandbox.Info, error) { - return proxy.Create(ctx, sandbox.Bootstrap{Reference: ref}) + return proxy.Create(ctx, sandbox.Bootstrap{Reference: ref, Harness: "codex"}) }, func(ctx context.Context) (sandbox.Info, error) { return proxy.GetInfo(ctx, ref) }, } { diff --git a/services/core/internal/sandbox/node/docker_live_test.go b/services/core/internal/sandbox/node/docker_live_test.go index 48695bfff..3c59d3bc7 100644 --- a/services/core/internal/sandbox/node/docker_live_test.go +++ b/services/core/internal/sandbox/node/docker_live_test.go @@ -87,7 +87,7 @@ func TestDockerNodeTransportLifecycle(t *testing.T) { }() ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-node-transport-credential", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-node-transport-credential", Harness: "codex", NetworkAccess: "enabled"} callCtx, callCancel := context.WithTimeout(ctx, 25*time.Second) info, err := proxy.Create(callCtx, b) callCancel() diff --git a/services/core/internal/sandbox/node/generation_connection_test.go b/services/core/internal/sandbox/node/generation_connection_test.go index 1c3e59d6c..efc41798d 100644 --- a/services/core/internal/sandbox/node/generation_connection_test.go +++ b/services/core/internal/sandbox/node/generation_connection_test.go @@ -69,7 +69,7 @@ func TestGenerationWireRoutesOldOwnershipAndCurrentTargetSeparately(t *testing.T t.Fatal("retained generation info failed", generation, err) } if generation != 9 { - if _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: ref}); err != nil { + if _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: ref, Harness: "codex"}); err != nil { t.Fatal("exact ready provider Create failed", generation, err) } } diff --git a/services/core/internal/sandbox/node/generation_json.go b/services/core/internal/sandbox/node/generation_json.go index 06ae10fff..d3a68ef7b 100644 --- a/services/core/internal/sandbox/node/generation_json.go +++ b/services/core/internal/sandbox/node/generation_json.go @@ -136,9 +136,15 @@ func validateGenerationJSON(raw []byte, kind string) error { } } if value := values["request"]; value != nil { - if _, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume snapshot observation", ""); err != nil { + request, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume retained observation", "") + if err != nil { return err } + if bootstrap := request["bootstrap"]; bootstrap != nil { + if _, err := generationObject(bootstrap, "TenantID EnvironmentID AllocationID SessionID DeviceID CoreURL Credential Harness NetworkAccess AllowedDomains", "Workspace", "AllowedDomains Workspace"); err != nil { + return err + } + } } if value := values["response"]; value != nil { if _, err := generationObject(value, "id connection_id", "error_code unsupported info compute state command sample", ""); err != nil { diff --git a/services/core/internal/sandbox/node/generation_json_test.go b/services/core/internal/sandbox/node/generation_json_test.go index 2f0f7e069..37fbd8a3f 100644 --- a/services/core/internal/sandbox/node/generation_json_test.go +++ b/services/core/internal/sandbox/node/generation_json_test.go @@ -64,7 +64,7 @@ func TestGenerationHealthRejectsUnboundedOrAmbiguousNumbers(t *testing.T) { } func TestNodeProtocolRejectsHistoricalVersions(t *testing.T) { - for _, version := range []int{1, 2, 3, 4, 5} { + for _, version := range []int{1, 2, 3, 4, 5, 6, 7} { raw, _ := json.Marshal(frame{Version: version, Type: "hello", Identity: new(Identity), Health: &Health{ObservedAt: time.Now().UTC()}}) if _, err := decodeFrame(raw); err == nil { t.Fatalf("accepted historical protocol %d", version) @@ -93,6 +93,26 @@ func TestNodeGenerationManagementIsAnExplicitCurrentCapability(t *testing.T) { } } +func TestNodeBootstrapRejectsAmbiguousHarness(t *testing.T) { + r := sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()} + b := sandbox.Bootstrap{Reference: r, Harness: "codex"} + f := frame{Version: ProtocolVersion, Type: "request", Request: &request{DeploymentGeneration: 1, ID: uuid.NewString(), Sequence: 1, ConnectionID: uuid.NewString(), OwnerEpoch: 1, Operation: "create", TimeoutMillis: 1000, Reference: r, Bootstrap: &b}} + raw, _ := json.Marshal(f) + good := string(raw) + if _, err := decodeFrame(raw); err != nil { + t.Fatal("valid selected bootstrap", err) + } + for _, replacement := range []string{`"Harness":null`, `"Harness":""`, `"harness":"codex"`, `"Harness":"codex","Harness":"mcode"`, `"Harness":"codex","harness":"mcode"`, `"Harness":"Codex"`, `"Harness":42`} { + bad := strings.Replace(good, `"Harness":"codex"`, replacement, 1) + if decoded, err := decodeFrame([]byte(bad)); err == nil && decoded.Request.validate() == nil { + t.Fatal("ambiguous selection reached dispatch", replacement) + } + } + if _, err := decodeFrame([]byte(strings.Replace(good, `"Harness":"codex",`, "", 1))); err == nil { + t.Fatal("missing Harness accepted") + } +} + func TestCheckpointGenerationHealthJSONRoundTrip(t *testing.T) { for _, managed := range []bool{false, true} { for _, kind := range []string{"hello", "heartbeat"} { diff --git a/services/core/internal/sandbox/node/generations.go b/services/core/internal/sandbox/node/generations.go index 9e44ef959..c28029a8d 100644 --- a/services/core/internal/sandbox/node/generations.go +++ b/services/core/internal/sandbox/node/generations.go @@ -362,7 +362,7 @@ func (m *GenerationManager) probeLoop() { } ctx, cancel := context.WithTimeout(m.ctx, 5*time.Second) checkpoint, err := g.value.Probe(ctx) - if err == nil && ((sandbox.SupportsCheckpoint(g.value.Provider) && (checkpoint == nil || checkpoint.Validate() != nil)) || (!sandbox.SupportsCheckpoint(g.value.Provider) && checkpoint != nil)) { + if err == nil && ((sandbox.SupportsSuspension(g.value.Provider) && (checkpoint == nil || checkpoint.Validate() != nil)) || (!sandbox.SupportsSuspension(g.value.Provider) && checkpoint != nil)) { err = sandbox.ErrInvalid } cancel() diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index 946470389..1b21674ca 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -129,7 +129,10 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing createCtx, stopCreate := context.WithTimeout(ctx, 150*time.Millisecond) defer stopCreate() createDone := make(chan error, 1) - go func() { _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r}); createDone <- err }() + go func() { + _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r, Harness: "codex"}) + createDone <- err + }() <-p.started if err := <-createDone; !errors.Is(err, sandbox.ErrComputeUnconfirmed) { t.Fatalf("lost reply = %v", err) @@ -166,11 +169,11 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing } } -func TestOfflineIsUnknownAndDockerDoesNotAdvertiseCheckpoint(t *testing.T) { +func TestOfflineIsUnknownAndDockerDoesNotAdvertiseSuspension(t *testing.T) { h := NewHub(HubOptions{OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) p := h.Proxy(uuid.NewString(), docker.Operations(), 1) - if sandbox.SupportsCheckpoint(p) { - t.Fatal("docker advertised checkpoint") + if sandbox.SupportsSuspension(p) { + t.Fatal("docker advertised suspension") } _, err := p.GetInfo(context.Background(), reference()) if !errors.Is(err, sandbox.ErrComputeUnconfirmed) || errors.Is(err, sandbox.ErrNotFound) { diff --git a/services/core/internal/sandbox/node/operations.go b/services/core/internal/sandbox/node/operations.go index 8f016664b..aca02dcb5 100644 --- a/services/core/internal/sandbox/node/operations.go +++ b/services/core/internal/sandbox/node/operations.go @@ -10,10 +10,11 @@ var operationMethods = map[string]string{ "initial": "Initial", "new_compute": "NewCompute", "compute": "GetCompute", + "renew_compute": "RenewCompute", "suspend": "Suspend", "resume": "Resume", "kill_compute": "KillCompute", - "delete_snapshot": "DeleteSnapshot", + "delete_retained": "DeleteRetained", "command_compute": "RunCommandCompute", "resume_compute": "ResumeCompute", "observe": "Observe", diff --git a/services/core/internal/sandbox/node/operations_test.go b/services/core/internal/sandbox/node/operations_test.go index e508988b5..f0567ad1c 100644 --- a/services/core/internal/sandbox/node/operations_test.go +++ b/services/core/internal/sandbox/node/operations_test.go @@ -49,9 +49,63 @@ func TestUnsupportedProxyRejectsBeforeNodeResolution(t *testing.T) { } } func TestNodeOperationMappingCoversForwardedMethods(t *testing.T) { - for _, method := range []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand", "Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute", "Observe"} { + for _, method := range []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand", "Initial", "NewCompute", "GetCompute", "RenewCompute", "Suspend", "Resume", "KillCompute", "DeleteRetained", "RunCommandCompute", "ResumeCompute", "Observe"} { if wire := operationWire(method); wire == "" || operationMethod(wire) != method { t.Fatal(method) } } } + +// Retained handles and renew are part of the allocation protocol, including on +// nodes whose implementation embeds no optional lifecycle interface. +type retainedWireProvider struct { + fakeProvider + renewed sandbox.Compute + deleted sandbox.RetainedState +} + +func (p *retainedWireProvider) ProviderOperations() providercontract.Operations { + operations := p.fakeProvider.ProviderOperations() + operations["RenewCompute"] = providercontract.Support{State: providercontract.Supported} + operations["DeleteRetained"] = providercontract.Support{State: providercontract.Supported} + return operations +} +func (p *retainedWireProvider) RenewCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + p.renewed = c + return sandbox.ComputeState{Compute: c, Status: "running"}, nil +} +func (p *retainedWireProvider) DeleteRetained(_ context.Context, _ sandbox.Reference, retained sandbox.RetainedState) error { + p.deleted = retained + return nil +} +func TestRetainedLifecycleDispatchSurvivesStrictWire(t *testing.T) { + ref := reference() + retained := sandbox.RetainedState{Reference: ref.AllocationID, ID: uuid.NewString(), OperationID: uuid.NewString(), SourceName: ref.AllocationID, SourceID: "native", Data: "opaque"} + compute := sandbox.Compute{Generation: 1, Name: ref.AllocationID, ID: "native", RestoredFrom: &retained} + p := new(retainedWireProvider) + for _, operation := range []string{"renew_compute", "delete_retained"} { + q := request{DeploymentGeneration: 1, ID: uuid.NewString(), Sequence: 1, ConnectionID: uuid.NewString(), OwnerEpoch: 1, Operation: operation, TimeoutMillis: 1000, Reference: ref} + if operation == "renew_compute" { + q.Compute = &compute + } else { + q.Retained = &retained + } + raw, err := json.Marshal(frame{Version: ProtocolVersion, Type: "request", Request: &q}) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeFrame(raw) + if err != nil { + t.Fatal(err) + } + if err := decoded.Request.validate(); err != nil { + t.Fatal(err) + } + if out := execute(t.Context(), p, *decoded.Request); responseError(out) != nil { + t.Fatal(out) + } + } + if p.renewed.ID != compute.ID || p.renewed.RestoredFrom == nil || *p.renewed.RestoredFrom != retained || p.deleted != retained { + t.Fatal("retained evidence lost in transport") + } +} diff --git a/services/core/internal/sandbox/node/provider_operations_fixture_test.go b/services/core/internal/sandbox/node/provider_operations_fixture_test.go index 3f892e9bb..fdcb82c42 100644 --- a/services/core/internal/sandbox/node/provider_operations_fixture_test.go +++ b/services/core/internal/sandbox/node/provider_operations_fixture_test.go @@ -17,11 +17,12 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -30,7 +31,7 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { func (*fakeProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*fakeProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*fakeProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*fakeProvider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (*fakeProvider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Com func (*fakeProvider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*fakeProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*fakeProvider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*fakeProvider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -66,13 +67,18 @@ func (*observationProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Supported}, } } + +func (*fakeProvider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/sandbox/node/proxy.go b/services/core/internal/sandbox/node/proxy.go index 4949c3795..c345a50c1 100644 --- a/services/core/internal/sandbox/node/proxy.go +++ b/services/core/internal/sandbox/node/proxy.go @@ -105,8 +105,8 @@ func (p *provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Co } return *out.Compute, nil } -func (p *provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.SnapshotIdentity) (sandbox.Compute, error) { - out, e := p.call(ctx, request{Operation: "new_compute", Reference: r, Generation: g, Snapshot: s}) +func (p *provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.RetainedState) (sandbox.Compute, error) { + out, e := p.call(ctx, request{Operation: "new_compute", Reference: r, Generation: g, Retained: s}) if e != nil { return sandbox.Compute{}, e } @@ -147,8 +147,8 @@ func (p *provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandb _, e := p.call(ctx, request{Operation: "kill_compute", Reference: r, Compute: &c}) return e } -func (p *provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s sandbox.SnapshotIdentity) error { - _, e := p.call(ctx, request{Operation: "delete_snapshot", Reference: r, Snapshot: &s}) +func (p *provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + _, e := p.call(ctx, request{Operation: "delete_retained", Reference: r, Retained: &s}) return e } func (p *provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, v sandbox.Command) (sandbox.CommandResult, error) { @@ -164,3 +164,7 @@ func (p *provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c san func (h *Hub) GenerationProvider(declared providercontract.Operations, resolve func(context.Context, sandbox.Reference) (string, uint64, error)) sandbox.SandboxProvider { return &provider{hub: h, operations: maps.Clone(declared), resolveGeneration: resolve} } + +func (p *provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, request{Operation: "renew_compute", Reference: r, Compute: &c}) +} diff --git a/services/core/internal/sandbox/node/recovery_test.go b/services/core/internal/sandbox/node/recovery_test.go index 78d106da3..c5039ea9d 100644 --- a/services/core/internal/sandbox/node/recovery_test.go +++ b/services/core/internal/sandbox/node/recovery_test.go @@ -236,7 +236,7 @@ func TestDegradedNodeRetainsObservationAndCleanup(t *testing.T) { } proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) r := reference() - if _, err = proxy.Create(ctx, sandbox.Bootstrap{Reference: r}); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { + if _, err = proxy.Create(ctx, sandbox.Bootstrap{Reference: r, Harness: "codex"}); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { t.Fatalf("create = %v", err) } if _, err = proxy.GetInfo(ctx, r); err != nil { diff --git a/services/core/internal/sandbox/node/timeout_test.go b/services/core/internal/sandbox/node/timeout_test.go index 15f8da3c5..aa6184be5 100644 --- a/services/core/internal/sandbox/node/timeout_test.go +++ b/services/core/internal/sandbox/node/timeout_test.go @@ -108,7 +108,10 @@ func TestQueuedMutationExpiresWithoutExecution(t *testing.T) { createCtx, stopCreate := context.WithTimeout(ctx, 3*time.Second) defer stopCreate() created := make(chan error, 1) - go func() { _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r}); created <- err }() + go func() { + _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r, Harness: "codex"}) + created <- err + }() <-p.started killCtx, stopKill := context.WithTimeout(ctx, 80*time.Millisecond) defer stopKill() diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index e2fe64b2c..31bfeefa6 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -11,6 +11,7 @@ import ( "io" "time" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" @@ -18,7 +19,7 @@ import ( "github.com/gorilla/websocket" ) -const ProtocolVersion = 6 +const ProtocolVersion = 8 const MaxControlFrameBytes = 32 * 1024 const MaxFrameBytes = 72 * 1024 * 1024 const maxPending = 32 @@ -86,15 +87,15 @@ type request struct { TimeoutMillis int64 `json:"timeout_ms"` // deadline is anchored to the receiving host and never crosses the wire. deadline time.Time - Reference sandbox.Reference `json:"reference"` - Bootstrap *sandbox.Bootstrap `json:"bootstrap,omitempty"` - Compute *sandbox.Compute `json:"compute,omitempty"` - Generation uint64 `json:"generation,omitempty"` - Command *sandbox.Command `json:"command,omitempty"` - Suspend *sandbox.SuspendRequest `json:"suspend,omitempty"` - Resume *sandbox.ResumeRequest `json:"resume,omitempty"` - Snapshot *sandbox.SnapshotIdentity `json:"snapshot,omitempty"` - Observation *runtimeobs.Target `json:"observation,omitempty"` + Reference sandbox.Reference `json:"reference"` + Bootstrap *sandbox.Bootstrap `json:"bootstrap,omitempty"` + Compute *sandbox.Compute `json:"compute,omitempty"` + Generation uint64 `json:"generation,omitempty"` + Command *sandbox.Command `json:"command,omitempty"` + Suspend *sandbox.SuspendRequest `json:"suspend,omitempty"` + Resume *sandbox.ResumeRequest `json:"resume,omitempty"` + Retained *sandbox.RetainedState `json:"retained,omitempty"` + Observation *runtimeobs.Target `json:"observation,omitempty"` } type response struct { @@ -243,7 +244,7 @@ func (q request) validate() error { return sandbox.ErrInvalid } count := 0 - for _, ok := range []bool{q.Bootstrap != nil, q.Compute != nil, q.Command != nil, q.Suspend != nil, q.Resume != nil, q.Snapshot != nil, q.Observation != nil} { + for _, ok := range []bool{q.Bootstrap != nil, q.Compute != nil, q.Command != nil, q.Suspend != nil, q.Resume != nil, q.Retained != nil, q.Observation != nil} { if ok { count++ } @@ -254,7 +255,7 @@ func (q request) validate() error { return nil } case "create": - if count == 1 && q.Bootstrap != nil && q.Bootstrap.Reference == q.Reference && sandbox.ValidateWorkspaceBinding(q.Reference, q.Bootstrap.Workspace) == nil { + if count == 1 && q.Bootstrap != nil && q.Bootstrap.Reference == q.Reference && runtimebootstrap.ValidHarness(q.Bootstrap.Harness) && sandbox.ValidateWorkspaceBinding(q.Reference, q.Bootstrap.Workspace) == nil { return nil } case "info", "renew", "kill", "initial": @@ -262,10 +263,10 @@ func (q request) validate() error { return nil } case "new_compute": - if count == 0 || count == 1 && q.Snapshot != nil { + if count == 0 || count == 1 && q.Retained != nil { return nil } - case "compute", "kill_compute", "resume_compute": + case "compute", "renew_compute", "kill_compute", "resume_compute": if count == 1 && q.Compute != nil { return nil } @@ -285,8 +286,8 @@ func (q request) validate() error { if count == 1 && q.Resume != nil && q.Resume.Reference == q.Reference && sandbox.ValidateWorkspaceBinding(q.Reference, q.Resume.Workspace) == nil { return nil } - case "delete_snapshot": - if count == 1 && q.Snapshot != nil { + case "delete_retained": + if count == 1 && q.Retained != nil { return nil } } @@ -336,11 +337,14 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response compute, err = p.Initial(ctx, q.Reference) out.Compute = &compute case "new_compute": - compute, err = p.NewCompute(ctx, q.Reference, q.Generation, q.Snapshot) + compute, err = p.NewCompute(ctx, q.Reference, q.Generation, q.Retained) out.Compute = &compute case "compute": state, err = p.GetCompute(ctx, q.Reference, *q.Compute) out.State = &state + case "renew_compute": + state, err = p.RenewCompute(ctx, q.Reference, *q.Compute) + out.State = &state case "suspend": state, err = p.Suspend(ctx, *q.Suspend) out.State = &state @@ -349,8 +353,8 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response out.State = &state case "kill_compute": err = p.KillCompute(ctx, q.Reference, *q.Compute) - case "delete_snapshot": - err = p.DeleteSnapshot(ctx, q.Reference, *q.Snapshot) + case "delete_retained": + err = p.DeleteRetained(ctx, q.Reference, *q.Retained) case "resume_compute": state, err = p.ResumeCompute(ctx, q.Reference, *q.Compute) out.State = &state @@ -381,7 +385,7 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response } func requiresReady(q request) bool { - return q.Operation == "create" || q.Operation == "resume" && q.Resume != nil && !q.Resume.ObserveOnly + return q.Operation == "create" || q.Operation == "resume" && q.Resume != nil && !q.Resume.ReconcileOnly } // setTimeout consumes sender queue time without comparing clocks across hosts. diff --git a/services/core/internal/sandbox/node/workspace_test.go b/services/core/internal/sandbox/node/workspace_test.go index cfcfe34e4..3c4ff1f7a 100644 --- a/services/core/internal/sandbox/node/workspace_test.go +++ b/services/core/internal/sandbox/node/workspace_test.go @@ -30,10 +30,29 @@ func TestWorkspaceWireBindsConfigurationAndEnvironment(t *testing.T) { case "object": b.Attachment.Reference.ObjectID = "invalid" } - q := request{ID: ref.AllocationID, Reference: ref, Operation: "create", TimeoutMillis: 1000, Bootstrap: &sandbox.Bootstrap{Reference: ref, Workspace: &b}} + q := request{ID: ref.AllocationID, Reference: ref, Operation: "create", TimeoutMillis: 1000, Bootstrap: &sandbox.Bootstrap{Harness: "codex", Reference: ref, Workspace: &b}} if (q.validate() == nil) != (fault == "match") { t.Fatal("invalid create binding forwarding", fault) } + for _, external := range []bool{false, true} { + wireRequest := q + bootstrap := *q.Bootstrap + wireRequest.Bootstrap = &bootstrap + if !external { + bootstrap.Workspace = nil + } + wireRequest.DeploymentGeneration, wireRequest.Sequence, wireRequest.OwnerEpoch = 1, 1, 1 + wireRequest.ConnectionID = ref.TenantID + raw, err := json.Marshal(frame{Version: ProtocolVersion, Type: "request", Request: &wireRequest}) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeFrame(raw) + want := !external || fault == "match" + if (err == nil && decoded.Request.validate() == nil) != want { + t.Fatalf("workspace wire external=%v fault=%s: %v", external, fault, err) + } + } q.Operation = "resume" q.Bootstrap = nil q.Resume = &sandbox.ResumeRequest{Reference: ref, Workspace: &b} @@ -75,7 +94,7 @@ func TestActualWorkspaceRefusalKeepsSettledAbsenceOnWire(t *testing.T) { } fsConfig := workspacefs.Configuration{ID: "44444444-4444-4444-8444-444444444444", Adapter: "fixture", Parameters: json.RawMessage(`{}`)} binding := &workspacefs.Binding{Configuration: fsConfig, Attachment: workspacefs.Attachment{Reference: workspacefs.Reference{TenantID: ref.TenantID, EnvironmentID: ref.EnvironmentID, ObjectID: "55555555-5555-4555-8555-555555555555"}, ConfigurationID: fsConfig.ID, Kind: workspacefs.AttachmentHostDirectory, Native: json.RawMessage(`{}`)}} - bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: binding} + bootstrap := sandbox.Bootstrap{Harness: "codex", Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: binding} ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() out := execute(ctx, provider, request{ID: ref.AllocationID, ConnectionID: ref.TenantID, Reference: ref, Operation: "create", TimeoutMillis: 1000, Bootstrap: &bootstrap}) diff --git a/services/core/internal/sandbox/operations_test.go b/services/core/internal/sandbox/operations_test.go index 62d2c5b3f..c8f494397 100644 --- a/services/core/internal/sandbox/operations_test.go +++ b/services/core/internal/sandbox/operations_test.go @@ -24,7 +24,7 @@ func TestDeclarationsRejectMissingUnknownAndContradictoryOperations(t *testing.T name string change func(providercontract.Operations) }{ - {"omitted", func(o providercontract.Operations) { delete(o, "DeleteSnapshot") }}, + {"omitted", func(o providercontract.Operations) { delete(o, "DeleteRetained") }}, {"zero", func(o providercontract.Operations) { o["Observe"] = providercontract.Support{} }}, {"unknown", func(o providercontract.Operations) { o["FutureOperation"] = providercontract.Support{State: providercontract.Supported} diff --git a/services/core/internal/sandbox/providers/registration.go b/services/core/internal/sandbox/providers/registration.go index 5a5668d35..eb7c455a7 100644 --- a/services/core/internal/sandbox/providers/registration.go +++ b/services/core/internal/sandbox/providers/registration.go @@ -55,11 +55,6 @@ func ValidateRegistration(a Adapter) error { if err := sandbox.ValidateOperations(operations); err != nil { return err } - // The common lifecycle suspends only node allocations, so only a nodes - // registration may declare checkpoint support. - if operations["Initial"].State == providercontract.Supported && a.Mode != sandbox.DeploymentNodes { - return invalid("checkpoint support outside nodes mode") - } if a.Policy.DefaultResources != nil && a.ValidateResources(*a.Policy.DefaultResources) != nil { return invalid("default resources") } diff --git a/services/core/internal/sandbox/providers/registration_test.go b/services/core/internal/sandbox/providers/registration_test.go index 592bdf313..e5c20bc80 100644 --- a/services/core/internal/sandbox/providers/registration_test.go +++ b/services/core/internal/sandbox/providers/registration_test.go @@ -173,12 +173,12 @@ func TestRegistrationRejectsInvalidDefaultResources(t *testing.T) { } } -// Only a nodes registration may declare checkpoint support. -func TestRegistrationCheckpointRequiresNodes(t *testing.T) { +// Complete suspension support is independent of deployment placement. +func TestRegistrationSuspensionSupportsDirect(t *testing.T) { registry := Builtin() a := registry.adapters["microsandbox"] a.Mode, a.BuildLocal, a.NodeArtifacts, a.BuildDirect = "direct", nil, nil, registry.adapters["e2b"].BuildDirect - if err := ValidateRegistration(a); !errors.Is(err, providercontract.ErrContract) || !strings.Contains(err.Error(), "checkpoint") { - t.Fatal("direct checkpoint registration accepted", err) + if err := ValidateRegistration(a); err != nil { + t.Fatal("direct suspension registration rejected", err) } } diff --git a/services/core/internal/sandbox/providers/registry.go b/services/core/internal/sandbox/providers/registry.go index a39c0429f..bd13da340 100644 --- a/services/core/internal/sandbox/providers/registry.go +++ b/services/core/internal/sandbox/providers/registry.go @@ -88,8 +88,8 @@ func (r *Registry) BuildDirect(c sandbox.DirectConfig) (sandbox.SandboxProvider, return p, nil } -// SupportsCheckpoint reports whether the provider declares checkpoint suspension. -func (r *Registry) SupportsCheckpoint(kind string) (bool, error) { +// SupportsSuspension reports whether the provider declares checkpoint suspension. +func (r *Registry) SupportsSuspension(kind string) (bool, error) { a, err := r.Lookup(kind) if err != nil { return false, err diff --git a/services/core/internal/sandbox/providers/registry_test.go b/services/core/internal/sandbox/providers/registry_test.go index 47a0629bb..3edefa87e 100644 --- a/services/core/internal/sandbox/providers/registry_test.go +++ b/services/core/internal/sandbox/providers/registry_test.go @@ -19,7 +19,7 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { }{ {"docker", "nodes", "nodes", false}, {"microsandbox", "nodes", "nodes", true}, - {"e2b", "direct", "e2b", false}, + {"e2b", "direct", "e2b", true}, } { t.Run(tc.kind, func(t *testing.T) { d, err := registry.Describe(tc.kind, installation) @@ -27,7 +27,7 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { t.Fatalf("wrong mode or namespace: %+v %v", d, err) } a, err := registry.Lookup(tc.kind) - checkpoint, checkpointErr := registry.SupportsCheckpoint(tc.kind) + checkpoint, checkpointErr := registry.SupportsSuspension(tc.kind) if err != nil || checkpointErr != nil || checkpoint != tc.checkpoint || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { t.Fatal("inconsistent construction/capability registration", err, checkpointErr) } @@ -70,7 +70,7 @@ func TestNewRegistrationDoesNotNeedCoreDispatchChanges(t *testing.T) { // Registration is test-local: production registrations are fixed, never plugins. registry.adapters[kind] = registry.adapters["docker"] s, err := registry.Normalize(sandbox.Selection{Provider: kind, DeploymentSpec: validRegistrationSpec()}) - checkpoint, checkpointErr := registry.SupportsCheckpoint(kind) + checkpoint, checkpointErr := registry.SupportsSuspension(kind) if err != nil || checkpointErr != nil || s.Provider != kind || checkpoint { t.Fatal("new entry did not follow shared boundary", err, checkpointErr) } @@ -111,8 +111,8 @@ func TestCapabilityLookupsReportFailures(t *testing.T) { invalid.Operations = nil registry.adapters["invalid-registration"] = invalid for kind, want := range map[string]error{"unregistered": ErrUnknownProvider, "invalid-registration": providercontract.ErrContract} { - if _, err := registry.SupportsCheckpoint(kind); !errors.Is(err, want) { - t.Fatal(kind, "SupportsCheckpoint", err) + if _, err := registry.SupportsSuspension(kind); !errors.Is(err, want) { + t.Fatal(kind, "SupportsSuspension", err) } if _, err := registry.RetainedLimit(kind, 1, 2); !errors.Is(err, want) { t.Fatal(kind, "RetainedLimit", err) diff --git a/services/core/internal/sandbox/retained_state_test.go b/services/core/internal/sandbox/retained_state_test.go new file mode 100644 index 000000000..c76f10010 --- /dev/null +++ b/services/core/internal/sandbox/retained_state_test.go @@ -0,0 +1,51 @@ +package sandbox + +import ( + "encoding/json" + "testing" +) + +func TestRetainedCompatibilityIsOptionalButNeverPartial(t *testing.T) { + legacy := `{"Reference":"native","ID":"one","Data":"opaque","OperationID":"capture","SourceGeneration":1,"SourceName":"source","SourceID":"one"}` + var retained RetainedState + if err := json.Unmarshal([]byte(legacy), &retained); err != nil || ValidateRetained(retained) != nil { + t.Fatal("legacy native pause envelope rejected", err) + } + for _, compatibility := range []CheckpointCompatibility{{}, {ArtifactDomain: "store"}, {ExecutionClass: "class"}, {ArtifactDomain: "store", ExecutionClass: "class"}} { + retained.Compatibility = compatibility + wantValid := compatibility == (CheckpointCompatibility{}) || compatibility.Validate() == nil + if (ValidateRetained(retained) == nil) != wantValid { + t.Fatal("partial compatibility accepted", compatibility) + } + } +} + +func TestRetainedClosureCannotAuthorizeAnotherRestore(t *testing.T) { + retained := RetainedState{Reference: "native", ID: "one", Data: "opaque", OperationID: "capture", SourceGeneration: 1, SourceName: "source", SourceID: "one"} + target := Compute{Generation: 2, Name: "target", RestoredFrom: &retained} + q := ResumeRequest{OperationID: "restore", Retained: retained, Target: target, ReconcileOnly: true} + good := ComputeState{Compute: target, Status: "absent", RestoreAttemptClosed: "restore"} + if !good.ClosesRestoreAttempt(q) { + t.Fatal("exact settled closure rejected") + } + for _, kind := range []string{"operation", "generation", "provenance", "dispatched", "fresh"} { + s, request := good, q + switch kind { + case "operation": + s.RestoreAttemptClosed = "other" + case "generation": + s.Compute.Generation++ + case "provenance": + other := retained + other.Data = "foreign" + s.Compute.RestoredFrom = &other + case "dispatched": + s.Compute.ID = "native-target" + case "fresh": + request.ReconcileOnly = false + } + if s.ClosesRestoreAttempt(request) { + t.Fatal("unbound closure accepted", kind) + } + } +} diff --git a/services/core/internal/sandbox/runtime_bootstrap.go b/services/core/internal/sandbox/runtime_bootstrap.go index a4a05ecad..66c83ee86 100644 --- a/services/core/internal/sandbox/runtime_bootstrap.go +++ b/services/core/internal/sandbox/runtime_bootstrap.go @@ -7,6 +7,6 @@ import "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" func (b Bootstrap) RuntimeConnection() runtimebootstrap.Connection { return runtimebootstrap.Connection{ Version: runtimebootstrap.Version, CoreURL: b.CoreURL, - DeviceID: b.DeviceID, Credential: b.Credential, + DeviceID: b.DeviceID, Credential: b.Credential, Harness: b.Harness, } } diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index 568c16b07..211a6053d 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -52,6 +52,7 @@ type Reference struct{ TenantID, EnvironmentID, AllocationID string } type Bootstrap struct { Workspace *workspacefs.Binding `json:",omitempty"` + Harness string Reference SessionID, DeviceID, CoreURL, Credential string NetworkAccess string @@ -108,12 +109,13 @@ type SandboxProvider interface { // The checkpoint lifecycle supplies exact-incarnation operations; Worker and // Store remain the lifecycle owner. Its operations share one declaration. Initial(context.Context, Reference) (Compute, error) - NewCompute(context.Context, Reference, uint64, *SnapshotIdentity) (Compute, error) + NewCompute(context.Context, Reference, uint64, *RetainedState) (Compute, error) GetCompute(context.Context, Reference, Compute) (ComputeState, error) + RenewCompute(context.Context, Reference, Compute) (ComputeState, error) Suspend(context.Context, SuspendRequest) (ComputeState, error) Resume(context.Context, ResumeRequest) (ComputeState, error) KillCompute(context.Context, Reference, Compute) error - DeleteSnapshot(context.Context, Reference, SnapshotIdentity) error + DeleteRetained(context.Context, Reference, RetainedState) error RunCommandCompute(context.Context, Reference, Compute, Command) (CommandResult, error) // ResumeCompute thaws only the same resident instance after an aborted pause. ResumeCompute(context.Context, Reference, Compute) (ComputeState, error) @@ -126,9 +128,20 @@ type SandboxProvider interface { // requiredOperations are supported by every Provider. var requiredOperations = []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand"} -// checkpointOperations are all supported or all unsupported: partial cleanup or +// suspensionOperations are all supported or all unsupported: partial cleanup or // restore support cannot safely own a compute incarnation. -var checkpointOperations = []string{"Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute"} +var suspensionOperations = []string{"Initial", "NewCompute", "GetCompute", "RenewCompute", "Suspend", "Resume", "KillCompute", "DeleteRetained", "RunCommandCompute", "ResumeCompute"} + +// ValidateRetained checks the shared envelope; only its adapter interprets Data. +func ValidateRetained(s RetainedState) error { + if s.Compatibility != (CheckpointCompatibility{}) && s.Compatibility.Validate() != nil { + return ErrInvalid + } + if s.Reference == "" || s.ID == "" || s.OperationID == "" || s.SourceID == "" || s.SourceName == "" || len(s.Data) == 0 || len(s.Data) > 64*1024 { + return ErrInvalid + } + return nil +} // Compute identifies one incarnation of an allocation. Name is provider-derived. // ID is empty only until the original create or restore result is observed. @@ -136,7 +149,7 @@ type Compute struct { Generation uint64 Name string ID string - RestoredFrom *SnapshotIdentity + RestoredFrom *RetainedState } // CheckpointCompatibility identifies an adapter-verified private archive domain @@ -153,16 +166,13 @@ func (c CheckpointCompatibility) Validate() error { return nil } -// SnapshotIdentity is provider evidence from a verified full snapshot. Core -// persists it unchanged and records consumption separately; it never invents -// paths, checksums, native checkpoint fields, or source identity. -type SnapshotIdentity struct { - Compatibility CheckpointCompatibility +// RetainedState is adapter-owned recoverable state; Data stays opaque to Core. +// Optional Compatibility declares portable checkpoint qualification. +type RetainedState struct { + Compatibility CheckpointCompatibility `json:",omitzero"` Reference string ID string - Digest string - CheckpointID string - CheckpointRoot string + Data string OperationID string SourceGeneration uint64 SourceName string @@ -170,45 +180,83 @@ type SnapshotIdentity struct { } type ComputeState struct { - Compute Compute - Status string - BootstrapComplete bool - Snapshot *SnapshotIdentity - // SourceStopped proves the archive is durable and all source-local compute - // and capture obligations have settled, so another node can own restoration. - SourceStopped bool - // RestoreAttemptClosed echoes the exact Resume OperationID only after a - // never-dispatched attempt is durably closed against late native execution. RestoreAttemptClosed string + Compute Compute + Status string + BootstrapComplete bool + Retained *RetainedState + ResourcesReleased bool + // SuspendSettled without Retained also durably fences every late dispatch + // of this exact source and operation; native absence alone is insufficient. + SuspendSettled bool } // ClosesRestoreAttempt validates a never-executed closure against the exact // persisted request. Absence without this operation-bound evidence is unknown. func (s ComputeState) ClosesRestoreAttempt(q ResumeRequest) bool { - return q.ObserveOnly && q.OperationID != "" && s.RestoreAttemptClosed == q.OperationID && s.Status == "absent" && - s.Compute.ID == "" && q.Target.ID == "" && s.Compute.Name == q.Target.Name && s.Compute.Generation == q.Target.Generation && - s.Compute.RestoredFrom != nil && q.Target.RestoredFrom != nil && *s.Compute.RestoredFrom == q.Snapshot && *q.Target.RestoredFrom == q.Snapshot && - !s.BootstrapComplete && !s.SourceStopped && s.Snapshot == nil + return q.ReconcileOnly && q.OperationID != "" && s.RestoreAttemptClosed == q.OperationID && s.Status == "absent" && + s.Compute.ID == q.Target.ID && s.Compute.Name == q.Target.Name && s.Compute.Generation == q.Target.Generation && + s.Compute.RestoredFrom != nil && q.Target.RestoredFrom != nil && *s.Compute.RestoredFrom == q.Retained && *q.Target.RestoredFrom == q.Retained && + !s.BootstrapComplete && !s.ResourcesReleased && s.Retained == nil } type SuspendRequest struct { Reference Reference OperationID string Source Compute - Snapshot *SnapshotIdentity - // Recovery observes the previous attempt and never starts a new capture. - ObserveOnly bool + Retained *RetainedState + // Recovery settles the previous attempt without another capture. + // Ownership-verified cleanup of a durable retained artifact may complete. + ReconcileOnly bool } type ResumeRequest struct { Workspace *workspacefs.Binding `json:",omitempty"` Reference Reference OperationID string - Snapshot SnapshotIdentity + Retained RetainedState Target Compute // Recovery observes the previous target and never starts a new restore. - ObserveOnly bool + ReconcileOnly bool +} + +// ValidateComputeResult binds an observation to its precommitted incarnation. +func ValidateComputeResult(want, got Compute) error { + if got.ID == "" || got.Name != want.Name || got.Generation != want.Generation || (want.ID != "" && got.ID != want.ID) || (want.RestoredFrom == nil) != (got.RestoredFrom == nil) { + return ErrOwnership + } + if want.RestoredFrom != nil && *want.RestoredFrom != *got.RestoredFrom { + return ErrOwnership + } + return nil +} + +// ValidateSuspendResult distinguishes settled rollback from uncertain native work. +func ValidateSuspendResult(q SuspendRequest, s ComputeState) error { + if ValidateComputeResult(q.Source, s.Compute) != nil { + return ErrOwnership + } + if s.Retained == nil { + if !s.SuspendSettled || !s.BootstrapComplete { + return ErrComputeUnconfirmed + } + if !q.ReconcileOnly || s.ResourcesReleased || (s.Status != "running" && s.Status != "paused") { + return ErrComputeUnconfirmed + } + return nil + } + v := s.Retained + if ValidateRetained(*v) != nil || v.OperationID != q.OperationID || v.SourceID != q.Source.ID || v.SourceName != q.Source.Name || v.SourceGeneration != q.Source.Generation || (q.Retained != nil && *q.Retained != *v) { + return ErrOwnership + } + if !s.SuspendSettled || !s.BootstrapComplete || !s.ResourcesReleased || s.Status != "suspended" { + return ErrComputeUnconfirmed + } + return nil } +// SuspensionStateVersion fences incompatible durable lifecycle shapes. +const SuspensionStateVersion = "1" + // ValidateProvider checks a constructed Provider's declaration. func ValidateProvider(p SandboxProvider) error { if p == nil { @@ -243,15 +291,15 @@ func ValidateOperations(operations providercontract.Operations) error { return fmt.Errorf("%w: required operation %s", providercontract.ErrContract, name) } } - for _, name := range checkpointOperations { + for _, name := range suspensionOperations { if operations[name].State != operations["Initial"].State { - return fmt.Errorf("%w: incomplete checkpoint lifecycle", providercontract.ErrContract) + return fmt.Errorf("%w: incomplete suspension lifecycle", providercontract.ErrContract) } } return nil } -func SupportsCheckpoint(p SandboxProvider) bool { +func SupportsSuspension(p SandboxProvider) bool { return providercontract.Require(p, "Initial") == nil } diff --git a/services/core/internal/sandbox/sandbox_provider_test.go b/services/core/internal/sandbox/sandbox_provider_test.go index 56c23b330..59806b0f3 100644 --- a/services/core/internal/sandbox/sandbox_provider_test.go +++ b/services/core/internal/sandbox/sandbox_provider_test.go @@ -16,7 +16,7 @@ func TestOperationGroupsPartitionTheInterface(t *testing.T) { methods = append(methods, name) } } - groups := slices.Concat(requiredOperations, checkpointOperations, []string{"Observe"}) + groups := slices.Concat(requiredOperations, suspensionOperations, []string{"Observe"}) slices.Sort(groups) if !slices.Equal(methods, groups) { t.Fatalf("SandboxProvider operations %v, groups %v", methods, groups) diff --git a/services/core/internal/sandbox/suspension_test.go b/services/core/internal/sandbox/suspension_test.go new file mode 100644 index 000000000..add0fb28b --- /dev/null +++ b/services/core/internal/sandbox/suspension_test.go @@ -0,0 +1,55 @@ +package sandbox + +import ( + "strings" + "testing" +) + +func TestSuspensionRequiresBoundSettledResourceRelease(t *testing.T) { + c := Compute{ID: "native", Name: "source", Generation: 2} + r := RetainedState{Reference: "allocation", ID: "retained", OperationID: "op", SourceGeneration: 2, SourceName: "source", SourceID: "native", Data: "private-proof"} + q := SuspendRequest{OperationID: "op", Source: c} + valid := ComputeState{Compute: c, Status: "suspended", BootstrapComplete: true, Retained: &r, ResourcesReleased: true, SuspendSettled: true} + if err := ValidateSuspendResult(q, valid); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + change func(*ComputeState) + }{ + {"unsettled", func(s *ComputeState) { s.SuspendSettled = false }}, + {"capacity still held", func(s *ComputeState) { s.ResourcesReleased = false }}, + {"running", func(s *ComputeState) { s.Status = "running" }}, + {"foreign source", func(s *ComputeState) { s.Compute.ID = "foreign" }}, + {"foreign handle", func(s *ComputeState) { v := *s.Retained; v.OperationID = "other"; s.Retained = &v }}, + {"oversized", func(s *ComputeState) { v := *s.Retained; v.Data = strings.Repeat("x", 65537); s.Retained = &v }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := valid + tc.change(&s) + if ValidateSuspendResult(q, s) == nil { + t.Fatal("invalid suspension accepted") + } + }) + } + rollback := ComputeState{Compute: c, Status: "running", BootstrapComplete: true, SuspendSettled: true} + if ValidateSuspendResult(q, rollback) == nil { + t.Fatal("fresh dispatch accepted rollback") + } + q.ReconcileOnly = true + if err := ValidateSuspendResult(q, rollback); err != nil { + t.Fatal(err) + } + rollback.SuspendSettled = false + if ValidateSuspendResult(q, rollback) == nil { + t.Fatal("running observation inferred settlement") + } +} +func TestComputeResultFencesSameNativeIDAcrossGenerations(t *testing.T) { + old := Compute{ID: "same-native", Name: "allocation", Generation: 1} + next := old + next.Generation++ + if ValidateComputeResult(old, next) == nil { + t.Fatal("stale logical generation accepted") + } +} diff --git a/services/core/tests/integration/admin_session_archive_worker_http_test.go b/services/core/tests/integration/admin_session_archive_worker_http_test.go index fb536b9e3..ac3b6761c 100644 --- a/services/core/tests/integration/admin_session_archive_worker_http_test.go +++ b/services/core/tests/integration/admin_session_archive_worker_http_test.go @@ -126,7 +126,7 @@ func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { t.Fatal("archive did not commit administrator audit", audits, err) } stop() - // Snapshot after shutdown: cancellation and lifecycle draining are complete. + // Retained after shutdown: cancellation and lifecycle draining are complete. // A stale handler must not fall back to the still-open admission Store. snapshot := func() string { t.Helper() diff --git a/services/core/tests/integration/archived_cancellation_migration_test.go b/services/core/tests/integration/archived_cancellation_migration_test.go index 705a695d6..805bc4591 100644 --- a/services/core/tests/integration/archived_cancellation_migration_test.go +++ b/services/core/tests/integration/archived_cancellation_migration_test.go @@ -22,6 +22,8 @@ func TestArchivedCancellationMigrationDoesNotAdoptOldRevocations(t *testing.T) { if _, err := deploymentExecution(t, w).ArchiveSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 1); err != nil { t.Fatal(err) } + // Exercise the historical migration with the target schema's disabled + // suspension policy, independently of the current provider defaults. db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() provider, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) diff --git a/services/core/tests/integration/environment_worker_scan_test.go b/services/core/tests/integration/environment_worker_scan_test.go index 5149190ff..779ff5cbb 100644 --- a/services/core/tests/integration/environment_worker_scan_test.go +++ b/services/core/tests/integration/environment_worker_scan_test.go @@ -8,7 +8,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -func TestWorkerEnvironmentRetriesNewlyReadyAtNextScan(t *testing.T) { +func TestWorkerEnvironmentRetriesNewlyReadyOnCapabilityHint(t *testing.T) { h := newDispatchHarness(t) enableWorkerEnvironment(t, h) pending := workerEnvironmentReservation(t, h) @@ -20,7 +20,6 @@ func TestWorkerEnvironmentRetriesNewlyReadyAtNextScan(t *testing.T) { h.session = publicSession(t, h, "scan-barrier") receipt := h.message("barrier", "ordinary work") - scanned := time.Now() _, stop := startEnvironmentExpiryWorker(t, h.s, h.d) // Dispatch starts only after selectWork has examined the pending input on // the same pass, while its exact Runtime is still incapable of preparation. @@ -28,13 +27,14 @@ func TestWorkerEnvironmentRetriesNewlyReadyAtNextScan(t *testing.T) { if barrier.ID != receipt.TurnID { t.Fatal("unexpected scan barrier") } + readyAt := time.Now() awaitFixtureCapabilities(t, runtime, workerEnvironmentCapabilities()) h.write(barrier.ID, proto.TypeDone, proto.DonePayload{Content: "complete"}) waitTurn(t, h, barrier.ID, sessions.TurnCompleted) prepare := nextWorkerFrame(t, frames, proto.TypeExecutionPrepare) - if elapsed := time.Since(scanned); elapsed < 750*time.Millisecond || elapsed > 3*time.Second { - t.Fatal("readiness retry must use the next scan, without an empty scan interval", elapsed) + if elapsed := time.Since(readyAt); elapsed >= 750*time.Millisecond { + t.Fatal("capability hint waited for the periodic scan", elapsed) } if workerRuntimeForPreparation(t, h, prepare) != runtime { t.Fatal("readiness retry moved Runtime ownership") diff --git a/services/core/tests/integration/provider_operations_fixture_test.go b/services/core/tests/integration/provider_operations_fixture_test.go index f528d5ae0..7f207632c 100644 --- a/services/core/tests/integration/provider_operations_fixture_test.go +++ b/services/core/tests/integration/provider_operations_fixture_test.go @@ -17,11 +17,12 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -30,7 +31,7 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { func (*lifecycleProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*lifecycleProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*lifecycleProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*lifecycleProvider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (*lifecycleProvider) Resume(context.Context, sandbox.ResumeRequest) (sandbo func (*lifecycleProvider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*lifecycleProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*lifecycleProvider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*lifecycleProvider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -57,7 +58,7 @@ func (*lifecycleProvider) ResumeCompute(context.Context, sandbox.Reference, sand func (*lifecycleProvider) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } -func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations { +func (*fakeSuspensionProvider) ProviderOperations() providercontract.Operations { return providercontract.Operations{ "Create": {State: providercontract.Supported}, "GetInfo": {State: providercontract.Supported}, @@ -66,13 +67,18 @@ func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Supported}, "NewCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "GetCompute": {State: providercontract.Supported}, "Suspend": {State: providercontract.Supported}, "Resume": {State: providercontract.Supported}, "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, } } + +func (*lifecycleProvider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/tests/integration/provider_registration_migration_test.go b/services/core/tests/integration/provider_registration_migration_test.go index bbc4cfb1e..1f10e1aea 100644 --- a/services/core/tests/integration/provider_registration_migration_test.go +++ b/services/core/tests/integration/provider_registration_migration_test.go @@ -29,6 +29,8 @@ func TestProviderRegistrationDowngradePreservesCustomEndpoints(t *testing.T) { if _, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, input); err != nil { t.Fatal(err) } + // Exercise the historical migration with the target schema's disabled + // suspension policy, independently of the current provider defaults. db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() migrations, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) diff --git a/services/core/tests/integration/runtime_compute_lifecycle_test.go b/services/core/tests/integration/runtime_compute_lifecycle_test.go index 4c27a8e40..e37f43ba7 100644 --- a/services/core/tests/integration/runtime_compute_lifecycle_test.go +++ b/services/core/tests/integration/runtime_compute_lifecycle_test.go @@ -15,6 +15,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" + db "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" @@ -26,11 +27,11 @@ import ( // The controlled provider records external effects independently of DB phases. // Lost replies retain those effects so recovery must use observation, not replay. -type fakeCheckpointProvider struct { +type fakeSuspensionProvider struct { preparation *initializationPeer lifecycleProvider computes map[string]sandbox.ComputeState - snapshots map[string]sandbox.SnapshotIdentity + snapshots map[string]sandbox.RetainedState bootstraps map[string]sandbox.Bootstrap peers map[string]*websocket.Conn registry *runtimegateway.Registry @@ -41,15 +42,16 @@ type fakeCheckpointProvider struct { quiesces, resumes atomic.Int32 loseCapture, loseRestore, rejectQuiesce bool beforeQuiesce func() + renewals atomic.Int32 } -func (p *fakeCheckpointProvider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { +func (p *fakeSuspensionProvider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{Name: r.AllocationID + "-g0"}, nil } -func (p *fakeCheckpointProvider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, parent *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *fakeSuspensionProvider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, parent *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{Generation: generation, Name: fmt.Sprintf("%s-g%d", r.AllocationID, generation), RestoredFrom: parent}, nil } -func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { +func (p *fakeSuspensionProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { info, err := p.lifecycleProvider.Create(ctx, b) p.mu.Lock() defer p.mu.Unlock() @@ -59,7 +61,7 @@ func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap p.bootstraps[b.AllocationID] = b return info, err } -func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[c.Name] @@ -71,28 +73,28 @@ func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Referen } return state, nil } -func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[q.Source.Name] if !ok { - if snapshot, found := p.snapshots[q.OperationID]; found && q.ObserveOnly { + if snapshot, exists := p.snapshots[q.OperationID]; exists && q.ReconcileOnly { p.captureObservations++ - return sandbox.ComputeState{Compute: q.Source, Snapshot: &snapshot, SourceStopped: true, Status: "suspended"}, nil + return sandbox.ComputeState{Compute: q.Source, Status: "suspended", Retained: &snapshot, BootstrapComplete: true, ResourcesReleased: true, SuspendSettled: true}, nil } return sandbox.ComputeState{}, sandbox.ErrNotFound } if state.Compute.ID != q.Source.ID { return sandbox.ComputeState{}, sandbox.ErrOwnership } - if q.ObserveOnly { + if q.ReconcileOnly { p.captureObservations++ } else { p.captures++ if _, exists := p.snapshots[q.OperationID]; exists { return sandbox.ComputeState{}, errors.New("capture replayed") } - p.snapshots[q.OperationID] = sandbox.SnapshotIdentity{Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}, Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Digest: "verified", CheckpointID: "checkpoint", CheckpointRoot: "private", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} + p.snapshots[q.OperationID] = sandbox.RetainedState{Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}, Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Data: "verified-native-state", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} state.Status = "paused" p.computes[q.Source.Name] = state if p.loseCapture { @@ -101,18 +103,20 @@ func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendReq } } if snapshot, exists := p.snapshots[q.OperationID]; exists { - state.Snapshot = &snapshot - state.SourceStopped = true - state.Status = "stopped" - p.computeKills++ + state.Retained = &snapshot + state.SuspendSettled = true + state.ResourcesReleased = true + state.Status = "suspended" delete(p.computes, q.Source.Name) + p.computeKills++ } + state.SuspendSettled = true return state, nil } -func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() - if q.ObserveOnly { + if q.ReconcileOnly { p.restoreObservations++ state, ok := p.computes[q.Target.Name] if !ok { @@ -134,7 +138,7 @@ func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeReque } return state, nil } -func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { +func (p *fakeSuspensionProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[c.Name] @@ -148,7 +152,7 @@ func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Refere delete(p.computes, c.Name) return nil } -func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Reference, s sandbox.SnapshotIdentity) error { +func (p *fakeSuspensionProvider) DeleteRetained(_ context.Context, _ sandbox.Reference, s sandbox.RetainedState) error { p.mu.Lock() defer p.mu.Unlock() old, ok := p.snapshots[s.OperationID] @@ -162,7 +166,7 @@ func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Ref delete(p.snapshots, s.OperationID) return nil } -func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { state, err := p.GetCompute(ctx, r, c) if err != nil { return state, err @@ -173,7 +177,7 @@ func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Re p.computes[c.Name] = state return state, nil } -func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { +func (p *fakeSuspensionProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { if _, err := p.GetCompute(ctx, r, c); err != nil { return sandbox.CommandResult{}, err } @@ -186,7 +190,7 @@ func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbo p.mu.Unlock() return sandbox.CommandResult{}, p.connect(ctx, b) } -func (p *fakeCheckpointProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { +func (p *fakeSuspensionProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { header := http.Header{"Authorization": []string{"Bearer " + b.Credential}} conn, _, err := websocket.DefaultDialer.DialContext(ctx, p.endpoint+"?device_id="+b.DeviceID+"&version="+proto.Version, header) if err != nil { @@ -264,7 +268,7 @@ func (p *fakeCheckpointProvider) connect(ctx context.Context, b sandbox.Bootstra type computeLifecycleFixture struct { t *testing.T store *Store - provider *fakeCheckpointProvider + provider *fakeSuspensionProvider worker *execution.Worker stop func() key string @@ -273,10 +277,13 @@ type computeLifecycleFixture struct { } func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *computeLifecycleFixture { + return computeFixtureForMode(t, maxActive, maxRetained, false) +} +func computeFixtureForMode(t *testing.T, maxActive, maxRetained int, direct bool) *computeLifecycleFixture { t.Helper() s, _ := newManagedTestStore(t) registry := runtimegateway.NewRegistry() - p := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} + p := &fakeSuspensionProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.RetainedState{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Authenticator: runtimegateway.NewAuthenticator(sessionAdapter(s)), Registry: registry}) server := httptest.NewServer(http.HandlerFunc(handler.WS)) p.endpoint = "ws" + strings.TrimPrefix(server.URL, "http") @@ -288,12 +295,18 @@ func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *compu } server.Close() }) - f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, "microsandbox"), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour}} + kind := "microsandbox" + if direct { + kind = "e2b" + } + f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, kind), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour, MaxActive: maxActive, MaxRetained: maxRetained}} view, err := deploymentService(t, s).View(t.Context()) if err != nil { t.Fatal(err) } - f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID + if !direct { + f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID + } f.start() return f } @@ -302,7 +315,9 @@ func (f *computeLifecycleFixture) start() { t.Helper() w := startWebWorker(t, f.store, f.provider.registry, f.key, f.provider, &f.policy) // The Worker's claim starts a new owner epoch, in which the node reconnects. - onlineManagerNode(t, f.store, f.node) + if f.node != "" { + onlineManagerNode(t, f.store, f.node) + } var once sync.Once stop := func() { once.Do(func() { ctx, cancel := context.WithCancel(context.Background()); cancel(); _ = w.Run(ctx) }) @@ -369,6 +384,12 @@ func (f *computeLifecycleFixture) queued(owner deployment.Allocation) string { func TestRuntimeComputeLifecycleIdleSuspendAndQueuedSameSessionWake(t *testing.T) { f := newComputeLifecycleFixture(t, 2, 4) tenant, session, env, owner := f.create() + for range 3 { + f.phase(tenant, env.ID, "running") + } + if f.provider.captures != 0 { + t.Fatal("recently-created Session suspended") + } completed := f.complete(owner) suspended := f.phase(tenant, env.ID, "suspended") if f.provider.captures != 1 || f.provider.computeKills != 1 || len(f.provider.computes) != 0 || len(f.provider.snapshots) != 1 { @@ -391,6 +412,28 @@ func TestRuntimeComputeLifecycleIdleSuspendAndQueuedSameSessionWake(t *testing.T } } +func TestRuntimeComputeLifecycleUnusedSessionSuspendsAndExpires(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, session, env, owner := f.create() + f.sql(`UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 minutes',compute_phase_changed_at=clock_timestamp()-interval '2 minutes' WHERE id=$1`, owner.ID) + suspended := f.phase(tenant, env.ID, "suspended") + if suspended.ComputeRetainedUntil == nil || f.provider.captures != 1 || f.provider.computeKills != 1 || len(f.provider.computes) != 0 || len(f.provider.snapshots) != 1 { + t.Fatal("unused Session did not suspend and release active compute") + } + var turns int + if err := f.store.pool.QueryRow(t.Context(), `SELECT count(*) FROM turns WHERE session_id=$1`, session.ID).Scan(&turns); err != nil || turns != 0 { + t.Fatal("unused Session gained a Turn", turns, err) + } + f.sql(`UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID) + reconcileManagedState(t, f.worker, f.store, tenant, env.ID, "released") + if len(f.provider.computes) != 0 || len(f.provider.snapshots) != 0 || f.provider.snapshotDeletes != 1 || f.provider.promptFrames.Load() != 0 { + t.Fatal("unused Session expiry retained resources or sent execution input") + } + if _, err := sessionAdapter(f.store).GetSession(t.Context(), tenant, session.ID); err != nil { + t.Fatal("resource cleanup removed Session history", err) + } +} + func TestRuntimeComputeLifecycleLostCaptureAndRestoreObserveWithoutReplay(t *testing.T) { f := newComputeLifecycleFixture(t, 1, 2) tenant, _, env, owner := f.create() @@ -516,3 +559,126 @@ func TestRuntimeComputeLifecycleSuspendedDeletionAndExpiryCleanup(t *testing.T) }) } } + +func TestRuntimeComputeLifecycleCapacityBoundsActiveAndRetained(t *testing.T) { + f := computeFixtureForMode(t, 1, 2, true) + tenant, _, env, owner := f.create() + tenant2, _, env2 := managedSession(t, f.store) + if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { + t.Fatalf("active capacity ignored: %v", err) + } + if f.provider.creates != 1 { + t.Fatal("capacity rejection allocated compute") + } + f.complete(owner) + f.phase(tenant, env.ID, "suspended") + second, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key) + if err != nil { + t.Fatal(err) + } + pending := f.queued(owner) + for range 4 { + f.worker.ReconcileManagedRuntimes(t.Context()) + } + first, err := deploymentStore(f.store).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}) + if err != nil || first.ComputePhase != "suspended" || f.provider.restores != 0 { + t.Fatal("wake exceeded active capacity", err) + } + f.sql(`UPDATE turns SET status='cancelled',completed_at=clock_timestamp() WHERE id=$1`, pending) + f.provider.mu.Lock() + b := f.provider.bootstraps[second.ID] + f.provider.mu.Unlock() + if err := f.provider.connect(t.Context(), b); err != nil { + t.Fatal(err) + } + second = f.phase(tenant2, env2.ID, "running") + f.complete(second) + f.phase(tenant2, env2.ID, "suspended") + // Both retained allocations count even when their source VMs are gone. + tenant3, _, env3 := managedSession(t, f.store) + if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant3, env3.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { + t.Fatalf("retained capacity ignored: %v", err) + } + if f.provider.creates != 2 { + t.Fatal("retained limit created a third allocation") + } +} + +func (p *fakeSuspensionProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + p.renewals.Add(1) + return p.GetCompute(ctx, r, c) +} + +func TestRuntimeComputeProtocolUpgradeRefusesOldReceiptsBeforeCleanup(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + f.complete(owner) + retained := f.phase(tenant, env.ID, "suspended") + f.stop() + for _, versioned := range []bool{false, true} { + f.sql(`UPDATE runtime_allocations SET compute_state=($2::jsonb-'retained') || jsonb_build_object('snapshot',$2::jsonb->'retained') WHERE id=$1`, owner.ID, retained.ComputeState) + if !versioned { + f.sql(`UPDATE runtime_allocations SET compute_state=compute_state-'protocol_version' WHERE id=$1`, owner.ID) + } + deletes := f.provider.snapshotDeletes + w, err := startNextWorker(t, t.Context(), f.store, &execution.Dispatcher{Registry: f.provider.registry, ManagedRuntimes: webRuntimes(t, f.store, f.key, f.provider, &f.policy)}) + if err == nil || w != nil || !strings.Contains(err.Error(), "previous release") { + t.Fatalf("incompatible state activated: %v", err) + } + if f.provider.snapshotDeletes != deletes || len(f.provider.snapshots) != 1 { + t.Fatal("upgrade lost owned artifact") + } + var state []byte + if err := f.store.pool.QueryRow(t.Context(), `SELECT compute_state FROM runtime_allocations WHERE id=$1`, owner.ID).Scan(&state); err != nil { + t.Fatal(err) + } + if !strings.Contains(string(state), `"snapshot"`) { + t.Fatal("old receipt was rewritten") + } + } + // A consumed legacy snapshot can survive only in current/target provenance. + for _, field := range []string{"current", "target"} { + f.sql(`UPDATE runtime_allocations SET compute_state=$2::jsonb || jsonb_build_object($3::text, jsonb_build_object('RestoredFrom', jsonb_build_object('Digest', 'legacy'))) WHERE id=$1`, owner.ID, retained.ComputeState, field) + incompatible, err := db.New(f.store.pool).HasIncompatibleRuntimeComputeState(t.Context(), sandbox.SuspensionStateVersion) + if err != nil || !incompatible { + t.Fatalf("legacy %s provenance accepted: incompatible=%v err=%v", field, incompatible, err) + } + } + // Adapter-owned data may contain any private field names. The activation + // fence must inspect only the shared envelope, never this opaque string. + f.sql(`UPDATE runtime_allocations SET compute_state=jsonb_set($2::jsonb, '{retained,Data}', to_jsonb($3::text)) WHERE id=$1`, owner.ID, retained.ComputeState, `{"snapshot":{"Digest":"private","CheckpointID":"native","CheckpointRoot":"owned"}}`) + incompatible, err := db.New(f.store.pool).HasIncompatibleRuntimeComputeState(t.Context(), sandbox.SuspensionStateVersion) + if err != nil || incompatible { + t.Fatalf("opaque native payload affected activation: incompatible=%v err=%v", incompatible, err) + } + f.sql(`UPDATE runtime_allocations SET compute_state=$2::jsonb WHERE id=$1`, owner.ID, retained.ComputeState) + f.start() +} + +func TestRuntimeComputeRepeatedWakeStillRenewsCurrentIncarnation(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + before := f.provider.renewals.Load() + for range 3 { + f.sql(`UPDATE runtime_allocations SET compute_wake_requested=true,compute_activity_at=clock_timestamp() WHERE id=$1`, owner.ID) + f.phase(tenant, env.ID, "running") + } + if f.provider.renewals.Load() < before+3 { + t.Fatal("wake requests bypassed native lease renewal") + } +} + +func TestRuntimeComputeProtocolRejectsOldDisabledAllocation(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + _, _, _, owner := f.create() + f.stop() + f.sql(`UPDATE runtime_allocations SET compute_phase='disabled',compute_state='{}'::jsonb WHERE id=$1`, owner.ID) + before := f.provider.renewals.Load() + w, err := startNextWorker(t, t.Context(), f.store, &execution.Dispatcher{Registry: f.provider.registry, ManagedRuntimes: webRuntimes(t, f.store, f.key, f.provider, &f.policy)}) + if err == nil || w != nil || !strings.Contains(err.Error(), "previous release") { + t.Fatal("old disabled allocation activated", err) + } + if f.provider.renewals.Load() != before || len(f.provider.computes) != 1 { + t.Fatal("upgrade changed owned native compute") + } +} diff --git a/services/core/tests/integration/runtime_idle_clock_test.go b/services/core/tests/integration/runtime_idle_clock_test.go index 298b2e082..715005a24 100644 --- a/services/core/tests/integration/runtime_idle_clock_test.go +++ b/services/core/tests/integration/runtime_idle_clock_test.go @@ -228,3 +228,24 @@ func TestManagedIdleClockReconnectPreservesReceipts(t *testing.T) { t.Fatal(err) } } + +func TestDirectManagedIdleClockIgnoresNativeClockSkew(t *testing.T) { + for _, skew := range []time.Duration{-269 * time.Second, 269 * time.Second} { + t.Run(skew.String(), func(t *testing.T) { + s, w, owner := managedIdleClockFixture(t) + runtimeSuspensionSQL(t, s.pool, "UPDATE runtime_allocations SET node_id=NULL WHERE id=$1", owner.ID) + owner.NodeID = "" + turn := uuid.NewString() + runtimeSuspensionSQL(t, s.pool, "INSERT INTO turns(id,session_id,status,started_at) VALUES($1,$2,'in_progress',clock_timestamp())", turn, owner.SessionID) + source := runtimeDatabaseTime(t, s).Add(skew).UnixMilli() + outcome := json.RawMessage(fmt.Sprintf(`{"done":{"source_completed_at_ms":%d}}`, source)) + before := runtimeDatabaseTime(t, s) + completed, err := sessionExecution(t, w.lease).CompleteExecution(t.Context(), owner.TenantID, owner.SessionID, turn, sessions.TurnCompleted, outcome, "", 0) + after := runtimeDatabaseTime(t, s) + if err != nil || completed.CompletedAt.UnixMilli() != source { + t.Fatal(completed, err) + } + verifyManagedIdleClock(t, s, w, owner, before, after) + }) + } +} diff --git a/services/core/tests/integration/runtime_lifecycle_test.go b/services/core/tests/integration/runtime_lifecycle_test.go index 4f28fc6f0..829f4fef7 100644 --- a/services/core/tests/integration/runtime_lifecycle_test.go +++ b/services/core/tests/integration/runtime_lifecycle_test.go @@ -28,6 +28,7 @@ type lifecycleProvider struct { loseCreate, absent, unavailable bool credentialHash string credential string + harness string } func (p *lifecycleProvider) Create(_ context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { @@ -36,6 +37,7 @@ func (p *lifecycleProvider) Create(_ context.Context, b sandbox.Bootstrap) (sand p.creates++ p.credentialHash = runtimedevice.HashCredential(b.Credential) p.credential = b.Credential + p.harness = b.Harness i := sandbox.Info{Reference: b.Reference, ProviderID: b.AllocationID, State: "running", BootstrapComplete: true} if !p.absent { p.resources[b.AllocationID] = i @@ -245,3 +247,32 @@ func TestManagedRuntimeStoppedComputeDoesNotRequestCleanup(t *testing.T) { } } } + +func TestManagedRuntimeBootstrapUsesSessionHarness(t *testing.T) { + for _, kind := range []string{"codex", "mcode", "claude_sdk"} { + t.Run(kind, func(t *testing.T) { + s, key := configuredStore(t) + tenant := uuid.NewString() + session, err := s.CreateSession(t.Context(), tenant, WithFixtureModelProvider(sessions.CreateSession{Creator: FixtureCreator(), Engine: kind, IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted","network":{"access":"enabled"}}}`)})) + if err != nil { + t.Fatal(err) + } + env, err := sessionAdapter(s).GetSessionEnvironment(t.Context(), tenant, session.ID) + if err != nil { + t.Fatal(err) + } + p := &lifecycleProvider{resources: map[string]sandbox.Info{}} + w, _ := managedWorker(t, s, key, p) + for range 2 { + if _, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key); err != nil { + t.Fatal(err) + } + } + p.mu.Lock() + defer p.mu.Unlock() + if p.harness != kind || p.creates != 1 { + t.Fatalf("bootstrap selection=%s creates=%d", p.harness, p.creates) + } + }) + } +} diff --git a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go index ca2573598..c9d0f2e07 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go @@ -25,7 +25,7 @@ import ( ) type nodeIsolationProvider struct { - *fakeCheckpointProvider + *fakeSuspensionProvider blockMu sync.Mutex blocked map[string]bool mode string @@ -49,7 +49,7 @@ func (p *nodeIsolationProvider) block(ctx context.Context, r sandbox.Reference, return ctx.Err() } func (p *nodeIsolationProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { - info, err := p.fakeCheckpointProvider.Create(ctx, b) + info, err := p.fakeSuspensionProvider.Create(ctx, b) if err == nil { err = p.connect(ctx, b) } @@ -59,13 +59,13 @@ func (p *nodeIsolationProvider) GetInfo(ctx context.Context, r sandbox.Reference if err := p.block(ctx, r, "observe"); err != nil { return sandbox.Info{}, err } - return p.fakeCheckpointProvider.GetInfo(ctx, r) + return p.fakeSuspensionProvider.GetInfo(ctx, r) } func (p *nodeIsolationProvider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { if err := p.block(ctx, r, "observe"); err != nil { return sandbox.ComputeState{}, err } - return p.fakeCheckpointProvider.GetCompute(ctx, r, c) + return p.fakeSuspensionProvider.GetCompute(ctx, r, c) } func (p *nodeIsolationProvider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { return p.preparation.RunCommand(ctx, r, c) @@ -91,8 +91,8 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { t.Helper() s, pool := newManagedTestStore(t) registry := runtimegateway.NewRegistry() - cp := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} - p := &nodeIsolationProvider{fakeCheckpointProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} + cp := &fakeSuspensionProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.RetainedState{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} + p := &nodeIsolationProvider{fakeSuspensionProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} preparationContext, cancelPreparation := context.WithCancel(t.Context()) t.Cleanup(cancelPreparation) cp.preparation = &initializationPeer{t: t, apply: func(request proto.RuntimePreparePayload, data []byte) proto.RuntimePrepareResultPayload { diff --git a/services/core/tests/integration/runtime_suspension_test.go b/services/core/tests/integration/runtime_suspension_test.go index e32df1a8c..a85b5e072 100644 --- a/services/core/tests/integration/runtime_suspension_test.go +++ b/services/core/tests/integration/runtime_suspension_test.go @@ -270,17 +270,60 @@ func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { } } +func TestRuntimeSuspensionCountsUncertainCapacityUntilReleased(t *testing.T) { + s, provider := configuredStore(t) + pool := s.pool + w := executionWriter(t, s) + cases := []struct { + state, phase string + count bool + }{ + {"creating", "disabled", true}, {"running", "running", true}, {"running", "quiescing", true}, {"running", "suspending", true}, {"running", "suspended", false}, {"running", "restoring", true}, {"running", "waking", true}, {"cleanup_pending", "restoring", true}, {"released", "running", false}, + } + want := int64(0) + wantRetained := int64(0) + for _, item := range cases { + tenant := uuid.NewString() + _, env := localEnvironment(t, s, tenant) + owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}, provider, runtimedevice.HashCredential(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET state=$2,compute_phase=$3,create_settled=($2<>'creating'),released_at=CASE WHEN $2='released' THEN clock_timestamp() END WHERE id=$1`, owner.ID, item.state, item.phase) + if item.count { + want++ + } + if item.state != "released" { + wantRetained++ + } + retained, err := deploymentStore(w).CountRetainedAllocations(t.Context(), provider) + if err != nil || retained != wantRetained { + t.Fatalf("retained capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, retained, wantRetained, err) + } + got, err := deploymentStore(w).CountComputeReservations(t.Context(), provider) + if err != nil || got != want { + t.Fatalf("capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, got, want, err) + } + } + got, err := deploymentStore(w).CountComputeReservations(t.Context(), uuid.NewString()) + if err != nil || got != 0 { + t.Fatal("capacity crossed installation boundary", got, err) + } +} + func TestRuntimeSuspensionIdleStartsAfterLastCompletion(t *testing.T) { - _, w, pool, owner := runtimeSuspensionFixture(t) - turn := runtimeSuspensionCompleted(t, pool, owner) + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 hours',compute_phase_changed_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) - var completed time.Time - if err := pool.QueryRow(t.Context(), `SELECT completed_at FROM turns WHERE id=$1`, turn).Scan(&completed); err != nil { + before := runtimeDatabaseTime(t, s) + id, _ := parseID(owner.SessionID) + if err := s.queries.RecordRuntimeTerminalActivity(t.Context(), id); err != nil { t.Fatal(err) } + after := runtimeDatabaseTime(t, s) activity, err := deploymentStore(w).Activity(t.Context(), owner.ID) - if err != nil || !activity.LastActivity.Equal(completed) { - t.Fatal("long Turn completion did not restart idle interval", activity, completed, err) + if err != nil || activity.LastActivity.Before(before) || activity.LastActivity.After(after) || activity.ReadyToSuspend(time.Minute) { + t.Fatal("long Turn completion did not restart the ingestion idle interval", activity, before, after, err) } } @@ -361,8 +404,14 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { default: runtimeSuspensionSQL(t, pool, `INSERT INTO environment_file_writes(id,environment_id,device_id,request_sha256,state,created_at,settled_at) VALUES($1,$2,$3,$4,$5,clock_timestamp()-interval '10 minutes',clock_timestamp())`, uuid.NewString(), owner.EnvironmentID, owner.DeviceID, strings.Repeat("a", 64), strings.TrimPrefix(kind, "file_")) } + if kind == "root" || kind == "subagent" { + id, _ := parseID(owner.SessionID) + if err := s.queries.RecordRuntimeTerminalActivity(t.Context(), id); err != nil { + t.Fatal(err) + } + } until := time.Now().Add(time.Hour) - if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleTimeout); !errors.Is(err, deployment.ErrNotIdle) { + if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleTimeout); !errors.Is(err, deployment.ErrNotIdle) && !errors.Is(err, deployment.ErrAllocationConflict) { t.Fatal("completion after idle observation did not fence quiesce", err) } activity, err := deploymentStore(w).Activity(t.Context(), owner.ID) @@ -372,6 +421,11 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 0); !errors.Is(err, deployment.ErrInvalidInput) { t.Fatal("missing idle timeout accepted", err) } + // Re-observe the allocation after terminal ingestion advanced its activity fence. + owner, err = deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: owner.TenantID, EnvironmentID: owner.EnvironmentID}) + if err != nil { + t.Fatal(err) + } if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Nanosecond); err != nil { t.Fatal("elapsed idle timeout rejected", err) } diff --git a/services/core/tests/integration/runtime_wake_hint_integration_test.go b/services/core/tests/integration/runtime_wake_hint_integration_test.go index aedeca68c..ebb976bd7 100644 --- a/services/core/tests/integration/runtime_wake_hint_integration_test.go +++ b/services/core/tests/integration/runtime_wake_hint_integration_test.go @@ -16,7 +16,7 @@ import ( ) type wakeHintScanProvider struct { - *fakeCheckpointProvider + *fakeSuspensionProvider sentinel string release chan struct{} scans chan int @@ -35,7 +35,7 @@ func (p *wakeHintScanProvider) GetCompute(ctx context.Context, reference sandbox } } } - return p.fakeCheckpointProvider.GetCompute(ctx, reference, compute) + return p.fakeSuspensionProvider.GetCompute(ctx, reference, compute) } type wakeHintIntegrationTarget struct { @@ -70,7 +70,7 @@ func newWakeHintIntegration(t *testing.T) *wakeHintIntegration { target.owner = f.phase(target.tenant, target.environment.ID, "suspended") f.stop() provider := &wakeHintScanProvider{ - fakeCheckpointProvider: f.provider, sentinel: sentinel.owner.ID, + fakeSuspensionProvider: f.provider, sentinel: sentinel.owner.ID, release: make(chan struct{}), scans: make(chan int, 16), } worker := startWebWorker(t, f.store, f.provider.registry, f.key, provider, &f.policy) diff --git a/services/core/tests/integration/runtime_worker_recovery_test.go b/services/core/tests/integration/runtime_worker_recovery_test.go index 7cb9c7f68..5fce9b5dd 100644 --- a/services/core/tests/integration/runtime_worker_recovery_test.go +++ b/services/core/tests/integration/runtime_worker_recovery_test.go @@ -2,12 +2,14 @@ package integration import ( "context" + "encoding/json" "errors" "testing" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" @@ -15,11 +17,24 @@ import ( func insertWorkerRuntimeAllocation(t *testing.T, pool *pgxpool.Pool, h *dispatchHarness, phase string) { t.Helper() - _, err := pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,compute_phase,compute_retained_until,deployment_generation) - VALUES($1,$2,$3,$4,'running',true,$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment))`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase) + state, err := json.Marshal(map[string]any{"protocol_version": sandbox.SuspensionStateVersion, "current": sandbox.Compute{Name: h.device.EnvironmentID, ID: h.device.EnvironmentID}}) if err != nil { t.Fatal(err) } + _, err = pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,compute_phase,compute_retained_until,deployment_generation,compute_state) + VALUES($1,$2,$3,$4,'running',true,$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment),$6)`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase, state) + if err != nil { + t.Fatal(err) + } + // This synthetic allocation must not outlive the test in the shared fixture + // database, where the next worker validates every retained protocol receipt. + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := pool.Exec(ctx, "DELETE FROM runtime_allocations WHERE environment_id=$1", h.device.EnvironmentID); err != nil { + t.Error(err) + } + }) } func runtimeWorkerHarness(t *testing.T) (*dispatchHarness, *pgxpool.Pool) { diff --git a/services/core/tests/integration/sandbox_deployment_view_test.go b/services/core/tests/integration/sandbox_deployment_view_test.go index ca1ed70b7..9f8051f31 100644 --- a/services/core/tests/integration/sandbox_deployment_view_test.go +++ b/services/core/tests/integration/sandbox_deployment_view_test.go @@ -38,7 +38,7 @@ func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { for _, want := range []string{ `"specification":{"resources":{"cpus":2,"memory_mib":2048}}`, `"template_build":{"status":"ready","resources":{"cpus":2,"memory_mib":2048,"root_disk_mib":24063}}`, - `"suspension":null`, + `"suspension":{"idle_seconds":300,"retention_seconds":86400}`, } { if !bytes.Contains(raw, []byte(want)) { t.Fatalf("E2B view lacks %s: %s", want, raw) diff --git a/services/core/tests/integration/sandbox_generations_test.go b/services/core/tests/integration/sandbox_generations_test.go index 2b08131df..9932151e8 100644 --- a/services/core/tests/integration/sandbox_generations_test.go +++ b/services/core/tests/integration/sandbox_generations_test.go @@ -3,6 +3,7 @@ package integration import ( "database/sql" "os" + "strings" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" @@ -218,14 +219,16 @@ func TestGenerationDowngradeRefusesOldAllocation(t *testing.T) { if _, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, input); err != nil { t.Fatal(err) } + // Isolate the generation downgrade guard using the target schema's disabled + // suspension policy. Active suspension itself is not representable there. db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() migrations, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) if err != nil { t.Fatal(err) } - if _, err = migrations.DownTo(t.Context(), 80); err == nil { - t.Fatal("downgrade erased old owned allocation") + if _, err = migrations.DownTo(t.Context(), 80); err == nil || !strings.Contains(err.Error(), "Cannot downgrade while retained ownership") { + t.Fatal("generation ownership guard did not reject downgrade", err) } // Earlier down migrations can commit before the generation guard vetoes // downgrade. Restore the current schema before invoking current Store code. diff --git a/services/core/tests/integration/worker_wakeup_test.go b/services/core/tests/integration/worker_wakeup_test.go index 5ab16df8d..cea2990fd 100644 --- a/services/core/tests/integration/worker_wakeup_test.go +++ b/services/core/tests/integration/worker_wakeup_test.go @@ -76,6 +76,12 @@ func TestWorkerSchedulerCommittedAdmissionWakesBeforeMaintenance(t *testing.T) { if err != nil { t.Fatal(err) } + // Fixture Runtime enrollment also emits a capability hint. Isolate + // the admission signal under test from that already committed work. + select { + case <-h.d.Registry.CapabilityHints(): + default: + } trace.armed.Store(true) started = true go func() { done <- worker.Run(ctx) }() diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index 1e7588c76..d4d8d8e51 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -30,7 +30,7 @@ Compatible endpoints must return the SDK 2.51.0 template-list and template-build Run `go generate ./services/core/internal/sandbox/e2b` from the repository root after changing these declarations. `make check-e2b-provider` and the Go adapter tests reject stale projections; both languages consume generated valid and invalid exchanges covering wire types, extra fields, operation/reference bounds and managed-bootstrap fields. The helper build copies those fixtures with its source before running the pinned-SDK suite. -The boundary has version 1. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. +The boundary has version 4. Managed bootstrap and Resume accept an omitted or null `Workspace`; E2B rejects non-null external workspace bindings before native calls. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. ## Receipts and state directory @@ -42,15 +42,15 @@ A helper holds its allocation's lock until the SDK operation returns, even after 1. Record `create_pending` with the bootstrap identity, then call `Sandbox.create` with the template, the configured timeout, the ownership metadata, `on_timeout=kill` and auto-resume disabled. A definite rejection records a settled `rejected` receipt with no sandbox IDs. 2. Record the sandbox ID and connection material, check the sandbox domain, then read the sandbox by ID and check its ownership metadata, template and resources before writing any credential. A mismatch records a settled rejection and returns `CreateSettled` with the error. -3. Check that `/opt/oac-e2b/managed_init.py` is readable, write the managed bootstrap input to `/root/.oac/e2b/managed-bootstrap.json` and run `managed_init.py` as root. +3. Check that `/opt/oac-e2b/managed_init.py` has regular-file type through SDK file information, then open it through the SDK streaming file API as root and close the response without downloading its contents. This verifies readability without a remote probe process. Then write the managed bootstrap input to `/root/.oac/e2b/managed-bootstrap.json` and run `managed_init.py` as root. -`managed_init.py` prepares the image as the [application-managed startup](../../deploy/e2b/README.md#startup-and-security-boundary) does, writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json` (mode 0600, owned by UID 1000), sets the Environment, Session and network variables and starts `oac-daemon connect --profile default --bootstrap-file /home/runtime/runtime-bootstrap.json` as UID/GID 1000. It records process handoff in `/root/.oac/e2b/managed-ready.json` and refuses to run again once any launch record exists. `BootstrapComplete` becomes true when a later inspection reads that record with the expected identity; it does not prove enrollment or native readiness. +`managed_init.py` prepares the image as the [application-managed startup](../../deploy/e2b/README.md#startup-and-security-boundary) does, writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json` (mode 0600, owned by UID 1000), sets the Environment, Session and network variables and starts `oac-daemon connect --profile default --bootstrap-file /home/runtime/runtime-bootstrap.json` as UID/GID 1000. It records process handoff in `/root/.oac/e2b/managed-ready.json` and refuses to run again once any launch record exists. `BootstrapComplete` becomes true when Core validates that record with the expected identity. The initialization command returns the bounded durable receipt on its existing output stream, avoiding a separate file read on successful Create; Core still rereads cloud ownership and configuration before returning. If the command response is lost or its receipt is unreadable, inspection can recover the record without replaying startup; it does not prove enrollment or native readiness. An unknown Create is never repeated. A Create whose connection material was lost can be discovered and destroyed but cannot resume bootstrap, and an unconfirmed startup requires reclaiming the whole allocation. ## Inspection and cleanup -Inspection uses SDK metadata and ID reads only. SDK `connect` is never used because it can resume paused compute. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. +Inspection uses SDK metadata and ID reads only. Inspection never uses SDK `connect`, which is reserved for the explicit managed Resume operation. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. `CreateSettled` proves that the original Create and bootstrap can no longer mutate; it is independent of `BootstrapComplete`. An empty lookup never settles an unknown Create. Kill destroys every matching sandbox, confirms that none remains and only then records a settled tombstone; it returns `State=absent` with `CreateSettled`. A settled rejected Create with no sandbox IDs proves absence without a cloud request, so GetInfo and Kill still succeed when the key is invalid. Ordinary missing compute has no such proof. @@ -71,3 +71,17 @@ make check-e2b-provider ``` With `OAC_TEST_E2B_SDK_PYTHON` pointing at the pinned SDK environment, this runs this directory's tests and the [template scripts' tests](../../deploy/e2b/README.md#tests). The helper build runs this directory's suite and checks the relocated helper's `--check` report. These tests create no cloud resources. + +## Managed suspension + +The adapter implements the complete shared suspension group using opaque retained state. A native pause retains the same sandbox; it is not a portable filesystem snapshot. Core's common activity rule quiesces the Runtime, then the adapter calls the pinned SDK's memory-preserving `Sandbox.pause`. The private allocation receipt commits the operation before native I/O and reports resource release only after observing the exact sandbox paused. + +Resume calls `Sandbox.connect` once with `on_resume=restore` and the existing native timeout. It preserves the native sandbox ID while advancing the shared logical generation. Fresh connection material is persisted before returning running. Unknown pause and connect outcomes are observed without replay; a missing connect receipt keeps execution unavailable. The adapter fences stale generations before commands and deletion. Consumed pause receipt cleanup preserves running compute. + +Before native connect is admitted, recovery may durably close an exact restore operation under the allocation lock. Its `RestoreAttemptClosed` response binds the operation, target logical generation, same native ID and retained provenance; it never infers no dispatch from cloud absence. Closed IDs are kept for that retained generation (maximum 64, no eviction). A late fresh request for a closed ID is rejected. An additional closure at the bound fails closed; an already admitted or uncertain connect is never closed or replayed. The next retained generation replaces this history only after old retained requests can no longer match. + +Deleting an unconsumed settled pause checks the exact source incarnation and requires the final deletion candidates to be absent or the same paused native ID. It then confirms absence and consumes the receipt. Unknown deletion preserves the receipt and may settle after a lost reply or local write failure. Consumed cleanup still preserves a running resumed incarnation. The SDK has no atomic state-conditional deletion; manual cloud lifecycle changes outside the managed allocation lock are not supported. + +The shared registration owns the 300-second idle duration and 86400-second retention default. Native timeout remains 3600 seconds and automatic native resume remains disabled. SDK calls use the configured official-compatible endpoint. Generated declarations and fixtures own the private helper shape. + +A no-intent suspension observation durably fences its OperationID before returning `SuspendSettled`. The private source-bound fence holds at most 64 closed attempts without eviction; a full journal fails closed. Rollback keeps the same generation and its fences. Only a strictly newer, exact owned compute incarnation replaces the fence, while stale-source requests fail ownership checks before native calls. diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index 77d1869fd..d454cdd8c 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -1,5 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" +COMPUTE_FIELDS = ["Generation","Name","ID","RestoredFrom"] ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] @@ -10,9 +11,13 @@ MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 1 +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute_info","compute_renew","suspend","resume","compute_kill","delete_retained","compute_command","resume_compute"] +PROTOCOL_VERSION = 4 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Retained","Deadline"] +RESPONSE_FIELDS = ["Version","State","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +RESUME_FIELDS = ["Workspace","Reference","OperationID","Retained","Target","ReconcileOnly"] +RETAINED_FIELDS = ["Compatibility","Reference","ID","Data","OperationID","SourceGeneration","SourceName","SourceID"] SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" +SUSPEND_FIELDS = ["Reference","OperationID","Source","Retained","ReconcileOnly"] diff --git a/services/core/tools/e2b-provider/observation_test.py b/services/core/tools/e2b-provider/observation_test.py index ef204d6d7..3a773ca24 100644 --- a/services/core/tools/e2b-provider/observation_test.py +++ b/services/core/tools/e2b-provider/observation_test.py @@ -7,6 +7,7 @@ from provider import Provider from provider_test import ProviderTest from state import Failure +from helper_contract_generated import PROTOCOL_VERSION class ObservationTest(ProviderTest): @@ -15,7 +16,7 @@ def observe(self, reference=None): try: return Provider(request).execute() except Failure as error: - return {'Version': 1, 'ErrorCode': error.code} + return {'Version': PROTOCOL_VERSION, 'ErrorCode': error.code} def listing(self, *pages): pages = [list(page) for page in pages] or [[]] diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index 76557b500..fbc595fef 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -7,7 +7,7 @@ from datetime import datetime, timezone from uuid import UUID -from e2b import Sandbox, SandboxQuery, SandboxState +from e2b import FileType, Sandbox, SandboxQuery, SandboxState from e2b.api.client.models.sandbox_metric import SandboxMetric from e2b.exceptions import AuthenticationException, FileNotFoundException, SandboxNotFoundException @@ -19,6 +19,20 @@ PREFIX = 'oac_' FIELDS = ('InstallationID', *REFERENCE_FIELDS) +# Execute the existing protected entry point and return its durable receipt on +# the same command stream. A read failure leaves successful startup recoverable +# through Inspect, without replaying initialization. +BOOTSTRAP_SCRIPT = """import runpy,sys +runpy.run_path('/opt/oac-e2b/managed_init.py', run_name='__main__') +try: + with open('/root/.oac/e2b/managed-ready.json', 'rb') as source: + receipt = source.read(4097) + if len(receipt) <= 4096: + sys.stdout.buffer.write(receipt) +except OSError: + pass +""" + def valid_id(value): try: @@ -91,6 +105,9 @@ def __init__(self, request): bootstrap = request.get('Bootstrap') if bootstrap is not None and (not isinstance(bootstrap, dict) or bootstrap.get('Workspace') is not None): raise Failure('invalid') + resume = request.get('Resume') + if resume is not None and (not isinstance(resume, dict) or resume.get('Workspace') is not None): + raise Failure('invalid') deadline = datetime.fromisoformat(request['Deadline'].replace('Z', '+00:00')) self.deadline = time.monotonic() + (deadline - datetime.now(timezone.utc)).total_seconds() self.metadata = {PREFIX + field.lower(): value for field, value in @@ -196,13 +213,17 @@ def inspect(self): except FileNotFoundException: receipt = None if receipt is not None: - expected = record.get('bootstrap_identity') - if (receipt.get('identity') != expected or receipt.get('status') != 'daemon_started' or - type(receipt.get('daemon_pid')) is not int or receipt['daemon_pid'] <= 0): - raise Failure('ownership') - self.receipt.save(settled=True, bootstrap_complete=True) + self.accept_bootstrap_receipt(receipt) return cloud + def accept_bootstrap_receipt(self, receipt): + expected = (self.receipt.data or {}).get('bootstrap_identity') + if (not isinstance(receipt, dict) or receipt.get('identity') != expected or + receipt.get('status') != 'daemon_started' or + type(receipt.get('daemon_pid')) is not int or receipt['daemon_pid'] <= 0): + raise Failure('ownership') + self.receipt.save(settled=True, bootstrap_complete=True) + def create(self): if self.receipt.data is not None: raise Failure('exists') @@ -220,7 +241,9 @@ def create(self): if definitely_rejected(error): self.receipt.save(status='rejected', settled=True) raise Failure('unconfirmed') from None - self.receipt.save(status='created', ids=[cloud.sandbox_id], connection=connection_material(cloud)) + self.receipt.save(status='created', ids=[cloud.sandbox_id], connection=connection_material(cloud), + compute={'Generation': 0, 'Name': self.reference['AllocationID'], + 'ID': cloud.sandbox_id, 'RestoredFrom': None}) # A create response must not steer envd traffic to an unrelated host. self.check_domain(cloud) # SDK Create returns connection material, but no metadata or resources. @@ -233,27 +256,40 @@ def create(self): self.receipt.save(status='configuration_rejected', settled=True) raise # Validate the current template entry point before writing any credential. - check = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', - "import os,sys; sys.exit(78 if not os.path.isfile('/opt/oac-e2b/managed_init.py') or not os.access('/opt/oac-e2b/managed_init.py', os.R_OK) else 0)"]}, - self.remaining, user='root') - if check['ExitCode'] != 0: + try: + entry = cloud.files.get_info('/opt/oac-e2b/managed_init.py', + user='root', request_timeout=self.remaining()) + if entry.type != FileType.FILE: + self.receipt.save(status='bootstrap_failed', settled=True) + raise Failure('template_invalid') + # A successful download can also refer to a special file. Check + # the type first, then verify readability without a probe process. + with cloud.files.read('/opt/oac-e2b/managed_init.py', format='stream', + user='root', request_timeout=self.remaining()): + pass + except FileNotFoundException: self.receipt.save(status='bootstrap_failed', settled=True) - raise Failure('template_invalid' if check['ExitCode'] == 78 else 'unconfirmed') + raise Failure('template_invalid') from None payload = dict(bootstrap, InstallationID=self.config['InstallationID'], RuntimeBootstrap=self.q['RuntimeBootstrap']) - del payload['CoreURL'], payload['Credential'] + del payload['CoreURL'], payload['Credential'], payload['Harness'] if (set(payload) - set(MANAGED_BOOTSTRAP_FIELDS) or not set(MANAGED_BOOTSTRAP_REQUIRED_FIELDS) <= set(payload)): raise Failure('invalid') cloud.files.write('/root/.oac/e2b/managed-bootstrap.json', json.dumps(payload), user='root', request_timeout=self.remaining()) self.receipt.save(status='bootstrap_pending') - result = run(cloud, {'Args': ['/usr/bin/python3', '-I', '/opt/oac-e2b/managed_init.py']}, + result = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', BOOTSTRAP_SCRIPT]}, self.remaining, user='root') if result['ExitCode'] != 0: self.receipt.save(status='bootstrap_failed', settled=True) raise Failure('unconfirmed') self.receipt.save(status='bootstrap_exited', settled=True) + try: + receipt = json.loads(result['Stdout']) + except (ValueError, KeyError): + raise Failure('unconfirmed') from None + self.accept_bootstrap_receipt(receipt) return self.inspect() def renew(self): @@ -263,10 +299,13 @@ def renew(self): Sandbox.set_timeout(cloud.sandbox_id, self.config['TimeoutSeconds'], **self.options()) return self.qualified(self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options()))) - def kill(self): + def kill(self, expected_paused_id=None): if self.rejected_absence(): return found = self.discover() + if expected_paused_id is not None and (len(found) > 1 or any( + cloud.sandbox_id != expected_paused_id or cloud.state != 'paused' for cloud in found)): + raise Failure('unconfirmed') # The allocation flock excludes any still-running local Create helper. # A matching actual VM proves the original request reached allocation. known = bool((self.receipt.data or {}).get('ids')) @@ -382,6 +421,10 @@ def execute(self): with Receipt(self.q, self.remaining) as self.receipt: try: operation = self.q['Operation'] + if operation in ('compute_info', 'compute_renew', 'suspend', 'resume', 'compute_kill', + 'delete_retained', 'compute_command', 'resume_compute'): + from suspension import Suspension + return Suspension(self, Sandbox, connection_material, run).execute() if operation == 'kill': self.kill() return {'Version': PROTOCOL_VERSION, 'Info': self.info(absent=True), 'ErrorCode': ''} diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 74d708547..9bd122dea 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -1,22 +1,63 @@ """Controlled SDK boundary failures; live qualification remains separate.""" import copy +import io +import httpx from datetime import datetime, timedelta, timezone import json from pathlib import Path import tempfile +import subprocess +import sys from types import SimpleNamespace import unittest -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch from uuid import uuid4 -from e2b import SandboxState -from e2b.exceptions import AuthenticationException, SandboxNotFoundException +from e2b import FileType, SandboxState +from e2b.connection_config import ConnectionConfig +from e2b.sandbox_sync.filesystem.filesystem import Filesystem +from packaging.version import Version +from e2b.exceptions import AuthenticationException, FileNotFoundException, SandboxNotFoundException -from provider import Provider +from provider import BOOTSTRAP_SCRIPT, Provider +from helper_contract_generated import PROTOCOL_VERSION from sdk import restore, run from state import Failure, Receipt +class BootstrapScriptTest(unittest.TestCase): + def execute(self, startup, receipt=None): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + entry = root / 'managed_init.py' + ready = root / 'managed-ready.json' + entry.write_text(startup.replace('READY_PATH', repr(str(ready)))) + if receipt is not None: + ready.write_bytes(receipt) + script = BOOTSTRAP_SCRIPT.replace('/opt/oac-e2b/managed_init.py', str(entry)).replace( + '/root/.oac/e2b/managed-ready.json', str(ready)) + return subprocess.run([sys.executable, '-I', '-c', script], + capture_output=True, timeout=5) + + def test_success_reads_receipt_after_initialization(self): + result = self.execute("from pathlib import Path; Path(READY_PATH).write_bytes(b'new receipt')", + receipt=b'stale receipt') + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, b'new receipt') + + def test_failed_initialization_cannot_return_old_receipt(self): + result = self.execute('raise SystemExit(7)', receipt=b'stale receipt') + self.assertEqual(result.returncode, 7) + self.assertEqual(result.stdout, b'') + + def test_missing_and_oversized_receipts_leave_startup_recoverable(self): + for receipt in (None, b'x' * 4097): + with self.subTest(size=len(receipt) if receipt else None): + result = self.execute('pass', receipt=receipt) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, b'') + + class ProviderTest(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() @@ -24,13 +65,13 @@ def setUp(self): self.reference = {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']} self.config = {'StateDir': self.temporary.name, 'InstallationID': str(uuid4()), 'APIKey': 'private-account-secret', 'Template': 'test:' + str(uuid4()), 'TimeoutSeconds': 120} - self.request = {'Version': 1, 'Operation': 'create', 'Config': self.config, + self.request = {'Version': PROTOCOL_VERSION, 'Operation': 'create', 'Config': self.config, 'Reference': self.reference, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=30)).isoformat(), 'Bootstrap': dict(self.reference, SessionID=str(uuid4()), DeviceID=str(uuid4()), CoreURL='https://core.example/api/v1', Credential='private-runtime-secret', - NetworkAccess='enabled', AllowedDomains=[])} + Harness='codex', NetworkAccess='enabled', AllowedDomains=[])} self.request['RuntimeBootstrap'] = { - 'version': 1, 'core_url': self.request['Bootstrap']['CoreURL'], + 'version': 2, 'harness': 'codex', 'core_url': self.request['Bootstrap']['CoreURL'], 'device_id': self.request['Bootstrap']['DeviceID'], 'credential': self.request['Bootstrap']['Credential']} self.cloud = Mock(sandbox_id='owned-id', sandbox_domain='e2b.app', _envd_version='0.5.0', @@ -40,7 +81,12 @@ def setUp(self): self.identity = dict(self.reference, InstallationID=self.config['InstallationID'], SessionID=self.request['Bootstrap']['SessionID'], DeviceID=self.request['Bootstrap']['DeviceID']) self.ready = json.dumps({'identity': self.identity, 'status': 'daemon_started', 'daemon_pid': 123}) + self.cloud.files.get_info.return_value = SimpleNamespace(type=FileType.FILE) + self.template_stream = MagicMock() self.cloud.files.read.return_value = self.ready + self.cloud.files.read.side_effect = lambda path, **kwargs: ( + self.template_stream if path == '/opt/oac-e2b/managed_init.py' + else self.cloud.files.read.return_value) self.api = Mock() self.api.create.return_value = self.cloud self.api.get_info.return_value = self.cloud @@ -51,7 +97,8 @@ def setUp(self): self.runtime = patch('provider.restore', return_value=self.cloud) self.runtime.start() self.addCleanup(self.runtime.stop) - self.command = patch('provider.run', return_value={'Stdout': '', 'Stderr': '', 'ExitCode': 0}) + self.command = patch('provider.run', side_effect=lambda *a, **k: { + 'Stdout': self.cloud.files.read.return_value, 'Stderr': '', 'ExitCode': 0}) self.command.start() self.addCleanup(self.command.stop) @@ -108,6 +155,7 @@ def test_create_refuses_other_owner_before_credentials(self): self.assertEqual(self.call('create')['ErrorCode'], 'ownership') self.api.get_info.assert_called_once() self.cloud.files.write.assert_not_called() + self.cloud.files.read.assert_not_called() def test_qualified_gateway_template_id_accepts_only_selected_build(self): self.config['Resources'] = {'cpus': 2, 'memory_mib': 2048} @@ -145,7 +193,7 @@ def test_create_refuses_foreign_data_plane_before_envd(self): self.cloud.commands.run.assert_not_called() def test_template_invalid_refuses_before_credentials_and_retains_owned_cleanup(self): - with patch('provider.run', return_value={'ExitCode': 78, 'Stdout': '', 'Stderr': ''}): + with patch.object(self.cloud.files, 'read', side_effect=FileNotFoundException('missing')): result = self.call('create') self.assertEqual(result['ErrorCode'], 'template_invalid') self.assertTrue(result['Info']['CreateSettled']) @@ -155,6 +203,75 @@ def test_template_invalid_refuses_before_credentials_and_retains_owned_cleanup(s self.api.kill.assert_not_called() self.assertEqual(self.call('create')['ErrorCode'], 'exists') + def test_template_non_regular_files_are_rejected_before_open_or_credentials(self): + for kind in (FileType.DIR, None): + with self.subTest(kind=kind): + self.reference['AllocationID'] = str(uuid4()) + self.request['Bootstrap']['AllocationID'] = self.reference['AllocationID'] + self.cloud.metadata = Provider(self.request).metadata + self.cloud.files.get_info.return_value = SimpleNamespace(type=kind) + self.assertEqual(self.call('create')['ErrorCode'], 'template_invalid') + self.cloud.files.read.assert_not_called() + self.cloud.files.write.assert_not_called() + + def test_template_read_is_closed_and_does_not_launch_a_probe(self): + with patch('provider.run', return_value={'ExitCode': 0, 'Stdout': self.ready}) as command: + self.assertEqual(self.call('create')['ErrorCode'], '') + self.template_stream.__enter__.assert_called_once() + self.template_stream.__exit__.assert_called_once() + self.template_stream.__iter__.assert_not_called() + command.assert_called_once() + self.assertEqual(command.call_args.args[1]['Args'], + ['/usr/bin/python3', '-I', '-c', BOOTSTRAP_SCRIPT]) + args, options = self.cloud.files.read.call_args_list[0] + self.assertEqual(args, ('/opt/oac-e2b/managed_init.py',)) + self.assertEqual(options['format'], 'stream') + self.assertEqual(options['user'], 'root') + self.assertGreater(options['request_timeout'], 0) + + def test_template_probe_uses_sdk_stream_and_closes_without_downloading(self): + class Body(httpx.SyncByteStream): + closed = False + + def __iter__(self): + raise AssertionError('template contents must not be downloaded') + yield b'' + + def close(self): + self.closed = True + + body = Body() + requests = [] + + def respond(request): + requests.append(request) + return httpx.Response(200, stream=body) + + with httpx.Client(base_url='https://fixture.invalid', + transport=httpx.MockTransport(respond)) as client: + with patch('e2b.sandbox_sync.filesystem.filesystem.get_envd_api', return_value=client), \ + patch('e2b.sandbox_sync.filesystem.filesystem.create_rpc_client'): + files = Filesystem('https://fixture.invalid', Version('0.5.0'), + ConnectionConfig(api_key='fixture'), client) + original_read = self.cloud.files.read.side_effect + self.cloud.files.read.side_effect = lambda path, **kwargs: ( + files.read(path, **kwargs) if path == '/opt/oac-e2b/managed_init.py' + else original_read(path, **kwargs)) + self.assertEqual(self.call('create')['ErrorCode'], '') + self.assertTrue(body.closed) + self.assertEqual(len(requests), 1) + self.assertEqual(requests[0].method, 'GET') + self.assertEqual(requests[0].url.params['path'], '/opt/oac-e2b/managed_init.py') + self.assertEqual(requests[0].url.params['username'], 'root') + + def test_uncertain_template_read_never_writes_credentials_or_replays_create(self): + with patch.object(self.cloud.files, 'read', side_effect=TimeoutError('private secret')): + self.assertEqual(self.call('create')['ErrorCode'], 'unconfirmed') + self.cloud.files.write.assert_not_called() + self.assertFalse(self.record().get('bootstrap_complete', False)) + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + self.api.create.assert_called_once() + def test_unknown_create_empty_lookup_never_proves_cleanup(self): self.api.create.side_effect = TimeoutError('confidential SDK diagnostic') self.assertEqual(self.call('create')['ErrorCode'], 'unconfirmed') @@ -239,6 +356,23 @@ def test_mismatched_receipt_cannot_prove_bootstrap_complete(self): self.assertTrue(self.record()['settled']) self.assertFalse(self.record()['bootstrap_complete']) + def test_successful_create_returns_receipt_without_remote_receipt_read(self): + self.assertTrue(self.call('create')['Info']['BootstrapComplete']) + self.assertEqual([call.args[0] for call in self.cloud.files.read.call_args_list], + ['/opt/oac-e2b/managed_init.py']) + self.assertEqual(self.api.get_info.call_count, 2) + + def test_lost_command_receipt_recovers_from_file_without_replaying_startup(self): + with patch('provider.run', return_value={'ExitCode': 0, 'Stdout': ''}) as command: + result = self.call('create') + self.assertEqual(result['ErrorCode'], 'unconfirmed') + self.assertTrue(result['Info']['CreateSettled']) + self.assertFalse(result['Info']['BootstrapComplete']) + self.assertTrue(self.call('inspect')['Info']['BootstrapComplete']) + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + command.assert_called_once() + self.api.create.assert_called_once() + def test_foreign_owner_prevents_renew_and_delete(self): self.call('create') self.cloud.metadata = {} diff --git a/services/core/tools/e2b-provider/state_test.py b/services/core/tools/e2b-provider/state_test.py index ae573370a..7de903990 100644 --- a/services/core/tools/e2b-provider/state_test.py +++ b/services/core/tools/e2b-provider/state_test.py @@ -11,6 +11,7 @@ from unittest.mock import patch from provider import Provider +from helper_contract_generated import PROTOCOL_VERSION from state import Failure, Receipt @@ -18,7 +19,7 @@ class StateTest(unittest.TestCase): def setUp(self): self.root = tempfile.TemporaryDirectory() self.addCleanup(self.root.cleanup) - self.request = {'Version': 1, 'Operation': 'kill', + self.request = {'Version': PROTOCOL_VERSION, 'Operation': 'kill', 'Config': {'StateDir': self.root.name, 'InstallationID': str(uuid4()), 'APIKey': 'test'}, 'Reference': {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']}, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=2)).isoformat()} diff --git a/services/core/tools/e2b-provider/suspension.py b/services/core/tools/e2b-provider/suspension.py new file mode 100644 index 000000000..453fb9ac6 --- /dev/null +++ b/services/core/tools/e2b-provider/suspension.py @@ -0,0 +1,269 @@ +"""E2B's implementation of exact-incarnation suspension under the allocation lock.""" +import json + +from helper_contract_generated import PROTOCOL_VERSION, COMPUTE_FIELDS, SUSPEND_FIELDS, RESUME_FIELDS +from state import Failure + + +class Suspension: + def __init__(self, provider, sdk, material, command): + self.p, self.sdk, self.material, self.command = provider, sdk, material, command + + @property + def record(self): + record = self.p.receipt.data + if not record or record.get('settled') is not True: + raise Failure('unconfirmed') + return record + + def current(self): + current = self.record.get('compute') + if not isinstance(current, dict) or not current.get('ID'): + raise Failure('unconfirmed') + return current + + def matches(self, wanted, actual, unresolved=False): + if (not isinstance(wanted, dict) or set(wanted) != set(COMPUTE_FIELDS) or + type(wanted['Generation']) is not int or wanted['Generation'] < 0 or + wanted['Name'] != self.p.reference['AllocationID'] or + any(wanted[k] != actual[k] for k in ('Generation', 'Name', 'RestoredFrom')) or + (wanted['ID'] != actual['ID'] and not (unresolved and wanted['ID'] == ''))): + raise Failure('ownership') + + def cloud(self, execution=True): + found = self.p.discover() + if len(found) != 1: + raise Failure('unconfirmed') + cloud = found[0] + if cloud.sandbox_id != self.current()['ID']: + raise Failure('ownership') + if execution: + self.p.qualified(cloud) + return cloud + + def state(self, current, cloud): + return {'Compute': current, 'Status': cloud.state, + 'BootstrapComplete': self.record.get('bootstrap_complete') is True, + 'Retained': None, 'ResourcesReleased': False, 'SuspendSettled': False} + + def retained(self, operation, current): + return {'Reference': self.p.reference['AllocationID'], 'ID': operation, 'OperationID': operation, + 'SourceGeneration': current['Generation'], 'SourceName': current['Name'], + 'SourceID': current['ID'], + 'Data': json.dumps({'version': 1, 'sandbox_id': current['ID']}, sort_keys=True, separators=(',', ':'))} + + def request(self, field): + q = self.p.q.get(field) + fields = set(SUSPEND_FIELDS if field == 'Suspend' else RESUME_FIELDS) + required = fields - ({'Workspace'} if field == 'Resume' else set()) + if (not isinstance(q, dict) or set(q) - fields or not required <= set(q) or + (field == 'Resume' and q.get('Workspace') is not None) or + q.get('Reference') != self.p.reference): + raise Failure('invalid') + # UUID parsing is shared with the allocation envelope validator. + from provider import valid_id + if not valid_id(q.get('OperationID')) or type(q.get('ReconcileOnly')) is not bool: + raise Failure('invalid') + return q + + def inspect(self, operation): + current = self.current() + self.matches(self.p.q.get('Compute'), current, unresolved=operation == 'compute_info') + if self.record.get('status') == 'killed': + raise Failure('not_found') + cloud = self.cloud() + if operation != 'compute_info': + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + pending = self.record.get('suspension') + if pending and pending['phase'] not in ('resumed', 'consumed'): + raise Failure('unconfirmed') + if operation == 'compute_renew': + self.sdk.set_timeout(cloud.sandbox_id, self.p.config['TimeoutSeconds'], **self.p.options()) + cloud = self.cloud() + if operation == 'compute_command': + return {'Command': self.command(self.p.client(cloud), self.p.q['Command'], self.p.remaining)} + return {'State': self.state(current, cloud)} + + def suspend(self): + q = self.request('Suspend') + current = self.current() + self.matches(q.get('Source'), current) + fence = self.record.get('suspend_fence') + closed = [] + if fence: + if fence['source'] == current: + closed = fence['closed'] + elif fence['source']['Generation'] >= current['Generation']: + raise Failure('ownership') + if q['OperationID'] in closed and not q['ReconcileOnly']: + raise Failure('ownership') + handle = self.retained(q['OperationID'], current) + if q.get('Retained') not in (None, handle): + raise Failure('ownership') + entry = self.record.get('suspension') + if entry and entry['retained'] == handle: + if entry['phase'] not in ('pause_pending', 'paused'): + raise Failure('ownership') + elif q['ReconcileOnly']: + # No durable native intent: the allocation lock proves this helper + # never issued pause. Only a qualified running source can roll back. + if entry and entry['phase'] != 'consumed': + raise Failure('ownership') + cloud = self.cloud() + if cloud.state != 'running': + raise Failure('unconfirmed') + if q['OperationID'] not in closed: + if len(closed) >= 64: + raise Failure('unconfirmed') + closed = closed + [q['OperationID']] + # Close the exact attempt durably before Core may thaw this source. + # Rollback does not advance the generation and cannot clear fences. + self.p.receipt.save(suspend_fence={'source': current, 'closed': closed}) + state = self.state(current, cloud) + state['SuspendSettled'] = True + return {'State': state} + else: + if entry and entry['phase'] != 'consumed': + raise Failure('ownership') + cloud = self.cloud() + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + entry = {'retained': handle, 'phase': 'pause_pending'} + self.p.receipt.save(suspension=entry) + # An error remains pending. A later observation may prove paused; + # a running observation never proves a timed-out pause cannot land. + self.sdk.pause(current['ID'], keep_memory=True, **self.p.options()) + cloud = self.cloud() + if cloud.state != 'paused': + raise Failure('unconfirmed') + self.p.receipt.save(suspension=dict(entry, phase='paused')) + state = self.state(current, cloud) + state.update(Status='suspended', Retained=handle, ResourcesReleased=True, SuspendSettled=True) + return {'State': state} + + def resume(self): + q = self.request('Resume') + entry = self.record.get('suspension') + if not entry or entry['retained'] != q.get('Retained'): + raise Failure('ownership') + retained = entry['retained'] + target = {'Generation': retained['SourceGeneration'] + 1, 'Name': retained['SourceName'], + 'ID': retained['SourceID'], 'RestoredFrom': retained} + self.matches(q.get('Target'), target) + closed = entry.get('closed_resume_ids', []) + if q['OperationID'] in closed: + if not q['ReconcileOnly'] or entry['phase'] != 'paused': + raise Failure('ownership') + return {'State': {'Compute': target, 'Status': 'absent', 'BootstrapComplete': False, + 'Retained': None, 'ResourcesReleased': False, 'SuspendSettled': False, + 'RestoreAttemptClosed': q['OperationID']}} + if entry['phase'] == 'paused' and q['ReconcileOnly']: + # Under the allocation flock no helper has persisted native dispatch + # for this attempt. Fence its late request durably before reporting it. + # Scope the bounded history to this retained generation; never evict. + if len(closed) >= 64: + raise Failure('unconfirmed') + self.p.receipt.save(suspension=dict(entry, closed_resume_ids=closed + [q['OperationID']])) + return {'State': {'Compute': target, 'Status': 'absent', 'BootstrapComplete': False, + 'Retained': None, 'ResourcesReleased': False, 'SuspendSettled': False, + 'RestoreAttemptClosed': q['OperationID']}} + if entry['phase'] in ('resume_pending', 'resumed', 'consumed'): + if entry.get('resume_id') != q['OperationID'] or entry.get('target') != target: + raise Failure('ownership') + elif entry['phase'] == 'paused' and not q['ReconcileOnly']: + cloud = self.cloud() + if cloud.state != 'paused': + raise Failure('unconfirmed') + entry = dict(entry, phase='resume_pending', resume_id=q['OperationID'], target=target, connected=False) + self.p.receipt.save(suspension=entry) + connected = self.sdk.connect(target['ID'], timeout=self.p.config['TimeoutSeconds'], + on_resume='restore', **self.p.options()) + if connected.sandbox_id != target['ID']: + raise Failure('ownership') + self.p.check_domain(connected) + entry = dict(entry, connected=True) + self.p.receipt.save(suspension=entry, connection=self.material(connected)) + else: + raise Failure('unconfirmed') + # Never repeat connect after an uncertain reply. Fresh connection material + # must have been durably received before execution can resume. + if not entry.get('connected'): + raise Failure('unconfirmed') + cloud = self.cloud() + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + if entry['phase'] != 'consumed': + self.p.receipt.save(compute=target, suspension=dict(entry, phase='resumed')) + return {'State': self.state(target, cloud)} + + def kill(self): + wanted = self.p.q.get('Compute') + current = self.current() + entry = self.record.get('suspension') + target = (entry or {}).get('target') + # Target-first cleanup may arrive before Resume was ever dispatched. + planned = None + if entry: + r = entry['retained'] + planned = {'Generation': r['SourceGeneration'] + 1, 'Name': r['SourceName'], + 'ID': r['SourceID'], 'RestoredFrom': r} + if wanted == planned and target is None and entry['phase'] == 'paused': + return {} + if wanted != current and wanted != target: + # Once destruction is confirmed, stale source cleanup is harmless. + if self.record.get('status') == 'killed' and isinstance(wanted, dict): + if wanted['ID'] == current['ID'] and wanted['Name'] == current['Name'] and wanted['Generation'] < current['Generation']: + return {} + raise Failure('ownership') + if entry and entry['phase'] == 'resume_pending' and entry.get('connected'): + # The SDK reply already settled the original connect. Cleanup owns + # recovery once Core leaves restoring, so reconcile that saved target + # here without another connect or a resource-qualification gate. + if wanted != target: + raise Failure('ownership') + found = self.p.discover() + if len(found) > 1 or any(cloud.sandbox_id != target['ID'] for cloud in found): + raise Failure('ownership') + entry = dict(entry, phase='resumed') + self.p.receipt.save(compute=target, suspension=entry) + if entry and entry['phase'] in ('pause_pending', 'resume_pending'): + # No successful native reply or settled pause has been observed. + raise Failure('unconfirmed') + self.p.kill() + return {} + + def delete(self): + wanted = self.p.q.get('Retained') + entry = self.record.get('suspension') + if not entry or wanted != entry['retained']: + raise Failure('ownership') + if entry['phase'] == 'consumed': + return {} + if self.record.get('status') != 'killed' and entry['phase'] == 'paused': + source = {'Generation': wanted['SourceGeneration'], 'Name': wanted['SourceName'], + 'ID': wanted['SourceID'], 'RestoredFrom': self.current()['RestoredFrom']} + self.matches(source, self.current()) + # A settled, unconsumed pause owns this exact native resource. The + # final discovery in kill rechecks paused ownership; unknown deletion + # keeps this receipt, and confirmed absence can settle a retry. + self.p.kill(expected_paused_id=source['ID']) + if self.record.get('status') != 'killed': + if entry['phase'] != 'resumed': + raise Failure('unconfirmed') + self.matches(entry['target'], self.current()) + if self.cloud().state != 'running': + raise Failure('unconfirmed') + # E2B consumed the pause image on restore. Keep its minimal generation + # receipt to reject delayed resume/cleanup; never kill running compute. + self.p.receipt.save(suspension=dict(entry, phase='consumed')) + return {} + + def execute(self): + operation = self.p.q['Operation'] + if operation in ('compute_info', 'compute_renew', 'compute_command', 'resume_compute'): + result = self.inspect(operation) + else: + result = {'suspend': self.suspend, 'resume': self.resume, + 'compute_kill': self.kill, 'delete_retained': self.delete}[operation]() + return dict(result, Version=PROTOCOL_VERSION, ErrorCode='') diff --git a/services/core/tools/e2b-provider/suspension_test.py b/services/core/tools/e2b-provider/suspension_test.py new file mode 100644 index 000000000..de1d6c5e0 --- /dev/null +++ b/services/core/tools/e2b-provider/suspension_test.py @@ -0,0 +1,288 @@ +"""Native lifecycle effects and durable recovery are tested through the helper.""" +import unittest +from uuid import uuid4 + +from provider import Provider +import provider_test + + +class SuspensionTest(unittest.TestCase): + setUp = provider_test.ProviderTest.setUp + call = provider_test.ProviderTest.call + record = provider_test.ProviderTest.record + + def prepare(self): + self.assertEqual(self.call('create')['ErrorCode'], '') + self.api.pause.side_effect = lambda *a, **k: setattr(self.cloud, 'state', 'paused') + def connect(*args, **kwargs): + self.cloud.state = 'running' + return self.cloud + self.api.connect.side_effect = connect + return self.record()['compute'] + + def invoke(self, operation, **payload): + return Provider(dict(self.request, Operation=operation, **payload)).execute() + + def pause(self, source): + q = {'Reference': self.reference, 'OperationID': str(uuid4()), 'Source': source, + 'Retained': None, 'ReconcileOnly': False} + response = self.invoke('suspend', Suspend=q) + return q, response + + def resume_request(self, retained): + target = {'Generation': retained['SourceGeneration'] + 1, 'Name': retained['SourceName'], + 'ID': retained['SourceID'], 'RestoredFrom': retained} + return {'Reference': self.reference, 'OperationID': str(uuid4()), 'Target': target, + 'Retained': retained, 'ReconcileOnly': False} + + def test_resume_rejects_external_workspace_before_sdk(self): + from state import Failure + with self.assertRaises(Failure) as raised: + Provider(dict(self.request, Operation='resume', Resume={'Workspace': {}})) + self.assertEqual(raised.exception.code, 'invalid') + self.api.connect.assert_not_called() + + def test_resume_accepts_null_workspace(self): + current = self.prepare() + _, paused = self.pause(current) + request = dict(self.resume_request(paused['State']['Retained']), Workspace=None) + self.assertEqual(self.invoke('resume', Resume=request)['ErrorCode'], '') + + def test_two_cycles_keep_native_id_and_fence_old_generation(self): + current = self.prepare() + for generation in range(2): + old = current + q, paused = self.pause(current) + self.assertEqual(paused['ErrorCode'], '') + self.assertTrue(paused['State']['ResourcesReleased']) + retained = paused['State']['Retained'] + request = self.resume_request(retained) + result = self.invoke('resume', Resume=request) + self.assertEqual(result['ErrorCode'], '') + current = result['State']['Compute'] + self.assertEqual(current['ID'], old['ID']) + self.assertEqual(current['Generation'], generation + 1) + self.assertEqual(self.invoke('compute_kill', Compute=old)['ErrorCode'], 'ownership') + self.api.kill.assert_not_called() + # Core wakes the parked daemon before deleting the consumed receipt. + self.assertEqual(self.invoke('compute_command', Compute=current, + Command={'Args': ['oac-daemon', 'resume']})['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.assertEqual(self.invoke('compute_renew', Compute=current)['ErrorCode'], '') + self.assertEqual(self.api.pause.call_count, 2) + self.assertEqual(self.api.connect.call_count, 2) + self.assertEqual(self.api.connect.call_args.kwargs['on_resume'], 'restore') + self.assertEqual(self.api.connect.call_args.kwargs['timeout'], self.config['TimeoutSeconds']) + self.assertTrue(self.api.pause.call_args.kwargs['keep_memory']) + + def test_lost_pause_reply_observes_without_replay(self): + current = self.prepare() + def lost(*args, **kwargs): + self.cloud.state = 'paused' + raise TimeoutError() + self.api.pause.side_effect = lost + q, result = self.pause(current) + self.assertEqual(result['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('suspend', Suspend=dict(q, ReconcileOnly=True))['ErrorCode'], '') + self.api.pause.assert_called_once() + + def test_pending_pause_running_cannot_roll_back_or_delete(self): + current = self.prepare() + self.api.pause.side_effect = TimeoutError() + q, result = self.pause(current) + self.assertEqual(result['ErrorCode'], 'unconfirmed') + for _ in range(2): + self.assertEqual(self.invoke('suspend', Suspend=dict(q, ReconcileOnly=True))['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], 'unconfirmed') + self.api.pause.assert_called_once() + self.api.kill.assert_not_called() + + def test_missing_native_pause_intent_can_settle_running_rollback(self): + current = self.prepare() + q = {'Reference': self.reference, 'OperationID': str(uuid4()), 'Source': current, + 'Retained': None, 'ReconcileOnly': True} + state = self.invoke('suspend', Suspend=q)['State'] + self.assertTrue(state['SuspendSettled']) + self.assertFalse(state['ResourcesReleased']) + self.assertIsNone(state['Retained']) + self.api.pause.assert_not_called() + + def test_lost_resume_reply_never_replays_connect(self): + current = self.prepare() + _, paused = self.pause(current) + q = self.resume_request(paused['State']['Retained']) + def lost(*a, **k): + self.cloud.state = 'running' + raise TimeoutError() + self.api.connect.side_effect = lost + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('resume', Resume=dict(q, ReconcileOnly=True))['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('compute_kill', Compute=q['Target'])['ErrorCode'], 'unconfirmed') + self.api.connect.assert_called_once() + self.api.kill.assert_not_called() + + def test_lost_core_resume_reply_adopts_saved_target_without_connect(self): + current = self.prepare() + _, paused = self.pause(current) + q = self.resume_request(paused['State']['Retained']) + first = self.invoke('resume', Resume=q) + self.assertEqual(first['ErrorCode'], '') + second = self.invoke('resume', Resume=dict(q, ReconcileOnly=True)) + self.assertEqual(first, second) + self.api.connect.assert_called_once() + + def test_foreign_retained_and_incarnation_cannot_mutate(self): + current = self.prepare() + _, paused = self.pause(current) + retained = paused['State']['Retained'] + q = self.resume_request(dict(retained, SourceID='foreign-id')) + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'ownership') + self.assertEqual(self.invoke('delete_retained', Retained=dict(retained, ID='foreign'))['ErrorCode'], 'ownership') + self.api.connect.assert_not_called() + self.api.kill.assert_not_called() + + def test_archive_recovers_saved_connect_reply_before_cleanup(self): + from e2b.exceptions import SandboxNotFoundException + current = self.prepare() + _, paused = self.pause(current) + retained = paused['State']['Retained'] + q = self.resume_request(retained) + def connected_then_observation_lost(*args, **kwargs): + self.cloud.state = 'running' + self.api.get_info.side_effect = TimeoutError() + return self.cloud + self.api.connect.side_effect = connected_then_observation_lost + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'unconfirmed') + self.assertTrue(self.record()['suspension']['connected']) + self.assertEqual(self.record()['suspension']['phase'], 'resume_pending') + self.api.get_info.side_effect = None + # The old incarnation remains fenced until exact target cleanup settles. + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], 'ownership') + def killed(*args, **kwargs): + self.api.get_info.side_effect = SandboxNotFoundException('gone') + self.api.kill.side_effect = killed + self.assertEqual(self.invoke('compute_kill', Compute=q['Target'])['ErrorCode'], '') + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.api.connect.assert_called_once() + self.api.kill.assert_called_once() + self.assertEqual(self.record()['status'], 'killed') + def test_paused_delete_reclaims_same_native_id_and_settles_lost_reply(self): + from e2b.exceptions import SandboxNotFoundException + source = self.prepare() + _, paused = self.pause(source) + retained = paused['State']['Retained'] + def killed_then_lost(*args, **kwargs): + self.api.get_info.side_effect = SandboxNotFoundException('gone') + raise TimeoutError() + self.api.kill.side_effect = killed_then_lost + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], 'unconfirmed') + self.assertEqual(self.record()['suspension']['phase'], 'paused') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.assertEqual(self.record()['status'], 'killed') + self.assertEqual(self.record()['suspension']['phase'], 'consumed') + self.api.kill.assert_called_once() + self.assertEqual(self.api.kill.call_args.args[0], source['ID']) + + def test_paused_delete_rejects_running_source(self): + source = self.prepare() + _, paused = self.pause(source) + self.cloud.state = 'running' + self.assertEqual(self.invoke('delete_retained', Retained=paused['State']['Retained'])['ErrorCode'], 'unconfirmed') + self.assertEqual(self.record()['suspension']['phase'], 'paused') + self.api.kill.assert_not_called() + + def test_never_dispatched_restore_closes_durably_and_rejects_late_request(self): + source = self.prepare() + _, paused = self.pause(source) + q = self.resume_request(paused['State']['Retained']) + observed = self.invoke('resume', Resume=dict(q, ReconcileOnly=True)) + self.assertEqual(observed['ErrorCode'], '') + self.assertEqual(observed['State']['RestoreAttemptClosed'], q['OperationID']) + self.assertEqual(observed['State']['Compute'], q['Target']) + self.api.connect.assert_not_called() + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'ownership') + self.assertEqual(self.invoke('resume', Resume=dict(q, ReconcileOnly=True)), observed) + fresh = dict(q, OperationID=str(uuid4())) + self.assertEqual(self.invoke('resume', Resume=fresh)['ErrorCode'], '') + self.assertEqual(self.invoke('resume', Resume=dict(q, ReconcileOnly=True))['ErrorCode'], 'ownership') + self.api.connect.assert_called_once() + + def test_closed_restore_history_is_bounded_without_eviction(self): + source = self.prepare() + _, paused = self.pause(source) + requests = [] + for _ in range(64): + q = self.resume_request(paused['State']['Retained']) + requests.append(q) + self.assertEqual(self.invoke('resume', Resume=dict(q, ReconcileOnly=True))['ErrorCode'], '') + extra = self.resume_request(paused['State']['Retained']) + self.assertEqual(self.invoke('resume', Resume=dict(extra, ReconcileOnly=True))['ErrorCode'], 'unconfirmed') + self.assertEqual(len(self.record()['suspension']['closed_resume_ids']), 64) + self.assertEqual(self.invoke('resume', Resume=requests[0])['ErrorCode'], 'ownership') + self.api.connect.assert_not_called() + # A fresh, authorized attempt can still proceed; closing unknown attempts + # never consumes the original native retained image. + self.assertEqual(self.invoke('resume', Resume=extra)['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=paused['State']['Retained'])['ErrorCode'], '') + current = self.record()['compute'] + _, next_pause = self.pause(current) + self.assertNotIn('closed_resume_ids', self.record()['suspension']) + self.assertEqual(self.invoke('resume', Resume=requests[0])['ErrorCode'], 'ownership') + + def test_paused_delete_settles_after_killed_receipt_write_fails(self): + from e2b.exceptions import SandboxNotFoundException + from state import Receipt + from unittest.mock import patch + source = self.prepare() + _, paused = self.pause(source) + retained = paused['State']['Retained'] + def killed(*args, **kwargs): + self.api.get_info.side_effect = SandboxNotFoundException('gone') + self.api.kill.side_effect = killed + save = Receipt.save + def fail_killed(receipt, **values): + if values.get('status') == 'killed': + raise OSError('fixture durable write failed') + return save(receipt, **values) + with patch.object(Receipt, 'save', fail_killed): + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.api.kill.assert_called_once() + self.assertEqual(self.record()['suspension']['phase'], 'consumed') + + def test_rollback_closes_late_suspend_across_helper_reopen(self): + source = self.prepare() + q = {'Reference': self.reference, 'OperationID': str(uuid4()), 'Source': source, + 'Retained': None, 'ReconcileOnly': True} + result = self.invoke('suspend', Suspend=q) + self.assertTrue(result['State']['SuspendSettled']) + self.assertIsNone(result['State']['Retained']) + self.assertEqual(self.invoke('suspend', Suspend=dict(q, ReconcileOnly=False))['ErrorCode'], 'ownership') + self.assertEqual(self.cloud.state, 'running') + self.api.pause.assert_not_called() + self.assertEqual(self.invoke('suspend', Suspend=q), result) + + def test_suspend_fences_bound_and_survive_same_generation_rollback(self): + source = self.prepare() + requests = [] + for _ in range(64): + q = {'Reference': self.reference, 'OperationID': str(uuid4()), 'Source': source, + 'Retained': None, 'ReconcileOnly': True} + requests.append(q) + self.assertTrue(self.invoke('suspend', Suspend=q)['State']['SuspendSettled']) + extra = dict(q, OperationID=str(uuid4())) + self.assertEqual(self.invoke('suspend', Suspend=extra)['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('suspend', Suspend=dict(requests[0], ReconcileOnly=False))['ErrorCode'], 'ownership') + self.assertEqual(len(self.record()['suspend_fence']['closed']), 64) + # Only a real resume advances the incarnation; the next observation then + # replaces the journal while stale source requests remain fenced. + _, paused = self.pause(source) + resume = self.resume_request(paused['State']['Retained']) + current = self.invoke('resume', Resume=resume)['State']['Compute'] + self.assertEqual(self.invoke('delete_retained', Retained=paused['State']['Retained'])['ErrorCode'], '') + fresh = dict(extra, Source=current, OperationID=str(uuid4())) + self.assertTrue(self.invoke('suspend', Suspend=fresh)['State']['SuspendSettled']) + self.assertEqual(len(self.record()['suspend_fence']['closed']), 1) + self.assertEqual(self.invoke('suspend', Suspend=dict(requests[0], ReconcileOnly=False))['ErrorCode'], 'ownership') diff --git a/services/core/tools/e2b-provider/testdata/contract.json b/services/core/tools/e2b-provider/testdata/contract.json index 80d09bf84..bd22db5b3 100644 --- a/services/core/tools/e2b-provider/testdata/contract.json +++ b/services/core/tools/e2b-provider/testdata/contract.json @@ -1,66 +1,76 @@ [ -{"kind":"managed","name":"disabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"disabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"enabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":"example.com","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":[1],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"unknown","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":true,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-AllocationID","payload":{"AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"disabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"disabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"enabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":"example.com","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":[1],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"unknown","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":true,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-AllocationID","payload":{"AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, {"kind":"managed","name":"missing-RuntimeBootstrap","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444"},"valid":false}, -{"kind":"managed","name":"restricted","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":["example.com"],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"restricted","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, -{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, -{"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"create","payload":{"Version":1,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"inspect","payload":{"Version":1,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"kill","payload":{"Version":1,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_builds","payload":{"Version":1,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_templates","payload":{"Version":1,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"renew","payload":{"Version":1,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"validate_deployment","payload":{"Version":1,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-0","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-32","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-33","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":true},"valid":false}, -{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":null},"valid":false}, -{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":"1"},"valid":false}, -{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, -{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":true}, -{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"response","name":"error-","payload":{"Version":1,"ErrorCode":""},"valid":true}, -{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":1,"ErrorCode":"command_unconfirmed"},"valid":true}, -{"kind":"response","name":"error-exists","payload":{"Version":1,"ErrorCode":"exists"},"valid":true}, -{"kind":"response","name":"error-invalid","payload":{"Version":1,"ErrorCode":"invalid"},"valid":true}, -{"kind":"response","name":"error-not_found","payload":{"Version":1,"ErrorCode":"not_found"},"valid":true}, -{"kind":"response","name":"error-ownership","payload":{"Version":1,"ErrorCode":"ownership"},"valid":true}, -{"kind":"response","name":"error-team_mismatch","payload":{"Version":1,"ErrorCode":"team_mismatch"},"valid":true}, -{"kind":"response","name":"error-template_invalid","payload":{"Version":1,"ErrorCode":"template_invalid"},"valid":true}, -{"kind":"response","name":"error-unauthorized","payload":{"Version":1,"ErrorCode":"unauthorized"},"valid":true}, -{"kind":"response","name":"error-unconfirmed","payload":{"Version":1,"ErrorCode":"unconfirmed"},"valid":true}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":1},"valid":false}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":1},"valid":false}, -{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":1},"valid":false}, +{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444"},"valid":false}, +{"kind":"managed","name":"restricted","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":["example.com"],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"restricted","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, +{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, +{"kind":"request","name":"command","payload":{"Version":4,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_command","payload":{"Version":4,"Operation":"compute_command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Command":{"Args":["true"],"Directory":"","Stdin":null},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_info","payload":{"Version":4,"Operation":"compute_info","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_kill","payload":{"Version":4,"Operation":"compute_kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_renew","payload":{"Version":4,"Operation":"compute_renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"create","payload":{"Version":4,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"delete_retained","payload":{"Version":4,"Operation":"delete_retained","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Retained":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"inspect","payload":{"Version":4,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill","payload":{"Version":4,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_builds","payload":{"Version":4,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_templates","payload":{"Version":4,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"observe","payload":{"Version":4,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"renew","payload":{"Version":4,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume","payload":{"Version":4,"Operation":"resume","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Resume":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"66666666-6666-4666-8666-666666666666","Retained":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"},"Target":{"Generation":1,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"}},"ReconcileOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume-workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"resume","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Resume":{"OperationID":"66666666-6666-4666-8666-666666666666","ReconcileOnly":false,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Retained":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"},"Target":{"Generation":1,"ID":"native-fixture","Name":"33333333-3333-4333-8333-333333333333","RestoredFrom":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"}},"Workspace":null},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":true}, +{"kind":"request","name":"resume-workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"resume","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Resume":{"OperationID":"66666666-6666-4666-8666-666666666666","ReconcileOnly":false,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Retained":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"},"Target":{"Generation":1,"ID":"native-fixture","Name":"33333333-3333-4333-8333-333333333333","RestoredFrom":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"}},"Workspace":{}},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"resume_compute","payload":{"Version":4,"Operation":"resume_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"suspend","payload":{"Version":4,"Operation":"suspend","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Suspend":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"66666666-6666-4666-8666-666666666666","Source":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Retained":null,"ReconcileOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"validate_deployment","payload":{"Version":4,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-0","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-32","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-33","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":true},"valid":false}, +{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":null},"valid":false}, +{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":"1"},"valid":false}, +{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":5},"valid":false}, +{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":true}, +{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"response","name":"error-","payload":{"Version":4,"ErrorCode":""},"valid":true}, +{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":4,"ErrorCode":"command_unconfirmed"},"valid":true}, +{"kind":"response","name":"error-exists","payload":{"Version":4,"ErrorCode":"exists"},"valid":true}, +{"kind":"response","name":"error-invalid","payload":{"Version":4,"ErrorCode":"invalid"},"valid":true}, +{"kind":"response","name":"error-not_found","payload":{"Version":4,"ErrorCode":"not_found"},"valid":true}, +{"kind":"response","name":"error-ownership","payload":{"Version":4,"ErrorCode":"ownership"},"valid":true}, +{"kind":"response","name":"error-team_mismatch","payload":{"Version":4,"ErrorCode":"team_mismatch"},"valid":true}, +{"kind":"response","name":"error-template_invalid","payload":{"Version":4,"ErrorCode":"template_invalid"},"valid":true}, +{"kind":"response","name":"error-unauthorized","payload":{"Version":4,"ErrorCode":"unauthorized"},"valid":true}, +{"kind":"response","name":"error-unconfirmed","payload":{"Version":4,"ErrorCode":"unconfirmed"},"valid":true}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":4},"valid":false}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":4},"valid":false}, +{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":4},"valid":false}, {"kind":"response","name":"invalid-Version","payload":{"ErrorCode":"","Version":true},"valid":false} ] diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index f30db5987..6db1c1822 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -1,6 +1,6 @@ # microsandbox Sandbox Provider helper -microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 5; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.8 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. +microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 5; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.8 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the suspension operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. [Add a Sandbox Provider](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources, Runtime release and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy. @@ -36,7 +36,7 @@ Core persists operation IDs, source and target generations, exact identities and - Native restore leaves the managed root size unset because the target inherits the verified full snapshot. The helper accepts that only with a matching snapshot resource proof and the exact source and target identities, and it checks the target's CPU, memory and Environment disk before keeping the inherited proof. A missing root size never counts as unlimited capacity, and retained state is never resized. - Fresh restore and retry share one completion: verify the original artifact and resource proof, inspect the running target's resources and ancestry, persist its missing derived resource-proof label, then strictly reread the same native ID ([`restore_completion.go`](restore_completion.go)). A conflicting proof is an error. Native restore does not copy the source's ownership labels; ancestry supplies that evidence. There is no ordinary Start, replacement, disk-only restore or cold boot. - **ResumeCompute** thaws the same resident source after an aborted suspension. The pinned SDK handle method is name-based; the allocation lock and ID checks before and after the call fence every managed replacement. Manual lifecycle changes in the managed namespace are unsupported. -- **DeleteSnapshot** accepts only the derived operation selector and matching artifact identity. It fences delayed publication and restoration with a durable deletion receipt before deleting the local SDK snapshot and portable archive. An unsettled admitted restore blocks deletion. The small ownership tombstone remains; Core owns retention and cleanup order. +- **DeleteRetained** accepts only the derived operation selector and matching artifact identity. It fences delayed publication and restoration with a durable deletion receipt before deleting the local SDK snapshot and portable archive. An unsettled admitted restore blocks deletion. The small ownership tombstone remains; Core owns retention and cleanup order. After a lost response Core uses `ObserveOnly`. It never starts a capture or restore. For an existing verified capture it may complete archive publication and exact paused-source cleanup, but never captures again or kills a running source. Source termination is recorded before local removal so a lost cleanup response can be settled from the archive receipt. A valid ownership receipt remains discoverable when archive bytes are missing or corrupt: observation returns its snapshot identity with unknown source state and `SourceStopped=false`, so ordinary explicit cleanup can proceed. Restore still requires complete archive verification. An interrupted restore may finish the derived proof on the exact running target, but never restarts a stopped target or restores again. @@ -68,3 +68,7 @@ The helper returns only the native observation time, exact uptime, cumulative vC ## Tests `make check-microsandbox-provider`, part of `make check`, runs the pure-Go adapter tests everywhere and this module's tests on Linux; other hosts print an explicit skip for the Linux-only module. + +The native helper wire uses version 5 and carries the Session-selected Harness into Runtime bootstrap version 2. Upgrade the helper with its node and Core. The Go adapter wraps native snapshot identity in the shared opaque retained-state handle and completes exact-source cleanup through the existing ownership-checked helper operations. Reconciliation never repeats snapshot capture. SuspendSettled is reported only after these serialized operations complete; captured-artifact cleanup does not require execution resource qualification. + +Capture admission uses one adapter-private `capture-admission.json` per allocation, with the exact source and at most 64 operation entries. Before any pause/capture, a fresh call persists open admission. A no-archive observation closes only an operation with no admission; an open admission without an artifact stays unknown. Closure is durable before rollback evidence is returned and cannot be overwritten by a late fresh call. The source generation acts as a monotonic fence: only an exact owned newer source can replace the bounded journal; same-generation rollback cannot clear it. Existing verified archives and source-settlement receipts retain their original recovery path even after the source is stopped. diff --git a/services/core/tools/microsandbox-provider/archive.go b/services/core/tools/microsandbox-provider/archive.go index 608a5bdf7..2eba80326 100644 --- a/services/core/tools/microsandbox-provider/archive.go +++ b/services/core/tools/microsandbox-provider/archive.go @@ -492,3 +492,47 @@ func (b backend) pinRestoreCleanup(target *wire.Compute) error { } return nil } + +// One bounded journal fences capture admission across process restarts. Source +// generations only advance after suspend verified the new exact native source. +// Keeping that high-water identity rejects delayed requests from older writers. +type captureAdmission struct { + Source wire.Compute + Operations map[string]bool // true means durably closed without dispatch +} + +func (b backend) admitSuspend(q wire.SuspendRequest) (bool, error) { + path := filepath.Join(b.storeDirectory(), "capture-admission.json") + var prior captureAdmission + err := readPrivateJSON(path, &prior) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return false, err + } + if err == nil { + if prior.Source.ID == "" || prior.Operations == nil || len(prior.Operations) > 64 { + return false, sandbox.ErrOwnership + } + if !reflect.DeepEqual(prior.Source, q.Source) { + if q.Source.Generation <= prior.Source.Generation { + return false, sandbox.ErrOwnership + } + prior = captureAdmission{Source: q.Source, Operations: map[string]bool{}} + } + } else { + prior = captureAdmission{Source: q.Source, Operations: map[string]bool{}} + } + if closed, exists := prior.Operations[q.OperationID]; exists { + if !closed && !q.ObserveOnly { + return false, wire.ErrUnconfirmed + } + return closed, nil + } + if len(prior.Operations) >= 64 { + return false, wire.ErrUnconfirmed + } + prior.Operations[q.OperationID] = q.ObserveOnly + if err := durableJSON(path, prior); err != nil { + return false, err + } + return q.ObserveOnly, nil +} diff --git a/services/core/tools/microsandbox-provider/bootstrap.go b/services/core/tools/microsandbox-provider/bootstrap.go index 79125a12d..d7911a82c 100644 --- a/services/core/tools/microsandbox-provider/bootstrap.go +++ b/services/core/tools/microsandbox-provider/bootstrap.go @@ -8,6 +8,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentnetwork" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" wire "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" sdk "github.com/superradcompany/microsandbox/sdk/go" @@ -18,7 +19,7 @@ import ( const bootstrapScript = ` import ctypes,json,os,stat,subprocess,sys b=json.load(sys.stdin) -for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages','/run/oac']: +for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages',os.path.dirname(os.environ['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'])]: os.makedirs(p,mode=0o700,exist_ok=True) if not stat.S_ISDIR(os.lstat(p).st_mode): raise RuntimeError('invalid bootstrap directory') os.chmod(p,0o700);os.chown(p,1000,1000) @@ -71,7 +72,7 @@ func (b backend) create(ctx context.Context) (wire.Response, error) { "HOME": "/home/runtime", "OAC_RUNTIME_HOME": "/home/runtime/.oac", "OAC_RUNTIME_ENVIRONMENT_ID": bootstrap.EnvironmentID, "OAC_RUNTIME_SESSION_ID": bootstrap.SessionID, "OAC_RUNTIME_NETWORK_ACCESS": policy.Access, "OAC_RUNTIME_ALLOWED_DOMAINS": string(domains), - "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": "/run/oac/daemon-suspend.json", + "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": runtimebootstrap.SuspendControlFile, })) if e != nil { return wire.Response{}, e diff --git a/services/core/tools/microsandbox-provider/snapshot.go b/services/core/tools/microsandbox-provider/snapshot.go index 01475786b..36ee2e4e4 100644 --- a/services/core/tools/microsandbox-provider/snapshot.go +++ b/services/core/tools/microsandbox-provider/snapshot.go @@ -84,7 +84,27 @@ func (b backend) suspend(ctx context.Context, q wire.SuspendRequest) (wire.State // A surviving completed artifact is observed, never overwritten or recaptured. artifact, snap, e := b.inspectSnapshot(ctx, q.OperationID, q.Source) if sdk.IsKind(e, sdk.ErrSnapshotNotFound) { + // Native absence cannot settle an admitted capture. Only a durable + // never-dispatched closure permits rollback of this exact source. + // Existing archives/receipts above retain their source-cleanup path. + _, sourceState, inspectErr := b.inspectOwned(ctx, q.Source) + if inspectErr != nil { + return wire.State{}, inspectErr + } + if !sourceState.BootstrapComplete || (sourceState.Status != "running" && sourceState.Status != "paused") { + return wire.State{}, wire.ErrUnconfirmed + } + closed, admissionErr := b.admitSuspend(q) + if admissionErr != nil { + return wire.State{}, admissionErr + } + if !q.ObserveOnly && closed { + return wire.State{}, wire.ErrUnconfirmed + } if q.ObserveOnly { + if !closed { + return wire.State{}, wire.ErrUnconfirmed + } if q.Snapshot != nil { return wire.State{}, wire.ErrUnconfirmed } diff --git a/services/core/tools/microsandbox-provider/snapshot_observation_test.go b/services/core/tools/microsandbox-provider/snapshot_observation_test.go index 74810cc67..7ebd62a6d 100644 --- a/services/core/tools/microsandbox-provider/snapshot_observation_test.go +++ b/services/core/tools/microsandbox-provider/snapshot_observation_test.go @@ -511,3 +511,79 @@ func TestFreshAdmissionClosesOnlyAfterRequestDeadline(t *testing.T) { t.Fatal("expired dispatch not fenced", r, err) } } + +func TestCaptureAdmissionClosesNeverDispatchedOperationDurably(t *testing.T) { + request := initialInfoRequest(t) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + b := backend{q: request} + source := request.Compute + source.ID = "local:exact" + q := wire.SuspendRequest{OperationID: "66666666-6666-4666-8666-666666666666", Source: source, ObserveOnly: true} + if closed, err := b.admitSuspend(q); err != nil || !closed { + t.Fatal(closed, err) + } + guard.Close() + guard, err = allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + q.ObserveOnly = false + if closed, err := b.admitSuspend(q); err != nil || !closed { + t.Fatal("late fresh capture crossed durable closure", closed, err) + } + q.OperationID = "77777777-7777-4777-8777-777777777777" + if closed, err := b.admitSuspend(q); err != nil || closed { + t.Fatal(closed, err) + } + // A crash after open admission remains unknown even without a visible archive. + q.ObserveOnly = true + if closed, err := b.admitSuspend(q); err != nil || closed { + t.Fatal("unknown capture manufactured no-effect proof", closed, err) + } + q.ObserveOnly = false + if _, err := b.admitSuspend(q); !errors.Is(err, wire.ErrUnconfirmed) { + t.Fatal("admitted capture replayed", err) + } +} + +func TestCaptureAdmissionBoundAndSourceGenerationFence(t *testing.T) { + request := initialInfoRequest(t) + guard, err := allocationLock(request) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + b := backend{q: request} + source := request.Compute + source.ID = "local:exact" + q := wire.SuspendRequest{Source: source, ObserveOnly: true} + for i := range 64 { + q.OperationID = fmt.Sprintf("%08x-6666-4666-8666-666666666666", i) + if closed, err := b.admitSuspend(q); err != nil || !closed { + t.Fatal(i, closed, err) + } + } + q.OperationID = "ffffffff-6666-4666-8666-666666666666" + if _, err := b.admitSuspend(q); !errors.Is(err, wire.ErrUnconfirmed) { + t.Fatal("capture journal exceeded its bound", err) + } + old := q + q.Source.Generation++ + q.Source.Name = wire.Name(request.Config, request.Reference, q.Source.Generation) + q.Source.ID = "local:next" + if closed, err := b.admitSuspend(q); err != nil || !closed { + t.Fatal("verified next source could not advance journal", closed, err) + } + old.ObserveOnly = false + if _, err := b.admitSuspend(old); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal("late older source admitted", err) + } + var journal captureAdmission + if err := readPrivateJSON(filepath.Join(b.storeDirectory(), "capture-admission.json"), &journal); err != nil || len(journal.Operations) != 1 || journal.Source.Generation != q.Source.Generation { + t.Fatal("generation journal not bounded", journal, err) + } +}