From 4d9f26308ab8efd845279ca557742fcc7c23963d Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 10 Oct 2026 23:14:49 +0800 Subject: [PATCH 1/4] perf(runtime): reduce managed Session startup waits --- apps/daemon/internal/cli/connect.go | 42 ++- apps/daemon/internal/cli/connect_bootstrap.go | 3 +- .../internal/cli/connect_bootstrap_test.go | 12 +- .../internal/cli/connect_environment.go | 7 +- apps/daemon/internal/cli/connect_startup.go | 50 ++++ .../internal/cli/connect_startup_test.go | 266 ++++++++++++++++++ .../internal/cli/native_discovery_test.go | 20 ++ .../agents-api/node-generation-protocol.md | 4 +- .../agents-api/zh/node-generation-protocol.md | 6 +- docs/getting-started/operations.md | 7 + docs/runtime-bootstrap.md | 3 + docs/sandbox-provider.md | 4 + docs/zh/getting-started/operations.md | 9 +- docs/zh/runtime-bootstrap.md | 5 +- docs/zh/sandbox-provider.md | 6 +- internal/runtimebootstrap/bootstrap.go | 22 +- internal/runtimebootstrap/bootstrap_test.go | 11 +- .../cmd/server/managed_generations_test.go | 4 +- .../server/managed_setup_preflight_test.go | 10 +- .../deploy/e2b/helper_contract_generated.py | 2 +- services/core/deploy/e2b/managed_init_test.py | 2 +- .../execution/environment_admission.go | 2 +- .../execution/runtime_fresh_hint_test.go | 208 ++++++++++++++ .../internal/execution/runtime_lifecycle.go | 6 +- .../internal/execution/runtime_wake_hint.go | 33 ++- .../execution/runtime_workspace_test.go | 4 + services/core/internal/execution/worker.go | 10 +- .../execution/worker_capability_test.go | 63 +++++ .../internal/execution/worker_schedule.go | 5 +- .../runtimegateway/capability_hint_test.go | 138 +++++++++ .../core/internal/runtimegateway/registry.go | 38 ++- .../core/internal/runtimegateway/session.go | 25 +- .../internal/sandbox/docker/bootstrap_test.go | 2 +- .../internal/sandbox/docker/contract_test.go | 2 +- .../sandbox/docker/deployment_test.go | 2 +- .../internal/sandbox/docker/provider_test.go | 2 +- .../internal/sandbox/docker/recovery_test.go | 2 +- .../internal/sandbox/e2b/contract_test.go | 4 +- .../internal/sandbox/e2b/helper_contract.go | 2 +- .../sandbox/e2b/helper_contract_test.go | 1 + .../sandbox/e2b/installed_paths_test.go | 2 +- .../sandbox/e2b/internal/contractgen/main.go | 7 +- .../core/internal/sandbox/e2b/process_test.go | 8 +- .../internal/sandbox/e2b/provider_test.go | 4 +- .../sandbox/microsandbox/contract_test.go | 23 +- .../microsandbox/creation_settlement_test.go | 2 +- .../internal/sandbox/microsandbox/types.go | 2 +- .../sandbox/microsandbox/workspace_test.go | 4 +- .../internal/sandbox/node/connection_test.go | 2 +- .../sandbox/node/creation_settlement_test.go | 2 +- .../internal/sandbox/node/docker_live_test.go | 2 +- .../node/generation_connection_test.go | 2 +- .../internal/sandbox/node/generation_json.go | 8 +- .../sandbox/node/generation_json_test.go | 20 ++ .../core/internal/sandbox/node/node_test.go | 5 +- .../internal/sandbox/node/recovery_test.go | 2 +- .../internal/sandbox/node/timeout_test.go | 5 +- services/core/internal/sandbox/node/wire.go | 5 +- .../internal/sandbox/node/workspace_test.go | 23 +- .../internal/sandbox/runtime_bootstrap.go | 2 +- .../core/internal/sandbox/sandbox_provider.go | 6 +- .../integration/runtime_lifecycle_test.go | 31 ++ services/core/tools/e2b-provider/README.md | 6 +- .../e2b-provider/helper_contract_generated.py | 2 +- .../tools/e2b-provider/observation_test.py | 2 +- services/core/tools/e2b-provider/provider.py | 57 +++- .../core/tools/e2b-provider/provider_test.py | 152 +++++++++- .../core/tools/e2b-provider/state_test.py | 2 +- .../tools/e2b-provider/testdata/contract.json | 124 ++++---- .../tools/microsandbox-provider/README.md | 4 +- 70 files changed, 1355 insertions(+), 205 deletions(-) create mode 100644 apps/daemon/internal/cli/connect_startup.go create mode 100644 apps/daemon/internal/cli/connect_startup_test.go create mode 100644 services/core/internal/execution/runtime_fresh_hint_test.go create mode 100644 services/core/internal/execution/worker_capability_test.go create mode 100644 services/core/internal/runtimegateway/capability_hint_test.go diff --git a/apps/daemon/internal/cli/connect.go b/apps/daemon/internal/cli/connect.go index dc87bb4a9..0ec595e66 100644 --- a/apps/daemon/internal/cli/connect.go +++ b/apps/daemon/internal/cli/connect.go @@ -70,7 +70,7 @@ func runConnect(ctx *runContext, args []string) error { if *remote != "" || *environment != "" || *credentialFile != "" || fs.NArg() != 0 { return errors.New("connect: bootstrap input cannot be combined with enrollment options") } - bootstrapped, err = bootstrapProfile(*bootstrapFile) + bootstrapped, err = bootstrapProfile(*bootstrapFile, ctx) if err != nil { return err } @@ -98,13 +98,6 @@ func runConnect(ctx *runContext, args []string) error { return spawnBackground(context.Background(), ctx, *profile, os.Args) } - // Self-check before loading credentials so a machine with no - // supported agent CLI fails fast. - agentCLIs, err := preflightAgentCLIs(context.Background(), ctx, *profile) - if err != nil { - return err - } - var prof auth.Profile if bootstrapped != nil { prof = *bootstrapped @@ -115,7 +108,7 @@ func runConnect(ctx *runContext, args []string) error { } } - return mainLoop(ctx, *profile, prof, agentCLIs) + return mainLoop(ctx, *profile, prof) } // spawnBackground forks the daemon into the background. Parent @@ -186,11 +179,16 @@ func spawnBackground(ctx context.Context, rc *runContext, profile string, argv [ // background process. SIGINT / SIGTERM cancels the root context, which // unblocks the read pump and any in-flight Send so the daemon exits // without orphaning agent subprocesses. -func mainLoop(rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery) error { - return mainLoopRemote(context.Background(), rc, profile, prof, agentCLIs, "") +func mainLoop(rc *runContext, profile string, prof auth.Profile) error { + return mainLoopRemote(context.Background(), rc, profile, prof, "") } -func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery, remote string) error { +func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof auth.Profile, remote string) error { + return mainLoopRemoteWithDiscovery(parent, rc, profile, prof, remote, func(ctx context.Context) (agentCLIDiscovery, error) { + return preflightAgentCLIs(ctx, rc, profile) + }) +} +func mainLoopRemoteWithDiscovery(parent context.Context, rc *runContext, profile string, prof auth.Profile, remote string, discover func(context.Context) (agentCLIDiscovery, error)) error { // Route through obs/log so daemon log lines pick up the same // trace_id / span_id auto-injection as the server side — when the // daemon adopts an envelope's trace, every log call under that ctx @@ -205,18 +203,18 @@ func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof rootCtx, cancel := daemonize.NotifyContext(parent) defer cancel() - bootCtx, bootCancel := context.WithTimeout(rootCtx, bootstrapTimeout) - var boot *transport.BootstrapResponse - var err error - if remote == "" { - boot, err = transport.Bootstrap(bootCtx, prof.ServerURL, prof.RuntimeID, prof.RunnerCredential, Version) - } else { - boot, err = environmentBootstrap(bootCtx, prof, remote) - } - bootCancel() + boot, agentCLIs, err := prepareConnection(rootCtx, discover, func(ctx context.Context) (*transport.BootstrapResponse, error) { + bootCtx, stop := context.WithTimeout(ctx, bootstrapTimeout) + defer stop() + if remote != "" { + return environmentBootstrap(bootCtx, prof, remote) + } + return transport.Bootstrap(bootCtx, prof.ServerURL, prof.RuntimeID, prof.RunnerCredential, Version) + }) if err != nil { - return fmt.Errorf("connect: bootstrap: %w", err) + return err } + wsURL, err := transport.DeriveWSURL(*boot, prof.ServerURL) if err != nil { return fmt.Errorf("connect: derive ws url: %w", err) diff --git a/apps/daemon/internal/cli/connect_bootstrap.go b/apps/daemon/internal/cli/connect_bootstrap.go index 1e7848808..27705a1c8 100644 --- a/apps/daemon/internal/cli/connect_bootstrap.go +++ b/apps/daemon/internal/cli/connect_bootstrap.go @@ -9,7 +9,7 @@ import ( // The launch file is the sole credential source for this connection. Reopening // it on process restart never reads or overwrites an auth profile. -func bootstrapProfile(path string) (*auth.Profile, error) { +func bootstrapProfile(path string, rc *runContext) (*auth.Profile, error) { raw, err := runtimefs.ReadPrivatePath(path, runtimebootstrap.MaxBytes) if err != nil { return nil, errors.New("connect: Runtime bootstrap file unavailable") @@ -18,5 +18,6 @@ func bootstrapProfile(path string) (*auth.Profile, error) { if err != nil { return nil, err } + rc.installedKinds = map[string]bool{input.Harness: true} return &auth.Profile{ServerURL: input.CoreURL, RuntimeID: input.DeviceID, RunnerCredential: input.Credential}, nil } diff --git a/apps/daemon/internal/cli/connect_bootstrap_test.go b/apps/daemon/internal/cli/connect_bootstrap_test.go index 7f510d699..fbfb704ab 100644 --- a/apps/daemon/internal/cli/connect_bootstrap_test.go +++ b/apps/daemon/internal/cli/connect_bootstrap_test.go @@ -31,7 +31,7 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { if err = os.WriteFile(filepath.Join(profileDir, "auth.json"), priorRaw, 0600); err != nil { t.Fatal(err) } - input := runtimebootstrap.Connection{Version: runtimebootstrap.Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "bootstrap-secret"} + input := runtimebootstrap.Connection{Version: runtimebootstrap.Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "bootstrap-secret", Harness: "codex"} raw, err := input.Marshal() if err != nil { t.Fatal(err) @@ -41,7 +41,11 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { t.Fatal(err) } for range 2 { - p, err := bootstrapProfile(path) + rc := &runContext{} + p, err := bootstrapProfile(path, rc) + if !rc.installedKinds["codex"] || len(rc.installedKinds) != 1 { + t.Fatal("bootstrap did not scope discovery") + } if err != nil || p.ServerURL != input.CoreURL || p.RuntimeID != input.DeviceID || p.RunnerCredential != input.Credential { t.Fatal("failed bootstrap/restart", err) } @@ -53,13 +57,13 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { if err = os.WriteFile(path, []byte("bootstrap-secret"), 0600); err != nil { t.Fatal(err) } - if _, err = bootstrapProfile(path); err == nil || strings.Contains(err.Error(), input.Credential) { + if _, err = bootstrapProfile(path, &runContext{}); err == nil || strings.Contains(err.Error(), input.Credential) { t.Fatal("invalid input fell back or leaked") } if err = os.Remove(path); err != nil { t.Fatal(err) } - if _, err = bootstrapProfile(path); err == nil { + if _, err = bootstrapProfile(path, &runContext{}); err == nil { t.Fatal("missing input fell back to private auth") } } diff --git a/apps/daemon/internal/cli/connect_environment.go b/apps/daemon/internal/cli/connect_environment.go index 0b1c3b4b1..4e9628e9e 100644 --- a/apps/daemon/internal/cli/connect_environment.go +++ b/apps/daemon/internal/cli/connect_environment.go @@ -163,13 +163,8 @@ func runEnvironmentConnect(parent context.Context, rc *runContext, profile strin if background && !daemonize.IsBackgroundChild() { return spawnBackground(parent, rc, profile, os.Args) } - // Discovery consumes the immutable Runtime binding; it must follow enrollment. - discovery, err := preflightAgentCLIs(parent, rc, profile) - if err != nil { - return err - } prof := auth.Profile{ServerURL: base, RuntimeID: bound.DeviceID, RunnerCredential: credential} - return rejected(mainLoopRemote(parent, rc, profile, prof, discovery, remote)) + return rejected(mainLoopRemote(parent, rc, profile, prof, remote)) } var ( diff --git a/apps/daemon/internal/cli/connect_startup.go b/apps/daemon/internal/cli/connect_startup.go new file mode 100644 index 000000000..68258f297 --- /dev/null +++ b/apps/daemon/internal/cli/connect_startup.go @@ -0,0 +1,50 @@ +package cli + +import ( + "context" + "errors" + "fmt" + + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/transport" +) + +// prepareConnection overlaps independent preflight work after local credentials +// and enrollment are resolved. Both workers are joined before returning, including +// on failure. Their temporary context never owns the live connection or executor. +func prepareConnection(parent context.Context, discover func(context.Context) (agentCLIDiscovery, error), bootstrap func(context.Context) (*transport.BootstrapResponse, error)) (*transport.BootstrapResponse, agentCLIDiscovery, error) { + if err := parent.Err(); err != nil { + return nil, nil, err + } + ctx, cancel := context.WithCancel(parent) + defer cancel() + var agents agentCLIDiscovery + var boot *transport.BootstrapResponse + done := make(chan error, 2) + go func() { + var err error + agents, err = discover(ctx) + done <- err + }() + go func() { + var err error + boot, err = bootstrap(ctx) + if err != nil { + err = fmt.Errorf("connect: bootstrap: %w", err) + } + done <- err + }() + var result error + for range 2 { + if err := <-done; err != nil { + result = errors.Join(result, err) + cancel() + } + } + if result != nil { + return nil, nil, result + } + if err := parent.Err(); err != nil { + return nil, nil, err + } + return boot, agents, nil +} diff --git a/apps/daemon/internal/cli/connect_startup_test.go b/apps/daemon/internal/cli/connect_startup_test.go new file mode 100644 index 000000000..6b5bca1b6 --- /dev/null +++ b/apps/daemon/internal/cli/connect_startup_test.go @@ -0,0 +1,266 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent" + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/auth" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/transport" +) + +func awaitStartup(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatal("startup worker did not reach barrier") + } +} + +func TestConnectionPreflightOverlapsAndJoins(t *testing.T) { + for _, first := range []string{"bootstrap", "discovery"} { + t.Run(first, func(t *testing.T) { + discoveryStarted, bootstrapStarted := make(chan struct{}), make(chan struct{}) + releaseDiscovery, releaseBootstrap := make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + boot := &transport.BootstrapResponse{DeviceID: "device"} + go func() { + result, _, err := prepareConnection(t.Context(), func(context.Context) (agentCLIDiscovery, error) { + close(discoveryStarted) + <-releaseDiscovery + return agentCLIDiscovery{}, nil + }, func(context.Context) (*transport.BootstrapResponse, error) { + close(bootstrapStarted) + <-releaseBootstrap + return boot, nil + }) + if err == nil && result != boot { + err = errors.New("bootstrap result lost") + } + done <- err + }() + awaitStartup(t, discoveryStarted) + awaitStartup(t, bootstrapStarted) + if first == "bootstrap" { + close(releaseBootstrap) + } else { + close(releaseDiscovery) + } + select { + case <-done: + t.Fatal("returned before both prerequisites completed") + default: + } + if first == "bootstrap" { + close(releaseDiscovery) + } else { + close(releaseBootstrap) + } + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("join blocked") + } + }) + } +} + +func TestConnectionPreflightFailureCancelsAndJoinsSibling(t *testing.T) { + for _, failing := range []string{"bootstrap", "discovery"} { + t.Run(failing, func(t *testing.T) { + cause := errors.New("rejected") + started, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + wait := func(ctx context.Context) error { + close(started) + <-ctx.Done() + close(canceled) + <-release + return ctx.Err() + } + fail := func() error { <-started; return cause } + go func() { + boot, agents, err := prepareConnection(t.Context(), func(ctx context.Context) (agentCLIDiscovery, error) { + if failing == "discovery" { + return nil, fail() + } + return nil, wait(ctx) + }, func(ctx context.Context) (*transport.BootstrapResponse, error) { + if failing == "bootstrap" { + return nil, fail() + } + return nil, wait(ctx) + }) + if boot != nil || agents != nil { + err = errors.New("partial successful result escaped") + } + done <- err + }() + awaitStartup(t, canceled) + select { + case <-done: + t.Fatal("returned before sibling cleanup") + default: + } + close(release) + select { + case err := <-done: + if !errors.Is(err, cause) { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("join blocked") + } + }) + } +} + +func TestConnectionPreflightParentCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + started := make(chan struct{}, 2) + done := make(chan error, 1) + go func() { + _, _, err := prepareConnection(ctx, func(ctx context.Context) (agentCLIDiscovery, error) { + started <- struct{}{} + <-ctx.Done() + return nil, ctx.Err() + }, func(ctx context.Context) (*transport.BootstrapResponse, error) { + started <- struct{}{} + <-ctx.Done() + return nil, ctx.Err() + }) + done <- err + }() + awaitStartup(t, started) + awaitStartup(t, started) + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("cancellation blocked") + } + _, _, err := prepareConnection(ctx, func(context.Context) (agentCLIDiscovery, error) { + t.Error("discovery started after cancellation") + return nil, nil + }, func(context.Context) (*transport.BootstrapResponse, error) { + t.Error("bootstrap started after cancellation") + return nil, nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +// Controlled waits model independent work, not production latency. Compare the +// same two operations to demonstrate the removed dependency edge. +func BenchmarkConnectionPreflight(b *testing.B) { + for _, parallel := range []bool{false, true} { + name := "serial" + if parallel { + name = "overlap" + } + b.Run(name, func(b *testing.B) { + discover := func(context.Context) (agentCLIDiscovery, error) { + time.Sleep(5 * time.Millisecond) + return agentCLIDiscovery{}, nil + } + bootstrap := func(context.Context) (*transport.BootstrapResponse, error) { + time.Sleep(5 * time.Millisecond) + return &transport.BootstrapResponse{}, nil + } + for b.Loop() { + if parallel { + _, _, _ = prepareConnection(b.Context(), discover, bootstrap) + } else { + _, _ = discover(b.Context()) + _, _ = bootstrap(b.Context()) + } + } + }) + } +} + +func TestMainLoopOverlapsPreflightWithoutEarlyRegistration(t *testing.T) { + for _, failure := range []string{"", "discovery", "bootstrap"} { + t.Run(failure, func(t *testing.T) { + t.Setenv("OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE", "") + original := harnessDeclarations + defer func() { harnessDeclarations = original }() + started, release, completed := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var probes, boots, dials atomic.Int32 + declaration := original[0] + declaration.Discover = func(ctx context.Context, _ agent.DiscoveryOptions, info proto.SupportedAgentKind) *agent.Runtime { + probes.Add(1) + close(started) + select { + case <-release: + case <-ctx.Done(): + } + defer close(completed) + info.Available = failure != "discovery" && ctx.Err() == nil + return &agent.Runtime{Info: info} + } + harnessDeclarations = []agent.Declaration{declaration} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/agent-daemon/bootstrap": + boots.Add(1) + select { + case <-started: + case <-r.Context().Done(): + return + } + if dials.Load() != 0 { + t.Error("dial before discovery completed") + } + close(release) + if failure == "bootstrap" { + http.Error(w, "rejected", http.StatusUnauthorized) + return + } + _ = json.NewEncoder(w).Encode(transport.BootstrapResponse{DeviceID: "device"}) + case "/agent-daemon/ws": + select { + case <-completed: + default: + t.Error("registration before discovery joined") + } + dials.Add(1) + http.Error(w, "end test", http.StatusUnauthorized) + default: + t.Errorf("unexpected route %s", r.URL.Path) + http.NotFound(w, r) + } + })) + defer server.Close() + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + err := mainLoopRemote(ctx, &runContext{stdout: io.Discard, stderr: io.Discard}, "default", auth.Profile{ServerURL: server.URL, RuntimeID: "device", RunnerCredential: "fixture"}, "") + if err == nil || ctx.Err() != nil { + t.Fatalf("unexpected completion: %v, context %v", err, ctx.Err()) + } + expected := int32(0) + if failure == "" { + expected = 1 + } + if probes.Load() != 1 || boots.Load() != 1 || dials.Load() != expected { + t.Fatalf("calls discovery=%d bootstrap=%d dial=%d", probes.Load(), boots.Load(), dials.Load()) + } + }) + } +} diff --git a/apps/daemon/internal/cli/native_discovery_test.go b/apps/daemon/internal/cli/native_discovery_test.go index 5f3592c9d..29776bcde 100644 --- a/apps/daemon/internal/cli/native_discovery_test.go +++ b/apps/daemon/internal/cli/native_discovery_test.go @@ -73,3 +73,23 @@ func TestDiscoveryUnavailableAndCancelled(t *testing.T) { t.Fatal("unconfigured adapter retained") } } + +func TestSelectedHarnessUnavailableDoesNotProbeOthers(t *testing.T) { + declarations := append([]agent.Declaration(nil), harnessDeclarations...) + for i := range declarations { + declarations[i].Discover = func(_ context.Context, _ agent.DiscoveryOptions, info proto.SupportedAgentKind) *agent.Runtime { + if info.Kind != "codex" { + t.Fatalf("unselected Harness was probed: %s", info.Kind) + } + return &agent.Runtime{Info: info} + } + } + rc := &runContext{stdout: io.Discard, stderr: io.Discard, installedKinds: map[string]bool{"codex": true}} + if _, err := discoverAgentCLIs(t.Context(), rc, "default", declarations); err == nil { + t.Fatal("unavailable selection was accepted") + } + rc.installedKinds = map[string]bool{"unknown": true} + if _, err := discoverAgentCLIs(t.Context(), rc, "default", declarations); err == nil { + t.Fatal("unknown selection fell back") + } +} diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index de65b9069..5b7892cac 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -123,4 +123,6 @@ The readiness classes, one exported error and one code each, are authored in `se Preparation diagnostics keep fixed typed causes. Only artifact transfer, checksum or release-provenance failures report `runtime_download_failed`; the private preparer signals that class through its exit category, without Core or the node parsing stderr. Provider, ownership, cancellation and unclassified failures keep their typed code or `provider_unavailable`. No raw provider text crosses the protocol. -The current wire version is 5. Create bootstrap and Resume requests may carry an optional workspace filesystem binding. The node validates its tenant and Environment against the allocation reference, its immutable ObjectID, and its attachment configuration ID against the supplied configuration before forwarding it. The filesystem resolver validates adapter-native ownership. A missing binding selects owned storage; a present binding cannot fall back. Error responses may retain a Resume target only when its nonempty native ID, name, generation and snapshot provenance match the requested target; this is cleanup evidence, never successful restore. +The current wire version is 6. Create bootstrap and Resume requests may carry an optional workspace filesystem binding. The node validates its tenant and Environment against the allocation reference, its immutable ObjectID, and its attachment configuration ID against the supplied configuration before forwarding it. The filesystem resolver validates adapter-native ownership. A missing binding selects owned storage; a present binding cannot fall back. Error responses may retain a Resume target only when its nonempty native ID, name, generation and snapshot provenance match the requested target; this is cleanup evidence, never successful restore. + +Creation carries the Session-selected `Bootstrap.Harness` into Runtime bootstrap version 2. Core and nodes use protocol version 6; upgrade matching nodes before publishing this Core release. diff --git a/contracts/agents-api/zh/node-generation-protocol.md b/contracts/agents-api/zh/node-generation-protocol.md index 481dbc4af..b24278ee8 100644 --- a/contracts/agents-api/zh/node-generation-protocol.md +++ b/contracts/agents-api/zh/node-generation-protocol.md @@ -1,7 +1,7 @@ --- title: "沙箱节点协议" source: contracts/agents-api/node-generation-protocol.md -source_hash: 3fa1d9257acefa2d144f6e76ac5c2d3ff1d074ebb2b21fb48a2f79ef58ad6073 +source_hash: 961c7993fa9c75ddadb6604fece87fb556bbcc40d4f190d03d2e90927c5be7bb --- 沙箱节点在其主机上运行 Docker 或 microsandbox Provider，并通过一个 WebSocket 与 Core 相连。Core 通过该连接发送 Provider 操作；节点针对本地 Provider 执行这些操作，并报告就绪状态、主机测量值及其持有的部署代次。Core 始终是唯一的生命周期所有者：节点绝不重试变更操作或调度工作。帧和校验器位于 [`services/core/internal/sandbox/node`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/sandbox/node)（`wire.go`、`generation_wire.go`）；节点用于注册和读取配置的 HTTP 路由位于[机器连接 API](machine-api.md#node-routes)。 @@ -125,4 +125,6 @@ Runtime 字节缺失时，绝不将固定的放置实例迁移到当前 Runtime 准备诊断使用固定的类型化原因。只有制品传输、校验和或版本来源验证失败才会报告 `runtime_download_failed`；私有准备器通过退出类别指示这一类失败，Core 和节点都不解析 stderr。Provider 故障、所有权故障、取消和未分类故障保留其类型化代码，或使用 `provider_unavailable`。协议中不会传输任何 Provider 原始文本。 -当前线协议版本为 5。Create 引导和 Resume 请求可携带可选的工作区文件系统绑定。节点在转发前校验其租户及 Environment 与分配引用一致、ObjectID 不可变且有效，以及挂载配置 ID 与所传配置一致。文件系统解析器负责适配器原生所有权校验。未提供绑定时选择自有存储；已提供绑定时不得回退。错误响应仅可保留原生 ID 非空且名称、代次、快照来源均匹配请求的 Resume 目标；这仅为清理证据，不代表恢复成功。 +当前线协议版本为 6。Create 引导和 Resume 请求可携带可选的工作区文件系统绑定。节点在转发前校验其租户及 Environment 与分配引用一致、ObjectID 不可变且有效，以及挂载配置 ID 与所传配置一致。文件系统解析器负责适配器原生所有权校验。未提供绑定时选择自有存储；已提供绑定时不得回退。错误响应仅可保留原生 ID 非空且名称、代次、快照来源均匹配请求的 Resume 目标；这仅为清理证据，不代表恢复成功。 + +创建操作通过 `Bootstrap.Harness` 将会话选择传递到 Runtime 启动协议版本 2。Core 和节点使用协议版本 6；发布此 Core 版本前需升级配套节点。 diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index 55760e76b..0ea13768f 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -24,6 +24,9 @@ docker compose -f ~/.oac/core/compose.yaml ps The examples use the default installation directory. On Windows, invoke the management command with `& "$HOME/.oac/core/oac.exe"` followed by the same arguments. For a custom installation directory, replace the path in each command. +After local credentials and enrollment are resolved, Runtime Harness discovery and the authenticated bootstrap HTTP request run concurrently. Both must succeed before the Runtime opens its connection or publishes capabilities. Failure cancels the sibling operation and waits for cleanup. The executor remains owned by the connection lifetime. Runtime startup changes require a rebuilt, qualified Runtime template. + + ## Service health Use these observations for different questions: @@ -187,3 +190,7 @@ Installation and mutating `oac` commands share the [installation lock](../config Web signs administrators in with the Core key, checks the origin of every request, and forwards signed-in `/core/v1` requests to Core with the Core key, which stays on the server. It forwards `/v1` and `/api/v1` to Core unchanged, with the caller's own credential, serves only the non-secret node payload at `/node-install/`, and has no Docker or KVM access. Machine routes under `/api/v1` use their own enrollment and connection credentials. No service receives a Docker socket. Sandboxes are the isolation boundary ([Runtime and outer isolation](../concepts.md#runtime-and-outer-isolation)). Docker sandboxes share the node's kernel, and a Docker node is [root-equivalent](./nodes.md#what-the-installer-sets-up) on its host; microsandbox gives each sandbox a microVM with an explicit [network policy](./nodes.md#what-the-installer-sets-up). Core itself has no Docker socket or KVM access. + +## Runtime availability probes + +Codex CLI availability emits `runtime version probe` records for `process_spawn` and `process_wait`, with duration and outcome only. The wait interval includes executable loading and the version command; neither interval is model execution. diff --git a/docs/runtime-bootstrap.md b/docs/runtime-bootstrap.md index 7397e07fa..8284d9f51 100644 --- a/docs/runtime-bootstrap.md +++ b/docs/runtime-bootstrap.md @@ -17,6 +17,7 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json | `version` | The exact bootstrap version, `runtimebootstrap.Version` | | `core_url` | HTTP(S) machine API base ending in `/api/v1`, without credentials, query or fragment | | `device_id` | Canonical nonzero UUID of the daemon identity Core issued | +| `harness` | The immutable Session Harness identifier; only this Harness is probed and registered by a managed Runtime | | `credential` | Nonempty daemon credential Core issued, without whitespace or NUL | The decoder rejects unknown, duplicate, missing and case-aliased fields, other versions and documents larger than `runtimebootstrap.MaxBytes` (16 KiB). Errors never include submitted values. A missing or malformed file fails before the daemon connects. @@ -27,6 +28,8 @@ The file is the only authentication input for this launch: the daemon refuses to The provider creates the account, mounts and workspace, delivers this file, sets the Runtime's resource and Environment binding settings, and starts the daemon as the unprivileged Runtime account. Docker writes the file into the Runtime's owned home volume; microsandbox and E2B deliver it before launching the same command. +Core derives `harness` from the Session that owns the allocation. A combined image can contain several Harnesses, but its managed Runtime discovers only this selection; an unknown, missing or unavailable selection fails without probing another Harness. The bootstrap version is 2. Upgrade Core, its provider helpers and newly launched Runtime images together; retained allocations keep their existing bootstrap and Runtime. Self-hosted installations continue to discover their installed Harness set. + The Runtime validates the input and owns authentication and connection. A successful launch proves only the handoff: an authenticated connection, prepared capabilities and execution readiness are separate observations under the [Core–Runtime protocol](./runtime-protocol.md), and the [Sandbox Provider guide](./sandbox-provider.md#four-distinct-readiness-facts) lists what each one proves. Self-hosted executors and operator-provisioned devices get their daemon identity in other ways; the [machine connection API](../contracts/agents-api/machine-api.md#credentials) lists every credential source. All of them enter the same Runtime execution loop. diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 0eaa49ff4..149b7c352 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -160,6 +160,8 @@ Node readiness binds to the exact generation, the current connection and the own ### Allocation lifecycle +Core includes the owning Session’s immutable Harness in `Bootstrap.Harness`; every provider projects it into the [Runtime bootstrap](./runtime-bootstrap.md) without choosing an implementation. + The allocation, its dedicated daemon credential digest and the exact Session binding commit atomically before `Create`, under the execution lease and the Session lock. Only a fresh allocation receipt permits `Create`; retries and a Core restart observe the same reference without replaying it or rotating the credential. An allocation is private compute ownership, separate from public Environment connection and native readiness; adapters qualify bootstrap completion, and Core never infers it from an engine or provider name. With a configured provider, the Worker scans committed pending hosted Environments that have no allocation, which covers idle Session creation and recovery after an interruption between commit and bootstrap; an existing allocation never re-enters that path. The scan is bounded and serialized by the lifecycle owner and needs no caller action. An initial reservation without a Turn leaves its Session idle, and a daemon connection is never treated as native readiness. The same scan publishes authenticated connection observations with durable generations, after verifying the exact Session and device binding and a settled bootstrap. @@ -172,6 +174,8 @@ Terminal cleanup atomically revokes the device's authority, records the Environm Each registered node has one serial lifecycle worker that owns its gate, allocation and pending cursors, connections and wake hints; E2B allocations share one serial lifecycle without a node. A thin coordinator discovers nodes and shuts workers down, and never holds its map mutex during database, provider or wait operations. Workers advance independently, so a stuck provider on one online node never stalls another: lifecycle concurrency is one operation per node and grows with the node count. Offline workers stay, so their retained resources remain observable after reconnection. +After a hosted Environment commits, Core sends a bounded hint to its placement’s lifecycle worker. A committed pending input also sends a hint, recovering a missed creation notification. Hints use the existing serial gate, lease, capacity checks and one-shot allocation receipts; they do not provision inline in the admission handler. Each normal maintenance period permits at most one extra hinted scan, and periodic scans recover missed or coalesced hints. Existing allocation observations still precede pending provisioning, so a slow observation on the same node can delay a fresh Environment. + Allocation scans filter by node before their 32-row page limit, and pending scans join the unreleased committed placement. Each node advances its own cursor, including past failed observations, and wraps once at the end. Direct provisioning resolves the tenant-scoped placement before entering that node's gate, and an existing allocation must agree with it; Core never chooses another node. Before releasing the execution lease, the coordinator stops accepting work and cancels and drains every node worker and direct caller. Lease loss affects everything; ordinary provider failures stay within their node. A planned deployment drain or node retirement cancels lifecycle contexts synchronously between leased operations, through the lease gate with its five-second bound, including an active manual reconcile, and never cancels an in-flight leased query just to change configuration. A failed cancellation fence closes manager admission and reports owner failure. A failed retirement keeps the original lifecycle identity and gate until owner shutdown, and the drain barrier stays closed. Session locks, deployment capacity transactions and revision-checked receipts stay authoritative, and no external operation holds a database lock. diff --git a/docs/zh/getting-started/operations.md b/docs/zh/getting-started/operations.md index e1b4d1112..cfe626664 100644 --- a/docs/zh/getting-started/operations.md +++ b/docs/zh/getting-started/operations.md @@ -1,7 +1,7 @@ --- title: "运维" source: docs/getting-started/operations.md -source_hash: 97f5d49b9b39441a78026433d662120d46471ef5f9a0b6b4508998a3b4491d09 +source_hash: f6ce9d76fdd074d9e7e7da4a69da6a4583a67848b86aa641ad09f127fcae346b --- 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。 @@ -26,6 +26,9 @@ docker compose -f ~/.oac/core/compose.yaml ps 示例使用默认安装目录。Windows 上使用 `& "$HOME/.oac/core/oac.exe"` 调用管理命令，后接相同参数。使用自定义安装目录时，替换各命令中的路径。 +本地凭据和注册解析完成后，Runtime 的 Harness 发现与已认证的引导 HTTP 请求并发执行。两者都成功后才建立连接并发布能力。失败会取消另一项操作并等待其清理。执行器仍由连接生命周期持有。Runtime 启动变更需要重新构建并验证 Runtime 模板。 + + ## 服务健康状态 {#service-health} 根据不同问题使用这些观察： @@ -190,3 +193,7 @@ cd && rm -rf ~/.oac/core Web 使用 Core 密钥让管理员登录，检查每个请求来源，并用保留在服务器上的 Core 密钥将已登录的 `/core/v1` 请求转发到 Core。它把 `/v1` 和 `/api/v1` 原样转发给 Core，使用调用方自己的凭据；Web 仅在 `/node-install/` 提供不含密钥的节点文件，没有 Docker 或 KVM 访问权限。`/api/v1` 机器路由使用独立注册和连接凭据。没有服务持有 Docker 套接字。 沙箱是隔离边界（[Runtime 与外层隔离](../concepts.md#runtime-and-outer-isolation)）。Docker 沙箱共享节点内核，Docker 节点在主机上[等同于 root 权限](nodes.md#what-the-installer-sets-up)；microsandbox 为每个沙箱提供具有显式[网络策略](nodes.md#what-the-installer-sets-up)的 microVM。Core 自身无 Docker 套接字或 KVM 访问权限。 + +## Runtime 可用性探测 {#runtime-availability-probes} + +Codex CLI 可用性探测为 `process_spawn` 和 `process_wait` 输出 `runtime version probe` 记录，只包含耗时和结果。等待区间包含可执行文件加载和版本命令，两者都不是模型执行。 diff --git a/docs/zh/runtime-bootstrap.md b/docs/zh/runtime-bootstrap.md index d3338fa61..5bd813902 100644 --- a/docs/zh/runtime-bootstrap.md +++ b/docs/zh/runtime-bootstrap.md @@ -1,7 +1,7 @@ --- title: "Runtime 引导" source: docs/runtime-bootstrap.md -source_hash: f7e9275feee79ac1fb1299227e148b85c26efc8b8af5ca0e8f498126c563d182 +source_hash: 73fb9968007f83d35f1e6fe185d9e376fa81fc7683fe822426b290632ca8de24 --- Sandbox Provider 通过交付一个引导文件来启动托管 Runtime。本文负责 Provider 到 Runtime 的启动输入。类型与验证器位于 [`internal/runtimebootstrap`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/runtimebootstrap/bootstrap.go)；Go provider 使用 [`runtime_bootstrap.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/runtime_bootstrap.go) 中的 `sandbox.Bootstrap.RuntimeConnection()` 构造输入，SDK helper 原样转发序列化对象。provider 不读取或写入 Runtime 的私有认证存储。 @@ -19,6 +19,7 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json | `version` | 精确的引导版本 `runtimebootstrap.Version` | | `core_url` | 以 `/api/v1` 结尾的 HTTP(S) 机器 API 基址，不含凭据、查询或片段 | | `device_id` | Core 签发的 daemon 身份的规范非零 UUID | +| `harness` | 会话固定选择的 Harness 标识；托管 Runtime 只探测并注册此 Harness | | `credential` | Core 签发的非空 daemon 凭据，不含空白或 NUL | 解码器拒绝未知、重复、缺失和大小写别名字段，拒绝其他版本及超过 `runtimebootstrap.MaxBytes`（16 KiB）的文档。错误不包含提交的值。文件缺失或格式错误时，daemon 在连接前失败。 @@ -29,6 +30,8 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json provider 创建账户、挂载和工作区，交付该文件，设置 Runtime 的资源与 Environment 绑定配置，然后以无特权 Runtime 账户启动 daemon。Docker 将文件写入 Runtime 拥有的 home volume；microsandbox 和 E2B 在启动同一命令之前交付文件。 +Core 从分配所属会话派生 `harness`。组合镜像可以包含多个 Harness，但托管 Runtime 只探测所选 Harness；未知、缺失或不可用的选择会失败，不探测其他 Harness。启动协议版本为 2，Core、提供商辅助程序和新启动的 Runtime 镜像需配套升级；已有分配保留原启动文件和 Runtime。自托管安装仍探测已安装的 Harness 集合。 + Runtime 验证输入，并负责认证与连接。启动成功仅证明交付完成：经过认证的连接、已准备的能力和执行就绪是 [Core–Runtime 协议](runtime-protocol.md) 下的独立观测；[Sandbox Provider 指南](sandbox-provider.md#four-distinct-readiness-facts) 列出各自证明的事实。 自托管 executor 和运维人员供应的设备通过其他方式获取 daemon 身份；[机器连接 API](../../contracts/agents-api/zh/machine-api.md#credentials) 列出所有凭据来源。它们都进入同一 Runtime 执行循环。 diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 67e1d5f25..90b262e43 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: ec6c807e262a37d2ecd456101dadf7ae4818f1bd711b4af5f08e27c9d1690d83 +source_hash: a69ba63ef2b5bd883e0a235807df9d270c23dab9b6df11cf6019534281ced8e9 --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -162,6 +162,8 @@ Node readiness 绑定到精确 generation、当前连接和 owner epoch。持久 ### Allocation 生命周期 {#allocation-lifecycle} +Core 在 `Bootstrap.Harness` 中传递所属会话固定选择的 Harness；所有提供商都将其投影到 [Runtime 启动输入](./runtime-bootstrap.md)，不自行选择实现。 + allocation、专用 daemon credential digest 和精确 Session binding 在 `Create` 前、execution lease 与 Session lock 下原子提交。只有新 allocation receipt 允许 `Create`；重试和 Core 重启观察同一 reference，不重放或轮换凭据。allocation 是私有计算资源所有权，与公开 Environment connection 和原生 readiness 独立；adapter 验证 bootstrap completion，Core 不从 engine 或 provider name 推断。 配置 provider 后，Worker 扫描已提交且没有 allocation 的 pending hosted Environment，涵盖空闲 Session 创建以及 commit 与 bootstrap 之间中断后的恢复；已有 allocation 不重新进入此路径。scan 有界，由 lifecycle owner 串行化，不需要调用方操作。没有 Turn 的初始预约让 Session 保持空闲，daemon 连接不被当作原生 readiness。同一 scan 在验证精确 Session、device binding 和已结算 bootstrap 后，发布带持久 generation 的认证连接观测。 @@ -174,6 +176,8 @@ Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 每个注册 node 有一个串行 lifecycle worker，负责 gate、allocation 与 pending cursor、connection 和 wake hint；E2B allocation 共享一个没有 node 的串行 lifecycle。薄 coordinator 发现 node 并关闭 worker，数据库、provider 或等待操作期间不持有 map mutex。worker 独立推进，因此一个在线 node 的 provider 卡住不会阻塞其他 node：lifecycle 并发为每 node 一项操作，随 node 数量增长。离线 worker 保留，因此其保留资源在重连后仍可观察。 +托管 Environment 提交后，Core 向其 placement 的生命周期 worker 发送有界提示。已提交的待处理输入也会发送提示，以恢复遗漏的创建通知。提示沿用串行 gate、租约、容量检查和一次性 allocation 收据，不在准入处理器内直接创建资源。每个正常维护周期最多允许一次额外提示扫描；周期扫描恢复遗漏或合并的提示。现有 allocation 观察仍先于待创建资源扫描，因此同一节点上缓慢的观察可能延迟新 Environment。 + allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan join 尚未释放的已提交 placement。每个 node 推进自己的 cursor，包括越过失败观测，并在末尾回绕一次。direct provisioning 在进入该 node gate 前解析 tenant 范围内 placement，已有 allocation 必须与其一致；Core 不选择另一 node。 释放 execution lease 前，coordinator 停止接受工作，取消并排空每个 node worker 和 direct caller。lease 丢失影响全部；普通 provider failure 限于所属 node。计划 deployment drain 或 node retirement 通过五秒有界 lease gate，在 leased operation 之间同步取消 lifecycle context，包括活动 manual reconcile，不仅为了改变配置就取消进行中的 leased query。失败的 cancellation fence 关闭 manager admission 并报告 owner failure。失败的 retirement 保留原 lifecycle identity 和 gate，直到 owner shutdown，drain barrier 保持关闭。Session lock、deployment capacity transaction 和 revision-checked receipt 仍是权威依据，外部操作不持有数据库锁。 diff --git a/internal/runtimebootstrap/bootstrap.go b/internal/runtimebootstrap/bootstrap.go index bab5e7b16..1beabb40f 100644 --- a/internal/runtimebootstrap/bootstrap.go +++ b/internal/runtimebootstrap/bootstrap.go @@ -14,7 +14,7 @@ import ( "github.com/google/uuid" ) -const Version = 1 +const Version = 2 const MaxBytes = 16 * 1024 var ErrInvalid = errors.New("invalid Runtime bootstrap input") @@ -27,12 +27,13 @@ type Connection struct { CoreURL string `json:"core_url"` DeviceID string `json:"device_id"` Credential string `json:"credential"` + Harness string `json:"harness"` } func (c Connection) Validate() error { u, err := url.Parse(c.CoreURL) id, idErr := uuid.Parse(c.DeviceID) - if c.Version != Version || err != nil || (u.Scheme != "https" && u.Scheme != "http") || + if c.Version != Version || !ValidHarness(c.Harness) || err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.RawPath != "" || u.Path != "/api/v1" || strings.ContainsAny(c.CoreURL, "?#") || strings.ContainsFunc(c.CoreURL, unicode.IsSpace) || @@ -83,6 +84,8 @@ func Decode(raw []byte) (Connection, error) { err = d.Decode(&c.DeviceID) case "credential": err = d.Decode(&c.Credential) + case "harness": + err = d.Decode(&c.Harness) default: return Connection{}, ErrInvalid } @@ -93,8 +96,21 @@ func Decode(raw []byte) (Connection, error) { if token, err = d.Token(); err != nil || token != json.Delim('}') { return Connection{}, ErrInvalid } - if len(seen) != 4 || d.Decode(new(any)) != io.EOF || c.Validate() != nil { + if len(seen) != 5 || d.Decode(new(any)) != io.EOF || c.Validate() != nil { return Connection{}, ErrInvalid } return c, nil } + +// ValidHarness checks the identifier syntax shared by startup projections. +func ValidHarness(kind string) bool { + if len(kind) == 0 || len(kind) > 64 || kind[0] < 'a' || kind[0] > 'z' { + return false + } + for _, c := range kind { + if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '_') { + return false + } + } + return true +} diff --git a/internal/runtimebootstrap/bootstrap_test.go b/internal/runtimebootstrap/bootstrap_test.go index 132e5659a..7fb194495 100644 --- a/internal/runtimebootstrap/bootstrap_test.go +++ b/internal/runtimebootstrap/bootstrap_test.go @@ -7,7 +7,7 @@ import ( ) func TestConnectionRoundTrip(t *testing.T) { - input := Connection{Version: Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-credential"} + input := Connection{Version: Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-credential", Harness: "codex"} raw, err := input.Marshal() if err != nil { t.Fatal(err) @@ -19,11 +19,11 @@ func TestConnectionRoundTrip(t *testing.T) { } func TestConnectionRejectsAmbiguousOrForeignInput(t *testing.T) { - good := `{"version":1,"core_url":"https://core.example/api/v1","device_id":"da912024-1543-4242-a2c1-5f4f7ebbc6c7","credential":"test-secret"}` + good := `{"version":2,"harness":"codex","core_url":"https://core.example/api/v1","device_id":"da912024-1543-4242-a2c1-5f4f7ebbc6c7","credential":"test-secret"}` for name, raw := range map[string]string{ "null": "null", "array": "[]", "trailing": good + "{}", - "duplicate": strings.Replace(good, `"version":1`, `"version":1,"version":1`, 1), - "unknown": strings.Replace(good, `"version":1`, `"extra":1,"version":1`, 1), + "duplicate": strings.Replace(good, `"version":2`, `"version":2,"version":2`, 1), + "unknown": strings.Replace(good, `"version":2`, `"extra":1,"version":2`, 1), "case": strings.Replace(good, "credential", "Credential", 1), "old auth": `{"server_url":"https://core.example","runtime_id":"old","runner_credential":"test-secret"}`, "oversized": strings.Repeat(" ", MaxBytes) + good, @@ -35,9 +35,10 @@ func TestConnectionRejectsAmbiguousOrForeignInput(t *testing.T) { }) } for field, values := range map[string][]any{ - "version": {nil, 0, 2, "1"}, + "version": {nil, 0, 1, 3, "2"}, "core_url": {"http://user:pass@core.example/api/v1", "https://core.example/api/v1?", "https://core.example/api/v1#", "https://core.example", "https://core.example/api/v1/", "https://core.example/api/v1?q=1", ""}, "device_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, + "harness": {nil, "", "Codex", "unknown-kind", "../codex", strings.Repeat("x", 65)}, "credential": {nil, "", "test\nsecret", "test\x00secret"}, } { for _, value := range values { diff --git a/services/core/cmd/server/managed_generations_test.go b/services/core/cmd/server/managed_generations_test.go index 7e5855f6b..4b1c57aac 100644 --- a/services/core/cmd/server/managed_generations_test.go +++ b/services/core/cmd/server/managed_generations_test.go @@ -29,7 +29,7 @@ q=json.load(sys.stdin) with (pathlib.Path(q['Config']['StateDir'])/'requests').open('a') as f: f.write(json.dumps(q)+'\n') info=dict(q['Reference'],State='running',ProviderID='owned',CreateSettled=True) if q['Operation']=='kill': info['State']='absent' -print(json.dumps({'Version':1,'Info':info})) +print(json.dumps({'Version':2,'Info':info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -132,7 +132,7 @@ code = '' if k == 'revoked': code = 'unauthorized' elif k == 'unconfirmed': code = 'unconfirmed' elif not (k.startswith('team-a') and template in ('owned-a', 'new-a') or k == 'team-b' and template == 'public-b'): code = 'team_mismatch' -result = {'Version': 1, 'ErrorCode': code} +result = {'Version': 2, 'ErrorCode': code} if not code: result['DeploymentValid'] = True if q['Operation'] == 'validate_deployment': diff --git a/services/core/cmd/server/managed_setup_preflight_test.go b/services/core/cmd/server/managed_setup_preflight_test.go index dc977e41a..884b89385 100644 --- a/services/core/cmd/server/managed_setup_preflight_test.go +++ b/services/core/cmd/server/managed_setup_preflight_test.go @@ -19,7 +19,7 @@ import ( func TestE2BRejectedSpecificationHasSafeActionableDiagnostic(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") @@ -49,12 +49,12 @@ op = q['Operation'] with (pathlib.Path(q['Config']['StateDir']) / 'operations').open('a') as log: log.write(op + '\n') if op == 'validate_deployment': - print(json.dumps({'Version': 1, 'ErrorCode': 'invalid'})) + print(json.dumps({'Version': 2, 'ErrorCode': 'invalid'})) else: info = dict(q['Reference'], ProviderID='owned-compute', State='stopped', CreateSettled=True) if op == 'kill': info.update(ProviderID='', State='absent') - print(json.dumps({'Version': 1, 'Info': info})) + print(json.dumps({'Version': 2, 'Info': info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -101,7 +101,7 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { } helper := filepath.Join(t.TempDir(), "provider") requests := filepath.Join(state, "requests") - script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" + script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -133,7 +133,7 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index 77d1869fd..0b7635c78 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -11,7 +11,7 @@ MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 1 +PROTOCOL_VERSION = 2 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] diff --git a/services/core/deploy/e2b/managed_init_test.py b/services/core/deploy/e2b/managed_init_test.py index d9017ba82..f38a92ffd 100644 --- a/services/core/deploy/e2b/managed_init_test.py +++ b/services/core/deploy/e2b/managed_init_test.py @@ -16,7 +16,7 @@ def payload(): value = {key: str(uuid4()) for key in ['InstallationID', 'TenantID', 'EnvironmentID', 'AllocationID', 'SessionID', 'DeviceID']} - return dict(value, RuntimeBootstrap={'version': 1, 'core_url': 'https://core.example/api/v1', + return dict(value, RuntimeBootstrap={'version': 2, 'harness': 'codex', 'core_url': 'https://core.example/api/v1', 'device_id': value['DeviceID'], 'credential': 'private-managed-token'}, NetworkAccess='restricted', AllowedDomains=['example.com']) diff --git a/services/core/internal/execution/environment_admission.go b/services/core/internal/execution/environment_admission.go index 0f34d9f47..18ec95ac5 100644 --- a/services/core/internal/execution/environment_admission.go +++ b/services/core/internal/execution/environment_admission.go @@ -99,7 +99,7 @@ func (w *Worker) submitEnvironmentInputs(ctx context.Context, session sessions.S return nil, err } w.wakeScheduler() - if reservation.State == sessions.EnvironmentInputPending && !reservation.IsInitial { + if reservation.State == sessions.EnvironmentInputPending { w.hintRuntimeWake(ctx, session) } ticker := time.NewTicker(250 * time.Millisecond) diff --git a/services/core/internal/execution/runtime_fresh_hint_test.go b/services/core/internal/execution/runtime_fresh_hint_test.go new file mode 100644 index 000000000..334052b4e --- /dev/null +++ b/services/core/internal/execution/runtime_fresh_hint_test.go @@ -0,0 +1,208 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + projectpg "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/projectpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/projects" + "sync" + "sync/atomic" + "testing" + "time" + + v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" + "github.com/google/uuid" +) + +type freshHintProvider struct { + sandbox.SandboxProvider + creates atomic.Int32 + created chan struct{} +} + +type hintSessionReader struct { + sessions.Reader + environment sessions.Environment +} + +func (r hintSessionReader) GetSessionEnvironment(ctx context.Context, _, _ string) (sessions.Environment, error) { + return r.environment, ctx.Err() +} + +func TestFreshHintRoutesAndPreservesLifecycleGuards(t *testing.T) { + for _, scenario := range []string{"direct", "node", "closed", "switching", "cancelled", "self_hosted", "released_placement", "lookup_failed", "suspended"} { + t.Run(scenario, func(t *testing.T) { + m := testRuntimeManager(t) + m.setupGate = make(chan struct{}, 1) + m.config = RuntimeProvider{InstallationID: uuid.NewString(), Mode: "direct", Provider: &freshHintProvider{}} + nodeID := "" + if scenario == "node" || scenario == "released_placement" { + m.config.ProviderKind = "docker" + m.config.Mode = "nodes" + nodeID = uuid.NewString() + } + node, err := m.node(nodeID) + if err != nil { + t.Fatal(err) + } + environment := sessions.Environment{ID: uuid.NewString(), Configuration: json.RawMessage(`{"type":"openai_hosted"}`)} + if scenario == "self_hosted" { + environment.Configuration = json.RawMessage(`{"type":"self_hosted","workspace_directory":"/workspace"}`) + } + if scenario == "suspended" { + environment.Initialization = "complete" + m.config.Suspension = &RuntimeSuspensionPolicy{} + } + reader := &strictDeploymentReader{t: t, + environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + if scenario == "lookup_failed" { + return deployment.Allocation{}, errors.New("database unavailable") + } + if scenario == "suspended" { + return deployment.Allocation{ProviderKey: m.config.InstallationID, State: "running", CreateSettled: true, ComputePhase: "suspended"}, nil + } + return deployment.Allocation{}, deployment.ErrNotFound + }, + lifecyclePlacement: func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) { + return deployment.LifecyclePlacement{Provider: m.config.ProviderKind, PlacementNodeID: nodeID, PlacementReleased: scenario == "released_placement"}, nil + }, + } + m.deploymentService, _ = deploymentOperations(t, &strictDeploymentStorage{t: t}, reader, &strictExecutionStorage{t: t}) + worker := &Worker{runtimes: m, dispatcher: &Dispatcher{SessionsReader: hintSessionReader{environment: environment}, DeploymentReader: reader}} + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + switch scenario { + case "closed": + m.closed = true + case "switching": + m.switching = true + case "cancelled": + cancel() + } + worker.hintRuntimeWake(ctx, sessions.Session{TenantID: uuid.NewString(), ID: uuid.NewString()}) + want := 0 + if scenario == "direct" || scenario == "node" || scenario == "suspended" { + want = 1 + } + if got := len(node.lifecycle.wakeHints); got != want { + t.Fatalf("hints = %d, want %d", got, want) + } + }) + } +} + +func (p *freshHintProvider) Create(_ context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { + p.creates.Add(1) + select { + case p.created <- struct{}{}: + default: + } + return sandbox.Info{Reference: b.Reference, ProviderID: b.AllocationID, State: "running", BootstrapComplete: true, CreateSettled: true}, nil +} + +func TestFreshEnvironmentHintProvisionsWithoutMaintenanceTick(t *testing.T) { + for _, mode := range []string{"create", "recovered_input", "recovered_initial"} { + t.Run(mode, func(t *testing.T) { + owner, deployments, reader, pool := resetManagerDB(t, nil) + installation := initializeE2BDeployment(t, owner) + sessionReader, s := testSessions(t, pool, pgtest.CredentialKey(t)) + provider := &freshHintProvider{created: make(chan struct{}, 1)} + m := testRuntimeManager(t) + m.setupGate = make(chan struct{}, 1) + m.sessions, m.sessionExecution = sessionReader, owner.Sessions + m.deployment, m.deploymentService, m.deploymentReader = owner.Deployment, deployments, reader + m.lease, m.registry = owner.Lease, runtimegateway.NewRegistry() + m.config = RuntimeProvider{Mode: "direct", InstallationID: installation, CoreURL: fixturePublicURL + "/api/v1", Provider: provider} + worker := &Worker{lease: owner.Lease, runtimes: m, dispatcher: &Dispatcher{Sessions: s, sessionExecution: owner.Sessions, SessionsReader: sessionReader, DeploymentReader: reader, notifications: &executionNotifications{}}, scheduleWake: make(chan struct{}, 1)} + node, err := m.node("") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + done, scans := make(chan error, 1), make(chan struct{}, 4) + // No value is ever sent to ticks: provisioning must come from a hint. + ticks := make(chan time.Time) + go func() { + done <- runRuntimeMaintenance(ctx, ticks, node.lifecycle.wakeHints, func(ctx context.Context) error { + err := node.lifecycle.reconcile(ctx) + scans <- struct{}{} + return err + }) + }() + t.Cleanup(func() { cancel(); <-done }) + <-scans // Startup scan finishes before any Session is committed. + projectID := uuid.NewString() + audit := adminaudit.WithSource(ctx, adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) + management, err := projects.NewService(projectpg.New(pgunit.NewPool(pool))) + if err != nil { + t.Fatal(err) + } + project, err := management.CreateProject(audit, projects.CreateProject{ID: projectID, Name: "Fresh hint"}) + if err != nil { + t.Fatal(err) + } + tenant := project.TenantID + input := sessions.CreateSession{Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted"}}`), ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ExecutionSourceSession} + messages := []sessions.Input{{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"fixture"}]}]}`)}} + if mode == "recovered_initial" { + input.InitialInputs = messages + } + var creation sessions.Creation + switch mode { + case "create": + creation, err = worker.CreateSession(ctx, tenant, input) + case "recovered_input", "recovered_initial": + creation, err = s.CreateSession(ctx, tenant, input) + } + if err != nil { + t.Fatal(err) + } + session := creation.Session + if mode == "recovered_input" || mode == "recovered_initial" { + key := uuid.NewString() + if mode == "recovered_initial" { + if err := pool.QueryRow(ctx, "SELECT idempotency_key FROM environment_input_reservations WHERE session_id=$1 AND is_initial", session.ID).Scan(&key); err != nil { + t.Fatal(err) + } + } + inputDone := make(chan error, 20) + inputCtx, stopInput := context.WithCancel(ctx) + for range 20 { + go func() { + _, inputErr := worker.submitEnvironmentInputs(inputCtx, session, key, messages) + inputDone <- inputErr + }() + } + t.Cleanup(func() { + stopInput() + for range 20 { + <-inputDone + } + }) + } + select { + case <-provider.created: + case <-time.After(2 * time.Second): + t.Fatal("committed Environment waited for a maintenance tick") + } + var requests sync.WaitGroup + for range 20 { + requests.Go(func() { worker.hintRuntimeWake(ctx, session) }) + } + requests.Wait() + <-scans // Wait for the hinted scan to settle its durable allocation. + if got := provider.creates.Load(); got != 1 { + t.Fatalf("concurrent hints issued %d Creates", got) + } + }) + } +} diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 49e23afd4..8b0eea006 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -202,6 +202,10 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p } } key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} + session, err := r.sessions.GetSession(ctx, tenant, environmentValue.SessionID) + if err != nil { + return deployment.Allocation{}, err + } secret := make([]byte, 32) if _, err := rand.Read(secret); err != nil { return deployment.Allocation{}, err @@ -222,7 +226,7 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p } info, err := provider.Create(ctx, sandbox.Bootstrap{ Reference: runtimeReference(owner), SessionID: owner.SessionID, DeviceID: owner.DeviceID, - Workspace: workspace, CoreURL: r.config.CoreURL, Credential: token, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, + Workspace: workspace, CoreURL: r.config.CoreURL, Credential: token, Harness: session.Engine, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, }) if info.Reference == runtimeReference(owner) && info.CreateSettled && info.State == "absent" { record, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) diff --git a/services/core/internal/execution/runtime_wake_hint.go b/services/core/internal/execution/runtime_wake_hint.go index ed7c9f1d6..b25fb4d03 100644 --- a/services/core/internal/execution/runtime_wake_hint.go +++ b/services/core/internal/execution/runtime_wake_hint.go @@ -2,14 +2,15 @@ package execution import ( "context" + "errors" "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -// A hint only accelerates observation of an already committed input. Lookup or -// delivery failure leaves that input for the normal maintenance scan. +// Hints only accelerate lifecycle work for committed Environments and inputs. +// Lookup or delivery failure leaves that work for the normal maintenance scan. func (w *Worker) hintRuntimeWake(ctx context.Context, session sessions.Session) { r := w.runtimes if r == nil { @@ -19,16 +20,40 @@ func (w *Worker) hintRuntimeWake(ctx context.Context, session sessions.Session) config := r.config available := !r.closed && !r.switching r.mu.Unlock() - if !available || config.Suspension == nil || r.ctx.Err() != nil { + if !available || r.ctx.Err() != nil { return } lookup, cancel := context.WithTimeout(ctx, time.Second) defer cancel() environment, err := w.dispatcher.SessionsReader.GetSessionEnvironment(lookup, session.TenantID, session.ID) - if err != nil || environment.Initialization != "complete" { + if err != nil { + return + } + placement, err := parseEnvironmentPlacement(environment.Configuration) + if err != nil || placement.Type != "openai_hosted" { return } owner, err := w.dispatcher.DeploymentReader.EnvironmentAllocation(lookup, deployment.AllocationKey{TenantID: session.TenantID, EnvironmentID: environment.ID}) + if errors.Is(err, deployment.ErrNotFound) { + // Placement exists before allocation. Route through its lifecycle owner; + // the scan still enforces lease, capacity and one-shot Create receipts. + nodeID, err := r.deploymentService.LifecycleNode(lookup, session.TenantID, environment.ID) + if err != nil || lookup.Err() != nil { + return + } + node, err := r.node(nodeID) + if err != nil { + return + } + select { + case node.lifecycle.wakeHints <- struct{}{}: + default: + } + return + } + if config.Suspension == nil || environment.Initialization != "complete" { + return + } if err != nil || owner.ProviderKey != config.InstallationID || owner.State != "running" || !owner.CreateSettled || owner.SessionDeleted || owner.Expired { return diff --git a/services/core/internal/execution/runtime_workspace_test.go b/services/core/internal/execution/runtime_workspace_test.go index 54ca2141c..8c52d590f 100644 --- a/services/core/internal/execution/runtime_workspace_test.go +++ b/services/core/internal/execution/runtime_workspace_test.go @@ -16,6 +16,10 @@ import ( type workspaceEnvironmentReader struct{ sessions.Reader } +func (workspaceEnvironmentReader) GetSession(context.Context, string, string) (sessions.Session, error) { + return sessions.Session{Engine: "codex"}, nil +} + func (workspaceEnvironmentReader) GetEnvironment(context.Context, string, string) (sessions.Environment, error) { return sessions.Environment{Configuration: []byte(`{"type":"openai_hosted"}`)}, nil } diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 92fabc153..6d67ff3cb 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -134,6 +134,9 @@ func (w *Worker) CreateSession(ctx context.Context, tenant string, input session return sessions.Creation{}, err } creation, err := w.dispatcher.Sessions.CreateSession(ctx, tenant, input) + if err == nil { + w.hintRuntimeWake(ctx, creation.Session) + } if err == nil && len(input.InitialInputs) > 0 { w.wakeScheduler() } @@ -264,6 +267,8 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { rescanOnCompletion = false case <-w.scheduleWake: rescanOnCompletion = true + case <-w.dispatcher.Registry.CapabilityHints(): + rescanOnCompletion = true case <-ticker.C: maintenance = true } @@ -293,10 +298,7 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { w.observeSchedulerPoll(0, nil) continue } - if !maintenance { - schedule.nextEnvironmentScan = time.Time{} - } - work, err := schedule.selectWork(ctx, w, devices, active) + work, err := schedule.selectWork(ctx, w, devices, active, !maintenance) w.observeSlots(len(active)) if err != nil { w.observeSchedulerPoll(0, err) diff --git a/services/core/internal/execution/worker_capability_test.go b/services/core/internal/execution/worker_capability_test.go new file mode 100644 index 000000000..d4d76c5bd --- /dev/null +++ b/services/core/internal/execution/worker_capability_test.go @@ -0,0 +1,63 @@ +package execution + +import ( + "context" + "encoding/json" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" +) + +type candidateQueryObserver struct { + sessions.Reader + scans atomic.Int32 +} + +func (o *candidateQueryObserver) ListExecutionWork(context.Context, string, []string, []string) ([]sessions.ExecutionWork, error) { + return nil, nil +} +func (o *candidateQueryObserver) ListEnvironmentInputWork(context.Context, string, []string) ([]sessions.EnvironmentInputWork, error) { + o.scans.Add(1) + return nil, nil +} + +func TestSchedulingHintScansBeforeDeadlineAndRestoresPollingDelay(t *testing.T) { + observer := &candidateQueryObserver{} + worker := &Worker{dispatcher: &Dispatcher{SessionsReader: observer}, concurrency: 1} + schedule := workerSchedule{nextEnvironmentScan: time.Now().Add(time.Hour)} + for _, hinted := range []bool{false, true, false} { + if _, err := schedule.selectWork(t.Context(), worker, nil, map[string]bool{}, hinted); err != nil { + t.Fatal(err) + } + } + if observer.scans.Load() != 1 { + t.Fatalf("candidate scans = %d; hint must bypass the deadline once", observer.scans.Load()) + } +} + +type phaseSessionReader struct{ sessions.Reader } + +func (phaseSessionReader) GetSessionEnvironment(context.Context, string, string) (sessions.Environment, error) { + return sessions.Environment{ID: "environment", Configuration: json.RawMessage(`{"type":"openai_hosted"}`)}, nil +} +func (phaseSessionReader) GetSessionDevice(context.Context, string, string) (sessions.ExecutionDevice, error) { + panic("non-running compute must not reach device eligibility") +} +func TestCapabilityHintDoesNotBypassComputePhase(t *testing.T) { + for _, phase := range []string{"quiescing", "suspending", "suspended", "restoring", "waking"} { + t.Run(phase, func(t *testing.T) { + reader := &strictDeploymentReader{t: t, environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + return deployment.Allocation{ComputePhase: phase}, nil + }} + worker := &Worker{dispatcher: &Dispatcher{SessionsReader: phaseSessionReader{}, DeploymentReader: reader}} + session := sessions.Session{Configuration: json.RawMessage(`{"environment":{"type":"openai_hosted"}}`)} + ready, err := worker.bindSessionDevice(t.Context(), session, func(string) bool { t.Fatal("phase bypassed"); return true }) + if err != nil || ready { + t.Fatal("non-running compute selected", ready, err) + } + }) + } +} diff --git a/services/core/internal/execution/worker_schedule.go b/services/core/internal/execution/worker_schedule.go index bf3a821cc..c38c07077 100644 --- a/services/core/internal/execution/worker_schedule.go +++ b/services/core/internal/execution/worker_schedule.go @@ -20,7 +20,10 @@ type scheduledWork struct { reservationID string } -func (s *workerSchedule) selectWork(ctx context.Context, w *Worker, devices []string, active map[string]bool) ([]scheduledWork, error) { +func (s *workerSchedule) selectWork(ctx context.Context, w *Worker, devices []string, active map[string]bool, hinted bool) ([]scheduledWork, error) { + if hinted { + s.nextEnvironmentScan = time.Time{} + } turns, err := w.dispatcher.SessionsReader.ListExecutionWork(ctx, s.turnCursor, []string{sessions.TurnQueued}, devices) if err != nil { return nil, err diff --git a/services/core/internal/runtimegateway/capability_hint_test.go b/services/core/internal/runtimegateway/capability_hint_test.go new file mode 100644 index 000000000..ba4fbd9e3 --- /dev/null +++ b/services/core/internal/runtimegateway/capability_hint_test.go @@ -0,0 +1,138 @@ +package runtimegateway + +import ( + "context" + "encoding/json" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" + "testing" + "time" +) + +func expectCapabilityHint(t *testing.T, reg *Registry, want bool) { + t.Helper() + select { + case <-reg.CapabilityHints(): + if !want { + t.Fatal("unexpected capability hint") + } + default: + if want { + t.Fatal("missing capability hint") + } + } +} + +func TestCapabilityHintsRequireCurrentAvailableSnapshot(t *testing.T) { + reg := NewRegistry() + old := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(old) + expectCapabilityHint(t, reg, false) // Transport alone is insufficient. + publishCapabilityHeartbeat(t, old, []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: false}}) + expectCapabilityHint(t, reg, false) + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: true}} + publishCapabilityHeartbeat(t, old, kinds) + expectCapabilityHint(t, reg, true) + publishCapabilityHeartbeat(t, old, kinds) + expectCapabilityHint(t, reg, false) // Ordinary heartbeats do not rescan. + newer := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(newer) + kinds[0].Version = "changed" + publishCapabilityHeartbeat(t, old, kinds) + expectCapabilityHint(t, reg, false) // Superseded socket cannot accelerate work. + publishCapabilityHeartbeat(t, newer, kinds) + expectCapabilityHint(t, reg, true) +} + +func TestCapabilityHintsCoalesceAndIgnoreDisconnectedPeers(t *testing.T) { + reg := NewRegistry() + s := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(s) + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: true}} + publishCapabilityHeartbeat(t, s, kinds) + kinds[0].Capabilities.Streaming = true + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, true) + expectCapabilityHint(t, reg, false) + reg.Deregister(s) + kinds[0].Capabilities.Steering = true + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, false) +} + +func TestInvalidHeartbeatCannotWakeScheduler(t *testing.T) { + reg := NewRegistry() + s := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(s) + s.handleHeartbeat(proto.Envelope{Type: proto.TypeHeartbeat, Payload: json.RawMessage(`{"ts":1,"active_requests":0,"supported_agent_kinds":[{"kind":"fake","available":true,"capabilities":{"streaming":"invented"}}]}`)}) + expectCapabilityHint(t, reg, false) + if !s.IsClosed() { + t.Fatal("malformed capabilities did not close transport") + } +} + +func publishCapabilityHeartbeat(t *testing.T, s *Session, kinds []runtimedevice.SupportedAgentKind) { + t.Helper() + advertised := make([]proto.SupportedAgentKind, len(kinds)) + for i, k := range kinds { + advertised[i] = proto.SupportedAgentKind{Kind: k.Kind, Available: k.Available, Version: k.Version, + Capabilities: prototest.Capabilities(proto.AgentKindCapabilities{ + Streaming: proto.CapabilityFromBool(k.Capabilities.Streaming), + Steering: proto.CapabilityFromBool(k.Capabilities.Steering), + })} + } + env, err := proto.NewEnvelope(proto.TypeHeartbeat, "", proto.HeartbeatPayload{SupportedAgentKinds: advertised}) + if err != nil { + t.Fatal(err) + } + s.handleHeartbeat(env) +} + +type capabilityAuthorityStore struct { + HeartbeatTouch + deleted bool + err error +} + +func (a *capabilityAuthorityStore) TouchAgentDaemonHeartbeat(context.Context, runtimedevice.Heartbeat) (runtimedevice.HeartbeatStatus, error) { + return runtimedevice.HeartbeatStatus{Deleted: a.deleted}, a.err +} +func TestCapabilityConfirmationRetriesWithoutObservingRejectedAuthority(t *testing.T) { + for _, mode := range []string{"error", "deleted", "draining"} { + t.Run(mode, func(t *testing.T) { + reg := NewRegistry() + s := NewSession(newFakeConn(), "device", "tenant", "version", reg, nil) + reg.Register(s) + t.Cleanup(func() { s.Close("test finished") }) + authority := &capabilityAuthorityStore{deleted: mode != "error"} + if mode == "error" { + authority.err = errors.New("private credential must not be observed") + } + s.heartbeat = authority + if mode == "draining" { + s.credentialHash = "original-hash" + s.archivedCancellations = &receiptStore{receipt: runtimedevice.ArchivedCancellationReceipt{RunID: "run", Deadline: time.Now().Add(time.Minute)}} + release, err := s.TrackExecutionDelivery("run") + if err != nil { + t.Fatal(err) + } + t.Cleanup(release) + } + kinds := []runtimedevice.SupportedAgentKind{{Kind: "fake", Available: true}} + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, false) + if mode == "draining" && s.IsClosed() { + t.Fatal("existing receipt drain was interrupted") + } + if mode == "error" { + authority.err = nil + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, true) + publishCapabilityHeartbeat(t, s, kinds) + expectCapabilityHint(t, reg, false) + } + }) + } +} diff --git a/services/core/internal/runtimegateway/registry.go b/services/core/internal/runtimegateway/registry.go index b5984821e..c30084280 100644 --- a/services/core/internal/runtimegateway/registry.go +++ b/services/core/internal/runtimegateway/registry.go @@ -12,6 +12,8 @@ import ( "errors" "sync" "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" ) // ErrDeviceNotRegistered is returned by Registry lookups when a caller @@ -36,16 +38,18 @@ type Registry struct { // is inserted into byDevice. Buffered(1) so a Register that // happens between WaitForDevice registering and selecting on the // chan still wakes the waiter. - waiters map[string][]chan *Session + waiters map[string][]chan *Session + capabilityHints chan struct{} } // NewRegistry returns an empty registry. The zero value would also // work but the constructor avoids accidental nil-map panics. func NewRegistry() *Registry { return &Registry{ - byDevice: map[string]*Session{}, - byRun: map[string]*Session{}, - waiters: map[string][]chan *Session{}, + byDevice: map[string]*Session{}, + byRun: map[string]*Session{}, + waiters: map[string][]chan *Session{}, + capabilityHints: make(chan struct{}, 1), } } @@ -237,3 +241,29 @@ func (r *Registry) removeWaiter(deviceID string, ch chan *Session) { r.waiters[deviceID] = filtered } } + +// CapabilityHints coalesces capability changes for the single execution Worker. +// A hint grants no execution authority; the Worker rechecks its normal gates. +func (r *Registry) CapabilityHints() <-chan struct{} { return r.capabilityHints } + +func (r *Registry) observeCapabilitySnapshot(sess *Session, kinds []runtimedevice.SupportedAgentKind) bool { + r.mu.RLock() + defer r.mu.RUnlock() + if r.byDevice[sess.DeviceID] != sess || sess.IsClosed() { + return false + } + available := 0 + for _, kind := range kinds { + if kind.Available { + available++ + } + } + if available == 0 { + return true + } + select { + case r.capabilityHints <- struct{}{}: + default: + } + return true +} diff --git a/services/core/internal/runtimegateway/session.go b/services/core/internal/runtimegateway/session.go index 9032c2f48..ebcf0da07 100644 --- a/services/core/internal/runtimegateway/session.go +++ b/services/core/internal/runtimegateway/session.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "strings" "sync" "time" @@ -100,9 +101,10 @@ type Session struct { // supportedKinds is the latest daemon-advertised agent_kind snapshot, // updated from heartbeat frames and read by the connector before // sending execution_prepare so unsupported engines fail on the server. - kindsMu sync.RWMutex - kindsSeen bool - supportedKinds []runtimedevice.SupportedAgentKind + kindsMu sync.RWMutex + kindsSeen bool + supportedKinds []runtimedevice.SupportedAgentKind + capabilityObservationPending bool // Subscribers keyed by runID. The read loop only sends on these // channels; Unsubscribe is the only place that closes them. @@ -222,11 +224,25 @@ func (s *Session) setSupportedAgentKinds(kinds []runtimedevice.SupportedAgentKin copyKinds := make([]runtimedevice.SupportedAgentKind, len(kinds)) copy(copyKinds, kinds) s.kindsMu.Lock() + changed := !s.kindsSeen || !reflect.DeepEqual(s.supportedKinds, copyKinds) s.kindsSeen = true s.supportedKinds = copyKinds + if changed { + s.capabilityObservationPending = true + } s.kindsMu.Unlock() } +// A failed authorization or persistence check retains the pending hint +// for the next confirmed heartbeat without changing capability publication. +func (s *Session) observeConfirmedCapabilities(kinds []runtimedevice.SupportedAgentKind) { + s.kindsMu.Lock() + defer s.kindsMu.Unlock() + if s.capabilityObservationPending && s.reg.observeCapabilitySnapshot(s, kinds) { + s.capabilityObservationPending = false + } +} + // Close closes the transport and subscriptions with ErrSessionClosed, then // releases connection ownership. It establishes no execution outcome. Idempotent. func (s *Session) Close(reason string) { @@ -453,6 +469,7 @@ func (s *Session) handleHeartbeat(env proto.Envelope) { kinds := deviceKindsFromHeartbeat(p) s.setSupportedAgentKinds(kinds) if s.heartbeat == nil { + s.observeConfirmedCapabilities(kinds) return } @@ -481,7 +498,9 @@ func (s *Session) handleHeartbeat(env proto.Envelope) { // actual admin action; the daemon only sees it's no longer // the current owner. s.CloseWithCode(CloseRuntimeDeleted, "runtime retired") + return } + s.observeConfirmedCapabilities(kinds) } func deviceKindsFromHeartbeat(p proto.HeartbeatPayload) []runtimedevice.SupportedAgentKind { diff --git a/services/core/internal/sandbox/docker/bootstrap_test.go b/services/core/internal/sandbox/docker/bootstrap_test.go index 31d2845ff..5d0b0f088 100644 --- a/services/core/internal/sandbox/docker/bootstrap_test.go +++ b/services/core/internal/sandbox/docker/bootstrap_test.go @@ -14,7 +14,7 @@ import ( ) func TestBootstrapDeliversOnlyPublicConnectionInput(t *testing.T) { - b := sandbox.Bootstrap{CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-secret"} + b := sandbox.Bootstrap{CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-secret", Harness: "codex"} found := false server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != "PUT" || !strings.HasSuffix(r.URL.Path, "/containers/test/archive") { diff --git a/services/core/internal/sandbox/docker/contract_test.go b/services/core/internal/sandbox/docker/contract_test.go index 09db3fef8..8105d6d40 100644 --- a/services/core/internal/sandbox/docker/contract_test.go +++ b/services/core/internal/sandbox/docker/contract_test.go @@ -26,7 +26,7 @@ type contractStep struct { func dockerContractFixture(t *testing.T, cancel context.CancelFunc, script func(*Provider, sandbox.Reference) []contractStep) contracttest.Fixture { t.Helper() - b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} var mu sync.Mutex var calls []string var steps []contractStep diff --git a/services/core/internal/sandbox/docker/deployment_test.go b/services/core/internal/sandbox/docker/deployment_test.go index 9df0727d2..7dfa1283c 100644 --- a/services/core/internal/sandbox/docker/deployment_test.go +++ b/services/core/internal/sandbox/docker/deployment_test.go @@ -87,7 +87,7 @@ func TestManagedDriftRejectsBeforeBootstrapAndRetainsCleanup(t *testing.T) { t.Fatal(err) } resources.CPUs = 9 - b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "secret", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "secret", Harness: "codex", NetworkAccess: "enabled"} info, err := p.Create(t.Context(), b) if !errors.Is(err, sandbox.ErrInvalid) { t.Fatal("drift accepted", err) diff --git a/services/core/internal/sandbox/docker/provider_test.go b/services/core/internal/sandbox/docker/provider_test.go index a7f71289b..0df733084 100644 --- a/services/core/internal/sandbox/docker/provider_test.go +++ b/services/core/internal/sandbox/docker/provider_test.go @@ -76,7 +76,7 @@ func TestDockerProviderLifecycle(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() bootstrap := func() sandbox.Bootstrap { - return sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-test-credential", NetworkAccess: "enabled"} + return sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-test-credential", Harness: "codex", NetworkAccess: "enabled"} } b := bootstrap() b.NetworkAccess, b.AllowedDomains = "restricted", []string{"Example.com", "api.example.com"} diff --git a/services/core/internal/sandbox/docker/recovery_test.go b/services/core/internal/sandbox/docker/recovery_test.go index 06ce2e8ae..9cb813598 100644 --- a/services/core/internal/sandbox/docker/recovery_test.go +++ b/services/core/internal/sandbox/docker/recovery_test.go @@ -87,7 +87,7 @@ func TestDockerProviderRecoveryObservations(t *testing.T) { if e != nil { t.Fatal(e) } - b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-recovery-token", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()}, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-recovery-token", Harness: "codex", NetworkAccess: "enabled"} direct, e := client.New(client.WithHost("unix:///var/run/docker.sock")) if e != nil { t.Fatal(e) diff --git a/services/core/internal/sandbox/e2b/contract_test.go b/services/core/internal/sandbox/e2b/contract_test.go index b4d64227b..0216cfc23 100644 --- a/services/core/internal/sandbox/e2b/contract_test.go +++ b/services/core/internal/sandbox/e2b/contract_test.go @@ -16,7 +16,7 @@ func (f contractCaller) Call(ctx context.Context, q Request) (Response, error) { func TestProviderContract(t *testing.T) { contracttest.RunFailures(t, func(t *testing.T, s contracttest.Scenario, cancel context.CancelFunc) contracttest.Fixture { p, _, r := fixture(t) - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} var calls []string p.caller = contractCaller(func(ctx context.Context, q Request) (Response, error) { calls = append(calls, q.Operation) @@ -47,7 +47,7 @@ func TestProviderContract(t *testing.T) { func TestProviderContractObservation(t *testing.T) { p, f, r := fixture(t) f.response.Info = &sandbox.Info{Reference: r, ProviderID: "native-owned", State: "running", CreateSettled: true, BootstrapComplete: true} - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} got, err := p.Create(bounded(t), b) contracttest.AssertObservation(t, got, err, r, "native-owned", "running") got, err = p.GetInfo(bounded(t), r) diff --git a/services/core/internal/sandbox/e2b/helper_contract.go b/services/core/internal/sandbox/e2b/helper_contract.go index f4cf30ffb..f6fe533b4 100644 --- a/services/core/internal/sandbox/e2b/helper_contract.go +++ b/services/core/internal/sandbox/e2b/helper_contract.go @@ -11,7 +11,7 @@ import ( //go:generate go run ./internal/contractgen // This adapter-private boundary is documented in tools/e2b-provider/README.md. -const ProtocolVersion = 1 +const ProtocolVersion = 2 const MaxOutputBytes = 1024 * 1024 const MaxRequestBytes = 72 * 1024 * 1024 const MaxResponseBytes = 16 * 1024 * 1024 diff --git a/services/core/internal/sandbox/e2b/helper_contract_test.go b/services/core/internal/sandbox/e2b/helper_contract_test.go index 686321cb1..de319aa09 100644 --- a/services/core/internal/sandbox/e2b/helper_contract_test.go +++ b/services/core/internal/sandbox/e2b/helper_contract_test.go @@ -67,6 +67,7 @@ func validManagedExchange(data []byte) bool { _ = json.Unmarshal(bootstrapBytes, &expected) delete(expected, "CoreURL") delete(expected, "Credential") + delete(expected, "Harness") expected["InstallationID"] = nil expected["RuntimeBootstrap"] = nil if workspace, present := fields["Workspace"]; present { diff --git a/services/core/internal/sandbox/e2b/installed_paths_test.go b/services/core/internal/sandbox/e2b/installed_paths_test.go index 5012e0d97..60d5b467b 100644 --- a/services/core/internal/sandbox/e2b/installed_paths_test.go +++ b/services/core/internal/sandbox/e2b/installed_paths_test.go @@ -31,7 +31,7 @@ func TestConfigurationDiscoveryUsesSuppliedProcessPaths(t *testing.T) { if err := os.MkdirAll(filepath.Dir(binary), 0700); err != nil { t.Fatal(err) } - if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":1,\"Templates\":[]}'\n"), 0700); err != nil { + if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"Templates\":[]}'\n"), 0700); err != nil { t.Fatal(err) } input := sandbox.ConfigurationDiscoveryInput{Credential: json.RawMessage(`{"api_key":"synthetic-key"}`)} diff --git a/services/core/internal/sandbox/e2b/internal/contractgen/main.go b/services/core/internal/sandbox/e2b/internal/contractgen/main.go index 118fe1c3f..a7a9c788f 100644 --- a/services/core/internal/sandbox/e2b/internal/contractgen/main.go +++ b/services/core/internal/sandbox/e2b/internal/contractgen/main.go @@ -71,9 +71,9 @@ func main() { write("services/core/internal/sandbox/e2b/helper_sdk_generated.go", goCode) bootstrap := fields(reflect.TypeFor[sandbox.Bootstrap]()) requiredBootstrap := fieldNames(reflect.TypeFor[sandbox.Bootstrap](), true) - requiredBootstrap = slices.DeleteFunc(requiredBootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" }) + requiredBootstrap = slices.DeleteFunc(requiredBootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" || s == "Harness" }) requiredBootstrap = append(requiredBootstrap, "InstallationID", "RuntimeBootstrap") - bootstrap = slices.DeleteFunc(bootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" }) + bootstrap = slices.DeleteFunc(bootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" || s == "Harness" }) bootstrap = append(bootstrap, "InstallationID", "RuntimeBootstrap") identity := fields(reflect.TypeFor[sandbox.Reference]()) for _, name := range []string{"SessionID", "DeviceID"} { @@ -115,7 +115,7 @@ func main() { // envelopes and managed bootstrap inputs in both implementations. func fixtures() []byte { r := sandbox.Reference{TenantID: "11111111-1111-4111-8111-111111111111", EnvironmentID: "22222222-2222-4222-8222-222222222222", AllocationID: "33333333-3333-4333-8333-333333333333"} - b := sandbox.Bootstrap{Reference: r, SessionID: "44444444-4444-4444-8444-444444444444", DeviceID: "55555555-5555-4555-8555-555555555555", CoreURL: "https://core.example/api/v1", Credential: "fixture-only", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: "44444444-4444-4444-8444-444444444444", DeviceID: "55555555-5555-4555-8555-555555555555", CoreURL: "https://core.example/api/v1", Credential: "fixture-only", Harness: "codex", NetworkAccess: "enabled"} installation := "66666666-6666-4666-8666-666666666666" connection := b.RuntimeConnection() q := e2b.Request{Version: e2b.ProtocolVersion, Operation: "create", Config: e2b.Config{InstallationID: installation}, Reference: r, Bootstrap: &b, RuntimeBootstrap: &connection, Deadline: time.Date(2099, 1, 1, 0, 0, 0, 0, time.UTC)} @@ -187,6 +187,7 @@ func fixtures() []byte { managed := object(b) delete(managed, "CoreURL") delete(managed, "Credential") + delete(managed, "Harness") managed["InstallationID"] = installation managed["RuntimeBootstrap"] = runtimebootstrap.Connection(connection) for _, policy := range []agentnetwork.Policy{{Access: "enabled"}, {Access: "disabled"}, {Access: "restricted", AllowedDomains: []string{"example.com"}}} { diff --git a/services/core/internal/sandbox/e2b/process_test.go b/services/core/internal/sandbox/e2b/process_test.go index 2d15997f4..a2e90a5b4 100644 --- a/services/core/internal/sandbox/e2b/process_test.go +++ b/services/core/internal/sandbox/e2b/process_test.go @@ -12,7 +12,7 @@ import ( ) func TestProcessWireRejectsUnknownAndTrailingOutput(t *testing.T) { - for _, output := range []string{`{"Version":1,"Secret":"private"}`, `{"Version":1} {"Version":1}`} { + for _, output := range []string{`{"Version":2,"Secret":"private"}`, `{"Version":2} {"Version":2}`} { root := t.TempDir() binary := filepath.Join(root, "helper") if err := os.WriteFile(binary, []byte("#!/bin/sh\nprintf '%s' '"+output+"'\n"), 0700); err != nil { @@ -29,7 +29,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { marker := filepath.Join(root, "settled") // The private test path contains no shell syntax; the real adapter never puts // credentials or request contents in argv or inherited environment. - script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":2}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -50,7 +50,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { func TestEnvironmentDropsProviderSelectorsAndCredentials(t *testing.T) { root := t.TempDir() binary := filepath.Join(root, "helper") - script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":2}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -67,7 +67,7 @@ func TestCredentialFenceWaitsForActualHelperExitAfterCancellation(t *testing.T) binary := filepath.Join(root, "helper") marker := filepath.Join(root, "started") finish := filepath.Join(root, "finish") - script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":1}'\n" + script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":2}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } diff --git a/services/core/internal/sandbox/e2b/provider_test.go b/services/core/internal/sandbox/e2b/provider_test.go index 406160671..6263ede3e 100644 --- a/services/core/internal/sandbox/e2b/provider_test.go +++ b/services/core/internal/sandbox/e2b/provider_test.go @@ -103,7 +103,7 @@ func TestKillRequiresTerminalProof(t *testing.T) { } func TestCreateAndCommandUseOnlyPrivateRequest(t *testing.T) { p, f, r := fixture(t) - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private-bootstrap", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private-bootstrap", Harness: "codex", NetworkAccess: "enabled"} f.response.Info = &sandbox.Info{Reference: r, State: "running", ProviderID: "native-id", CreateSettled: true, BootstrapComplete: true} if _, err := p.Create(bounded(t), b); err != nil { t.Fatal(err) @@ -143,7 +143,7 @@ func TestDeadlineAndCommandAdmission(t *testing.T) { func TestDefinitePreHelperCreateFailureCarriesAbsenceProof(t *testing.T) { p, f, r := fixture(t) - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "private", Harness: "codex", NetworkAccess: "enabled"} for _, err := range []error{errHelperNotStarted, context.DeadlineExceeded} { f.err = err info, gotErr := p.Create(bounded(t), b) diff --git a/services/core/internal/sandbox/microsandbox/contract_test.go b/services/core/internal/sandbox/microsandbox/contract_test.go index 0733824a9..7ecfc5da8 100644 --- a/services/core/internal/sandbox/microsandbox/contract_test.go +++ b/services/core/internal/sandbox/microsandbox/contract_test.go @@ -7,12 +7,13 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/contracttest" "github.com/google/uuid" "testing" + "time" ) func TestProviderContract(t *testing.T) { contracttest.RunFailures(t, func(t *testing.T, s contracttest.Scenario, cancel context.CancelFunc) contracttest.Fixture { c, r := testConfig(), testRef() - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} var calls []string p, err := NewWithCaller(c, callerFunc(func(ctx context.Context, q Request) (Response, error) { calls = append(calls, q.Operation) @@ -62,7 +63,7 @@ func TestProviderContractObservation(t *testing.T) { if err != nil { t.Fatal(err) } - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "synthetic", Harness: "codex", NetworkAccess: "enabled"} got, err := p.Create(deadline(t), b) contracttest.AssertObservation(t, got, err, r, "native-owned", "running") got, err = p.GetInfo(deadline(t), r) @@ -70,3 +71,21 @@ func TestProviderContractObservation(t *testing.T) { got, err = p.Renew(deadline(t), r) contracttest.AssertObservation(t, got, err, r, "native-owned", "stopped") } + +func TestBootstrapWireRejectsSupersededVersion(t *testing.T) { + c, r := testConfig(), testRef() + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "enabled"} + q := Request{Version: ProtocolVersion, Operation: "create", Config: c, Reference: r, Bootstrap: &b, Deadline: time.Now().Add(time.Minute)} + if err := ValidateRequest(q); err != nil { + t.Fatal(err) + } + q.Version = 2 + if err := ValidateRequest(q); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("superseded bootstrap wire accepted", err) + } + q.Version = ProtocolVersion + b.Harness = "" + if err := ValidateRequest(q); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("missing selected Harness accepted", err) + } +} diff --git a/services/core/internal/sandbox/microsandbox/creation_settlement_test.go b/services/core/internal/sandbox/microsandbox/creation_settlement_test.go index db52bcfae..6fab743b3 100644 --- a/services/core/internal/sandbox/microsandbox/creation_settlement_test.go +++ b/services/core/internal/sandbox/microsandbox/creation_settlement_test.go @@ -11,7 +11,7 @@ import ( func TestCreateConfigurationRejectionSettlement(t *testing.T) { config, ref := testConfig(), testRef() bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, - CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled"} + CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "disabled"} for _, test := range []struct { name string change func(*Response) diff --git a/services/core/internal/sandbox/microsandbox/types.go b/services/core/internal/sandbox/microsandbox/types.go index bb8d9246d..0a04f9e53 100644 --- a/services/core/internal/sandbox/microsandbox/types.go +++ b/services/core/internal/sandbox/microsandbox/types.go @@ -9,7 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -const ProtocolVersion = 4 +const ProtocolVersion = 5 const SDKVersion = "v0.7.2" const MaxOutputBytes = 1024 * 1024 const MaxRequestBytes = 72 * 1024 * 1024 diff --git a/services/core/internal/sandbox/microsandbox/workspace_test.go b/services/core/internal/sandbox/microsandbox/workspace_test.go index 64ba57717..f345df118 100644 --- a/services/core/internal/sandbox/microsandbox/workspace_test.go +++ b/services/core/internal/sandbox/microsandbox/workspace_test.go @@ -62,7 +62,7 @@ func TestWorkspaceDoesNotFallbackWithoutResolver(t *testing.T) { config := testConfig() config.ExternalWorkspace = true p, _ := NewWithCaller(config, callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) - _, err := p.Create(deadline(t), sandbox.Bootstrap{Reference: testRef(), SessionID: testRef().TenantID, DeviceID: testRef().EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()}) + _, err := p.Create(deadline(t), sandbox.Bootstrap{Harness: "codex", Reference: testRef(), SessionID: testRef().TenantID, DeviceID: testRef().EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()}) if !errors.Is(err, workspacefs.ErrUnsupported) || calls != 0 { t.Fatal(err, calls) } @@ -81,7 +81,7 @@ func TestPreNativeWorkspaceFailureSettlesAbsentCreation(t *testing.T) { provider.workspace = workspaceResolverFunc(func(context.Context, workspacefs.Binding) (workspacefs.Directory, error) { return workspacefs.Directory{}, workspacefs.ErrUnavailable }) - bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()} + bootstrap := sandbox.Bootstrap{Harness: "codex", Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()} ctx := deadline(t) switch failure { case "mode": diff --git a/services/core/internal/sandbox/node/connection_test.go b/services/core/internal/sandbox/node/connection_test.go index abee501b3..931d1e78e 100644 --- a/services/core/internal/sandbox/node/connection_test.go +++ b/services/core/internal/sandbox/node/connection_test.go @@ -179,7 +179,7 @@ func TestCopiedIdentityCannotReplaceNodeWithInflightCreate(t *testing.T) { proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) r := reference() created := make(chan error, 1) - go func() { _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: r}); created <- err }() + go func() { _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: r, Harness: "codex"}); created <- err }() <-original.started stopDuplicate, duplicateDone := start(duplicateDir, duplicate) defer func() { diff --git a/services/core/internal/sandbox/node/creation_settlement_test.go b/services/core/internal/sandbox/node/creation_settlement_test.go index 09d8daec5..cc0048506 100644 --- a/services/core/internal/sandbox/node/creation_settlement_test.go +++ b/services/core/internal/sandbox/node/creation_settlement_test.go @@ -76,7 +76,7 @@ func TestNodeCarriesCreationSettlementWithoutConvertingFailureToSuccess(t *testi proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) for _, operation := range []func(context.Context) (sandbox.Info, error){ func(ctx context.Context) (sandbox.Info, error) { - return proxy.Create(ctx, sandbox.Bootstrap{Reference: ref}) + return proxy.Create(ctx, sandbox.Bootstrap{Reference: ref, Harness: "codex"}) }, func(ctx context.Context) (sandbox.Info, error) { return proxy.GetInfo(ctx, ref) }, } { diff --git a/services/core/internal/sandbox/node/docker_live_test.go b/services/core/internal/sandbox/node/docker_live_test.go index 48695bfff..3c59d3bc7 100644 --- a/services/core/internal/sandbox/node/docker_live_test.go +++ b/services/core/internal/sandbox/node/docker_live_test.go @@ -87,7 +87,7 @@ func TestDockerNodeTransportLifecycle(t *testing.T) { }() ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() - b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-node-transport-credential", NetworkAccess: "enabled"} + b := sandbox.Bootstrap{Reference: r, SessionID: uuid.NewString(), DeviceID: uuid.NewString(), CoreURL: "http://core.invalid/api/v1", Credential: "synthetic-node-transport-credential", Harness: "codex", NetworkAccess: "enabled"} callCtx, callCancel := context.WithTimeout(ctx, 25*time.Second) info, err := proxy.Create(callCtx, b) callCancel() diff --git a/services/core/internal/sandbox/node/generation_connection_test.go b/services/core/internal/sandbox/node/generation_connection_test.go index 1c3e59d6c..efc41798d 100644 --- a/services/core/internal/sandbox/node/generation_connection_test.go +++ b/services/core/internal/sandbox/node/generation_connection_test.go @@ -69,7 +69,7 @@ func TestGenerationWireRoutesOldOwnershipAndCurrentTargetSeparately(t *testing.T t.Fatal("retained generation info failed", generation, err) } if generation != 9 { - if _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: ref}); err != nil { + if _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: ref, Harness: "codex"}); err != nil { t.Fatal("exact ready provider Create failed", generation, err) } } diff --git a/services/core/internal/sandbox/node/generation_json.go b/services/core/internal/sandbox/node/generation_json.go index 5336a320f..661d177b2 100644 --- a/services/core/internal/sandbox/node/generation_json.go +++ b/services/core/internal/sandbox/node/generation_json.go @@ -130,9 +130,15 @@ func validateGenerationJSON(raw []byte, kind string) error { } } if value := values["request"]; value != nil { - if _, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume snapshot observation", ""); err != nil { + request, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume snapshot observation", "") + if err != nil { return err } + if bootstrap := request["bootstrap"]; bootstrap != nil { + if _, err := generationObject(bootstrap, "TenantID EnvironmentID AllocationID SessionID DeviceID CoreURL Credential Harness NetworkAccess AllowedDomains", "Workspace", "AllowedDomains"); err != nil { + return err + } + } } if value := values["response"]; value != nil { if _, err := generationObject(value, "id connection_id", "error_code unsupported info compute state command sample", ""); err != nil { diff --git a/services/core/internal/sandbox/node/generation_json_test.go b/services/core/internal/sandbox/node/generation_json_test.go index b6fef3a10..e9f52cbf9 100644 --- a/services/core/internal/sandbox/node/generation_json_test.go +++ b/services/core/internal/sandbox/node/generation_json_test.go @@ -90,3 +90,23 @@ func TestNodeGenerationManagementIsAnExplicitCurrentCapability(t *testing.T) { } } } + +func TestNodeBootstrapRejectsAmbiguousHarness(t *testing.T) { + r := sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()} + b := sandbox.Bootstrap{Reference: r, Harness: "codex"} + f := frame{Version: ProtocolVersion, Type: "request", Request: &request{DeploymentGeneration: 1, ID: uuid.NewString(), Sequence: 1, ConnectionID: uuid.NewString(), OwnerEpoch: 1, Operation: "create", TimeoutMillis: 1000, Reference: r, Bootstrap: &b}} + raw, _ := json.Marshal(f) + good := string(raw) + if _, err := decodeFrame(raw); err != nil { + t.Fatal("valid selected bootstrap", err) + } + for _, replacement := range []string{`"Harness":null`, `"Harness":""`, `"harness":"codex"`, `"Harness":"codex","Harness":"mcode"`, `"Harness":"codex","harness":"mcode"`, `"Harness":"Codex"`, `"Harness":42`} { + bad := strings.Replace(good, `"Harness":"codex"`, replacement, 1) + if decoded, err := decodeFrame([]byte(bad)); err == nil && decoded.Request.validate() == nil { + t.Fatal("ambiguous selection reached dispatch", replacement) + } + } + if _, err := decodeFrame([]byte(strings.Replace(good, `,"Harness":"codex"`, "", 1))); err == nil { + t.Fatal("missing Harness accepted") + } +} diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index 946470389..a8a21fbce 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -129,7 +129,10 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing createCtx, stopCreate := context.WithTimeout(ctx, 150*time.Millisecond) defer stopCreate() createDone := make(chan error, 1) - go func() { _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r}); createDone <- err }() + go func() { + _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r, Harness: "codex"}) + createDone <- err + }() <-p.started if err := <-createDone; !errors.Is(err, sandbox.ErrComputeUnconfirmed) { t.Fatalf("lost reply = %v", err) diff --git a/services/core/internal/sandbox/node/recovery_test.go b/services/core/internal/sandbox/node/recovery_test.go index 78d106da3..c5039ea9d 100644 --- a/services/core/internal/sandbox/node/recovery_test.go +++ b/services/core/internal/sandbox/node/recovery_test.go @@ -236,7 +236,7 @@ func TestDegradedNodeRetainsObservationAndCleanup(t *testing.T) { } proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) r := reference() - if _, err = proxy.Create(ctx, sandbox.Bootstrap{Reference: r}); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { + if _, err = proxy.Create(ctx, sandbox.Bootstrap{Reference: r, Harness: "codex"}); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { t.Fatalf("create = %v", err) } if _, err = proxy.GetInfo(ctx, r); err != nil { diff --git a/services/core/internal/sandbox/node/timeout_test.go b/services/core/internal/sandbox/node/timeout_test.go index 15f8da3c5..aa6184be5 100644 --- a/services/core/internal/sandbox/node/timeout_test.go +++ b/services/core/internal/sandbox/node/timeout_test.go @@ -108,7 +108,10 @@ func TestQueuedMutationExpiresWithoutExecution(t *testing.T) { createCtx, stopCreate := context.WithTimeout(ctx, 3*time.Second) defer stopCreate() created := make(chan error, 1) - go func() { _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r}); created <- err }() + go func() { + _, err := proxy.Create(createCtx, sandbox.Bootstrap{Reference: r, Harness: "codex"}) + created <- err + }() <-p.started killCtx, stopKill := context.WithTimeout(ctx, 80*time.Millisecond) defer stopKill() diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index d4ab402ab..e7a9d80f2 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -11,6 +11,7 @@ import ( "io" "time" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" @@ -18,7 +19,7 @@ import ( "github.com/gorilla/websocket" ) -const ProtocolVersion = 5 +const ProtocolVersion = 6 const MaxControlFrameBytes = 32 * 1024 const MaxFrameBytes = 72 * 1024 * 1024 const maxPending = 32 @@ -254,7 +255,7 @@ func (q request) validate() error { return nil } case "create": - if count == 1 && q.Bootstrap != nil && q.Bootstrap.Reference == q.Reference && sandbox.ValidateWorkspaceBinding(q.Reference, q.Bootstrap.Workspace) == nil { + if count == 1 && q.Bootstrap != nil && q.Bootstrap.Reference == q.Reference && runtimebootstrap.ValidHarness(q.Bootstrap.Harness) && sandbox.ValidateWorkspaceBinding(q.Reference, q.Bootstrap.Workspace) == nil { return nil } case "info", "renew", "kill", "initial": diff --git a/services/core/internal/sandbox/node/workspace_test.go b/services/core/internal/sandbox/node/workspace_test.go index 35ab50569..787447247 100644 --- a/services/core/internal/sandbox/node/workspace_test.go +++ b/services/core/internal/sandbox/node/workspace_test.go @@ -30,10 +30,29 @@ func TestWorkspaceWireBindsConfigurationAndEnvironment(t *testing.T) { case "object": b.Attachment.Reference.ObjectID = "invalid" } - q := request{ID: ref.AllocationID, Reference: ref, Operation: "create", TimeoutMillis: 1000, Bootstrap: &sandbox.Bootstrap{Reference: ref, Workspace: &b}} + q := request{ID: ref.AllocationID, Reference: ref, Operation: "create", TimeoutMillis: 1000, Bootstrap: &sandbox.Bootstrap{Harness: "codex", Reference: ref, Workspace: &b}} if (q.validate() == nil) != (fault == "match") { t.Fatal("invalid create binding forwarding", fault) } + for _, external := range []bool{false, true} { + wireRequest := q + bootstrap := *q.Bootstrap + wireRequest.Bootstrap = &bootstrap + if !external { + bootstrap.Workspace = nil + } + wireRequest.DeploymentGeneration, wireRequest.Sequence, wireRequest.OwnerEpoch = 1, 1, 1 + wireRequest.ConnectionID = ref.TenantID + raw, err := json.Marshal(frame{Version: ProtocolVersion, Type: "request", Request: &wireRequest}) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeFrame(raw) + want := !external || fault == "match" + if (err == nil && decoded.Request.validate() == nil) != want { + t.Fatalf("workspace wire external=%v fault=%s: %v", external, fault, err) + } + } q.Operation = "resume" q.Bootstrap = nil q.Resume = &sandbox.ResumeRequest{Reference: ref, Workspace: &b} @@ -75,7 +94,7 @@ func TestActualWorkspaceRefusalKeepsSettledAbsenceOnWire(t *testing.T) { } fsConfig := workspacefs.Configuration{ID: "44444444-4444-4444-8444-444444444444", Adapter: "fixture", Parameters: json.RawMessage(`{}`)} binding := &workspacefs.Binding{Configuration: fsConfig, Attachment: workspacefs.Attachment{Reference: workspacefs.Reference{TenantID: ref.TenantID, EnvironmentID: ref.EnvironmentID, ObjectID: "55555555-5555-4555-8555-555555555555"}, ConfigurationID: fsConfig.ID, Kind: workspacefs.AttachmentHostDirectory, Native: json.RawMessage(`{}`)}} - bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: binding} + bootstrap := sandbox.Bootstrap{Harness: "codex", Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: binding} ctx, cancel := context.WithTimeout(t.Context(), time.Second) defer cancel() out := execute(ctx, provider, request{ID: ref.AllocationID, ConnectionID: ref.TenantID, Reference: ref, Operation: "create", TimeoutMillis: 1000, Bootstrap: &bootstrap}) diff --git a/services/core/internal/sandbox/runtime_bootstrap.go b/services/core/internal/sandbox/runtime_bootstrap.go index a4a05ecad..66c83ee86 100644 --- a/services/core/internal/sandbox/runtime_bootstrap.go +++ b/services/core/internal/sandbox/runtime_bootstrap.go @@ -7,6 +7,6 @@ import "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" func (b Bootstrap) RuntimeConnection() runtimebootstrap.Connection { return runtimebootstrap.Connection{ Version: runtimebootstrap.Version, CoreURL: b.CoreURL, - DeviceID: b.DeviceID, Credential: b.Credential, + DeviceID: b.DeviceID, Credential: b.Credential, Harness: b.Harness, } } diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index 8d311697f..e5770790e 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -53,9 +53,9 @@ type Reference struct{ TenantID, EnvironmentID, AllocationID string } type Bootstrap struct { Workspace *workspacefs.Binding `json:",omitempty"` Reference - SessionID, DeviceID, CoreURL, Credential string - NetworkAccess string - AllowedDomains []string + SessionID, DeviceID, CoreURL, Credential, Harness string + NetworkAccess string + AllowedDomains []string } // Info describes compute only. Running does not establish daemon authentication, diff --git a/services/core/tests/integration/runtime_lifecycle_test.go b/services/core/tests/integration/runtime_lifecycle_test.go index 4f28fc6f0..829f4fef7 100644 --- a/services/core/tests/integration/runtime_lifecycle_test.go +++ b/services/core/tests/integration/runtime_lifecycle_test.go @@ -28,6 +28,7 @@ type lifecycleProvider struct { loseCreate, absent, unavailable bool credentialHash string credential string + harness string } func (p *lifecycleProvider) Create(_ context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { @@ -36,6 +37,7 @@ func (p *lifecycleProvider) Create(_ context.Context, b sandbox.Bootstrap) (sand p.creates++ p.credentialHash = runtimedevice.HashCredential(b.Credential) p.credential = b.Credential + p.harness = b.Harness i := sandbox.Info{Reference: b.Reference, ProviderID: b.AllocationID, State: "running", BootstrapComplete: true} if !p.absent { p.resources[b.AllocationID] = i @@ -245,3 +247,32 @@ func TestManagedRuntimeStoppedComputeDoesNotRequestCleanup(t *testing.T) { } } } + +func TestManagedRuntimeBootstrapUsesSessionHarness(t *testing.T) { + for _, kind := range []string{"codex", "mcode", "claude_sdk"} { + t.Run(kind, func(t *testing.T) { + s, key := configuredStore(t) + tenant := uuid.NewString() + session, err := s.CreateSession(t.Context(), tenant, WithFixtureModelProvider(sessions.CreateSession{Creator: FixtureCreator(), Engine: kind, IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted","network":{"access":"enabled"}}}`)})) + if err != nil { + t.Fatal(err) + } + env, err := sessionAdapter(s).GetSessionEnvironment(t.Context(), tenant, session.ID) + if err != nil { + t.Fatal(err) + } + p := &lifecycleProvider{resources: map[string]sandbox.Info{}} + w, _ := managedWorker(t, s, key, p) + for range 2 { + if _, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key); err != nil { + t.Fatal(err) + } + } + p.mu.Lock() + defer p.mu.Unlock() + if p.harness != kind || p.creates != 1 { + t.Fatalf("bootstrap selection=%s creates=%d", p.harness, p.creates) + } + }) + } +} diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index 1e7588c76..46f1e9f15 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -30,7 +30,7 @@ Compatible endpoints must return the SDK 2.51.0 template-list and template-build Run `go generate ./services/core/internal/sandbox/e2b` from the repository root after changing these declarations. `make check-e2b-provider` and the Go adapter tests reject stale projections; both languages consume generated valid and invalid exchanges covering wire types, extra fields, operation/reference bounds and managed-bootstrap fields. The helper build copies those fixtures with its source before running the pinned-SDK suite. -The boundary has version 1. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. +The boundary has version 2. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. ## Receipts and state directory @@ -42,9 +42,9 @@ A helper holds its allocation's lock until the SDK operation returns, even after 1. Record `create_pending` with the bootstrap identity, then call `Sandbox.create` with the template, the configured timeout, the ownership metadata, `on_timeout=kill` and auto-resume disabled. A definite rejection records a settled `rejected` receipt with no sandbox IDs. 2. Record the sandbox ID and connection material, check the sandbox domain, then read the sandbox by ID and check its ownership metadata, template and resources before writing any credential. A mismatch records a settled rejection and returns `CreateSettled` with the error. -3. Check that `/opt/oac-e2b/managed_init.py` is readable, write the managed bootstrap input to `/root/.oac/e2b/managed-bootstrap.json` and run `managed_init.py` as root. +3. Check that `/opt/oac-e2b/managed_init.py` has regular-file type through SDK file information, then open it through the SDK streaming file API as root and close the response without downloading its contents. This verifies readability without a remote probe process. Then write the managed bootstrap input to `/root/.oac/e2b/managed-bootstrap.json` and run `managed_init.py` as root. -`managed_init.py` prepares the image as the [application-managed startup](../../deploy/e2b/README.md#startup-and-security-boundary) does, writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json` (mode 0600, owned by UID 1000), sets the Environment, Session and network variables and starts `oac-daemon connect --profile default --bootstrap-file /home/runtime/runtime-bootstrap.json` as UID/GID 1000. It records process handoff in `/root/.oac/e2b/managed-ready.json` and refuses to run again once any launch record exists. `BootstrapComplete` becomes true when a later inspection reads that record with the expected identity; it does not prove enrollment or native readiness. +`managed_init.py` prepares the image as the [application-managed startup](../../deploy/e2b/README.md#startup-and-security-boundary) does, writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json` (mode 0600, owned by UID 1000), sets the Environment, Session and network variables and starts `oac-daemon connect --profile default --bootstrap-file /home/runtime/runtime-bootstrap.json` as UID/GID 1000. It records process handoff in `/root/.oac/e2b/managed-ready.json` and refuses to run again once any launch record exists. `BootstrapComplete` becomes true when Core validates that record with the expected identity. The initialization command returns the bounded durable receipt on its existing output stream, avoiding a separate file read on successful Create; Core still rereads cloud ownership and configuration before returning. If the command response is lost or its receipt is unreadable, inspection can recover the record without replaying startup; it does not prove enrollment or native readiness. An unknown Create is never repeated. A Create whose connection material was lost can be discovered and destroyed but cannot resume bootstrap, and an unconfirmed startup requires reclaiming the whole allocation. diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index 77d1869fd..0b7635c78 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -11,7 +11,7 @@ MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 1 +PROTOCOL_VERSION = 2 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] diff --git a/services/core/tools/e2b-provider/observation_test.py b/services/core/tools/e2b-provider/observation_test.py index ef204d6d7..f72554e8d 100644 --- a/services/core/tools/e2b-provider/observation_test.py +++ b/services/core/tools/e2b-provider/observation_test.py @@ -15,7 +15,7 @@ def observe(self, reference=None): try: return Provider(request).execute() except Failure as error: - return {'Version': 1, 'ErrorCode': error.code} + return {'Version': 2, 'ErrorCode': error.code} def listing(self, *pages): pages = [list(page) for page in pages] or [[]] diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index 76557b500..264b39489 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -7,7 +7,7 @@ from datetime import datetime, timezone from uuid import UUID -from e2b import Sandbox, SandboxQuery, SandboxState +from e2b import FileType, Sandbox, SandboxQuery, SandboxState from e2b.api.client.models.sandbox_metric import SandboxMetric from e2b.exceptions import AuthenticationException, FileNotFoundException, SandboxNotFoundException @@ -19,6 +19,20 @@ PREFIX = 'oac_' FIELDS = ('InstallationID', *REFERENCE_FIELDS) +# Execute the existing protected entry point and return its durable receipt on +# the same command stream. A read failure leaves successful startup recoverable +# through Inspect, without replaying initialization. +BOOTSTRAP_SCRIPT = """import runpy,sys +runpy.run_path('/opt/oac-e2b/managed_init.py', run_name='__main__') +try: + with open('/root/.oac/e2b/managed-ready.json', 'rb') as source: + receipt = source.read(4097) + if len(receipt) <= 4096: + sys.stdout.buffer.write(receipt) +except OSError: + pass +""" + def valid_id(value): try: @@ -196,13 +210,17 @@ def inspect(self): except FileNotFoundException: receipt = None if receipt is not None: - expected = record.get('bootstrap_identity') - if (receipt.get('identity') != expected or receipt.get('status') != 'daemon_started' or - type(receipt.get('daemon_pid')) is not int or receipt['daemon_pid'] <= 0): - raise Failure('ownership') - self.receipt.save(settled=True, bootstrap_complete=True) + self.accept_bootstrap_receipt(receipt) return cloud + def accept_bootstrap_receipt(self, receipt): + expected = (self.receipt.data or {}).get('bootstrap_identity') + if (not isinstance(receipt, dict) or receipt.get('identity') != expected or + receipt.get('status') != 'daemon_started' or + type(receipt.get('daemon_pid')) is not int or receipt['daemon_pid'] <= 0): + raise Failure('ownership') + self.receipt.save(settled=True, bootstrap_complete=True) + def create(self): if self.receipt.data is not None: raise Failure('exists') @@ -233,27 +251,40 @@ def create(self): self.receipt.save(status='configuration_rejected', settled=True) raise # Validate the current template entry point before writing any credential. - check = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', - "import os,sys; sys.exit(78 if not os.path.isfile('/opt/oac-e2b/managed_init.py') or not os.access('/opt/oac-e2b/managed_init.py', os.R_OK) else 0)"]}, - self.remaining, user='root') - if check['ExitCode'] != 0: + try: + entry = cloud.files.get_info('/opt/oac-e2b/managed_init.py', + user='root', request_timeout=self.remaining()) + if entry.type != FileType.FILE: + self.receipt.save(status='bootstrap_failed', settled=True) + raise Failure('template_invalid') + # A successful download can also refer to a special file. Check + # the type first, then verify readability without a probe process. + with cloud.files.read('/opt/oac-e2b/managed_init.py', format='stream', + user='root', request_timeout=self.remaining()): + pass + except FileNotFoundException: self.receipt.save(status='bootstrap_failed', settled=True) - raise Failure('template_invalid' if check['ExitCode'] == 78 else 'unconfirmed') + raise Failure('template_invalid') from None payload = dict(bootstrap, InstallationID=self.config['InstallationID'], RuntimeBootstrap=self.q['RuntimeBootstrap']) - del payload['CoreURL'], payload['Credential'] + del payload['CoreURL'], payload['Credential'], payload['Harness'] if (set(payload) - set(MANAGED_BOOTSTRAP_FIELDS) or not set(MANAGED_BOOTSTRAP_REQUIRED_FIELDS) <= set(payload)): raise Failure('invalid') cloud.files.write('/root/.oac/e2b/managed-bootstrap.json', json.dumps(payload), user='root', request_timeout=self.remaining()) self.receipt.save(status='bootstrap_pending') - result = run(cloud, {'Args': ['/usr/bin/python3', '-I', '/opt/oac-e2b/managed_init.py']}, + result = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', BOOTSTRAP_SCRIPT]}, self.remaining, user='root') if result['ExitCode'] != 0: self.receipt.save(status='bootstrap_failed', settled=True) raise Failure('unconfirmed') self.receipt.save(status='bootstrap_exited', settled=True) + try: + receipt = json.loads(result['Stdout']) + except (ValueError, KeyError): + raise Failure('unconfirmed') from None + self.accept_bootstrap_receipt(receipt) return self.inspect() def renew(self): diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 74d708547..240e05eab 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -1,22 +1,63 @@ """Controlled SDK boundary failures; live qualification remains separate.""" import copy +import io +import httpx from datetime import datetime, timedelta, timezone import json from pathlib import Path import tempfile +import subprocess +import sys from types import SimpleNamespace import unittest -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch from uuid import uuid4 -from e2b import SandboxState -from e2b.exceptions import AuthenticationException, SandboxNotFoundException +from e2b import FileType, SandboxState +from e2b.connection_config import ConnectionConfig +from e2b.sandbox_sync.filesystem.filesystem import Filesystem +from packaging.version import Version +from e2b.exceptions import AuthenticationException, FileNotFoundException, SandboxNotFoundException -from provider import Provider +from provider import BOOTSTRAP_SCRIPT, Provider +from helper_contract_generated import PROTOCOL_VERSION from sdk import restore, run from state import Failure, Receipt +class BootstrapScriptTest(unittest.TestCase): + def execute(self, startup, receipt=None): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + entry = root / 'managed_init.py' + ready = root / 'managed-ready.json' + entry.write_text(startup.replace('READY_PATH', repr(str(ready)))) + if receipt is not None: + ready.write_bytes(receipt) + script = BOOTSTRAP_SCRIPT.replace('/opt/oac-e2b/managed_init.py', str(entry)).replace( + '/root/.oac/e2b/managed-ready.json', str(ready)) + return subprocess.run([sys.executable, '-I', '-c', script], + capture_output=True, timeout=5) + + def test_success_reads_receipt_after_initialization(self): + result = self.execute("from pathlib import Path; Path(READY_PATH).write_bytes(b'new receipt')", + receipt=b'stale receipt') + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, b'new receipt') + + def test_failed_initialization_cannot_return_old_receipt(self): + result = self.execute('raise SystemExit(7)', receipt=b'stale receipt') + self.assertEqual(result.returncode, 7) + self.assertEqual(result.stdout, b'') + + def test_missing_and_oversized_receipts_leave_startup_recoverable(self): + for receipt in (None, b'x' * 4097): + with self.subTest(size=len(receipt) if receipt else None): + result = self.execute('pass', receipt=receipt) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, b'') + + class ProviderTest(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() @@ -24,13 +65,13 @@ def setUp(self): self.reference = {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']} self.config = {'StateDir': self.temporary.name, 'InstallationID': str(uuid4()), 'APIKey': 'private-account-secret', 'Template': 'test:' + str(uuid4()), 'TimeoutSeconds': 120} - self.request = {'Version': 1, 'Operation': 'create', 'Config': self.config, + self.request = {'Version': 2, 'Operation': 'create', 'Config': self.config, 'Reference': self.reference, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=30)).isoformat(), 'Bootstrap': dict(self.reference, SessionID=str(uuid4()), DeviceID=str(uuid4()), CoreURL='https://core.example/api/v1', Credential='private-runtime-secret', - NetworkAccess='enabled', AllowedDomains=[])} + Harness='codex', NetworkAccess='enabled', AllowedDomains=[])} self.request['RuntimeBootstrap'] = { - 'version': 1, 'core_url': self.request['Bootstrap']['CoreURL'], + 'version': 2, 'harness': 'codex', 'core_url': self.request['Bootstrap']['CoreURL'], 'device_id': self.request['Bootstrap']['DeviceID'], 'credential': self.request['Bootstrap']['Credential']} self.cloud = Mock(sandbox_id='owned-id', sandbox_domain='e2b.app', _envd_version='0.5.0', @@ -40,7 +81,12 @@ def setUp(self): self.identity = dict(self.reference, InstallationID=self.config['InstallationID'], SessionID=self.request['Bootstrap']['SessionID'], DeviceID=self.request['Bootstrap']['DeviceID']) self.ready = json.dumps({'identity': self.identity, 'status': 'daemon_started', 'daemon_pid': 123}) + self.cloud.files.get_info.return_value = SimpleNamespace(type=FileType.FILE) + self.template_stream = MagicMock() self.cloud.files.read.return_value = self.ready + self.cloud.files.read.side_effect = lambda path, **kwargs: ( + self.template_stream if path == '/opt/oac-e2b/managed_init.py' + else self.cloud.files.read.return_value) self.api = Mock() self.api.create.return_value = self.cloud self.api.get_info.return_value = self.cloud @@ -51,7 +97,8 @@ def setUp(self): self.runtime = patch('provider.restore', return_value=self.cloud) self.runtime.start() self.addCleanup(self.runtime.stop) - self.command = patch('provider.run', return_value={'Stdout': '', 'Stderr': '', 'ExitCode': 0}) + self.command = patch('provider.run', side_effect=lambda *a, **k: { + 'Stdout': self.cloud.files.read.return_value, 'Stderr': '', 'ExitCode': 0}) self.command.start() self.addCleanup(self.command.stop) @@ -108,6 +155,7 @@ def test_create_refuses_other_owner_before_credentials(self): self.assertEqual(self.call('create')['ErrorCode'], 'ownership') self.api.get_info.assert_called_once() self.cloud.files.write.assert_not_called() + self.cloud.files.read.assert_not_called() def test_qualified_gateway_template_id_accepts_only_selected_build(self): self.config['Resources'] = {'cpus': 2, 'memory_mib': 2048} @@ -145,7 +193,7 @@ def test_create_refuses_foreign_data_plane_before_envd(self): self.cloud.commands.run.assert_not_called() def test_template_invalid_refuses_before_credentials_and_retains_owned_cleanup(self): - with patch('provider.run', return_value={'ExitCode': 78, 'Stdout': '', 'Stderr': ''}): + with patch.object(self.cloud.files, 'read', side_effect=FileNotFoundException('missing')): result = self.call('create') self.assertEqual(result['ErrorCode'], 'template_invalid') self.assertTrue(result['Info']['CreateSettled']) @@ -155,6 +203,75 @@ def test_template_invalid_refuses_before_credentials_and_retains_owned_cleanup(s self.api.kill.assert_not_called() self.assertEqual(self.call('create')['ErrorCode'], 'exists') + def test_template_non_regular_files_are_rejected_before_open_or_credentials(self): + for kind in (FileType.DIR, None): + with self.subTest(kind=kind): + self.reference['AllocationID'] = str(uuid4()) + self.request['Bootstrap']['AllocationID'] = self.reference['AllocationID'] + self.cloud.metadata = Provider(self.request).metadata + self.cloud.files.get_info.return_value = SimpleNamespace(type=kind) + self.assertEqual(self.call('create')['ErrorCode'], 'template_invalid') + self.cloud.files.read.assert_not_called() + self.cloud.files.write.assert_not_called() + + def test_template_read_is_closed_and_does_not_launch_a_probe(self): + with patch('provider.run', return_value={'ExitCode': 0, 'Stdout': self.ready}) as command: + self.assertEqual(self.call('create')['ErrorCode'], '') + self.template_stream.__enter__.assert_called_once() + self.template_stream.__exit__.assert_called_once() + self.template_stream.__iter__.assert_not_called() + command.assert_called_once() + self.assertEqual(command.call_args.args[1]['Args'], + ['/usr/bin/python3', '-I', '-c', BOOTSTRAP_SCRIPT]) + args, options = self.cloud.files.read.call_args_list[0] + self.assertEqual(args, ('/opt/oac-e2b/managed_init.py',)) + self.assertEqual(options['format'], 'stream') + self.assertEqual(options['user'], 'root') + self.assertGreater(options['request_timeout'], 0) + + def test_template_probe_uses_sdk_stream_and_closes_without_downloading(self): + class Body(httpx.SyncByteStream): + closed = False + + def __iter__(self): + raise AssertionError('template contents must not be downloaded') + yield b'' + + def close(self): + self.closed = True + + body = Body() + requests = [] + + def respond(request): + requests.append(request) + return httpx.Response(200, stream=body) + + with httpx.Client(base_url='https://fixture.invalid', + transport=httpx.MockTransport(respond)) as client: + with patch('e2b.sandbox_sync.filesystem.filesystem.get_envd_api', return_value=client), \ + patch('e2b.sandbox_sync.filesystem.filesystem.create_rpc_client'): + files = Filesystem('https://fixture.invalid', Version('0.5.0'), + ConnectionConfig(api_key='fixture'), client) + original_read = self.cloud.files.read.side_effect + self.cloud.files.read.side_effect = lambda path, **kwargs: ( + files.read(path, **kwargs) if path == '/opt/oac-e2b/managed_init.py' + else original_read(path, **kwargs)) + self.assertEqual(self.call('create')['ErrorCode'], '') + self.assertTrue(body.closed) + self.assertEqual(len(requests), 1) + self.assertEqual(requests[0].method, 'GET') + self.assertEqual(requests[0].url.params['path'], '/opt/oac-e2b/managed_init.py') + self.assertEqual(requests[0].url.params['username'], 'root') + + def test_uncertain_template_read_never_writes_credentials_or_replays_create(self): + with patch.object(self.cloud.files, 'read', side_effect=TimeoutError('private secret')): + self.assertEqual(self.call('create')['ErrorCode'], 'unconfirmed') + self.cloud.files.write.assert_not_called() + self.assertFalse(self.record().get('bootstrap_complete', False)) + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + self.api.create.assert_called_once() + def test_unknown_create_empty_lookup_never_proves_cleanup(self): self.api.create.side_effect = TimeoutError('confidential SDK diagnostic') self.assertEqual(self.call('create')['ErrorCode'], 'unconfirmed') @@ -239,6 +356,23 @@ def test_mismatched_receipt_cannot_prove_bootstrap_complete(self): self.assertTrue(self.record()['settled']) self.assertFalse(self.record()['bootstrap_complete']) + def test_successful_create_returns_receipt_without_remote_receipt_read(self): + self.assertTrue(self.call('create')['Info']['BootstrapComplete']) + self.assertEqual([call.args[0] for call in self.cloud.files.read.call_args_list], + ['/opt/oac-e2b/managed_init.py']) + self.assertEqual(self.api.get_info.call_count, 2) + + def test_lost_command_receipt_recovers_from_file_without_replaying_startup(self): + with patch('provider.run', return_value={'ExitCode': 0, 'Stdout': ''}) as command: + result = self.call('create') + self.assertEqual(result['ErrorCode'], 'unconfirmed') + self.assertTrue(result['Info']['CreateSettled']) + self.assertFalse(result['Info']['BootstrapComplete']) + self.assertTrue(self.call('inspect')['Info']['BootstrapComplete']) + self.assertEqual(self.call('create')['ErrorCode'], 'exists') + command.assert_called_once() + self.api.create.assert_called_once() + def test_foreign_owner_prevents_renew_and_delete(self): self.call('create') self.cloud.metadata = {} diff --git a/services/core/tools/e2b-provider/state_test.py b/services/core/tools/e2b-provider/state_test.py index ae573370a..9ff5b89b4 100644 --- a/services/core/tools/e2b-provider/state_test.py +++ b/services/core/tools/e2b-provider/state_test.py @@ -18,7 +18,7 @@ class StateTest(unittest.TestCase): def setUp(self): self.root = tempfile.TemporaryDirectory() self.addCleanup(self.root.cleanup) - self.request = {'Version': 1, 'Operation': 'kill', + self.request = {'Version': 2, 'Operation': 'kill', 'Config': {'StateDir': self.root.name, 'InstallationID': str(uuid4()), 'APIKey': 'test'}, 'Reference': {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']}, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=2)).isoformat()} diff --git a/services/core/tools/e2b-provider/testdata/contract.json b/services/core/tools/e2b-provider/testdata/contract.json index 80d09bf84..0efaa314d 100644 --- a/services/core/tools/e2b-provider/testdata/contract.json +++ b/services/core/tools/e2b-provider/testdata/contract.json @@ -1,66 +1,66 @@ [ -{"kind":"managed","name":"disabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"disabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"enabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":"example.com","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":[1],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"unknown","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":true,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-AllocationID","payload":{"AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"disabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"disabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"enabled","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":"example.com","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":[1],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"unknown","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":true,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-AllocationID","payload":{"AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-AllowedDomains","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-NetworkAccess","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, {"kind":"managed","name":"missing-RuntimeBootstrap","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444"},"valid":false}, -{"kind":"managed","name":"restricted","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":["example.com"],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"restricted","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, -{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, -{"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"create","payload":{"Version":1,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"inspect","payload":{"Version":1,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"kill","payload":{"Version":1,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_builds","payload":{"Version":1,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_templates","payload":{"Version":1,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"renew","payload":{"Version":1,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"validate_deployment","payload":{"Version":1,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-0","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-32","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-33","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":true},"valid":false}, -{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":null},"valid":false}, -{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":"1"},"valid":false}, -{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":2},"valid":false}, -{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":true}, -{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"response","name":"error-","payload":{"Version":1,"ErrorCode":""},"valid":true}, -{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":1,"ErrorCode":"command_unconfirmed"},"valid":true}, -{"kind":"response","name":"error-exists","payload":{"Version":1,"ErrorCode":"exists"},"valid":true}, -{"kind":"response","name":"error-invalid","payload":{"Version":1,"ErrorCode":"invalid"},"valid":true}, -{"kind":"response","name":"error-not_found","payload":{"Version":1,"ErrorCode":"not_found"},"valid":true}, -{"kind":"response","name":"error-ownership","payload":{"Version":1,"ErrorCode":"ownership"},"valid":true}, -{"kind":"response","name":"error-team_mismatch","payload":{"Version":1,"ErrorCode":"team_mismatch"},"valid":true}, -{"kind":"response","name":"error-template_invalid","payload":{"Version":1,"ErrorCode":"template_invalid"},"valid":true}, -{"kind":"response","name":"error-unauthorized","payload":{"Version":1,"ErrorCode":"unauthorized"},"valid":true}, -{"kind":"response","name":"error-unconfirmed","payload":{"Version":1,"ErrorCode":"unconfirmed"},"valid":true}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":1},"valid":false}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":1},"valid":false}, -{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":1},"valid":false}, +{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444"},"valid":false}, +{"kind":"managed","name":"restricted","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":["example.com"],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"restricted","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, +{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, +{"kind":"request","name":"command","payload":{"Version":2,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"create","payload":{"Version":2,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"inspect","payload":{"Version":2,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill","payload":{"Version":2,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_builds","payload":{"Version":2,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_templates","payload":{"Version":2,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"observe","payload":{"Version":2,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"renew","payload":{"Version":2,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"validate_deployment","payload":{"Version":2,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-0","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-32","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-33","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":true},"valid":false}, +{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":null},"valid":false}, +{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":"1"},"valid":false}, +{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":3},"valid":false}, +{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":true}, +{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"response","name":"error-","payload":{"Version":2,"ErrorCode":""},"valid":true}, +{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":2,"ErrorCode":"command_unconfirmed"},"valid":true}, +{"kind":"response","name":"error-exists","payload":{"Version":2,"ErrorCode":"exists"},"valid":true}, +{"kind":"response","name":"error-invalid","payload":{"Version":2,"ErrorCode":"invalid"},"valid":true}, +{"kind":"response","name":"error-not_found","payload":{"Version":2,"ErrorCode":"not_found"},"valid":true}, +{"kind":"response","name":"error-ownership","payload":{"Version":2,"ErrorCode":"ownership"},"valid":true}, +{"kind":"response","name":"error-team_mismatch","payload":{"Version":2,"ErrorCode":"team_mismatch"},"valid":true}, +{"kind":"response","name":"error-template_invalid","payload":{"Version":2,"ErrorCode":"template_invalid"},"valid":true}, +{"kind":"response","name":"error-unauthorized","payload":{"Version":2,"ErrorCode":"unauthorized"},"valid":true}, +{"kind":"response","name":"error-unconfirmed","payload":{"Version":2,"ErrorCode":"unconfirmed"},"valid":true}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":2},"valid":false}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":2},"valid":false}, +{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":2},"valid":false}, {"kind":"response","name":"invalid-Version","payload":{"ErrorCode":"","Version":true},"valid":false} ] diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index ac7e22c2c..58e190c59 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -1,6 +1,6 @@ # microsandbox Sandbox Provider helper -microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 4; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. +microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 5; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. [Add a Sandbox Provider](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources, Runtime release and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy. @@ -64,3 +64,5 @@ The helper returns only the native observation time, exact uptime, cumulative vC ## Tests `make check-microsandbox-provider`, part of `make check`, runs the pure-Go adapter tests everywhere and this module's tests on Linux; other hosts print an explicit skip for the Linux-only module. + +The native helper wire uses version 5 and carries the Session-selected Harness into Runtime bootstrap version 2. Upgrade the helper with its node and Core. From 96bcbcd2534a2083b1bbf869da8b3d8e7ddf606f Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 10 Oct 2026 23:25:15 +0800 Subject: [PATCH 2/4] test(core): account for capability-driven scheduler wakeups --- .../tests/integration/environment_worker_scan_test.go | 8 ++++---- services/core/tests/integration/worker_wakeup_test.go | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/services/core/tests/integration/environment_worker_scan_test.go b/services/core/tests/integration/environment_worker_scan_test.go index 5149190ff..779ff5cbb 100644 --- a/services/core/tests/integration/environment_worker_scan_test.go +++ b/services/core/tests/integration/environment_worker_scan_test.go @@ -8,7 +8,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -func TestWorkerEnvironmentRetriesNewlyReadyAtNextScan(t *testing.T) { +func TestWorkerEnvironmentRetriesNewlyReadyOnCapabilityHint(t *testing.T) { h := newDispatchHarness(t) enableWorkerEnvironment(t, h) pending := workerEnvironmentReservation(t, h) @@ -20,7 +20,6 @@ func TestWorkerEnvironmentRetriesNewlyReadyAtNextScan(t *testing.T) { h.session = publicSession(t, h, "scan-barrier") receipt := h.message("barrier", "ordinary work") - scanned := time.Now() _, stop := startEnvironmentExpiryWorker(t, h.s, h.d) // Dispatch starts only after selectWork has examined the pending input on // the same pass, while its exact Runtime is still incapable of preparation. @@ -28,13 +27,14 @@ func TestWorkerEnvironmentRetriesNewlyReadyAtNextScan(t *testing.T) { if barrier.ID != receipt.TurnID { t.Fatal("unexpected scan barrier") } + readyAt := time.Now() awaitFixtureCapabilities(t, runtime, workerEnvironmentCapabilities()) h.write(barrier.ID, proto.TypeDone, proto.DonePayload{Content: "complete"}) waitTurn(t, h, barrier.ID, sessions.TurnCompleted) prepare := nextWorkerFrame(t, frames, proto.TypeExecutionPrepare) - if elapsed := time.Since(scanned); elapsed < 750*time.Millisecond || elapsed > 3*time.Second { - t.Fatal("readiness retry must use the next scan, without an empty scan interval", elapsed) + if elapsed := time.Since(readyAt); elapsed >= 750*time.Millisecond { + t.Fatal("capability hint waited for the periodic scan", elapsed) } if workerRuntimeForPreparation(t, h, prepare) != runtime { t.Fatal("readiness retry moved Runtime ownership") diff --git a/services/core/tests/integration/worker_wakeup_test.go b/services/core/tests/integration/worker_wakeup_test.go index 5ab16df8d..cea2990fd 100644 --- a/services/core/tests/integration/worker_wakeup_test.go +++ b/services/core/tests/integration/worker_wakeup_test.go @@ -76,6 +76,12 @@ func TestWorkerSchedulerCommittedAdmissionWakesBeforeMaintenance(t *testing.T) { if err != nil { t.Fatal(err) } + // Fixture Runtime enrollment also emits a capability hint. Isolate + // the admission signal under test from that already committed work. + select { + case <-h.d.Registry.CapabilityHints(): + default: + } trace.armed.Store(true) started = true go func() { done <- worker.Run(ctx) }() From f3b50eeb20f42a1c47c6271592c8fec6c43eaaa5 Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 10 Oct 2026 23:58:44 +0800 Subject: [PATCH 3/4] feat(sandbox): support native E2B suspension and retained capacity --- .../agents-api/node-generation-protocol.md | 15 +- .../agents-api/zh/node-generation-protocol.md | 17 +- deploy/compose/compose.yaml | 2 + deploy/compose/test_compose.py | 9 +- docs/configuration.md | 8 +- docs/getting-started/operations.md | 4 - docs/runtime-bootstrap.md | 4 + docs/sandbox-provider.md | 44 ++-- docs/zh/configuration.md | 10 +- docs/zh/getting-started/operations.md | 6 +- docs/zh/runtime-bootstrap.md | 6 +- docs/zh/sandbox-provider.md | 48 ++-- internal/runtimebootstrap/bootstrap.go | 3 + .../server/managed_generation_operations.go | 20 +- .../cmd/server/managed_generations_test.go | 8 +- services/core/cmd/server/managed_nodes.go | 2 +- services/core/cmd/server/managed_setup.go | 4 +- .../server/managed_setup_preflight_test.go | 21 +- .../core/cmd/server/managed_setup_test.go | 15 +- .../core/cmd/server/sandbox_capacity_test.go | 16 ++ services/core/deploy/e2b/build-template.py | 2 + .../core/deploy/e2b/build_template_test.py | 3 + .../deploy/e2b/helper_contract_generated.py | 13 +- services/core/deploy/e2b/init.py | 2 + services/core/deploy/e2b/init_test.py | 4 +- services/core/deploy/e2b/managed_init.py | 7 + services/core/deploy/e2b/managed_init_test.py | 8 +- .../db/queries/runtime_allocations.sql | 4 +- .../db/queries/runtime_suspension.sql | 25 +- .../db/sqlc/runtime_allocations.sql.go | 6 +- .../db/sqlc/runtime_suspension.sql.go | 51 +++- .../core/internal/deployment/activity_test.go | 27 +-- .../core/internal/deployment/allocation.go | 6 +- .../core/internal/deployment/execution.go | 14 ++ .../core/internal/deployment/fakes_test.go | 67 +++-- services/core/internal/deployment/service.go | 2 +- services/core/internal/deployment/storage.go | 8 + .../archive_cancellation_cleanup_test.go | 2 +- .../execution/deployment_fixture_test.go | 58 +++-- .../provider_operations_fixture_test.go | 13 +- .../internal/execution/runtime_compute.go | 154 ++++++++---- .../runtime_compute_compatibility_test.go | 60 +++++ .../execution/runtime_compute_wake.go | 17 +- .../execution/runtime_fresh_hint_test.go | 44 ++-- .../internal/execution/runtime_lifecycle.go | 48 +++- .../runtime_workspace_settlement_test.go | 2 +- .../execution/runtime_workspace_test.go | 10 +- services/core/internal/execution/worker.go | 4 + .../postgres/deploymentpg/allocations.go | 20 +- .../persistence/postgres/deploymentpg/tx.go | 4 + .../core/internal/processconfig/config.go | 6 + .../processconfig/sandbox_capacity.go | 41 ++++ .../processconfig/sandbox_capacity_test.go | 53 ++++ .../internal/sandbox/docker/operations.go | 13 +- .../internal/sandbox/e2b/helper_contract.go | 31 ++- .../sandbox/e2b/installed_paths_test.go | 3 +- .../sandbox/e2b/internal/contractgen/main.go | 40 ++- .../core/internal/sandbox/e2b/operations.go | 49 +--- .../core/internal/sandbox/e2b/process_test.go | 9 +- .../core/internal/sandbox/e2b/provider.go | 12 +- .../core/internal/sandbox/e2b/suspension.go | 115 +++++++++ .../internal/sandbox/e2b/suspension_test.go | 48 ++++ .../internal/sandbox/e2b/workspace_test.go | 8 + .../microsandbox/creation_settlement_test.go | 2 +- .../sandbox/microsandbox/operations.go | 3 +- .../sandbox/microsandbox/process_test.go | 6 +- .../internal/sandbox/microsandbox/provider.go | 33 +-- .../sandbox/microsandbox/provider_test.go | 14 +- .../sandbox/microsandbox/suspension.go | 146 +++++++++++ .../microsandbox/suspension_contract_test.go | 57 +++++ .../internal/sandbox/microsandbox/types.go | 54 ++++- .../sandbox/microsandbox/workspace_test.go | 10 +- .../internal/sandbox/node/generation_json.go | 4 +- .../sandbox/node/generation_json_test.go | 4 +- .../core/internal/sandbox/node/node_test.go | 6 +- .../core/internal/sandbox/node/operations.go | 3 +- .../internal/sandbox/node/operations_test.go | 56 ++++- .../node/provider_operations_fixture_test.go | 16 +- services/core/internal/sandbox/node/proxy.go | 12 +- services/core/internal/sandbox/node/wire.go | 41 ++-- .../core/internal/sandbox/operations_test.go | 2 +- .../sandbox/providers/registration.go | 5 - .../sandbox/providers/registration_test.go | 8 +- .../internal/sandbox/providers/registry.go | 4 +- .../sandbox/providers/registry_test.go | 10 +- .../core/internal/sandbox/sandbox_provider.go | 97 ++++++-- .../internal/sandbox/sandbox_provider_test.go | 2 +- .../core/internal/sandbox/suspension_test.go | 55 +++++ .../admin_session_archive_worker_http_test.go | 2 +- .../archived_cancellation_migration_test.go | 2 + .../provider_operations_fixture_test.go | 18 +- .../provider_registration_migration_test.go | 2 + .../runtime_compute_lifecycle_test.go | 216 +++++++++++++++-- .../integration/runtime_idle_clock_test.go | 23 +- .../runtime_node_lifecycle_fixture_test.go | 12 +- .../integration/runtime_suspension_test.go | 72 +++++- .../runtime_wake_hint_integration_test.go | 6 +- .../runtime_worker_recovery_test.go | 19 +- .../sandbox_deployment_view_test.go | 2 +- .../integration/sandbox_generations_test.go | 7 +- .../session_execution_configuration_test.go | 2 +- services/core/tools/e2b-provider/README.md | 12 +- .../e2b-provider/helper_contract_generated.py | 13 +- .../tools/e2b-provider/observation_test.py | 3 +- services/core/tools/e2b-provider/provider.py | 11 +- .../core/tools/e2b-provider/provider_test.py | 2 +- .../core/tools/e2b-provider/state_test.py | 3 +- .../core/tools/e2b-provider/suspension.py | 228 ++++++++++++++++++ .../tools/e2b-provider/suspension_test.py | 170 +++++++++++++ .../tools/e2b-provider/testdata/contract.json | 86 ++++--- .../tools/microsandbox-provider/README.md | 10 +- .../tools/microsandbox-provider/bootstrap.go | 5 +- 112 files changed, 2405 insertions(+), 546 deletions(-) create mode 100644 services/core/cmd/server/sandbox_capacity_test.go create mode 100644 services/core/internal/execution/runtime_compute_compatibility_test.go create mode 100644 services/core/internal/processconfig/sandbox_capacity.go create mode 100644 services/core/internal/processconfig/sandbox_capacity_test.go create mode 100644 services/core/internal/sandbox/e2b/suspension.go create mode 100644 services/core/internal/sandbox/e2b/suspension_test.go create mode 100644 services/core/internal/sandbox/microsandbox/suspension.go create mode 100644 services/core/internal/sandbox/microsandbox/suspension_contract_test.go create mode 100644 services/core/internal/sandbox/suspension_test.go create mode 100644 services/core/tools/e2b-provider/suspension.py create mode 100644 services/core/tools/e2b-provider/suspension_test.py diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index 5b7892cac..cc2d0f76f 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -44,20 +44,21 @@ Each operation carries its own arguments and returns the following result on suc | `command` | `RunCommand` | `command` | `command` | | `observe` | `Observe` | `observation` | `sample` | | `initial` | `Initial` | None | `compute` | -| `new_compute` | `NewCompute` | Positive compute `generation` and optional `snapshot` | `compute` | +| `new_compute` | `NewCompute` | Positive `generation` and optional `retained` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | Exact current `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | None | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` and `command` | `command` | | `suspend` | `Suspend` | `suspend` | `state` | | `resume` | `Resume` | `resume` | `state` | -| `delete_snapshot` | `DeleteSnapshot` | `snapshot` | None | +| `delete_retained` | `DeleteRetained` | `retained` | None | -A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. A node without generation management accepts only its enrolled `deployment_generation`; a generation-managing node runs the request on that generation's provider and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not observe-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. +A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. A node without generation management accepts only its enrolled `deployment_generation`; a generation-managing node runs the request on that generation's provider and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not reconciliation-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. The budget is relative: the node anchors `timeout_ms` to its own clock on receipt and consumes it while the request waits in its queue, so the hosts' clocks need not agree. Core still bounds its own wait. A full node queue closes the connection. -The `response` frame carries `id` and `connection_id`. A successful response carries the result named in the operation table, with no result field for `kill`, `kill_compute` or `delete_snapshot`. A failed response carries an `error_code`: +The `response` frame carries `id` and `connection_id`. A successful response carries the result named in the operation table, with no result field for `kill`, `kill_compute` or `delete_retained`. A failed response carries an `error_code`: | `error_code` | Meaning | | --- | --- | @@ -67,7 +68,7 @@ The `response` frame carries `id` and `connection_id`. A successful response car | `unsupported` | The operation is declared unsupported; see below | | `unconfirmed`, or any other value | The outcome is unknown | -A failed response carries no result, except an `info` that is an exact-reference `CreateSettled` receipt: a confirmed native Create that failed a later check can still prove that the attempt settled. A timeout, a lost response or a disconnect is unavailable or uncertain, never evidence of absence, and Core never replays a mutation after one; it observes the original operation instead. The [Sandbox Provider guide](../../docs/sandbox-provider.md#operation-outcomes-and-retries) defines each outcome. +A failed response carries no result except an exact-provenance Resume target for cleanup (as specified below), or an `info` that is an exact-reference `CreateSettled` receipt: a confirmed native Create that failed a later check can still prove that the attempt settled. A timeout, a lost response or a disconnect is unavailable or uncertain, never evidence of absence, and Core never replays a mutation after one; it observes the original operation instead. The [Sandbox Provider guide](../../docs/sandbox-provider.md#operation-outcomes-and-retries) defines each outcome. Node startup and generation loading validate complete Provider operation declarations before accepting work, and the Core proxy uses the same registered declaration, so an unsupported operation rejects before node resolution or native I/O. The [operation contract](../../docs/sandbox-provider.md#explicit-operation-contracts) owns the inventory. An `unsupported` response carries an `unsupported` object with the exact method `operation` and an authored safe `reason`; the proxy checks both against the request. Missing, malformed or mismatched evidence is an unconfirmed result, never proof that a mutation was rejected. Unsupported stays distinct from observation unavailability and unknown compute or command results, and it neither settles resource ownership nor authorizes a replay. @@ -123,6 +124,6 @@ The readiness classes, one exported error and one code each, are authored in `se Preparation diagnostics keep fixed typed causes. Only artifact transfer, checksum or release-provenance failures report `runtime_download_failed`; the private preparer signals that class through its exit category, without Core or the node parsing stderr. Provider, ownership, cancellation and unclassified failures keep their typed code or `provider_unavailable`. No raw provider text crosses the protocol. -The current wire version is 6. Create bootstrap and Resume requests may carry an optional workspace filesystem binding. The node validates its tenant and Environment against the allocation reference, its immutable ObjectID, and its attachment configuration ID against the supplied configuration before forwarding it. The filesystem resolver validates adapter-native ownership. A missing binding selects owned storage; a present binding cannot fall back. Error responses may retain a Resume target only when its nonempty native ID, name, generation and snapshot provenance match the requested target; this is cleanup evidence, never successful restore. +Creation carries the Session-selected `Bootstrap.Harness` into Runtime bootstrap version 2. Core and nodes use protocol version 7 and require a coordinated upgrade. `Bootstrap.Harness` is required. All ten suspension operations belong to the same `SandboxProvider` and are declared supported or unsupported together; dispatch uses no optional interface. `Observe` reads one allocation per request. -Creation carries the Session-selected `Bootstrap.Harness` into Runtime bootstrap version 2. Core and nodes use protocol version 6; upgrade matching nodes before publishing this Core release. +The current wire version is 7. Create bootstrap and Resume requests may carry an optional workspace filesystem binding. The node validates its tenant and Environment against the allocation reference, its immutable ObjectID, and its attachment configuration ID against the supplied configuration before forwarding it. The filesystem resolver validates adapter-native ownership. A missing binding selects owned storage; a present binding cannot fall back. Error responses may retain a Resume target only when its nonempty native ID, name, generation and retained-state provenance match the requested target; this is cleanup evidence, never successful restore. diff --git a/contracts/agents-api/zh/node-generation-protocol.md b/contracts/agents-api/zh/node-generation-protocol.md index b24278ee8..99998679f 100644 --- a/contracts/agents-api/zh/node-generation-protocol.md +++ b/contracts/agents-api/zh/node-generation-protocol.md @@ -1,7 +1,7 @@ --- title: "沙箱节点协议" source: contracts/agents-api/node-generation-protocol.md -source_hash: 961c7993fa9c75ddadb6604fece87fb556bbcc40d4f190d03d2e90927c5be7bb +source_hash: e9a9e3d1027d797bdf467d867023fa11792866ea93252f94904a66f7be45af1e --- 沙箱节点在其主机上运行 Docker 或 microsandbox Provider，并通过一个 WebSocket 与 Core 相连。Core 通过该连接发送 Provider 操作；节点针对本地 Provider 执行这些操作，并报告就绪状态、主机测量值及其持有的部署代次。Core 始终是唯一的生命周期所有者：节点绝不重试变更操作或调度工作。帧和校验器位于 [`services/core/internal/sandbox/node`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/sandbox/node)（`wire.go`、`generation_wire.go`）；节点用于注册和读取配置的 HTTP 路由位于[机器连接 API](machine-api.md#node-routes)。 @@ -46,20 +46,21 @@ Core 发送包含以下内容的 `request` 帧： | `command` | `RunCommand` | `command` | `command` | | `observe` | `Observe` | `observation` | `sample` | | `initial` | `Initial` | 无 | `compute` | -| `new_compute` | `NewCompute` | 大于零的计算 `generation` 和可选的 `snapshot` | `compute` | +| `new_compute` | `NewCompute` | 大于零的计算 `generation` 和可选的 `retained` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | 精确的当前 `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | 无 | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` 和 `command` | `command` | | `suspend` | `Suspend` | `suspend` | `state` | | `resume` | `Resume` | `resume` | `state` | -| `delete_snapshot` | `DeleteSnapshot` | `snapshot` | 无 | +| `delete_retained` | `DeleteRetained` | `retained` | 无 | -只要 `connection_id`、`owner_epoch` 或 `sequence` 中任一值不匹配，请求就会关闭连接。格式错误的请求会得到 `invalid` 响应。未启用代次管理的节点仅接受其登记的 `deployment_generation`；支持代次管理的节点在对应代次的 Provider 上运行请求，无法运行时回复 `unconfirmed`。Core 仅向节点上已就绪的代次发送 `create` 和非 observe-only 的 `resume`，并且每条连接最多保留 32 个待处理请求。 +只要 `connection_id`、`owner_epoch` 或 `sequence` 中任一值不匹配，请求就会关闭连接。格式错误的请求会得到 `invalid` 响应。未启用代次管理的节点仅接受其登记的 `deployment_generation`；支持代次管理的节点在对应代次的 Provider 上运行请求，无法运行时回复 `unconfirmed`。Core 仅向节点上已就绪的代次发送 `create` 和非 reconciliation-only 的 `resume`，并且每条连接最多保留 32 个待处理请求。 预算采用相对计时：节点收到请求时以自己的时钟为基准锚定 `timeout_ms`，并在请求排队等待期间持续消耗该预算，因此各主机的时钟无需保持一致。Core 仍会限制自身等待时长。节点队列已满时会关闭连接。 -`response` 帧包含 `id` 和 `connection_id`。成功响应携带操作表中指定的结果；对于 `kill`、`kill_compute` 或 `delete_snapshot`，响应不含结果字段。失败响应携带一个 `error_code`： +`response` 帧包含 `id` 和 `connection_id`。成功响应携带操作表中指定的结果；对于 `kill`、`kill_compute` 或 `delete_retained`，响应不含结果字段。失败响应携带一个 `error_code`： | `error_code` | 含义 | | --- | --- | @@ -69,7 +70,7 @@ Core 发送包含以下内容的 `request` 帧： | `unsupported` | 该操作被声明为不支持；见下文 | | `unconfirmed` 或任何其他值 | 结果未知 | -失败响应不携带结果，唯一的例外是作为精确引用 `CreateSettled` 回执的 `info` 结果：即便已确认的原生 Create 在后续检查中失败，仍可证明该尝试已有确定结果。超时、响应丢失或断连属于不可用或不确定情况，绝不能证明资源不存在；发生这些情况后，Core 绝不重放变更操作，而是改为观察原始操作。[Sandbox Provider 指南](../../../docs/zh/sandbox-provider.md#operation-outcomes-and-retries) 定义了每种结果。 +失败响应不携带结果；例外包括下文规定的精确来源 Resume 目标清理证据，以及作为精确引用 `CreateSettled` 回执的 `info` 结果：即便已确认的原生 Create 在后续检查中失败，仍可证明该尝试已有确定结果。超时、响应丢失或断连属于不可用或不确定情况，绝不能证明资源不存在；发生这些情况后，Core 绝不重放变更操作，而是改为观察原始操作。[Sandbox Provider 指南](../../../docs/zh/sandbox-provider.md#operation-outcomes-and-retries) 定义了每种结果。 节点启动和代次加载会在接受工作前验证完整的 Provider 操作声明，Core 代理使用同一份已注册声明，因此不支持的操作会在节点解析或原生 I/O 之前被拒绝。操作清单由[操作契约](../../../docs/zh/sandbox-provider.md#explicit-operation-contracts)维护。`unsupported` 响应包含一个 `unsupported` 对象，其中有精确的方法 `operation` 和经作者编写且安全的 `reason`；代理会将两者与请求进行核对。证据缺失、格式错误或不匹配会得到 `unconfirmed` 结果，而绝不会证明变更操作被拒绝。`unsupported` 始终不同于观察不可用，也不同于计算或命令结果未知；它既不确定资源所有权，也不授权重放。 @@ -125,6 +126,6 @@ Runtime 字节缺失时，绝不将固定的放置实例迁移到当前 Runtime 准备诊断使用固定的类型化原因。只有制品传输、校验和或版本来源验证失败才会报告 `runtime_download_failed`；私有准备器通过退出类别指示这一类失败，Core 和节点都不解析 stderr。Provider 故障、所有权故障、取消和未分类故障保留其类型化代码，或使用 `provider_unavailable`。协议中不会传输任何 Provider 原始文本。 -当前线协议版本为 6。Create 引导和 Resume 请求可携带可选的工作区文件系统绑定。节点在转发前校验其租户及 Environment 与分配引用一致、ObjectID 不可变且有效，以及挂载配置 ID 与所传配置一致。文件系统解析器负责适配器原生所有权校验。未提供绑定时选择自有存储；已提供绑定时不得回退。错误响应仅可保留原生 ID 非空且名称、代次、快照来源均匹配请求的 Resume 目标；这仅为清理证据，不代表恢复成功。 +创建操作通过 `Bootstrap.Harness` 将会话选择传递到 Runtime 启动协议版本 2。Core 和节点使用协议版本 7，需协调升级配套组件。`Bootstrap.Harness` 是必填字段。十项暂停操作均属于同一个 `SandboxProvider`，必须完整声明支持或不支持；不通过可选接口分派。`Observe` 每次只观测一个 allocation。 -创建操作通过 `Bootstrap.Harness` 将会话选择传递到 Runtime 启动协议版本 2。Core 和节点使用协议版本 6；发布此 Core 版本前需升级配套节点。 +当前线协议版本为 7。Create 引导和 Resume 请求可携带可选的工作区文件系统绑定。节点在转发前校验其租户及 Environment 与分配引用一致、ObjectID 不可变且有效，以及挂载配置 ID 与所传配置一致。文件系统解析器负责适配器原生所有权校验。未提供绑定时选择自有存储；已提供绑定时不得回退。错误响应仅可保留原生 ID 非空且名称、代次、保留状态来源均匹配请求的 Resume 目标；这仅为清理证据，不代表恢复成功。 diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml index 70471dc31..d91f0d8cd 100644 --- a/deploy/compose/compose.yaml +++ b/deploy/compose/compose.yaml @@ -59,6 +59,8 @@ services: OAC_CREDENTIAL_KEY_FILE: /run/oac/credential.key OAC_CORE_KEY_DIGESTS_FILE: /run/oac/core-key-digests.json OAC_EXECUTION_CONCURRENCY: ${OAC_EXECUTION_CONCURRENCY:-} + OAC_SANDBOX_MAX_ACTIVE: ${OAC_SANDBOX_MAX_ACTIVE:-} + OAC_SANDBOX_MAX_RETAINED: ${OAC_SANDBOX_MAX_RETAINED:-} OAC_DEFAULT_HARNESS: ${OAC_DEFAULT_HARNESS:-} OAC_HARNESSES: ${OAC_HARNESSES:-} OAC_WRITE_AUDIT_RETENTION: ${OAC_WRITE_AUDIT_RETENTION:-} diff --git a/deploy/compose/test_compose.py b/deploy/compose/test_compose.py index 4f14ef726..7a4fcc6b3 100644 --- a/deploy/compose/test_compose.py +++ b/deploy/compose/test_compose.py @@ -71,9 +71,16 @@ def test_compose_uses_private_services_and_ordered_initialization(self): self.assertEqual(services['web']['healthcheck']['test'], ['CMD', '/usr/local/bin/oac-web', 'healthcheck']) self.assertNotIn('python3', json.dumps(self.compose)) self.assertEqual(services['init']['environment']['OAC_REVISION'], 'd' * 40) - for name in ('OAC_EXECUTION_CONCURRENCY', 'OAC_DEFAULT_HARNESS', 'OAC_HARNESSES', 'OAC_WRITE_AUDIT_RETENTION', 'OAC_LOG_LEVEL'): + for name in ('OAC_SANDBOX_MAX_ACTIVE', 'OAC_SANDBOX_MAX_RETAINED', 'OAC_EXECUTION_CONCURRENCY', 'OAC_DEFAULT_HARNESS', 'OAC_HARNESSES', 'OAC_WRITE_AUDIT_RETENTION', 'OAC_LOG_LEVEL'): self.assertEqual(services['core']['environment'][name], '', name) + def test_direct_sandbox_capacity_reaches_core(self): + from unittest.mock import patch + with patch.dict(os.environ, {'OAC_SANDBOX_MAX_ACTIVE': '17', 'OAC_SANDBOX_MAX_RETAINED': '61'}): + configured = self.render()['services']['core']['environment'] + self.assertEqual(configured['OAC_SANDBOX_MAX_ACTIVE'], '17') + self.assertEqual(configured['OAC_SANDBOX_MAX_RETAINED'], '61') + def test_public_url_can_be_configured_after_initial_startup(self): for value in (None, '', 'https://oac.example.test', 'http://localhost:9080'): with self.subTest(public_url=value): diff --git a/docs/configuration.md b/docs/configuration.md index 02ca6682e..3f4f38c0f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -50,6 +50,8 @@ Model providers are not process settings; see [Default models](#default-models). | `OAC_LOG_FORMAT` | `auto` | `auto`, `text` or `json` | | `OAC_LOG_ADD_SOURCE` | unset | `1` adds source locations | | `OAC_EXECUTION_CONCURRENCY` | `4` | Concurrent execution work, from 1 to 1024 | +| `OAC_SANDBOX_MAX_ACTIVE` | `100` | Active sandbox limit for direct Providers with suspension, from 1 to 100000. Independent of execution concurrency and node capacity | +| `OAC_SANDBOX_MAX_RETAINED` | `400` | Retained sandbox limit for direct Providers, from 1 to 100000, including active, suspended and unconfirmed cleanup. Must be at least the active limit | | `OAC_DEFAULT_HARNESS` | `codex` | Harness used when a request does not name one | | `OAC_HARNESSES` | Every registered Harness | Comma-separated Harnesses to enable besides the default one. Unknown names stop startup | | `OAC_WRITE_AUDIT_RETENTION` | `2160h` | Minimum `1h` | @@ -87,6 +89,10 @@ Runtime settings live in Core's database. Change them in Web; scripts use the sa Which harnesses are enabled, and the default one, are process settings (`core.harnesses`, `core.default_harness`); System shows them read-only. The [Core administration API](../contracts/agents-api/admin-api.md) lists every Core API route, and the [deployment contract](../contracts/agents-api/sandbox-deployment.md) defines the sandbox fields, limits and change rules. +### Direct Provider capacity + +Set `OAC_SANDBOX_MAX_ACTIVE` and `OAC_SANDBOX_MAX_RETAINED` in `.env`, then run `oac apply`. Core uses these limits for direct Providers with suspension enabled. Every unreleased allocation consumes retained capacity. Lowering a limit stops no existing sandbox; new allocations wait until usage falls below both limits. These settings are independent of `OAC_EXECUTION_CONCURRENCY` and enrolled node capacity. + ### Independent workspace storage The initial supported independent filesystem combination is microsandbox with the [kernel NFS adapter](./workspace-provider.md#kernel-nfs-adapter). Mount the same NFSv4.2 export on the Linux Core host and every participating node before starting their services, at the same absolute path, for example `/srv/oac-workspaces`. The operator manages the export, mount availability and service startup ordering. Use a trusted-client AUTH_SYS export with `root_squash`; do not enable `no_root_squash` or broaden permissions to make a check pass. Prepare the namespace and service principals according to the adapter's [ownership requirements](./workspace-provider.md#kernel-nfs-adapter). @@ -193,7 +199,7 @@ Core reads its process environment. Compose interpolates `.env` into it and moun | `OAC_CREDENTIAL_KEY_FILE` | Required. `/run/oac/credential.key`: a base64-encoded random 32-byte key. Core seals stored credentials with it | | `OAC_CORE_KEY_DIGESTS_FILE` | Required. `/run/oac/core-key-digests.json`: a JSON array with the SHA-256 of the Core key | | `OAC_INSTALLATION_ID_FILE` | Required. `/run/oac/installation.id`: the installation ID, a canonical UUID. Core refuses an ID other than the one its database recorded | -| `OAC_EXECUTION_CONCURRENCY`, `OAC_DEFAULT_HARNESS`, `OAC_HARNESSES`, `OAC_WRITE_AUDIT_RETENTION`, `OAC_OAUTH_TRUSTED_ORIGINS`, `OAC_HISTORY_SETTINGS_FILE`, `OAC_LOG_LEVEL`, `OAC_LOG_FORMAT`, `OAC_LOG_ADD_SOURCE` | The matching [process settings](#settings). Web reads the three log settings too | +| `OAC_SANDBOX_MAX_ACTIVE`, `OAC_SANDBOX_MAX_RETAINED`, `OAC_EXECUTION_CONCURRENCY`, `OAC_DEFAULT_HARNESS`, `OAC_HARNESSES`, `OAC_WRITE_AUDIT_RETENTION`, `OAC_OAUTH_TRUSTED_ORIGINS`, `OAC_HISTORY_SETTINGS_FILE`, `OAC_LOG_LEVEL`, `OAC_LOG_FORMAT`, `OAC_LOG_ADD_SOURCE` | The matching [process settings](#settings). Web reads the three log settings too | | `OAC_PROVIDER_ROOT` | Absolute adapter artifact root. The Core image sets `/opt/oac`. Each adapter owns its helper paths beneath this root. Core serves self-hosted daemon installers from its `native-installers/` directory when that holds a `catalog.json`, after checking the catalog against its own release. Adapter state lives at `/state`, the data volume's [`state/`](#compose-installations) | Core logs the history file path it loads, never environment values or file contents. diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index 0ea13768f..023581438 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -190,7 +190,3 @@ Installation and mutating `oac` commands share the [installation lock](../config Web signs administrators in with the Core key, checks the origin of every request, and forwards signed-in `/core/v1` requests to Core with the Core key, which stays on the server. It forwards `/v1` and `/api/v1` to Core unchanged, with the caller's own credential, serves only the non-secret node payload at `/node-install/`, and has no Docker or KVM access. Machine routes under `/api/v1` use their own enrollment and connection credentials. No service receives a Docker socket. Sandboxes are the isolation boundary ([Runtime and outer isolation](../concepts.md#runtime-and-outer-isolation)). Docker sandboxes share the node's kernel, and a Docker node is [root-equivalent](./nodes.md#what-the-installer-sets-up) on its host; microsandbox gives each sandbox a microVM with an explicit [network policy](./nodes.md#what-the-installer-sets-up). Core itself has no Docker socket or KVM access. - -## Runtime availability probes - -Codex CLI availability emits `runtime version probe` records for `process_spawn` and `process_wait`, with duration and outcome only. The wait interval includes executable loading and the version command; neither interval is model execution. diff --git a/docs/runtime-bootstrap.md b/docs/runtime-bootstrap.md index 8284d9f51..5a788ddf5 100644 --- a/docs/runtime-bootstrap.md +++ b/docs/runtime-bootstrap.md @@ -34,6 +34,10 @@ The Runtime validates the input and owns authentication and connection. A succes Self-hosted executors and operator-provisioned devices get their daemon identity in other ways; the [machine connection API](../contracts/agents-api/machine-api.md#credentials) lists every credential source. All of them enter the same Runtime execution loop. +## Hosted suspension control + +The private hosted park/wake control-file path is authored as `SuspendControlFile` in `internal/runtimebootstrap/bootstrap.go`. Core recovery and native Go adapters read that value; the E2B helper contract generator projects it into the template builder. Managed startup prepares its private directory and supplies `OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE` to enable Runtime suspension. This is a packaged protocol setting. The shared Sandbox Provider registration owns idle and retention defaults; the adapter owns its native lease timeout. + ## Verification `go test ./internal/runtimebootstrap ./apps/daemon/internal/cli` covers the input contract, the exclusivity of credential sources and restart behavior. Provider tests verify delivery and file permissions without relying on the Runtime's private storage. diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 149b7c352..6554d8360 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -25,7 +25,7 @@ Core owns durable Environment, allocation, placement and cleanup state; the Prov ## Implement the interface -`sandbox_provider.go` holds the Core–Sandbox Provider protocol: the `SandboxProvider` interface for allocation, checkpoint and observation, its request and result types, and the setup-time `ConfigurationAdapter` with its typed errors. Value types that Core also uses beyond this boundary, such as `DeploymentSpec` and `CallFence`, live in their own files of the same package. Every method is required at compile time, and `ProviderOperations()` declares which ones the Provider supports. Five operations are always supported: +`sandbox_provider.go` holds the Core–Sandbox Provider protocol: the `SandboxProvider` interface for allocation, suspension and observation, its request and result types, and the setup-time `ConfigurationAdapter` with its typed errors. Value types that Core also uses beyond this boundary, such as `DeploymentSpec` and `CallFence`, live in their own files of the same package. Every method is required at compile time, and `ProviderOperations()` declares which ones the Provider supports. Five operations are always supported: | Operation | Purpose | | --- | --- | @@ -45,7 +45,7 @@ Every provider returns a complete `ProviderOperations()` declaration with one en | --- | --- | --- | | `Create`, `GetInfo`, `Renew`, `Kill`, `RunCommand` | Supported | Allocation lifecycle and bounded commands | | `Observe` | Explicit decision | Ownership-checked read-only observation of one allocation | -| `Initial`, `NewCompute`, `GetCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute` | The same decision for every checkpoint method; supported only by a `nodes` registration | Exact compute incarnations, capture and restore, retained-source resume and cleanup | +| `Initial`, `NewCompute`, `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteRetained`, `RunCommandCompute` | The same decision for all ten suspension methods in `nodes` or `direct` mode | Exact compute incarnations, capture and restore, retained-source resume and cleanup | `Initial` and `NewCompute` construct compute references without allocating, `ResumeCompute` thaws only the same resident instance after an aborted pause, and `RunCommandCompute` runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)). @@ -89,7 +89,7 @@ Every call receives a bounded context. Expiry or cancellation ends the caller's `ErrInvalid`, `ErrOwnership`, `ErrExists`, `ErrNotFound`, `ErrComputeUnconfirmed` and `ErrCommandUnconfirmed` keep their defined meanings. An unclassified native or transport error is unknown, never permission to retry a mutation. Core never reads provider diagnostics as lifecycle truth or exposes native error text or credentials; the node transport maps errors to fixed codes, and direct SDK details stay private. -Checkpoint support adds `Compute` generation, name and ID and `SnapshotIdentity`; persist operation IDs and the provider's snapshot provenance unchanged. `ObserveOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and snapshot, not whatever instance now has the same name. Read [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) and its failure tests before declaring checkpoint support. +Suspension support adds `Compute` generation, name and ID and `RetainedState`; persist operation IDs and the provider's retained-state provenance unchanged. `ReconcileOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and retained state, not whatever instance now has the same name. Read [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) and its failure tests before declaring suspension support. ### Four distinct readiness facts @@ -112,7 +112,7 @@ A new provider takes these steps: 2. Add its specification and resource validators. 3. Implement `sandbox.ConfigurationAdapter` over a typed native configuration. `DecodeInput` strictly parses the separate public `configuration` and write-only `credential` objects of a request. `Encode` produces whitelisted public selectors, read-only observations and separate secret bytes, and never passes request JSON through. `Decode` restores stored selectors, and keeps access to owned resources, without remote admission or new template validation. `Normalize` copies its input before changing it. `ResolveChange`, `Equal` and `WithCredential` own inheritance, identity and credential composition. `Requirements` declares whether a credential and a public Core origin are required, and which setup operations are supported: `Discovery` for `DiscoverConfiguration`, `SelectionDiscovery` for `DiscoverSelection` and `CredentialVerification` for `VerifyCredential`. `DiscoverConfiguration` validates the query and returns a safe catalog, never a mutation or an admission decision, while Core keeps authorization, input limits and deadlines. `DiscoverSelection` resolves a candidate's omitted native values before commit, and `VerifyCredential` verifies a credential's access to owned resources without mutation. Both receive the candidate's `sandbox.DirectConfig` and build any native client for that call only. A node provider accepts only an empty public object, rejects credentials and returns Unsupported for every setup operation and for credential replacement. 4. For a node adapter, export from its package the `BuildLocal` constructor, the typed `native` object it decodes and the native files it adds to the shared node artifacts. Node-local settings, such as host paths, live only in that object; resources and the Runtime release are read from the node configuration's `specification`. -5. Register its constructor, policies, configuration adapter and operation declaration in `providers/registry.go`. Its key is the provider kind, which also labels the Provider's observations, and checkpoint support reads this entry. The generated projections combine each registered mode and deployment policy with the shared field bounds in `sandbox/deployment_contract.go`: the installer reads them from `deploy/node/node_spec.py`, and the TypeScript client and Web from `packages/agents-client/src/deployment-contract.ts`, so Web reads these declarations instead of comparing provider kinds. Regenerate both with `go run ./services/core/cmd/specification-contract -write`. +5. Register its constructor, policies, configuration adapter and operation declaration in `providers/registry.go`. Its key is the provider kind, which also labels the Provider's observations, and suspension support reads this entry. The generated projections combine each registered mode and deployment policy with the shared field bounds in `sandbox/deployment_contract.go`: the installer reads them from `deploy/node/node_spec.py`, and the TypeScript client and Web from `packages/agents-client/src/deployment-contract.ts`, so Web reads these declarations instead of comparing provider kinds. Regenerate both with `go run ./services/core/cmd/specification-contract -write`. 6. Supply the distribution artifacts for the adapter and its helper, and offer the provider to operators through the registered configuration contract. **Known design gap:** Web still names providers in the setup wizard's backend choice, the Docker confirmation and E2B's configuration fields wherever Web shows or parses them (the setup step with its service presets, the deployment summary and the client's deployment projection), because the protocol declares no configuration fields yet. Exposing another provider through that surface currently requires a shared Web edit. This coupling does not meet [Complexity stays in the adapter](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#complexity-stays-in-the-adapter); new integrations must express their configuration through the protocol and keep vendor-specific behavior in the adapter. Never add a Session or Turn scheduling path, a vendor column or API field, or a vendor switch in the store. @@ -126,7 +126,7 @@ A node configuration, `sandbox.NodeConfig`, holds `provider`, `generation`, `ins - A `nodes` registration has only `BuildLocal`, and a `direct` registration only `BuildDirect`; missing, mixed or unknown modes are rejected. - The specification and resource validators, the configuration adapter and a complete operation declaration are mandatory, so an incomplete registration cannot publish a partial projection. A declared default size must pass the adapter's resource validator. - The Runtime input policy either accepts the pinned Runtime or gives the adapter's fixed reason for rejecting it, never both. -- Checkpoint is admitted only for a `nodes` registration, because the common lifecycle suspends only node allocations; registration rejects a `direct` Provider that declares it. A registration carries no suspension values: Core applies its one [suspension policy](#suspension) to every Provider that declares checkpoint support. +- Suspension is admitted in both `nodes` and `direct` modes when the complete lifecycle is declared supported. A registration carries no suspension values: Core applies its one [suspension policy](#suspension) to every supported Provider. The configuration adapter must be non-nil, including its concrete value. Every `ConfigurationRequirements` field needs an explicit valid decision: `Credential` and `PublicOrigin` are `Required` or `NotRequired`, and `Discovery`, `SelectionDiscovery` and `CredentialVerification` use the shared supported or unsupported declaration with a safe reason. A new requirement field needs an explicit validation update and never inherits an existing decision. Requiring a credential does not promise the `VerifyCredential` operation. These checks establish complete registration, not correct native SDK behavior; constructor and adapter contract tests still apply. @@ -136,7 +136,7 @@ Preview and persistence use `providers.Normalize` and `providers.Describe`. `pro A direct adapter with a credential verifies all retained generations and allocation references before a key is replaced. The common `sandbox.CallFence` excludes native calls and waits for helper completion, including calls whose callers timed out; execution invokes the prepared verification and fencing callbacks without branching on a vendor. -Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `sandbox.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and a `Quiescent` check when a helper can outlive its caller; generation collection waits for it. The factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes checkpoint operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through the checkpoint declaration, never through a provider name. +Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `sandbox.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and a `Quiescent` check when a helper can outlive its caller; generation collection waits for it. The factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes suspension operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through the suspension declaration, never through a provider name. The backend fingerprint identifies a native resource namespace, not capacity. Core keeps deployment generations so that owned allocations keep resolving to their original backend; never repoint retained allocations at a replacement backend. @@ -166,15 +166,15 @@ The allocation, its dedicated daemon credential digest and the exact Session bin With a configured provider, the Worker scans committed pending hosted Environments that have no allocation, which covers idle Session creation and recovery after an interruption between commit and bootstrap; an existing allocation never re-enters that path. The scan is bounded and serialized by the lifecycle owner and needs no caller action. An initial reservation without a Turn leaves its Session idle, and a daemon connection is never treated as native readiness. The same scan publishes authenticated connection observations with durable generations, after verifying the exact Session and device binding and a settled bootstrap. -Between Turns, Core checks that connected, observed compute is still its Session's running allocation; the check changes nothing and never revives a cleanup request. Running compute never expires: explicit deletion and the snapshot retention authorize its cleanup. A stopped or missing container never authorizes discarding retained workspace or history. Disabling the provider stops new hosted admission and bootstrap but never blocks cancellation, function results or input retry outcomes of existing Sessions. +Between Turns, Core checks that connected, observed compute is still its Session's running allocation; the check changes nothing and never revives a cleanup request. Running compute never expires: explicit deletion and the retained-state retention authorize its cleanup. A stopped or missing container never authorizes discarding retained workspace or history. Disabling the provider stops new hosted admission and bootstrap but never blocks cancellation, function results or input retry outcomes of existing Sessions. Terminal cleanup atomically revokes the device's authority, records the Environment's failure or expiry, settles pending input and requests cancellation, and only then calls `Kill`; original input deadlines and retry outcomes are kept. Temporary provider outages, unknown Create results and stopped compute never prove a permanent failure. After public Session deletion Core keeps the allocation and marks it released only after owned compute and volume cleanup and proof that the original Create settled; an unknown creation keeps cleanup ownership even after an absence observation, and bounded scans continue to catch late resources without another `Create`. ### Per-node lifecycle workers -Each registered node has one serial lifecycle worker that owns its gate, allocation and pending cursors, connections and wake hints; E2B allocations share one serial lifecycle without a node. A thin coordinator discovers nodes and shuts workers down, and never holds its map mutex during database, provider or wait operations. Workers advance independently, so a stuck provider on one online node never stalls another: lifecycle concurrency is one operation per node and grows with the node count. Offline workers stay, so their retained resources remain observable after reconnection. +After a hosted Environment commits, Core sends a bounded hint to the lifecycle worker for its placement. Committed pending input also sends a hint to recover a missed creation notification. Hints reuse the serial gate, execution lease, capacity checks and one-shot allocation receipt; admission handlers never create a sandbox directly. Each normal maintenance period admits at most one extra hinted scan, and periodic scans recover missed or coalesced hints. Existing allocations are observed before pending provisioning, so a slow observation on the same node can still delay a fresh Environment. -After a hosted Environment commits, Core sends a bounded hint to its placement’s lifecycle worker. A committed pending input also sends a hint, recovering a missed creation notification. Hints use the existing serial gate, lease, capacity checks and one-shot allocation receipts; they do not provision inline in the admission handler. Each normal maintenance period permits at most one extra hinted scan, and periodic scans recover missed or coalesced hints. Existing allocation observations still precede pending provisioning, so a slow observation on the same node can delay a fresh Environment. +Each registered node has one serial lifecycle worker that owns its gate, allocation and pending cursors, connections and wake hints; E2B allocations share one serial lifecycle without a node. A thin coordinator discovers nodes and shuts workers down, and never holds its map mutex during database, provider or wait operations. Workers advance independently, so a stuck provider on one online node never stalls another: lifecycle concurrency is one operation per node and grows with the node count. Offline workers stay, so their retained resources remain observable after reconnection. Allocation scans filter by node before their 32-row page limit, and pending scans join the unreleased committed placement. Each node advances its own cursor, including past failed observations, and wraps once at the end. Direct provisioning resolves the tenant-scoped placement before entering that node's gate, and an existing allocation must agree with it; Core never chooses another node. @@ -184,15 +184,31 @@ Before releasing the execution lease, the coordinator stops accepting work and c Placement is automatic: the environment-to-node placement commits with Session creation and its retry identity, callers cannot choose a node, and a retry keeps its original node even while it is offline. Node capacity counts pending reservations and unresolved resources, and new placement and a suspended-to-restoring transition share a database lock. Unknown operations keep their reservations, source teardown must be confirmed before active capacity is released, and confirmed cleanup releases placement capacity. Retained ownership needs exact provider evidence: a socket path, a missing instance or an empty listing never proves cleanup or authorizes a replacement. -The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and the snapshot retention bound each node. There is no node-level drain, cross-node Session migration, multi-active Core, autoscaling or snapshot replication. Node removal is refused while the node holds allocations, snapshots, reservations, unknown results or cleanup, and offline ownership is kept. +The deployment's CPU, memory and disk settings, `max_active`, `max_retained` and the retained-state retention bound each node. There is no node-level drain, cross-node Session migration, multi-active Core, autoscaling or snapshot replication. Node removal is refused while the node holds allocations, retained states, reservations, unknown results or cleanup, and offline ownership is kept. ### Suspension -Core suspends the idle work of every provider that declares checkpoint support, with one fixed policy: it suspends work idle for 5 minutes (300 seconds) and keeps the snapshot for 24 hours (86400 seconds). The deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) reports these values. Core suspends only after at least one Turn is terminal, when no root or Subagent Turn is queued, in progress or waiting, no input, file operation or initialization is pending, and real activity has been idle for that time. For node allocations Core records the first root or child terminal transition with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial snapshot retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. +The single `SandboxProvider` contract declares `Initial`, `NewCompute`, `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `KillCompute`, `DeleteRetained`, `RunCommandCompute` and `ResumeCompute` together: all are supported or all are unsupported. No optional suspension interface or provider-name branch selects this lifecycle. + +`RetainedState` is an opaque adapter-owned envelope with `Reference`, `ID`, `Data`, `OperationID`, `SourceGeneration`, `SourceName` and `SourceID`. `Data` is nonempty and at most 64 KiB. Core preserves the envelope unchanged and validates operation and source ownership; the immutable allocation provider/generation binding selects the only adapter allowed to interpret it. A retained state need not be an independent snapshot. Microsandbox retains its complete native snapshot identity in `Data`; E2B retains its versioned native pause receipt and cannot promise a separate disk image. Each adapter validates its native payload and ownership before any effect. + +`Compute.RestoredFrom`, `ComputeState.Retained`, `ResourcesReleased`, `SuspendSettled` and recovery's `ReconcileOnly` are authoritative settlement evidence. A missing resource is not proof that an unknown suspension settled. `RenewCompute` renews only the exact incarnation; an uncertain result never authorizes a second create, capture or restore. Durable `runtime_compute` uses `protocol_version: "1"`; startup rejects missing or unknown versions. Version-1 envelopes matching the current retained-state field shape remain readable without a rewrite. Snapshot-only durable shapes are rejected even when labeled version 1; equal version numbers do not establish compatibility. Upgrade performs no migration, synthetic replay or retained-state discard. + +`ResumeRequest.Workspace` carries the allocation's independently owned filesystem binding. Core obtains a ready binding before restore. Suspension must prove the source released active execution before a target can become a writer. Adapters validate the binding before native effects; an adapter without independent filesystem support rejects a non-null binding. `DeleteRetained` deletes only adapter-owned retained compute state, never the workspace. Explicit Session deletion first settles compute cleanup and then deletes independent storage, preserving a single active writer throughout recovery. + +Core suspends the idle work of every provider that declares suspension support, with one fixed policy: it suspends work idle for 5 minutes (300 seconds) and keeps retained state for 24 hours (86400 seconds). The deployment's [`suspension`](../contracts/agents-api/sandbox-deployment.md#safe-response) reports these values. Core suspends initialized Environments, including those with no Turn yet, when no root or Subagent Turn is queued, in progress or waiting, no input, file operation or initialization is pending, and real activity has been idle for that time. For allocations Core records initialization completion and root or child terminal transitions with the database clock in the same transaction. Candidate filtering and the Session-locked recheck compare elapsed database time with the idle duration, and the initial retained-state retention deadline is anchored to the same database observation, so Core and database host clocks need not agree. Native completion timestamps stay unchanged in public history but never drive idle admission, and heartbeats never reset activity. Before acknowledging a planned suspension, the daemon closes admission and drains native cleanup, output receipts and file work. + +The Worker lease, the Session lock and the per-node gates own suspension for every provider. New Turn claims, file-write intents and capture admission serialize under the Session lock and share one compute-phase check; new pending work cancels a capture and wakes the same source. Normal preparation waits for the compute phase to be running, after the authenticated resume handshake, and pending input stays pending when its promotion conflicts with a lifecycle transition. Compute phases and revision-checked receipts live on the allocation. Core persists quiesce, capture and restore intent before the effect, only a fresh receipt performs a capture or restore, and recovery observes the exact attempt without retrying an unknown creation, capture or restore. Consumed retained state never rolls a running generation back. Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap, and unknown cleanup identities are kept until owned resources are confirmed absent. Consumed artifacts and old compute are deleted, so suspension cycles never build a chain of writable disks. + +Queued work and live Environment file access wake a suspended Environment; history and published Artifact reads do not. Planned suspension uses an Environment and suspension token on the daemon connection. A PID and start-time fenced local control signal (`RunCommandCompute`) wakes the parked daemon, which authenticates again before admitting work. A transient disconnect before confirmation retries the same armed suspension with bounded attempts and backoff; a permanent authentication or protocol rejection closes it. Core owns the retained state's retention deadline, and the daemon has no timer for it. A lost quiesce acknowledgement may thaw the same source through explicit rollback but never authorizes capturing it. + +`Suspend` owns native resource release. Core releases active capacity only after receiving a bound retained handle, suspended state, `ResourcesReleased` and `SuspendSettled`. `ReconcileOnly` prohibits replaying the original capture or pause but permits adapter cleanup justified by durable retained evidence. An outcome without retained state permits rollback only with `SuspendSettled` and an exact source that can resume; other uncertainty retains ownership and closes admission. Core never unconditionally destroys the source after `Suspend`. + +`Resume` consumes retained state once into the precommitted target. Recovery observes that same attempt. Core persists `waking`, authenticates and resumes the daemon, deletes consumed retained resources, then commits `running` and admits work. `DeleteRetained` is idempotent cleanup that preserves running compute. Cleanup failure keeps `waking` and cannot trigger another restore. `KillCompute` remains destructive; old-generation cleanup must not kill a newer active incarnation sharing its native ID. Every unreleased allocation, including a running one, consumes `max_retained`. Every reservation records the shared compute protocol version, including allocations whose compute phase is `disabled`. Activation rejects an unreleased allocation with a missing or different version; the old version must complete its normal cleanup before upgrade. Session history remains. -The Worker lease, the Session lock and the per-node gates own suspension for every provider. New Turn claims, file-write intents and capture admission serialize under the Session lock and share one compute-phase check; new pending work cancels a capture and wakes the same source. Normal preparation waits for the compute phase to be running, after the authenticated resume handshake, and pending input stays pending when its promotion conflicts with a lifecycle transition. Compute phases and revision-checked receipts live on the allocation. Core persists quiesce, capture and restore intent before the effect, only a fresh receipt performs a capture or restore, and recovery observes the exact attempt without retrying an unknown creation, capture or restore. A consumed snapshot never rolls a running generation back. Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap, and unknown cleanup identities are kept until owned resources are confirmed absent. Consumed artifacts and old compute are deleted, so suspension cycles never build a chain of writable disks. +### Coordinated protocol upgrade -Queued work and live Environment file access wake a suspended Environment; history and published Artifact reads do not. Planned suspension uses an Environment and suspension token on the daemon connection. A PID and start-time fenced local control signal (`RunCommandCompute`) wakes the parked daemon, which authenticates again before admitting work. A transient disconnect before confirmation retries the same armed suspension with bounded attempts and backoff; a permanent authentication or protocol rejection closes it. Core owns the snapshot's retention deadline, and the daemon has no timer for it. A lost quiesce acknowledgement may thaw the same source through explicit rollback but never authorizes capturing it. +Upgrade Core and nodes together with node wire version 7, the E2B helper with private wire version 4 and its rebuilt template containing hosted suspension control, and the microsandbox helper with private wire version 5. All Providers deliver Runtime bootstrap version 2. Each boundary validates its own contract; version numbers are not interchangeable across boundaries. Before activation, use the previous compatible release to complete normal cleanup of incompatible unreleased allocations. The startup fence preserves their stored receipts and Session history and makes no Provider calls to migrate them. ### Reset and archive @@ -200,7 +216,7 @@ A [reset](../contracts/agents-api/sandbox-deployment.md#reset) is durable execut One snapshot and timestamp partition the held resources. Offline ownership comes from the allocation's or active placement's node, with the same 45-second connection and owner-epoch predicate as online presence, independently of provider readiness; no cleanup failure, offline state or empty read authorizes a synthetic release. At zero held resources, Core drains outside database transactions, rechecks under the deployment lock and clears the deployment in one transaction, then publishes a generation-bearing empty provider without fallible work. If the final write or drain fails, it restores the committed provider with a bounded owner context before releasing the mutation gate; if that recovery fails, admission stays fenced and the owner stops. -An administrator [Session archive](../contracts/agents-api/admin-api.md#session-archive) keeps its Project scope and hosted eligibility checks in a Session-first transaction, together with Environment expiry, cancellation, Runtime authority revocation and audit; a reset's background archive reconstructs the actual Project scope from trusted records and keeps the requester's provenance. The ordinary provider lifecycle releases compute and snapshots. Archive cancellation keeps a healthy receipt path until the terminal commit: only the archive that first revokes a device records its exact `archive_cancel_turn_id` (ordinary revocation clears it, and repeated cleanup keeps it), and the existing authenticated delivery may drain that cancellation for at most 20 seconds from the Turn's original `cancel_requested_at`. Core tracks the delivery through `done`, the cancellation acknowledgement and the terminal commit, independently of subscription removal, and grants no new connection, input, file or MCP authority or lease renewal. No transaction or lifecycle gate waits for the receipt, and a lost peer, expiry or restart falls back to ordinary failure and cleanup, never a fabricated cancelled outcome. +An administrator [Session archive](../contracts/agents-api/admin-api.md#session-archive) keeps its Project scope and hosted eligibility checks in a Session-first transaction, together with Environment expiry, cancellation, Runtime authority revocation and audit; a reset's background archive reconstructs the actual Project scope from trusted records and keeps the requester's provenance. The ordinary provider lifecycle releases compute and retained states. Archive cancellation keeps a healthy receipt path until the terminal commit: only the archive that first revokes a device records its exact `archive_cancel_turn_id` (ordinary revocation clears it, and repeated cleanup keeps it), and the existing authenticated delivery may drain that cancellation for at most 20 seconds from the Turn's original `cancel_requested_at`. Core tracks the delivery through `done`, the cancellation acknowledgement and the terminal commit, independently of subscription removal, and grants no new connection, input, file or MCP authority or lease renewal. No transaction or lifecycle gate waits for the receipt, and a lost peer, expiry or restart falls back to ordinary failure and cleanup, never a fabricated cancelled outcome. ## Validate the integration diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 793535d52..18a33cf31 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: eeb840aefb1eab56b16b5185c9fb6b3fcba5f0e9e666eaa32cf9e02f1e2fe134 +source_hash: 2c38905154d559c2a8def738e4514b4b32ca563f34d20d84407af8ad87b210b3 --- Core 安装的每项设置都恰好只有一个归属位置，分属以下三类： @@ -54,6 +54,8 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 | `OAC_LOG_FORMAT` | `auto` | `auto`, `text` or `json` | | `OAC_LOG_ADD_SOURCE` | unset | `1` adds source locations | | `OAC_EXECUTION_CONCURRENCY` | `4` | Concurrent execution work, from 1 to 1024 | +| `OAC_SANDBOX_MAX_ACTIVE` | `100` | 启用暂停能力的直接 Provider 的活跃沙箱上限，范围 1–100000；独立于执行并发度和节点容量 | +| `OAC_SANDBOX_MAX_RETAINED` | `400` | 直接 Provider 的保留沙箱上限，范围 1–100000，包括活跃、已暂停以及尚未确认清理完成的分配；必须不小于活跃上限 | | `OAC_DEFAULT_HARNESS` | `codex` | Harness used when a request does not name one | | `OAC_HARNESSES` | Every registered Harness | Comma-separated Harnesses to enable besides the default one. Unknown names stop startup | | `OAC_WRITE_AUDIT_RETENTION` | `2160h` | Minimum `1h` | @@ -91,6 +93,10 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 哪些 Harness 已启用以及默认 Harness 属于进程设置（`core.harnesses`、`core.default_harness`）；System 会以只读方式显示它们。[Core 管理 API](../../contracts/agents-api/zh/admin-api.md) 列出了所有 Core API 路由，[部署契约](../../contracts/agents-api/zh/sandbox-deployment.md) 定义了沙箱字段、限制和更改规则。 +### 直接 Provider 容量 {#direct-provider-capacity} + +在 `.env` 中设置 `OAC_SANDBOX_MAX_ACTIVE` 和 `OAC_SANDBOX_MAX_RETAINED`，然后运行 `oac apply`。Core 将这些限制用于启用暂停能力的直接 Provider。每个尚未释放的分配都占用保留容量。降低上限不会终止已有沙箱；新分配会等待使用量低于两项上限。这些设置独立于 `OAC_EXECUTION_CONCURRENCY` 和已注册节点的容量。 + ### 独立工作区存储 {#independent-workspace-storage} 首个支持的独立文件系统组合是 microsandbox 与[内核 NFS 适配器](./workspace-provider.md#kernel-nfs-adapter)。启动服务前，在 Linux Core 主机和所有参与节点上将同一个 NFSv4.2 导出挂载到相同的绝对路径，例如 `/srv/oac-workspaces`。运维人员负责导出、挂载可用性和服务启动顺序。使用带有 `root_squash` 的受信任客户端 AUTH_SYS 导出；不要启用 `no_root_squash` 或放宽权限来使检查通过。按照适配器的[所有权要求](./workspace-provider.md#kernel-nfs-adapter)准备命名空间和服务身份。 @@ -197,7 +203,7 @@ Core 读取进程环境。Compose 将 `.env` 插值到环境中，并把机密 | `OAC_CREDENTIAL_KEY_FILE` | 必填。`/run/oac/credential.key`：Base64 编码的 32 字节随机密钥。Core 用它加密存储的凭据 | | `OAC_CORE_KEY_DIGESTS_FILE` | 必填。`/run/oac/core-key-digests.json`：一个包含 Core 密钥 SHA-256 的 JSON 数组 | | `OAC_INSTALLATION_ID_FILE` | 必填。`/run/oac/installation.id`：安装 ID，采用规范 UUID 格式。如果 ID 与数据库记录的 ID 不一致，Core 会拒绝它 | -| `OAC_EXECUTION_CONCURRENCY`、`OAC_DEFAULT_HARNESS`、`OAC_HARNESSES`、`OAC_WRITE_AUDIT_RETENTION`、`OAC_OAUTH_TRUSTED_ORIGINS`、`OAC_HISTORY_SETTINGS_FILE`、`OAC_LOG_LEVEL`、`OAC_LOG_FORMAT`、`OAC_LOG_ADD_SOURCE` | 对应的[进程设置](#settings)。Web 也读取三个日志设置 | +| `OAC_SANDBOX_MAX_ACTIVE`、`OAC_SANDBOX_MAX_RETAINED`、`OAC_EXECUTION_CONCURRENCY`、`OAC_DEFAULT_HARNESS`、`OAC_HARNESSES`、`OAC_WRITE_AUDIT_RETENTION`、`OAC_OAUTH_TRUSTED_ORIGINS`、`OAC_HISTORY_SETTINGS_FILE`、`OAC_LOG_LEVEL`、`OAC_LOG_FORMAT`、`OAC_LOG_ADD_SOURCE` | 对应的[进程设置](#settings)。Web 也读取三个日志设置 | | `OAC_PROVIDER_ROOT` | 适配器构件的绝对根目录。Core 镜像设置为 `/opt/oac`。每个适配器都拥有此根目录下的辅助路径。当其中的 `native-installers/` 目录包含 `catalog.json` 时，Core 在核对该目录清单与自身发行版后提供自托管守护进程安装程序。适配器状态位于 `/state`，即数据卷的 [`state/`](#compose-installations) | Core 会记录所加载的历史文件路径，但绝不记录环境变量的值或文件内容。 diff --git a/docs/zh/getting-started/operations.md b/docs/zh/getting-started/operations.md index cfe626664..8a24fb8d9 100644 --- a/docs/zh/getting-started/operations.md +++ b/docs/zh/getting-started/operations.md @@ -1,7 +1,7 @@ --- title: "运维" source: docs/getting-started/operations.md -source_hash: f6ce9d76fdd074d9e7e7da4a69da6a4583a67848b86aa641ad09f127fcae346b +source_hash: c04a4d8fedef58355dfe5e7537415d81aed55420346bc99263157c04c8b60daf --- 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。 @@ -193,7 +193,3 @@ cd && rm -rf ~/.oac/core Web 使用 Core 密钥让管理员登录，检查每个请求来源，并用保留在服务器上的 Core 密钥将已登录的 `/core/v1` 请求转发到 Core。它把 `/v1` 和 `/api/v1` 原样转发给 Core，使用调用方自己的凭据；Web 仅在 `/node-install/` 提供不含密钥的节点文件，没有 Docker 或 KVM 访问权限。`/api/v1` 机器路由使用独立注册和连接凭据。没有服务持有 Docker 套接字。 沙箱是隔离边界（[Runtime 与外层隔离](../concepts.md#runtime-and-outer-isolation)）。Docker 沙箱共享节点内核，Docker 节点在主机上[等同于 root 权限](nodes.md#what-the-installer-sets-up)；microsandbox 为每个沙箱提供具有显式[网络策略](nodes.md#what-the-installer-sets-up)的 microVM。Core 自身无 Docker 套接字或 KVM 访问权限。 - -## Runtime 可用性探测 {#runtime-availability-probes} - -Codex CLI 可用性探测为 `process_spawn` 和 `process_wait` 输出 `runtime version probe` 记录，只包含耗时和结果。等待区间包含可执行文件加载和版本命令，两者都不是模型执行。 diff --git a/docs/zh/runtime-bootstrap.md b/docs/zh/runtime-bootstrap.md index 5bd813902..aeecb495e 100644 --- a/docs/zh/runtime-bootstrap.md +++ b/docs/zh/runtime-bootstrap.md @@ -1,7 +1,7 @@ --- title: "Runtime 引导" source: docs/runtime-bootstrap.md -source_hash: 73fb9968007f83d35f1e6fe185d9e376fa81fc7683fe822426b290632ca8de24 +source_hash: 537197f388a57bb983c0e7e0b8a3bfe66b1e8e0ef7f4f1fef572ad7dcf724cb3 --- Sandbox Provider 通过交付一个引导文件来启动托管 Runtime。本文负责 Provider 到 Runtime 的启动输入。类型与验证器位于 [`internal/runtimebootstrap`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/runtimebootstrap/bootstrap.go)；Go provider 使用 [`runtime_bootstrap.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/runtime_bootstrap.go) 中的 `sandbox.Bootstrap.RuntimeConnection()` 构造输入，SDK helper 原样转发序列化对象。provider 不读取或写入 Runtime 的私有认证存储。 @@ -36,6 +36,10 @@ Runtime 验证输入，并负责认证与连接。启动成功仅证明交付完 自托管 executor 和运维人员供应的设备通过其他方式获取 daemon 身份；[机器连接 API](../../contracts/agents-api/zh/machine-api.md#credentials) 列出所有凭据来源。它们都进入同一 Runtime 执行循环。 +## 托管暂停控制 {#hosted-suspension-control} + +私有托管暂停/唤醒控制文件的路径在 `internal/runtimebootstrap/bootstrap.go` 中以 `SuspendControlFile` 定义。Core 恢复逻辑和原生 Go adapter 读取该值；E2B helper 契约生成器将其投射到模板构建器。托管启动准备私有目录并提供 `OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE`，以启用 Runtime 暂停。这是随软件包发布的协议设置。共享 Sandbox Provider 注册负责空闲和保留默认值；adapter 负责自身原生租约超时。 + ## 验证 {#verification} `go test ./internal/runtimebootstrap ./apps/daemon/internal/cli` 覆盖输入契约、凭据来源互斥规则和重启行为。Provider 测试验证交付与文件权限，不依赖 Runtime 的私有存储。 diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 90b262e43..13ca28591 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: a69ba63ef2b5bd883e0a235807df9d270c23dab9b6df11cf6019534281ced8e9 +source_hash: e49b04aa281a5f8bba5838b14b9a3a7ab607a5484d79cd451fcd8bbcc77142d5 --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -27,7 +27,7 @@ Core 拥有持久 Environment、allocation、placement 和 cleanup 状态；Prov ## 实现接口 {#implement-the-interface} -`sandbox_provider.go` 包含 Core–Sandbox Provider 协议：负责 allocation、checkpoint 和观测的 `SandboxProvider` 接口及其请求与结果类型，以及 setup 阶段的 `ConfigurationAdapter` 及其类型化错误。Core 在此边界之外也使用的值类型，例如 `DeploymentSpec` 和 `CallFence`，位于同一 package 的独立文件中。每个方法在编译期都必须实现，`ProviderOperations()` 声明 Provider 支持哪些方法。以下五项操作始终支持： +`sandbox_provider.go` 包含 Core–Sandbox Provider 协议：负责 allocation、暂停和观测的 `SandboxProvider` 接口及其请求与结果类型，以及 setup 阶段的 `ConfigurationAdapter` 及其类型化错误。Core 在此边界之外也使用的值类型，例如 `DeploymentSpec` 和 `CallFence`，位于同一 package 的独立文件中。每个方法在编译期都必须实现，`ProviderOperations()` 声明 Provider 支持哪些方法。以下五项操作始终支持： | 操作 | 用途 | | --- | --- | @@ -47,7 +47,7 @@ Docker 等没有原生可续期租约的 backend 仍遵守 Core 的 hosted expir | --- | --- | --- | | `Create`、`GetInfo`、`Renew`、`Kill`、`RunCommand` | 支持 | Allocation 生命周期与有界命令 | | `Observe` | 明确决定 | 检查所有权、只读地观测一个 allocation | -| `Initial`、`NewCompute`、`GetCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute` | 所有 checkpoint 方法决定一致；仅 `nodes` 注册可以支持 | 精确计算实例、捕获与恢复、保留源恢复和清理 | +| `Initial`、`NewCompute`、`GetCompute`、`RenewCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteRetained`、`RunCommandCompute` | 全部十项暂停方法决定一致；`nodes` 与 `direct` 模式均可支持 | 精确计算实例、捕获与恢复、保留源恢复和清理 | `Initial` 和 `NewCompute` 构造 compute reference，不分配资源；`ResumeCompute` 在暂停中止后仅解冻同一驻留实例；`RunCommandCompute` 在一个精确 compute incarnation 中运行有界命令。Core 使用 `RunCommandCompute` 在恢复后唤醒 parked daemon（[`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)）。 @@ -91,7 +91,7 @@ Microsandbox 在返回创建不存在回执前永久关闭初始 Create 准入 `ErrInvalid`、`ErrOwnership`、`ErrExists`、`ErrNotFound`、`ErrComputeUnconfirmed` 和 `ErrCommandUnconfirmed` 保持其定义含义。未分类原生或 transport error 表示未知，不授权重试 mutation。Core 不将 provider diagnostics 读作生命周期事实，也不暴露原生错误文本或凭据；node transport 将错误映射为固定 code，直接 SDK 细节保持私有。 -Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity`；原样持久化 operation ID 与 provider snapshot provenance。suspend 或 resume 的 `ObserveOnly` 仅观察上次尝试，不启动另一 capture 或 restore。`ResumeCompute` 仅解冻保留源，不冷启动已停止源。清理针对精确 compute incarnation 和 snapshot，不针对当前同名实例。声明 checkpoint 支持前阅读 [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) 及其失败测试。 +暂停支持增加精确的 `Compute` 代次、名称和 ID，以及不透明的 `RetainedState` 收据。原样持久化操作 ID 和 Provider 保留状态的来源。暂停或恢复中的 `ReconcileOnly` 观察原始尝试，不会开始另一次捕获或恢复。`ResumeCompute` 仅解冻保留的源实例，绝不冷启动已停止的源实例。清理针对精确计算实例和保留状态，而非当前恰好同名的实例。声明支持前请阅读 [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) 及其失败测试。 ### 四个独立就绪事实 {#four-distinct-readiness-facts} @@ -114,7 +114,7 @@ Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity` 2. 添加 specification 和 resource validator。 3. 基于类型化原生配置实现 `sandbox.ConfigurationAdapter`。`DecodeInput` 严格解析请求中独立的公开 `configuration` 与只写 `credential` 对象。`Encode` 生成白名单公开 selector、只读观测和独立 secret bytes，不透传请求 JSON。`Decode` 恢复已存储 selector 并保留对所属资源的访问，不做远程 admission 或新模板验证。`Normalize` 修改前复制输入。`ResolveChange`、`Equal` 和 `WithCredential` 负责继承、身份与凭据组合。`Requirements` 声明是否需要凭据和公开 Core origin，以及支持哪些 setup 操作：`Discovery` 对应 `DiscoverConfiguration`，`SelectionDiscovery` 对应 `DiscoverSelection`，`CredentialVerification` 对应 `VerifyCredential`。`DiscoverConfiguration` 验证 query 并返回安全 catalog，不做 mutation 或 admission decision；Core 保留授权、输入限制与 deadline。`DiscoverSelection` 在提交前解析候选项省略的原生值，`VerifyCredential` 验证凭据对所属资源的访问，不修改资源。两者都接收候选项的 `sandbox.DirectConfig`，原生 client 只为该次调用构造。node provider 仅接受空公开对象，拒绝凭据，对每项 setup 操作和 credential replacement 返回 Unsupported。 4. node adapter 从自己的包中导出 `BuildLocal` constructor、它解码的类型化 `native` 对象，以及它在共享 node artifact 之外添加的原生文件。node-local 设置（如主机路径）只存放在该对象中；resources 和 Runtime release 从 node 配置的 `specification` 读取。 -5. 在 `providers/registry.go` 中注册 constructor、policy、configuration adapter 和 operation 声明。其键即 provider kind，也用于标记该 Provider 的观测；checkpoint 支持读取此项。生成的投影组合每个已注册的部署模式和 deployment policy 与 `sandbox/deployment_contract.go` 中的共享 field bound：installer 从 `deploy/node/node_spec.py` 读取，TypeScript 客户端和 Web 从 `packages/agents-client/src/deployment-contract.ts` 读取，因此 Web 读取这些声明，而不比较 provider kind。通过 `go run ./services/core/cmd/specification-contract -write` 重新生成两者。 +5. 在 `providers/registry.go` 中注册 constructor、policy、configuration adapter 和 operation 声明。其键即 provider kind，也用于标记该 Provider 的观测；暂停支持读取此项。生成的投影组合每个已注册的部署模式和 deployment policy 与 `sandbox/deployment_contract.go` 中的共享 field bound：installer 从 `deploy/node/node_spec.py` 读取，TypeScript 客户端和 Web 从 `packages/agents-client/src/deployment-contract.ts` 读取，因此 Web 读取这些声明，而不比较 provider kind。通过 `go run ./services/core/cmd/specification-contract -write` 重新生成两者。 6. 提供 adapter 和 helper 的发行产物，通过已注册 configuration 契约向运维人员提供 provider。 **已知设计缺口：** Web 仍在 setup 向导的后端选择、Docker 确认，以及所有显示或解析 E2B 配置字段的地方（带服务预设的 setup 步骤、部署摘要和客户端的部署投影）中指名 provider，因为协议尚未声明配置字段。通过该界面提供另一 provider 目前需要修改共享的 Web。此耦合不符合[复杂性留在 adapter 内](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#complexity-stays-in-the-adapter)；新集成必须通过协议表达配置，把厂商专有行为留在 adapter。不得添加 Session 或 Turn 调度路径、厂商专有 column 或 API field，或 store 中的厂商 switch。 @@ -128,7 +128,7 @@ node 配置 `sandbox.NodeConfig` 包含 `provider`、`generation`、`installatio - `nodes` 注册仅有 `BuildLocal`，`direct` 注册仅有 `BuildDirect`；缺失、混合或未知 mode 被拒绝。 - specification 和 resource validator、configuration adapter 与完整 operation 声明都是必需项，因此不完整注册不能发布部分投影。声明的默认大小必须通过 adapter 的 resource validator。 - Runtime input policy 要么接受固定 Runtime，要么给出 adapter 拒绝它的固定原因，不能两者兼有。 -- Checkpoint 仅准入 `nodes` 注册，因为公共 lifecycle 只暂停 node allocation；声明 checkpoint 的 `direct` Provider 会被注册拒绝。注册不携带 suspension 数值：Core 对每个声明 checkpoint 支持的 Provider 应用同一个 [suspension policy](#suspension)。 +- 完整生命周期声明支持时，`nodes` 与 `direct` 模式都准入暂停。注册不携带 suspension 数值；Core 对所有支持的 Provider 应用同一个 [暂停策略](#suspension)。 configuration adapter 必须非 nil，包括其具体值。每个 `ConfigurationRequirements` 字段都需要明确有效的决定：`Credential` 和 `PublicOrigin` 为 `Required` 或 `NotRequired`，`Discovery`、`SelectionDiscovery` 和 `CredentialVerification` 使用共享 supported 或 unsupported 声明并携带安全 reason。新增 requirement field 需要明确更新验证，不继承已有决定。要求凭据不承诺支持 `VerifyCredential` 操作。这些检查证明注册完整，不证明原生 SDK 行为正确；constructor 和 adapter 契约测试仍然适用。 @@ -138,7 +138,7 @@ configuration adapter 必须非 nil，包括其具体值。每个 `Configuration 具有凭据的 direct adapter 在替换 key 前验证全部保留 generation 与 allocation reference。公共 `sandbox.CallFence` 排除原生调用并等待 helper 完成，包括调用方已超时的调用；execution 调用已准备的 verification 和 fencing callback，不按厂商分支。 -厂商部署验证和 SDK setup 留在构造边界，构造不创建 Environment。node-local adapter 的 `sandbox.Built` 返回 provider、probe、installation identity、backend fingerprint 和 specification digest；helper 可能比调用方存活更久时，还返回 `Quiescent` 检查，generation 回收会等待它。factory 还返回 close 函数。`execution.RuntimeProvider` 将 adapter 绑定到 kind、installation ID、backend fingerprint、generation、mode 和 node ownership；选择由数据库负责，内存副本不构成另一权限来源。Docker 与 microsandbox 在 node 上运行，E2B 直接构造。node proxy 仅对注册声明支持 checkpoint 的 backend 暴露 checkpoint 操作，公共 lifecycle 通过 checkpoint 声明准入 suspension，不通过 provider name。 +厂商部署验证和 SDK setup 留在构造边界，构造不创建 Environment。node-local adapter 的 `sandbox.Built` 返回 provider、probe、installation identity、backend fingerprint 和 specification digest；helper 可能比调用方存活更久时，还返回 `Quiescent` 检查，generation 回收会等待它。factory 还返回 close 函数。`execution.RuntimeProvider` 将 adapter 绑定到 kind、installation ID、backend fingerprint、generation、mode 和 node ownership；选择由数据库负责，内存副本不构成另一权限来源。Docker 与 microsandbox 在 node 上运行，E2B 直接构造。node proxy 仅对注册声明支持 暂停的 backend 暴露暂停操作，公共 lifecycle 通过暂停声明准入 suspension，不通过 provider name。 backend fingerprint 标识原生资源命名空间，不表示容量。Core 保留部署 generation，使所属 allocation 继续解析到原 backend；不要将保留 allocation 重新指向替代 backend。 @@ -168,15 +168,15 @@ allocation、专用 daemon credential digest 和精确 Session binding 在 `Crea 配置 provider 后，Worker 扫描已提交且没有 allocation 的 pending hosted Environment，涵盖空闲 Session 创建以及 commit 与 bootstrap 之间中断后的恢复；已有 allocation 不重新进入此路径。scan 有界，由 lifecycle owner 串行化，不需要调用方操作。没有 Turn 的初始预约让 Session 保持空闲，daemon 连接不被当作原生 readiness。同一 scan 在验证精确 Session、device binding 和已结算 bootstrap 后，发布带持久 generation 的认证连接观测。 -Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 正在运行的 allocation；该检查不做任何修改，也不复活 cleanup 请求。正在运行的计算资源不会到期：显式删除和 snapshot retention 授权其清理。停止或缺失 container 不授权丢弃保留工作区或历史。禁用 provider 停止新 hosted admission 与 bootstrap，但不阻止现有 Session 的取消、function result 或 input retry outcome。 +Core 在 Turn 之间检查已连接且已观察的计算资源仍是其 Session 正在运行的 allocation；该检查不做任何修改，也不复活 cleanup 请求。正在运行的计算资源不会到期：显式删除和保留状态期限授权其清理。停止或缺失 container 不授权丢弃保留工作区或历史。禁用 provider 停止新 hosted admission 与 bootstrap，但不阻止现有 Session 的取消、function result 或 input retry outcome。 终结清理原子撤销 device authority、记录 Environment 失败或到期、结算 pending input 并请求取消，然后才调用 `Kill`；原 input deadline 与 retry outcome 保留。临时 provider outage、未知 Create result 和停止的计算资源不证明永久失败。公开 Session 删除后 Core 保留 allocation，仅在所属 compute 与 volume 清理完成且原 Create 已结算的证明成立后标记 released；未知创建即使观察到不存在也保留 cleanup ownership，有界 scan 继续捕捉延迟资源，不再调用 `Create`。 ### 每节点生命周期 worker {#per-node-lifecycle-workers} -每个注册 node 有一个串行 lifecycle worker，负责 gate、allocation 与 pending cursor、connection 和 wake hint；E2B allocation 共享一个没有 node 的串行 lifecycle。薄 coordinator 发现 node 并关闭 worker，数据库、provider 或等待操作期间不持有 map mutex。worker 独立推进，因此一个在线 node 的 provider 卡住不会阻塞其他 node：lifecycle 并发为每 node 一项操作，随 node 数量增长。离线 worker 保留，因此其保留资源在重连后仍可观察。 +托管 Environment 提交后，Core 会向其放置节点的生命周期 worker 发送有界提示。已提交的待处理输入也会发送提示，以恢复遗漏的创建通知。提示复用现有串行门控、执行租约、容量检查和一次性分配凭据，不会在准入处理器内直接创建沙箱。每个正常维护周期最多允许一次额外提示扫描；周期扫描负责恢复遗漏或合并的提示。已有分配的观察仍先于待创建资源处理，因此同一节点上的慢观察仍可能延迟新 Environment。 -托管 Environment 提交后，Core 向其 placement 的生命周期 worker 发送有界提示。已提交的待处理输入也会发送提示，以恢复遗漏的创建通知。提示沿用串行 gate、租约、容量检查和一次性 allocation 收据，不在准入处理器内直接创建资源。每个正常维护周期最多允许一次额外提示扫描；周期扫描恢复遗漏或合并的提示。现有 allocation 观察仍先于待创建资源扫描，因此同一节点上缓慢的观察可能延迟新 Environment。 +每个注册 node 有一个串行 lifecycle worker，负责 gate、allocation 与 pending cursor、connection 和 wake hint；E2B allocation 共享一个没有 node 的串行 lifecycle。薄 coordinator 发现 node 并关闭 worker，数据库、provider 或等待操作期间不持有 map mutex。worker 独立推进，因此一个在线 node 的 provider 卡住不会阻塞其他 node：lifecycle 并发为每 node 一项操作，随 node 数量增长。离线 worker 保留，因此其保留资源在重连后仍可观察。 allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan join 尚未释放的已提交 placement。每个 node 推进自己的 cursor，包括越过失败观测，并在末尾回绕一次。direct provisioning 在进入该 node gate 前解析 tenant 范围内 placement，已有 allocation 必须与其一致；Core 不选择另一 node。 @@ -186,15 +186,33 @@ allocation scan 在应用 32 行分页限制前按 node 过滤，pending scan jo placement 自动完成：environment-to-node placement 与 Session 创建及其 retry identity 一起提交，调用方不能选择 node，重试即使 node 离线也保留原 node。Node capacity 计入 pending reservation 和未决资源，新 placement 与 suspended-to-restoring 转移共享数据库锁。未知操作保留预约，source teardown 必须确认后才能释放 active capacity，确认 cleanup 后释放 placement capacity。保留所有权需要精确 provider evidence：socket path、缺失 instance 或空列表都不证明 cleanup，也不授权 replacement。 -部署的 CPU、memory、disk 设置、`max_active`、`max_retained` 和 snapshot retention 限制每个 node。不提供 node-level drain、跨 node Session migration、multi-active Core、autoscaling 或 snapshot replication。node 持有 allocation、snapshot、reservation、unknown result 或 cleanup 时拒绝移除 node，离线 ownership 保留。 +部署的 CPU、memory、disk 设置、`max_active`、`max_retained` 和保留状态期限限制每个 node。不提供 node-level drain、跨 node Session migration、multi-active Core、autoscaling 或 snapshot replication。node 持有 allocation、保留状态、reservation、unknown result 或 cleanup 时拒绝移除 node，离线 ownership 保留。 ### 暂停 {#suspension} -Core 用同一个固定 policy 暂停每个声明 checkpoint 支持的 provider 的空闲工作：工作空闲 5 分钟（300 秒）后暂停，snapshot 保留 24 小时（86400 秒）。部署的 [`suspension`](../../contracts/agents-api/zh/sandbox-deployment.md#safe-response) 报告这两个值。Core 仅在至少一个 Turn 已终结、没有 root 或 Subagent Turn 排队、进行中或等待、没有 pending input、file operation 或 initialization，且真实 activity 已空闲达到该时间后暂停。对于 node allocation，Core 在同一事务中用数据库时钟记录首个 root 或 child terminal transition。candidate filter 和 Session-locked recheck 比较数据库已过时间与 idle duration，初始 snapshot retention deadline 也锚定同一数据库观测，因此 Core 与数据库主机时钟无需一致。原生 completion timestamp 在公开历史中保持不变，但不驱动 idle admission，heartbeat 不重置 activity。确认计划暂停前，daemon 关闭 admission 并排空 native cleanup、output receipt 和 file work。 +单一 `SandboxProvider` 一并声明 `Initial`、`NewCompute`、`GetCompute`、`RenewCompute`、`Suspend`、`Resume`、`KillCompute`、`DeleteRetained`、`RunCommandCompute` 和 `ResumeCompute`：十项全部支持或全部不支持。生命周期不通过可选暂停接口或 Provider 名称分支选择。 + +`RetainedState` 是 adapter 拥有的不透明信封，包含 `Reference`、`ID`、`Data`、`OperationID`、`SourceGeneration`、`SourceName` 和 `SourceID`。`Data` 必须非空且不超过 64 KiB。Core 原样保留信封并校验操作与源实例归属；不可变的 allocation Provider/代次绑定决定唯一有权解释它的 adapter。保留状态不必是独立快照：microsandbox 将完整原生快照身份放入 `Data`，E2B 保留带版本的原生暂停收据，不承诺独立磁盘镜像。adapter 在任何副作用前校验原生内容及归属。 + +`Compute.RestoredFrom`、`ComputeState.Retained`、`ResourcesReleased`、`SuspendSettled` 和恢复的 `ReconcileOnly` 是结算证据。资源缺失不证明未知暂停已结束。`RenewCompute` 只续租精确实例；不确定结果不授权第二次创建、捕获或恢复。持久 `runtime_compute` 使用 `protocol_version: "1"`；启动拒绝缺失或未知版本。仅匹配当前 retained-state 字段形状的版本 1 信封可直接读取，无需改写。旧 snapshot-only 形状即使标记相同的版本 1 也会被拒绝；数字相同不证明互通。升级不做迁移、合成重放或丢弃保留状态。 + +`ResumeRequest.Workspace` 携带 allocation 独立拥有的文件系统绑定；Core 在恢复前获取 ready 绑定。源实例必须已释放活跃执行能力，目标才可成为写者。adapter 在原生副作用前校验绑定；不支持独立文件系统的 adapter 拒绝非空绑定。`DeleteRetained` 仅删除 adapter 的计算保留状态，绝不删除 workspace。显式 Session 删除先完成 compute 清理，再删除独立存储，恢复期间保持单一活跃写者。 + +Core 对所有声明暂停支持的 Provider 应用统一固定策略：工作空闲 5 分钟（300 秒）后暂停，保留状态期限为 24 小时（86400 秒）；部署的 [`suspension`](../../contracts/agents-api/zh/sandbox-deployment.md#safe-response) 返回这些值。Core 暂停已经初始化的 Environment，包括尚未执行 Turn 的 Environment，前提是没有 root 或 Subagent Turn 排队、执行或等待，没有输入、文件操作或初始化待处理，且真实活动已空闲达到该时长。Core 在同一事务中使用数据库时钟记录 allocation 的初始化完成和 root 或 child 终结转换。候选筛选和持有 Session 锁的复查比较数据库经过时间与空闲时长，初始保留状态期限也锚定同一数据库观测，因此 Core 与数据库主机的时钟无需一致。公开历史中的原生完成时间戳保持不变，但不驱动空闲准入，心跳也不重置活动时间。确认计划暂停前，daemon 关闭准入并排空原生清理、输出收据和文件工作。 + +Worker 租约、Session 锁和每个 node 的 gate 负责所有 Provider 的暂停。新 Turn claim、文件写入意图和捕获准入在 Session 锁下串行化，共享计算阶段检查；新待处理工作取消捕获并唤醒同一源实例。正常准备在经过认证的恢复握手后等待计算阶段变为 running；待处理输入的提升与生命周期转换冲突时，输入保持 pending。计算阶段和经 revision 校验的收据存储在 allocation 中。Core 在副作用前持久化静止、捕获和恢复意图，仅新收据执行捕获或恢复；恢复流程观察精确尝试，不重试结果未知的创建、捕获或恢复。已消费的保留状态不会使运行中的代次回滚。删除、撤销和保留期到期始终优先于唤醒，直至最后的数据库 compare-and-swap；未知清理身份会一直保留，直到确认所属资源已不存在。已消费产物和旧计算实例会被删除，因此暂停循环不会累积可写磁盘链。 + +排队工作和实时 Environment 文件访问会唤醒暂停的 Environment；历史和已发布 Artifact 的读取不会唤醒。计划暂停在 daemon 连接上使用 Environment 与 suspension token。由 PID 和启动时间隔离的本地控制信号（`RunCommandCompute`）唤醒 parked daemon，daemon 在准入工作前重新认证。确认前的临时断连通过有界尝试和退避重试同一已准备好的暂停；永久认证或协议拒绝会将其关闭。Core 负责保留状态的到期期限，daemon 没有相应定时器。静止确认丢失时可以通过明确回滚解冻同一源实例，但不授权捕获。 + +`Suspend` 负责原生资源释放，返回绑定的保留句柄、suspended 状态、`ResourcesReleased` 和 `SuspendSettled` 后，Core 才释放活跃容量。`ReconcileOnly` 禁止重放原始捕获或暂停，但允许完成由持久保留产物证明安全的 adapter 清理。无保留状态的结果只有在带有 `SuspendSettled` 且源实例处于可恢复的运行或暂停状态时才允许回滚。其他所有不确定结果均保留所有权并关闭准入。Core 从不在 `Suspend` 后无条件销毁源实例。 + +`Resume` 将保留状态恰好消费一次并恢复到预先提交的目标。恢复逻辑观察同一次尝试。Core 持久化 `waking`、认证并恢复 daemon、删除已消费的保留资源，然后提交 `running` 并准入工作。`DeleteRetained` 是幂等产物清理，会保留运行中的计算资源。清理失败会保持 `waking` 阶段，不能触发再次恢复。`KillCompute` 仍然是破坏性操作；清理旧代次时不得终止共享原生 ID 的较新活跃实例。 + +现有 Session 锁、生命周期租约、空闲规则、容量查询和清理顺序继续作为权威。每个尚未释放的分配都占用 `max_retained`，包括运行中的分配。每个分配在预留时都写入共享计算协议版本，包括暂停阶段为 `disabled` 的分配。激活会拒绝任何协议版本缺失或不同的未释放分配；升级前必须由旧版本完成普通清理。Session 历史保留。 -Worker lease、Session lock 与 per-node gate 对每个 provider 负责 suspension。新 Turn claim、file-write intent 和 capture admission 在 Session lock 下串行化，共享一个 compute-phase 检查；新 pending work 取消 capture 并唤醒同一 source。正常 preparation 在经过认证的 resume handshake 后等待 compute phase 为 running；pending input 的 promotion 与 lifecycle transition 冲突时保持 pending。compute phase 和 revision-checked receipt 位于 allocation。Core 在 effect 前持久化 quiesce、capture 和 restore intent，仅新 receipt 执行 capture 或 restore，恢复观察精确 attempt，不重试未知 creation、capture 或 restore。已消费 snapshot 不让 running generation 回滚。删除、撤销和 retention expiry 优先于 wake，一直持续到最终数据库 compare-and-swap；未知 cleanup identity 保留，直到确认所属资源不存在。已消费 artifact 和旧 compute 被删除，因此暂停循环不累积可写磁盘链。 +### 协调协议升级 {#coordinated-protocol-upgrade} -排队工作和实时 Environment file access 唤醒 suspended Environment；history 和已发布 Artifact read 不唤醒。计划暂停在 daemon 连接上使用 Environment 和 suspension token。受 PID 与 start-time fencing 的本地 control signal（`RunCommandCompute`）唤醒 parked daemon，daemon 在准入工作前重新认证。确认前临时断连通过有界 attempt 和 backoff 重试同一已 armed suspension；永久认证或协议拒绝则关闭。Core 负责 snapshot retention deadline，daemon 没有相应 timer。quiesce 确认丢失时可以通过明确 rollback 解冻同一 source，但不授权 capture。 +Core 与 node 一同升级到 node wire version 7；E2B helper 使用 private wire version 4，并重建包含托管暂停控制的模板；microsandbox helper 使用 private wire version 5。所有 Provider 交付 Runtime bootstrap version 2。每个边界独立验证自身契约，不同边界的版本号不能互换。激活前，使用此前兼容的版本对不兼容的未释放 allocation 完成正常清理。启动栅栏保留其持久收据及 Session 历史，不通过 Provider 调用迁移它们。 ### 重置与归档 {#reset-and-archive} @@ -202,7 +220,7 @@ Worker lease、Session lock 与 per-node gate 对每个 provider 负责 suspensi 一个 snapshot 和 timestamp 对持有资源分区。离线 ownership 来自 allocation 或 active placement 的 node，与 online presence 使用同一 45 秒 connection 和 owner-epoch predicate，独立于 provider readiness；cleanup failure、offline state 或空 read 都不授权合成 release。held resource 为零时，Core 在数据库事务外 drain，在 deployment lock 下复查，并在一个事务中清空 deployment，再发布携带 generation 的空 provider，没有可失败工作。最终 write 或 drain 失败时，在释放 mutation gate 前使用有界 owner context 恢复已提交 provider；恢复失败时 admission 保持 fenced，owner 停止。 -管理员 [Session archive](../../contracts/agents-api/zh/admin-api.md#session-archive) 在 Session-first transaction 中保留 Project scope 与 hosted eligibility 检查，并一起处理 Environment expiry、cancellation、Runtime authority revocation 和 audit；reset 的后台 archive 从可信记录重建实际 Project scope，保留请求方 provenance。普通 provider lifecycle 释放 compute 和 snapshot。archive cancellation 在 terminal commit 前保留健康 receipt path：仅首次撤销 device 的 archive 记录精确 `archive_cancel_turn_id`（普通 revocation 清空它，重复 cleanup 保留它），现有已认证 delivery 可从 Turn 原 `cancel_requested_at` 起最多 20 秒排空该 cancellation。Core 独立于 subscription removal，通过 `done`、cancellation acknowledgement 和 terminal commit 跟踪 delivery，不授予新 connection、input、file 或 MCP authority，也不续期 lease。事务或 lifecycle gate 不等待 receipt，peer 丢失、到期或重启回到普通 failure 与 cleanup，不虚构 cancelled outcome。 +管理员 [Session archive](../../contracts/agents-api/zh/admin-api.md#session-archive) 在 Session-first transaction 中保留 Project scope 与 hosted eligibility 检查，并一起处理 Environment expiry、cancellation、Runtime authority revocation 和 audit；reset 的后台 archive 从可信记录重建实际 Project scope，保留请求方 provenance。普通 provider lifecycle 释放 compute 和保留状态。archive cancellation 在 terminal commit 前保留健康 receipt path：仅首次撤销 device 的 archive 记录精确 `archive_cancel_turn_id`（普通 revocation 清空它，重复 cleanup 保留它），现有已认证 delivery 可从 Turn 原 `cancel_requested_at` 起最多 20 秒排空该 cancellation。Core 独立于 subscription removal，通过 `done`、cancellation acknowledgement 和 terminal commit 跟踪 delivery，不授予新 connection、input、file 或 MCP authority，也不续期 lease。事务或 lifecycle gate 不等待 receipt，peer 丢失、到期或重启回到普通 failure 与 cleanup，不虚构 cancelled outcome。 ## 验证集成 {#validate-the-integration} diff --git a/internal/runtimebootstrap/bootstrap.go b/internal/runtimebootstrap/bootstrap.go index 1beabb40f..183f5b054 100644 --- a/internal/runtimebootstrap/bootstrap.go +++ b/internal/runtimebootstrap/bootstrap.go @@ -14,6 +14,9 @@ import ( "github.com/google/uuid" ) +// SuspendControlFile is the packaged private hosted Runtime park/wake location. +const SuspendControlFile = "/run/oac/daemon-suspend.json" + const Version = 2 const MaxBytes = 16 * 1024 diff --git a/services/core/cmd/server/managed_generation_operations.go b/services/core/cmd/server/managed_generation_operations.go index 2f7c99b65..aba23a97c 100644 --- a/services/core/cmd/server/managed_generation_operations.go +++ b/services/core/cmd/server/managed_generation_operations.go @@ -17,7 +17,7 @@ func (p *generationRouter) Initial(ctx context.Context, r sandbox.Reference) (sa defer done() return v.Initial(ctx, r) } -func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, snapshot *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, snapshot *sandbox.RetainedState) (sandbox.Compute, error) { if err := providercontract.Require(p, "NewCompute"); err != nil { return sandbox.Compute{}, err } @@ -72,8 +72,8 @@ func (p *generationRouter) KillCompute(ctx context.Context, r sandbox.Reference, defer done() return v.KillCompute(ctx, r, c) } -func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Reference, snapshot sandbox.SnapshotIdentity) error { - if err := providercontract.Require(p, "DeleteSnapshot"); err != nil { +func (p *generationRouter) DeleteRetained(ctx context.Context, r sandbox.Reference, snapshot sandbox.RetainedState) error { + if err := providercontract.Require(p, "DeleteRetained"); err != nil { return err } v, done, err := p.route(ctx, r) @@ -81,7 +81,7 @@ func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Referen return err } defer done() - return v.DeleteSnapshot(ctx, r, snapshot) + return v.DeleteRetained(ctx, r, snapshot) } func (p *generationRouter) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { if err := providercontract.Require(p, "RunCommandCompute"); err != nil { @@ -105,3 +105,15 @@ func (p *generationRouter) ResumeCompute(ctx context.Context, r sandbox.Referenc defer done() return v.ResumeCompute(ctx, r, c) } + +func (p *generationRouter) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + if err := providercontract.Require(p, "RenewCompute"); err != nil { + return sandbox.ComputeState{}, err + } + v, done, err := p.route(ctx, r) + if err != nil { + return sandbox.ComputeState{}, err + } + defer done() + return v.RenewCompute(ctx, r, c) +} diff --git a/services/core/cmd/server/managed_generations_test.go b/services/core/cmd/server/managed_generations_test.go index 4b1c57aac..341d27cd5 100644 --- a/services/core/cmd/server/managed_generations_test.go +++ b/services/core/cmd/server/managed_generations_test.go @@ -29,7 +29,7 @@ q=json.load(sys.stdin) with (pathlib.Path(q['Config']['StateDir'])/'requests').open('a') as f: f.write(json.dumps(q)+'\n') info=dict(q['Reference'],State='running',ProviderID='owned',CreateSettled=True) if q['Operation']=='kill': info['State']='absent' -print(json.dumps({'Version':2,'Info':info})) +print(json.dumps({'Version':q['Version'],'Info':info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -54,7 +54,7 @@ print(json.dumps({'Version':2,'Info':info})) value.Configuration = &key return value, nil } - setup := &managedSetup{processPaths: paths, registry: providers.Builtin(), deployment: &fakeDeploymentSetups{t: t, allocationSetup: allocation}, installationID: id} + setup := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), deployment: &fakeDeploymentSetups{t: t, allocationSetup: allocation}, installationID: id} // A facade retained by a generation-one lifecycle still reads current credentials. router := &generationRouter{setup: setup} ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) @@ -132,7 +132,7 @@ code = '' if k == 'revoked': code = 'unauthorized' elif k == 'unconfirmed': code = 'unconfirmed' elif not (k.startswith('team-a') and template in ('owned-a', 'new-a') or k == 'team-b' and template == 'public-b'): code = 'team_mismatch' -result = {'Version': 2, 'ErrorCode': code} +result = {'Version': q['Version'], 'ErrorCode': code} if not code: result['DeploymentValid'] = True if q['Operation'] == 'validate_deployment': @@ -151,7 +151,7 @@ print(json.dumps(result)) setups := &fakeDeploymentSetups{t: t, setup: committedSetup(&committed), withCredential: credentialService(t), generationPage: func(context.Context, int64) ([]deployment.Setup, error) { return nil, nil }} allocations := &fakeGenerationAllocations{t: t, credentialAllocations: func(context.Context, string) ([]deployment.Allocation, error) { return nil, nil }} - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: setups, allocations: allocations} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: setups, allocations: allocations} loaded, err := s.load(t.Context()) if err != nil { t.Fatal(err) diff --git a/services/core/cmd/server/managed_nodes.go b/services/core/cmd/server/managed_nodes.go index b908bbab6..78a5639f7 100644 --- a/services/core/cmd/server/managed_nodes.go +++ b/services/core/cmd/server/managed_nodes.go @@ -70,7 +70,7 @@ func configureManagedNodes(nodes *deployment.Service, reader deployment.Reader, return nodes.Heartbeat(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health)) }, }) - result.setup = &managedSetup{processPaths: config.ProviderPaths, registry: registry, deployment: nodes, allocations: reader, hub: result.hub, installationID: config.InstallationID, runtimeAPI: config.PublicOrigin.RuntimeAPI()} + result.setup = &managedSetup{capacity: config.SandboxCapacity, processPaths: config.ProviderPaths, registry: registry, deployment: nodes, allocations: reader, hub: result.hub, installationID: config.InstallationID, runtimeAPI: config.PublicOrigin.RuntimeAPI()} result.runtime = execution.NewDeferredRuntimeProvider(config.InstallationID, result.setup.load, result.setup.prepare) result.runtime.PublishUnconfigured = result.setup.publishUnconfigured return result diff --git a/services/core/cmd/server/managed_setup.go b/services/core/cmd/server/managed_setup.go index 758ff5db0..f913b508c 100644 --- a/services/core/cmd/server/managed_setup.go +++ b/services/core/cmd/server/managed_setup.go @@ -11,6 +11,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/processconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -22,6 +23,7 @@ import ( // observation. The database owns the selection; this cache is never a writer. type managedSetup struct { processPaths sandbox.ProcessPaths + capacity processconfig.SandboxLimits // registry builds the selected direct provider and discovers configuration; // the deployment setup reports what the registration declares. registry *providers.Registry @@ -159,7 +161,7 @@ func (s *managedSetup) configuration(setup deployment.Setup) (execution.Prepared Resources: setup.Specification.Resources, WorkspaceRequirements: adapter.Policy.Workspace, Workspace: setup.Specification.Workspace} if setup.Suspension != nil { selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.Suspension.IdleSeconds) * time.Second, - Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second} + Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second, MaxActive: s.capacity.MaxActive, MaxRetained: s.capacity.MaxRetained} } return execution.PreparedRuntimeDeployment{Config: selected, Publish: s.publish}, nil } diff --git a/services/core/cmd/server/managed_setup_preflight_test.go b/services/core/cmd/server/managed_setup_preflight_test.go index 884b89385..959fb49fd 100644 --- a/services/core/cmd/server/managed_setup_preflight_test.go +++ b/services/core/cmd/server/managed_setup_preflight_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -19,7 +20,7 @@ import ( func TestE2BRejectedSpecificationHasSafeActionableDiagnostic(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(e2b.ProtocolVersion)+",\"ErrorCode\":\"invalid\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") @@ -28,7 +29,7 @@ func TestE2BRejectedSpecificationHasSafeActionableDiagnostic(t *testing.T) { } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id} selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 3, MemoryMiB: 3072}}, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} _, err := s.prepare(t.Context(), selection) if !errors.Is(err, sandbox.ErrConfigurationSelection) || strings.Contains(err.Error(), "synthetic-private-key") || s.selected.Load() != nil { @@ -49,12 +50,12 @@ op = q['Operation'] with (pathlib.Path(q['Config']['StateDir']) / 'operations').open('a') as log: log.write(op + '\n') if op == 'validate_deployment': - print(json.dumps({'Version': 2, 'ErrorCode': 'invalid'})) + print(json.dumps({'Version': q['Version'], 'ErrorCode': 'invalid'})) else: info = dict(q['Reference'], ProviderID='owned-compute', State='stopped', CreateSettled=True) if op == 'kill': info.update(ProviderID='', State='absent') - print(json.dumps({'Version': 2, 'Info': info})) + print(json.dumps({'Version': q['Version'], 'Info': info})) ` if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) @@ -69,7 +70,7 @@ else: Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} allocation := func(context.Context, sandbox.Reference) (deployment.Setup, error) { return selection, nil } - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: committedSetup(&selection), allocationSetup: allocation}} if _, err := s.prepare(t.Context(), selection); err == nil || s.selected.Load() != nil { t.Fatal("invalid new template selection was published", err) @@ -101,13 +102,13 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { } helper := filepath.Join(t.TempDir(), "provider") requests := filepath.Join(state, "requests") - script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" + script := "#!/bin/sh\ncat >>" + requests + "\necho >>" + requests + "\nprintf '%s' '{\"Version\":" + strconv.Itoa(e2b.ProtocolVersion) + ",\"ErrorCode\":\"\",\"DeploymentValid\":true,\"TemplateBuild\":{\"Status\":\"ready\",\"CPUs\":4,\"MemoryMiB\":4096,\"RootDiskMiB\":24063}}'\n" if err := os.WriteFile(helper, []byte(script), 0700); err != nil { t.Fatal(err) } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-private-key", Template: "runtime:" + uuid.NewString()}} candidate, err := s.prepare(t.Context(), selection) disk := int32(24063) @@ -133,7 +134,7 @@ func TestE2BCandidateAdoptsTemplateBuildForOmittedResources(t *testing.T) { func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { helper := filepath.Join(t.TempDir(), "provider") - if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { + if err := os.WriteFile(helper, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(e2b.ProtocolVersion)+",\"ErrorCode\":\"team_mismatch\"}'\n"), 0700); err != nil { t.Fatal(err) } state := filepath.Join(t.TempDir(), "e2b") @@ -142,7 +143,7 @@ func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { } paths := testProviderPaths(t, helper, state) id := uuid.NewString() - s := &managedSetup{processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: paths, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t}} selection := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-team-a", Template: "public-team-b:" + uuid.NewString()}} if _, err := s.prepare(t.Context(), selection); !errors.Is(err, sandbox.ErrCredentialOwnership) || s.selected.Load() != nil { t.Fatal("public readability accepted as team ownership", err) @@ -150,7 +151,7 @@ func TestInitialE2BPublicTemplateOutsideTeamIsRejected(t *testing.T) { } func TestManagedSetupRoutesProviderWithoutCredentialRequirement(t *testing.T) { - s := &managedSetup{} + s := &managedSetup{capacity: testSandboxCapacity(t)} config := &execution.RuntimeProvider{ProviderKind: "docker"} candidate, err := s.routeGenerations( execution.PreparedRuntimeDeployment{Config: config}, diff --git a/services/core/cmd/server/managed_setup_test.go b/services/core/cmd/server/managed_setup_test.go index aaaf9cd02..d2f95a334 100644 --- a/services/core/cmd/server/managed_setup_test.go +++ b/services/core/cmd/server/managed_setup_test.go @@ -122,7 +122,7 @@ func TestManagedSetupNeverReusesAnotherGenerationOrUnverifiedState(t *testing.T) value := deployment.Setup{InstallationID: "installation", Provider: "docker", Mode: "nodes", Generation: 1} var loadErr error setups := &fakeDeploymentSetups{t: t, setup: func(context.Context) (deployment.Setup, error) { return value, loadErr }} - s := &managedSetup{registry: providers.Builtin(), deployment: setups, allocations: &fakeGenerationAllocations{t: t}, installationID: "installation"} + s := &managedSetup{capacity: testSandboxCapacity(t), registry: providers.Builtin(), deployment: setups, allocations: &fakeGenerationAllocations{t: t}, installationID: "installation"} cached := &execution.RuntimeProvider{InstallationID: "installation", ProviderKind: "docker", Generation: 1} s.publish(cached) if got, err := s.load(t.Context()); err != nil || got != cached { @@ -153,7 +153,7 @@ func TestMissingE2BHelperReportsProviderUnavailable(t *testing.T) { id := uuid.NewString() committed := deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", Generation: 1, UsesCredential: true, Configuration: &e2b.DeploymentConfiguration{APIKey: "synthetic-key", Template: "runtime:" + uuid.NewString()}} - s := &managedSetup{processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id, + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: committedSetup(&committed)}} if _, err := s.load(t.Context()); !errors.Is(err, execution.ErrExecutionUnavailable) { t.Fatal("missing local helper must leave administrative recovery available", err) @@ -164,10 +164,12 @@ func TestMissingE2BHelperReportsProviderUnavailable(t *testing.T) { } func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "7") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "31") id := uuid.NewString() hub := node.NewHub(node.HubOptions{}) defer hub.Close() - s := &managedSetup{registry: providers.Builtin(), installationID: id, hub: hub, deployment: &fakeDeploymentSetups{t: t}, allocations: &fakeGenerationAllocations{t: t}, runtimeAPI: "https://core.example/api/v1"} + s := &managedSetup{capacity: testSandboxCapacity(t), registry: providers.Builtin(), installationID: id, hub: hub, deployment: &fakeDeploymentSetups{t: t}, allocations: &fakeGenerationAllocations{t: t}, runtimeAPI: "https://core.example/api/v1"} previous := &execution.RuntimeProvider{InstallationID: id, Generation: 1, ProviderKind: "docker"} s.publish(previous) candidate, err := s.prepare(t.Context(), deployment.Setup{InstallationID: id, Provider: "microsandbox", Mode: "nodes", Operations: microsandbox.Operations(), Suspension: &deployment.Suspension{IdleSeconds: 300, RetentionSeconds: 86400}}) @@ -177,6 +179,9 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { if s.selected.Load().Config != previous || candidate.Config.ProviderKind != "microsandbox" || candidate.Config.Suspension == nil || candidate.Config.CoreURL != "https://core.example/api/v1" { t.Fatal("preparation published or lost candidate configuration") } + if candidate.Config.Suspension.MaxActive != 7 || candidate.Config.Suspension.MaxRetained != 31 { + t.Fatal("configured capacity was not propagated") + } committed := *candidate.Config committed.Generation = 2 candidate.Publish(&committed) @@ -187,7 +192,7 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { func TestManagedSetupRejectedCandidateRetainsSelection(t *testing.T) { id := uuid.NewString() - s := &managedSetup{processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id} + s := &managedSetup{capacity: testSandboxCapacity(t), processPaths: sandbox.ProcessPaths{ArtifactRoot: t.TempDir(), StateRoot: t.TempDir()}, registry: providers.Builtin(), installationID: id} previous := &execution.RuntimeProvider{InstallationID: id, Generation: 1, ProviderKind: "docker"} s.publish(previous) _, err := s.prepare(t.Context(), deployment.Setup{InstallationID: id, Provider: "e2b", Mode: "direct", UsesCredential: true, @@ -210,7 +215,7 @@ func TestManagedSetupResetTombstoneRejectsDelayedProviderLoad(t *testing.T) { return deployment.Setup{}, ctx.Err() } } - s := &managedSetup{registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: delayed}} + s := &managedSetup{capacity: testSandboxCapacity(t), registry: providers.Builtin(), installationID: id, deployment: &fakeDeploymentSetups{t: t, setup: delayed}} done := make(chan error, 1) go func() { provider, err := s.load(t.Context()) diff --git a/services/core/cmd/server/sandbox_capacity_test.go b/services/core/cmd/server/sandbox_capacity_test.go new file mode 100644 index 000000000..46ad1cc0d --- /dev/null +++ b/services/core/cmd/server/sandbox_capacity_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/processconfig" +) + +func testSandboxCapacity(t testing.TB) processconfig.SandboxLimits { + t.Helper() + c, err := processconfig.SandboxCapacity() + if err != nil { + t.Fatal(err) + } + return c +} diff --git a/services/core/deploy/e2b/build-template.py b/services/core/deploy/e2b/build-template.py index d78926213..30616a94d 100644 --- a/services/core/deploy/e2b/build-template.py +++ b/services/core/deploy/e2b/build-template.py @@ -10,6 +10,7 @@ import tempfile from e2b import Template +from helper_contract_generated import SUSPEND_CONTROL_FILE BASE = 'node:22.23.1-bookworm-slim@sha256:8607a9064d4a571140998ae9e52a3b3fcf9cff361d04642d5971e6cd76d39e27' parser = argparse.ArgumentParser() @@ -27,6 +28,7 @@ if value.startswith(('HOME=', 'OAC_'))) if environment.get('OAC_RUNTIME_WORKSPACE') != '/environment/workspace': parser.error('Image does not use the colocated Runtime layout') +environment['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'] = SUSPEND_CONTROL_FILE dev_home = Path(os.environ.get('OAC_DEV_HOME') or Path.home() / '.oac') if not dev_home.is_absolute(): parser.error('OAC_DEV_HOME must be absolute') diff --git a/services/core/deploy/e2b/build_template_test.py b/services/core/deploy/e2b/build_template_test.py index 9a3b3034f..3e4f90508 100644 --- a/services/core/deploy/e2b/build_template_test.py +++ b/services/core/deploy/e2b/build_template_test.py @@ -64,6 +64,9 @@ def build(instance, **kwargs): self.assertIs(instance, template) context = Path(factory.call_args.kwargs['file_context_path']) self.assertEqual(stat.S_IMODE(context.stat().st_mode), 0o700) + from helper_contract_generated import SUSPEND_CONTROL_FILE + runtime_env = json.loads((context / 'runtime-env.json').read_text()) + self.assertEqual(runtime_env['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'], SUSPEND_CONTROL_FILE) with tarfile.open(context / 'runtime.tar.gz') as archive: modes = {m.name: stat.S_IMODE(m.mode) for m in archive.getmembers()} for parent in ('usr', 'usr/local', 'etc'): diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index 0b7635c78..7f420e0aa 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -1,5 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" +COMPUTE_FIELDS = ["Generation","Name","ID","RestoredFrom"] ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] @@ -10,9 +11,13 @@ MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 2 +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute_info","compute_renew","suspend","resume","compute_kill","delete_retained","compute_command","resume_compute"] +PROTOCOL_VERSION = 4 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Retained","Deadline"] +RESPONSE_FIELDS = ["Version","State","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +RESUME_FIELDS = ["Workspace","Reference","OperationID","Retained","Target","ReconcileOnly"] +RETAINED_FIELDS = ["Reference","ID","Data","OperationID","SourceGeneration","SourceName","SourceID"] SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" +SUSPEND_FIELDS = ["Reference","OperationID","Source","Retained","ReconcileOnly"] diff --git a/services/core/deploy/e2b/init.py b/services/core/deploy/e2b/init.py index 75d8e1f07..fa8171c29 100644 --- a/services/core/deploy/e2b/init.py +++ b/services/core/deploy/e2b/init.py @@ -106,6 +106,8 @@ def initialize(): # Claim before any side effect. An interrupted attempt must never start twice. write_private(ROOT / 'launch.json', identity) environment = prepare_runtime() + # Application-owned startup does not participate in Core-managed suspension. + environment.pop('OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE', None) credential = PROFILE.parent / 'executor-key.json' write_private(credential, payload['executor_key'], owner=1000) source.unlink() diff --git a/services/core/deploy/e2b/init_test.py b/services/core/deploy/e2b/init_test.py index 3c416c13f..31a1489d3 100644 --- a/services/core/deploy/e2b/init_test.py +++ b/services/core/deploy/e2b/init_test.py @@ -37,7 +37,8 @@ def test_launch_handoff_is_private_and_cannot_replay(self): profile = Path(temporary) / 'private/default' environment_file = Path(temporary) / 'image.json' environment_file.write_text(json.dumps({'HOME': '/home/runtime', - 'OAC_RUNTIME_HOME': '/home/runtime/.oac', 'OAC_RUNTIME_WORKSPACE': '/environment/workspace'})) + 'OAC_RUNTIME_HOME': '/home/runtime/.oac', 'OAC_RUNTIME_WORKSPACE': '/environment/workspace', + 'OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE': '/private/control/fixture.json'})) (root / 'bootstrap.json').write_text(json.dumps(PAYLOAD)) real_chmod = Path.chmod @@ -57,6 +58,7 @@ def chmod(path, mode): self.assertEqual(argv[argv.index('--remote') + 1], PAYLOAD['remote_url']) self.assertNotIn('test-private-key', repr(popen.call_args)) self.assertNotIn('OAC_RUNTIME_SESSION_ID', options['env']) + self.assertNotIn('OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE', options['env']) self.assertEqual((options['user'], options['group'], options['extra_groups']), (1000, 1000, [])) self.assertEqual(options['umask'], 0o077) key = profile.parent / 'executor-key.json' diff --git a/services/core/deploy/e2b/managed_init.py b/services/core/deploy/e2b/managed_init.py index 588f69d80..8c004102e 100644 --- a/services/core/deploy/e2b/managed_init.py +++ b/services/core/deploy/e2b/managed_init.py @@ -49,6 +49,13 @@ def initialize(): binding = identity(payload) shared.write_private(root / 'managed-launch.json', binding) environment = shared.prepare_runtime() + control_file = Path(environment['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE']) + if not control_file.is_absolute() or '..' in control_file.parts: + raise ValueError('Absolute private suspend control file required') + control_directory = control_file.parent + control_directory.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chown(control_directory, 1000, 1000) + control_directory.chmod(0o700) environment.update(OAC_RUNTIME_ENVIRONMENT_ID=payload['EnvironmentID'], OAC_RUNTIME_SESSION_ID=payload['SessionID'], OAC_RUNTIME_NETWORK_ACCESS=payload['NetworkAccess'], diff --git a/services/core/deploy/e2b/managed_init_test.py b/services/core/deploy/e2b/managed_init_test.py index f38a92ffd..c6d6a957e 100644 --- a/services/core/deploy/e2b/managed_init_test.py +++ b/services/core/deploy/e2b/managed_init_test.py @@ -63,9 +63,11 @@ def exercise(self, failed=False): if failed: process.side_effect = RuntimeError('private process diagnostic') image_env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'OAC_RUNTIME_HOME': str(Path(temporary) / '.oac'), - 'OAC_RUNTIME_WORKSPACE': '/environment/workspace'} + 'OAC_RUNTIME_WORKSPACE': '/environment/workspace', + 'OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE': str(Path(temporary) / 'control' / 'custom-suspend.json')} with patch.object(managed_init.shared, 'ROOT', root), patch.object(managed_init.shared, 'PROFILE', profile), \ patch.object(managed_init.shared, 'prepare_runtime', return_value=image_env), \ + patch.object(managed_init.os, 'chown') as chown, \ patch.object(managed_init.os, 'fchown'), patch.object(managed_init.subprocess, 'Popen', process): if failed: with self.assertRaises(RuntimeError): @@ -83,6 +85,10 @@ def exercise(self, failed=False): self.assertNotIn(data['RuntimeBootstrap']['credential'], json.dumps(process.call_args.kwargs['env'])) self.assertEqual(process.call_args.kwargs['env']['OAC_RUNTIME_ENVIRONMENT_ID'], data['EnvironmentID']) self.assertEqual(process.call_args.kwargs['user'], 1000) + control = Path(temporary) / 'control' + self.assertEqual(control.stat().st_mode & 0o777, 0o700) + chown.assert_called_once_with(control, 1000, 1000) + self.assertEqual(process.call_args.kwargs['env']['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'], str(control / 'custom-suspend.json')) if failed: self.assertFalse((root / 'managed-ready.json').exists()) else: diff --git a/services/core/internal/db/queries/runtime_allocations.sql b/services/core/internal/db/queries/runtime_allocations.sql index 6902e54ab..4eeccef61 100644 --- a/services/core/internal/db/queries/runtime_allocations.sql +++ b/services/core/internal/db/queries/runtime_allocations.sql @@ -1,6 +1,6 @@ -- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) -VALUES ($1, $2, $3, $4, $5, $6) RETURNING *; +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, compute_state) +VALUES ($1, $2, $3, $4, $5, $6, jsonb_build_object('protocol_version', sqlc.arg(protocol_version)::text)) RETURNING *; -- name: GetRuntimeAllocation :one SELECT sqlc.embed(a), e.session_id, s.tenant_id, s.deleted_at, (a.compute_phase NOT IN ('disabled', 'running') AND a.compute_retained_until IS NOT NULL AND a.compute_retained_until <= clock_timestamp())::boolean AS expired diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index 83b5a38ca..379687964 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -25,15 +25,12 @@ WHERE id = $1 AND compute_phase = 'running' AND compute_activity_at <= $2; -- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, - CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, - CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = e.session_id AND r.state = 'pending') OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = e.id AND f.state = 'pending'))::boolean AS busy, - a.compute_wake_requested, - EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.completed_at IS NOT NULL)::boolean AS has_completed_turn + a.compute_wake_requested FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE a.id = $1; @@ -47,10 +44,28 @@ SELECT EXISTS ( UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e WHERE a.environment_id = e.id AND e.session_id = $1 - AND a.node_id IS NOT NULL AND a.state = 'running'; + AND a.state = 'running'; -- name: SessionHasRuntimeNode :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE e.session_id = $1 AND a.node_id IS NOT NULL )::boolean; + +-- name: HasIncompatibleRuntimeComputeState :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations + WHERE state <> 'released' + AND ((compute_state->>'protocol_version') IS DISTINCT FROM sqlc.arg(protocol_version)::text + OR compute_state ? 'snapshot' + OR (compute_state #> '{current,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot'] + OR (compute_state #> '{target,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot']) +)::boolean; + +-- name: CountRuntimeComputeReservations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended'; + +-- name: CountRuntimeRetainedAllocations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released'; diff --git a/services/core/internal/db/sqlc/runtime_allocations.sql.go b/services/core/internal/db/sqlc/runtime_allocations.sql.go index 4725ab1a1..2eed749c1 100644 --- a/services/core/internal/db/sqlc/runtime_allocations.sql.go +++ b/services/core/internal/db/sqlc/runtime_allocations.sql.go @@ -12,8 +12,8 @@ import ( ) const createRuntimeAllocation = `-- name: CreateRuntimeAllocation :one -INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation) -VALUES ($1, $2, $3, $4, $5, $6) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation +INSERT INTO runtime_allocations (id, environment_id, device_id, provider_key, node_id, deployment_generation, compute_state) +VALUES ($1, $2, $3, $4, $5, $6, jsonb_build_object('protocol_version', $7::text)) RETURNING id, environment_id, device_id, provider_key, state, create_settled, created_at, released_at, compute_phase, compute_revision, compute_state, compute_activity_at, compute_wake_requested, compute_retained_until, node_id, observation_error, compute_phase_changed_at, deployment_generation ` type CreateRuntimeAllocationParams struct { @@ -23,6 +23,7 @@ type CreateRuntimeAllocationParams struct { ProviderKey pgtype.UUID `json:"provider_key"` NodeID pgtype.UUID `json:"node_id"` DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` + ProtocolVersion string `json:"protocol_version"` } func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntimeAllocationParams) (RuntimeAllocation, error) { @@ -33,6 +34,7 @@ func (q *Queries) CreateRuntimeAllocation(ctx context.Context, arg CreateRuntime arg.ProviderKey, arg.NodeID, arg.DeploymentGeneration, + arg.ProtocolVersion, ) var i RuntimeAllocation err := row.Scan( diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index 8b01bf155..f0ac40ce3 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -27,18 +27,39 @@ func (q *Queries) ClearRuntimeWake(ctx context.Context, arg ClearRuntimeWakePara return err } +const countRuntimeComputeReservations = `-- name: CountRuntimeComputeReservations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended' +` + +func (q *Queries) CountRuntimeComputeReservations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { + row := q.db.QueryRow(ctx, countRuntimeComputeReservations, providerKey) + var count int64 + err := row.Scan(&count) + return count, err +} + +const countRuntimeRetainedAllocations = `-- name: CountRuntimeRetainedAllocations :one +SELECT count(*) FROM runtime_allocations +WHERE provider_key = $1 AND state <> 'released' +` + +func (q *Queries) CountRuntimeRetainedAllocations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { + row := q.db.QueryRow(ctx, countRuntimeRetainedAllocations, providerKey) + var count int64 + err := row.Scan(&count) + return count, err +} + const getRuntimeActivity = `-- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, - CASE WHEN a.node_id IS NULL THEN COALESCE((SELECT max(t.completed_at) FROM turns t WHERE t.session_id = e.session_id), a.created_at) END, - CASE WHEN a.node_id IS NULL THEN (SELECT max(t.completed_at) FROM subagent_turns t WHERE t.session_id = e.session_id) END, (SELECT max(f.settled_at) FROM environment_file_writes f WHERE f.environment_id = e.id))::timestamptz AS last_activity, (EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM subagent_turns t WHERE t.session_id = e.session_id AND t.status IN ('queued','in_progress','waiting')) OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = e.session_id AND r.state = 'pending') OR EXISTS (SELECT 1 FROM environment_file_writes f WHERE f.environment_id = e.id AND f.state = 'pending'))::boolean AS busy, - a.compute_wake_requested, - EXISTS (SELECT 1 FROM turns t WHERE t.session_id = e.session_id AND t.completed_at IS NOT NULL)::boolean AS has_completed_turn + a.compute_wake_requested FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE a.id = $1 ` @@ -48,7 +69,6 @@ type GetRuntimeActivityRow struct { LastActivity pgtype.Timestamptz `json:"last_activity"` Busy bool `json:"busy"` ComputeWakeRequested bool `json:"compute_wake_requested"` - HasCompletedTurn bool `json:"has_completed_turn"` } func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRuntimeActivityRow, error) { @@ -59,16 +79,33 @@ func (q *Queries) GetRuntimeActivity(ctx context.Context, id pgtype.UUID) (GetRu &i.LastActivity, &i.Busy, &i.ComputeWakeRequested, - &i.HasCompletedTurn, ) return i, err } +const hasIncompatibleRuntimeComputeState = `-- name: HasIncompatibleRuntimeComputeState :one +SELECT EXISTS ( + SELECT 1 FROM runtime_allocations + WHERE state <> 'released' + AND ((compute_state->>'protocol_version') IS DISTINCT FROM $1::text + OR compute_state ? 'snapshot' + OR (compute_state #> '{current,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot'] + OR (compute_state #> '{target,RestoredFrom}') ?| ARRAY['Digest', 'CheckpointID', 'CheckpointRoot']) +)::boolean +` + +func (q *Queries) HasIncompatibleRuntimeComputeState(ctx context.Context, protocolVersion string) (bool, error) { + row := q.db.QueryRow(ctx, hasIncompatibleRuntimeComputeState, protocolVersion) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + const recordRuntimeTerminalActivity = `-- name: RecordRuntimeTerminalActivity :exec UPDATE runtime_allocations a SET compute_activity_at = clock_timestamp() FROM environments e WHERE a.environment_id = e.id AND e.session_id = $1 - AND a.node_id IS NOT NULL AND a.state = 'running' + AND a.state = 'running' ` func (q *Queries) RecordRuntimeTerminalActivity(ctx context.Context, sessionID pgtype.UUID) error { diff --git a/services/core/internal/deployment/activity_test.go b/services/core/internal/deployment/activity_test.go index 2f308dfb2..a1b5d0c02 100644 --- a/services/core/internal/deployment/activity_test.go +++ b/services/core/internal/deployment/activity_test.go @@ -12,23 +12,22 @@ func TestRuntimeIdleAdmissionUsesDatabaseClock(t *testing.T) { t.Run(skew.String(), func(t *testing.T) { observed := coreNow.Add(skew) for _, test := range []struct { - name string - age time.Duration - busy, wake, completed bool - timeout time.Duration - want bool + name string + age time.Duration + busy, wake bool + timeout time.Duration + want bool }{ - {"recent", timeout - time.Second, false, false, true, timeout, false}, - {"boundary", timeout, false, false, true, timeout, true}, - {"elapsed", timeout + time.Second, false, false, true, timeout, true}, - {"busy", timeout + time.Second, true, false, true, timeout, false}, - {"wake", timeout + time.Second, false, true, true, timeout, false}, - {"never_completed", timeout + time.Second, false, false, false, timeout, false}, - {"missing_policy", timeout + time.Second, false, false, true, 0, false}, - {"invalid_policy", timeout + time.Second, false, false, true, -time.Second, false}, + {"recent", timeout - time.Second, false, false, timeout, false}, + {"boundary", timeout, false, false, timeout, true}, + {"elapsed", timeout + time.Second, false, false, timeout, true}, + {"busy", timeout + time.Second, true, false, timeout, false}, + {"wake", timeout + time.Second, false, true, timeout, false}, + {"missing_policy", timeout + time.Second, false, false, 0, false}, + {"invalid_policy", timeout + time.Second, false, false, -time.Second, false}, } { t.Run(test.name, func(t *testing.T) { - activity := Activity{ObservedAt: observed, LastActivity: observed.Add(-test.age), Busy: test.busy, WakeRequested: test.wake, HasCompletedTurn: test.completed} + activity := Activity{ObservedAt: observed, LastActivity: observed.Add(-test.age), Busy: test.busy, WakeRequested: test.wake} if got := activity.ReadyToSuspend(test.timeout); got != test.want { t.Fatalf("suspension admission = %v, want %v with database clock skew %s", got, test.want, skew) } diff --git a/services/core/internal/deployment/allocation.go b/services/core/internal/deployment/allocation.go index c51f2527b..820c398f2 100644 --- a/services/core/internal/deployment/allocation.go +++ b/services/core/internal/deployment/allocation.go @@ -76,13 +76,13 @@ type SessionDevice struct{ ID, EnvironmentID string } // Activity separates real work from the connection keepalive. ObservedAt is // the database clock when it was read. type Activity struct { - LastActivity, ObservedAt time.Time - Busy, WakeRequested, HasCompletedTurn bool + LastActivity, ObservedAt time.Time + Busy, WakeRequested bool } // ReadyToSuspend evaluates the idle policy on one database-clock observation. func (a Activity) ReadyToSuspend(idleTimeout time.Duration) bool { - return idleTimeout > 0 && a.HasCompletedTurn && !a.Busy && !a.WakeRequested && + return idleTimeout > 0 && !a.Busy && !a.WakeRequested && a.ObservedAt.Sub(a.LastActivity) >= idleTimeout } diff --git a/services/core/internal/deployment/execution.go b/services/core/internal/deployment/execution.go index 6af321418..e2541d406 100644 --- a/services/core/internal/deployment/execution.go +++ b/services/core/internal/deployment/execution.go @@ -289,3 +289,17 @@ func (e *ExecutionOperations) recordMetadata(tx DeploymentTx, input sandbox.Sele } return tx.RecordConfigurationMetadata(record.Metadata) } + +// CheckRuntimeComputeProtocol refuses incompatible allocation receipts before activation. +func (e *ExecutionOperations) CheckRuntimeComputeProtocol(ctx context.Context, version string) error { + return e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { + incompatible, err := tx.HasIncompatibleComputeState(version) + if err != nil { + return err + } + if incompatible { + return errors.New("incompatible retained runtime state: use the previous release to archive allocations before upgrading; history is preserved") + } + return nil + }) +} diff --git a/services/core/internal/deployment/fakes_test.go b/services/core/internal/deployment/fakes_test.go index 3ccdb1d13..0317764a1 100644 --- a/services/core/internal/deployment/fakes_test.go +++ b/services/core/internal/deployment/fakes_test.go @@ -76,27 +76,29 @@ func (f *fakeExecutionStorage) WithDeployment(ctx context.Context, apply func(De } type fakeReader struct { - t testing.TB - deployment func(context.Context) (Record, error) - snapshot func(context.Context) (Snapshot, error) - ownerEpoch func(context.Context) (uint64, error) - allocation func(context.Context, sandbox.Reference) (AllocationRecord, error) - generations func(context.Context, int64) ([]GenerationRecord, error) - nodes func(context.Context) ([]NodeRecord, error) - nodeHistory func(context.Context, string, coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) - readNodes func(context.Context, func(NodeReads) error) error - resetSessions func(context.Context, string, bool) ([]ResetSession, error) - addressBindings func(context.Context, string) (AddressBindings, error) - environmentAllocation func(context.Context, AllocationKey) (Allocation, error) - credentialAllocations func(context.Context, string) ([]Allocation, error) - observationSessions func(context.Context, string, int) (ObservationSessionPage, error) - nodeAllocations func(context.Context, string) ([]NodeAllocation, error) - nodeOnline func(context.Context, string) (bool, error) - lifecycleNodes func(context.Context) ([]string, error) - lifecycleAllocations func(context.Context, string, string) ([]Allocation, error) - unallocatedEnvironments func(context.Context, string, string) ([]UnallocatedEnvironment, error) - lifecyclePlacement func(context.Context, AllocationKey) (LifecyclePlacement, error) - activity func(context.Context, string) (Activity, error) + countRetainedAllocations func(context.Context, string) (int64, error) + countComputeReservations func(context.Context, string) (int64, error) + t testing.TB + deployment func(context.Context) (Record, error) + snapshot func(context.Context) (Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (AllocationRecord, error) + generations func(context.Context, int64) ([]GenerationRecord, error) + nodes func(context.Context) ([]NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) + readNodes func(context.Context, func(NodeReads) error) error + resetSessions func(context.Context, string, bool) ([]ResetSession, error) + addressBindings func(context.Context, string) (AddressBindings, error) + environmentAllocation func(context.Context, AllocationKey) (Allocation, error) + credentialAllocations func(context.Context, string) ([]Allocation, error) + observationSessions func(context.Context, string, int) (ObservationSessionPage, error) + nodeAllocations func(context.Context, string) ([]NodeAllocation, error) + nodeOnline func(context.Context, string) (bool, error) + lifecycleNodes func(context.Context) ([]string, error) + lifecycleAllocations func(context.Context, string, string) ([]Allocation, error) + unallocatedEnvironments func(context.Context, string, string) ([]UnallocatedEnvironment, error) + lifecyclePlacement func(context.Context, AllocationKey) (LifecyclePlacement, error) + activity func(context.Context, string) (Activity, error) } func (f *fakeReader) Deployment(ctx context.Context) (Record, error) { @@ -346,6 +348,7 @@ func (f *fakeNodeTx) RefreshServingReadiness(nodeID string, protocol int) error } type fakeDeploymentTx struct { + hasIncompatibleComputeState func(string) (bool, error) t testing.TB loadDeployment func() (Record, error) loadSnapshot func() (Snapshot, error) @@ -637,3 +640,25 @@ func (f *fakeSessionReader) GetManagedSessionArchive(context.Context, string, st unexpected(f.t, "GetManagedSessionArchive") return sessions.ManagedArchive{}, nil } + +func (f *fakeDeploymentTx) HasIncompatibleComputeState(version string) (bool, error) { + if f.hasIncompatibleComputeState == nil { + f.t.Fatal("unexpected HasIncompatibleComputeState") + return false, nil + } + return f.hasIncompatibleComputeState(version) +} + +func (f *fakeReader) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { + if f.countComputeReservations == nil { + unexpected(f.t, "CountComputeReservations") + } + return f.countComputeReservations(ctx, installationID) +} + +func (f *fakeReader) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { + if f.countRetainedAllocations == nil { + unexpected(f.t, "CountRetainedAllocations") + } + return f.countRetainedAllocations(ctx, installationID) +} diff --git a/services/core/internal/deployment/service.go b/services/core/internal/deployment/service.go index 022a0d359..582b67960 100644 --- a/services/core/internal/deployment/service.go +++ b/services/core/internal/deployment/service.go @@ -253,7 +253,7 @@ func (s *Service) SetupForSelection(installationID string, input sandbox.Selecti // suspension returns Core's idle suspension policy for a provider that // declares checkpoint support, and nil for any other provider. func (s *Service) suspension(provider string) (*Suspension, error) { - checkpoint, err := s.registry.SupportsCheckpoint(provider) + checkpoint, err := s.registry.SupportsSuspension(provider) if err != nil || !checkpoint { return nil, err } diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index 2eb526b14..2364910bb 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -247,6 +247,12 @@ type Reader interface { // Activity returns the allocation's activity; a missing allocation is // ErrNotFound. Activity(ctx context.Context, allocationID string) (Activity, error) + // CountComputeReservations counts the installation's allocations that + // reserve active compute. + CountComputeReservations(ctx context.Context, installationID string) (int64, error) + // CountRetainedAllocations counts the installation's allocations that + // retain resources. + CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) } // NodeReads loads node authentication facts. @@ -292,6 +298,8 @@ type NodeTx interface { // DeploymentTx is one leased deployment change. type DeploymentTx interface { + // HasIncompatibleComputeState checks all unreleased allocation protocol receipts. + HasIncompatibleComputeState(version string) (bool, error) LoadDeployment() (Record, error) LoadSnapshot() (Snapshot, error) CountResources() (Resources, error) diff --git a/services/core/internal/execution/archive_cancellation_cleanup_test.go b/services/core/internal/execution/archive_cancellation_cleanup_test.go index 3ac684477..0823c6cc0 100644 --- a/services/core/internal/execution/archive_cancellation_cleanup_test.go +++ b/services/core/internal/execution/archive_cancellation_cleanup_test.go @@ -107,7 +107,7 @@ func TestArchiveWaitingCleanupReceiptBarrier(t *testing.T) { } currentCompute := sandbox.Compute{ID: uuid.NewString(), Name: owner.ID + "-g0"} if checkpoint { - state, _ := json.Marshal(runtimeCompute{Current: currentCompute}) + state, _ := json.Marshal(runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: currentCompute}) owner, err = leased.Deployment.SetCompute(t.Context(), owner, "running", state, nil, 0) if err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/deployment_fixture_test.go b/services/core/internal/execution/deployment_fixture_test.go index 1cb8c27b5..67716db1b 100644 --- a/services/core/internal/execution/deployment_fixture_test.go +++ b/services/core/internal/execution/deployment_fixture_test.go @@ -140,27 +140,29 @@ func (s *strictExecutionStorage) WithDeployment(ctx context.Context, apply func( // strictDeploymentReader runs each set func; any other call fails the test. type strictDeploymentReader struct { - t *testing.T - deployment func(context.Context) (deployment.Record, error) - snapshot func(context.Context) (deployment.Snapshot, error) - ownerEpoch func(context.Context) (uint64, error) - allocation func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) - generations func(context.Context, int64) ([]deployment.GenerationRecord, error) - nodes func(context.Context) ([]deployment.NodeRecord, error) - nodeHistory func(context.Context, string, coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) - readNodes func(context.Context, func(deployment.NodeReads) error) error - resetSessions func(context.Context, string, bool) ([]deployment.ResetSession, error) - addressBindings func(context.Context, string) (deployment.AddressBindings, error) - environmentAllocation func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) - credentialAllocations func(context.Context, string) ([]deployment.Allocation, error) - observationSessions func(context.Context, string, int) (deployment.ObservationSessionPage, error) - nodeAllocations func(context.Context, string) ([]deployment.NodeAllocation, error) - nodeOnline func(context.Context, string) (bool, error) - lifecycleNodes func(context.Context) ([]string, error) - lifecycleAllocations func(context.Context, string, string) ([]deployment.Allocation, error) - unallocatedEnvironments func(context.Context, string, string) ([]deployment.UnallocatedEnvironment, error) - lifecyclePlacement func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) - activity func(context.Context, string) (deployment.Activity, error) + countRetainedAllocations func(context.Context, string) (int64, error) + countComputeReservations func(context.Context, string) (int64, error) + t *testing.T + deployment func(context.Context) (deployment.Record, error) + snapshot func(context.Context) (deployment.Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) + generations func(context.Context, int64) ([]deployment.GenerationRecord, error) + nodes func(context.Context) ([]deployment.NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) + readNodes func(context.Context, func(deployment.NodeReads) error) error + resetSessions func(context.Context, string, bool) ([]deployment.ResetSession, error) + addressBindings func(context.Context, string) (deployment.AddressBindings, error) + environmentAllocation func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) + credentialAllocations func(context.Context, string) ([]deployment.Allocation, error) + observationSessions func(context.Context, string, int) (deployment.ObservationSessionPage, error) + nodeAllocations func(context.Context, string) ([]deployment.NodeAllocation, error) + nodeOnline func(context.Context, string) (bool, error) + lifecycleNodes func(context.Context) ([]string, error) + lifecycleAllocations func(context.Context, string, string) ([]deployment.Allocation, error) + unallocatedEnvironments func(context.Context, string, string) ([]deployment.UnallocatedEnvironment, error) + lifecyclePlacement func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) + activity func(context.Context, string) (deployment.Activity, error) } func (r *strictDeploymentReader) Deployment(ctx context.Context) (deployment.Record, error) { @@ -341,3 +343,17 @@ func (r *strictDeploymentReader) Activity(ctx context.Context, allocationID stri } return r.activity(ctx, allocationID) } + +func (r *strictDeploymentReader) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { + if r.countComputeReservations == nil { + return 0, unexpectedDeploymentCall(r.t, "CountComputeReservations") + } + return r.countComputeReservations(ctx, installationID) +} + +func (r *strictDeploymentReader) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { + if r.countRetainedAllocations == nil { + return 0, unexpectedDeploymentCall(r.t, "CountRetainedAllocations") + } + return r.countRetainedAllocations(ctx, installationID) +} diff --git a/services/core/internal/execution/provider_operations_fixture_test.go b/services/core/internal/execution/provider_operations_fixture_test.go index 98e951c04..c1a132c1a 100644 --- a/services/core/internal/execution/provider_operations_fixture_test.go +++ b/services/core/internal/execution/provider_operations_fixture_test.go @@ -17,11 +17,12 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -30,7 +31,7 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { func (*lifecycleOnlySandbox) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*lifecycleOnlySandbox) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*lifecycleOnlySandbox) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*lifecycleOnlySandbox) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (*lifecycleOnlySandbox) Resume(context.Context, sandbox.ResumeRequest) (san func (*lifecycleOnlySandbox) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*lifecycleOnlySandbox) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*lifecycleOnlySandbox) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*lifecycleOnlySandbox) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -57,3 +58,7 @@ func (*lifecycleOnlySandbox) ResumeCompute(context.Context, sandbox.Reference, s func (*lifecycleOnlySandbox) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } + +func (*lifecycleOnlySandbox) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 20d00602b..cbf208557 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -1,9 +1,11 @@ package execution import ( + "bytes" "context" "encoding/json" "errors" + "io" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" @@ -17,20 +19,24 @@ import ( // RuntimeSuspensionPolicy is the idle suspension policy of a provider that // suspends sandboxes. type RuntimeSuspensionPolicy struct { + MaxActive int + MaxRetained int IdleTimeout time.Duration Retention time.Duration } type runtimeCompute struct { - Current sandbox.Compute `json:"current"` - Target *sandbox.Compute `json:"target,omitempty"` - Snapshot *sandbox.SnapshotIdentity `json:"snapshot,omitempty"` - SuspendID string `json:"suspend_id,omitempty"` - RestoreID string `json:"restore_id,omitempty"` - Rollback bool `json:"rollback,omitempty"` + Version string `json:"protocol_version"` + Current sandbox.Compute `json:"current"` + Target *sandbox.Compute `json:"target,omitempty"` + Retained *sandbox.RetainedState `json:"retained,omitempty"` + SuspendID string `json:"suspend_id,omitempty"` + RestoreID string `json:"restore_id,omitempty"` + Rollback bool `json:"rollback,omitempty"` } func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner deployment.Allocation, phase string, state runtimeCompute, until *time.Time) (deployment.Allocation, error) { + state.Version = sandbox.SuspensionStateVersion raw, err := json.Marshal(state) if err != nil { return owner, err @@ -71,7 +77,7 @@ func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment. return err } var state runtimeCompute - if json.Unmarshal(owner.ComputeState, &state) != nil || state.Current.ID == "" { + if decodeRuntimeCompute(owner.ComputeState, &state) != nil || state.Current.ID == "" { return sandbox.ErrOwnership } if owner.SessionDeleted || owner.Expired || owner.State == "cleanup_pending" { @@ -113,6 +119,13 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro if compute.Status != "running" || !compute.BootstrapComplete { return sandbox.ErrComputeUnconfirmed } + renewed, err := p.RenewCompute(ctx, runtimeReference(owner), state.Current) + if err != nil { + return err + } + if sandbox.ValidateComputeResult(state.Current, renewed.Compute) != nil || renewed.Status != "running" || !renewed.BootstrapComplete { + return sandbox.ErrComputeUnconfirmed + } peer, err := authorizedRuntimePeer(ctx, r.sessions, r.registry, owner.DeviceID) if err != nil { return err @@ -173,15 +186,15 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro } func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) error { - result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Snapshot: state.Snapshot, ObserveOnly: observeOnly}) + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Retained: state.Retained, ReconcileOnly: observeOnly}) if err != nil { return err } - if result.Compute.ID != state.Current.ID { - return sandbox.ErrOwnership + if err := sandbox.ValidateSuspendResult(sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Retained: state.Retained, ReconcileOnly: observeOnly}, result); err != nil { + return err } - if result.Snapshot == nil { - if !observeOnly || result.SourceStopped || (result.Status != "running" && result.Status != "paused") { + if result.Retained == nil { + if !observeOnly || !result.SuspendSettled || result.ResourcesReleased || (result.Status != "running" && result.Status != "paused") { return sandbox.ErrComputeUnconfirmed } state.Rollback = true @@ -191,17 +204,8 @@ func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.Sandbox } return r.wakeCompute(ctx, p, next, state) } - state.Snapshot = result.Snapshot - // Store the verified artifact before any recovery-path kill. Snapshot failure - // or an unknown result cannot silently fall back to a cold Environment. - next, err := r.saveCompute(ctx, owner, "suspending", state, owner.ComputeRetainedUntil) - if err != nil { - return err - } - if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { - return err - } - _, err = r.saveCompute(ctx, next, "suspended", state, next.ComputeRetainedUntil) + state.Retained = result.Retained + _, err = r.saveCompute(ctx, owner, "suspended", state, owner.ComputeRetainedUntil) return err } @@ -213,13 +217,19 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San if !activity.Busy && !activity.WakeRequested { return nil } - if state.Snapshot == nil || state.Target != nil { + if err := r.computeCapacityForAllocation(ctx, owner); err != nil { + return err + } + if state.Retained == nil || state.Target != nil { return sandbox.ErrOwnership } - target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Snapshot) + target, err := p.NewCompute(ctx, runtimeReference(owner), state.Current.Generation+1, state.Retained) if err != nil { return err } + if target.Name == "" || target.Generation != state.Current.Generation+1 || target.RestoredFrom == nil || *target.RestoredFrom != *state.Retained { + return sandbox.ErrOwnership + } state.Target, state.RestoreID = &target, uuid.NewString() next, err := r.saveCompute(ctx, owner, "restoring", state, owner.ComputeRetainedUntil) if err != nil { @@ -228,28 +238,18 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.San return r.restoreCompute(ctx, p, next, state, false) } func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) error { - if state.Target == nil || state.Snapshot == nil || state.Rollback { + if state.Target == nil || state.Retained == nil || state.Rollback { return sandbox.ErrOwnership } - var workspace *workspacefs.Binding - if r.config.Workspace != nil { - if r.workspaces == nil { - return workspacefs.ErrUnavailable - } - var err error - workspace, err = r.workspaces.GetReady(ctx, owner.TenantID, owner.EnvironmentID, r.config.WorkspaceRequirements, r.config.Resources.EnvironmentDiskMiB) - if err == nil && workspace == nil { - err = workspacefs.ErrNotFound - } - if err != nil { - return err - } + workspace, err := r.restoreWorkspace(ctx, owner) + if err != nil { + return err } - result, err := p.Resume(ctx, sandbox.ResumeRequest{Workspace: workspace, Reference: runtimeReference(owner), OperationID: state.RestoreID, Snapshot: *state.Snapshot, Target: *state.Target, ObserveOnly: observeOnly}) + result, err := p.Resume(ctx, sandbox.ResumeRequest{Workspace: workspace, Reference: runtimeReference(owner), OperationID: state.RestoreID, Retained: *state.Retained, Target: *state.Target, ReconcileOnly: observeOnly}) if err != nil { return err } - if result.Status != "running" || result.Compute.ID == "" { + if result.Status != "running" || !result.BootstrapComplete || sandbox.ValidateComputeResult(*state.Target, result.Compute) != nil { return sandbox.ErrComputeUnconfirmed } state.Current, state.Target = result.Compute, nil @@ -268,3 +268,75 @@ func ignoreComputeAbsent(err error) error { } return err } + +// Node-backed restores reserve capacity atomically in SetCompute. +func (r *runtimeLifecycle) computeCapacityForAllocation(ctx context.Context, owner deployment.Allocation) error { + if owner.NodeID != "" { + return nil + } + return r.computeCapacity(ctx, owner.ProviderKey) +} + +func decodeRuntimeCompute(raw []byte, state *runtimeCompute) error { + d := json.NewDecoder(bytes.NewReader(raw)) + d.DisallowUnknownFields() + if err := d.Decode(state); err != nil { + return err + } + if d.Decode(new(any)) != io.EOF || state.Version != sandbox.SuspensionStateVersion { + return sandbox.ErrOwnership + } + return nil +} + +func (r *runtimeLifecycle) computeCapacity(ctx context.Context, key string) error { + policy := r.config.Suspension + if key != r.config.InstallationID { + return sandbox.ErrOwnership + } + if policy == nil { + return nil + } + count, err := r.reader.CountComputeReservations(ctx, key) + if err != nil { + return err + } + if count >= int64(policy.MaxActive) { + return ErrExecutionUnavailable + } + return nil +} + +// restoreWorkspace follows the allocation's immutable generation, never the +// current deployment's filesystem selection or sizing. +func (r *runtimeLifecycle) restoreWorkspace(ctx context.Context, owner deployment.Allocation) (*workspacefs.Binding, error) { + record, err := r.reader.Allocation(ctx, runtimeReference(owner)) + if err != nil { + return nil, err + } + if record.ID != owner.ID || record.InstallationID != owner.ProviderKey || record.Generation != owner.DeploymentGeneration || record.Released { + return nil, sandbox.ErrOwnership + } + raw := record.Deployment.Specification + if record.Generation != record.Deployment.Generation { + if record.Retained == nil || record.Retained.Generation != record.Generation { + return nil, sandbox.ErrOwnership + } + raw = record.Retained.Specification + } + var spec sandbox.DeploymentSpec + if err := json.Unmarshal(raw, &spec); err != nil { + return nil, sandbox.ErrInvalid + } + if spec.Workspace == nil { + return nil, nil + } + if r.workspaces == nil { + return nil, workspacefs.ErrUnavailable + } + binding, err := r.workspaces.GetReady(ctx, owner.TenantID, owner.EnvironmentID, r.config.WorkspaceRequirements, spec.Resources.EnvironmentDiskMiB) + if err == nil && binding == nil { + return nil, workspacefs.ErrNotFound + } + return binding, err +} diff --git a/services/core/internal/execution/runtime_compute_compatibility_test.go b/services/core/internal/execution/runtime_compute_compatibility_test.go new file mode 100644 index 000000000..cdf5ba6eb --- /dev/null +++ b/services/core/internal/execution/runtime_compute_compatibility_test.go @@ -0,0 +1,60 @@ +package execution + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" + "reflect" + "testing" +) + +// This literal freezes the deployed beta v1 shape independently of current types. +func TestBetaRetainedV1RoundTripPreservesEveryField(t *testing.T) { + const raw = `{"protocol_version":"1","current":{"Generation":4,"Name":"owned-source","ID":"native-source","RestoredFrom":{"Reference":"owned/reference","ID":"native-parent","Data":"{\"version\":1,\"sandbox_id\":\"native-source\"}","OperationID":"prior-operation","SourceGeneration":3,"SourceName":"prior-source","SourceID":"native-parent"}},"target":{"Generation":5,"Name":"owned-target","ID":"","RestoredFrom":{"Reference":"owned/reference","ID":"native-source","Data":"{\"version\":1,\"sandbox_id\":\"native-source\"}","OperationID":"pause-operation","SourceGeneration":4,"SourceName":"owned-source","SourceID":"native-source"}},"retained":{"Reference":"owned/reference","ID":"native-source","Data":"{\"version\":1,\"sandbox_id\":\"native-source\"}","OperationID":"pause-operation","SourceGeneration":4,"SourceName":"owned-source","SourceID":"native-source"},"suspend_id":"pause-operation","restore_id":"resume-operation","rollback":true}` + var state runtimeCompute + if err := decodeRuntimeCompute([]byte(raw), &state); err != nil { + t.Fatal(err) + } + out, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + var before, after any + if json.Unmarshal([]byte(raw), &before) != nil || json.Unmarshal(out, &after) != nil || !reflect.DeepEqual(before, after) { + t.Fatal("beta v1 field loss or rewrite") + } + for _, invalid := range []string{`{}`, `{"protocol_version":"2"}`, `{"protocol_version":"1","snapshot":{}}`, `{"protocol_version":"1","current":{"RestoredFrom":{"Digest":"legacy"}}}`, `{"protocol_version":"1","target":{"RestoredFrom":{"CheckpointRoot":"legacy"}}}`, raw + `{}`} { + if decodeRuntimeCompute([]byte(invalid), new(runtimeCompute)) == nil { + t.Fatal("incompatible receipt accepted") + } + } +} + +func TestRestoreUsesAllocationGenerationWorkspaceAfterDeploymentChange(t *testing.T) { + owner := deployment.Allocation{ID: "allocation", ProviderKey: "installation", DeploymentGeneration: 4} + for _, originalExternal := range []bool{false, true} { + t.Run(fmt.Sprint(originalExternal), func(t *testing.T) { + original, current := sandbox.DeploymentSpec{}, sandbox.DeploymentSpec{Workspace: &workspacefs.Declaration{}} + if originalExternal { + original, current = current, original + } + prior, _ := json.Marshal(original) + latest, _ := json.Marshal(current) + r := runtimeLifecycle{config: RuntimeProvider{Workspace: current.Workspace}, reader: &strictDeploymentReader{t: t, allocation: func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) { + return deployment.AllocationRecord{ID: owner.ID, InstallationID: owner.ProviderKey, Generation: 4, Deployment: deployment.Record{Generation: 5, Specification: latest}, Retained: &deployment.GenerationRecord{Generation: 4, Specification: prior}}, nil + }}} + binding, err := r.restoreWorkspace(t.Context(), owner) + if originalExternal { + if !errors.Is(err, workspacefs.ErrUnavailable) { + t.Fatal("original external storage was bypassed", err) + } + } else if err != nil || binding != nil { + t.Fatal("old owned disk depended on new workspace", err) + } + }) + } +} diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index 0dd7f915d..3d485e21d 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -6,6 +6,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -25,7 +26,7 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxPro } // This idempotent control signal is fenced by guest PID/start time and the // suspension token. It cannot execute or replay an agent request. - result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", "/run/oac/daemon-suspend.json", "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) + result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", runtimebootstrap.SuspendControlFile, "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) if err != nil { return err } @@ -55,8 +56,8 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxPro } // The artifact has been consumed. Never restore it after this generation // admits work, even if garbage collection or the final database commit fails. - if state.Snapshot != nil { - if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + if state.Retained != nil { + if err := ignoreComputeAbsent(p.DeleteRetained(ctx, runtimeReference(owner), *state.Retained)); err != nil { return err } } @@ -76,13 +77,13 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Sandbox } // An uncommitted artifact is found by its persisted attempt, never a directory // glob. The helper's allocation lock also waits for an earlier unknown call. - if owner.ComputePhase == "suspending" && state.Snapshot == nil { - result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, ObserveOnly: true}) + if owner.ComputePhase == "suspending" && state.Retained == nil { + result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, ReconcileOnly: true}) if err != nil && !errors.Is(err, sandbox.ErrNotFound) { return err } if err == nil { - state.Snapshot = result.Snapshot + state.Retained = result.Retained } } if state.Target != nil { @@ -93,8 +94,8 @@ func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.Sandbox if err := ignoreComputeAbsent(p.KillCompute(ctx, runtimeReference(owner), state.Current)); err != nil { return err } - if state.Snapshot != nil { - if err := ignoreComputeAbsent(p.DeleteSnapshot(ctx, runtimeReference(owner), *state.Snapshot)); err != nil { + if state.Retained != nil { + if err := ignoreComputeAbsent(p.DeleteRetained(ctx, runtimeReference(owner), *state.Retained)); err != nil { return err } } diff --git a/services/core/internal/execution/runtime_fresh_hint_test.go b/services/core/internal/execution/runtime_fresh_hint_test.go index 334052b4e..fec4b3817 100644 --- a/services/core/internal/execution/runtime_fresh_hint_test.go +++ b/services/core/internal/execution/runtime_fresh_hint_test.go @@ -4,11 +4,6 @@ import ( "context" "encoding/json" "errors" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/adminaudit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" - projectpg "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/projectpg" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/projects" "sync" "sync/atomic" "testing" @@ -17,6 +12,7 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/identity" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -43,11 +39,10 @@ func TestFreshHintRoutesAndPreservesLifecycleGuards(t *testing.T) { t.Run(scenario, func(t *testing.T) { m := testRuntimeManager(t) m.setupGate = make(chan struct{}, 1) - m.config = RuntimeProvider{InstallationID: uuid.NewString(), Mode: "direct", Provider: &freshHintProvider{}} + m.config = RuntimeProvider{InstallationID: uuid.NewString(), ProviderKind: "e2b", Mode: "direct", Provider: &freshHintProvider{}} nodeID := "" if scenario == "node" || scenario == "released_placement" { - m.config.ProviderKind = "docker" - m.config.Mode = "nodes" + m.config.ProviderKind, m.config.Mode = "docker", "nodes" nodeID = uuid.NewString() } node, err := m.node(nodeID) @@ -56,7 +51,7 @@ func TestFreshHintRoutesAndPreservesLifecycleGuards(t *testing.T) { } environment := sessions.Environment{ID: uuid.NewString(), Configuration: json.RawMessage(`{"type":"openai_hosted"}`)} if scenario == "self_hosted" { - environment.Configuration = json.RawMessage(`{"type":"self_hosted","workspace_directory":"/workspace"}`) + environment.Configuration = json.RawMessage(`{"type":"self_hosted"}`) } if scenario == "suspended" { environment.Initialization = "complete" @@ -73,7 +68,7 @@ func TestFreshHintRoutesAndPreservesLifecycleGuards(t *testing.T) { return deployment.Allocation{}, deployment.ErrNotFound }, lifecyclePlacement: func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) { - return deployment.LifecyclePlacement{Provider: m.config.ProviderKind, PlacementNodeID: nodeID, PlacementReleased: scenario == "released_placement"}, nil + return deployment.LifecyclePlacement{Provider: m.config.ProviderKind, Mode: m.config.Mode, PlacementNodeID: nodeID, PlacementReleased: scenario == "released_placement"}, nil }, } m.deploymentService, _ = deploymentOperations(t, &strictDeploymentStorage{t: t}, reader, &strictExecutionStorage{t: t}) @@ -110,19 +105,19 @@ func (p *freshHintProvider) Create(_ context.Context, b sandbox.Bootstrap) (sand } func TestFreshEnvironmentHintProvisionsWithoutMaintenanceTick(t *testing.T) { - for _, mode := range []string{"create", "recovered_input", "recovered_initial"} { + for _, mode := range []string{"create", "create_initial", "recovered_input", "recovered_initial"} { t.Run(mode, func(t *testing.T) { owner, deployments, reader, pool := resetManagerDB(t, nil) installation := initializeE2BDeployment(t, owner) - sessionReader, s := testSessions(t, pool, pgtest.CredentialKey(t)) + sessionReader, sessionService := testSessions(t, pool, pgtest.CredentialKey(t)) provider := &freshHintProvider{created: make(chan struct{}, 1)} m := testRuntimeManager(t) m.setupGate = make(chan struct{}, 1) m.sessions, m.sessionExecution = sessionReader, owner.Sessions m.deployment, m.deploymentService, m.deploymentReader = owner.Deployment, deployments, reader m.lease, m.registry = owner.Lease, runtimegateway.NewRegistry() - m.config = RuntimeProvider{Mode: "direct", InstallationID: installation, CoreURL: fixturePublicURL + "/api/v1", Provider: provider} - worker := &Worker{lease: owner.Lease, runtimes: m, dispatcher: &Dispatcher{Sessions: s, sessionExecution: owner.Sessions, SessionsReader: sessionReader, DeploymentReader: reader, notifications: &executionNotifications{}}, scheduleWake: make(chan struct{}, 1)} + m.config = RuntimeProvider{InstallationID: installation, Generation: 1, ProviderKind: "e2b", Mode: "direct", CoreURL: fixturePublicURL + "/api/v1", Provider: provider} + worker := &Worker{lease: owner.Lease, runtimes: m, dispatcher: &Dispatcher{Sessions: sessionService, SessionsReader: sessionReader, DeploymentReader: reader, notifications: &executionNotifications{}}, scheduleWake: make(chan struct{}, 1)} node, err := m.node("") if err != nil { t.Fatal(err) @@ -140,33 +135,24 @@ func TestFreshEnvironmentHintProvisionsWithoutMaintenanceTick(t *testing.T) { }() t.Cleanup(func() { cancel(); <-done }) <-scans // Startup scan finishes before any Session is committed. - projectID := uuid.NewString() - audit := adminaudit.WithSource(ctx, adminaudit.Source{CredentialID: "fixture-admin", ProjectID: projectID, RequestID: uuid.NewString(), TraceID: uuid.NewString()}) - management, err := projects.NewService(projectpg.New(pgunit.NewPool(pool))) - if err != nil { - t.Fatal(err) - } - project, err := management.CreateProject(audit, projects.CreateProject{ID: projectID, Name: "Fresh hint"}) - if err != nil { - t.Fatal(err) - } - tenant := project.TenantID + tenant := uuid.NewString() input := sessions.CreateSession{Creator: identity.Subject{Kind: "service_account", ID: "fixture"}, Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test-model"},"environment":{"type":"openai_hosted"}}`), ModelProvider: &v1.ModelProviderInput{Protocol: "responses", BaseURL: "https://model.fixture.example/v1", APIKey: "fixture-key"}, ModelProviderSource: v1.ExecutionSourceSession} messages := []sessions.Input{{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"fixture"}]}]}`)}} - if mode == "recovered_initial" { + if mode == "recovered_initial" || mode == "create_initial" { input.InitialInputs = messages } var creation sessions.Creation switch mode { - case "create": + case "create", "create_initial": + // Ordinary and streaming HTTP creation share this Worker entrypoint. creation, err = worker.CreateSession(ctx, tenant, input) case "recovered_input", "recovered_initial": - creation, err = s.CreateSession(ctx, tenant, input) + creation, err = sessionService.CreateSession(ctx, tenant, input) } + session := creation.Session if err != nil { t.Fatal(err) } - session := creation.Session if mode == "recovered_input" || mode == "recovered_initial" { key := uuid.NewString() if mode == "recovered_initial" { diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 8b0eea006..feca79400 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -101,12 +101,9 @@ func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway. if copied.ProviderKind == "" || (copied.Mode != string(sandbox.DeploymentNodes) && copied.Mode != string(sandbox.DeploymentDirect)) { return RuntimeProvider{}, sandbox.ErrInvalid } - if copied.Mode == string(sandbox.DeploymentDirect) && copied.Suspension != nil { - return RuntimeProvider{}, sandbox.ErrInvalid - } if config.Suspension != nil { policy := *config.Suspension - if !sandbox.SupportsCheckpoint(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second { + if !sandbox.SupportsSuspension(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second || (copied.Mode == string(sandbox.DeploymentDirect) && (policy.MaxActive < 1 || policy.MaxRetained < policy.MaxActive)) { return RuntimeProvider{}, sandbox.ErrInvalid } copied.Suspension = &policy @@ -169,6 +166,10 @@ func (w *Worker) ProvisionEnvironment(ctx context.Context, tenant, environment, // provision runs under the lifecycle gate and uses the durable one-shot receipt. func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, providerKey string) (deployment.Allocation, error) { provider := r.config.Provider + // The lifecycle lane, not provider configuration, owns node identity. + if (r.nodeID == "") != (r.config.Mode == string(sandbox.DeploymentDirect)) { + return deployment.Allocation{}, sandbox.ErrOwnership + } if providerKey != r.config.InstallationID { return deployment.Allocation{}, sandbox.ErrInvalid } @@ -180,6 +181,26 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if err != nil || placement.Type != "openai_hosted" { return deployment.Allocation{}, sandbox.ErrInvalid } + key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} + if _, err := r.reader.EnvironmentAllocation(ctx, key); errors.Is(err, deployment.ErrNotFound) { + if r.config.Generation == 0 { + return deployment.Allocation{}, ErrExecutionUnavailable + } + if err := r.computeFreshCapacity(ctx, providerKey); err != nil { + return deployment.Allocation{}, err + } + if policy := r.config.Suspension; policy != nil && (r.config.ProviderKind == "" || r.config.Mode == "direct") { + count, err := r.reader.CountRetainedAllocations(ctx, providerKey) + if err != nil { + return deployment.Allocation{}, err + } + if count >= int64(policy.MaxRetained) { + return deployment.Allocation{}, ErrExecutionUnavailable + } + } + } else if err != nil { + return deployment.Allocation{}, err + } var workspace *workspacefs.Binding if r.config.Workspace != nil { if r.workspaces == nil { @@ -201,11 +222,6 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p return existing, sandbox.ErrOwnership } } - key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} - session, err := r.sessions.GetSession(ctx, tenant, environmentValue.SessionID) - if err != nil { - return deployment.Allocation{}, err - } secret := make([]byte, 32) if _, err := rand.Read(secret); err != nil { return deployment.Allocation{}, err @@ -224,9 +240,13 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if err := r.lease.CheckOwnership(ctx); err != nil { return owner, err } + session, err := r.sessions.GetSession(ctx, tenant, environmentValue.SessionID) + if err != nil { + return deployment.Allocation{}, err + } info, err := provider.Create(ctx, sandbox.Bootstrap{ Reference: runtimeReference(owner), SessionID: owner.SessionID, DeviceID: owner.DeviceID, - Workspace: workspace, CoreURL: r.config.CoreURL, Credential: token, Harness: session.Engine, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, + Workspace: workspace, Harness: session.Engine, CoreURL: r.config.CoreURL, Credential: token, NetworkAccess: placement.NetworkAccess, AllowedDomains: placement.AllowedDomains, }) if info.Reference == runtimeReference(owner) && info.CreateSettled && info.State == "absent" { record, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) @@ -432,3 +452,11 @@ func runtimeReference(owner deployment.Allocation) sandbox.Reference { func (w *Worker) runManagedRuntimes(ctx context.Context) error { return w.runtimes.run(ctx) } + +// Manager deployments reserve capacity with Session placement before provisioning. +func (r *runtimeLifecycle) computeFreshCapacity(ctx context.Context, key string) error { + if r.config.ProviderKind != "" && r.config.Mode != "direct" { + return nil + } + return r.computeCapacity(ctx, key) +} diff --git a/services/core/internal/execution/runtime_workspace_settlement_test.go b/services/core/internal/execution/runtime_workspace_settlement_test.go index 101e76529..d4cf55108 100644 --- a/services/core/internal/execution/runtime_workspace_settlement_test.go +++ b/services/core/internal/execution/runtime_workspace_settlement_test.go @@ -107,7 +107,7 @@ func newWorkspaceSettlementFixture(t *testing.T, provider sandbox.SandboxProvide control := &settlementWorkspaceControl{configuration: configuration} filesystems := workspaces.NewExecution(storage, writer, settlementWorkspaceControls{control}, lease) declaration := control.Declaration() - lifecycle := &runtimeLifecycle{registry: runtimegateway.NewRegistry(), sessions: sessionReader, sessionExecution: owner.Sessions, deployment: operations, deployments: deployments, reader: reader, lease: lease, workspaces: filesystems, config: RuntimeProvider{InstallationID: installation, Provider: provider, Workspace: &declaration, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} + lifecycle := &runtimeLifecycle{registry: runtimegateway.NewRegistry(), sessions: sessionReader, sessionExecution: owner.Sessions, deployment: operations, deployments: deployments, reader: reader, lease: lease, workspaces: filesystems, config: RuntimeProvider{InstallationID: installation, Mode: "direct", Generation: 1, Provider: provider, Workspace: &declaration, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} return workspaceSettlementFixture{lifecycle: lifecycle, sessions: service, storage: storage, control: control, session: session} } diff --git a/services/core/internal/execution/runtime_workspace_test.go b/services/core/internal/execution/runtime_workspace_test.go index 8c52d590f..0999b7c99 100644 --- a/services/core/internal/execution/runtime_workspace_test.go +++ b/services/core/internal/execution/runtime_workspace_test.go @@ -16,10 +16,6 @@ import ( type workspaceEnvironmentReader struct{ sessions.Reader } -func (workspaceEnvironmentReader) GetSession(context.Context, string, string) (sessions.Session, error) { - return sessions.Session{Engine: "codex"}, nil -} - func (workspaceEnvironmentReader) GetEnvironment(context.Context, string, string) (sessions.Environment, error) { return sessions.Environment{Configuration: []byte(`{"type":"openai_hosted"}`)}, nil } @@ -99,7 +95,7 @@ func TestRuntimeWorkspaceConvergesBeforeComputeReservation(t *testing.T) { }}) r := &runtimeLifecycle{sessions: workspaceEnvironmentReader{}, deployment: operations, reader: &strictDeploymentReader{t: t, environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { return deployment.Allocation{}, deployment.ErrNotFound - }}, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{InstallationID: uuid.NewString(), Workspace: &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory}, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} + }}, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{Mode: "direct", Generation: 1, InstallationID: uuid.NewString(), Workspace: &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory}, WorkspaceRequirements: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory}}} _, err := r.provision(t.Context(), f.record.Reference.TenantID, f.record.Reference.EnvironmentID, r.config.InstallationID) if fail { if !errors.Is(err, workspacefs.ErrUnavailable) || reserved { @@ -155,7 +151,9 @@ func TestOwnedGenerationDoesNotAdoptLaterFilesystemSelection(t *testing.T) { _, operations := deploymentOperations(t, &strictDeploymentStorage{t: t}, &strictDeploymentReader{t: t}, &strictExecutionStorage{t: t, withReservation: func(context.Context, deployment.AllocationKey, func(sessions.LockedSession, deployment.ReservationTx) error) error { return stop }}) - r := &runtimeLifecycle{sessions: workspaceEnvironmentReader{}, deployment: operations, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{InstallationID: uuid.NewString()}} + r := &runtimeLifecycle{sessions: workspaceEnvironmentReader{}, deployment: operations, reader: &strictDeploymentReader{t: t, environmentAllocation: func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + return deployment.Allocation{}, deployment.ErrNotFound + }}, workspaces: workspaces.NewExecution(f, f, workspaceControls{f}, heldLease{}), config: RuntimeProvider{Mode: "direct", Generation: 1, InstallationID: uuid.NewString()}} _, err := r.provision(t.Context(), f.record.Reference.TenantID, f.record.Reference.EnvironmentID, r.config.InstallationID) if !errors.Is(err, stop) || f.binds != 0 || f.creates != 0 { t.Fatal("owned generation retroactively attached filesystem", err) diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 6d67ff3cb..fd4328c8a 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -9,6 +9,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -75,6 +76,9 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher, owner Owner) (_ *W if owner.Deployment == nil { return nil, errors.New("execution worker requires the deployment execution operations") } + if err := owner.Deployment.CheckRuntimeComputeProtocol(ctx, sandbox.SuspensionStateVersion); err != nil { + return nil, err + } owned.notifications = &executionNotifications{} worker := &Worker{concurrency: dispatcher.MaxConcurrentExecutions, dispatcher: owned, lease: owner.Lease, directoryReads: make(chan directoryReadRequest), fileWrites: make(chan fileWriteRequest), stopped: make(chan struct{}), scheduleWake: make(chan struct{}, 1), enrolledConnections: make(map[string]*runtimeConnection)} worker.runtimes, err = newRuntimeManager(owner, owned.Deployment, owned.DeploymentReader, owned.SessionsReader, owned.Registry, owned.ManagedRuntimes) diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index ee99f8f4d..b65e310ee 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -14,6 +14,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/placementpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/sessionpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -217,6 +218,7 @@ func (t *reservationTx) InsertAllocation(a deployment.NewAllocation) (deployment row, err := t.q.CreateRuntimeAllocation(t.ctx, sqlc.CreateRuntimeAllocationParams{ ID: id, EnvironmentID: t.environment, DeviceID: device, ProviderKey: provider, NodeID: node, DeploymentGeneration: pgtype.Int8{Int64: int64(a.Generation), Valid: true}, + ProtocolVersion: sandbox.SuspensionStateVersion, }) if err != nil { return deployment.Allocation{}, err @@ -352,7 +354,7 @@ func loadActivity(ctx context.Context, q *sqlc.Queries, id pgtype.UUID) (deploym if err != nil { return deployment.Activity{}, err } - return deployment.Activity{LastActivity: row.LastActivity.Time, ObservedAt: row.ObservedAt.Time, Busy: row.Busy, WakeRequested: row.ComputeWakeRequested, HasCompletedTurn: row.HasCompletedTurn}, nil + return deployment.Activity{LastActivity: row.LastActivity.Time, ObservedAt: row.ObservedAt.Time, Busy: row.Busy, WakeRequested: row.ComputeWakeRequested}, nil } func (s *Store) EnvironmentAllocation(ctx context.Context, key deployment.AllocationKey) (deployment.Allocation, error) { @@ -536,3 +538,19 @@ func (s *Store) Activity(ctx context.Context, allocationID string) (deployment.A } return loadActivity(ctx, s.pool.Queries(), id) } + +func (s *Store) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { + id, err := parseID(installationID) + if err != nil { + return 0, err + } + return s.pool.Queries().CountRuntimeComputeReservations(ctx, id) +} + +func (s *Store) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { + id, err := parseID(installationID) + if err != nil { + return 0, err + } + return s.pool.Queries().CountRuntimeRetainedAllocations(ctx, id) +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/tx.go b/services/core/internal/persistence/postgres/deploymentpg/tx.go index 28f35e1c5..7e865c437 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/tx.go +++ b/services/core/internal/persistence/postgres/deploymentpg/tx.go @@ -354,3 +354,7 @@ func (t *deploymentTx) RecordAudit(action, installationID string) error { func (t *deploymentTx) RecordAuditAs(source adminaudit.Source, action, installationID string) error { return auditpg.RecordDeploymentMutation(adminaudit.WithSource(t.ctx, source), t.q, action, "sandbox_deployment", installationID) } + +func (t *deploymentTx) HasIncompatibleComputeState(version string) (bool, error) { + return t.q.HasIncompatibleRuntimeComputeState(t.ctx, version) +} diff --git a/services/core/internal/processconfig/config.go b/services/core/internal/processconfig/config.go index 6a5b75edf..6bc5c9b72 100644 --- a/services/core/internal/processconfig/config.go +++ b/services/core/internal/processconfig/config.go @@ -41,6 +41,7 @@ const ( // Config is Core's process configuration. type Config struct { + SandboxCapacity SandboxLimits // Addr is OAC_ADDR, the listener address. Addr string // PublicOrigin is OAC_PUBLIC_URL. @@ -102,6 +103,9 @@ type runtimeHistoryFile struct { func Load() (Config, error) { var c Config var err error + if c.SandboxCapacity, err = SandboxCapacity(); err != nil { + return Config{}, err + } if c.Log, err = log.LoadConfig(); err != nil { return Config{}, err } @@ -170,6 +174,8 @@ func (c Config) Settings() []api.InstallationSetting { setting("log.level", strings.ToLower(c.Log.Level.String()), "info", []api.InstallationService{api.InstallationCore, api.InstallationWeb}), setting("log.format", format, "auto", []api.InstallationService{api.InstallationCore, api.InstallationWeb}), setting("log.add_source", c.Log.AddSource, false, []api.InstallationService{api.InstallationCore, api.InstallationWeb}), + setting("core.sandbox_capacity.max_active", c.SandboxCapacity.MaxActive, defaultSandboxMaxActive, []api.InstallationService{api.InstallationCore}), + setting("core.sandbox_capacity.max_retained", c.SandboxCapacity.MaxRetained, defaultSandboxMaxRetained, []api.InstallationService{api.InstallationCore}), setting("core.execution_concurrency", c.ExecutionConcurrency, execution.DefaultExecutionConcurrency, []api.InstallationService{api.InstallationCore}), setting("core.harnesses", c.Harnesses, (engine.Catalog{}).Kinds(), []api.InstallationService{api.InstallationCore}), setting("core.default_harness", c.DefaultHarness, defaultHarness, []api.InstallationService{api.InstallationCore}), diff --git a/services/core/internal/processconfig/sandbox_capacity.go b/services/core/internal/processconfig/sandbox_capacity.go new file mode 100644 index 000000000..742b057ff --- /dev/null +++ b/services/core/internal/processconfig/sandbox_capacity.go @@ -0,0 +1,41 @@ +package processconfig + +import ( + "os" + "strconv" +) + +const defaultSandboxMaxActive = 100 +const defaultSandboxMaxRetained = 400 + +// SandboxLimits bounds direct Providers with suspension independently of +// execution concurrency and each enrolled node's capacity. +type SandboxLimits struct { + MaxActive int + MaxRetained int +} + +// SandboxCapacity reads the process-owned active and retained limits. Unset or +// empty variables select the same defaults as the other process settings. +func SandboxCapacity() (SandboxLimits, error) { + capacity := SandboxLimits{MaxActive: defaultSandboxMaxActive, MaxRetained: defaultSandboxMaxRetained} + for _, setting := range []struct { + name string + target *int + }{ + {"OAC_SANDBOX_MAX_ACTIVE", &capacity.MaxActive}, + {"OAC_SANDBOX_MAX_RETAINED", &capacity.MaxRetained}, + } { + if raw := os.Getenv(setting.name); raw != "" { + value, err := strconv.Atoi(raw) + if err != nil || value < 1 || value > 100000 { + return SandboxLimits{}, configError(setting.name + " must be an integer between 1 and 100000") + } + *setting.target = value + } + } + if capacity.MaxRetained < capacity.MaxActive { + return SandboxLimits{}, configError("OAC_SANDBOX_MAX_RETAINED must be at least OAC_SANDBOX_MAX_ACTIVE") + } + return capacity, nil +} diff --git a/services/core/internal/processconfig/sandbox_capacity_test.go b/services/core/internal/processconfig/sandbox_capacity_test.go new file mode 100644 index 000000000..6c06fd7f0 --- /dev/null +++ b/services/core/internal/processconfig/sandbox_capacity_test.go @@ -0,0 +1,53 @@ +package processconfig + +import ( + "strings" + "testing" +) + +func TestSandboxCapacityDefaultsAndSettings(t *testing.T) { + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "") + capacity, err := SandboxCapacity() + if err != nil || capacity.MaxActive != 100 || capacity.MaxRetained != 400 { + t.Fatal("empty Compose settings lost defaults", capacity, err) + } + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "7") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "31") + capacity, err = SandboxCapacity() + settings := (Config{SandboxCapacity: capacity}).Settings() + if err != nil { + t.Fatal(err) + } + found := map[string]any{} + for _, setting := range settings { + found[setting.Key] = setting.Value + } + if found["core.sandbox_capacity.max_active"] != 7 || found["core.sandbox_capacity.max_retained"] != 31 { + t.Fatal("installation omitted effective direct capacity", found) + } +} + +func TestCheckValidatesSandboxCapacity(t *testing.T) { + for _, key := range []string{"OAC_SANDBOX_MAX_ACTIVE", "OAC_SANDBOX_MAX_RETAINED"} { + for _, tc := range []struct { + value string + valid bool + }{{"0", false}, {"-1", false}, {"1.5", false}, {"synthetic-secret-value", false}, {"100001", false}, {"100000", true}, {"1", true}, {"", true}} { + t.Run(key+"/"+tc.value, func(t *testing.T) { + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "1") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "100000") + t.Setenv(key, tc.value) + _, err := SandboxCapacity() + if (err == nil) != tc.valid || err != nil && (!strings.Contains(err.Error(), key) || strings.Contains(err.Error(), "synthetic-secret-value")) { + t.Fatal("wrong process validation", err) + } + }) + } + } + t.Setenv("OAC_SANDBOX_MAX_ACTIVE", "100") + t.Setenv("OAC_SANDBOX_MAX_RETAINED", "99") + if _, err := SandboxCapacity(); err == nil { + t.Fatal("accepted retained limit below active limit") + } +} diff --git a/services/core/internal/sandbox/docker/operations.go b/services/core/internal/sandbox/docker/operations.go index 2722da0dd..0d479f3f1 100644 --- a/services/core/internal/sandbox/docker/operations.go +++ b/services/core/internal/sandbox/docker/operations.go @@ -16,11 +16,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "NewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "GetCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Suspend": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Resume": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "KillCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "Observe": {State: providercontract.Supported}, @@ -30,7 +31,7 @@ func (*Provider) ProviderOperations() providercontract.Operations { return Opera func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} } -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} } func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Compu func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} } -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} +func (p *Provider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: Operations()["DeleteRetained"].Reason} } func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} @@ -54,3 +55,7 @@ func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} } + +func (p *Provider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: Operations()["RenewCompute"].Reason} +} diff --git a/services/core/internal/sandbox/e2b/helper_contract.go b/services/core/internal/sandbox/e2b/helper_contract.go index f6fe533b4..517357622 100644 --- a/services/core/internal/sandbox/e2b/helper_contract.go +++ b/services/core/internal/sandbox/e2b/helper_contract.go @@ -11,7 +11,7 @@ import ( //go:generate go run ./internal/contractgen // This adapter-private boundary is documented in tools/e2b-provider/README.md. -const ProtocolVersion = 2 +const ProtocolVersion = 4 const MaxOutputBytes = 1024 * 1024 const MaxRequestBytes = 72 * 1024 * 1024 const MaxResponseBytes = 16 * 1024 * 1024 @@ -20,7 +20,7 @@ const MaxCommandInputBytes = sandbox.MaxCommandInputBytes // HelperOperations declares the complete set of one-shot helper operations. func HelperOperations() []string { - return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential"} + return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential", "compute_info", "compute_renew", "suspend", "resume", "compute_kill", "delete_retained", "compute_command", "resume_compute"} } // HelperErrors are sanitized wire outcomes; an empty code denotes success. @@ -56,6 +56,28 @@ func (q Request) Validate() error { return sandbox.ErrInvalid } } + + switch q.Operation { + case "compute_info", "compute_renew", "compute_kill", "compute_command", "resume_compute": + if q.Compute == nil || q.Compute.Name != q.Reference.AllocationID || (q.Operation != "compute_info" && q.Compute.ID == "") { + return sandbox.ErrInvalid + } + if q.Operation == "compute_command" && q.Command == nil { + return sandbox.ErrInvalid + } + case "suspend": + if q.Suspend == nil || q.Suspend.Reference != q.Reference || !validID(q.Suspend.OperationID) || q.Suspend.Source.Name != q.Reference.AllocationID || q.Suspend.Source.ID == "" { + return sandbox.ErrInvalid + } + case "resume": + if q.Resume == nil || q.Resume.Workspace != nil || q.Resume.Reference != q.Reference || !validID(q.Resume.OperationID) || sandbox.ValidateRetained(q.Resume.Retained) != nil || q.Resume.Retained.Reference != q.Reference.AllocationID { + return sandbox.ErrInvalid + } + case "delete_retained": + if q.Retained == nil || sandbox.ValidateRetained(*q.Retained) != nil || q.Retained.Reference != q.Reference.AllocationID { + return sandbox.ErrInvalid + } + } return nil } @@ -69,10 +91,15 @@ type Request struct { Bootstrap *sandbox.Bootstrap `json:",omitempty"` RuntimeBootstrap *runtimebootstrap.Connection `json:",omitempty"` Command *sandbox.Command `json:",omitempty"` + Compute *sandbox.Compute `json:",omitempty"` + Suspend *sandbox.SuspendRequest `json:",omitempty"` + Resume *sandbox.ResumeRequest `json:",omitempty"` + Retained *sandbox.RetainedState `json:",omitempty"` Deadline time.Time } type Response struct { Version int + State *sandbox.ComputeState `json:",omitempty"` Info *sandbox.Info `json:",omitempty"` Command *sandbox.CommandResult `json:",omitempty"` ErrorCode string diff --git a/services/core/internal/sandbox/e2b/installed_paths_test.go b/services/core/internal/sandbox/e2b/installed_paths_test.go index 60d5b467b..35a813ef9 100644 --- a/services/core/internal/sandbox/e2b/installed_paths_test.go +++ b/services/core/internal/sandbox/e2b/installed_paths_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "strconv" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -31,7 +32,7 @@ func TestConfigurationDiscoveryUsesSuppliedProcessPaths(t *testing.T) { if err := os.MkdirAll(filepath.Dir(binary), 0700); err != nil { t.Fatal(err) } - if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":2,\"Templates\":[]}'\n"), 0700); err != nil { + if err := os.WriteFile(binary, []byte("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{\"Version\":"+strconv.Itoa(ProtocolVersion)+",\"Templates\":[]}'\n"), 0700); err != nil { t.Fatal(err) } input := sandbox.ConfigurationDiscoveryInput{Credential: json.RawMessage(`{"api_key":"synthetic-key"}`)} diff --git a/services/core/internal/sandbox/e2b/internal/contractgen/main.go b/services/core/internal/sandbox/e2b/internal/contractgen/main.go index a7a9c788f..de3685b1e 100644 --- a/services/core/internal/sandbox/e2b/internal/contractgen/main.go +++ b/services/core/internal/sandbox/e2b/internal/contractgen/main.go @@ -85,11 +85,17 @@ func main() { identity = append(identity, "InstallationID") values := map[string]any{ "PROTOCOL_VERSION": e2b.ProtocolVersion, "SDK_VERSION": sdk, - "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, - "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, - "OPERATIONS": e2b.HelperOperations(), "ERROR_CODES": e2b.HelperErrors(), + "SUSPEND_CONTROL_FILE": runtimebootstrap.SuspendControlFile, + "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, + "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, + "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, + "OPERATIONS": e2b.HelperOperations(), "ERROR_CODES": e2b.HelperErrors(), "REQUEST_FIELDS": fields(reflect.TypeFor[e2b.Request]()), "RESPONSE_FIELDS": fields(reflect.TypeFor[e2b.Response]()), "REFERENCE_FIELDS": fields(reflect.TypeFor[sandbox.Reference]()), + "COMPUTE_FIELDS": fields(reflect.TypeFor[sandbox.Compute]()), + "RETAINED_FIELDS": fields(reflect.TypeFor[sandbox.RetainedState]()), + "SUSPEND_FIELDS": fields(reflect.TypeFor[sandbox.SuspendRequest]()), + "RESUME_FIELDS": fields(reflect.TypeFor[sandbox.ResumeRequest]()), "MANAGED_BOOTSTRAP_FIELDS": bootstrap, "MANAGED_BOOTSTRAP_REQUIRED_FIELDS": requiredBootstrap, "MANAGED_IDENTITY_FIELDS": identity, "NETWORK_ACCESS": networkValues(filepath.Join(root, "internal/agentnetwork/policy.go")), } @@ -132,7 +138,24 @@ func fixtures() []byte { } for _, operation := range e2b.HelperOperations() { copy := q + copy.Operation = operation + c := sandbox.Compute{Name: r.AllocationID, ID: "native-fixture"} + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: installation, OperationID: installation, SourceName: c.Name, SourceID: c.ID, Data: "opaque"} + switch operation { + case "compute_info", "compute_renew", "compute_kill", "compute_command", "resume_compute": + copy.Compute = &c + if operation == "compute_command" { + copy.Command = &sandbox.Command{Args: []string{"true"}} + } + case "suspend": + copy.Suspend = &sandbox.SuspendRequest{Reference: r, OperationID: installation, Source: c} + case "resume": + target := sandbox.Compute{Generation: 1, Name: c.Name, ID: c.ID, RestoredFrom: &retained} + copy.Resume = &sandbox.ResumeRequest{Reference: r, OperationID: installation, Retained: retained, Target: target} + case "delete_retained": + copy.Retained = &retained + } if operation == "verify_credential" { copy.References = []sandbox.Reference{r} } @@ -157,6 +180,17 @@ func fixtures() []byte { value["Bootstrap"] = bootstrapValue add("request", fmt.Sprintf("workspace-%v", workspace), workspace == nil, value) } + for _, workspace := range []any{nil, map[string]any{}} { + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: installation, OperationID: installation, SourceName: r.AllocationID, SourceID: "native-fixture", Data: "opaque"} + copy := q + copy.Operation = "resume" + copy.Resume = &sandbox.ResumeRequest{Reference: r, OperationID: installation, Retained: retained, Target: sandbox.Compute{Generation: 1, Name: r.AllocationID, ID: "native-fixture", RestoredFrom: &retained}} + value := object(copy) + resume := object(copy.Resume) + resume["Workspace"] = workspace + value["Resume"] = resume + add("request", fmt.Sprintf("resume-workspace-%v", workspace), workspace == nil, value) + } for _, count := range []int{0, e2b.MaxCredentialReferences, e2b.MaxCredentialReferences + 1} { copy := q copy.Operation = "verify_credential" diff --git a/services/core/internal/sandbox/e2b/operations.go b/services/core/internal/sandbox/e2b/operations.go index a3da4688e..7bb90e068 100644 --- a/services/core/internal/sandbox/e2b/operations.go +++ b/services/core/internal/sandbox/e2b/operations.go @@ -1,11 +1,12 @@ package e2b import ( - "context" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) +var _ sandbox.SandboxProvider = (*Provider)(nil) + // Operations is this adapter's complete authored resource contract. func Operations() providercontract.Operations { return providercontract.Operations{ @@ -14,43 +15,17 @@ func Operations() providercontract.Operations { "Renew": {State: providercontract.Supported}, "Kill": {State: providercontract.Supported}, "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Suspend": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Resume": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "Initial": {State: providercontract.Supported}, + "NewCompute": {State: providercontract.Supported}, + "GetCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, + "Suspend": {State: providercontract.Supported}, + "Resume": {State: providercontract.Supported}, + "KillCompute": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, + "RunCommandCompute": {State: providercontract.Supported}, + "ResumeCompute": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, } } func (*Provider) ProviderOperations() providercontract.Operations { return Operations() } -func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} -} -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} -} -func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "GetCompute", Reason: Operations()["GetCompute"].Reason} -} -func (p *Provider) Suspend(context.Context, sandbox.SuspendRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Suspend", Reason: Operations()["Suspend"].Reason} -} -func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Resume", Reason: Operations()["Resume"].Reason} -} -func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { - return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} -} -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} -} -func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { - return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} -} -func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} -} diff --git a/services/core/internal/sandbox/e2b/process_test.go b/services/core/internal/sandbox/e2b/process_test.go index a2e90a5b4..695333885 100644 --- a/services/core/internal/sandbox/e2b/process_test.go +++ b/services/core/internal/sandbox/e2b/process_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -12,7 +13,7 @@ import ( ) func TestProcessWireRejectsUnknownAndTrailingOutput(t *testing.T) { - for _, output := range []string{`{"Version":2,"Secret":"private"}`, `{"Version":2} {"Version":2}`} { + for _, output := range []string{`{"Version":1,"Secret":"private"}`, `{"Version":1} {"Version":1}`} { root := t.TempDir() binary := filepath.Join(root, "helper") if err := os.WriteFile(binary, []byte("#!/bin/sh\nprintf '%s' '"+output+"'\n"), 0700); err != nil { @@ -29,7 +30,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { marker := filepath.Join(root, "settled") // The private test path contains no shell syntax; the real adapter never puts // credentials or request contents in argv or inherited environment. - script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":2}'\n" + script := "#!/bin/sh\nsleep 0.15\ntouch '" + marker + "'\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -50,7 +51,7 @@ func TestTimeoutDoesNotTerminateOwnedHelper(t *testing.T) { func TestEnvironmentDropsProviderSelectorsAndCredentials(t *testing.T) { root := t.TempDir() binary := filepath.Join(root, "helper") - script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":2}'\n" + script := "#!/bin/sh\nif test -n \"${E2B_API_KEY:-}${E2B_API_URL:-}${PRIVATE_MODEL_KEY:-}\"; then exit 1; fi\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } @@ -67,7 +68,7 @@ func TestCredentialFenceWaitsForActualHelperExitAfterCancellation(t *testing.T) binary := filepath.Join(root, "helper") marker := filepath.Join(root, "started") finish := filepath.Join(root, "finish") - script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":2}'\n" + script := "#!/bin/sh\ntouch '" + marker + "'\nwhile ! test -f '" + finish + "'; do sleep 0.01; done\nprintf '%s' '{\"Version\":" + strconv.Itoa(ProtocolVersion) + "}'\n" if err := os.WriteFile(binary, []byte(script), 0700); err != nil { t.Fatal(err) } diff --git a/services/core/internal/sandbox/e2b/provider.go b/services/core/internal/sandbox/e2b/provider.go index 81e74cdba..4c022070e 100644 --- a/services/core/internal/sandbox/e2b/provider.go +++ b/services/core/internal/sandbox/e2b/provider.go @@ -195,12 +195,20 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen } connection = &value } - out, err := p.caller.Call(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) + return p.callRequest(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) +} + +func (p *Provider) callRequest(ctx context.Context, q Request) (Response, error) { + operation, r := q.Operation, q.Reference + if q.Validate() != nil { + return Response{}, sandbox.ErrInvalid + } + out, err := p.caller.Call(ctx, q) if errors.Is(err, errHelperNotStarted) { return unstarted(operation, r), sandbox.ErrComputeUnconfirmed } if err != nil || out.Version != ProtocolVersion { - if operation == "command" { + if operation == "command" || operation == "compute_command" { return Response{}, sandbox.ErrCommandUnconfirmed } return Response{}, sandbox.ErrComputeUnconfirmed diff --git a/services/core/internal/sandbox/e2b/suspension.go b/services/core/internal/sandbox/e2b/suspension.go new file mode 100644 index 000000000..5237ea082 --- /dev/null +++ b/services/core/internal/sandbox/e2b/suspension.go @@ -0,0 +1,115 @@ +package e2b + +import ( + "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +func (p *Provider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { + if !validReference(r) { + return sandbox.Compute{}, sandbox.ErrInvalid + } + return sandbox.Compute{Name: r.AllocationID}, nil +} +func (p *Provider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, retained *sandbox.RetainedState) (sandbox.Compute, error) { + if !validReference(r) || retained == nil || sandbox.ValidateRetained(*retained) != nil || retained.Reference != r.AllocationID || retained.SourceName != r.AllocationID || generation == 0 || generation != retained.SourceGeneration+1 { + return sandbox.Compute{}, sandbox.ErrInvalid + } + value := *retained + return sandbox.Compute{Generation: generation, Name: r.AllocationID, ID: retained.SourceID, RestoredFrom: &value}, nil +} +func (p *Provider) computeCall(ctx context.Context, q Request) (Response, error) { + deadline, ok := ctx.Deadline() + if !ok || ctx.Err() != nil { + return Response{}, sandbox.ErrInvalid + } + q.Version, q.Config, q.Deadline = ProtocolVersion, p.config, deadline + return p.callRequest(ctx, q) +} +func (p *Provider) computeState(ctx context.Context, operation string, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + if c.Name != r.AllocationID { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + out, err := p.computeCall(ctx, Request{Operation: operation, Reference: r, Compute: &c}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State == nil || sandbox.ValidateComputeResult(c, out.State.Compute) != nil { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return *out.State, nil +} +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "compute_info", r, c) +} +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "compute_renew", r, c) +} +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.computeState(ctx, "resume_compute", r, c) +} +func (p *Provider) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + if !validID(q.OperationID) || q.Source.Name != q.Reference.AllocationID || q.Source.ID == "" { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State == nil { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + if err = sandbox.ValidateSuspendResult(q, *out.State); err != nil { + return sandbox.ComputeState{}, err + } + if out.State.Retained != nil && out.State.Retained.Reference != q.Reference.AllocationID { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + return *out.State, nil +} +func (p *Provider) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + if q.Workspace != nil { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Resume", Reason: "external_workspace_unsupported"} + } + planned, err := p.NewCompute(ctx, q.Reference, q.Target.Generation, &q.Retained) + if err != nil || sandbox.ValidateComputeResult(planned, q.Target) != nil || !validID(q.OperationID) { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q}) + if err != nil { + return sandbox.ComputeState{}, err + } + if out.State == nil || sandbox.ValidateComputeResult(q.Target, out.State.Compute) != nil || out.State.Status != "running" || !out.State.BootstrapComplete { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return *out.State, nil +} +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { + if c.Name != r.AllocationID || c.ID == "" { + return sandbox.ErrInvalid + } + _, err := p.computeCall(ctx, Request{Operation: "compute_kill", Reference: r, Compute: &c}) + return err +} +func (p *Provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + if sandbox.ValidateRetained(s) != nil || s.Reference != r.AllocationID { + return sandbox.ErrInvalid + } + _, err := p.computeCall(ctx, Request{Operation: "delete_retained", Reference: r, Retained: &s}) + return err +} +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { + if c.Name != r.AllocationID || c.ID == "" { + return sandbox.CommandResult{}, sandbox.ErrInvalid + } + out, err := p.computeCall(ctx, Request{Operation: "compute_command", Reference: r, Compute: &c, Command: &command}) + if err != nil { + return sandbox.CommandResult{}, err + } + if out.Command == nil { + return sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed + } + return *out.Command, nil +} diff --git a/services/core/internal/sandbox/e2b/suspension_test.go b/services/core/internal/sandbox/e2b/suspension_test.go new file mode 100644 index 000000000..d4a0c0c06 --- /dev/null +++ b/services/core/internal/sandbox/e2b/suspension_test.go @@ -0,0 +1,48 @@ +package e2b + +import ( + "errors" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" +) + +func TestSuspensionPlansSameNativeIDWithoutCallingHelper(t *testing.T) { + p, f, r := fixture(t) + initial, err := p.Initial(bounded(t), r) + if err != nil || initial.Name != r.AllocationID || initial.ID != "" { + t.Fatal(initial, err) + } + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: uuid.NewString(), OperationID: uuid.NewString(), SourceName: r.AllocationID, SourceID: "native-id", Data: "opaque"} + target, err := p.NewCompute(bounded(t), r, 1, &retained) + if err != nil || target.ID != retained.SourceID || target.Generation != 1 || *target.RestoredFrom != retained || len(f.requests) != 0 { + t.Fatal(target, err) + } + if _, err = p.NewCompute(bounded(t), r, 2, &retained); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("generation jump accepted", err) + } + retained.Reference = uuid.NewString() + if _, err = p.NewCompute(bounded(t), r, 1, &retained); !errors.Is(err, sandbox.ErrInvalid) { + t.Fatal("foreign allocation accepted", err) + } +} +func TestSuspensionRejectsUnsettledAndForeignHelperOutcomes(t *testing.T) { + p, f, r := fixture(t) + current := sandbox.Compute{Name: r.AllocationID, ID: "native-id"} + q := sandbox.SuspendRequest{Reference: r, Source: current, OperationID: uuid.NewString()} + retained := sandbox.RetainedState{Reference: r.AllocationID, ID: q.OperationID, OperationID: q.OperationID, SourceName: current.Name, SourceID: current.ID, Data: "opaque"} + f.response.State = &sandbox.ComputeState{Compute: current, Status: "suspended", BootstrapComplete: true, Retained: &retained, ResourcesReleased: true, SuspendSettled: true} + if _, err := p.Suspend(bounded(t), q); err != nil { + t.Fatal(err) + } + f.response.State.SuspendSettled = false + if _, err := p.Suspend(bounded(t), q); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { + t.Fatal(err) + } + f.response.State.SuspendSettled = true + f.response.State.Compute.Generation = 1 + if _, err := p.Suspend(bounded(t), q); !errors.Is(err, sandbox.ErrOwnership) { + t.Fatal(err) + } +} diff --git a/services/core/internal/sandbox/e2b/workspace_test.go b/services/core/internal/sandbox/e2b/workspace_test.go index 0f6ed46d2..5acc16786 100644 --- a/services/core/internal/sandbox/e2b/workspace_test.go +++ b/services/core/internal/sandbox/e2b/workspace_test.go @@ -15,3 +15,11 @@ func TestCreateExplicitlyRejectsExternalWorkspace(t *testing.T) { t.Fatal("external workspace not rejected before native access", err) } } + +func TestResumeExplicitlyRejectsExternalWorkspace(t *testing.T) { + provider := new(Provider) + _, err := provider.Resume(context.Background(), sandbox.ResumeRequest{Workspace: &workspacefs.Binding{}}) + if reason, ok := providercontract.UnsupportedReason(err, "Resume"); !ok || reason != "external_workspace_unsupported" { + t.Fatal("external workspace not rejected before native access", err) + } +} diff --git a/services/core/internal/sandbox/microsandbox/creation_settlement_test.go b/services/core/internal/sandbox/microsandbox/creation_settlement_test.go index 6fab743b3..f088e8308 100644 --- a/services/core/internal/sandbox/microsandbox/creation_settlement_test.go +++ b/services/core/internal/sandbox/microsandbox/creation_settlement_test.go @@ -150,7 +150,7 @@ func TestOrdinaryComputeInspectionDoesNotRequestInitialSettlement(t *testing.T) if err != nil { t.Fatal(err) } - _, err = provider.GetCompute(deadline(t), ref, Compute{Name: Name(config, ref, 0)}) + _, err = provider.nativeGetCompute(deadline(t), ref, Compute{Name: Name(config, ref, 0)}) if !errors.Is(err, sandbox.ErrNotFound) { t.Fatal(err) } diff --git a/services/core/internal/sandbox/microsandbox/operations.go b/services/core/internal/sandbox/microsandbox/operations.go index 13e81ff96..e3742747e 100644 --- a/services/core/internal/sandbox/microsandbox/operations.go +++ b/services/core/internal/sandbox/microsandbox/operations.go @@ -12,11 +12,12 @@ func Operations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Supported}, "NewCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "GetCompute": {State: providercontract.Supported}, "Suspend": {State: providercontract.Supported}, "Resume": {State: providercontract.Supported}, "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, diff --git a/services/core/internal/sandbox/microsandbox/process_test.go b/services/core/internal/sandbox/microsandbox/process_test.go index 68021cd2d..894568e83 100644 --- a/services/core/internal/sandbox/microsandbox/process_test.go +++ b/services/core/internal/sandbox/microsandbox/process_test.go @@ -335,7 +335,11 @@ identity={"installation_id":"test-installation","generation":1,"specification_di with g.collection_lease(Path(sys.argv[2]),1,i,identity,initialize=sys.argv[3]=="init"): pass ` run := func(mode string) error { - return exec.Command("python3", "-c", script, installer, root, mode, testLeaseIdentity().SpecificationDigest).Run() + output, err := exec.Command("python3", "-c", script, installer, root, mode, testLeaseIdentity().SpecificationDigest).CombinedOutput() + if err != nil { + t.Log(string(output)) + } + return err } if err := run("init"); err != nil { t.Fatal("Python initialization", err) diff --git a/services/core/internal/sandbox/microsandbox/provider.go b/services/core/internal/sandbox/microsandbox/provider.go index 0b9f62800..88ce87292 100644 --- a/services/core/internal/sandbox/microsandbox/provider.go +++ b/services/core/internal/sandbox/microsandbox/provider.go @@ -26,7 +26,7 @@ func NewWithCaller(c Config, caller Caller) (*Provider, error) { c.Network.Rules = append([]NetworkRule(nil), c.Network.Rules...) return &Provider{config: c, caller: caller}, nil } -func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (Compute, error) { +func (p *Provider) nativeInitial(ctx context.Context, r sandbox.Reference) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err } @@ -93,7 +93,7 @@ func (p *Provider) responseState(ctx context.Context, q Request, out Response) ( } want := q.Compute if q.Operation == "create" { - want, e = p.Initial(ctx, q.Reference) + want, e = p.nativeInitial(ctx, q.Reference) if e != nil { return State{}, e } @@ -166,7 +166,7 @@ func (p *Provider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Inf return result, err } func (p *Provider) GetInfo(ctx context.Context, r sandbox.Reference) (sandbox.Info, error) { - c, e := p.Initial(ctx, r) + c, e := p.nativeInitial(ctx, r) if e != nil { return sandbox.Info{}, e } @@ -185,34 +185,37 @@ func (p *Provider) Renew(ctx context.Context, r sandbox.Reference) (sandbox.Info return p.GetInfo(ctx, r) } func (p *Provider) Kill(ctx context.Context, r sandbox.Reference) error { - c, e := p.Initial(ctx, r) + c, e := p.nativeInitial(ctx, r) if e != nil { return e } - return p.KillCompute(ctx, r, c) + return p.nativeKillCompute(ctx, r, c) } func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { - compute, e := p.Initial(ctx, r) + compute, e := p.nativeInitial(ctx, r) if e != nil { return sandbox.CommandResult{}, e } - return p.RunCommandCompute(ctx, r, compute, c) + return p.nativeRunCommandCompute(ctx, r, compute, c) } -func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { +func (p *Provider) nativeGetCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { return p.state(ctx, Request{Operation: "inspect", Reference: r, Compute: c}) } -func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c Compute) error { +func (p *Provider) nativeKillCompute(ctx context.Context, r sandbox.Reference, c Compute) error { _, e := p.call(ctx, Request{Operation: "kill", Reference: r, Compute: c}) return e } -func (p *Provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s SnapshotIdentity) error { +func (p *Provider) nativeDeleteSnapshot(ctx context.Context, r sandbox.Reference, s SnapshotIdentity) error { _, e := p.call(ctx, Request{Operation: "delete_snapshot", Reference: r, Snapshot: &s}) return e } -func (p *Provider) Suspend(ctx context.Context, q SuspendRequest) (State, error) { +func (p *Provider) nativeSuspend(ctx context.Context, q SuspendRequest) (State, error) { return p.state(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}) } -func (p *Provider) Resume(ctx context.Context, q ResumeRequest) (State, error) { +func (p *Provider) nativeResume(ctx context.Context, q ResumeRequest) (State, error) { + if p.config.ExternalWorkspace != (q.Workspace != nil) { + return State{}, sandbox.ErrInvalid + } workspace, err := p.resolveWorkspace(ctx, q.Reference, q.Workspace) if err != nil { return State{}, err @@ -220,7 +223,7 @@ func (p *Provider) Resume(ctx context.Context, q ResumeRequest) (State, error) { q.Workspace = nil return p.state(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q, Workspace: workspace}) } -func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c Compute, command sandbox.Command) (sandbox.CommandResult, error) { +func (p *Provider) nativeRunCommandCompute(ctx context.Context, r sandbox.Reference, c Compute, command sandbox.Command) (sandbox.CommandResult, error) { out, e := p.call(ctx, Request{Operation: "command", Reference: r, Compute: c, Command: &command}) if e != nil { return sandbox.CommandResult{}, e @@ -233,11 +236,11 @@ func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c // ResumeCompute thaws the exact resident source after an aborted suspension. // It never starts stopped compute or restores a checkpoint. -func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { +func (p *Provider) nativeResumeCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { return p.state(ctx, Request{Operation: "resume_compute", Reference: r, Compute: c}) } -func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { +func (p *Provider) nativeNewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err } diff --git a/services/core/internal/sandbox/microsandbox/provider_test.go b/services/core/internal/sandbox/microsandbox/provider_test.go index 94bfa9881..42f955dc1 100644 --- a/services/core/internal/sandbox/microsandbox/provider_test.go +++ b/services/core/internal/sandbox/microsandbox/provider_test.go @@ -44,7 +44,7 @@ func TestRejectsChangedComputeIdentity(t *testing.T) { if e != nil { t.Fatal(e) } - _, e = p.GetCompute(deadline(t), r, Compute{Name: Name(c, r, 0), ID: "local:4"}) + _, e = p.nativeGetCompute(deadline(t), r, Compute{Name: Name(c, r, 0), ID: "local:4"}) if !errors.Is(e, sandbox.ErrOwnership) { t.Fatalf("foreign identity accepted: %v", e) } @@ -60,7 +60,7 @@ func TestRestoreMustRetainExactProvenance(t *testing.T) { got.RestoredFrom = &v return Response{Version: ProtocolVersion, State: &State{Compute: got, Status: "running", BootstrapComplete: true}}, nil })) - _, e := p.Resume(deadline(t), ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: target}) + _, e := p.nativeResume(deadline(t), ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: target}) if !errors.Is(e, sandbox.ErrOwnership) { t.Fatalf("different snapshot accepted: %v", e) } @@ -70,7 +70,7 @@ func TestRejectsSnapshotPathBeforeHelper(t *testing.T) { s.Reference = "/foreign/checkpoint" calls := 0 p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) - if e := p.DeleteSnapshot(deadline(t), r, s); !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { + if e := p.nativeDeleteSnapshot(deadline(t), r, s); !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { t.Fatalf("e=%v calls=%d", e, calls) } } @@ -83,7 +83,7 @@ func TestObserveOnlyPreservesCapturedButResidentState(t *testing.T) { } return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: "paused", Snapshot: &s}}, nil })) - got, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + got, e := p.nativeSuspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) if e != nil || got.SourceStopped || got.Status != "paused" { t.Fatalf("state=%+v error=%v", got, e) } @@ -117,11 +117,11 @@ func TestNoDeadlineOrForeignAllocationNeverCallsHelper(t *testing.T) { r := testRef() foreign := r foreign.EnvironmentID = "77777777-7777-4777-8777-777777777777" - initial, initialErr := p.Initial(deadline(t), r) + initial, initialErr := p.nativeInitial(deadline(t), r) if initialErr != nil { t.Fatal(initialErr) } - _, e = p.GetCompute(deadline(t), foreign, initial) + _, e = p.nativeGetCompute(deadline(t), foreign, initial) if !errors.Is(e, sandbox.ErrInvalid) || calls != 0 { t.Fatalf("e=%v calls=%d", e, calls) } @@ -146,7 +146,7 @@ func TestMissingSnapshotObservationAllowsOnlyIntactSourceRollback(t *testing.T) p, _ := NewWithCaller(c, callerFunc(func(context.Context, Request) (Response, error) { return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: status, BootstrapComplete: true}}, nil })) - _, e := p.Suspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) + _, e := p.nativeSuspend(deadline(t), SuspendRequest{Reference: r, OperationID: s.OperationID, Source: source, ObserveOnly: true}) if status == "running" || status == "paused" { if e != nil { t.Fatal(e) diff --git a/services/core/internal/sandbox/microsandbox/suspension.go b/services/core/internal/sandbox/microsandbox/suspension.go new file mode 100644 index 000000000..46954f196 --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/suspension.go @@ -0,0 +1,146 @@ +package microsandbox + +import ( + "context" + "encoding/json" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// retained translates a verified native snapshot without exposing its schema to Core. +func retained(s SnapshotIdentity) sandbox.RetainedState { + raw, _ := json.Marshal(s) + return sandbox.RetainedState{Reference: s.Reference, ID: s.ID, OperationID: s.OperationID, SourceGeneration: s.SourceGeneration, SourceName: s.SourceName, SourceID: s.SourceID, Data: string(raw)} +} +func (p *Provider) snapshot(r sandbox.Reference, s sandbox.RetainedState) (SnapshotIdentity, error) { + var native SnapshotIdentity + if sandbox.ValidateRetained(s) != nil || json.Unmarshal([]byte(s.Data), &native) != nil || retained(native) != s || ValidateSnapshot(p.config, r, native) != nil { + return native, sandbox.ErrOwnership + } + return native, nil +} +func compute(c Compute) sandbox.Compute { + out := sandbox.Compute{Generation: c.Generation, Name: c.Name, ID: c.ID} + if c.RestoredFrom != nil { + s := retained(*c.RestoredFrom) + out.RestoredFrom = &s + } + return out +} +func (p *Provider) nativeCompute(r sandbox.Reference, c sandbox.Compute) (Compute, error) { + out := Compute{Generation: c.Generation, Name: c.Name, ID: c.ID} + if c.RestoredFrom != nil { + s, e := p.snapshot(r, *c.RestoredFrom) + if e != nil { + return out, e + } + out.RestoredFrom = &s + } + if ValidateCompute(p.config, r, out) != nil { + return out, sandbox.ErrOwnership + } + return out, nil +} +func state(s State) sandbox.ComputeState { + out := sandbox.ComputeState{Compute: compute(s.Compute), Status: s.Status, BootstrapComplete: s.BootstrapComplete, ResourcesReleased: s.SourceStopped} + if s.Snapshot != nil { + v := retained(*s.Snapshot) + out.Retained = &v + } + return out +} +func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Compute, error) { + c, e := p.nativeInitial(ctx, r) + return compute(c), e +} +func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.RetainedState) (sandbox.Compute, error) { + var snap *SnapshotIdentity + if s != nil { + v, e := p.snapshot(r, *s) + if e != nil { + return sandbox.Compute{}, e + } + snap = &v + } + c, e := p.nativeNewCompute(ctx, r, g, snap) + return compute(c), e +} +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.ComputeState{}, e + } + s, e := p.nativeGetCompute(ctx, r, n) + return state(s), e +} +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.GetCompute(ctx, r, c) +} +func (p *Provider) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + c, e := p.nativeCompute(q.Reference, q.Source) + if e != nil { + return sandbox.ComputeState{}, e + } + n := SuspendRequest{Reference: q.Reference, OperationID: q.OperationID, Source: c, ObserveOnly: q.ReconcileOnly} + if q.Retained != nil { + s, e := p.snapshot(q.Reference, *q.Retained) + if e != nil { + return sandbox.ComputeState{}, e + } + n.Snapshot = &s + } + s, e := p.nativeSuspend(ctx, n) + if e == nil && s.Snapshot != nil { + // Complete the same exact-source cleanup previously performed by Core. + // Native KillCompute still verifies ownership and removes its disks. + e = p.nativeKillCompute(ctx, q.Reference, c) + if e == nil { + s.SourceStopped = true + s.Status = "suspended" + } + } + out := state(s) + // The helper holds the allocation lock until the original native work settles. + out.SuspendSettled = e == nil + return out, e +} +func (p *Provider) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + c, e := p.nativeCompute(q.Reference, q.Target) + if e != nil { + return sandbox.ComputeState{}, e + } + s, e := p.snapshot(q.Reference, q.Retained) + if e != nil { + return sandbox.ComputeState{}, e + } + v, e := p.nativeResume(ctx, ResumeRequest{Workspace: q.Workspace, Reference: q.Reference, OperationID: q.OperationID, Snapshot: s, Target: c, ObserveOnly: q.ReconcileOnly}) + return state(v), e +} +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { + n, e := p.nativeCompute(r, c) + if e != nil { + return e + } + return p.nativeKillCompute(ctx, r, n) +} +func (p *Provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + n, e := p.snapshot(r, s) + if e != nil { + return e + } + return p.nativeDeleteSnapshot(ctx, r, n) +} +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, q sandbox.Command) (sandbox.CommandResult, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.CommandResult{}, e + } + return p.nativeRunCommandCompute(ctx, r, n, q) +} +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + n, e := p.nativeCompute(r, c) + if e != nil { + return sandbox.ComputeState{}, e + } + v, e := p.nativeResumeCompute(ctx, r, n) + return state(v), e +} diff --git a/services/core/internal/sandbox/microsandbox/suspension_contract_test.go b/services/core/internal/sandbox/microsandbox/suspension_contract_test.go new file mode 100644 index 000000000..284bc2159 --- /dev/null +++ b/services/core/internal/sandbox/microsandbox/suspension_contract_test.go @@ -0,0 +1,57 @@ +package microsandbox + +import ( + "context" + "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "testing" +) + +func TestSharedSuspendSettlesExactSourceCleanupInsideAdapter(t *testing.T) { + for _, reconcile := range []bool{false, true} { + for _, cleanupFails := range []bool{false, true} { + c, r, snap := testConfig(), testRef(), testSnapshot() + source := Compute{Name: snap.SourceName, ID: snap.SourceID} + calls := []string{} + p, e := NewWithCaller(c, callerFunc(func(_ context.Context, q Request) (Response, error) { + calls = append(calls, q.Operation) + if q.Operation == "suspend" { + if q.Suspend.ObserveOnly != reconcile { + t.Fatal("reconcile intent changed") + } + return Response{Version: ProtocolVersion, State: &State{Compute: source, Status: "suspended", BootstrapComplete: true, Snapshot: &snap, SourceStopped: !reconcile}}, nil + } + if q.Operation != "kill" || q.Compute.ID != source.ID { + t.Fatal("wrong source cleanup", q.Operation) + } + if cleanupFails { + return Response{}, errors.New("lost cleanup") + } + return Response{Version: ProtocolVersion}, nil + })) + if e != nil { + t.Fatal(e) + } + q := sandbox.SuspendRequest{Reference: r, OperationID: snap.OperationID, Source: compute(source), ReconcileOnly: reconcile} + got, e := p.Suspend(deadline(t), q) + if len(calls) != 2 || calls[0] != "suspend" || calls[1] != "kill" { + t.Fatal(calls) + } + if cleanupFails { + if e == nil || got.SuspendSettled { + t.Fatal("failed cleanup released capacity") + } + continue + } + if e != nil { + t.Fatal(e) + } + if e = sandbox.ValidateSuspendResult(q, got); e != nil { + t.Fatal(e) + } + if got.Retained == nil || got.Retained.Data == "" { + t.Fatal("missing native proof") + } + } + } +} diff --git a/services/core/internal/sandbox/microsandbox/types.go b/services/core/internal/sandbox/microsandbox/types.go index 0a04f9e53..a947b3b47 100644 --- a/services/core/internal/sandbox/microsandbox/types.go +++ b/services/core/internal/sandbox/microsandbox/types.go @@ -4,6 +4,7 @@ package microsandbox import ( "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -42,12 +43,53 @@ type NetworkPolicy struct { } type NetworkRule struct{ Action, Direction, Destination, Protocol, Port string } -// These aliases keep the helper wire private while Core uses provider-neutral types. -type Compute = sandbox.Compute -type SnapshotIdentity = sandbox.SnapshotIdentity -type State = sandbox.ComputeState -type SuspendRequest = sandbox.SuspendRequest -type ResumeRequest = sandbox.ResumeRequest +// Native snapshot wire types belong to this adapter. +type Compute struct { + Generation uint64 + Name string + ID string + RestoredFrom *SnapshotIdentity +} + +// SnapshotIdentity is provider evidence from a verified full snapshot. Core +// persists it unchanged and records consumption separately; it never invents +// paths, checksums, native checkpoint fields, or source identity. +type SnapshotIdentity struct { + Reference string + ID string + Digest string + CheckpointID string + CheckpointRoot string + OperationID string + SourceGeneration uint64 + SourceName string + SourceID string +} + +type State struct { + Compute Compute + Status string + BootstrapComplete bool + Snapshot *SnapshotIdentity + SourceStopped bool +} +type SuspendRequest struct { + Reference sandbox.Reference + OperationID string + Source Compute + Snapshot *SnapshotIdentity + // Recovery observes the previous attempt and never starts a new capture. + ObserveOnly bool +} +type ResumeRequest struct { + Workspace *workspacefs.Binding `json:",omitempty"` + Reference sandbox.Reference + OperationID string + Snapshot SnapshotIdentity + Target Compute + // Recovery observes the previous target and never starts a new restore. + ObserveOnly bool +} // Request and Response are the finite, private helper boundary. Confidential // Bootstrap and Command bytes travel only through stdin and are never logged. diff --git a/services/core/internal/sandbox/microsandbox/workspace_test.go b/services/core/internal/sandbox/microsandbox/workspace_test.go index f345df118..bc74b2aec 100644 --- a/services/core/internal/sandbox/microsandbox/workspace_test.go +++ b/services/core/internal/sandbox/microsandbox/workspace_test.go @@ -42,17 +42,17 @@ func TestWorkspaceResolveBeforeRestoreAndRetainPartialTarget(t *testing.T) { return workspacefs.Directory{Path: "/resolved/environment"}, nil }) q := ResumeRequest{Reference: r, OperationID: "66666666-6666-4666-8666-666666666666", Snapshot: s, Target: Compute{Generation: 1, Name: Name(c, r, 1), RestoredFrom: &s}, Workspace: workspaceBinding()} - state, err := p.Resume(deadline(t), q) + state, err := p.nativeResume(deadline(t), q) if !errors.Is(err, ErrUnconfirmed) || state.Compute.ID != "local:partial" || resolves != 1 || calls != 1 { t.Fatal(state, err, resolves, calls) } q.ObserveOnly = true - _, _ = p.Resume(deadline(t), q) + _, _ = p.nativeResume(deadline(t), q) if resolves != 2 { t.Fatal("restore observation did not resolve original binding") } q.Workspace.Attachment.Reference.EnvironmentID = r.TenantID - _, err = p.Resume(deadline(t), q) + _, err = p.nativeResume(deadline(t), q) if !errors.Is(err, workspacefs.ErrOwnership) || calls != 2 { t.Fatal("foreign binding reached helper", err, calls) } @@ -62,7 +62,7 @@ func TestWorkspaceDoesNotFallbackWithoutResolver(t *testing.T) { config := testConfig() config.ExternalWorkspace = true p, _ := NewWithCaller(config, callerFunc(func(context.Context, Request) (Response, error) { calls++; return Response{}, nil })) - _, err := p.Create(deadline(t), sandbox.Bootstrap{Harness: "codex", Reference: testRef(), SessionID: testRef().TenantID, DeviceID: testRef().EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()}) + _, err := p.Create(deadline(t), sandbox.Bootstrap{Reference: testRef(), SessionID: testRef().TenantID, DeviceID: testRef().EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "disabled", Workspace: workspaceBinding()}) if !errors.Is(err, workspacefs.ErrUnsupported) || calls != 0 { t.Fatal(err, calls) } @@ -81,7 +81,7 @@ func TestPreNativeWorkspaceFailureSettlesAbsentCreation(t *testing.T) { provider.workspace = workspaceResolverFunc(func(context.Context, workspacefs.Binding) (workspacefs.Directory, error) { return workspacefs.Directory{}, workspacefs.ErrUnavailable }) - bootstrap := sandbox.Bootstrap{Harness: "codex", Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", NetworkAccess: "disabled", Workspace: workspaceBinding()} + bootstrap := sandbox.Bootstrap{Reference: ref, SessionID: ref.TenantID, DeviceID: ref.EnvironmentID, CoreURL: "https://core.example/api/v1", Credential: "fixture", Harness: "codex", NetworkAccess: "disabled", Workspace: workspaceBinding()} ctx := deadline(t) switch failure { case "mode": diff --git a/services/core/internal/sandbox/node/generation_json.go b/services/core/internal/sandbox/node/generation_json.go index 661d177b2..75e85ea36 100644 --- a/services/core/internal/sandbox/node/generation_json.go +++ b/services/core/internal/sandbox/node/generation_json.go @@ -130,12 +130,12 @@ func validateGenerationJSON(raw []byte, kind string) error { } } if value := values["request"]; value != nil { - request, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume snapshot observation", "") + request, err := generationObject(value, "deployment_generation id sequence connection_id owner_epoch operation timeout_ms reference", "bootstrap compute generation command suspend resume retained observation", "") if err != nil { return err } if bootstrap := request["bootstrap"]; bootstrap != nil { - if _, err := generationObject(bootstrap, "TenantID EnvironmentID AllocationID SessionID DeviceID CoreURL Credential Harness NetworkAccess AllowedDomains", "Workspace", "AllowedDomains"); err != nil { + if _, err := generationObject(bootstrap, "TenantID EnvironmentID AllocationID SessionID DeviceID CoreURL Credential Harness NetworkAccess AllowedDomains", "Workspace", "AllowedDomains Workspace"); err != nil { return err } } diff --git a/services/core/internal/sandbox/node/generation_json_test.go b/services/core/internal/sandbox/node/generation_json_test.go index e9f52cbf9..8fa769f8e 100644 --- a/services/core/internal/sandbox/node/generation_json_test.go +++ b/services/core/internal/sandbox/node/generation_json_test.go @@ -63,7 +63,7 @@ func TestGenerationHealthRejectsUnboundedOrAmbiguousNumbers(t *testing.T) { } func TestNodeProtocolRejectsHistoricalVersions(t *testing.T) { - for _, version := range []int{1, 2, 3, 4} { + for _, version := range []int{1, 2, 3, 4, 5, 6} { raw, _ := json.Marshal(frame{Version: version, Type: "hello", Identity: new(Identity), Health: &Health{ObservedAt: time.Now().UTC()}}) if _, err := decodeFrame(raw); err == nil { t.Fatalf("accepted historical protocol %d", version) @@ -106,7 +106,7 @@ func TestNodeBootstrapRejectsAmbiguousHarness(t *testing.T) { t.Fatal("ambiguous selection reached dispatch", replacement) } } - if _, err := decodeFrame([]byte(strings.Replace(good, `,"Harness":"codex"`, "", 1))); err == nil { + if _, err := decodeFrame([]byte(strings.Replace(good, `"Harness":"codex",`, "", 1))); err == nil { t.Fatal("missing Harness accepted") } } diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index a8a21fbce..1b21674ca 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -169,11 +169,11 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing } } -func TestOfflineIsUnknownAndDockerDoesNotAdvertiseCheckpoint(t *testing.T) { +func TestOfflineIsUnknownAndDockerDoesNotAdvertiseSuspension(t *testing.T) { h := NewHub(HubOptions{OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) p := h.Proxy(uuid.NewString(), docker.Operations(), 1) - if sandbox.SupportsCheckpoint(p) { - t.Fatal("docker advertised checkpoint") + if sandbox.SupportsSuspension(p) { + t.Fatal("docker advertised suspension") } _, err := p.GetInfo(context.Background(), reference()) if !errors.Is(err, sandbox.ErrComputeUnconfirmed) || errors.Is(err, sandbox.ErrNotFound) { diff --git a/services/core/internal/sandbox/node/operations.go b/services/core/internal/sandbox/node/operations.go index 8f016664b..aca02dcb5 100644 --- a/services/core/internal/sandbox/node/operations.go +++ b/services/core/internal/sandbox/node/operations.go @@ -10,10 +10,11 @@ var operationMethods = map[string]string{ "initial": "Initial", "new_compute": "NewCompute", "compute": "GetCompute", + "renew_compute": "RenewCompute", "suspend": "Suspend", "resume": "Resume", "kill_compute": "KillCompute", - "delete_snapshot": "DeleteSnapshot", + "delete_retained": "DeleteRetained", "command_compute": "RunCommandCompute", "resume_compute": "ResumeCompute", "observe": "Observe", diff --git a/services/core/internal/sandbox/node/operations_test.go b/services/core/internal/sandbox/node/operations_test.go index e508988b5..f0567ad1c 100644 --- a/services/core/internal/sandbox/node/operations_test.go +++ b/services/core/internal/sandbox/node/operations_test.go @@ -49,9 +49,63 @@ func TestUnsupportedProxyRejectsBeforeNodeResolution(t *testing.T) { } } func TestNodeOperationMappingCoversForwardedMethods(t *testing.T) { - for _, method := range []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand", "Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute", "Observe"} { + for _, method := range []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand", "Initial", "NewCompute", "GetCompute", "RenewCompute", "Suspend", "Resume", "KillCompute", "DeleteRetained", "RunCommandCompute", "ResumeCompute", "Observe"} { if wire := operationWire(method); wire == "" || operationMethod(wire) != method { t.Fatal(method) } } } + +// Retained handles and renew are part of the allocation protocol, including on +// nodes whose implementation embeds no optional lifecycle interface. +type retainedWireProvider struct { + fakeProvider + renewed sandbox.Compute + deleted sandbox.RetainedState +} + +func (p *retainedWireProvider) ProviderOperations() providercontract.Operations { + operations := p.fakeProvider.ProviderOperations() + operations["RenewCompute"] = providercontract.Support{State: providercontract.Supported} + operations["DeleteRetained"] = providercontract.Support{State: providercontract.Supported} + return operations +} +func (p *retainedWireProvider) RenewCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + p.renewed = c + return sandbox.ComputeState{Compute: c, Status: "running"}, nil +} +func (p *retainedWireProvider) DeleteRetained(_ context.Context, _ sandbox.Reference, retained sandbox.RetainedState) error { + p.deleted = retained + return nil +} +func TestRetainedLifecycleDispatchSurvivesStrictWire(t *testing.T) { + ref := reference() + retained := sandbox.RetainedState{Reference: ref.AllocationID, ID: uuid.NewString(), OperationID: uuid.NewString(), SourceName: ref.AllocationID, SourceID: "native", Data: "opaque"} + compute := sandbox.Compute{Generation: 1, Name: ref.AllocationID, ID: "native", RestoredFrom: &retained} + p := new(retainedWireProvider) + for _, operation := range []string{"renew_compute", "delete_retained"} { + q := request{DeploymentGeneration: 1, ID: uuid.NewString(), Sequence: 1, ConnectionID: uuid.NewString(), OwnerEpoch: 1, Operation: operation, TimeoutMillis: 1000, Reference: ref} + if operation == "renew_compute" { + q.Compute = &compute + } else { + q.Retained = &retained + } + raw, err := json.Marshal(frame{Version: ProtocolVersion, Type: "request", Request: &q}) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeFrame(raw) + if err != nil { + t.Fatal(err) + } + if err := decoded.Request.validate(); err != nil { + t.Fatal(err) + } + if out := execute(t.Context(), p, *decoded.Request); responseError(out) != nil { + t.Fatal(out) + } + } + if p.renewed.ID != compute.ID || p.renewed.RestoredFrom == nil || *p.renewed.RestoredFrom != retained || p.deleted != retained { + t.Fatal("retained evidence lost in transport") + } +} diff --git a/services/core/internal/sandbox/node/provider_operations_fixture_test.go b/services/core/internal/sandbox/node/provider_operations_fixture_test.go index 3f892e9bb..fdcb82c42 100644 --- a/services/core/internal/sandbox/node/provider_operations_fixture_test.go +++ b/services/core/internal/sandbox/node/provider_operations_fixture_test.go @@ -17,11 +17,12 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -30,7 +31,7 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { func (*fakeProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*fakeProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*fakeProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*fakeProvider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (*fakeProvider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.Com func (*fakeProvider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*fakeProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*fakeProvider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*fakeProvider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -66,13 +67,18 @@ func (*observationProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Supported}, } } + +func (*fakeProvider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/internal/sandbox/node/proxy.go b/services/core/internal/sandbox/node/proxy.go index e064f5b38..9271fb418 100644 --- a/services/core/internal/sandbox/node/proxy.go +++ b/services/core/internal/sandbox/node/proxy.go @@ -105,8 +105,8 @@ func (p *provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Co } return *out.Compute, nil } -func (p *provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.SnapshotIdentity) (sandbox.Compute, error) { - out, e := p.call(ctx, request{Operation: "new_compute", Reference: r, Generation: g, Snapshot: s}) +func (p *provider) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, s *sandbox.RetainedState) (sandbox.Compute, error) { + out, e := p.call(ctx, request{Operation: "new_compute", Reference: r, Generation: g, Retained: s}) if e != nil { return sandbox.Compute{}, e } @@ -141,8 +141,8 @@ func (p *provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandb _, e := p.call(ctx, request{Operation: "kill_compute", Reference: r, Compute: &c}) return e } -func (p *provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s sandbox.SnapshotIdentity) error { - _, e := p.call(ctx, request{Operation: "delete_snapshot", Reference: r, Snapshot: &s}) +func (p *provider) DeleteRetained(ctx context.Context, r sandbox.Reference, s sandbox.RetainedState) error { + _, e := p.call(ctx, request{Operation: "delete_retained", Reference: r, Retained: &s}) return e } func (p *provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, v sandbox.Command) (sandbox.CommandResult, error) { @@ -158,3 +158,7 @@ func (p *provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c san func (h *Hub) GenerationProvider(declared providercontract.Operations, resolve func(context.Context, sandbox.Reference) (string, uint64, error)) sandbox.SandboxProvider { return &provider{hub: h, operations: maps.Clone(declared), resolveGeneration: resolve} } + +func (p *provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, request{Operation: "renew_compute", Reference: r, Compute: &c}) +} diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index e7a9d80f2..58d51b97d 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -19,7 +19,7 @@ import ( "github.com/gorilla/websocket" ) -const ProtocolVersion = 6 +const ProtocolVersion = 7 const MaxControlFrameBytes = 32 * 1024 const MaxFrameBytes = 72 * 1024 * 1024 const maxPending = 32 @@ -87,15 +87,15 @@ type request struct { TimeoutMillis int64 `json:"timeout_ms"` // deadline is anchored to the receiving host and never crosses the wire. deadline time.Time - Reference sandbox.Reference `json:"reference"` - Bootstrap *sandbox.Bootstrap `json:"bootstrap,omitempty"` - Compute *sandbox.Compute `json:"compute,omitempty"` - Generation uint64 `json:"generation,omitempty"` - Command *sandbox.Command `json:"command,omitempty"` - Suspend *sandbox.SuspendRequest `json:"suspend,omitempty"` - Resume *sandbox.ResumeRequest `json:"resume,omitempty"` - Snapshot *sandbox.SnapshotIdentity `json:"snapshot,omitempty"` - Observation *runtimeobs.Target `json:"observation,omitempty"` + Reference sandbox.Reference `json:"reference"` + Bootstrap *sandbox.Bootstrap `json:"bootstrap,omitempty"` + Compute *sandbox.Compute `json:"compute,omitempty"` + Generation uint64 `json:"generation,omitempty"` + Command *sandbox.Command `json:"command,omitempty"` + Suspend *sandbox.SuspendRequest `json:"suspend,omitempty"` + Resume *sandbox.ResumeRequest `json:"resume,omitempty"` + Retained *sandbox.RetainedState `json:"retained,omitempty"` + Observation *runtimeobs.Target `json:"observation,omitempty"` } type response struct { @@ -244,7 +244,7 @@ func (q request) validate() error { return sandbox.ErrInvalid } count := 0 - for _, ok := range []bool{q.Bootstrap != nil, q.Compute != nil, q.Command != nil, q.Suspend != nil, q.Resume != nil, q.Snapshot != nil, q.Observation != nil} { + for _, ok := range []bool{q.Bootstrap != nil, q.Compute != nil, q.Command != nil, q.Suspend != nil, q.Resume != nil, q.Retained != nil, q.Observation != nil} { if ok { count++ } @@ -263,10 +263,10 @@ func (q request) validate() error { return nil } case "new_compute": - if count == 0 || count == 1 && q.Snapshot != nil { + if count == 0 || count == 1 && q.Retained != nil { return nil } - case "compute", "kill_compute", "resume_compute": + case "compute", "renew_compute", "kill_compute", "resume_compute": if count == 1 && q.Compute != nil { return nil } @@ -286,8 +286,8 @@ func (q request) validate() error { if count == 1 && q.Resume != nil && q.Resume.Reference == q.Reference && sandbox.ValidateWorkspaceBinding(q.Reference, q.Resume.Workspace) == nil { return nil } - case "delete_snapshot": - if count == 1 && q.Snapshot != nil { + case "delete_retained": + if count == 1 && q.Retained != nil { return nil } } @@ -337,11 +337,14 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response compute, err = p.Initial(ctx, q.Reference) out.Compute = &compute case "new_compute": - compute, err = p.NewCompute(ctx, q.Reference, q.Generation, q.Snapshot) + compute, err = p.NewCompute(ctx, q.Reference, q.Generation, q.Retained) out.Compute = &compute case "compute": state, err = p.GetCompute(ctx, q.Reference, *q.Compute) out.State = &state + case "renew_compute": + state, err = p.RenewCompute(ctx, q.Reference, *q.Compute) + out.State = &state case "suspend": state, err = p.Suspend(ctx, *q.Suspend) out.State = &state @@ -350,8 +353,8 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response out.State = &state case "kill_compute": err = p.KillCompute(ctx, q.Reference, *q.Compute) - case "delete_snapshot": - err = p.DeleteSnapshot(ctx, q.Reference, *q.Snapshot) + case "delete_retained": + err = p.DeleteRetained(ctx, q.Reference, *q.Retained) case "resume_compute": state, err = p.ResumeCompute(ctx, q.Reference, *q.Compute) out.State = &state @@ -382,7 +385,7 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response } func requiresReady(q request) bool { - return q.Operation == "create" || q.Operation == "resume" && q.Resume != nil && !q.Resume.ObserveOnly + return q.Operation == "create" || q.Operation == "resume" && q.Resume != nil && !q.Resume.ReconcileOnly } // setTimeout consumes sender queue time without comparing clocks across hosts. diff --git a/services/core/internal/sandbox/operations_test.go b/services/core/internal/sandbox/operations_test.go index 62d2c5b3f..c8f494397 100644 --- a/services/core/internal/sandbox/operations_test.go +++ b/services/core/internal/sandbox/operations_test.go @@ -24,7 +24,7 @@ func TestDeclarationsRejectMissingUnknownAndContradictoryOperations(t *testing.T name string change func(providercontract.Operations) }{ - {"omitted", func(o providercontract.Operations) { delete(o, "DeleteSnapshot") }}, + {"omitted", func(o providercontract.Operations) { delete(o, "DeleteRetained") }}, {"zero", func(o providercontract.Operations) { o["Observe"] = providercontract.Support{} }}, {"unknown", func(o providercontract.Operations) { o["FutureOperation"] = providercontract.Support{State: providercontract.Supported} diff --git a/services/core/internal/sandbox/providers/registration.go b/services/core/internal/sandbox/providers/registration.go index 5a5668d35..eb7c455a7 100644 --- a/services/core/internal/sandbox/providers/registration.go +++ b/services/core/internal/sandbox/providers/registration.go @@ -55,11 +55,6 @@ func ValidateRegistration(a Adapter) error { if err := sandbox.ValidateOperations(operations); err != nil { return err } - // The common lifecycle suspends only node allocations, so only a nodes - // registration may declare checkpoint support. - if operations["Initial"].State == providercontract.Supported && a.Mode != sandbox.DeploymentNodes { - return invalid("checkpoint support outside nodes mode") - } if a.Policy.DefaultResources != nil && a.ValidateResources(*a.Policy.DefaultResources) != nil { return invalid("default resources") } diff --git a/services/core/internal/sandbox/providers/registration_test.go b/services/core/internal/sandbox/providers/registration_test.go index 592bdf313..e5c20bc80 100644 --- a/services/core/internal/sandbox/providers/registration_test.go +++ b/services/core/internal/sandbox/providers/registration_test.go @@ -173,12 +173,12 @@ func TestRegistrationRejectsInvalidDefaultResources(t *testing.T) { } } -// Only a nodes registration may declare checkpoint support. -func TestRegistrationCheckpointRequiresNodes(t *testing.T) { +// Complete suspension support is independent of deployment placement. +func TestRegistrationSuspensionSupportsDirect(t *testing.T) { registry := Builtin() a := registry.adapters["microsandbox"] a.Mode, a.BuildLocal, a.NodeArtifacts, a.BuildDirect = "direct", nil, nil, registry.adapters["e2b"].BuildDirect - if err := ValidateRegistration(a); !errors.Is(err, providercontract.ErrContract) || !strings.Contains(err.Error(), "checkpoint") { - t.Fatal("direct checkpoint registration accepted", err) + if err := ValidateRegistration(a); err != nil { + t.Fatal("direct suspension registration rejected", err) } } diff --git a/services/core/internal/sandbox/providers/registry.go b/services/core/internal/sandbox/providers/registry.go index a39c0429f..bd13da340 100644 --- a/services/core/internal/sandbox/providers/registry.go +++ b/services/core/internal/sandbox/providers/registry.go @@ -88,8 +88,8 @@ func (r *Registry) BuildDirect(c sandbox.DirectConfig) (sandbox.SandboxProvider, return p, nil } -// SupportsCheckpoint reports whether the provider declares checkpoint suspension. -func (r *Registry) SupportsCheckpoint(kind string) (bool, error) { +// SupportsSuspension reports whether the provider declares checkpoint suspension. +func (r *Registry) SupportsSuspension(kind string) (bool, error) { a, err := r.Lookup(kind) if err != nil { return false, err diff --git a/services/core/internal/sandbox/providers/registry_test.go b/services/core/internal/sandbox/providers/registry_test.go index 47a0629bb..3edefa87e 100644 --- a/services/core/internal/sandbox/providers/registry_test.go +++ b/services/core/internal/sandbox/providers/registry_test.go @@ -19,7 +19,7 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { }{ {"docker", "nodes", "nodes", false}, {"microsandbox", "nodes", "nodes", true}, - {"e2b", "direct", "e2b", false}, + {"e2b", "direct", "e2b", true}, } { t.Run(tc.kind, func(t *testing.T) { d, err := registry.Describe(tc.kind, installation) @@ -27,7 +27,7 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { t.Fatalf("wrong mode or namespace: %+v %v", d, err) } a, err := registry.Lookup(tc.kind) - checkpoint, checkpointErr := registry.SupportsCheckpoint(tc.kind) + checkpoint, checkpointErr := registry.SupportsSuspension(tc.kind) if err != nil || checkpointErr != nil || checkpoint != tc.checkpoint || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { t.Fatal("inconsistent construction/capability registration", err, checkpointErr) } @@ -70,7 +70,7 @@ func TestNewRegistrationDoesNotNeedCoreDispatchChanges(t *testing.T) { // Registration is test-local: production registrations are fixed, never plugins. registry.adapters[kind] = registry.adapters["docker"] s, err := registry.Normalize(sandbox.Selection{Provider: kind, DeploymentSpec: validRegistrationSpec()}) - checkpoint, checkpointErr := registry.SupportsCheckpoint(kind) + checkpoint, checkpointErr := registry.SupportsSuspension(kind) if err != nil || checkpointErr != nil || s.Provider != kind || checkpoint { t.Fatal("new entry did not follow shared boundary", err, checkpointErr) } @@ -111,8 +111,8 @@ func TestCapabilityLookupsReportFailures(t *testing.T) { invalid.Operations = nil registry.adapters["invalid-registration"] = invalid for kind, want := range map[string]error{"unregistered": ErrUnknownProvider, "invalid-registration": providercontract.ErrContract} { - if _, err := registry.SupportsCheckpoint(kind); !errors.Is(err, want) { - t.Fatal(kind, "SupportsCheckpoint", err) + if _, err := registry.SupportsSuspension(kind); !errors.Is(err, want) { + t.Fatal(kind, "SupportsSuspension", err) } if _, err := registry.RetainedLimit(kind, 1, 2); !errors.Is(err, want) { t.Fatal(kind, "RetainedLimit", err) diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index e5770790e..b95182b6e 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -52,10 +52,11 @@ type Reference struct{ TenantID, EnvironmentID, AllocationID string } type Bootstrap struct { Workspace *workspacefs.Binding `json:",omitempty"` + Harness string Reference - SessionID, DeviceID, CoreURL, Credential, Harness string - NetworkAccess string - AllowedDomains []string + SessionID, DeviceID, CoreURL, Credential string + NetworkAccess string + AllowedDomains []string } // Info describes compute only. Running does not establish daemon authentication, @@ -108,12 +109,13 @@ type SandboxProvider interface { // The checkpoint lifecycle supplies exact-incarnation operations; Worker and // Store remain the lifecycle owner. Its operations share one declaration. Initial(context.Context, Reference) (Compute, error) - NewCompute(context.Context, Reference, uint64, *SnapshotIdentity) (Compute, error) + NewCompute(context.Context, Reference, uint64, *RetainedState) (Compute, error) GetCompute(context.Context, Reference, Compute) (ComputeState, error) + RenewCompute(context.Context, Reference, Compute) (ComputeState, error) Suspend(context.Context, SuspendRequest) (ComputeState, error) Resume(context.Context, ResumeRequest) (ComputeState, error) KillCompute(context.Context, Reference, Compute) error - DeleteSnapshot(context.Context, Reference, SnapshotIdentity) error + DeleteRetained(context.Context, Reference, RetainedState) error RunCommandCompute(context.Context, Reference, Compute, Command) (CommandResult, error) // ResumeCompute thaws only the same resident instance after an aborted pause. ResumeCompute(context.Context, Reference, Compute) (ComputeState, error) @@ -126,9 +128,17 @@ type SandboxProvider interface { // requiredOperations are supported by every Provider. var requiredOperations = []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand"} -// checkpointOperations are all supported or all unsupported: partial cleanup or +// suspensionOperations are all supported or all unsupported: partial cleanup or // restore support cannot safely own a compute incarnation. -var checkpointOperations = []string{"Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute"} +var suspensionOperations = []string{"Initial", "NewCompute", "GetCompute", "RenewCompute", "Suspend", "Resume", "KillCompute", "DeleteRetained", "RunCommandCompute", "ResumeCompute"} + +// ValidateRetained checks the shared envelope; only its adapter interprets Data. +func ValidateRetained(s RetainedState) error { + if s.Reference == "" || s.ID == "" || s.OperationID == "" || s.SourceID == "" || s.SourceName == "" || len(s.Data) == 0 || len(s.Data) > 64*1024 { + return ErrInvalid + } + return nil +} // Compute identifies one incarnation of an allocation. Name is provider-derived. // ID is empty only until the original create or restore result is observed. @@ -136,18 +146,15 @@ type Compute struct { Generation uint64 Name string ID string - RestoredFrom *SnapshotIdentity + RestoredFrom *RetainedState } -// SnapshotIdentity is provider evidence from a verified full snapshot. Core -// persists it unchanged and records consumption separately; it never invents -// paths, checksums, native checkpoint fields, or source identity. -type SnapshotIdentity struct { +// RetainedState is adapter-owned recoverable state. Data is opaque to Core. +// A retained state does not imply an independent snapshot. +type RetainedState struct { Reference string ID string - Digest string - CheckpointID string - CheckpointRoot string + Data string OperationID string SourceGeneration uint64 SourceName string @@ -158,27 +165,67 @@ type ComputeState struct { Compute Compute Status string BootstrapComplete bool - Snapshot *SnapshotIdentity - SourceStopped bool + Retained *RetainedState + ResourcesReleased bool + SuspendSettled bool } type SuspendRequest struct { Reference Reference OperationID string Source Compute - Snapshot *SnapshotIdentity - // Recovery observes the previous attempt and never starts a new capture. - ObserveOnly bool + Retained *RetainedState + // Recovery settles the previous attempt without another capture. + // Ownership-verified cleanup of a durable retained artifact may complete. + ReconcileOnly bool } type ResumeRequest struct { Workspace *workspacefs.Binding `json:",omitempty"` Reference Reference OperationID string - Snapshot SnapshotIdentity + Retained RetainedState Target Compute // Recovery observes the previous target and never starts a new restore. - ObserveOnly bool + ReconcileOnly bool +} + +// ValidateComputeResult binds an observation to its precommitted incarnation. +func ValidateComputeResult(want, got Compute) error { + if got.ID == "" || got.Name != want.Name || got.Generation != want.Generation || (want.ID != "" && got.ID != want.ID) || (want.RestoredFrom == nil) != (got.RestoredFrom == nil) { + return ErrOwnership + } + if want.RestoredFrom != nil && *want.RestoredFrom != *got.RestoredFrom { + return ErrOwnership + } + return nil +} + +// ValidateSuspendResult distinguishes settled rollback from uncertain native work. +func ValidateSuspendResult(q SuspendRequest, s ComputeState) error { + if ValidateComputeResult(q.Source, s.Compute) != nil { + return ErrOwnership + } + if !s.SuspendSettled || !s.BootstrapComplete { + return ErrComputeUnconfirmed + } + if s.Retained == nil { + if !q.ReconcileOnly || s.ResourcesReleased || (s.Status != "running" && s.Status != "paused") { + return ErrComputeUnconfirmed + } + return nil + } + v := s.Retained + if ValidateRetained(*v) != nil || v.OperationID != q.OperationID || v.SourceID != q.Source.ID || v.SourceName != q.Source.Name || v.SourceGeneration != q.Source.Generation || (q.Retained != nil && *q.Retained != *v) { + return ErrOwnership + } + if !s.ResourcesReleased || s.Status != "suspended" { + return ErrComputeUnconfirmed + } + return nil } +// SuspensionStateVersion fences incompatible durable lifecycle shapes. +const SuspensionStateVersion = "1" + // ValidateProvider checks a constructed Provider's declaration. func ValidateProvider(p SandboxProvider) error { if p == nil { @@ -213,15 +260,15 @@ func ValidateOperations(operations providercontract.Operations) error { return fmt.Errorf("%w: required operation %s", providercontract.ErrContract, name) } } - for _, name := range checkpointOperations { + for _, name := range suspensionOperations { if operations[name].State != operations["Initial"].State { - return fmt.Errorf("%w: incomplete checkpoint lifecycle", providercontract.ErrContract) + return fmt.Errorf("%w: incomplete suspension lifecycle", providercontract.ErrContract) } } return nil } -func SupportsCheckpoint(p SandboxProvider) bool { +func SupportsSuspension(p SandboxProvider) bool { return providercontract.Require(p, "Initial") == nil } diff --git a/services/core/internal/sandbox/sandbox_provider_test.go b/services/core/internal/sandbox/sandbox_provider_test.go index 56c23b330..59806b0f3 100644 --- a/services/core/internal/sandbox/sandbox_provider_test.go +++ b/services/core/internal/sandbox/sandbox_provider_test.go @@ -16,7 +16,7 @@ func TestOperationGroupsPartitionTheInterface(t *testing.T) { methods = append(methods, name) } } - groups := slices.Concat(requiredOperations, checkpointOperations, []string{"Observe"}) + groups := slices.Concat(requiredOperations, suspensionOperations, []string{"Observe"}) slices.Sort(groups) if !slices.Equal(methods, groups) { t.Fatalf("SandboxProvider operations %v, groups %v", methods, groups) diff --git a/services/core/internal/sandbox/suspension_test.go b/services/core/internal/sandbox/suspension_test.go new file mode 100644 index 000000000..add0fb28b --- /dev/null +++ b/services/core/internal/sandbox/suspension_test.go @@ -0,0 +1,55 @@ +package sandbox + +import ( + "strings" + "testing" +) + +func TestSuspensionRequiresBoundSettledResourceRelease(t *testing.T) { + c := Compute{ID: "native", Name: "source", Generation: 2} + r := RetainedState{Reference: "allocation", ID: "retained", OperationID: "op", SourceGeneration: 2, SourceName: "source", SourceID: "native", Data: "private-proof"} + q := SuspendRequest{OperationID: "op", Source: c} + valid := ComputeState{Compute: c, Status: "suspended", BootstrapComplete: true, Retained: &r, ResourcesReleased: true, SuspendSettled: true} + if err := ValidateSuspendResult(q, valid); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + change func(*ComputeState) + }{ + {"unsettled", func(s *ComputeState) { s.SuspendSettled = false }}, + {"capacity still held", func(s *ComputeState) { s.ResourcesReleased = false }}, + {"running", func(s *ComputeState) { s.Status = "running" }}, + {"foreign source", func(s *ComputeState) { s.Compute.ID = "foreign" }}, + {"foreign handle", func(s *ComputeState) { v := *s.Retained; v.OperationID = "other"; s.Retained = &v }}, + {"oversized", func(s *ComputeState) { v := *s.Retained; v.Data = strings.Repeat("x", 65537); s.Retained = &v }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := valid + tc.change(&s) + if ValidateSuspendResult(q, s) == nil { + t.Fatal("invalid suspension accepted") + } + }) + } + rollback := ComputeState{Compute: c, Status: "running", BootstrapComplete: true, SuspendSettled: true} + if ValidateSuspendResult(q, rollback) == nil { + t.Fatal("fresh dispatch accepted rollback") + } + q.ReconcileOnly = true + if err := ValidateSuspendResult(q, rollback); err != nil { + t.Fatal(err) + } + rollback.SuspendSettled = false + if ValidateSuspendResult(q, rollback) == nil { + t.Fatal("running observation inferred settlement") + } +} +func TestComputeResultFencesSameNativeIDAcrossGenerations(t *testing.T) { + old := Compute{ID: "same-native", Name: "allocation", Generation: 1} + next := old + next.Generation++ + if ValidateComputeResult(old, next) == nil { + t.Fatal("stale logical generation accepted") + } +} diff --git a/services/core/tests/integration/admin_session_archive_worker_http_test.go b/services/core/tests/integration/admin_session_archive_worker_http_test.go index fb536b9e3..ac3b6761c 100644 --- a/services/core/tests/integration/admin_session_archive_worker_http_test.go +++ b/services/core/tests/integration/admin_session_archive_worker_http_test.go @@ -126,7 +126,7 @@ func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { t.Fatal("archive did not commit administrator audit", audits, err) } stop() - // Snapshot after shutdown: cancellation and lifecycle draining are complete. + // Retained after shutdown: cancellation and lifecycle draining are complete. // A stale handler must not fall back to the still-open admission Store. snapshot := func() string { t.Helper() diff --git a/services/core/tests/integration/archived_cancellation_migration_test.go b/services/core/tests/integration/archived_cancellation_migration_test.go index 705a695d6..805bc4591 100644 --- a/services/core/tests/integration/archived_cancellation_migration_test.go +++ b/services/core/tests/integration/archived_cancellation_migration_test.go @@ -22,6 +22,8 @@ func TestArchivedCancellationMigrationDoesNotAdoptOldRevocations(t *testing.T) { if _, err := deploymentExecution(t, w).ArchiveSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 1); err != nil { t.Fatal(err) } + // Exercise the historical migration with the target schema's disabled + // suspension policy, independently of the current provider defaults. db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() provider, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) diff --git a/services/core/tests/integration/provider_operations_fixture_test.go b/services/core/tests/integration/provider_operations_fixture_test.go index f528d5ae0..7f207632c 100644 --- a/services/core/tests/integration/provider_operations_fixture_test.go +++ b/services/core/tests/integration/provider_operations_fixture_test.go @@ -17,11 +17,12 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteRetained": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -30,7 +31,7 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { func (*lifecycleProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: "fixture_operation_not_supported"} } -func (*lifecycleProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (*lifecycleProvider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: "fixture_operation_not_supported"} } func (*lifecycleProvider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { @@ -45,8 +46,8 @@ func (*lifecycleProvider) Resume(context.Context, sandbox.ResumeRequest) (sandbo func (*lifecycleProvider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: "fixture_operation_not_supported"} } -func (*lifecycleProvider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: "fixture_operation_not_supported"} +func (*lifecycleProvider) DeleteRetained(context.Context, sandbox.Reference, sandbox.RetainedState) error { + return &providercontract.UnsupportedError{Operation: "DeleteRetained", Reason: "fixture_operation_not_supported"} } func (*lifecycleProvider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: "fixture_operation_not_supported"} @@ -57,7 +58,7 @@ func (*lifecycleProvider) ResumeCompute(context.Context, sandbox.Reference, sand func (*lifecycleProvider) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } -func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations { +func (*fakeSuspensionProvider) ProviderOperations() providercontract.Operations { return providercontract.Operations{ "Create": {State: providercontract.Supported}, "GetInfo": {State: providercontract.Supported}, @@ -66,13 +67,18 @@ func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations "RunCommand": {State: providercontract.Supported}, "Initial": {State: providercontract.Supported}, "NewCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "GetCompute": {State: providercontract.Supported}, "Suspend": {State: providercontract.Supported}, "Resume": {State: providercontract.Supported}, "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, + "DeleteRetained": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, } } + +func (*lifecycleProvider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} diff --git a/services/core/tests/integration/provider_registration_migration_test.go b/services/core/tests/integration/provider_registration_migration_test.go index bbc4cfb1e..1f10e1aea 100644 --- a/services/core/tests/integration/provider_registration_migration_test.go +++ b/services/core/tests/integration/provider_registration_migration_test.go @@ -29,6 +29,8 @@ func TestProviderRegistrationDowngradePreservesCustomEndpoints(t *testing.T) { if _, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, input); err != nil { t.Fatal(err) } + // Exercise the historical migration with the target schema's disabled + // suspension policy, independently of the current provider defaults. db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() migrations, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) diff --git a/services/core/tests/integration/runtime_compute_lifecycle_test.go b/services/core/tests/integration/runtime_compute_lifecycle_test.go index 73f5e29d6..f188a5448 100644 --- a/services/core/tests/integration/runtime_compute_lifecycle_test.go +++ b/services/core/tests/integration/runtime_compute_lifecycle_test.go @@ -15,6 +15,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" + db "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" @@ -26,11 +27,11 @@ import ( // The controlled provider records external effects independently of DB phases. // Lost replies retain those effects so recovery must use observation, not replay. -type fakeCheckpointProvider struct { +type fakeSuspensionProvider struct { preparation *initializationPeer lifecycleProvider computes map[string]sandbox.ComputeState - snapshots map[string]sandbox.SnapshotIdentity + snapshots map[string]sandbox.RetainedState bootstraps map[string]sandbox.Bootstrap peers map[string]*websocket.Conn registry *runtimegateway.Registry @@ -41,15 +42,16 @@ type fakeCheckpointProvider struct { quiesces, resumes atomic.Int32 loseCapture, loseRestore, rejectQuiesce bool beforeQuiesce func() + renewals atomic.Int32 } -func (p *fakeCheckpointProvider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { +func (p *fakeSuspensionProvider) Initial(_ context.Context, r sandbox.Reference) (sandbox.Compute, error) { return sandbox.Compute{Name: r.AllocationID + "-g0"}, nil } -func (p *fakeCheckpointProvider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, parent *sandbox.SnapshotIdentity) (sandbox.Compute, error) { +func (p *fakeSuspensionProvider) NewCompute(_ context.Context, r sandbox.Reference, generation uint64, parent *sandbox.RetainedState) (sandbox.Compute, error) { return sandbox.Compute{Generation: generation, Name: fmt.Sprintf("%s-g%d", r.AllocationID, generation), RestoredFrom: parent}, nil } -func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { +func (p *fakeSuspensionProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { info, err := p.lifecycleProvider.Create(ctx, b) p.mu.Lock() defer p.mu.Unlock() @@ -59,7 +61,7 @@ func (p *fakeCheckpointProvider) Create(ctx context.Context, b sandbox.Bootstrap p.bootstraps[b.AllocationID] = b return info, err } -func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[c.Name] @@ -71,24 +73,28 @@ func (p *fakeCheckpointProvider) GetCompute(_ context.Context, _ sandbox.Referen } return state, nil } -func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) Suspend(_ context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[q.Source.Name] if !ok { + if snapshot, exists := p.snapshots[q.OperationID]; exists && q.ReconcileOnly { + p.captureObservations++ + return sandbox.ComputeState{Compute: q.Source, Status: "suspended", Retained: &snapshot, BootstrapComplete: true, ResourcesReleased: true, SuspendSettled: true}, nil + } return sandbox.ComputeState{}, sandbox.ErrNotFound } if state.Compute.ID != q.Source.ID { return sandbox.ComputeState{}, sandbox.ErrOwnership } - if q.ObserveOnly { + if q.ReconcileOnly { p.captureObservations++ } else { p.captures++ if _, exists := p.snapshots[q.OperationID]; exists { return sandbox.ComputeState{}, errors.New("capture replayed") } - p.snapshots[q.OperationID] = sandbox.SnapshotIdentity{Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Digest: "verified", CheckpointID: "checkpoint", CheckpointRoot: "private", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} + p.snapshots[q.OperationID] = sandbox.RetainedState{Reference: "snapshot-" + q.OperationID, ID: uuid.NewString(), Data: "verified-native-state", OperationID: q.OperationID, SourceGeneration: q.Source.Generation, SourceName: q.Source.Name, SourceID: q.Source.ID} state.Status = "paused" p.computes[q.Source.Name] = state if p.loseCapture { @@ -97,14 +103,20 @@ func (p *fakeCheckpointProvider) Suspend(_ context.Context, q sandbox.SuspendReq } } if snapshot, exists := p.snapshots[q.OperationID]; exists { - state.Snapshot = &snapshot - } + state.Retained = &snapshot + state.SuspendSettled = true + state.ResourcesReleased = true + state.Status = "suspended" + delete(p.computes, q.Source.Name) + p.computeKills++ + } + state.SuspendSettled = true return state, nil } -func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) Resume(_ context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { p.mu.Lock() defer p.mu.Unlock() - if q.ObserveOnly { + if q.ReconcileOnly { p.restoreObservations++ state, ok := p.computes[q.Target.Name] if !ok { @@ -126,7 +138,7 @@ func (p *fakeCheckpointProvider) Resume(_ context.Context, q sandbox.ResumeReque } return state, nil } -func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { +func (p *fakeSuspensionProvider) KillCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) error { p.mu.Lock() defer p.mu.Unlock() state, ok := p.computes[c.Name] @@ -140,7 +152,7 @@ func (p *fakeCheckpointProvider) KillCompute(_ context.Context, _ sandbox.Refere delete(p.computes, c.Name) return nil } -func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Reference, s sandbox.SnapshotIdentity) error { +func (p *fakeSuspensionProvider) DeleteRetained(_ context.Context, _ sandbox.Reference, s sandbox.RetainedState) error { p.mu.Lock() defer p.mu.Unlock() old, ok := p.snapshots[s.OperationID] @@ -154,7 +166,7 @@ func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Ref delete(p.snapshots, s.OperationID) return nil } -func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { +func (p *fakeSuspensionProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { state, err := p.GetCompute(ctx, r, c) if err != nil { return state, err @@ -165,7 +177,7 @@ func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Re p.computes[c.Name] = state return state, nil } -func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { +func (p *fakeSuspensionProvider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { if _, err := p.GetCompute(ctx, r, c); err != nil { return sandbox.CommandResult{}, err } @@ -178,7 +190,7 @@ func (p *fakeCheckpointProvider) RunCommandCompute(ctx context.Context, r sandbo p.mu.Unlock() return sandbox.CommandResult{}, p.connect(ctx, b) } -func (p *fakeCheckpointProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { +func (p *fakeSuspensionProvider) connect(ctx context.Context, b sandbox.Bootstrap) error { header := http.Header{"Authorization": []string{"Bearer " + b.Credential}} conn, _, err := websocket.DefaultDialer.DialContext(ctx, p.endpoint+"?device_id="+b.DeviceID+"&version="+proto.Version, header) if err != nil { @@ -256,7 +268,7 @@ func (p *fakeCheckpointProvider) connect(ctx context.Context, b sandbox.Bootstra type computeLifecycleFixture struct { t *testing.T store *Store - provider *fakeCheckpointProvider + provider *fakeSuspensionProvider worker *execution.Worker stop func() key string @@ -265,10 +277,13 @@ type computeLifecycleFixture struct { } func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *computeLifecycleFixture { + return computeFixtureForMode(t, maxActive, maxRetained, false) +} +func computeFixtureForMode(t *testing.T, maxActive, maxRetained int, direct bool) *computeLifecycleFixture { t.Helper() s, _ := newManagedTestStore(t) registry := runtimegateway.NewRegistry() - p := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} + p := &fakeSuspensionProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.RetainedState{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} handler := runtimegateway.NewHandler(runtimegateway.HandlerConfig{Authenticator: runtimegateway.NewAuthenticator(sessionAdapter(s)), Registry: registry}) server := httptest.NewServer(http.HandlerFunc(handler.WS)) p.endpoint = "ws" + strings.TrimPrefix(server.URL, "http") @@ -280,12 +295,18 @@ func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *compu } server.Close() }) - f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, "microsandbox"), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour}} + kind := "microsandbox" + if direct { + kind = "e2b" + } + f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, kind), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour, MaxActive: maxActive, MaxRetained: maxRetained}} view, err := deploymentService(t, s).View(t.Context()) if err != nil { t.Fatal(err) } - f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID + if !direct { + f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID + } f.start() return f } @@ -294,7 +315,9 @@ func (f *computeLifecycleFixture) start() { t.Helper() w := startWebWorker(t, f.store, f.provider.registry, f.key, f.provider, &f.policy) // The Worker's claim starts a new owner epoch, in which the node reconnects. - onlineManagerNode(t, f.store, f.node) + if f.node != "" { + onlineManagerNode(t, f.store, f.node) + } var once sync.Once stop := func() { once.Do(func() { ctx, cancel := context.WithCancel(context.Background()); cancel(); _ = w.Run(ctx) }) @@ -365,7 +388,7 @@ func TestRuntimeComputeLifecycleIdleSuspendAndQueuedSameSessionWake(t *testing.T f.phase(tenant, env.ID, "running") } if f.provider.captures != 0 { - t.Fatal("never-used Session suspended") + t.Fatal("recently-created Session suspended") } completed := f.complete(owner) suspended := f.phase(tenant, env.ID, "suspended") @@ -389,6 +412,28 @@ func TestRuntimeComputeLifecycleIdleSuspendAndQueuedSameSessionWake(t *testing.T } } +func TestRuntimeComputeLifecycleUnusedSessionSuspendsAndExpires(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, session, env, owner := f.create() + f.sql(`UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 minutes' WHERE id=$1`, owner.ID) + suspended := f.phase(tenant, env.ID, "suspended") + if suspended.ComputeRetainedUntil == nil || f.provider.captures != 1 || f.provider.computeKills != 1 || len(f.provider.computes) != 0 || len(f.provider.snapshots) != 1 { + t.Fatal("unused Session did not suspend and release active compute") + } + var turns int + if err := f.store.pool.QueryRow(t.Context(), `SELECT count(*) FROM turns WHERE session_id=$1`, session.ID).Scan(&turns); err != nil || turns != 0 { + t.Fatal("unused Session gained a Turn", turns, err) + } + f.sql(`UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1`, owner.ID) + reconcileManagedState(t, f.worker, f.store, tenant, env.ID, "released") + if len(f.provider.computes) != 0 || len(f.provider.snapshots) != 0 || f.provider.snapshotDeletes != 1 || f.provider.promptFrames.Load() != 0 { + t.Fatal("unused Session expiry retained resources or sent execution input") + } + if _, err := sessionAdapter(f.store).GetSession(t.Context(), tenant, session.ID); err != nil { + t.Fatal("resource cleanup removed Session history", err) + } +} + func TestRuntimeComputeLifecycleLostCaptureAndRestoreObserveWithoutReplay(t *testing.T) { f := newComputeLifecycleFixture(t, 1, 2) tenant, _, env, owner := f.create() @@ -514,3 +559,126 @@ func TestRuntimeComputeLifecycleSuspendedDeletionAndExpiryCleanup(t *testing.T) }) } } + +func TestRuntimeComputeLifecycleCapacityBoundsActiveAndRetained(t *testing.T) { + f := computeFixtureForMode(t, 1, 2, true) + tenant, _, env, owner := f.create() + tenant2, _, env2 := managedSession(t, f.store) + if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { + t.Fatalf("active capacity ignored: %v", err) + } + if f.provider.creates != 1 { + t.Fatal("capacity rejection allocated compute") + } + f.complete(owner) + f.phase(tenant, env.ID, "suspended") + second, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key) + if err != nil { + t.Fatal(err) + } + pending := f.queued(owner) + for range 4 { + f.worker.ReconcileManagedRuntimes(t.Context()) + } + first, err := deploymentStore(f.store).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}) + if err != nil || first.ComputePhase != "suspended" || f.provider.restores != 0 { + t.Fatal("wake exceeded active capacity", err) + } + f.sql(`UPDATE turns SET status='cancelled',completed_at=clock_timestamp() WHERE id=$1`, pending) + f.provider.mu.Lock() + b := f.provider.bootstraps[second.ID] + f.provider.mu.Unlock() + if err := f.provider.connect(t.Context(), b); err != nil { + t.Fatal(err) + } + second = f.phase(tenant2, env2.ID, "running") + f.complete(second) + f.phase(tenant2, env2.ID, "suspended") + // Both retained allocations count even when their source VMs are gone. + tenant3, _, env3 := managedSession(t, f.store) + if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant3, env3.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { + t.Fatalf("retained capacity ignored: %v", err) + } + if f.provider.creates != 2 { + t.Fatal("retained limit created a third allocation") + } +} + +func (p *fakeSuspensionProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + p.renewals.Add(1) + return p.GetCompute(ctx, r, c) +} + +func TestRuntimeComputeProtocolUpgradeRefusesOldReceiptsBeforeCleanup(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + f.complete(owner) + retained := f.phase(tenant, env.ID, "suspended") + f.stop() + for _, versioned := range []bool{false, true} { + f.sql(`UPDATE runtime_allocations SET compute_state=($2::jsonb-'retained') || jsonb_build_object('snapshot',$2::jsonb->'retained') WHERE id=$1`, owner.ID, retained.ComputeState) + if !versioned { + f.sql(`UPDATE runtime_allocations SET compute_state=compute_state-'protocol_version' WHERE id=$1`, owner.ID) + } + deletes := f.provider.snapshotDeletes + w, err := startNextWorker(t, t.Context(), f.store, &execution.Dispatcher{Registry: f.provider.registry, ManagedRuntimes: webRuntimes(t, f.store, f.key, f.provider, &f.policy)}) + if err == nil || w != nil || !strings.Contains(err.Error(), "previous release") { + t.Fatalf("incompatible state activated: %v", err) + } + if f.provider.snapshotDeletes != deletes || len(f.provider.snapshots) != 1 { + t.Fatal("upgrade lost owned artifact") + } + var state []byte + if err := f.store.pool.QueryRow(t.Context(), `SELECT compute_state FROM runtime_allocations WHERE id=$1`, owner.ID).Scan(&state); err != nil { + t.Fatal(err) + } + if !strings.Contains(string(state), `"snapshot"`) { + t.Fatal("old receipt was rewritten") + } + } + // A consumed legacy snapshot can survive only in current/target provenance. + for _, field := range []string{"current", "target"} { + f.sql(`UPDATE runtime_allocations SET compute_state=$2::jsonb || jsonb_build_object($3::text, jsonb_build_object('RestoredFrom', jsonb_build_object('Digest', 'legacy'))) WHERE id=$1`, owner.ID, retained.ComputeState, field) + incompatible, err := db.New(f.store.pool).HasIncompatibleRuntimeComputeState(t.Context(), sandbox.SuspensionStateVersion) + if err != nil || !incompatible { + t.Fatalf("legacy %s provenance accepted: incompatible=%v err=%v", field, incompatible, err) + } + } + // Adapter-owned data may contain any private field names. The activation + // fence must inspect only the shared envelope, never this opaque string. + f.sql(`UPDATE runtime_allocations SET compute_state=jsonb_set($2::jsonb, '{retained,Data}', to_jsonb($3::text)) WHERE id=$1`, owner.ID, retained.ComputeState, `{"snapshot":{"Digest":"private","CheckpointID":"native","CheckpointRoot":"owned"}}`) + incompatible, err := db.New(f.store.pool).HasIncompatibleRuntimeComputeState(t.Context(), sandbox.SuspensionStateVersion) + if err != nil || incompatible { + t.Fatalf("opaque native payload affected activation: incompatible=%v err=%v", incompatible, err) + } + f.sql(`UPDATE runtime_allocations SET compute_state=$2::jsonb WHERE id=$1`, owner.ID, retained.ComputeState) + f.start() +} + +func TestRuntimeComputeRepeatedWakeStillRenewsCurrentIncarnation(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + tenant, _, env, owner := f.create() + before := f.provider.renewals.Load() + for range 3 { + f.sql(`UPDATE runtime_allocations SET compute_wake_requested=true,compute_activity_at=clock_timestamp() WHERE id=$1`, owner.ID) + f.phase(tenant, env.ID, "running") + } + if f.provider.renewals.Load() < before+3 { + t.Fatal("wake requests bypassed native lease renewal") + } +} + +func TestRuntimeComputeProtocolRejectsOldDisabledAllocation(t *testing.T) { + f := newComputeLifecycleFixture(t, 1, 2) + _, _, _, owner := f.create() + f.stop() + f.sql(`UPDATE runtime_allocations SET compute_phase='disabled',compute_state='{}'::jsonb WHERE id=$1`, owner.ID) + before := f.provider.renewals.Load() + w, err := startNextWorker(t, t.Context(), f.store, &execution.Dispatcher{Registry: f.provider.registry, ManagedRuntimes: webRuntimes(t, f.store, f.key, f.provider, &f.policy)}) + if err == nil || w != nil || !strings.Contains(err.Error(), "previous release") { + t.Fatal("old disabled allocation activated", err) + } + if f.provider.renewals.Load() != before || len(f.provider.computes) != 1 { + t.Fatal("upgrade changed owned native compute") + } +} diff --git a/services/core/tests/integration/runtime_idle_clock_test.go b/services/core/tests/integration/runtime_idle_clock_test.go index b792c00e5..d2cfdffc5 100644 --- a/services/core/tests/integration/runtime_idle_clock_test.go +++ b/services/core/tests/integration/runtime_idle_clock_test.go @@ -64,7 +64,7 @@ func verifyManagedIdleClock(t *testing.T, s, w *Store, owner deployment.Allocati observedBefore := runtimeDatabaseTime(t, s) activity, err := deploymentStore(w).Activity(t.Context(), owner.ID) observedAfter := runtimeDatabaseTime(t, s) - if err != nil || activity.ObservedAt.Before(observedBefore) || activity.ObservedAt.After(observedAfter) || activity.ReadyToSuspend(idleTimeout) || activity.LastActivity.Before(before) || activity.LastActivity.After(after) || activity.Busy || activity.WakeRequested || !activity.HasCompletedTurn { + if err != nil || activity.ObservedAt.Before(observedBefore) || activity.ObservedAt.After(observedAfter) || activity.ReadyToSuspend(idleTimeout) || activity.LastActivity.Before(before) || activity.LastActivity.After(after) || activity.Busy || activity.WakeRequested { t.Fatal("idle clock did not use committed terminal ingestion", activity, before, after, err) } until := runtimeDatabaseTime(t, s).Add(time.Hour) @@ -225,3 +225,24 @@ func TestManagedIdleClockReconnectPreservesReceipts(t *testing.T) { t.Fatal(err) } } + +func TestDirectManagedIdleClockIgnoresNativeClockSkew(t *testing.T) { + for _, skew := range []time.Duration{-269 * time.Second, 269 * time.Second} { + t.Run(skew.String(), func(t *testing.T) { + s, w, owner := managedIdleClockFixture(t) + runtimeSuspensionSQL(t, s.pool, "UPDATE runtime_allocations SET node_id=NULL WHERE id=$1", owner.ID) + owner.NodeID = "" + turn := uuid.NewString() + runtimeSuspensionSQL(t, s.pool, "INSERT INTO turns(id,session_id,status,started_at) VALUES($1,$2,'in_progress',clock_timestamp())", turn, owner.SessionID) + source := runtimeDatabaseTime(t, s).Add(skew).UnixMilli() + outcome := json.RawMessage(fmt.Sprintf(`{"done":{"source_completed_at_ms":%d}}`, source)) + before := runtimeDatabaseTime(t, s) + completed, err := sessionExecution(t, w.lease).CompleteExecution(t.Context(), owner.TenantID, owner.SessionID, turn, sessions.TurnCompleted, outcome, "", 0) + after := runtimeDatabaseTime(t, s) + if err != nil || completed.CompletedAt.UnixMilli() != source { + t.Fatal(completed, err) + } + verifyManagedIdleClock(t, s, w, owner, before, after) + }) + } +} diff --git a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go index 055ac6383..837cb4279 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go @@ -25,7 +25,7 @@ import ( ) type nodeIsolationProvider struct { - *fakeCheckpointProvider + *fakeSuspensionProvider blockMu sync.Mutex blocked map[string]bool mode string @@ -49,7 +49,7 @@ func (p *nodeIsolationProvider) block(ctx context.Context, r sandbox.Reference, return ctx.Err() } func (p *nodeIsolationProvider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { - info, err := p.fakeCheckpointProvider.Create(ctx, b) + info, err := p.fakeSuspensionProvider.Create(ctx, b) if err == nil { err = p.connect(ctx, b) } @@ -59,13 +59,13 @@ func (p *nodeIsolationProvider) GetInfo(ctx context.Context, r sandbox.Reference if err := p.block(ctx, r, "observe"); err != nil { return sandbox.Info{}, err } - return p.fakeCheckpointProvider.GetInfo(ctx, r) + return p.fakeSuspensionProvider.GetInfo(ctx, r) } func (p *nodeIsolationProvider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { if err := p.block(ctx, r, "observe"); err != nil { return sandbox.ComputeState{}, err } - return p.fakeCheckpointProvider.GetCompute(ctx, r, c) + return p.fakeSuspensionProvider.GetCompute(ctx, r, c) } func (p *nodeIsolationProvider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { return p.preparation.RunCommand(ctx, r, c) @@ -90,8 +90,8 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { t.Helper() s, pool := newManagedTestStore(t) registry := runtimegateway.NewRegistry() - cp := &fakeCheckpointProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.SnapshotIdentity{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} - p := &nodeIsolationProvider{fakeCheckpointProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} + cp := &fakeSuspensionProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, computes: map[string]sandbox.ComputeState{}, snapshots: map[string]sandbox.RetainedState{}, bootstraps: map[string]sandbox.Bootstrap{}, peers: map[string]*websocket.Conn{}, registry: registry} + p := &nodeIsolationProvider{fakeSuspensionProvider: cp, blocked: map[string]bool{}, mode: mode, entered: make(chan struct{})} preparationContext, cancelPreparation := context.WithCancel(t.Context()) t.Cleanup(cancelPreparation) cp.preparation = &initializationPeer{t: t, apply: func(request proto.RuntimePreparePayload, data []byte) proto.RuntimePrepareResultPayload { diff --git a/services/core/tests/integration/runtime_suspension_test.go b/services/core/tests/integration/runtime_suspension_test.go index 1d5eabad8..3e2a9cf4c 100644 --- a/services/core/tests/integration/runtime_suspension_test.go +++ b/services/core/tests/integration/runtime_suspension_test.go @@ -64,7 +64,7 @@ func runtimeSuspensionStep(t *testing.T, w *Store, owner deployment.Allocation, return next } -func TestRuntimeSuspensionRequiresCompletedIdleAndNoPendingWork(t *testing.T) { +func TestRuntimeSuspensionRequiresIdleAndNoPendingWork(t *testing.T) { cases := []string{"no_completed_turn", "queued", "in_progress", "waiting", "subagent_queued", "subagent_in_progress", "subagent_waiting", "input_reservation", "file_write", "idle"} for _, kind := range cases { t.Run(kind, func(t *testing.T) { @@ -91,12 +91,12 @@ func TestRuntimeSuspensionRequiresCompletedIdleAndNoPendingWork(t *testing.T) { t.Fatal(err) } wantBusy := kind != "idle" && kind != "no_completed_turn" - if activity.Busy != wantBusy || activity.HasCompletedTurn != (kind != "no_completed_turn") { + if activity.Busy != wantBusy { t.Fatalf("activity lost pending work: %+v", activity) } until := time.Now().Add(time.Hour) _, err = deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Nanosecond) - if kind == "idle" { + if kind == "idle" || kind == "no_completed_turn" { if err != nil { t.Fatal(err) } @@ -261,17 +261,60 @@ func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { } } +func TestRuntimeSuspensionCountsUncertainCapacityUntilReleased(t *testing.T) { + s, provider := configuredStore(t) + pool := s.pool + w := executionWriter(t, s) + cases := []struct { + state, phase string + count bool + }{ + {"creating", "disabled", true}, {"running", "running", true}, {"running", "quiescing", true}, {"running", "suspending", true}, {"running", "suspended", false}, {"running", "restoring", true}, {"running", "waking", true}, {"cleanup_pending", "restoring", true}, {"released", "running", false}, + } + want := int64(0) + wantRetained := int64(0) + for _, item := range cases { + tenant := uuid.NewString() + _, env := localEnvironment(t, s, tenant) + owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}, provider, runtimedevice.HashCredential(uuid.NewString())) + if err != nil { + t.Fatal(err) + } + runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET state=$2,compute_phase=$3,create_settled=($2<>'creating'),released_at=CASE WHEN $2='released' THEN clock_timestamp() END WHERE id=$1`, owner.ID, item.state, item.phase) + if item.count { + want++ + } + if item.state != "released" { + wantRetained++ + } + retained, err := deploymentStore(w).CountRetainedAllocations(t.Context(), provider) + if err != nil || retained != wantRetained { + t.Fatalf("retained capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, retained, wantRetained, err) + } + got, err := deploymentStore(w).CountComputeReservations(t.Context(), provider) + if err != nil || got != want { + t.Fatalf("capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, got, want, err) + } + } + got, err := deploymentStore(w).CountComputeReservations(t.Context(), uuid.NewString()) + if err != nil || got != 0 { + t.Fatal("capacity crossed installation boundary", got, err) + } +} + func TestRuntimeSuspensionIdleStartsAfterLastCompletion(t *testing.T) { - _, w, pool, owner := runtimeSuspensionFixture(t) - turn := runtimeSuspensionCompleted(t, pool, owner) + s, w, pool, owner := runtimeSuspensionFixture(t) + runtimeSuspensionCompleted(t, pool, owner) runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET compute_activity_at=clock_timestamp()-interval '2 hours' WHERE id=$1`, owner.ID) - var completed time.Time - if err := pool.QueryRow(t.Context(), `SELECT completed_at FROM turns WHERE id=$1`, turn).Scan(&completed); err != nil { + before := runtimeDatabaseTime(t, s) + id, _ := parseID(owner.SessionID) + if err := s.queries.RecordRuntimeTerminalActivity(t.Context(), id); err != nil { t.Fatal(err) } + after := runtimeDatabaseTime(t, s) activity, err := deploymentStore(w).Activity(t.Context(), owner.ID) - if err != nil || !activity.LastActivity.Equal(completed) { - t.Fatal("long Turn completion did not restart idle interval", activity, completed, err) + if err != nil || activity.LastActivity.Before(before) || activity.LastActivity.After(after) || activity.ReadyToSuspend(time.Minute) { + t.Fatal("long Turn completion did not restart the ingestion idle interval", activity, before, after, err) } } @@ -352,6 +395,12 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { default: runtimeSuspensionSQL(t, pool, `INSERT INTO environment_file_writes(id,environment_id,device_id,request_sha256,state,created_at,settled_at) VALUES($1,$2,$3,$4,$5,clock_timestamp()-interval '10 minutes',clock_timestamp())`, uuid.NewString(), owner.EnvironmentID, owner.DeviceID, strings.Repeat("a", 64), strings.TrimPrefix(kind, "file_")) } + if kind == "root" || kind == "subagent" { + id, _ := parseID(owner.SessionID) + if err := s.queries.RecordRuntimeTerminalActivity(t.Context(), id); err != nil { + t.Fatal(err) + } + } until := time.Now().Add(time.Hour) if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, idleTimeout); !errors.Is(err, deployment.ErrAllocationConflict) { t.Fatal("completion after idle observation did not fence quiesce", err) @@ -363,6 +412,11 @@ func TestRuntimeSuspensionRechecksCompletionAgainstIdleTimeout(t *testing.T) { if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, 0); !errors.Is(err, deployment.ErrInvalidInput) { t.Fatal("missing idle timeout accepted", err) } + // Re-observe the allocation after terminal ingestion advanced its activity fence. + owner, err = deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: owner.TenantID, EnvironmentID: owner.EnvironmentID}) + if err != nil { + t.Fatal(err) + } if _, err := deploymentExecution(t, w).SetCompute(t.Context(), owner, "quiescing", json.RawMessage(`{}`), &until, time.Nanosecond); err != nil { t.Fatal("elapsed idle timeout rejected", err) } diff --git a/services/core/tests/integration/runtime_wake_hint_integration_test.go b/services/core/tests/integration/runtime_wake_hint_integration_test.go index aedeca68c..ebb976bd7 100644 --- a/services/core/tests/integration/runtime_wake_hint_integration_test.go +++ b/services/core/tests/integration/runtime_wake_hint_integration_test.go @@ -16,7 +16,7 @@ import ( ) type wakeHintScanProvider struct { - *fakeCheckpointProvider + *fakeSuspensionProvider sentinel string release chan struct{} scans chan int @@ -35,7 +35,7 @@ func (p *wakeHintScanProvider) GetCompute(ctx context.Context, reference sandbox } } } - return p.fakeCheckpointProvider.GetCompute(ctx, reference, compute) + return p.fakeSuspensionProvider.GetCompute(ctx, reference, compute) } type wakeHintIntegrationTarget struct { @@ -70,7 +70,7 @@ func newWakeHintIntegration(t *testing.T) *wakeHintIntegration { target.owner = f.phase(target.tenant, target.environment.ID, "suspended") f.stop() provider := &wakeHintScanProvider{ - fakeCheckpointProvider: f.provider, sentinel: sentinel.owner.ID, + fakeSuspensionProvider: f.provider, sentinel: sentinel.owner.ID, release: make(chan struct{}), scans: make(chan int, 16), } worker := startWebWorker(t, f.store, f.provider.registry, f.key, provider, &f.policy) diff --git a/services/core/tests/integration/runtime_worker_recovery_test.go b/services/core/tests/integration/runtime_worker_recovery_test.go index 7cb9c7f68..5fce9b5dd 100644 --- a/services/core/tests/integration/runtime_worker_recovery_test.go +++ b/services/core/tests/integration/runtime_worker_recovery_test.go @@ -2,12 +2,14 @@ package integration import ( "context" + "encoding/json" "errors" "testing" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" @@ -15,11 +17,24 @@ import ( func insertWorkerRuntimeAllocation(t *testing.T, pool *pgxpool.Pool, h *dispatchHarness, phase string) { t.Helper() - _, err := pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,compute_phase,compute_retained_until,deployment_generation) - VALUES($1,$2,$3,$4,'running',true,$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment))`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase) + state, err := json.Marshal(map[string]any{"protocol_version": sandbox.SuspensionStateVersion, "current": sandbox.Compute{Name: h.device.EnvironmentID, ID: h.device.EnvironmentID}}) if err != nil { t.Fatal(err) } + _, err = pool.Exec(t.Context(), `INSERT INTO runtime_allocations(id,environment_id,device_id,provider_key,state,create_settled,compute_phase,compute_retained_until,deployment_generation,compute_state) + VALUES($1,$2,$3,$4,'running',true,$5,clock_timestamp()+interval '1 hour',(SELECT generation FROM runtime_deployment),$6)`, uuid.NewString(), h.device.EnvironmentID, h.device.ID, uuid.NewString(), phase, state) + if err != nil { + t.Fatal(err) + } + // This synthetic allocation must not outlive the test in the shared fixture + // database, where the next worker validates every retained protocol receipt. + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := pool.Exec(ctx, "DELETE FROM runtime_allocations WHERE environment_id=$1", h.device.EnvironmentID); err != nil { + t.Error(err) + } + }) } func runtimeWorkerHarness(t *testing.T) (*dispatchHarness, *pgxpool.Pool) { diff --git a/services/core/tests/integration/sandbox_deployment_view_test.go b/services/core/tests/integration/sandbox_deployment_view_test.go index ca1ed70b7..9f8051f31 100644 --- a/services/core/tests/integration/sandbox_deployment_view_test.go +++ b/services/core/tests/integration/sandbox_deployment_view_test.go @@ -38,7 +38,7 @@ func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { for _, want := range []string{ `"specification":{"resources":{"cpus":2,"memory_mib":2048}}`, `"template_build":{"status":"ready","resources":{"cpus":2,"memory_mib":2048,"root_disk_mib":24063}}`, - `"suspension":null`, + `"suspension":{"idle_seconds":300,"retention_seconds":86400}`, } { if !bytes.Contains(raw, []byte(want)) { t.Fatalf("E2B view lacks %s: %s", want, raw) diff --git a/services/core/tests/integration/sandbox_generations_test.go b/services/core/tests/integration/sandbox_generations_test.go index 580459b2a..da94f3457 100644 --- a/services/core/tests/integration/sandbox_generations_test.go +++ b/services/core/tests/integration/sandbox_generations_test.go @@ -3,6 +3,7 @@ package integration import ( "database/sql" "os" + "strings" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" @@ -215,14 +216,16 @@ func TestGenerationDowngradeRefusesOldAllocation(t *testing.T) { if _, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, input); err != nil { t.Fatal(err) } + // Isolate the generation downgrade guard using the target schema's disabled + // suspension policy. Active suspension itself is not representable there. db := sql.OpenDB(stdlib.GetConnector(*s.pool.Config().ConnConfig)) defer db.Close() migrations, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) if err != nil { t.Fatal(err) } - if _, err = migrations.DownTo(t.Context(), 80); err == nil { - t.Fatal("downgrade erased old owned allocation") + if _, err = migrations.DownTo(t.Context(), 80); err == nil || !strings.Contains(err.Error(), "Cannot downgrade while retained ownership") { + t.Fatal("generation ownership guard did not reject downgrade", err) } // Earlier down migrations can commit before the generation guard vetoes // downgrade. Restore the current schema before invoking current Store code. diff --git a/services/core/tests/integration/session_execution_configuration_test.go b/services/core/tests/integration/session_execution_configuration_test.go index 54dea62c3..e4a1c168c 100644 --- a/services/core/tests/integration/session_execution_configuration_test.go +++ b/services/core/tests/integration/session_execution_configuration_test.go @@ -282,7 +282,7 @@ func TestSessionExecutionConfigurationSurvivesSuspendResume(t *testing.T) { t.Fatal("runtime transition changed execution projection", phase, err) } after, err := deploymentStore(w).Activity(t.Context(), owner.ID) - if err != nil || !before.LastActivity.Equal(after.LastActivity) || before.WakeRequested != after.WakeRequested || before.Busy != after.Busy || before.HasCompletedTurn != after.HasCompletedTurn { + if err != nil || !before.LastActivity.Equal(after.LastActivity) || before.WakeRequested != after.WakeRequested || before.Busy != after.Busy { t.Fatal("configuration read changed runtime activity", err) } } diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index 46f1e9f15..2f8b9a8a0 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -30,7 +30,7 @@ Compatible endpoints must return the SDK 2.51.0 template-list and template-build Run `go generate ./services/core/internal/sandbox/e2b` from the repository root after changing these declarations. `make check-e2b-provider` and the Go adapter tests reject stale projections; both languages consume generated valid and invalid exchanges covering wire types, extra fields, operation/reference bounds and managed-bootstrap fields. The helper build copies those fixtures with its source before running the pinned-SDK suite. -The boundary has version 2. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. +The boundary has version 4. Managed bootstrap and Resume accept an omitted or null `Workspace`; E2B rejects non-null external workspace bindings before native calls. The request and credentials arrive on standard input; standard output carries one bounded response with a sanitized error code. The helper removes ambient `E2B_*` and `PYTHON*` variables and calls the SDK only with the request's explicit API origin and sandbox domain. Each receipt is bound to those selectors, and a receipt without them belongs to the official endpoints (`https://api.e2b.app`, `e2b.app`). An endpoint change keeps earlier generations on their original API and sandbox domain; the candidate key must verify all retained ownership before an online switch. ## Receipts and state directory @@ -50,7 +50,7 @@ An unknown Create is never repeated. A Create whose connection material was lost ## Inspection and cleanup -Inspection uses SDK metadata and ID reads only. SDK `connect` is never used because it can resume paused compute. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. +Inspection uses SDK metadata and ID reads only. Inspection never uses SDK `connect`, which is reserved for the explicit managed Resume operation. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. `CreateSettled` proves that the original Create and bootstrap can no longer mutate; it is independent of `BootstrapComplete`. An empty lookup never settles an unknown Create. Kill destroys every matching sandbox, confirms that none remains and only then records a settled tombstone; it returns `State=absent` with `CreateSettled`. A settled rejected Create with no sandbox IDs proves absence without a cloud request, so GetInfo and Kill still succeed when the key is invalid. Ordinary missing compute has no such proof. @@ -71,3 +71,11 @@ make check-e2b-provider ``` With `OAC_TEST_E2B_SDK_PYTHON` pointing at the pinned SDK environment, this runs this directory's tests and the [template scripts' tests](../../deploy/e2b/README.md#tests). The helper build runs this directory's suite and checks the relocated helper's `--check` report. These tests create no cloud resources. + +## Managed suspension + +The adapter implements the complete shared suspension group using opaque retained state. A native pause retains the same sandbox; it is not a portable filesystem snapshot. Core's common activity rule quiesces the Runtime, then the adapter calls the pinned SDK's memory-preserving `Sandbox.pause`. The private allocation receipt commits the operation before native I/O and reports resource release only after observing the exact sandbox paused. + +Resume calls `Sandbox.connect` once with `on_resume=restore` and the existing native timeout. It preserves the native sandbox ID while advancing the shared logical generation. Fresh connection material is persisted before returning running. Unknown pause and connect outcomes are observed without replay; a missing connect receipt keeps execution unavailable. The adapter fences stale generations before commands and deletion. Consumed pause receipt cleanup preserves running compute. + +The shared registration owns the 300-second idle duration and 86400-second retention default. Native timeout remains 3600 seconds and automatic native resume remains disabled. SDK calls use the configured official-compatible endpoint. Generated declarations and fixtures own the private helper shape. diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index 0b7635c78..7f420e0aa 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -1,5 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" +COMPUTE_FIELDS = ["Generation","Name","ID","RestoredFrom"] ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","NetworkAccess","AllowedDomains","InstallationID","RuntimeBootstrap"] @@ -10,9 +11,13 @@ MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 NETWORK_ACCESS = ["enabled","disabled","restricted"] -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] -PROTOCOL_VERSION = 2 +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute_info","compute_renew","suspend","resume","compute_kill","delete_retained","compute_command","resume_compute"] +PROTOCOL_VERSION = 4 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Retained","Deadline"] +RESPONSE_FIELDS = ["Version","State","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +RESUME_FIELDS = ["Workspace","Reference","OperationID","Retained","Target","ReconcileOnly"] +RETAINED_FIELDS = ["Reference","ID","Data","OperationID","SourceGeneration","SourceName","SourceID"] SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" +SUSPEND_FIELDS = ["Reference","OperationID","Source","Retained","ReconcileOnly"] diff --git a/services/core/tools/e2b-provider/observation_test.py b/services/core/tools/e2b-provider/observation_test.py index f72554e8d..3a773ca24 100644 --- a/services/core/tools/e2b-provider/observation_test.py +++ b/services/core/tools/e2b-provider/observation_test.py @@ -7,6 +7,7 @@ from provider import Provider from provider_test import ProviderTest from state import Failure +from helper_contract_generated import PROTOCOL_VERSION class ObservationTest(ProviderTest): @@ -15,7 +16,7 @@ def observe(self, reference=None): try: return Provider(request).execute() except Failure as error: - return {'Version': 2, 'ErrorCode': error.code} + return {'Version': PROTOCOL_VERSION, 'ErrorCode': error.code} def listing(self, *pages): pages = [list(page) for page in pages] or [[]] diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index 264b39489..a75d09a67 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -105,6 +105,9 @@ def __init__(self, request): bootstrap = request.get('Bootstrap') if bootstrap is not None and (not isinstance(bootstrap, dict) or bootstrap.get('Workspace') is not None): raise Failure('invalid') + resume = request.get('Resume') + if resume is not None and (not isinstance(resume, dict) or resume.get('Workspace') is not None): + raise Failure('invalid') deadline = datetime.fromisoformat(request['Deadline'].replace('Z', '+00:00')) self.deadline = time.monotonic() + (deadline - datetime.now(timezone.utc)).total_seconds() self.metadata = {PREFIX + field.lower(): value for field, value in @@ -238,7 +241,9 @@ def create(self): if definitely_rejected(error): self.receipt.save(status='rejected', settled=True) raise Failure('unconfirmed') from None - self.receipt.save(status='created', ids=[cloud.sandbox_id], connection=connection_material(cloud)) + self.receipt.save(status='created', ids=[cloud.sandbox_id], connection=connection_material(cloud), + compute={'Generation': 0, 'Name': self.reference['AllocationID'], + 'ID': cloud.sandbox_id, 'RestoredFrom': None}) # A create response must not steer envd traffic to an unrelated host. self.check_domain(cloud) # SDK Create returns connection material, but no metadata or resources. @@ -413,6 +418,10 @@ def execute(self): with Receipt(self.q, self.remaining) as self.receipt: try: operation = self.q['Operation'] + if operation in ('compute_info', 'compute_renew', 'suspend', 'resume', 'compute_kill', + 'delete_retained', 'compute_command', 'resume_compute'): + from suspension import Suspension + return Suspension(self, Sandbox, connection_material, run).execute() if operation == 'kill': self.kill() return {'Version': PROTOCOL_VERSION, 'Info': self.info(absent=True), 'ErrorCode': ''} diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 240e05eab..9bd122dea 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -65,7 +65,7 @@ def setUp(self): self.reference = {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']} self.config = {'StateDir': self.temporary.name, 'InstallationID': str(uuid4()), 'APIKey': 'private-account-secret', 'Template': 'test:' + str(uuid4()), 'TimeoutSeconds': 120} - self.request = {'Version': 2, 'Operation': 'create', 'Config': self.config, + self.request = {'Version': PROTOCOL_VERSION, 'Operation': 'create', 'Config': self.config, 'Reference': self.reference, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=30)).isoformat(), 'Bootstrap': dict(self.reference, SessionID=str(uuid4()), DeviceID=str(uuid4()), CoreURL='https://core.example/api/v1', Credential='private-runtime-secret', diff --git a/services/core/tools/e2b-provider/state_test.py b/services/core/tools/e2b-provider/state_test.py index 9ff5b89b4..7de903990 100644 --- a/services/core/tools/e2b-provider/state_test.py +++ b/services/core/tools/e2b-provider/state_test.py @@ -11,6 +11,7 @@ from unittest.mock import patch from provider import Provider +from helper_contract_generated import PROTOCOL_VERSION from state import Failure, Receipt @@ -18,7 +19,7 @@ class StateTest(unittest.TestCase): def setUp(self): self.root = tempfile.TemporaryDirectory() self.addCleanup(self.root.cleanup) - self.request = {'Version': 2, 'Operation': 'kill', + self.request = {'Version': PROTOCOL_VERSION, 'Operation': 'kill', 'Config': {'StateDir': self.root.name, 'InstallationID': str(uuid4()), 'APIKey': 'test'}, 'Reference': {key: str(uuid4()) for key in ['TenantID', 'EnvironmentID', 'AllocationID']}, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=2)).isoformat()} diff --git a/services/core/tools/e2b-provider/suspension.py b/services/core/tools/e2b-provider/suspension.py new file mode 100644 index 000000000..d37b860da --- /dev/null +++ b/services/core/tools/e2b-provider/suspension.py @@ -0,0 +1,228 @@ +"""E2B's implementation of exact-incarnation suspension under the allocation lock.""" +import json + +from helper_contract_generated import PROTOCOL_VERSION, COMPUTE_FIELDS, SUSPEND_FIELDS, RESUME_FIELDS +from state import Failure + + +class Suspension: + def __init__(self, provider, sdk, material, command): + self.p, self.sdk, self.material, self.command = provider, sdk, material, command + + @property + def record(self): + record = self.p.receipt.data + if not record or record.get('settled') is not True: + raise Failure('unconfirmed') + return record + + def current(self): + current = self.record.get('compute') + if not isinstance(current, dict) or not current.get('ID'): + raise Failure('unconfirmed') + return current + + def matches(self, wanted, actual, unresolved=False): + if (not isinstance(wanted, dict) or set(wanted) != set(COMPUTE_FIELDS) or + type(wanted['Generation']) is not int or wanted['Generation'] < 0 or + wanted['Name'] != self.p.reference['AllocationID'] or + any(wanted[k] != actual[k] for k in ('Generation', 'Name', 'RestoredFrom')) or + (wanted['ID'] != actual['ID'] and not (unresolved and wanted['ID'] == ''))): + raise Failure('ownership') + + def cloud(self, execution=True): + found = self.p.discover() + if len(found) != 1: + raise Failure('unconfirmed') + cloud = found[0] + if cloud.sandbox_id != self.current()['ID']: + raise Failure('ownership') + if execution: + self.p.qualified(cloud) + return cloud + + def state(self, current, cloud): + return {'Compute': current, 'Status': cloud.state, + 'BootstrapComplete': self.record.get('bootstrap_complete') is True, + 'Retained': None, 'ResourcesReleased': False, 'SuspendSettled': False} + + def retained(self, operation, current): + return {'Reference': self.p.reference['AllocationID'], 'ID': operation, 'OperationID': operation, + 'SourceGeneration': current['Generation'], 'SourceName': current['Name'], + 'SourceID': current['ID'], + 'Data': json.dumps({'version': 1, 'sandbox_id': current['ID']}, sort_keys=True, separators=(',', ':'))} + + def request(self, field): + q = self.p.q.get(field) + fields = set(SUSPEND_FIELDS if field == 'Suspend' else RESUME_FIELDS) + required = fields - ({'Workspace'} if field == 'Resume' else set()) + if (not isinstance(q, dict) or set(q) - fields or not required <= set(q) or + (field == 'Resume' and q.get('Workspace') is not None) or + q.get('Reference') != self.p.reference): + raise Failure('invalid') + # UUID parsing is shared with the allocation envelope validator. + from provider import valid_id + if not valid_id(q.get('OperationID')) or type(q.get('ReconcileOnly')) is not bool: + raise Failure('invalid') + return q + + def inspect(self, operation): + current = self.current() + self.matches(self.p.q.get('Compute'), current, unresolved=operation == 'compute_info') + if self.record.get('status') == 'killed': + raise Failure('not_found') + cloud = self.cloud() + if operation != 'compute_info': + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + pending = self.record.get('suspension') + if pending and pending['phase'] not in ('resumed', 'consumed'): + raise Failure('unconfirmed') + if operation == 'compute_renew': + self.sdk.set_timeout(cloud.sandbox_id, self.p.config['TimeoutSeconds'], **self.p.options()) + cloud = self.cloud() + if operation == 'compute_command': + return {'Command': self.command(self.p.client(cloud), self.p.q['Command'], self.p.remaining)} + return {'State': self.state(current, cloud)} + + def suspend(self): + q = self.request('Suspend') + current = self.current() + self.matches(q.get('Source'), current) + handle = self.retained(q['OperationID'], current) + if q.get('Retained') not in (None, handle): + raise Failure('ownership') + entry = self.record.get('suspension') + if entry and entry['retained'] == handle: + if entry['phase'] not in ('pause_pending', 'paused'): + raise Failure('ownership') + elif q['ReconcileOnly']: + # No durable native intent: the allocation lock proves this helper + # never issued pause. Only a qualified running source can roll back. + if entry and entry['phase'] != 'consumed': + raise Failure('ownership') + cloud = self.cloud() + if cloud.state != 'running': + raise Failure('unconfirmed') + state = self.state(current, cloud) + state['SuspendSettled'] = True + return {'State': state} + else: + if entry and entry['phase'] != 'consumed': + raise Failure('ownership') + cloud = self.cloud() + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + entry = {'retained': handle, 'phase': 'pause_pending'} + self.p.receipt.save(suspension=entry) + # An error remains pending. A later observation may prove paused; + # a running observation never proves a timed-out pause cannot land. + self.sdk.pause(current['ID'], keep_memory=True, **self.p.options()) + cloud = self.cloud() + if cloud.state != 'paused': + raise Failure('unconfirmed') + self.p.receipt.save(suspension=dict(entry, phase='paused')) + state = self.state(current, cloud) + state.update(Status='suspended', Retained=handle, ResourcesReleased=True, SuspendSettled=True) + return {'State': state} + + def resume(self): + q = self.request('Resume') + entry = self.record.get('suspension') + if not entry or entry['retained'] != q.get('Retained'): + raise Failure('ownership') + retained = entry['retained'] + target = {'Generation': retained['SourceGeneration'] + 1, 'Name': retained['SourceName'], + 'ID': retained['SourceID'], 'RestoredFrom': retained} + self.matches(q.get('Target'), target) + if entry['phase'] in ('resume_pending', 'resumed', 'consumed'): + if entry.get('resume_id') != q['OperationID'] or entry.get('target') != target: + raise Failure('ownership') + elif entry['phase'] == 'paused' and not q['ReconcileOnly']: + cloud = self.cloud() + if cloud.state != 'paused': + raise Failure('unconfirmed') + entry = dict(entry, phase='resume_pending', resume_id=q['OperationID'], target=target, connected=False) + self.p.receipt.save(suspension=entry) + connected = self.sdk.connect(target['ID'], timeout=self.p.config['TimeoutSeconds'], + on_resume='restore', **self.p.options()) + if connected.sandbox_id != target['ID']: + raise Failure('ownership') + self.p.check_domain(connected) + entry = dict(entry, connected=True) + self.p.receipt.save(suspension=entry, connection=self.material(connected)) + else: + raise Failure('unconfirmed') + # Never repeat connect after an uncertain reply. Fresh connection material + # must have been durably received before execution can resume. + if not entry.get('connected'): + raise Failure('unconfirmed') + cloud = self.cloud() + if cloud.state != 'running' or not self.record.get('bootstrap_complete'): + raise Failure('unconfirmed') + if entry['phase'] != 'consumed': + self.p.receipt.save(compute=target, suspension=dict(entry, phase='resumed')) + return {'State': self.state(target, cloud)} + + def kill(self): + wanted = self.p.q.get('Compute') + current = self.current() + entry = self.record.get('suspension') + target = (entry or {}).get('target') + # Target-first cleanup may arrive before Resume was ever dispatched. + planned = None + if entry: + r = entry['retained'] + planned = {'Generation': r['SourceGeneration'] + 1, 'Name': r['SourceName'], + 'ID': r['SourceID'], 'RestoredFrom': r} + if wanted == planned and target is None and entry['phase'] == 'paused': + return {} + if wanted != current and wanted != target: + # Once destruction is confirmed, stale source cleanup is harmless. + if self.record.get('status') == 'killed' and isinstance(wanted, dict): + if wanted['ID'] == current['ID'] and wanted['Name'] == current['Name'] and wanted['Generation'] < current['Generation']: + return {} + raise Failure('ownership') + if entry and entry['phase'] == 'resume_pending' and entry.get('connected'): + # The SDK reply already settled the original connect. Cleanup owns + # recovery once Core leaves restoring, so reconcile that saved target + # here without another connect or a resource-qualification gate. + if wanted != target: + raise Failure('ownership') + found = self.p.discover() + if len(found) > 1 or any(cloud.sandbox_id != target['ID'] for cloud in found): + raise Failure('ownership') + entry = dict(entry, phase='resumed') + self.p.receipt.save(compute=target, suspension=entry) + if entry and entry['phase'] in ('pause_pending', 'resume_pending'): + # No successful native reply or settled pause has been observed. + raise Failure('unconfirmed') + self.p.kill() + return {} + + def delete(self): + wanted = self.p.q.get('Retained') + entry = self.record.get('suspension') + if not entry or wanted != entry['retained']: + raise Failure('ownership') + if entry['phase'] == 'consumed': + return {} + if self.record.get('status') != 'killed': + if entry['phase'] != 'resumed': + raise Failure('unconfirmed') + self.matches(entry['target'], self.current()) + if self.cloud().state != 'running': + raise Failure('unconfirmed') + # E2B consumed the pause image on restore. Keep its minimal generation + # receipt to reject delayed resume/cleanup; never kill running compute. + self.p.receipt.save(suspension=dict(entry, phase='consumed')) + return {} + + def execute(self): + operation = self.p.q['Operation'] + if operation in ('compute_info', 'compute_renew', 'compute_command', 'resume_compute'): + result = self.inspect(operation) + else: + result = {'suspend': self.suspend, 'resume': self.resume, + 'compute_kill': self.kill, 'delete_retained': self.delete}[operation]() + return dict(result, Version=PROTOCOL_VERSION, ErrorCode='') diff --git a/services/core/tools/e2b-provider/suspension_test.py b/services/core/tools/e2b-provider/suspension_test.py new file mode 100644 index 000000000..875c0d69e --- /dev/null +++ b/services/core/tools/e2b-provider/suspension_test.py @@ -0,0 +1,170 @@ +"""Native lifecycle effects and durable recovery are tested through the helper.""" +import unittest +from uuid import uuid4 + +from provider import Provider +import provider_test + + +class SuspensionTest(unittest.TestCase): + setUp = provider_test.ProviderTest.setUp + call = provider_test.ProviderTest.call + record = provider_test.ProviderTest.record + + def prepare(self): + self.assertEqual(self.call('create')['ErrorCode'], '') + self.api.pause.side_effect = lambda *a, **k: setattr(self.cloud, 'state', 'paused') + def connect(*args, **kwargs): + self.cloud.state = 'running' + return self.cloud + self.api.connect.side_effect = connect + return self.record()['compute'] + + def invoke(self, operation, **payload): + return Provider(dict(self.request, Operation=operation, **payload)).execute() + + def pause(self, source): + q = {'Reference': self.reference, 'OperationID': str(uuid4()), 'Source': source, + 'Retained': None, 'ReconcileOnly': False} + response = self.invoke('suspend', Suspend=q) + return q, response + + def resume_request(self, retained): + target = {'Generation': retained['SourceGeneration'] + 1, 'Name': retained['SourceName'], + 'ID': retained['SourceID'], 'RestoredFrom': retained} + return {'Reference': self.reference, 'OperationID': str(uuid4()), 'Target': target, + 'Retained': retained, 'ReconcileOnly': False} + + def test_resume_rejects_external_workspace_before_sdk(self): + from state import Failure + with self.assertRaises(Failure) as raised: + Provider(dict(self.request, Operation='resume', Resume={'Workspace': {}})) + self.assertEqual(raised.exception.code, 'invalid') + self.api.connect.assert_not_called() + + def test_resume_accepts_null_workspace(self): + current = self.prepare() + _, paused = self.pause(current) + request = dict(self.resume_request(paused['State']['Retained']), Workspace=None) + self.assertEqual(self.invoke('resume', Resume=request)['ErrorCode'], '') + + def test_two_cycles_keep_native_id_and_fence_old_generation(self): + current = self.prepare() + for generation in range(2): + old = current + q, paused = self.pause(current) + self.assertEqual(paused['ErrorCode'], '') + self.assertTrue(paused['State']['ResourcesReleased']) + retained = paused['State']['Retained'] + request = self.resume_request(retained) + result = self.invoke('resume', Resume=request) + self.assertEqual(result['ErrorCode'], '') + current = result['State']['Compute'] + self.assertEqual(current['ID'], old['ID']) + self.assertEqual(current['Generation'], generation + 1) + self.assertEqual(self.invoke('compute_kill', Compute=old)['ErrorCode'], 'ownership') + self.api.kill.assert_not_called() + # Core wakes the parked daemon before deleting the consumed receipt. + self.assertEqual(self.invoke('compute_command', Compute=current, + Command={'Args': ['oac-daemon', 'resume']})['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.assertEqual(self.invoke('compute_renew', Compute=current)['ErrorCode'], '') + self.assertEqual(self.api.pause.call_count, 2) + self.assertEqual(self.api.connect.call_count, 2) + self.assertEqual(self.api.connect.call_args.kwargs['on_resume'], 'restore') + self.assertEqual(self.api.connect.call_args.kwargs['timeout'], self.config['TimeoutSeconds']) + self.assertTrue(self.api.pause.call_args.kwargs['keep_memory']) + + def test_lost_pause_reply_observes_without_replay(self): + current = self.prepare() + def lost(*args, **kwargs): + self.cloud.state = 'paused' + raise TimeoutError() + self.api.pause.side_effect = lost + q, result = self.pause(current) + self.assertEqual(result['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('suspend', Suspend=dict(q, ReconcileOnly=True))['ErrorCode'], '') + self.api.pause.assert_called_once() + + def test_pending_pause_running_cannot_roll_back_or_delete(self): + current = self.prepare() + self.api.pause.side_effect = TimeoutError() + q, result = self.pause(current) + self.assertEqual(result['ErrorCode'], 'unconfirmed') + for _ in range(2): + self.assertEqual(self.invoke('suspend', Suspend=dict(q, ReconcileOnly=True))['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], 'unconfirmed') + self.api.pause.assert_called_once() + self.api.kill.assert_not_called() + + def test_missing_native_pause_intent_can_settle_running_rollback(self): + current = self.prepare() + q = {'Reference': self.reference, 'OperationID': str(uuid4()), 'Source': current, + 'Retained': None, 'ReconcileOnly': True} + state = self.invoke('suspend', Suspend=q)['State'] + self.assertTrue(state['SuspendSettled']) + self.assertFalse(state['ResourcesReleased']) + self.assertIsNone(state['Retained']) + self.api.pause.assert_not_called() + + def test_lost_resume_reply_never_replays_connect(self): + current = self.prepare() + _, paused = self.pause(current) + q = self.resume_request(paused['State']['Retained']) + def lost(*a, **k): + self.cloud.state = 'running' + raise TimeoutError() + self.api.connect.side_effect = lost + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('resume', Resume=dict(q, ReconcileOnly=True))['ErrorCode'], 'unconfirmed') + self.assertEqual(self.invoke('compute_kill', Compute=q['Target'])['ErrorCode'], 'unconfirmed') + self.api.connect.assert_called_once() + self.api.kill.assert_not_called() + + def test_lost_core_resume_reply_adopts_saved_target_without_connect(self): + current = self.prepare() + _, paused = self.pause(current) + q = self.resume_request(paused['State']['Retained']) + first = self.invoke('resume', Resume=q) + self.assertEqual(first['ErrorCode'], '') + second = self.invoke('resume', Resume=dict(q, ReconcileOnly=True)) + self.assertEqual(first, second) + self.api.connect.assert_called_once() + + def test_foreign_retained_and_incarnation_cannot_mutate(self): + current = self.prepare() + _, paused = self.pause(current) + retained = paused['State']['Retained'] + q = self.resume_request(dict(retained, SourceID='foreign-id')) + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'ownership') + self.assertEqual(self.invoke('delete_retained', Retained=dict(retained, ID='foreign'))['ErrorCode'], 'ownership') + self.api.connect.assert_not_called() + self.api.kill.assert_not_called() + + def test_archive_recovers_saved_connect_reply_before_cleanup(self): + from e2b.exceptions import SandboxNotFoundException + current = self.prepare() + _, paused = self.pause(current) + retained = paused['State']['Retained'] + q = self.resume_request(retained) + def connected_then_observation_lost(*args, **kwargs): + self.cloud.state = 'running' + self.api.get_info.side_effect = TimeoutError() + return self.cloud + self.api.connect.side_effect = connected_then_observation_lost + self.assertEqual(self.invoke('resume', Resume=q)['ErrorCode'], 'unconfirmed') + self.assertTrue(self.record()['suspension']['connected']) + self.assertEqual(self.record()['suspension']['phase'], 'resume_pending') + self.api.get_info.side_effect = None + # The old incarnation remains fenced until exact target cleanup settles. + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], 'ownership') + def killed(*args, **kwargs): + self.api.get_info.side_effect = SandboxNotFoundException('gone') + self.api.kill.side_effect = killed + self.assertEqual(self.invoke('compute_kill', Compute=q['Target'])['ErrorCode'], '') + self.assertEqual(self.invoke('compute_kill', Compute=current)['ErrorCode'], '') + self.assertEqual(self.invoke('delete_retained', Retained=retained)['ErrorCode'], '') + self.api.connect.assert_called_once() + self.api.kill.assert_called_once() + self.assertEqual(self.record()['status'], 'killed') diff --git a/services/core/tools/e2b-provider/testdata/contract.json b/services/core/tools/e2b-provider/testdata/contract.json index 0efaa314d..bd22db5b3 100644 --- a/services/core/tools/e2b-provider/testdata/contract.json +++ b/services/core/tools/e2b-provider/testdata/contract.json @@ -21,46 +21,56 @@ {"kind":"managed","name":"restricted","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":["example.com"],"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"restricted","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, {"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, {"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","NetworkAccess":"enabled","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, -{"kind":"request","name":"command","payload":{"Version":2,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"create","payload":{"Version":2,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"inspect","payload":{"Version":2,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"kill","payload":{"Version":2,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_builds","payload":{"Version":2,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_templates","payload":{"Version":2,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe","payload":{"Version":2,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"renew","payload":{"Version":2,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"validate_deployment","payload":{"Version":2,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-0","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-32","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-33","payload":{"Version":2,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","Harness":"codex","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, +{"kind":"request","name":"command","payload":{"Version":4,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_command","payload":{"Version":4,"Operation":"compute_command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Command":{"Args":["true"],"Directory":"","Stdin":null},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_info","payload":{"Version":4,"Operation":"compute_info","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_kill","payload":{"Version":4,"Operation":"compute_kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute_renew","payload":{"Version":4,"Operation":"compute_renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"create","payload":{"Version":4,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"delete_retained","payload":{"Version":4,"Operation":"delete_retained","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Retained":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"inspect","payload":{"Version":4,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill","payload":{"Version":4,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_builds","payload":{"Version":4,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_templates","payload":{"Version":4,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"observe","payload":{"Version":4,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"renew","payload":{"Version":4,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume","payload":{"Version":4,"Operation":"resume","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Resume":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"66666666-6666-4666-8666-666666666666","Retained":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"},"Target":{"Generation":1,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":{"Reference":"33333333-3333-4333-8333-333333333333","ID":"66666666-6666-4666-8666-666666666666","Data":"opaque","OperationID":"66666666-6666-4666-8666-666666666666","SourceGeneration":0,"SourceName":"33333333-3333-4333-8333-333333333333","SourceID":"native-fixture"}},"ReconcileOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume-workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"resume","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Resume":{"OperationID":"66666666-6666-4666-8666-666666666666","ReconcileOnly":false,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Retained":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"},"Target":{"Generation":1,"ID":"native-fixture","Name":"33333333-3333-4333-8333-333333333333","RestoredFrom":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"}},"Workspace":null},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":true}, +{"kind":"request","name":"resume-workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"resume","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Resume":{"OperationID":"66666666-6666-4666-8666-666666666666","ReconcileOnly":false,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"Retained":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"},"Target":{"Generation":1,"ID":"native-fixture","Name":"33333333-3333-4333-8333-333333333333","RestoredFrom":{"Data":"opaque","ID":"66666666-6666-4666-8666-666666666666","OperationID":"66666666-6666-4666-8666-666666666666","Reference":"33333333-3333-4333-8333-333333333333","SourceGeneration":0,"SourceID":"native-fixture","SourceName":"33333333-3333-4333-8333-333333333333"}},"Workspace":{}},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"resume_compute","payload":{"Version":4,"Operation":"resume_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Compute":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"suspend","payload":{"Version":4,"Operation":"suspend","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Suspend":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"66666666-6666-4666-8666-666666666666","Source":{"Generation":0,"Name":"33333333-3333-4333-8333-333333333333","ID":"native-fixture","RestoredFrom":null},"Retained":null,"ReconcileOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"validate_deployment","payload":{"Version":4,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-0","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-32","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-33","payload":{"Version":4,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"Harness":"codex","TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","harness":"codex"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, {"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":true},"valid":false}, {"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":null},"valid":false}, {"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":"1"},"valid":false}, -{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":3},"valid":false}, -{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":true}, -{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":2},"valid":false}, -{"kind":"response","name":"error-","payload":{"Version":2,"ErrorCode":""},"valid":true}, -{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":2,"ErrorCode":"command_unconfirmed"},"valid":true}, -{"kind":"response","name":"error-exists","payload":{"Version":2,"ErrorCode":"exists"},"valid":true}, -{"kind":"response","name":"error-invalid","payload":{"Version":2,"ErrorCode":"invalid"},"valid":true}, -{"kind":"response","name":"error-not_found","payload":{"Version":2,"ErrorCode":"not_found"},"valid":true}, -{"kind":"response","name":"error-ownership","payload":{"Version":2,"ErrorCode":"ownership"},"valid":true}, -{"kind":"response","name":"error-team_mismatch","payload":{"Version":2,"ErrorCode":"team_mismatch"},"valid":true}, -{"kind":"response","name":"error-template_invalid","payload":{"Version":2,"ErrorCode":"template_invalid"},"valid":true}, -{"kind":"response","name":"error-unauthorized","payload":{"Version":2,"ErrorCode":"unauthorized"},"valid":true}, -{"kind":"response","name":"error-unconfirmed","payload":{"Version":2,"ErrorCode":"unconfirmed"},"valid":true}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":2},"valid":false}, -{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":2},"valid":false}, -{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":2},"valid":false}, +{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":5},"valid":false}, +{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":true}, +{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Harness":"codex","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","harness":"codex","version":2},"Version":4},"valid":false}, +{"kind":"response","name":"error-","payload":{"Version":4,"ErrorCode":""},"valid":true}, +{"kind":"response","name":"error-command_unconfirmed","payload":{"Version":4,"ErrorCode":"command_unconfirmed"},"valid":true}, +{"kind":"response","name":"error-exists","payload":{"Version":4,"ErrorCode":"exists"},"valid":true}, +{"kind":"response","name":"error-invalid","payload":{"Version":4,"ErrorCode":"invalid"},"valid":true}, +{"kind":"response","name":"error-not_found","payload":{"Version":4,"ErrorCode":"not_found"},"valid":true}, +{"kind":"response","name":"error-ownership","payload":{"Version":4,"ErrorCode":"ownership"},"valid":true}, +{"kind":"response","name":"error-team_mismatch","payload":{"Version":4,"ErrorCode":"team_mismatch"},"valid":true}, +{"kind":"response","name":"error-template_invalid","payload":{"Version":4,"ErrorCode":"template_invalid"},"valid":true}, +{"kind":"response","name":"error-unauthorized","payload":{"Version":4,"ErrorCode":"unauthorized"},"valid":true}, +{"kind":"response","name":"error-unconfirmed","payload":{"Version":4,"ErrorCode":"unconfirmed"},"valid":true}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":"unknown","Version":4},"valid":false}, +{"kind":"response","name":"invalid-ErrorCode","payload":{"ErrorCode":1,"Version":4},"valid":false}, +{"kind":"response","name":"invalid-Extra","payload":{"ErrorCode":"","Extra":true,"Version":4},"valid":false}, {"kind":"response","name":"invalid-Version","payload":{"ErrorCode":"","Version":true},"valid":false} ] diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index 58e190c59..a487b26f6 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -1,6 +1,6 @@ # microsandbox Sandbox Provider helper -microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 5; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. +microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 5; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the suspension operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. [Add a Sandbox Provider](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources, Runtime release and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy. @@ -29,14 +29,14 @@ A helper response carries `CreateSettled` with a configuration rejection only af Core persists operation IDs, source and target generations, exact identities and snapshot evidence before it depends on them. `Initial` and `NewCompute` only construct references and allocate nothing. - **Suspend** pauses the exact VM, captures a full snapshot under the persisted operation's derived group and member, verifies the complete checkpoint closure, then force-stops the source. Pausing alone does not release memory. A completed matching artifact is inspected instead of captured again. A full snapshot records a resource proof only after the source's limits match. -- **KillCompute** checks the precise incarnation before it stops the VM and removes its writable disks. Core calls it after it has persisted the verified snapshot, even when Suspend already stopped the source, so no chain of old writable disks grows across suspension cycles. +- **KillCompute** checks the precise incarnation before it stops the VM and removes its writable disks. Suspend completes native source cleanup under the allocation lock before reporting resource release, including recovery of a captured artifact. Core uses KillCompute for allocation cleanup. - **Restore** verifies the exact artifact and creates the precommitted target name. An existing target is adopted only when its immutable ID, if known, and its persisted `snapshot_parent` agree. Upstream restore defaults to public networking, so restore passes the same explicit host policy as creation, and no undeclared host resource or mount is inherited. - Native restore leaves the managed root size unset because the target inherits the verified full snapshot. The helper accepts that only with a matching snapshot resource proof and the exact source and target identities, and it checks the target's CPU, memory and Environment disk before keeping the inherited proof. A missing root size never counts as unlimited capacity, and retained state is never resized. - Fresh restore and retry share one completion: verify the original artifact and resource proof, inspect the running target's resources and ancestry, persist its missing derived resource-proof label, then strictly reread the same native ID ([`restore_completion.go`](restore_completion.go)). A conflicting proof is an error. Native restore does not copy the source's ownership labels; ancestry supplies that evidence. There is no ordinary Start, replacement, disk-only restore or cold boot. - **ResumeCompute** thaws the same resident source after an aborted suspension. The pinned SDK handle method is name-based; the allocation lock and ID checks before and after the call fence every managed replacement. Manual lifecycle changes in the managed namespace are unsupported. -- **DeleteSnapshot** accepts only the derived operation selector and the matching full artifact identity, never an arbitrary path. Core owns retention, consumed snapshot generations and cleanup order. A checkpoint never rolls back work admitted after its first restore. +- **DeleteRetained** accepts only the derived operation selector and the matching full artifact identity, never an arbitrary path. Core owns retention, consumed snapshot generations and cleanup order. A checkpoint never rolls back work admitted after its first restore. -After a lost response Core uses `ObserveOnly`. It never starts a capture or restore, and observing a suspend operation never kills its source. Observation checks artifact integrity and source ownership independently of resource checks, so resource drift cannot hide a retained artifact from cleanup; Core can persist recovered snapshot evidence before KillCompute. If the artifact is absent but the exact source is still running or paused with a settled bootstrap, observation returns the source without a snapshot and Core can abort the suspension; thawing and further execution still require the resource checks. For an interrupted restore, `ObserveOnly` may finish the missing resource proof on the exact target but never restarts a stopped target, changes resources or restores again. Missing state never authorizes a replay. +After a lost response Core uses `ReconcileOnly`. The adapter observes the original capture or restore and never starts it again. When a complete matching snapshot exists, reconciliation finishes exact source cleanup before reporting settled suspension and released resources. Artifact integrity and source ownership are checked independently of resource qualification, so resource drift cannot prevent owned cleanup. If capture is settled without an artifact and the exact source is still running or paused with a settled bootstrap, the adapter reports that outcome so Core can abort suspension; thawing and further execution still require resource checks. For an interrupted restore, reconciliation may finish the missing resource proof on the exact target but never restarts a stopped target, changes resources or restores again. Missing state never authorizes a replay. GetCompute, commands, cleanup and the next suspension verify restored provenance from the persisted VM configuration after the consumed artifact is deleted. @@ -65,4 +65,4 @@ The helper returns only the native observation time, exact uptime, cumulative vC `make check-microsandbox-provider`, part of `make check`, runs the pure-Go adapter tests everywhere and this module's tests on Linux; other hosts print an explicit skip for the Linux-only module. -The native helper wire uses version 5 and carries the Session-selected Harness into Runtime bootstrap version 2. Upgrade the helper with its node and Core. +The native helper wire uses version 5 and carries the Session-selected Harness into Runtime bootstrap version 2. Upgrade the helper with its node and Core. The Go adapter wraps native snapshot identity in the shared opaque retained-state handle and completes exact-source cleanup through the existing ownership-checked helper operations. Reconciliation never repeats snapshot capture. SuspendSettled is reported only after these serialized operations complete; captured-artifact cleanup does not require execution resource qualification. diff --git a/services/core/tools/microsandbox-provider/bootstrap.go b/services/core/tools/microsandbox-provider/bootstrap.go index d1753e4f4..e6779414f 100644 --- a/services/core/tools/microsandbox-provider/bootstrap.go +++ b/services/core/tools/microsandbox-provider/bootstrap.go @@ -8,6 +8,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentnetwork" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" wire "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" sdk "github.com/superradcompany/microsandbox/sdk/go" @@ -18,7 +19,7 @@ import ( const bootstrapScript = ` import ctypes,json,os,stat,subprocess,sys b=json.load(sys.stdin) -for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages','/run/oac']: +for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages',os.path.dirname(os.environ['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'])]: os.makedirs(p,mode=0o700,exist_ok=True) if not stat.S_ISDIR(os.lstat(p).st_mode): raise RuntimeError('invalid bootstrap directory') os.chmod(p,0o700);os.chown(p,1000,1000) @@ -71,7 +72,7 @@ func (b backend) create(ctx context.Context) (wire.Response, error) { "HOME": "/home/runtime", "OAC_RUNTIME_HOME": "/home/runtime/.oac", "OAC_RUNTIME_ENVIRONMENT_ID": bootstrap.EnvironmentID, "OAC_RUNTIME_SESSION_ID": bootstrap.SessionID, "OAC_RUNTIME_NETWORK_ACCESS": policy.Access, "OAC_RUNTIME_ALLOWED_DOMAINS": string(domains), - "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": "/run/oac/daemon-suspend.json", + "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": runtimebootstrap.SuspendControlFile, })) if e != nil { return wire.Response{}, e From a166c8223bacd9e65949b42718abf50d46f15820 Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 11 Oct 2026 10:25:23 +0800 Subject: [PATCH 4/4] Fix retained checkpoint demand and node accounting queries --- .../internal/db/queries/runtime_nodes.sql | 2 +- .../queries/runtime_suspension_pressure.sql | 4 +- .../internal/db/sqlc/runtime_nodes.sql.go | 2 +- .../sqlc/runtime_suspension_pressure.sql.go | 4 +- .../runtime_retained_queries_test.go | 84 +++++++++++++++++++ .../deploymentpg/checkpoint_transfer_test.go | 2 +- .../deploymentpg/suspension_pressure_test.go | 2 +- 7 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 services/core/internal/execution/runtime_retained_queries_test.go diff --git a/services/core/internal/db/queries/runtime_nodes.sql b/services/core/internal/db/queries/runtime_nodes.sql index 6a24dbb20..44f68e596 100644 --- a/services/core/internal/db/queries/runtime_nodes.sql +++ b/services/core/internal/db/queries/runtime_nodes.sql @@ -19,7 +19,7 @@ SELECT n.*, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id AND a.state<>'released'))::bigint AS reserved, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='cleanup_pending')::bigint AS cleanup_pending, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='running' AND a.compute_phase IN('running','disabled'))::bigint AS running, - (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'snapshot' IS NOT NULL AND a.compute_state->'snapshot'<>'null'::jsonb)::bigint AS snapshots + (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'retained' IS NOT NULL AND a.compute_state->'retained'<>'null'::jsonb)::bigint AS snapshots FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND (sqlc.narg(node_id)::uuid IS NULL OR n.id=sqlc.narg(node_id)::uuid) ORDER BY n.id; diff --git a/services/core/internal/db/queries/runtime_suspension_pressure.sql b/services/core/internal/db/queries/runtime_suspension_pressure.sql index 580c00813..4c47d4caa 100644 --- a/services/core/internal/db/queries/runtime_suspension_pressure.sql +++ b/services/core/internal/db/queries/runtime_suspension_pressure.sql @@ -6,9 +6,9 @@ WHERE a.node_id IS NOT NULL AND a.state <> 'released' OR (a.compute_retained_until <= clock_timestamp() AND a.compute_phase <> 'disabled')); -- name: ListWaitingCheckpointRestores :many -SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'snapshot'->'Compatibility')::jsonb AS checkpoint +SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'retained'->'Compatibility')::jsonb AS checkpoint FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id WHERE a.node_id IS NOT NULL AND a.state='running' AND a.compute_phase='suspended' AND a.compute_retained_until>clock_timestamp() AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') - AND a.compute_state->'snapshot'->'Compatibility' IS NOT NULL + AND a.compute_state->'retained'->'Compatibility' IS NOT NULL AND (a.compute_wake_requested OR EXISTS(SELECT 1 FROM environment_input_reservations r WHERE r.session_id=s.id AND r.state='pending' AND r.deadline>clock_timestamp())); diff --git a/services/core/internal/db/sqlc/runtime_nodes.sql.go b/services/core/internal/db/sqlc/runtime_nodes.sql.go index e28353bcf..d664a022d 100644 --- a/services/core/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_nodes.sql.go @@ -388,7 +388,7 @@ SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha2 (SELECT count(*) FROM runtime_placements p WHERE p.node_id=n.id AND p.released_at IS NULL AND NOT EXISTS(SELECT 1 FROM runtime_allocations a WHERE a.environment_id=p.environment_id AND a.state<>'released'))::bigint AS reserved, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='cleanup_pending')::bigint AS cleanup_pending, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='running' AND a.compute_phase IN('running','disabled'))::bigint AS running, - (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'snapshot' IS NOT NULL AND a.compute_state->'snapshot'<>'null'::jsonb)::bigint AS snapshots + (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'retained' IS NOT NULL AND a.compute_state->'retained'<>'null'::jsonb)::bigint AS snapshots FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id AND ($1::uuid IS NULL OR n.id=$1::uuid) ORDER BY n.id diff --git a/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go index e192c879e..86b051015 100644 --- a/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension_pressure.sql.go @@ -40,11 +40,11 @@ func (q *Queries) ListRuntimeSuspensionInFlightNodes(ctx context.Context) ([]pgt } const listWaitingCheckpointRestores = `-- name: ListWaitingCheckpointRestores :many -SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'snapshot'->'Compatibility')::jsonb AS checkpoint +SELECT DISTINCT a.node_id, a.deployment_generation, (a.compute_state->'retained'->'Compatibility')::jsonb AS checkpoint FROM runtime_allocations a JOIN environments e ON e.id=a.environment_id JOIN sessions s ON s.id=e.session_id WHERE a.node_id IS NOT NULL AND a.state='running' AND a.compute_phase='suspended' AND a.compute_retained_until>clock_timestamp() AND s.deleted_at IS NULL AND e.status NOT IN ('failed','expired') - AND a.compute_state->'snapshot'->'Compatibility' IS NOT NULL + AND a.compute_state->'retained'->'Compatibility' IS NOT NULL AND (a.compute_wake_requested OR EXISTS(SELECT 1 FROM environment_input_reservations r WHERE r.session_id=s.id AND r.state='pending' AND r.deadline>clock_timestamp())) ` diff --git a/services/core/internal/execution/runtime_retained_queries_test.go b/services/core/internal/execution/runtime_retained_queries_test.go new file mode 100644 index 000000000..e2430706b --- /dev/null +++ b/services/core/internal/execution/runtime_retained_queries_test.go @@ -0,0 +1,84 @@ +package execution + +import ( + "encoding/json" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/db/sqlc" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +// Exercise the SQL projections with the same serialized envelope Core persists. +// Adapter-private data deliberately contains legacy-looking keys: queries must +// read only the shared retained envelope, never the opaque payload. +func TestRuntimeRetainedStateFeedsCheckpointDemandAndNodeCounts(t *testing.T) { + f := newWorkspaceSettlementFixture(t, &retentionProvider{}) + r, session := f.lifecycle, f.session + owner, err := r.provision(t.Context(), session.TenantID, session.Environment.ID, r.config.InstallationID) + if err != nil { + t.Fatal(err) + } + compatibility := sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"} + source := sandbox.Compute{Generation: 1, Name: "source", ID: "native-source"} + retained := sandbox.RetainedState{Reference: "fixture/retained", ID: "native-retained", Data: `{"snapshot":{"Compatibility":{"artifact_domain":"private","execution_class":"private"}}}`, OperationID: "suspend-operation", SourceGeneration: source.Generation, SourceName: source.Name, SourceID: source.ID, Compatibility: compatibility} + for _, tc := range []struct { + name string + wake, expired, released, noCompatibility, legacy bool + demand, count int + }{ + {name: "waiting", wake: true, demand: 1, count: 1}, + {name: "idle", count: 1}, + {name: "expired", wake: true, expired: true, count: 1}, + {name: "released", wake: true, released: true}, + {name: "no_compatibility", wake: true, noCompatibility: true, count: 1}, + {name: "legacy_shape", wake: true, legacy: true}, + } { + t.Run(tc.name, func(t *testing.T) { + state := runtimeCompute{Version: sandbox.SuspensionStateVersion, Current: source, Retained: &retained, SuspendID: retained.OperationID} + copied := retained + if tc.noCompatibility { + copied.Compatibility = sandbox.CheckpointCompatibility{} + state.Retained = &copied + } + raw, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if tc.legacy { + var envelope map[string]json.RawMessage + if err := json.Unmarshal(raw, &envelope); err != nil { + t.Fatal(err) + } + envelope["snapshot"] = envelope["retained"] + delete(envelope, "retained") + raw, err = json.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + } + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET state=CASE WHEN $3 THEN 'released' ELSE 'running' END,released_at=CASE WHEN $3 THEN clock_timestamp() END,compute_phase='suspended',compute_state=$2,compute_retained_until=clock_timestamp()+CASE WHEN $4 THEN interval '-1 hour' ELSE interval '1 hour' END,compute_wake_requested=$5 WHERE id=$1`, owner.ID, raw, tc.released, tc.expired, tc.wake); err != nil { + t.Fatal(err) + } + q := sqlc.New(f.pool) + demands, err := q.ListWaitingCheckpointRestores(t.Context()) + if err != nil || len(demands) != tc.demand { + t.Fatalf("checkpoint demand=%d want=%d: %v", len(demands), tc.demand, err) + } + if len(demands) == 1 { + var got sandbox.CheckpointCompatibility + if err := json.Unmarshal(demands[0].Checkpoint, &got); err != nil || got != compatibility { + t.Fatalf("shared compatibility lost: %+v %v", got, err) + } + } + nodes, err := q.ListRuntimeNodes(t.Context(), pgtype.UUID{Bytes: uuid.MustParse(owner.NodeID), Valid: true}) + if err != nil || len(nodes) != 1 { + t.Fatalf("node projection: %d %v", len(nodes), err) + } + if nodes[0].Snapshots != int64(tc.count) { + t.Fatalf("retained node count=%d want=%d", nodes[0].Snapshots, tc.count) + } + }) + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go b/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go index 919f2dac6..63f150c89 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go @@ -34,7 +34,7 @@ func readyCheckpointNode(t *testing.T, f fixture, node deployment.Enrollment, vi func suspendedCheckpointOwner(t *testing.T, f fixture, changes *deployment.ExecutionOperations, installation string, node deployment.Enrollment, view deployment.View) deployment.Allocation { t.Helper() owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) - if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"snapshot":{"ID":"snapshot","Compatibility":{"artifact_domain":"fixture-archive","execution_class":"fixture-machine"}}}',compute_retained_until=clock_timestamp()+interval '24 hours',compute_wake_requested=true WHERE id=$1`, owner.ID); err != nil { + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"protocol_version":"1","retained":{"ID":"snapshot","Compatibility":{"artifact_domain":"fixture-archive","execution_class":"fixture-machine"}}}',compute_retained_until=clock_timestamp()+interval '24 hours',compute_wake_requested=true WHERE id=$1`, owner.ID); err != nil { t.Fatal(err) } owner, err := f.adapter.EnvironmentAllocation(t.Context(), owner.Key()) diff --git a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go index 77fed6a34..67acb835e 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/suspension_pressure_test.go @@ -132,7 +132,7 @@ func TestPressureSuspensionServesRestoreOnItsOriginalNode(t *testing.T) { node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) f.connect(t, node.NodeID) waiting := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation) - if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"snapshot":{"Compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()+interval '1 hour',compute_wake_requested=true WHERE id=$1`, waiting.ID); err != nil { + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_phase='suspended',compute_state='{"protocol_version":"1","retained":{"Compatibility":{"artifact_domain":"fixture-store","execution_class":"fixture-runtime"}}}',compute_retained_until=clock_timestamp()+interval '1 hour',compute_wake_requested=true WHERE id=$1`, waiting.ID); err != nil { t.Fatal(err) } owner := runningPressureOwner(t, f, changes, installation, node.NodeID, view.Generation)