diff --git a/api/harness.go b/api/harness.go index 163eb26d..c75dd93f 100644 --- a/api/harness.go +++ b/api/harness.go @@ -18,6 +18,26 @@ type Harness interface { Run(ctx context.Context, s *Start, sink EventSink) error } +// HarnessRunPreparer optionally prepares the transport or other resources for one Run before the +// host records its execution. PrepareRun must not invoke the harness, send Start, or perform any +// model or tool effect. An error therefore proves that no execution was delegated. +// +// On error an implementation must release anything it acquired and return no prepared run. A +// successful call transfers one prepared run to the caller. The host either calls Run exactly once +// or closes it without running it. +type HarnessRunPreparer interface { + PrepareRun(ctx context.Context) (PreparedHarnessRun, error) +} + +// PreparedHarnessRun is one prepared Harness.Run. Run may be called exactly once. Close is +// idempotent and may race with Run: if Close wins, Run must fail without invoking the harness; if +// Run wins, Close cancels the active run. Close is also safe after Run returns. Implementations own +// cleanup and cannot report a second cleanup error. +type PreparedHarnessRun interface { + Run(ctx context.Context, s *Start, sink EventSink) error + Close() +} + // Descriptor is a harness's static contract. type Descriptor struct { ID string diff --git a/changelog/unreleased/harness-connect-preparation.md b/changelog/unreleased/harness-connect-preparation.md new file mode 100644 index 00000000..6b9a93b3 --- /dev/null +++ b/changelog/unreleased/harness-connect-preparation.md @@ -0,0 +1,10 @@ +--- +category: Fixed +--- + +- A new `Exec` over `Harness.Connect` now opens its actual stream before recording + `EXECUTION_START`. If Connect cannot be opened, no execution records are appended and retrying the + same session starts its first turn normally. The host still sends `Start` only after the start and + every input are durable; cancellation after successful preparation and send or receive failures + remain uncertain and keep any recorded attempt in later history. Go harness adapters can opt into + the same boundary with `api.HarnessRunPreparer` (`EXE-45`). diff --git a/changelog/unreleased/harness-run-preparer.md b/changelog/unreleased/harness-run-preparer.md new file mode 100644 index 00000000..0d47d340 --- /dev/null +++ b/changelog/unreleased/harness-run-preparer.md @@ -0,0 +1,9 @@ +--- +category: Added +--- + +- Go harness adapters can implement `api.HarnessRunPreparer` to acquire the transport for one + execution before it is recorded, without invoking the harness or sending `Start`. The returned + `api.PreparedHarnessRun` is used at most once or closed; a concurrent close either prevents Run + or cancels it. `harnesswire.ClientHarness` implements the extension for its `Harness.Connect` + stream (`EXE-45`). diff --git a/changelog/unreleased/harnesswire-terminal-compat.md b/changelog/unreleased/harnesswire-terminal-compat.md new file mode 100644 index 00000000..7ef0813d --- /dev/null +++ b/changelog/unreleased/harnesswire-terminal-compat.md @@ -0,0 +1,12 @@ +--- +category: Compatibility +--- + +- `Harness.Connect` now fails an execution for four wire shapes the host previously accepted: clean + closure without `EVENT_END`; harness-sent `EVENT_INPUT`, `EVENT_LIFECYCLE`, `EVENT_ERROR`, or an + unspecified/unknown kind; any frame after `EVENT_END`; and a harness-owned non-OK final RPC status + after `EVENT_END{COMPLETED}` while the caller remains live. Caller cancellation or deadline during + the bounded post-END telemetry flush does not replace a completed harness result (`WIRE-12`). +- A new `Exec` whose Connect preparation fails with `UNAVAILABLE` or `DEADLINE_EXCEEDED` while the + caller remains live now returns `UNAVAILABLE` with `agentsessions/HARNESS_UNAVAILABLE` and fixed + host text. Before this change it returned `INTERNAL` with the raw Connect diagnostic. diff --git a/controller/controller.go b/controller/controller.go index 7b04be11..0c1e46db 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -43,6 +43,11 @@ var ErrInvalidExecutionLog = errors.New("controller: invalid execution log") // The original cause is retained for callers to distinguish cancellation from endpoint failure. var ErrHarnessDescribeFailed = errors.New("controller: describe harness failed") +// ErrHarnessPrepareFailed identifies failure to prepare a harness run before its execution is +// recorded. The original cause is retained for callers to distinguish cancellation from endpoint +// failure. No Start send was attempted. +var ErrHarnessPrepareFailed = errors.New("controller: prepare harness run failed") + // ErrHarnessMismatch rejects replay/recovery with a different resolved harness name. var ErrHarnessMismatch = errors.New("controller: harness does not match recorded invocation") @@ -339,6 +344,22 @@ func (c *Controller) Exec(ctx context.Context, har api.Harness, inputs []api.Mes } } + run := har.Run + if preparer, ok := har.(api.HarnessRunPreparer); ok { + prepared, prepareErr := preparer.PrepareRun(ctx) + if prepareErr != nil { + if prepared != nil { + prepared.Close() + } + return fmt.Errorf("%w: %w", ErrHarnessPrepareFailed, prepareErr) + } + if prepared == nil { + return fmt.Errorf("%w: preparer returned nil", ErrHarnessPrepareFailed) + } + defer prepared.Close() + run = prepared.Run + } + // Record every invocation before inputs so recovery can detect partially committed turns. config := bytes.Clone(c.startConfig) last := expectedLastSeq @@ -388,7 +409,7 @@ func (c *Controller) Exec(ctx context.Context, har api.Harness, inputs []api.Mes ResumeFromSeq: c.startResumeFromSeq, } sink := &liveSink{c: c, executionID: executionID, modelDefault: model, modelPolicy: c.invocationPolicy(model), guard: &sinkGuard{}, desc: &desc, har: har, calls: make(map[string]bool), runCtx: runCtx} - runErr := har.Run(runCtx, start, sink) + runErr := run(runCtx, start, sink) stop := sink.guard.close() if stop != nil { if runErr == nil || !errors.Is(stop, api.ErrApprovalParked) { diff --git a/controller/prepare_run_test.go b/controller/prepare_run_test.go new file mode 100644 index 00000000..88767182 --- /dev/null +++ b/controller/prepare_run_test.go @@ -0,0 +1,186 @@ +package controller_test + +import ( + "context" + "errors" + "testing" + + "github.com/agent-primitives/sessions/api" + "github.com/agent-primitives/sessions/controller" + "github.com/agent-primitives/sessions/eventlog" +) + +type preparedRunHarness struct { + executionConfigHarness + prepareErr error + prepareValueOnErr bool + prepare, runs, closes int + beforeRun func() + runErr error +} + +func (h *preparedRunHarness) PrepareRun(context.Context) (api.PreparedHarnessRun, error) { + h.prepare++ + if h.prepareErr != nil { + if h.prepareValueOnErr { + return &testPreparedRun{h: h}, h.prepareErr + } + return nil, h.prepareErr + } + return &testPreparedRun{h: h}, nil +} + +type testPreparedRun struct { + h *preparedRunHarness + used bool + closed bool +} + +func (r *testPreparedRun) Run(ctx context.Context, start *api.Start, sink api.EventSink) error { + if r.used { + return errors.New("prepared run reused") + } + r.used = true + r.h.runs++ + if r.h.beforeRun != nil { + r.h.beforeRun() + } + if r.h.runErr != nil { + return r.h.runErr + } + return r.h.Run(ctx, start, sink) +} + +func (r *testPreparedRun) Close() { + if !r.closed { + r.closed = true + r.h.closes++ + } +} + +func TestExecPreparesHarnessBeforeFirstAppend(t *testing.T) { + log := eventlog.AsStore(eventlog.New()) + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + h := &preparedRunHarness{} + h.beforeRun = func() { + recs, err := log.Read(1) + if err != nil { + t.Fatal(err) + } + if len(recs) != 2 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventInput { + t.Fatalf("prepared Run saw records %+v, want committed start and input", recs) + } + } + if err := c.Exec(t.Context(), h, []api.Message{msg("hello")}, 0); err != nil { + t.Fatal(err) + } + if h.prepare != 1 || h.runs != 1 || h.closes != 1 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/1/1", h.prepare, h.runs, h.closes) + } +} + +func TestExecPrepareFailureLeavesJournalUntouched(t *testing.T) { + log := eventlog.AsStore(eventlog.New()) + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + cause := errors.New("connect failed before Start") + h := &preparedRunHarness{prepareErr: cause} + err = c.Exec(t.Context(), h, []api.Message{msg("hello")}, 0) + if !errors.Is(err, controller.ErrHarnessPrepareFailed) || !errors.Is(err, cause) { + t.Fatalf("Exec=%v, want preparation failure retaining %v", err, cause) + } + if head, headErr := log.Head(); headErr != nil || head != 0 { + t.Fatalf("head=%d err=%v, want untouched journal", head, headErr) + } + if h.prepare != 1 || h.runs != 0 || h.closes != 0 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/0/0", h.prepare, h.runs, h.closes) + } +} + +func TestExecDefensivelyClosesPreparedValueReturnedWithError(t *testing.T) { + log := eventlog.AsStore(eventlog.New()) + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + cause := errors.New("prepare failed after allocation") + h := &preparedRunHarness{prepareErr: cause, prepareValueOnErr: true} + err = c.Exec(t.Context(), h, nil, 0) + if !errors.Is(err, controller.ErrHarnessPrepareFailed) || !errors.Is(err, cause) { + t.Fatalf("Exec=%v, want preparation failure retaining %v", err, cause) + } + if h.prepare != 1 || h.runs != 0 || h.closes != 1 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/0/1", h.prepare, h.runs, h.closes) + } +} + +func TestExecClosesPreparedRunWhenAppendFails(t *testing.T) { + cause := errors.New("append failed") + log := rejectAppendStore{Store: eventlog.AsStore(eventlog.New()), err: cause} + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + h := &preparedRunHarness{} + err = c.Exec(t.Context(), h, []api.Message{msg("hello")}, 0) + if !errors.Is(err, cause) || errors.Is(err, controller.ErrHarnessPrepareFailed) { + t.Fatalf("Exec=%v, want append failure %v", err, cause) + } + if h.prepare != 1 || h.runs != 0 || h.closes != 1 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/0/1", h.prepare, h.runs, h.closes) + } +} + +func TestExecClosesPreparedRunWhenInputAppendFails(t *testing.T) { + cause := errors.New("input append failed") + base := eventlog.AsStore(eventlog.New()) + log := &failInputStore{Store: base, failAt: 2, err: cause} + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + h := &preparedRunHarness{} + err = c.Exec(t.Context(), h, []api.Message{msg("one"), msg("two")}, 0) + if !errors.Is(err, cause) || errors.Is(err, controller.ErrHarnessPrepareFailed) { + t.Fatalf("Exec=%v, want input append failure %v", err, cause) + } + if h.prepare != 1 || h.runs != 0 || h.closes != 1 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/0/1", h.prepare, h.runs, h.closes) + } + recs, readErr := base.Read(1) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 2 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventInput { + t.Fatalf("records=%+v, want existing incomplete invocation", recs) + } +} + +func TestExecPreparedRunFailureRemainsDurable(t *testing.T) { + log := eventlog.AsStore(eventlog.New()) + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + cause := errors.New("Start send outcome unknown") + h := &preparedRunHarness{runErr: cause} + err = c.Exec(t.Context(), h, []api.Message{msg("hello")}, 0) + if !errors.Is(err, cause) || errors.Is(err, controller.ErrHarnessPrepareFailed) { + t.Fatalf("Exec=%v, want Run failure %v without preparation marker", err, cause) + } + recs, readErr := log.Read(1) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 3 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventInput || recs[2].Event.Kind != api.EventError { + t.Fatalf("records=%+v, want durable failed execution", recs) + } + if h.prepare != 1 || h.runs != 1 || h.closes != 1 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/1/1", h.prepare, h.runs, h.closes) + } +} diff --git a/docs/harness-authoring.md b/docs/harness-authoring.md index 92834c5f..d25d9ac5 100644 --- a/docs/harness-authoring.md +++ b/docs/harness-authoring.md @@ -135,6 +135,17 @@ empty/`echo` fallback. Old sessions/journals stay unchanged; see The host hands you a `Start` and an `EventSink`. +An out-of-process adapter can implement the optional Go interface `api.HarnessRunPreparer` to open +the actual transport for one `Run` before the host records a new execution. Preparation must not +invoke the harness or send `Start`; its only positive guarantee is that a preparation error happened +before delegation. The host closes an unused preparation and calls a successful prepared run at most +once, after `EXECUTION_START` and every `INPUT` are durable. `harnesswire.ClientHarness` implements +this for `Harness.Connect`. A wrapper around it must forward the optional interface to keep that +pre-journal boundary. A failed `Send(Start)` or later receive is not proof of non-delivery and stays +a durable failed execution (`EXE-45`). In particular, successfully opening a client stream does not +prove that a router or harness will accept `Start`; a refusal surfaced by Send or Recv remains +uncertain and is not filtered from later history. + ```go type Start struct { diff --git a/harnesswire/harnesswire.go b/harnesswire/harnesswire.go index 93583ab0..895ed229 100644 --- a/harnesswire/harnesswire.go +++ b/harnesswire/harnesswire.go @@ -21,6 +21,8 @@ import ( "errors" "fmt" "io" + "sync" + "sync/atomic" "time" "go.opentelemetry.io/otel/attribute" @@ -445,6 +447,8 @@ type ClientHarness struct { client v1.HarnessClient } +var _ api.HarnessRunPreparer = (*ClientHarness)(nil) + // NewClientHarness wraps client as an api.Harness. func NewClientHarness(client v1.HarnessClient) *ClientHarness { return &ClientHarness{client: client} } @@ -457,19 +461,89 @@ func (h *ClientHarness) Describe(ctx context.Context) (api.Descriptor, error) { return descriptorFromProto(d), nil } +// PrepareRun opens the actual Connect stream without sending Start. A Connect failure therefore +// proves that the remote harness did not receive an execution. The returned stream is one-shot. +func (h *ClientHarness) PrepareRun(ctx context.Context) (api.PreparedHarnessRun, error) { + ctx, cancel := context.WithCancel(ctx) + stream, err := h.client.Connect(ctx) + if err != nil { + cancel() + return nil, err + } + return &preparedClientRun{stream: stream, ctx: ctx, cancel: cancel}, nil +} + +type preparedClientRun struct { + stream v1.Harness_ConnectClient + ctx context.Context + cancel context.CancelFunc + mu sync.Mutex + state uint8 // 0 prepared, 1 used, 2 closed unused + runCancel context.CancelFunc +} + +var _ api.PreparedHarnessRun = (*preparedClientRun)(nil) + +func (r *preparedClientRun) Close() { + r.mu.Lock() + if r.state == 0 { + r.state = 2 + } + runCancel := r.runCancel + r.mu.Unlock() + if runCancel != nil { + runCancel() + } + r.cancel() +} + +func (r *preparedClientRun) Run(ctx context.Context, start *api.Start, sink api.EventSink) error { + r.mu.Lock() + if r.state != 0 { + r.mu.Unlock() + return errors.New("harnesswire: prepared Connect stream is no longer available") + } + runCtx, runCancel := context.WithCancel(ctx) + r.state = 1 + r.runCancel = runCancel + r.mu.Unlock() + defer runCancel() + defer r.cancel() + stopPrepared := context.AfterFunc(r.ctx, func() { + runCancel() + r.cancel() + }) + defer stopPrepared() + if err := r.ctx.Err(); err != nil { + return err + } + if err := runCtx.Err(); err != nil { + return err + } + stopCancel := context.AfterFunc(runCtx, r.cancel) + defer stopCancel() + return runClientStream(runCtx, r.ctx, r.stream, r.cancel, start, sink) +} + // Run drives the remote harness over Connect, translating each emitted Event into a host-mediated // sink call and sending the result back. The sink (live or replay) is the controller's — the host // still mediates the model over the wire. func (h *ClientHarness) Run(ctx context.Context, start *api.Start, sink api.EventSink) error { - ctx, cancel := context.WithCancel(ctx) - defer cancel() // tear the Connect stream down (and the server's recv loop) when Run returns if start == nil || start.ExecutionID == "" { return errors.New("harnesswire: Start requires execution_id") } - stream, err := h.client.Connect(ctx) + prepared, err := h.PrepareRun(ctx) if err != nil { return err } + defer prepared.Close() + return prepared.Run(ctx, start, sink) +} + +func runClientStream(ctx, streamCtx context.Context, stream v1.Harness_ConnectClient, cancel context.CancelFunc, start *api.Start, sink api.EventSink) error { + if start == nil || start.ExecutionID == "" { + return errors.New("harnesswire: Start requires execution_id") + } if err := stream.Send(&v1.ControllerFrame{ Session: start.SessionUID, ExecutionId: start.ExecutionID, @@ -480,7 +554,7 @@ func (h *ClientHarness) Run(ctx context.Context, start *api.Start, sink api.Even for { ev, err := stream.Recv() if err == io.EOF { - return nil + return errors.New("harnesswire: stream closed before EVENT_END") } if err != nil { return err @@ -595,9 +669,14 @@ func (h *ClientHarness) Run(ctx context.Context, start *api.Start, sink api.Even case v1.EventKind_EVENT_APPROVAL_REQUEST, v1.EventKind_EVENT_APPROVAL_RESULT, v1.EventKind_EVENT_EXECUTION_START, v1.EventKind_EVENT_PARKED: return fmt.Errorf("harnesswire: harness emitted host-owned control event %s", ev.GetKind()) case v1.EventKind_EVENT_END: - err := endError(ev.GetEnd()) - awaitClose(stream, cancel) - return err + endErr := endError(ev.GetEnd()) + closeErr := awaitClose(ctx, streamCtx, stream, cancel) + if endErr != nil { + return endErr + } + return closeErr + default: + return fmt.Errorf("harnesswire: harness emitted unsupported event %s", ev.GetKind()) } } } @@ -610,15 +689,22 @@ const closeGrace = 2 * time.Second // harness gets to flush its telemetry before the host can suspend it, and the RPC ends with the // harness's own status rather than a host-side cancellation. A harness that never closes the // stream costs at most closeGrace. -func awaitClose(stream v1.Harness_ConnectClient, cancel context.CancelFunc) { +func awaitClose(ctx, streamCtx context.Context, stream v1.Harness_ConnectClient, cancel context.CancelFunc) error { _ = stream.CloseSend() - t := time.AfterFunc(closeGrace, cancel) + var timedOut atomic.Bool + t := time.AfterFunc(closeGrace, func() { + timedOut.Store(true) + cancel() + }) defer t.Stop() - for { - if _, err := stream.Recv(); err != nil { - return - } + ev, err := stream.Recv() + if err == nil { + return fmt.Errorf("harnesswire: harness emitted %s after EVENT_END", ev.GetKind()) } + if err == io.EOF || timedOut.Load() || ctx.Err() != nil || streamCtx.Err() != nil { + return nil + } + return err } // hostFailed ends a turn whose sink call failed on the host side, such as a provider error. While @@ -627,11 +713,25 @@ func awaitClose(stream v1.Harness_ConnectClient, cancel context.CancelFunc) { // returned unchanged. A cancelled caller ends the turn at once. func hostFailed(ctx context.Context, stream v1.Harness_ConnectClient, cancel context.CancelFunc, err error) error { if ctx.Err() == nil { - awaitClose(stream, cancel) + drainClose(stream, cancel) } return err } +// drainClose is the host-failure counterpart to awaitClose. The host's sink error is authoritative, +// so discard the harness's failed END and any other frames while waiting for the RPC and its +// telemetry flush to finish. +func drainClose(stream v1.Harness_ConnectClient, cancel context.CancelFunc) { + _ = stream.CloseSend() + t := time.AfterFunc(closeGrace, cancel) + defer t.Stop() + for { + if _, err := stream.Recv(); err != nil { + return + } + } +} + // exchangePark accepts only the transport acknowledgement followed by actual // clean closure. Nothing received here is dispatched to the journal sink. func exchangePark(ctx context.Context, stream v1.Harness_ConnectClient, start *api.Start, ref api.ApprovalRef) error { diff --git a/harnesswire/prepare_test.go b/harnesswire/prepare_test.go new file mode 100644 index 00000000..6fd695f6 --- /dev/null +++ b/harnesswire/prepare_test.go @@ -0,0 +1,320 @@ +package harnesswire + +import ( + "context" + "errors" + "io" + "net" + "path/filepath" + "testing" + "time" + + "github.com/agent-primitives/sessions/api" + v1 "github.com/agent-primitives/sessions/api/genpb" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type prepareClient struct { + v1.HarnessClient + stream *prepareClientStream + err error + connects int + ctx context.Context +} + +func (c *prepareClient) Connect(ctx context.Context, _ ...grpc.CallOption) (grpc.BidiStreamingClient[v1.ControllerFrame, v1.Event], error) { + c.connects++ + c.ctx = ctx + if c.err != nil { + return nil, c.err + } + return c.stream, nil +} + +type prepareClientStream struct { + grpc.ClientStream + sent []*v1.ControllerFrame + events []*v1.Event + sendErr error + recvErr error + closeSend int +} + +type blockingPreparedStream struct { + grpc.ClientStream + ctx context.Context + sent chan struct{} +} + +func (s *blockingPreparedStream) Send(*v1.ControllerFrame) error { + close(s.sent) + return nil +} + +func (s *blockingPreparedStream) Recv() (*v1.Event, error) { + <-s.ctx.Done() + return nil, s.ctx.Err() +} + +func (*blockingPreparedStream) CloseSend() error { return nil } + +func (s *prepareClientStream) Send(frame *v1.ControllerFrame) error { + s.sent = append(s.sent, frame) + return s.sendErr +} + +func (s *prepareClientStream) Recv() (*v1.Event, error) { + if len(s.events) > 0 { + ev := s.events[0] + s.events = s.events[1:] + return ev, nil + } + if s.recvErr != nil { + return nil, s.recvErr + } + return nil, io.EOF +} + +func (s *prepareClientStream) CloseSend() error { + s.closeSend++ + return nil +} + +type noOpSink struct{} + +func (noOpSink) Model(context.Context, api.ModelRequest) (api.ModelResponse, error) { + return api.ModelResponse{}, nil +} +func (noOpSink) Output(context.Context, string) error { return nil } +func (noOpSink) ToolCall(context.Context, api.ToolCall) (api.ToolResult, error) { + return api.ToolResult{}, nil +} +func (noOpSink) Report(context.Context, api.ToolResult) error { return nil } +func (noOpSink) Usage(context.Context, api.Usage) error { return nil } + +type blockingModelSink struct { + noOpSink + started chan struct{} +} + +func (s blockingModelSink) Model(ctx context.Context, _ api.ModelRequest) (api.ModelResponse, error) { + close(s.started) + <-ctx.Done() + return api.ModelResponse{}, ctx.Err() +} + +func TestPrepareRunOpensAndReusesOneConnectStream(t *testing.T) { + stream := &prepareClientStream{events: []*v1.Event{{ExecutionId: "e", Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}}} + client := &prepareClient{stream: stream} + h := NewClientHarness(client) + prepared, err := h.PrepareRun(t.Context()) + if err != nil { + t.Fatal(err) + } + defer prepared.Close() + if client.connects != 1 || len(stream.sent) != 0 { + t.Fatalf("after PrepareRun: connects=%d sent=%d, want 1/0", client.connects, len(stream.sent)) + } + if err := prepared.Run(t.Context(), &api.Start{SessionUID: "s", ExecutionID: "e"}, noOpSink{}); err != nil { + t.Fatal(err) + } + if client.connects != 1 || len(stream.sent) != 1 || stream.sent[0].GetStart() == nil { + t.Fatalf("after Run: connects=%d sent=%v, want one Start on prepared stream", client.connects, stream.sent) + } + if err := prepared.Run(t.Context(), &api.Start{SessionUID: "s", ExecutionID: "e"}, noOpSink{}); err == nil { + t.Fatal("second prepared Run succeeded") + } + if client.connects != 1 || len(stream.sent) != 1 { + t.Fatalf("second Run opened or sent again: connects=%d sent=%d", client.connects, len(stream.sent)) + } +} + +func TestPrepareRunFailureSendsNothing(t *testing.T) { + cause := errors.New("connect unavailable") + client := &prepareClient{err: cause} + prepared, err := NewClientHarness(client).PrepareRun(t.Context()) + if !errors.Is(err, cause) || prepared != nil || client.connects != 1 { + t.Fatalf("PrepareRun=(%v, %v), connects=%d; want nil, %v, 1", prepared, err, client.connects, cause) + } +} + +func TestPrepareRunMissingUnixSocketFailsBeforeStart(t *testing.T) { + missing := filepath.Join(t.TempDir(), "missing.sock") + conn, err := grpc.NewClient( + "passthrough:///missing-harness", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", missing) + }), + ) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + prepared, err := NewClientHarness(v1.NewHarnessClient(conn)).PrepareRun(ctx) + if err == nil || prepared != nil { + t.Fatalf("PrepareRun=(%v, %v), want pre-Start Connect failure", prepared, err) + } +} + +func TestPreparedRunCloseCancelsUnusedConnect(t *testing.T) { + client := &prepareClient{stream: &prepareClientStream{}} + prepared, err := NewClientHarness(client).PrepareRun(t.Context()) + if err != nil { + t.Fatal(err) + } + prepared.Close() + prepared.Close() + select { + case <-client.ctx.Done(): + default: + t.Fatal("Close did not cancel prepared Connect") + } + if len(client.stream.sent) != 0 { + t.Fatal("Close sent a frame") + } + if err := prepared.Run(t.Context(), &api.Start{ExecutionID: "e"}, noOpSink{}); err == nil { + t.Fatal("Run after Close succeeded") + } +} + +func TestPreparedRunCloseCancelsActiveRun(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + stream := &blockingPreparedStream{ctx: ctx, sent: make(chan struct{})} + prepared := &preparedClientRun{stream: stream, ctx: ctx, cancel: cancel} + done := make(chan error, 1) + go func() { + done <- prepared.Run(ctx, &api.Start{ExecutionID: "e"}, noOpSink{}) + }() + <-stream.sent + prepared.Close() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("Run=%v, want Close cancellation", err) + } +} + +func TestPreparedRunHonorsIndependentRunCancellation(t *testing.T) { + prepareCtx, prepareCancel := context.WithCancel(context.Background()) + stream := &blockingPreparedStream{ctx: prepareCtx, sent: make(chan struct{})} + prepared := &preparedClientRun{stream: stream, ctx: prepareCtx, cancel: prepareCancel} + runCtx, runCancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { + done <- prepared.Run(runCtx, &api.Start{ExecutionID: "e"}, noOpSink{}) + }() + <-stream.sent + runCancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("Run=%v, want Run context cancellation", err) + } +} + +func TestPreparedRunCloseCancelsBlockedSinkWithIndependentContext(t *testing.T) { + prepareCtx, prepareCancel := context.WithCancel(context.Background()) + stream := &prepareClientStream{events: []*v1.Event{{ + ExecutionId: "e", + Kind: v1.EventKind_EVENT_MODEL_CALL, + Body: &v1.Event_Model{Model: &v1.ModelCall{Id: "m", Model: "model"}}, + }}} + prepared := &preparedClientRun{stream: stream, ctx: prepareCtx, cancel: prepareCancel} + sink := blockingModelSink{started: make(chan struct{})} + done := make(chan error, 1) + go func() { + done <- prepared.Run(context.Background(), &api.Start{ExecutionID: "e"}, sink) + }() + <-sink.started + prepared.Close() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("Run=%v, want Close cancellation", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Close did not cancel blocked sink call") + } +} + +func TestClientRunInvalidStartDoesNotConnect(t *testing.T) { + client := &prepareClient{stream: &prepareClientStream{}} + err := NewClientHarness(client).Run(t.Context(), &api.Start{}, noOpSink{}) + if err == nil || client.connects != 0 { + t.Fatalf("Run=%v connects=%d, want local validation before Connect", err, client.connects) + } +} + +func TestPreparedSendAndRecvFailuresAreRunFailures(t *testing.T) { + for _, tc := range []struct { + name string + stream *prepareClientStream + cause error + }{ + {name: "send", cause: errors.New("send uncertain")}, + {name: "recv", cause: errors.New("recv uncertain")}, + } { + t.Run(tc.name, func(t *testing.T) { + stream := &prepareClientStream{} + if tc.name == "send" { + stream.sendErr = tc.cause + } else { + stream.recvErr = tc.cause + } + prepared, err := NewClientHarness(&prepareClient{stream: stream}).PrepareRun(t.Context()) + if err != nil { + t.Fatal(err) + } + defer prepared.Close() + err = prepared.Run(t.Context(), &api.Start{ExecutionID: "e"}, noOpSink{}) + if !errors.Is(err, tc.cause) { + t.Fatalf("Run=%v, want %v", err, tc.cause) + } + }) + } +} + +func TestPreparedRunRequiresEndAndRejectsUnsupportedEvents(t *testing.T) { + for _, tc := range []struct { + name string + events []*v1.Event + }{ + {name: "EOF before END"}, + {name: "host-owned error", events: []*v1.Event{{ExecutionId: "e", Kind: v1.EventKind_EVENT_ERROR, Body: &v1.Event_Error{Error: &v1.Error{Description: "peer error"}}}}}, + {name: "unspecified", events: []*v1.Event{{ExecutionId: "e", Kind: v1.EventKind_EVENT_KIND_UNSPECIFIED}}}, + } { + t.Run(tc.name, func(t *testing.T) { + stream := &prepareClientStream{events: tc.events} + prepared, err := NewClientHarness(&prepareClient{stream: stream}).PrepareRun(t.Context()) + if err != nil { + t.Fatal(err) + } + defer prepared.Close() + if err := prepared.Run(t.Context(), &api.Start{ExecutionID: "e"}, noOpSink{}); err == nil { + t.Fatal("Run accepted stream without EVENT_END") + } + }) + } +} + +func TestPreparedRunRejectsFramesAfterEnd(t *testing.T) { + for _, ev := range []*v1.Event{ + {ExecutionId: "e", Kind: v1.EventKind_EVENT_ERROR, Body: &v1.Event_Error{Error: &v1.Error{Description: "peer error"}}}, + {ExecutionId: "e", Kind: v1.EventKind_EVENT_KIND_UNSPECIFIED}, + {ExecutionId: "e", Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}, + } { + stream := &prepareClientStream{events: []*v1.Event{ + {ExecutionId: "e", Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}, + ev, + }} + prepared, err := NewClientHarness(&prepareClient{stream: stream}).PrepareRun(t.Context()) + if err != nil { + t.Fatal(err) + } + if err := prepared.Run(t.Context(), &api.Start{ExecutionID: "e"}, noOpSink{}); err == nil { + t.Fatalf("Run accepted %s after EVENT_END", ev.GetKind()) + } + prepared.Close() + } +} diff --git a/placement/describe_boundary_test.go b/placement/describe_boundary_test.go index 682aadae..82dd72e1 100644 --- a/placement/describe_boundary_test.go +++ b/placement/describe_boundary_test.go @@ -29,6 +29,15 @@ func (h describeBoundaryHarness) Describe(ctx context.Context) (api.Descriptor, return h.describe(ctx) } +type prepareBoundaryHarness struct { + api.Harness + prepare func(context.Context) (api.PreparedHarnessRun, error) +} + +func (h prepareBoundaryHarness) PrepareRun(ctx context.Context) (api.PreparedHarnessRun, error) { + return h.prepare(ctx) +} + func TestControllerDescribeBoundaryUsesAdmissionClassification(t *testing.T) { for _, operation := range []string{"Exec", "Resume"} { for _, tc := range []struct { @@ -119,3 +128,59 @@ func TestControllerDescribeBoundaryUsesAdmissionClassification(t *testing.T) { } } } + +func TestControllerPrepareBoundaryUsesAdmissionClassification(t *testing.T) { + for _, tc := range []struct { + name string + peerErr error + cancel bool + want error + wantCause error + }{ + {name: "unavailable", peerErr: status.Error(codes.Unavailable, "peer unavailable"), want: placement.ErrHarnessUnavailable}, + {name: "peer deadline", peerErr: status.Error(codes.DeadlineExceeded, "peer deadline"), want: placement.ErrHarnessUnavailable}, + {name: "plain deadline", peerErr: context.DeadlineExceeded, want: placement.ErrHarnessUnavailable}, + {name: "caller cancelled", peerErr: status.Error(codes.Unavailable, "peer unavailable"), cancel: true, want: placement.ErrAdmissionInterrupted, wantCause: context.Canceled}, + {name: "internal", peerErr: status.Error(codes.Internal, "peer internal"), want: controller.ErrHarnessPrepareFailed}, + } { + t.Run(tc.name, func(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + defer store.Close() + log := store.Session("prepare-boundary") + base := &identityToolHarness{version: "v1"} + desc, err := base.Describe(t.Context()) + if err != nil { + t.Fatal(err) + } + backend := &routedBackend{desc: desc} + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var closes atomic.Int32 + h := prepareBoundaryHarness{Harness: base, prepare: func(context.Context) (api.PreparedHarnessRun, error) { + if tc.cancel { + cancel() + } + return nil, tc.peerErr + }} + p := placement.New(backend, nil, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return h, func() error { closes.Add(1); return nil }, nil + })) + _, err = p.Exec(ctx, log, "prepare-boundary", nil, 0) + if !errors.Is(err, tc.want) || tc.wantCause != nil && !errors.Is(err, tc.wantCause) { + t.Fatalf("error=%v want=%v cause=%v", err, tc.want, tc.wantCause) + } + if tc.want == controller.ErrHarnessPrepareFailed && (errors.Is(err, placement.ErrHarnessUnavailable) || errors.Is(err, placement.ErrAdmissionInterrupted)) { + t.Fatalf("non-outage preparation error misclassified: %v", err) + } + if base.runs.Load() != 0 || closes.Load() != 1 { + t.Fatalf("preparation failure ran harness or leaked connection: runs=%d closes=%d", base.runs.Load(), closes.Load()) + } + if head, headErr := log.Head(); headErr != nil || head != 0 { + t.Fatalf("head=%d err=%v, want no execution records", head, headErr) + } + }) + } +} diff --git a/placement/placement.go b/placement/placement.go index 78d2df4a..0eaa7230 100644 --- a/placement/placement.go +++ b/placement/placement.go @@ -52,8 +52,9 @@ var ErrSessionBusy = errors.New("placement: another operation is in progress for var ErrDescriptorMismatch = errors.New("placement: harness is not the expected one") // ErrHarnessUnavailable is returned when the harness could not be reached to describe itself during -// admission or the controller's connected-harness identity check. Neither check runs the harness or -// appends journal records; the connected check may follow compute allocation and fence minting. +// admission or the controller's connected-harness identity check, or to prepare its Connect stream +// before a new execution. None runs the harness or appends execution records; the connected checks +// may follow compute allocation and fence minting. // session.Service surfaces it as codes.Unavailable. If the caller's own deadline or cancellation // ended the check, it is ErrAdmissionInterrupted instead: the same expired call cannot succeed. var ErrHarnessUnavailable = errors.New("placement: harness unavailable") @@ -166,7 +167,9 @@ func (s *sessionSet) busy() bool { // the connection. runtime/local passes a unix-socket address (unix://…); substrate passes the // atenet-router's host:port, dialed over h2c, plus CallMetadata naming the actor, which the client // must attach to every call. The default dialer handles both forms; WithDialer overrides it (tests). -// This is the one transport seam the harness rides unchanged. +// The closer may be called while PrepareRun or Run is blocked; it must interrupt the transport and +// return promptly rather than wait for that call to finish. This is the one transport seam the +// harness rides unchanged. type Dialer func(inc api.Incarnation) (api.Harness, func() error, error) // Option configures a Placer. @@ -645,13 +648,18 @@ func (p *Placer) refuseOverrideRepair(ctx context.Context, sessionUID string, ov return refuseStatefulOverride(desc, override) } -// controllerError maps only the controller's Describe boundary. Run, model and runtime errors -// with identical transport codes must keep their existing classification. +// controllerError maps only the controller's pre-execution Describe and Connect-preparation +// boundaries. Run, model and runtime errors with identical transport codes keep their existing +// classification because Start delivery may already be uncertain. func controllerError(ctx context.Context, err error) error { - if !errors.Is(err, controller.ErrHarnessDescribeFailed) { + switch { + case errors.Is(err, controller.ErrHarnessDescribeFailed): + return describeError(ctx, err, errors.Is(err, context.DeadlineExceeded)) + case errors.Is(err, controller.ErrHarnessPrepareFailed): + return prepareHarnessError(ctx, err) + default: return err } - return describeError(ctx, err, errors.Is(err, context.DeadlineExceeded)) } // describeError is shared by admission and connected-harness identity checks. @@ -662,13 +670,31 @@ func describeError(ctx context.Context, err error, timedOut bool) error { // context error cannot change the service's cancellation/deadline classification. return fmt.Errorf("%w: describe harness: %w: %v", ErrAdmissionInterrupted, ctx.Err(), err) case timedOut || c == codes.Unavailable || c == codes.DeadlineExceeded: - // A refused connection or a harness timeout while the caller is live is a retryable outage. return fmt.Errorf("%w: %w", ErrHarnessUnavailable, err) default: return err } } +func prepareHarnessError(ctx context.Context, err error) error { + if ctx.Err() != nil { + return &CallerContextError{CallerErr: ctx.Err(), Cause: err} + } + if c := status.Code(err); c == codes.Unavailable || c == codes.DeadlineExceeded || errors.Is(err, context.DeadlineExceeded) { + // Connect setup can name a private endpoint or carry credential-provider diagnostics. Keep the + // cause for direct Go callers but publish only the fixed host-owned outage text. + return &harnessUnavailableError{cause: err} + } + return err +} + +type harnessUnavailableError struct{ cause error } + +func (*harnessUnavailableError) Error() string { return ErrHarnessUnavailable.Error() } +func (e *harnessUnavailableError) Unwrap() []error { + return []error{ErrHarnessUnavailable, e.cause} +} + // DefaultDial reaches a harnesswire server by address form over plaintext: a unix-socket address // (unix://…, runtime/local) or a TCP host:port. It attaches the incarnation's CallMetadata to every // call. A backend whose harnesses need transport security (substrate's atenet-router serves only diff --git a/placement/transport_test.go b/placement/transport_test.go index ea1f0939..857bef4b 100644 --- a/placement/transport_test.go +++ b/placement/transport_test.go @@ -393,6 +393,65 @@ func TestForkEndsTheParentsOpenHarnessStreamBeforeTheCheckpoint(t *testing.T) { testCheckpointEndsOpenStream(t, forkCheckpoint) } +type blockedPrepareHarness struct { + api.Harness + started chan struct{} + done chan struct{} +} + +func (h *blockedPrepareHarness) PrepareRun(ctx context.Context) (api.PreparedHarnessRun, error) { + close(h.started) + <-ctx.Done() + close(h.done) + return nil, ctx.Err() +} + +func TestForkCancelsPreparedHarnessBeforeExecutionIsRecorded(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + defer store.Close() + log := store.Session("s1") + h := &blockedPrepareHarness{Harness: echoagent.Harness{}, started: make(chan struct{}), done: make(chan struct{})} + backend := &routedBackend{desc: memDescriptor} + p := placement.New(backend, echoagent.Model, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return h, func() error { return nil }, nil + })) + + execErr := make(chan error, 1) + go func() { + _, err := p.Exec(context.Background(), log, "s1", []api.Message{*api.TextMessage("user", "hi")}, 0) + execErr <- err + }() + select { + case <-h.started: + case <-time.After(10 * time.Second): + t.Fatal("turn never began Connect preparation") + } + + if err := forkCheckpoint(p, log, "s1"); err != nil { + t.Fatalf("checkpoint: %v", err) + } + select { + case <-h.done: + case <-time.After(10 * time.Second): + t.Fatal("checkpoint did not cancel prepared Connect") + } + select { + case err := <-execErr: + if !errors.Is(err, placement.ErrCheckpointing) || !errors.Is(err, eventlog.ErrFenced) { + t.Fatalf("turn returned %v, want ErrCheckpointing and ErrFenced", err) + } + case <-time.After(10 * time.Second): + t.Fatal("superseded turn did not return") + } + kinds := journalKinds(t, log) + if len(kinds) != 1 || kinds[0] != api.EventLifecycle { + t.Fatalf("journal=%v, want only checkpoint lifecycle record", kinds) + } +} + // The fork must end the turn's stream whichever Placer of the Registry runs the turn. func TestForkEndsAParentStreamOpenedByAnotherPlacerOfTheRegistry(t *testing.T) { testCheckpointEndsOpenStream(t, forkCheckpoint, registryPair) diff --git a/session/error_connect_test.go b/session/error_connect_test.go new file mode 100644 index 00000000..4c66bf29 --- /dev/null +++ b/session/error_connect_test.go @@ -0,0 +1,330 @@ +package session_test + +import ( + "context" + "errors" + "io" + "strings" + "sync/atomic" + "testing" + + "github.com/agent-primitives/sessions/api" + v1 "github.com/agent-primitives/sessions/api/genpb" + "github.com/agent-primitives/sessions/harness/echoagent" + "github.com/agent-primitives/sessions/harnesswire" + "github.com/agent-primitives/sessions/placement" + "github.com/agent-primitives/sessions/runtime/local" + "github.com/agent-primitives/sessions/session" + "github.com/agent-primitives/sessions/sqlitelog" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type retryConnectClient struct { + v1.HarnessClient + connects atomic.Int32 + starts []*v1.Start + firstErr error +} + +func (c *retryConnectClient) Describe(ctx context.Context, _ *v1.DescribeRequest, _ ...grpc.CallOption) (*v1.HarnessDescriptor, error) { + desc, err := (echoagent.Harness{}).Describe(ctx) + if err != nil { + return nil, err + } + return harnesswire.DescriptorToProto(desc), nil +} + +func (c *retryConnectClient) Connect(context.Context, ...grpc.CallOption) (grpc.BidiStreamingClient[v1.ControllerFrame, v1.Event], error) { + if c.connects.Add(1) == 1 { + return nil, c.firstErr + } + return &retryConnectStream{client: c}, nil +} + +type retryConnectStream struct { + grpc.ClientStream + client *retryConnectClient + executionID string + ended bool +} + +func (s *retryConnectStream) Send(frame *v1.ControllerFrame) error { + start := frame.GetStart() + if start == nil { + return errors.New("expected Start") + } + s.executionID = frame.GetExecutionId() + s.client.starts = append(s.client.starts, start) + return nil +} + +func (s *retryConnectStream) Recv() (*v1.Event, error) { + if s.ended { + return nil, io.EOF + } + s.ended = true + return &v1.Event{ExecutionId: s.executionID, Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}, nil +} + +func (*retryConnectStream) CloseSend() error { return nil } + +type postEndClient struct{ v1.HarnessClient } + +func (postEndClient) Describe(ctx context.Context, _ *v1.DescribeRequest, _ ...grpc.CallOption) (*v1.HarnessDescriptor, error) { + desc, err := (echoagent.Harness{}).Describe(ctx) + if err != nil { + return nil, err + } + return harnesswire.DescriptorToProto(desc), nil +} + +func (postEndClient) Connect(context.Context, ...grpc.CallOption) (grpc.BidiStreamingClient[v1.ControllerFrame, v1.Event], error) { + return &postEndStream{}, nil +} + +type postEndStream struct { + grpc.ClientStream + executionID string + step int +} + +func (s *postEndStream) Send(frame *v1.ControllerFrame) error { + s.executionID = frame.GetExecutionId() + return nil +} + +func (s *postEndStream) Recv() (*v1.Event, error) { + s.step++ + switch s.step { + case 1: + return &v1.Event{ExecutionId: s.executionID, Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}, nil + case 2: + return &v1.Event{ExecutionId: s.executionID, Kind: v1.EventKind_EVENT_ERROR, Body: &v1.Event_Error{Error: &v1.Error{Description: "contradictory"}}}, nil + default: + return nil, io.EOF + } +} + +func (*postEndStream) CloseSend() error { return nil } + +type endThenBlockClient struct { + v1.HarnessClient + waiting chan struct{} +} + +func (endThenBlockClient) Describe(ctx context.Context, _ *v1.DescribeRequest, _ ...grpc.CallOption) (*v1.HarnessDescriptor, error) { + desc, err := (echoagent.Harness{}).Describe(ctx) + if err != nil { + return nil, err + } + return harnesswire.DescriptorToProto(desc), nil +} + +func (c endThenBlockClient) Connect(ctx context.Context, _ ...grpc.CallOption) (grpc.BidiStreamingClient[v1.ControllerFrame, v1.Event], error) { + return &endThenBlockStream{ctx: ctx, waiting: c.waiting}, nil +} + +type endThenBlockStream struct { + grpc.ClientStream + ctx context.Context + waiting chan struct{} + executionID string + step int +} + +func (s *endThenBlockStream) Send(frame *v1.ControllerFrame) error { + s.executionID = frame.GetExecutionId() + return nil +} + +func (s *endThenBlockStream) Recv() (*v1.Event, error) { + s.step++ + if s.step == 1 { + return &v1.Event{ExecutionId: s.executionID, Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}, nil + } + close(s.waiting) + <-s.ctx.Done() + return nil, s.ctx.Err() +} + +func (*endThenBlockStream) CloseSend() error { return nil } + +func TestExecConnectPreparationFailureCanRetryFirstTurn(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + backend := local.New(echoagent.Harness{}) + t.Cleanup(func() { _ = backend.Close() }) + const private = "https://private.invalid/?token=connect-secret" + cause := status.Error(codes.Unavailable, private) + client := &retryConnectClient{firstErr: cause} + p := placement.New(backend, echoagent.Model, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return harnesswire.NewClientHarness(client), func() error { return nil }, nil + })) + registry, err := placement.NewRegistry("echo", map[string]*placement.Placer{"echo": p}) + if err != nil { + t.Fatal(err) + } + svc := session.NewService(store, registry) + created, err := svc.CreateSession(t.Context(), &v1.CreateSessionRequest{}) + if err != nil { + t.Fatal(err) + } + uid := created.GetMetadata().GetUid() + req := &v1.ExecRequest{Session: uid, Inputs: []*v1.Message{{Role: "user", Parts: []*v1.Part{{Part: &v1.Part_Text{Text: &v1.TextPart{Text: "hi"}}}}}}} + + err = svc.Exec(req, &rpcStream{ctx: t.Context()}) + if status.Code(err) != codes.Unavailable || !errors.Is(err, placement.ErrHarnessUnavailable) || !errors.Is(err, cause) { + t.Fatalf("first Exec=%v, want UNAVAILABLE/HARNESS_UNAVAILABLE", err) + } + if strings.Contains(err.Error(), private) { + t.Fatalf("Connect diagnostic leaked into public status: %v", err) + } + details := status.Convert(err).Details() + if len(details) != 1 { + t.Fatalf("details=%v, want HARNESS_UNAVAILABLE", details) + } + info, ok := details[0].(*errdetails.ErrorInfo) + if !ok || info.Domain != "agentsessions" || info.Reason != "HARNESS_UNAVAILABLE" || len(info.Metadata) != 0 { + t.Fatalf("details=%v, want agentsessions/HARNESS_UNAVAILABLE", details) + } + if head, headErr := store.Session(uid).Head(); headErr != nil || head != 0 { + t.Fatalf("head=%d err=%v, want no failed execution", head, headErr) + } + + if err := svc.Exec(req, &rpcStream{ctx: t.Context()}); err != nil { + t.Fatalf("retry Exec=%v", err) + } + if client.connects.Load() != 2 || len(client.starts) != 1 || len(client.starts[0].GetHistory()) != 0 { + t.Fatalf("connects=%d starts=%v, want one failed Connect and one first-turn Start", client.connects.Load(), client.starts) + } +} + +type unavailableRunHarness struct{ echoagent.Harness } + +func (h unavailableRunHarness) PrepareRun(context.Context) (api.PreparedHarnessRun, error) { + return unavailablePreparedRun{}, nil +} + +type unavailablePreparedRun struct{} + +func (unavailablePreparedRun) Run(context.Context, *api.Start, api.EventSink) error { + return status.Error(codes.Unavailable, "delivery outcome unknown") +} + +func (unavailablePreparedRun) Close() {} + +func TestExecPreparedRunFailureRemainsDurableAndUnclassified(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + backend := local.New(echoagent.Harness{}) + t.Cleanup(func() { _ = backend.Close() }) + p := placement.New(backend, echoagent.Model, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return unavailableRunHarness{}, func() error { return nil }, nil + })) + registry, err := placement.NewRegistry("echo", map[string]*placement.Placer{"echo": p}) + if err != nil { + t.Fatal(err) + } + svc := session.NewService(store, registry) + created, err := svc.CreateSession(t.Context(), &v1.CreateSessionRequest{}) + if err != nil { + t.Fatal(err) + } + uid := created.GetMetadata().GetUid() + err = svc.Exec(&v1.ExecRequest{Session: uid}, &rpcStream{ctx: t.Context()}) + if status.Code(err) != codes.Internal || errors.Is(err, placement.ErrHarnessUnavailable) { + t.Fatalf("Exec=%v, want INTERNAL without HARNESS_UNAVAILABLE", err) + } + if len(status.Convert(err).Details()) != 0 { + t.Fatalf("uncertain Run failure acquired ErrorInfo: %v", status.Convert(err).Details()) + } + recs, readErr := store.Session(uid).Read(1) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 2 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventError { + t.Fatalf("records=%+v, want durable start and error", recs) + } +} + +func TestExecFrameAfterEndIsDurableProtocolFailure(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + backend := local.New(echoagent.Harness{}) + t.Cleanup(func() { _ = backend.Close() }) + p := placement.New(backend, echoagent.Model, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return harnesswire.NewClientHarness(postEndClient{}), func() error { return nil }, nil + })) + registry, err := placement.NewRegistry("echo", map[string]*placement.Placer{"echo": p}) + if err != nil { + t.Fatal(err) + } + svc := session.NewService(store, registry) + created, err := svc.CreateSession(t.Context(), &v1.CreateSessionRequest{}) + if err != nil { + t.Fatal(err) + } + uid := created.GetMetadata().GetUid() + err = svc.Exec(&v1.ExecRequest{Session: uid}, &rpcStream{ctx: t.Context()}) + if status.Code(err) != codes.Internal { + t.Fatalf("Exec=%v, want INTERNAL protocol failure", err) + } + recs, readErr := store.Session(uid).Read(1) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 2 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventError { + t.Fatalf("records=%+v, want durable start and error without END", recs) + } +} + +func TestExecCancellationDuringPostEndFlushKeepsCompletedResult(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + backend := local.New(echoagent.Harness{}) + t.Cleanup(func() { _ = backend.Close() }) + waiting := make(chan struct{}) + p := placement.New(backend, echoagent.Model, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return harnesswire.NewClientHarness(endThenBlockClient{waiting: waiting}), func() error { return nil }, nil + })) + registry, err := placement.NewRegistry("echo", map[string]*placement.Placer{"echo": p}) + if err != nil { + t.Fatal(err) + } + svc := session.NewService(store, registry) + created, err := svc.CreateSession(t.Context(), &v1.CreateSessionRequest{}) + if err != nil { + t.Fatal(err) + } + uid := created.GetMetadata().GetUid() + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- svc.Exec(&v1.ExecRequest{Session: uid}, &rpcStream{ctx: ctx}) }() + <-waiting + cancel() + if err := <-done; err != nil { + t.Fatalf("Exec=%v, want completed result after caller cancellation during telemetry flush", err) + } + recs, readErr := store.Session(uid).Read(1) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 2 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventEnd { + t.Fatalf("records=%+v, want durable start and END without ERROR", recs) + } +} diff --git a/spec.md b/spec.md index e27b63e6..b0900796 100644 --- a/spec.md +++ b/spec.md @@ -641,6 +641,14 @@ reads the catalog that registration writes; the reference daemon does. - `EXE-7`: `Start.history` for a new execution MUST be every event before its `EXECUTION_START`, with `ApprovalResult.verified_approver` left empty (`APR-8`). The reference host always sends the full prior log, including after a memory restore. +- `EXE-45`: Before recording a new execution for a harness that supports run preparation, the host + MUST prepare the actual Run without invoking the harness or sending `Start`. A preparation failure + appends no execution records, so the caller may retry `Exec` at the same journal cursor. The + successful preparation MUST be used for that execution at most once, and MUST be closed unused if + a record append fails. Only after `EXECUTION_START` and every `INPUT` commit may the host send + `Start`. Once preparation succeeds, a caller cancellation, send failure or receive failure does + not prove non-delivery: any recorded execution remains durable and visible in later history under + `EXE-7` and `EXE-17`. ### 8.2 Effects @@ -903,10 +911,13 @@ cannot observe a model call that does not reach it. The class concerns model cal re-prompts. Once the compute that held the memory is gone (`RT-4e`), or replaced on a backend that records `ComputeInstance` (`RT-4h`, for example `runtime/substrate`), every later `Exec` and `Resume` on the session fails `FAILED_PRECONDITION`, so the re-prompt goes to a new session; the - failed session's log stays readable. When the compute and its memory survive, for example when - only the host or the harness stream failed, the re-prompt is an `Exec` on the same session. An - unmediated harness with no state for a session whose history records an earlier execution MUST - fail the run as possible state loss rather than start an empty conversation. + failed session's log stays readable. When the compute and its memory are known to survive, or + preparing the harness Run failed so no execution was recorded (`EXE-45`), the re-prompt is an + `Exec` on the same session. Cancellation after a successful preparation, or a stream failure once + sending begins, does not prove that memory survived or that the harness did no work; any recorded + attempt remains in later history. An unmediated harness with no state for a session whose history + records an earlier execution MUST fail the run as possible state loss rather than start an empty + conversation. - `EXE-44`: Wrapper delegation. When a host-side wrapper performs work before delegating to an unmediated harness, a recorded execution that the wrapper positively knows was not delegated MUST be omitted in full from `Start.history` on later calls to that underlying harness. This is a @@ -1507,7 +1518,8 @@ does not erase data or reclaim compute. | Harness `Describe` (admission `RT-1`, the `RT-3` recheck, or a `Describe` during the turn) could not reach the harness while the caller's context is live: the peer returned `UNAVAILABLE` or `DEADLINE_EXCEEDED`; or, on admission and the recheck, the host's own bound expired; or, during the turn, the error wraps `context.DeadlineExceeded` | `UNAVAILABLE` | | Any other `Describe` error on those paths, for example a peer `UNIMPLEMENTED` or `INTERNAL`, or a plain error from an in-process harness (during the turn, even one returned after the host's bound expired) | `INTERNAL` | | Any `Describe` error in the stored harness's half of the memory-tier override check (`OPS-7`) while the caller's context is live | `UNAVAILABLE` | -| Caller deadline or cancellation that ends admission, compute allocation/start, any `Describe` above, a `Suspend` or `Fork` runtime operation, or `DeleteSession`, unless a more specific lifecycle refusal above applies | `DEADLINE_EXCEEDED` or `CANCELLED` | +| Preparing `Harness.Connect` for a new `Exec` fails with `UNAVAILABLE` or `DEADLINE_EXCEEDED` while the caller is live (`EXE-45`) | `UNAVAILABLE` | +| Caller deadline or cancellation that ends admission, compute allocation/start, any `Describe` above, Connect preparation, a `Suspend` or `Fork` runtime operation, or `DeleteSession`, unless a more specific lifecycle refusal above applies | `DEADLINE_EXCEEDED` or `CANCELLED` | | Journal hash chain fails verification on `Exec`, `Resume` (including an `APR-13` park or request repair) or `Fork` (`CHAIN-12`) | `DATA_LOSS` | | `Cancel` | `UNIMPLEMENTED` | | Any other failure | `INTERNAL` | @@ -1618,10 +1630,11 @@ sequenceDiagram seconds for the harness to end the RPC before cancelling it, so the harness can flush telemetry and return its own status. It waits the same way when a sink call fails on the host side, unless the caller has already cancelled; the turn still fails with the host's error. After - `EVENT_PARKED` it already waits for the harness to close (`APR-10`). -- TODO(spike): The reference host client treats a stream that closes cleanly without `EVENT_END` - as success, and silently ignores harness-sent `EVENT_INPUT`, `EVENT_LIFECYCLE`, `EVENT_ERROR`, - and unspecified kinds. Should these be protocol errors, as the proto comment implies? + `EVENT_PARKED` it already waits for the harness to close (`APR-10`). A stream that closes cleanly + before `EVENT_END`, or sends a host-owned or unsupported event kind, fails the execution as a + protocol error; it is never inferred as successful completion. After `EVENT_END{COMPLETED}`, a + non-OK final RPC status fails the turn while the caller remains live, but caller cancellation or + deadline during the bounded telemetry-flush wait does not replace the harness's completed result. - `WIRE-13`: Harness dials in the reference host use no transport security (§17). ### 12.3 Wire conversion @@ -2435,7 +2448,6 @@ Each item is also marked inline where it applies. - TODO(spike): Repeated `Suspend` and `Resume` without `Suspend` (§11.5). - TODO(spike): Fork journal atomicity and `at_seq > head` (§11.7). - TODO(spike): Unknown `Resumability` numbers read as `STATELESS_REPLAY` (§12.1). -- TODO(spike): Stream close without `EVENT_END` and ignored harness event kinds (§12.2). - TODO(spike): Session check and stickiness for model reply correlation failures (§12.2). - TODO(spike): Black-box conformance packaging (§16.2). - TODO(spike): End-to-end egress credential injection and policy enforcement for an unmediated