From b0627ed0ad25d100828118463aeaa4d7fc54950b Mon Sep 17 00:00:00 2001 From: Tatsat Mishra Date: Tue, 17 Nov 2026 18:39:16 +1300 Subject: [PATCH 1/5] fix(harnesswire): prepare Connect before journaling --- api/harness.go | 20 ++ .../unreleased/harness-connect-preparation.md | 11 + changelog/unreleased/harness-run-preparer.md | 9 + controller/controller.go | 23 +- controller/prepare_run_test.go | 186 +++++++++++++ docs/harness-authoring.md | 9 + harnesswire/harnesswire.go | 99 ++++++- harnesswire/prepare_test.go | 258 ++++++++++++++++++ placement/describe_boundary_test.go | 65 +++++ placement/placement.go | 42 ++- placement/transport_test.go | 59 ++++ session/error_connect_test.go | 249 +++++++++++++++++ spec.md | 28 +- 13 files changed, 1027 insertions(+), 31 deletions(-) create mode 100644 changelog/unreleased/harness-connect-preparation.md create mode 100644 changelog/unreleased/harness-run-preparer.md create mode 100644 controller/prepare_run_test.go create mode 100644 harnesswire/prepare_test.go create mode 100644 session/error_connect_test.go diff --git a/api/harness.go b/api/harness.go index 163eb26d..c75dd93f 100644 --- a/api/harness.go +++ b/api/harness.go @@ -18,6 +18,26 @@ type Harness interface { Run(ctx context.Context, s *Start, sink EventSink) error } +// HarnessRunPreparer optionally prepares the transport or other resources for one Run before the +// host records its execution. PrepareRun must not invoke the harness, send Start, or perform any +// model or tool effect. An error therefore proves that no execution was delegated. +// +// On error an implementation must release anything it acquired and return no prepared run. A +// successful call transfers one prepared run to the caller. The host either calls Run exactly once +// or closes it without running it. +type HarnessRunPreparer interface { + PrepareRun(ctx context.Context) (PreparedHarnessRun, error) +} + +// PreparedHarnessRun is one prepared Harness.Run. Run may be called exactly once. Close is +// idempotent and may race with Run: if Close wins, Run must fail without invoking the harness; if +// Run wins, Close cancels the active run. Close is also safe after Run returns. Implementations own +// cleanup and cannot report a second cleanup error. +type PreparedHarnessRun interface { + Run(ctx context.Context, s *Start, sink EventSink) error + Close() +} + // Descriptor is a harness's static contract. type Descriptor struct { ID string diff --git a/changelog/unreleased/harness-connect-preparation.md b/changelog/unreleased/harness-connect-preparation.md new file mode 100644 index 00000000..379b496a --- /dev/null +++ b/changelog/unreleased/harness-connect-preparation.md @@ -0,0 +1,11 @@ +--- +category: Fixed +--- + +- A new `Exec` over `Harness.Connect` now opens its actual stream before recording + `EXECUTION_START`. If Connect cannot be opened, no execution records are appended and retrying the + same session starts its first turn normally. The host still sends `Start` only after the start and + every input are durable; send and receive failures remain uncertain and keep the attempted + execution in later history. A clean close without `EVENT_END`, or a host-owned or unsupported + harness event, is now a failed execution rather than implicit success. Go harness adapters can opt + into the same boundary with `api.HarnessRunPreparer` (`EXE-45`). diff --git a/changelog/unreleased/harness-run-preparer.md b/changelog/unreleased/harness-run-preparer.md new file mode 100644 index 00000000..0d47d340 --- /dev/null +++ b/changelog/unreleased/harness-run-preparer.md @@ -0,0 +1,9 @@ +--- +category: Added +--- + +- Go harness adapters can implement `api.HarnessRunPreparer` to acquire the transport for one + execution before it is recorded, without invoking the harness or sending `Start`. The returned + `api.PreparedHarnessRun` is used at most once or closed; a concurrent close either prevents Run + or cancels it. `harnesswire.ClientHarness` implements the extension for its `Harness.Connect` + stream (`EXE-45`). diff --git a/controller/controller.go b/controller/controller.go index 7b04be11..0c1e46db 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -43,6 +43,11 @@ var ErrInvalidExecutionLog = errors.New("controller: invalid execution log") // The original cause is retained for callers to distinguish cancellation from endpoint failure. var ErrHarnessDescribeFailed = errors.New("controller: describe harness failed") +// ErrHarnessPrepareFailed identifies failure to prepare a harness run before its execution is +// recorded. The original cause is retained for callers to distinguish cancellation from endpoint +// failure. No Start send was attempted. +var ErrHarnessPrepareFailed = errors.New("controller: prepare harness run failed") + // ErrHarnessMismatch rejects replay/recovery with a different resolved harness name. var ErrHarnessMismatch = errors.New("controller: harness does not match recorded invocation") @@ -339,6 +344,22 @@ func (c *Controller) Exec(ctx context.Context, har api.Harness, inputs []api.Mes } } + run := har.Run + if preparer, ok := har.(api.HarnessRunPreparer); ok { + prepared, prepareErr := preparer.PrepareRun(ctx) + if prepareErr != nil { + if prepared != nil { + prepared.Close() + } + return fmt.Errorf("%w: %w", ErrHarnessPrepareFailed, prepareErr) + } + if prepared == nil { + return fmt.Errorf("%w: preparer returned nil", ErrHarnessPrepareFailed) + } + defer prepared.Close() + run = prepared.Run + } + // Record every invocation before inputs so recovery can detect partially committed turns. config := bytes.Clone(c.startConfig) last := expectedLastSeq @@ -388,7 +409,7 @@ func (c *Controller) Exec(ctx context.Context, har api.Harness, inputs []api.Mes ResumeFromSeq: c.startResumeFromSeq, } sink := &liveSink{c: c, executionID: executionID, modelDefault: model, modelPolicy: c.invocationPolicy(model), guard: &sinkGuard{}, desc: &desc, har: har, calls: make(map[string]bool), runCtx: runCtx} - runErr := har.Run(runCtx, start, sink) + runErr := run(runCtx, start, sink) stop := sink.guard.close() if stop != nil { if runErr == nil || !errors.Is(stop, api.ErrApprovalParked) { diff --git a/controller/prepare_run_test.go b/controller/prepare_run_test.go new file mode 100644 index 00000000..4bd28469 --- /dev/null +++ b/controller/prepare_run_test.go @@ -0,0 +1,186 @@ +package controller_test + +import ( + "context" + "errors" + "testing" + + "github.com/agent-primitives/sessions/api" + "github.com/agent-primitives/sessions/controller" + "github.com/agent-primitives/sessions/eventlog" +) + +type preparedRunHarness struct { + executionConfigHarness + prepareErr error + prepareValueOnErr bool + prepare, runs, closes int + beforeRun func() + runErr error +} + +func (h *preparedRunHarness) PrepareRun(context.Context) (api.PreparedHarnessRun, error) { + h.prepare++ + if h.prepareErr != nil { + if h.prepareValueOnErr { + return &testPreparedRun{h: h}, h.prepareErr + } + return nil, h.prepareErr + } + return &testPreparedRun{h: h}, nil +} + +type testPreparedRun struct { + h *preparedRunHarness + used bool + closed bool +} + +func (r *testPreparedRun) Run(ctx context.Context, start *api.Start, sink api.EventSink) error { + if r.used { + return errors.New("prepared run reused") + } + r.used = true + r.h.runs++ + if r.h.beforeRun != nil { + r.h.beforeRun() + } + if r.h.runErr != nil { + return r.h.runErr + } + return r.h.executionConfigHarness.Run(ctx, start, sink) +} + +func (r *testPreparedRun) Close() { + if !r.closed { + r.closed = true + r.h.closes++ + } +} + +func TestExecPreparesHarnessBeforeFirstAppend(t *testing.T) { + log := eventlog.AsStore(eventlog.New()) + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + h := &preparedRunHarness{} + h.beforeRun = func() { + recs, err := log.Read(1) + if err != nil { + t.Fatal(err) + } + if len(recs) != 2 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventInput { + t.Fatalf("prepared Run saw records %+v, want committed start and input", recs) + } + } + if err := c.Exec(t.Context(), h, []api.Message{msg("hello")}, 0); err != nil { + t.Fatal(err) + } + if h.prepare != 1 || h.runs != 1 || h.closes != 1 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/1/1", h.prepare, h.runs, h.closes) + } +} + +func TestExecPrepareFailureLeavesJournalUntouched(t *testing.T) { + log := eventlog.AsStore(eventlog.New()) + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + cause := errors.New("connect failed before Start") + h := &preparedRunHarness{prepareErr: cause} + err = c.Exec(t.Context(), h, []api.Message{msg("hello")}, 0) + if !errors.Is(err, controller.ErrHarnessPrepareFailed) || !errors.Is(err, cause) { + t.Fatalf("Exec=%v, want preparation failure retaining %v", err, cause) + } + if head, headErr := log.Head(); headErr != nil || head != 0 { + t.Fatalf("head=%d err=%v, want untouched journal", head, headErr) + } + if h.prepare != 1 || h.runs != 0 || h.closes != 0 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/0/0", h.prepare, h.runs, h.closes) + } +} + +func TestExecDefensivelyClosesPreparedValueReturnedWithError(t *testing.T) { + log := eventlog.AsStore(eventlog.New()) + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + cause := errors.New("prepare failed after allocation") + h := &preparedRunHarness{prepareErr: cause, prepareValueOnErr: true} + err = c.Exec(t.Context(), h, nil, 0) + if !errors.Is(err, controller.ErrHarnessPrepareFailed) || !errors.Is(err, cause) { + t.Fatalf("Exec=%v, want preparation failure retaining %v", err, cause) + } + if h.prepare != 1 || h.runs != 0 || h.closes != 1 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/0/1", h.prepare, h.runs, h.closes) + } +} + +func TestExecClosesPreparedRunWhenAppendFails(t *testing.T) { + cause := errors.New("append failed") + log := rejectAppendStore{Store: eventlog.AsStore(eventlog.New()), err: cause} + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + h := &preparedRunHarness{} + err = c.Exec(t.Context(), h, []api.Message{msg("hello")}, 0) + if !errors.Is(err, cause) || errors.Is(err, controller.ErrHarnessPrepareFailed) { + t.Fatalf("Exec=%v, want append failure %v", err, cause) + } + if h.prepare != 1 || h.runs != 0 || h.closes != 1 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/0/1", h.prepare, h.runs, h.closes) + } +} + +func TestExecClosesPreparedRunWhenInputAppendFails(t *testing.T) { + cause := errors.New("input append failed") + base := eventlog.AsStore(eventlog.New()) + log := &failInputStore{Store: base, failAt: 2, err: cause} + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + h := &preparedRunHarness{} + err = c.Exec(t.Context(), h, []api.Message{msg("one"), msg("two")}, 0) + if !errors.Is(err, cause) || errors.Is(err, controller.ErrHarnessPrepareFailed) { + t.Fatalf("Exec=%v, want input append failure %v", err, cause) + } + if h.prepare != 1 || h.runs != 0 || h.closes != 1 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/0/1", h.prepare, h.runs, h.closes) + } + recs, readErr := base.Read(1) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 2 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventInput { + t.Fatalf("records=%+v, want existing incomplete invocation", recs) + } +} + +func TestExecPreparedRunFailureRemainsDurable(t *testing.T) { + log := eventlog.AsStore(eventlog.New()) + c, err := controller.New(log, echoModel) + if err != nil { + t.Fatal(err) + } + cause := errors.New("Start send outcome unknown") + h := &preparedRunHarness{runErr: cause} + err = c.Exec(t.Context(), h, []api.Message{msg("hello")}, 0) + if !errors.Is(err, cause) || errors.Is(err, controller.ErrHarnessPrepareFailed) { + t.Fatalf("Exec=%v, want Run failure %v without preparation marker", err, cause) + } + recs, readErr := log.Read(1) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 3 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventInput || recs[2].Event.Kind != api.EventError { + t.Fatalf("records=%+v, want durable failed execution", recs) + } + if h.prepare != 1 || h.runs != 1 || h.closes != 1 { + t.Fatalf("prepare/run/close=%d/%d/%d, want 1/1/1", h.prepare, h.runs, h.closes) + } +} diff --git a/docs/harness-authoring.md b/docs/harness-authoring.md index 92834c5f..efd01e7b 100644 --- a/docs/harness-authoring.md +++ b/docs/harness-authoring.md @@ -135,6 +135,15 @@ empty/`echo` fallback. Old sessions/journals stay unchanged; see The host hands you a `Start` and an `EventSink`. +An out-of-process adapter can implement the optional Go interface `api.HarnessRunPreparer` to open +the actual transport for one `Run` before the host records a new execution. Preparation must not +invoke the harness or send `Start`; its only positive guarantee is that a preparation error happened +before delegation. The host closes an unused preparation and calls a successful prepared run at most +once, after `EXECUTION_START` and every `INPUT` are durable. `harnesswire.ClientHarness` implements +this for `Harness.Connect`. A wrapper around it must forward the optional interface to keep that +pre-journal boundary. A failed `Send(Start)` or later receive is not proof of non-delivery and stays +a durable failed execution (`EXE-45`). + ```go type Start struct { diff --git a/harnesswire/harnesswire.go b/harnesswire/harnesswire.go index 93583ab0..96535af7 100644 --- a/harnesswire/harnesswire.go +++ b/harnesswire/harnesswire.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "sync/atomic" "time" "go.opentelemetry.io/otel/attribute" @@ -445,6 +446,8 @@ type ClientHarness struct { client v1.HarnessClient } +var _ api.HarnessRunPreparer = (*ClientHarness)(nil) + // NewClientHarness wraps client as an api.Harness. func NewClientHarness(client v1.HarnessClient) *ClientHarness { return &ClientHarness{client: client} } @@ -457,19 +460,63 @@ func (h *ClientHarness) Describe(ctx context.Context) (api.Descriptor, error) { return descriptorFromProto(d), nil } +// PrepareRun opens the actual Connect stream without sending Start. A Connect failure therefore +// proves that the remote harness did not receive an execution. The returned stream is one-shot. +func (h *ClientHarness) PrepareRun(ctx context.Context) (api.PreparedHarnessRun, error) { + ctx, cancel := context.WithCancel(ctx) + stream, err := h.client.Connect(ctx) + if err != nil { + cancel() + return nil, err + } + return &preparedClientRun{stream: stream, cancel: cancel}, nil +} + +type preparedClientRun struct { + stream v1.Harness_ConnectClient + cancel context.CancelFunc + state atomic.Uint32 // 0 prepared, 1 used, 2 closed unused +} + +var _ api.PreparedHarnessRun = (*preparedClientRun)(nil) + +func (r *preparedClientRun) Close() { + r.state.CompareAndSwap(0, 2) + r.cancel() +} + +func (r *preparedClientRun) Run(ctx context.Context, start *api.Start, sink api.EventSink) error { + if !r.state.CompareAndSwap(0, 1) { + return errors.New("harnesswire: prepared Connect stream is no longer available") + } + defer r.cancel() + if err := ctx.Err(); err != nil { + return err + } + stopCancel := context.AfterFunc(ctx, r.cancel) + defer stopCancel() + return runClientStream(ctx, r.stream, r.cancel, start, sink) +} + // Run drives the remote harness over Connect, translating each emitted Event into a host-mediated // sink call and sending the result back. The sink (live or replay) is the controller's — the host // still mediates the model over the wire. func (h *ClientHarness) Run(ctx context.Context, start *api.Start, sink api.EventSink) error { - ctx, cancel := context.WithCancel(ctx) - defer cancel() // tear the Connect stream down (and the server's recv loop) when Run returns if start == nil || start.ExecutionID == "" { return errors.New("harnesswire: Start requires execution_id") } - stream, err := h.client.Connect(ctx) + prepared, err := h.PrepareRun(ctx) if err != nil { return err } + defer prepared.Close() + return prepared.Run(ctx, start, sink) +} + +func runClientStream(ctx context.Context, stream v1.Harness_ConnectClient, cancel context.CancelFunc, start *api.Start, sink api.EventSink) error { + if start == nil || start.ExecutionID == "" { + return errors.New("harnesswire: Start requires execution_id") + } if err := stream.Send(&v1.ControllerFrame{ Session: start.SessionUID, ExecutionId: start.ExecutionID, @@ -480,7 +527,7 @@ func (h *ClientHarness) Run(ctx context.Context, start *api.Start, sink api.Even for { ev, err := stream.Recv() if err == io.EOF { - return nil + return errors.New("harnesswire: stream closed before EVENT_END") } if err != nil { return err @@ -595,9 +642,14 @@ func (h *ClientHarness) Run(ctx context.Context, start *api.Start, sink api.Even case v1.EventKind_EVENT_APPROVAL_REQUEST, v1.EventKind_EVENT_APPROVAL_RESULT, v1.EventKind_EVENT_EXECUTION_START, v1.EventKind_EVENT_PARKED: return fmt.Errorf("harnesswire: harness emitted host-owned control event %s", ev.GetKind()) case v1.EventKind_EVENT_END: - err := endError(ev.GetEnd()) - awaitClose(stream, cancel) - return err + endErr := endError(ev.GetEnd()) + closeErr := awaitClose(stream, cancel) + if endErr != nil { + return endErr + } + return closeErr + default: + return fmt.Errorf("harnesswire: harness emitted unsupported event %s", ev.GetKind()) } } } @@ -610,14 +662,23 @@ const closeGrace = 2 * time.Second // harness gets to flush its telemetry before the host can suspend it, and the RPC ends with the // harness's own status rather than a host-side cancellation. A harness that never closes the // stream costs at most closeGrace. -func awaitClose(stream v1.Harness_ConnectClient, cancel context.CancelFunc) { +func awaitClose(stream v1.Harness_ConnectClient, cancel context.CancelFunc) error { _ = stream.CloseSend() - t := time.AfterFunc(closeGrace, cancel) + var timedOut atomic.Bool + t := time.AfterFunc(closeGrace, func() { + timedOut.Store(true) + cancel() + }) defer t.Stop() for { - if _, err := stream.Recv(); err != nil { - return + ev, err := stream.Recv() + if err == nil { + return fmt.Errorf("harnesswire: harness emitted %s after EVENT_END", ev.GetKind()) + } + if err == io.EOF || timedOut.Load() { + return nil } + return err } } @@ -627,11 +688,25 @@ func awaitClose(stream v1.Harness_ConnectClient, cancel context.CancelFunc) { // returned unchanged. A cancelled caller ends the turn at once. func hostFailed(ctx context.Context, stream v1.Harness_ConnectClient, cancel context.CancelFunc, err error) error { if ctx.Err() == nil { - awaitClose(stream, cancel) + drainClose(stream, cancel) } return err } +// drainClose is the host-failure counterpart to awaitClose. The host's sink error is authoritative, +// so discard the harness's failed END and any other frames while waiting for the RPC and its +// telemetry flush to finish. +func drainClose(stream v1.Harness_ConnectClient, cancel context.CancelFunc) { + _ = stream.CloseSend() + t := time.AfterFunc(closeGrace, cancel) + defer t.Stop() + for { + if _, err := stream.Recv(); err != nil { + return + } + } +} + // exchangePark accepts only the transport acknowledgement followed by actual // clean closure. Nothing received here is dispatched to the journal sink. func exchangePark(ctx context.Context, stream v1.Harness_ConnectClient, start *api.Start, ref api.ApprovalRef) error { diff --git a/harnesswire/prepare_test.go b/harnesswire/prepare_test.go new file mode 100644 index 00000000..aeec5520 --- /dev/null +++ b/harnesswire/prepare_test.go @@ -0,0 +1,258 @@ +package harnesswire + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/agent-primitives/sessions/api" + v1 "github.com/agent-primitives/sessions/api/genpb" + "google.golang.org/grpc" +) + +type prepareClient struct { + v1.HarnessClient + stream *prepareClientStream + err error + connects int + ctx context.Context +} + +func (c *prepareClient) Connect(ctx context.Context, _ ...grpc.CallOption) (grpc.BidiStreamingClient[v1.ControllerFrame, v1.Event], error) { + c.connects++ + c.ctx = ctx + if c.err != nil { + return nil, c.err + } + return c.stream, nil +} + +type prepareClientStream struct { + grpc.ClientStream + sent []*v1.ControllerFrame + events []*v1.Event + sendErr error + recvErr error + closeSend int +} + +type blockingPreparedStream struct { + grpc.ClientStream + ctx context.Context + sent chan struct{} +} + +func (s *blockingPreparedStream) Send(*v1.ControllerFrame) error { + close(s.sent) + return nil +} + +func (s *blockingPreparedStream) Recv() (*v1.Event, error) { + <-s.ctx.Done() + return nil, s.ctx.Err() +} + +func (*blockingPreparedStream) CloseSend() error { return nil } + +func (s *prepareClientStream) Send(frame *v1.ControllerFrame) error { + s.sent = append(s.sent, frame) + return s.sendErr +} + +func (s *prepareClientStream) Recv() (*v1.Event, error) { + if len(s.events) > 0 { + ev := s.events[0] + s.events = s.events[1:] + return ev, nil + } + if s.recvErr != nil { + return nil, s.recvErr + } + return nil, io.EOF +} + +func (s *prepareClientStream) CloseSend() error { + s.closeSend++ + return nil +} + +type noOpSink struct{} + +func (noOpSink) Model(context.Context, api.ModelRequest) (api.ModelResponse, error) { + return api.ModelResponse{}, nil +} +func (noOpSink) Output(context.Context, string) error { return nil } +func (noOpSink) ToolCall(context.Context, api.ToolCall) (api.ToolResult, error) { + return api.ToolResult{}, nil +} +func (noOpSink) Report(context.Context, api.ToolResult) error { return nil } +func (noOpSink) Usage(context.Context, api.Usage) error { return nil } + +func TestPrepareRunOpensAndReusesOneConnectStream(t *testing.T) { + stream := &prepareClientStream{events: []*v1.Event{{ExecutionId: "e", Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}}} + client := &prepareClient{stream: stream} + h := NewClientHarness(client) + prepared, err := h.PrepareRun(t.Context()) + if err != nil { + t.Fatal(err) + } + defer prepared.Close() + if client.connects != 1 || len(stream.sent) != 0 { + t.Fatalf("after PrepareRun: connects=%d sent=%d, want 1/0", client.connects, len(stream.sent)) + } + if err := prepared.Run(t.Context(), &api.Start{SessionUID: "s", ExecutionID: "e"}, noOpSink{}); err != nil { + t.Fatal(err) + } + if client.connects != 1 || len(stream.sent) != 1 || stream.sent[0].GetStart() == nil { + t.Fatalf("after Run: connects=%d sent=%v, want one Start on prepared stream", client.connects, stream.sent) + } + if err := prepared.Run(t.Context(), &api.Start{SessionUID: "s", ExecutionID: "e"}, noOpSink{}); err == nil { + t.Fatal("second prepared Run succeeded") + } + if client.connects != 1 || len(stream.sent) != 1 { + t.Fatalf("second Run opened or sent again: connects=%d sent=%d", client.connects, len(stream.sent)) + } +} + +func TestPrepareRunFailureSendsNothing(t *testing.T) { + cause := errors.New("connect unavailable") + client := &prepareClient{err: cause} + prepared, err := NewClientHarness(client).PrepareRun(t.Context()) + if !errors.Is(err, cause) || prepared != nil || client.connects != 1 { + t.Fatalf("PrepareRun=(%v, %v), connects=%d; want nil, %v, 1", prepared, err, client.connects, cause) + } +} + +func TestPreparedRunCloseCancelsUnusedConnect(t *testing.T) { + client := &prepareClient{stream: &prepareClientStream{}} + prepared, err := NewClientHarness(client).PrepareRun(t.Context()) + if err != nil { + t.Fatal(err) + } + prepared.Close() + prepared.Close() + select { + case <-client.ctx.Done(): + default: + t.Fatal("Close did not cancel prepared Connect") + } + if len(client.stream.sent) != 0 { + t.Fatal("Close sent a frame") + } + if err := prepared.Run(t.Context(), &api.Start{ExecutionID: "e"}, noOpSink{}); err == nil { + t.Fatal("Run after Close succeeded") + } +} + +func TestPreparedRunCloseCancelsActiveRun(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + stream := &blockingPreparedStream{ctx: ctx, sent: make(chan struct{})} + prepared := &preparedClientRun{stream: stream, cancel: cancel} + done := make(chan error, 1) + go func() { + done <- prepared.Run(ctx, &api.Start{ExecutionID: "e"}, noOpSink{}) + }() + <-stream.sent + prepared.Close() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("Run=%v, want Close cancellation", err) + } +} + +func TestPreparedRunHonorsIndependentRunCancellation(t *testing.T) { + prepareCtx, prepareCancel := context.WithCancel(context.Background()) + stream := &blockingPreparedStream{ctx: prepareCtx, sent: make(chan struct{})} + prepared := &preparedClientRun{stream: stream, cancel: prepareCancel} + runCtx, runCancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { + done <- prepared.Run(runCtx, &api.Start{ExecutionID: "e"}, noOpSink{}) + }() + <-stream.sent + runCancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("Run=%v, want Run context cancellation", err) + } +} + +func TestClientRunInvalidStartDoesNotConnect(t *testing.T) { + client := &prepareClient{stream: &prepareClientStream{}} + err := NewClientHarness(client).Run(t.Context(), &api.Start{}, noOpSink{}) + if err == nil || client.connects != 0 { + t.Fatalf("Run=%v connects=%d, want local validation before Connect", err, client.connects) + } +} + +func TestPreparedSendAndRecvFailuresAreRunFailures(t *testing.T) { + for _, tc := range []struct { + name string + stream *prepareClientStream + cause error + }{ + {name: "send", cause: errors.New("send uncertain")}, + {name: "recv", cause: errors.New("recv uncertain")}, + } { + t.Run(tc.name, func(t *testing.T) { + stream := &prepareClientStream{} + if tc.name == "send" { + stream.sendErr = tc.cause + } else { + stream.recvErr = tc.cause + } + prepared, err := NewClientHarness(&prepareClient{stream: stream}).PrepareRun(t.Context()) + if err != nil { + t.Fatal(err) + } + defer prepared.Close() + err = prepared.Run(t.Context(), &api.Start{ExecutionID: "e"}, noOpSink{}) + if !errors.Is(err, tc.cause) { + t.Fatalf("Run=%v, want %v", err, tc.cause) + } + }) + } +} + +func TestPreparedRunRequiresEndAndRejectsUnsupportedEvents(t *testing.T) { + for _, tc := range []struct { + name string + events []*v1.Event + }{ + {name: "EOF before END"}, + {name: "host-owned error", events: []*v1.Event{{ExecutionId: "e", Kind: v1.EventKind_EVENT_ERROR, Body: &v1.Event_Error{Error: &v1.Error{Description: "peer error"}}}}}, + {name: "unspecified", events: []*v1.Event{{ExecutionId: "e", Kind: v1.EventKind_EVENT_KIND_UNSPECIFIED}}}, + } { + t.Run(tc.name, func(t *testing.T) { + stream := &prepareClientStream{events: tc.events} + prepared, err := NewClientHarness(&prepareClient{stream: stream}).PrepareRun(t.Context()) + if err != nil { + t.Fatal(err) + } + defer prepared.Close() + if err := prepared.Run(t.Context(), &api.Start{ExecutionID: "e"}, noOpSink{}); err == nil { + t.Fatal("Run accepted stream without EVENT_END") + } + }) + } +} + +func TestPreparedRunRejectsFramesAfterEnd(t *testing.T) { + for _, ev := range []*v1.Event{ + {ExecutionId: "e", Kind: v1.EventKind_EVENT_ERROR, Body: &v1.Event_Error{Error: &v1.Error{Description: "peer error"}}}, + {ExecutionId: "e", Kind: v1.EventKind_EVENT_KIND_UNSPECIFIED}, + {ExecutionId: "e", Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}, + } { + stream := &prepareClientStream{events: []*v1.Event{ + {ExecutionId: "e", Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}, + ev, + }} + prepared, err := NewClientHarness(&prepareClient{stream: stream}).PrepareRun(t.Context()) + if err != nil { + t.Fatal(err) + } + if err := prepared.Run(t.Context(), &api.Start{ExecutionID: "e"}, noOpSink{}); err == nil { + t.Fatalf("Run accepted %s after EVENT_END", ev.GetKind()) + } + prepared.Close() + } +} diff --git a/placement/describe_boundary_test.go b/placement/describe_boundary_test.go index 682aadae..82dd72e1 100644 --- a/placement/describe_boundary_test.go +++ b/placement/describe_boundary_test.go @@ -29,6 +29,15 @@ func (h describeBoundaryHarness) Describe(ctx context.Context) (api.Descriptor, return h.describe(ctx) } +type prepareBoundaryHarness struct { + api.Harness + prepare func(context.Context) (api.PreparedHarnessRun, error) +} + +func (h prepareBoundaryHarness) PrepareRun(ctx context.Context) (api.PreparedHarnessRun, error) { + return h.prepare(ctx) +} + func TestControllerDescribeBoundaryUsesAdmissionClassification(t *testing.T) { for _, operation := range []string{"Exec", "Resume"} { for _, tc := range []struct { @@ -119,3 +128,59 @@ func TestControllerDescribeBoundaryUsesAdmissionClassification(t *testing.T) { } } } + +func TestControllerPrepareBoundaryUsesAdmissionClassification(t *testing.T) { + for _, tc := range []struct { + name string + peerErr error + cancel bool + want error + wantCause error + }{ + {name: "unavailable", peerErr: status.Error(codes.Unavailable, "peer unavailable"), want: placement.ErrHarnessUnavailable}, + {name: "peer deadline", peerErr: status.Error(codes.DeadlineExceeded, "peer deadline"), want: placement.ErrHarnessUnavailable}, + {name: "plain deadline", peerErr: context.DeadlineExceeded, want: placement.ErrHarnessUnavailable}, + {name: "caller cancelled", peerErr: status.Error(codes.Unavailable, "peer unavailable"), cancel: true, want: placement.ErrAdmissionInterrupted, wantCause: context.Canceled}, + {name: "internal", peerErr: status.Error(codes.Internal, "peer internal"), want: controller.ErrHarnessPrepareFailed}, + } { + t.Run(tc.name, func(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + defer store.Close() + log := store.Session("prepare-boundary") + base := &identityToolHarness{version: "v1"} + desc, err := base.Describe(t.Context()) + if err != nil { + t.Fatal(err) + } + backend := &routedBackend{desc: desc} + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var closes atomic.Int32 + h := prepareBoundaryHarness{Harness: base, prepare: func(context.Context) (api.PreparedHarnessRun, error) { + if tc.cancel { + cancel() + } + return nil, tc.peerErr + }} + p := placement.New(backend, nil, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return h, func() error { closes.Add(1); return nil }, nil + })) + _, err = p.Exec(ctx, log, "prepare-boundary", nil, 0) + if !errors.Is(err, tc.want) || tc.wantCause != nil && !errors.Is(err, tc.wantCause) { + t.Fatalf("error=%v want=%v cause=%v", err, tc.want, tc.wantCause) + } + if tc.want == controller.ErrHarnessPrepareFailed && (errors.Is(err, placement.ErrHarnessUnavailable) || errors.Is(err, placement.ErrAdmissionInterrupted)) { + t.Fatalf("non-outage preparation error misclassified: %v", err) + } + if base.runs.Load() != 0 || closes.Load() != 1 { + t.Fatalf("preparation failure ran harness or leaked connection: runs=%d closes=%d", base.runs.Load(), closes.Load()) + } + if head, headErr := log.Head(); headErr != nil || head != 0 { + t.Fatalf("head=%d err=%v, want no execution records", head, headErr) + } + }) + } +} diff --git a/placement/placement.go b/placement/placement.go index 78d2df4a..0eaa7230 100644 --- a/placement/placement.go +++ b/placement/placement.go @@ -52,8 +52,9 @@ var ErrSessionBusy = errors.New("placement: another operation is in progress for var ErrDescriptorMismatch = errors.New("placement: harness is not the expected one") // ErrHarnessUnavailable is returned when the harness could not be reached to describe itself during -// admission or the controller's connected-harness identity check. Neither check runs the harness or -// appends journal records; the connected check may follow compute allocation and fence minting. +// admission or the controller's connected-harness identity check, or to prepare its Connect stream +// before a new execution. None runs the harness or appends execution records; the connected checks +// may follow compute allocation and fence minting. // session.Service surfaces it as codes.Unavailable. If the caller's own deadline or cancellation // ended the check, it is ErrAdmissionInterrupted instead: the same expired call cannot succeed. var ErrHarnessUnavailable = errors.New("placement: harness unavailable") @@ -166,7 +167,9 @@ func (s *sessionSet) busy() bool { // the connection. runtime/local passes a unix-socket address (unix://…); substrate passes the // atenet-router's host:port, dialed over h2c, plus CallMetadata naming the actor, which the client // must attach to every call. The default dialer handles both forms; WithDialer overrides it (tests). -// This is the one transport seam the harness rides unchanged. +// The closer may be called while PrepareRun or Run is blocked; it must interrupt the transport and +// return promptly rather than wait for that call to finish. This is the one transport seam the +// harness rides unchanged. type Dialer func(inc api.Incarnation) (api.Harness, func() error, error) // Option configures a Placer. @@ -645,13 +648,18 @@ func (p *Placer) refuseOverrideRepair(ctx context.Context, sessionUID string, ov return refuseStatefulOverride(desc, override) } -// controllerError maps only the controller's Describe boundary. Run, model and runtime errors -// with identical transport codes must keep their existing classification. +// controllerError maps only the controller's pre-execution Describe and Connect-preparation +// boundaries. Run, model and runtime errors with identical transport codes keep their existing +// classification because Start delivery may already be uncertain. func controllerError(ctx context.Context, err error) error { - if !errors.Is(err, controller.ErrHarnessDescribeFailed) { + switch { + case errors.Is(err, controller.ErrHarnessDescribeFailed): + return describeError(ctx, err, errors.Is(err, context.DeadlineExceeded)) + case errors.Is(err, controller.ErrHarnessPrepareFailed): + return prepareHarnessError(ctx, err) + default: return err } - return describeError(ctx, err, errors.Is(err, context.DeadlineExceeded)) } // describeError is shared by admission and connected-harness identity checks. @@ -662,13 +670,31 @@ func describeError(ctx context.Context, err error, timedOut bool) error { // context error cannot change the service's cancellation/deadline classification. return fmt.Errorf("%w: describe harness: %w: %v", ErrAdmissionInterrupted, ctx.Err(), err) case timedOut || c == codes.Unavailable || c == codes.DeadlineExceeded: - // A refused connection or a harness timeout while the caller is live is a retryable outage. return fmt.Errorf("%w: %w", ErrHarnessUnavailable, err) default: return err } } +func prepareHarnessError(ctx context.Context, err error) error { + if ctx.Err() != nil { + return &CallerContextError{CallerErr: ctx.Err(), Cause: err} + } + if c := status.Code(err); c == codes.Unavailable || c == codes.DeadlineExceeded || errors.Is(err, context.DeadlineExceeded) { + // Connect setup can name a private endpoint or carry credential-provider diagnostics. Keep the + // cause for direct Go callers but publish only the fixed host-owned outage text. + return &harnessUnavailableError{cause: err} + } + return err +} + +type harnessUnavailableError struct{ cause error } + +func (*harnessUnavailableError) Error() string { return ErrHarnessUnavailable.Error() } +func (e *harnessUnavailableError) Unwrap() []error { + return []error{ErrHarnessUnavailable, e.cause} +} + // DefaultDial reaches a harnesswire server by address form over plaintext: a unix-socket address // (unix://…, runtime/local) or a TCP host:port. It attaches the incarnation's CallMetadata to every // call. A backend whose harnesses need transport security (substrate's atenet-router serves only diff --git a/placement/transport_test.go b/placement/transport_test.go index ea1f0939..857bef4b 100644 --- a/placement/transport_test.go +++ b/placement/transport_test.go @@ -393,6 +393,65 @@ func TestForkEndsTheParentsOpenHarnessStreamBeforeTheCheckpoint(t *testing.T) { testCheckpointEndsOpenStream(t, forkCheckpoint) } +type blockedPrepareHarness struct { + api.Harness + started chan struct{} + done chan struct{} +} + +func (h *blockedPrepareHarness) PrepareRun(ctx context.Context) (api.PreparedHarnessRun, error) { + close(h.started) + <-ctx.Done() + close(h.done) + return nil, ctx.Err() +} + +func TestForkCancelsPreparedHarnessBeforeExecutionIsRecorded(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + defer store.Close() + log := store.Session("s1") + h := &blockedPrepareHarness{Harness: echoagent.Harness{}, started: make(chan struct{}), done: make(chan struct{})} + backend := &routedBackend{desc: memDescriptor} + p := placement.New(backend, echoagent.Model, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return h, func() error { return nil }, nil + })) + + execErr := make(chan error, 1) + go func() { + _, err := p.Exec(context.Background(), log, "s1", []api.Message{*api.TextMessage("user", "hi")}, 0) + execErr <- err + }() + select { + case <-h.started: + case <-time.After(10 * time.Second): + t.Fatal("turn never began Connect preparation") + } + + if err := forkCheckpoint(p, log, "s1"); err != nil { + t.Fatalf("checkpoint: %v", err) + } + select { + case <-h.done: + case <-time.After(10 * time.Second): + t.Fatal("checkpoint did not cancel prepared Connect") + } + select { + case err := <-execErr: + if !errors.Is(err, placement.ErrCheckpointing) || !errors.Is(err, eventlog.ErrFenced) { + t.Fatalf("turn returned %v, want ErrCheckpointing and ErrFenced", err) + } + case <-time.After(10 * time.Second): + t.Fatal("superseded turn did not return") + } + kinds := journalKinds(t, log) + if len(kinds) != 1 || kinds[0] != api.EventLifecycle { + t.Fatalf("journal=%v, want only checkpoint lifecycle record", kinds) + } +} + // The fork must end the turn's stream whichever Placer of the Registry runs the turn. func TestForkEndsAParentStreamOpenedByAnotherPlacerOfTheRegistry(t *testing.T) { testCheckpointEndsOpenStream(t, forkCheckpoint, registryPair) diff --git a/session/error_connect_test.go b/session/error_connect_test.go new file mode 100644 index 00000000..ed8f937f --- /dev/null +++ b/session/error_connect_test.go @@ -0,0 +1,249 @@ +package session_test + +import ( + "context" + "errors" + "io" + "strings" + "sync/atomic" + "testing" + + "github.com/agent-primitives/sessions/api" + v1 "github.com/agent-primitives/sessions/api/genpb" + "github.com/agent-primitives/sessions/harness/echoagent" + "github.com/agent-primitives/sessions/harnesswire" + "github.com/agent-primitives/sessions/placement" + "github.com/agent-primitives/sessions/runtime/local" + "github.com/agent-primitives/sessions/session" + "github.com/agent-primitives/sessions/sqlitelog" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type retryConnectClient struct { + v1.HarnessClient + connects atomic.Int32 + starts []*v1.Start + firstErr error +} + +func (c *retryConnectClient) Describe(ctx context.Context, _ *v1.DescribeRequest, _ ...grpc.CallOption) (*v1.HarnessDescriptor, error) { + desc, err := (echoagent.Harness{}).Describe(ctx) + if err != nil { + return nil, err + } + return harnesswire.DescriptorToProto(desc), nil +} + +func (c *retryConnectClient) Connect(context.Context, ...grpc.CallOption) (grpc.BidiStreamingClient[v1.ControllerFrame, v1.Event], error) { + if c.connects.Add(1) == 1 { + return nil, c.firstErr + } + return &retryConnectStream{client: c}, nil +} + +type retryConnectStream struct { + grpc.ClientStream + client *retryConnectClient + executionID string + ended bool +} + +func (s *retryConnectStream) Send(frame *v1.ControllerFrame) error { + start := frame.GetStart() + if start == nil { + return errors.New("expected Start") + } + s.executionID = frame.GetExecutionId() + s.client.starts = append(s.client.starts, start) + return nil +} + +func (s *retryConnectStream) Recv() (*v1.Event, error) { + if s.ended { + return nil, io.EOF + } + s.ended = true + return &v1.Event{ExecutionId: s.executionID, Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}, nil +} + +func (*retryConnectStream) CloseSend() error { return nil } + +type postEndClient struct{ v1.HarnessClient } + +func (postEndClient) Describe(ctx context.Context, _ *v1.DescribeRequest, _ ...grpc.CallOption) (*v1.HarnessDescriptor, error) { + desc, err := (echoagent.Harness{}).Describe(ctx) + if err != nil { + return nil, err + } + return harnesswire.DescriptorToProto(desc), nil +} + +func (postEndClient) Connect(context.Context, ...grpc.CallOption) (grpc.BidiStreamingClient[v1.ControllerFrame, v1.Event], error) { + return &postEndStream{}, nil +} + +type postEndStream struct { + grpc.ClientStream + executionID string + step int +} + +func (s *postEndStream) Send(frame *v1.ControllerFrame) error { + s.executionID = frame.GetExecutionId() + return nil +} + +func (s *postEndStream) Recv() (*v1.Event, error) { + s.step++ + switch s.step { + case 1: + return &v1.Event{ExecutionId: s.executionID, Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}, nil + case 2: + return &v1.Event{ExecutionId: s.executionID, Kind: v1.EventKind_EVENT_ERROR, Body: &v1.Event_Error{Error: &v1.Error{Description: "contradictory"}}}, nil + default: + return nil, io.EOF + } +} + +func (*postEndStream) CloseSend() error { return nil } + +func TestExecConnectPreparationFailureCanRetryFirstTurn(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + backend := local.New(echoagent.Harness{}) + t.Cleanup(func() { _ = backend.Close() }) + const private = "https://private.invalid/?token=connect-secret" + cause := status.Error(codes.Unavailable, private) + client := &retryConnectClient{firstErr: cause} + p := placement.New(backend, echoagent.Model, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return harnesswire.NewClientHarness(client), func() error { return nil }, nil + })) + registry, err := placement.NewRegistry("echo", map[string]*placement.Placer{"echo": p}) + if err != nil { + t.Fatal(err) + } + svc := session.NewService(store, registry) + created, err := svc.CreateSession(t.Context(), &v1.CreateSessionRequest{}) + if err != nil { + t.Fatal(err) + } + uid := created.GetMetadata().GetUid() + req := &v1.ExecRequest{Session: uid, Inputs: []*v1.Message{{Role: "user", Parts: []*v1.Part{{Part: &v1.Part_Text{Text: &v1.TextPart{Text: "hi"}}}}}}} + + err = svc.Exec(req, &rpcStream{ctx: t.Context()}) + if status.Code(err) != codes.Unavailable || !errors.Is(err, placement.ErrHarnessUnavailable) || !errors.Is(err, cause) { + t.Fatalf("first Exec=%v, want UNAVAILABLE/HARNESS_UNAVAILABLE", err) + } + if strings.Contains(err.Error(), private) { + t.Fatalf("Connect diagnostic leaked into public status: %v", err) + } + details := status.Convert(err).Details() + if len(details) != 1 { + t.Fatalf("details=%v, want HARNESS_UNAVAILABLE", details) + } + info, ok := details[0].(*errdetails.ErrorInfo) + if !ok || info.Domain != "agentsessions" || info.Reason != "HARNESS_UNAVAILABLE" || len(info.Metadata) != 0 { + t.Fatalf("details=%v, want agentsessions/HARNESS_UNAVAILABLE", details) + } + if head, headErr := store.Session(uid).Head(); headErr != nil || head != 0 { + t.Fatalf("head=%d err=%v, want no failed execution", head, headErr) + } + + if err := svc.Exec(req, &rpcStream{ctx: t.Context()}); err != nil { + t.Fatalf("retry Exec=%v", err) + } + if client.connects.Load() != 2 || len(client.starts) != 1 || len(client.starts[0].GetHistory()) != 0 { + t.Fatalf("connects=%d starts=%v, want one failed Connect and one first-turn Start", client.connects.Load(), client.starts) + } +} + +type unavailableRunHarness struct{ echoagent.Harness } + +func (h unavailableRunHarness) PrepareRun(context.Context) (api.PreparedHarnessRun, error) { + return unavailablePreparedRun{}, nil +} + +type unavailablePreparedRun struct{} + +func (unavailablePreparedRun) Run(context.Context, *api.Start, api.EventSink) error { + return status.Error(codes.Unavailable, "delivery outcome unknown") +} + +func (unavailablePreparedRun) Close() {} + +func TestExecPreparedRunFailureRemainsDurableAndUnclassified(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + backend := local.New(echoagent.Harness{}) + t.Cleanup(func() { _ = backend.Close() }) + p := placement.New(backend, echoagent.Model, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return unavailableRunHarness{}, func() error { return nil }, nil + })) + registry, err := placement.NewRegistry("echo", map[string]*placement.Placer{"echo": p}) + if err != nil { + t.Fatal(err) + } + svc := session.NewService(store, registry) + created, err := svc.CreateSession(t.Context(), &v1.CreateSessionRequest{}) + if err != nil { + t.Fatal(err) + } + uid := created.GetMetadata().GetUid() + err = svc.Exec(&v1.ExecRequest{Session: uid}, &rpcStream{ctx: t.Context()}) + if status.Code(err) != codes.Internal || errors.Is(err, placement.ErrHarnessUnavailable) { + t.Fatalf("Exec=%v, want INTERNAL without HARNESS_UNAVAILABLE", err) + } + if len(status.Convert(err).Details()) != 0 { + t.Fatalf("uncertain Run failure acquired ErrorInfo: %v", status.Convert(err).Details()) + } + recs, readErr := store.Session(uid).Read(1) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 2 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventError { + t.Fatalf("records=%+v, want durable start and error", recs) + } +} + +func TestExecFrameAfterEndIsDurableProtocolFailure(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + backend := local.New(echoagent.Harness{}) + t.Cleanup(func() { _ = backend.Close() }) + p := placement.New(backend, echoagent.Model, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return harnesswire.NewClientHarness(postEndClient{}), func() error { return nil }, nil + })) + registry, err := placement.NewRegistry("echo", map[string]*placement.Placer{"echo": p}) + if err != nil { + t.Fatal(err) + } + svc := session.NewService(store, registry) + created, err := svc.CreateSession(t.Context(), &v1.CreateSessionRequest{}) + if err != nil { + t.Fatal(err) + } + uid := created.GetMetadata().GetUid() + err = svc.Exec(&v1.ExecRequest{Session: uid}, &rpcStream{ctx: t.Context()}) + if status.Code(err) != codes.Internal { + t.Fatalf("Exec=%v, want INTERNAL protocol failure", err) + } + recs, readErr := store.Session(uid).Read(1) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 2 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventError { + t.Fatalf("records=%+v, want durable start and error without END", recs) + } +} diff --git a/spec.md b/spec.md index e27b63e6..fc057abd 100644 --- a/spec.md +++ b/spec.md @@ -641,6 +641,13 @@ reads the catalog that registration writes; the reference daemon does. - `EXE-7`: `Start.history` for a new execution MUST be every event before its `EXECUTION_START`, with `ApprovalResult.verified_approver` left empty (`APR-8`). The reference host always sends the full prior log, including after a memory restore. +- `EXE-45`: Before recording a new execution for a harness that supports run preparation, the host + MUST prepare the actual Run without invoking the harness or sending `Start`. A preparation failure + appends no execution records, so the caller may retry `Exec` at the same journal cursor. The + successful preparation MUST be used for that execution exactly once; only after + `EXECUTION_START` and every `INPUT` commit may the host send `Start`. Once sending `Start` is + attempted, a send or receive failure does not prove non-delivery: the execution remains durable + and visible in later history under `EXE-7` and `EXE-17`. ### 8.2 Effects @@ -903,10 +910,12 @@ cannot observe a model call that does not reach it. The class concerns model cal re-prompts. Once the compute that held the memory is gone (`RT-4e`), or replaced on a backend that records `ComputeInstance` (`RT-4h`, for example `runtime/substrate`), every later `Exec` and `Resume` on the session fails `FAILED_PRECONDITION`, so the re-prompt goes to a new session; the - failed session's log stays readable. When the compute and its memory survive, for example when - only the host or the harness stream failed, the re-prompt is an `Exec` on the same session. An - unmediated harness with no state for a session whose history records an earlier execution MUST - fail the run as possible state loss rather than start an empty conversation. + failed session's log stays readable. When the compute and its memory are known to survive, or a + prepared harness Run failed before `Start` was sent (`EXE-45`), the re-prompt is an `Exec` on the + same session. A stream failure after sending begins does not prove that memory survived or that + the harness did no work. An unmediated harness with no state for a session whose history records + an earlier execution MUST fail the run as possible state loss rather than start an empty + conversation. - `EXE-44`: Wrapper delegation. When a host-side wrapper performs work before delegating to an unmediated harness, a recorded execution that the wrapper positively knows was not delegated MUST be omitted in full from `Start.history` on later calls to that underlying harness. This is a @@ -1507,7 +1516,8 @@ does not erase data or reclaim compute. | Harness `Describe` (admission `RT-1`, the `RT-3` recheck, or a `Describe` during the turn) could not reach the harness while the caller's context is live: the peer returned `UNAVAILABLE` or `DEADLINE_EXCEEDED`; or, on admission and the recheck, the host's own bound expired; or, during the turn, the error wraps `context.DeadlineExceeded` | `UNAVAILABLE` | | Any other `Describe` error on those paths, for example a peer `UNIMPLEMENTED` or `INTERNAL`, or a plain error from an in-process harness (during the turn, even one returned after the host's bound expired) | `INTERNAL` | | Any `Describe` error in the stored harness's half of the memory-tier override check (`OPS-7`) while the caller's context is live | `UNAVAILABLE` | -| Caller deadline or cancellation that ends admission, compute allocation/start, any `Describe` above, a `Suspend` or `Fork` runtime operation, or `DeleteSession`, unless a more specific lifecycle refusal above applies | `DEADLINE_EXCEEDED` or `CANCELLED` | +| Preparing `Harness.Connect` for a new `Exec` fails with `UNAVAILABLE` or `DEADLINE_EXCEEDED` while the caller is live (`EXE-45`) | `UNAVAILABLE` | +| Caller deadline or cancellation that ends admission, compute allocation/start, any `Describe` above, Connect preparation, a `Suspend` or `Fork` runtime operation, or `DeleteSession`, unless a more specific lifecycle refusal above applies | `DEADLINE_EXCEEDED` or `CANCELLED` | | Journal hash chain fails verification on `Exec`, `Resume` (including an `APR-13` park or request repair) or `Fork` (`CHAIN-12`) | `DATA_LOSS` | | `Cancel` | `UNIMPLEMENTED` | | Any other failure | `INTERNAL` | @@ -1618,10 +1628,9 @@ sequenceDiagram seconds for the harness to end the RPC before cancelling it, so the harness can flush telemetry and return its own status. It waits the same way when a sink call fails on the host side, unless the caller has already cancelled; the turn still fails with the host's error. After - `EVENT_PARKED` it already waits for the harness to close (`APR-10`). -- TODO(spike): The reference host client treats a stream that closes cleanly without `EVENT_END` - as success, and silently ignores harness-sent `EVENT_INPUT`, `EVENT_LIFECYCLE`, `EVENT_ERROR`, - and unspecified kinds. Should these be protocol errors, as the proto comment implies? + `EVENT_PARKED` it already waits for the harness to close (`APR-10`). A stream that closes cleanly + before `EVENT_END`, or sends a host-owned or unsupported event kind, fails the execution as a + protocol error; it is never inferred as successful completion. - `WIRE-13`: Harness dials in the reference host use no transport security (§17). ### 12.3 Wire conversion @@ -2435,7 +2444,6 @@ Each item is also marked inline where it applies. - TODO(spike): Repeated `Suspend` and `Resume` without `Suspend` (§11.5). - TODO(spike): Fork journal atomicity and `at_seq > head` (§11.7). - TODO(spike): Unknown `Resumability` numbers read as `STATELESS_REPLAY` (§12.1). -- TODO(spike): Stream close without `EVENT_END` and ignored harness event kinds (§12.2). - TODO(spike): Session check and stickiness for model reply correlation failures (§12.2). - TODO(spike): Black-box conformance packaging (§16.2). - TODO(spike): End-to-end egress credential injection and policy enforcement for an unmediated From 58d4147031922747c062a9ade4b73464dd456e20 Mon Sep 17 00:00:00 2001 From: Tatsat Mishra Date: Tue, 17 Nov 2026 18:59:10 +1300 Subject: [PATCH 2/5] fix(harnesswire): cancel sink calls on close --- harnesswire/harnesswire.go | 16 +++++++++++---- harnesswire/prepare_test.go | 41 +++++++++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/harnesswire/harnesswire.go b/harnesswire/harnesswire.go index 96535af7..3fe93014 100644 --- a/harnesswire/harnesswire.go +++ b/harnesswire/harnesswire.go @@ -469,11 +469,12 @@ func (h *ClientHarness) PrepareRun(ctx context.Context) (api.PreparedHarnessRun, cancel() return nil, err } - return &preparedClientRun{stream: stream, cancel: cancel}, nil + return &preparedClientRun{stream: stream, ctx: ctx, cancel: cancel}, nil } type preparedClientRun struct { stream v1.Harness_ConnectClient + ctx context.Context cancel context.CancelFunc state atomic.Uint32 // 0 prepared, 1 used, 2 closed unused } @@ -490,12 +491,19 @@ func (r *preparedClientRun) Run(ctx context.Context, start *api.Start, sink api. return errors.New("harnesswire: prepared Connect stream is no longer available") } defer r.cancel() - if err := ctx.Err(); err != nil { + runCtx, runCancel := context.WithCancel(ctx) + defer runCancel() + stopPrepared := context.AfterFunc(r.ctx, runCancel) + defer stopPrepared() + if err := r.ctx.Err(); err != nil { + return err + } + if err := runCtx.Err(); err != nil { return err } - stopCancel := context.AfterFunc(ctx, r.cancel) + stopCancel := context.AfterFunc(runCtx, r.cancel) defer stopCancel() - return runClientStream(ctx, r.stream, r.cancel, start, sink) + return runClientStream(runCtx, r.stream, r.cancel, start, sink) } // Run drives the remote harness over Connect, translating each emitted Event into a host-mediated diff --git a/harnesswire/prepare_test.go b/harnesswire/prepare_test.go index aeec5520..01794dfe 100644 --- a/harnesswire/prepare_test.go +++ b/harnesswire/prepare_test.go @@ -5,6 +5,7 @@ import ( "errors" "io" "testing" + "time" "github.com/agent-primitives/sessions/api" v1 "github.com/agent-primitives/sessions/api/genpb" @@ -89,6 +90,17 @@ func (noOpSink) ToolCall(context.Context, api.ToolCall) (api.ToolResult, error) func (noOpSink) Report(context.Context, api.ToolResult) error { return nil } func (noOpSink) Usage(context.Context, api.Usage) error { return nil } +type blockingModelSink struct { + noOpSink + started chan struct{} +} + +func (s blockingModelSink) Model(ctx context.Context, _ api.ModelRequest) (api.ModelResponse, error) { + close(s.started) + <-ctx.Done() + return api.ModelResponse{}, ctx.Err() +} + func TestPrepareRunOpensAndReusesOneConnectStream(t *testing.T) { stream := &prepareClientStream{events: []*v1.Event{{ExecutionId: "e", Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}}} client := &prepareClient{stream: stream} @@ -148,7 +160,7 @@ func TestPreparedRunCloseCancelsUnusedConnect(t *testing.T) { func TestPreparedRunCloseCancelsActiveRun(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) stream := &blockingPreparedStream{ctx: ctx, sent: make(chan struct{})} - prepared := &preparedClientRun{stream: stream, cancel: cancel} + prepared := &preparedClientRun{stream: stream, ctx: ctx, cancel: cancel} done := make(chan error, 1) go func() { done <- prepared.Run(ctx, &api.Start{ExecutionID: "e"}, noOpSink{}) @@ -163,7 +175,7 @@ func TestPreparedRunCloseCancelsActiveRun(t *testing.T) { func TestPreparedRunHonorsIndependentRunCancellation(t *testing.T) { prepareCtx, prepareCancel := context.WithCancel(context.Background()) stream := &blockingPreparedStream{ctx: prepareCtx, sent: make(chan struct{})} - prepared := &preparedClientRun{stream: stream, cancel: prepareCancel} + prepared := &preparedClientRun{stream: stream, ctx: prepareCtx, cancel: prepareCancel} runCtx, runCancel := context.WithCancel(t.Context()) done := make(chan error, 1) go func() { @@ -176,6 +188,31 @@ func TestPreparedRunHonorsIndependentRunCancellation(t *testing.T) { } } +func TestPreparedRunCloseCancelsBlockedSinkWithIndependentContext(t *testing.T) { + prepareCtx, prepareCancel := context.WithCancel(context.Background()) + stream := &prepareClientStream{events: []*v1.Event{{ + ExecutionId: "e", + Kind: v1.EventKind_EVENT_MODEL_CALL, + Body: &v1.Event_Model{Model: &v1.ModelCall{Id: "m", Model: "model"}}, + }}} + prepared := &preparedClientRun{stream: stream, ctx: prepareCtx, cancel: prepareCancel} + sink := blockingModelSink{started: make(chan struct{})} + done := make(chan error, 1) + go func() { + done <- prepared.Run(context.Background(), &api.Start{ExecutionID: "e"}, sink) + }() + <-sink.started + prepared.Close() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("Run=%v, want Close cancellation", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Close did not cancel blocked sink call") + } +} + func TestClientRunInvalidStartDoesNotConnect(t *testing.T) { client := &prepareClient{stream: &prepareClientStream{}} err := NewClientHarness(client).Run(t.Context(), &api.Start{}, noOpSink{}) From 90ab5fac0a09b0a85cc5a41f600a0c82efd639cb Mon Sep 17 00:00:00 2001 From: Tatsat Mishra Date: Tue, 17 Nov 2026 19:05:16 +1300 Subject: [PATCH 3/5] fix(harnesswire): preserve completion during close --- .../unreleased/harness-connect-preparation.md | 3 +- docs/harness-authoring.md | 4 +- harnesswire/harnesswire.go | 45 ++++++++--- harnesswire/prepare_test.go | 25 ++++++ session/error_connect_test.go | 81 +++++++++++++++++++ spec.md | 4 +- 6 files changed, 146 insertions(+), 16 deletions(-) diff --git a/changelog/unreleased/harness-connect-preparation.md b/changelog/unreleased/harness-connect-preparation.md index 379b496a..452de6b5 100644 --- a/changelog/unreleased/harness-connect-preparation.md +++ b/changelog/unreleased/harness-connect-preparation.md @@ -8,4 +8,5 @@ category: Fixed every input are durable; send and receive failures remain uncertain and keep the attempted execution in later history. A clean close without `EVENT_END`, or a host-owned or unsupported harness event, is now a failed execution rather than implicit success. Go harness adapters can opt - into the same boundary with `api.HarnessRunPreparer` (`EXE-45`). + into the same boundary with `api.HarnessRunPreparer` (`EXE-45`). Caller cancellation during the + bounded post-END telemetry flush does not replace a completed harness result. diff --git a/docs/harness-authoring.md b/docs/harness-authoring.md index efd01e7b..d25d9ac5 100644 --- a/docs/harness-authoring.md +++ b/docs/harness-authoring.md @@ -142,7 +142,9 @@ before delegation. The host closes an unused preparation and calls a successful once, after `EXECUTION_START` and every `INPUT` are durable. `harnesswire.ClientHarness` implements this for `Harness.Connect`. A wrapper around it must forward the optional interface to keep that pre-journal boundary. A failed `Send(Start)` or later receive is not proof of non-delivery and stays -a durable failed execution (`EXE-45`). +a durable failed execution (`EXE-45`). In particular, successfully opening a client stream does not +prove that a router or harness will accept `Start`; a refusal surfaced by Send or Recv remains +uncertain and is not filtered from later history. ```go diff --git a/harnesswire/harnesswire.go b/harnesswire/harnesswire.go index 3fe93014..3d119a50 100644 --- a/harnesswire/harnesswire.go +++ b/harnesswire/harnesswire.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "sync" "sync/atomic" "time" @@ -473,27 +474,45 @@ func (h *ClientHarness) PrepareRun(ctx context.Context) (api.PreparedHarnessRun, } type preparedClientRun struct { - stream v1.Harness_ConnectClient - ctx context.Context - cancel context.CancelFunc - state atomic.Uint32 // 0 prepared, 1 used, 2 closed unused + stream v1.Harness_ConnectClient + ctx context.Context + cancel context.CancelFunc + mu sync.Mutex + state uint8 // 0 prepared, 1 used, 2 closed unused + runCancel context.CancelFunc } var _ api.PreparedHarnessRun = (*preparedClientRun)(nil) func (r *preparedClientRun) Close() { - r.state.CompareAndSwap(0, 2) + r.mu.Lock() + if r.state == 0 { + r.state = 2 + } + runCancel := r.runCancel + r.mu.Unlock() + if runCancel != nil { + runCancel() + } r.cancel() } func (r *preparedClientRun) Run(ctx context.Context, start *api.Start, sink api.EventSink) error { - if !r.state.CompareAndSwap(0, 1) { + r.mu.Lock() + if r.state != 0 { + r.mu.Unlock() return errors.New("harnesswire: prepared Connect stream is no longer available") } - defer r.cancel() runCtx, runCancel := context.WithCancel(ctx) + r.state = 1 + r.runCancel = runCancel + r.mu.Unlock() defer runCancel() - stopPrepared := context.AfterFunc(r.ctx, runCancel) + defer r.cancel() + stopPrepared := context.AfterFunc(r.ctx, func() { + runCancel() + r.cancel() + }) defer stopPrepared() if err := r.ctx.Err(); err != nil { return err @@ -503,7 +522,7 @@ func (r *preparedClientRun) Run(ctx context.Context, start *api.Start, sink api. } stopCancel := context.AfterFunc(runCtx, r.cancel) defer stopCancel() - return runClientStream(runCtx, r.stream, r.cancel, start, sink) + return runClientStream(runCtx, r.ctx, r.stream, r.cancel, start, sink) } // Run drives the remote harness over Connect, translating each emitted Event into a host-mediated @@ -521,7 +540,7 @@ func (h *ClientHarness) Run(ctx context.Context, start *api.Start, sink api.Even return prepared.Run(ctx, start, sink) } -func runClientStream(ctx context.Context, stream v1.Harness_ConnectClient, cancel context.CancelFunc, start *api.Start, sink api.EventSink) error { +func runClientStream(ctx, streamCtx context.Context, stream v1.Harness_ConnectClient, cancel context.CancelFunc, start *api.Start, sink api.EventSink) error { if start == nil || start.ExecutionID == "" { return errors.New("harnesswire: Start requires execution_id") } @@ -651,7 +670,7 @@ func runClientStream(ctx context.Context, stream v1.Harness_ConnectClient, cance return fmt.Errorf("harnesswire: harness emitted host-owned control event %s", ev.GetKind()) case v1.EventKind_EVENT_END: endErr := endError(ev.GetEnd()) - closeErr := awaitClose(stream, cancel) + closeErr := awaitClose(ctx, streamCtx, stream, cancel) if endErr != nil { return endErr } @@ -670,7 +689,7 @@ const closeGrace = 2 * time.Second // harness gets to flush its telemetry before the host can suspend it, and the RPC ends with the // harness's own status rather than a host-side cancellation. A harness that never closes the // stream costs at most closeGrace. -func awaitClose(stream v1.Harness_ConnectClient, cancel context.CancelFunc) error { +func awaitClose(ctx, streamCtx context.Context, stream v1.Harness_ConnectClient, cancel context.CancelFunc) error { _ = stream.CloseSend() var timedOut atomic.Bool t := time.AfterFunc(closeGrace, func() { @@ -683,7 +702,7 @@ func awaitClose(stream v1.Harness_ConnectClient, cancel context.CancelFunc) erro if err == nil { return fmt.Errorf("harnesswire: harness emitted %s after EVENT_END", ev.GetKind()) } - if err == io.EOF || timedOut.Load() { + if err == io.EOF || timedOut.Load() || ctx.Err() != nil || streamCtx.Err() != nil { return nil } return err diff --git a/harnesswire/prepare_test.go b/harnesswire/prepare_test.go index 01794dfe..6fd695f6 100644 --- a/harnesswire/prepare_test.go +++ b/harnesswire/prepare_test.go @@ -4,12 +4,15 @@ import ( "context" "errors" "io" + "net" + "path/filepath" "testing" "time" "github.com/agent-primitives/sessions/api" v1 "github.com/agent-primitives/sessions/api/genpb" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) type prepareClient struct { @@ -136,6 +139,28 @@ func TestPrepareRunFailureSendsNothing(t *testing.T) { } } +func TestPrepareRunMissingUnixSocketFailsBeforeStart(t *testing.T) { + missing := filepath.Join(t.TempDir(), "missing.sock") + conn, err := grpc.NewClient( + "passthrough:///missing-harness", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", missing) + }), + ) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + prepared, err := NewClientHarness(v1.NewHarnessClient(conn)).PrepareRun(ctx) + if err == nil || prepared != nil { + t.Fatalf("PrepareRun=(%v, %v), want pre-Start Connect failure", prepared, err) + } +} + func TestPreparedRunCloseCancelsUnusedConnect(t *testing.T) { client := &prepareClient{stream: &prepareClientStream{}} prepared, err := NewClientHarness(client).PrepareRun(t.Context()) diff --git a/session/error_connect_test.go b/session/error_connect_test.go index ed8f937f..4c66bf29 100644 --- a/session/error_connect_test.go +++ b/session/error_connect_test.go @@ -110,6 +110,48 @@ func (s *postEndStream) Recv() (*v1.Event, error) { func (*postEndStream) CloseSend() error { return nil } +type endThenBlockClient struct { + v1.HarnessClient + waiting chan struct{} +} + +func (endThenBlockClient) Describe(ctx context.Context, _ *v1.DescribeRequest, _ ...grpc.CallOption) (*v1.HarnessDescriptor, error) { + desc, err := (echoagent.Harness{}).Describe(ctx) + if err != nil { + return nil, err + } + return harnesswire.DescriptorToProto(desc), nil +} + +func (c endThenBlockClient) Connect(ctx context.Context, _ ...grpc.CallOption) (grpc.BidiStreamingClient[v1.ControllerFrame, v1.Event], error) { + return &endThenBlockStream{ctx: ctx, waiting: c.waiting}, nil +} + +type endThenBlockStream struct { + grpc.ClientStream + ctx context.Context + waiting chan struct{} + executionID string + step int +} + +func (s *endThenBlockStream) Send(frame *v1.ControllerFrame) error { + s.executionID = frame.GetExecutionId() + return nil +} + +func (s *endThenBlockStream) Recv() (*v1.Event, error) { + s.step++ + if s.step == 1 { + return &v1.Event{ExecutionId: s.executionID, Kind: v1.EventKind_EVENT_END, Body: &v1.Event_End{End: &v1.HarnessEnd{State: "COMPLETED"}}}, nil + } + close(s.waiting) + <-s.ctx.Done() + return nil, s.ctx.Err() +} + +func (*endThenBlockStream) CloseSend() error { return nil } + func TestExecConnectPreparationFailureCanRetryFirstTurn(t *testing.T) { store, err := sqlitelog.Open(":memory:") if err != nil { @@ -247,3 +289,42 @@ func TestExecFrameAfterEndIsDurableProtocolFailure(t *testing.T) { t.Fatalf("records=%+v, want durable start and error without END", recs) } } + +func TestExecCancellationDuringPostEndFlushKeepsCompletedResult(t *testing.T) { + store, err := sqlitelog.Open(":memory:") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + backend := local.New(echoagent.Harness{}) + t.Cleanup(func() { _ = backend.Close() }) + waiting := make(chan struct{}) + p := placement.New(backend, echoagent.Model, placement.WithDialer(func(api.Incarnation) (api.Harness, func() error, error) { + return harnesswire.NewClientHarness(endThenBlockClient{waiting: waiting}), func() error { return nil }, nil + })) + registry, err := placement.NewRegistry("echo", map[string]*placement.Placer{"echo": p}) + if err != nil { + t.Fatal(err) + } + svc := session.NewService(store, registry) + created, err := svc.CreateSession(t.Context(), &v1.CreateSessionRequest{}) + if err != nil { + t.Fatal(err) + } + uid := created.GetMetadata().GetUid() + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- svc.Exec(&v1.ExecRequest{Session: uid}, &rpcStream{ctx: ctx}) }() + <-waiting + cancel() + if err := <-done; err != nil { + t.Fatalf("Exec=%v, want completed result after caller cancellation during telemetry flush", err) + } + recs, readErr := store.Session(uid).Read(1) + if readErr != nil { + t.Fatal(readErr) + } + if len(recs) != 2 || recs[0].Event.Kind != api.EventExecutionStart || recs[1].Event.Kind != api.EventEnd { + t.Fatalf("records=%+v, want durable start and END without ERROR", recs) + } +} diff --git a/spec.md b/spec.md index fc057abd..72aad0da 100644 --- a/spec.md +++ b/spec.md @@ -1630,7 +1630,9 @@ sequenceDiagram the caller has already cancelled; the turn still fails with the host's error. After `EVENT_PARKED` it already waits for the harness to close (`APR-10`). A stream that closes cleanly before `EVENT_END`, or sends a host-owned or unsupported event kind, fails the execution as a - protocol error; it is never inferred as successful completion. + protocol error; it is never inferred as successful completion. After `EVENT_END{COMPLETED}`, a + non-OK final RPC status fails the turn while the caller remains live, but caller cancellation or + deadline during the bounded telemetry-flush wait does not replace the harness's completed result. - `WIRE-13`: Harness dials in the reference host use no transport security (§17). ### 12.3 Wire conversion From 0f78259fbc97fb3e114a989a9f208be4b54781a8 Mon Sep 17 00:00:00 2001 From: Tatsat Mishra Date: Tue, 17 Nov 2026 20:34:51 +1300 Subject: [PATCH 4/5] docs(harnesswire): clarify preparation boundary --- .../unreleased/harness-connect-preparation.md | 8 +++----- .../unreleased/harnesswire-terminal-compat.md | 9 +++++++++ spec.md | 20 ++++++++++--------- 3 files changed, 23 insertions(+), 14 deletions(-) create mode 100644 changelog/unreleased/harnesswire-terminal-compat.md diff --git a/changelog/unreleased/harness-connect-preparation.md b/changelog/unreleased/harness-connect-preparation.md index 452de6b5..6b9a93b3 100644 --- a/changelog/unreleased/harness-connect-preparation.md +++ b/changelog/unreleased/harness-connect-preparation.md @@ -5,8 +5,6 @@ category: Fixed - A new `Exec` over `Harness.Connect` now opens its actual stream before recording `EXECUTION_START`. If Connect cannot be opened, no execution records are appended and retrying the same session starts its first turn normally. The host still sends `Start` only after the start and - every input are durable; send and receive failures remain uncertain and keep the attempted - execution in later history. A clean close without `EVENT_END`, or a host-owned or unsupported - harness event, is now a failed execution rather than implicit success. Go harness adapters can opt - into the same boundary with `api.HarnessRunPreparer` (`EXE-45`). Caller cancellation during the - bounded post-END telemetry flush does not replace a completed harness result. + every input are durable; cancellation after successful preparation and send or receive failures + remain uncertain and keep any recorded attempt in later history. Go harness adapters can opt into + the same boundary with `api.HarnessRunPreparer` (`EXE-45`). diff --git a/changelog/unreleased/harnesswire-terminal-compat.md b/changelog/unreleased/harnesswire-terminal-compat.md new file mode 100644 index 00000000..1fae8ca2 --- /dev/null +++ b/changelog/unreleased/harnesswire-terminal-compat.md @@ -0,0 +1,9 @@ +--- +category: Compatibility +--- + +- `Harness.Connect` now fails an execution for four wire shapes the host previously accepted: clean + closure without `EVENT_END`, a host-owned or unsupported harness event, any frame after + `EVENT_END`, and a harness-owned non-OK final RPC status after `EVENT_END{COMPLETED}` while the + caller remains live. Caller cancellation or deadline during the bounded post-END telemetry flush + does not replace a completed harness result (`WIRE-12`). diff --git a/spec.md b/spec.md index 72aad0da..b0900796 100644 --- a/spec.md +++ b/spec.md @@ -644,10 +644,11 @@ reads the catalog that registration writes; the reference daemon does. - `EXE-45`: Before recording a new execution for a harness that supports run preparation, the host MUST prepare the actual Run without invoking the harness or sending `Start`. A preparation failure appends no execution records, so the caller may retry `Exec` at the same journal cursor. The - successful preparation MUST be used for that execution exactly once; only after - `EXECUTION_START` and every `INPUT` commit may the host send `Start`. Once sending `Start` is - attempted, a send or receive failure does not prove non-delivery: the execution remains durable - and visible in later history under `EXE-7` and `EXE-17`. + successful preparation MUST be used for that execution at most once, and MUST be closed unused if + a record append fails. Only after `EXECUTION_START` and every `INPUT` commit may the host send + `Start`. Once preparation succeeds, a caller cancellation, send failure or receive failure does + not prove non-delivery: any recorded execution remains durable and visible in later history under + `EXE-7` and `EXE-17`. ### 8.2 Effects @@ -910,11 +911,12 @@ cannot observe a model call that does not reach it. The class concerns model cal re-prompts. Once the compute that held the memory is gone (`RT-4e`), or replaced on a backend that records `ComputeInstance` (`RT-4h`, for example `runtime/substrate`), every later `Exec` and `Resume` on the session fails `FAILED_PRECONDITION`, so the re-prompt goes to a new session; the - failed session's log stays readable. When the compute and its memory are known to survive, or a - prepared harness Run failed before `Start` was sent (`EXE-45`), the re-prompt is an `Exec` on the - same session. A stream failure after sending begins does not prove that memory survived or that - the harness did no work. An unmediated harness with no state for a session whose history records - an earlier execution MUST fail the run as possible state loss rather than start an empty + failed session's log stays readable. When the compute and its memory are known to survive, or + preparing the harness Run failed so no execution was recorded (`EXE-45`), the re-prompt is an + `Exec` on the same session. Cancellation after a successful preparation, or a stream failure once + sending begins, does not prove that memory survived or that the harness did no work; any recorded + attempt remains in later history. An unmediated harness with no state for a session whose history + records an earlier execution MUST fail the run as possible state loss rather than start an empty conversation. - `EXE-44`: Wrapper delegation. When a host-side wrapper performs work before delegating to an unmediated harness, a recorded execution that the wrapper positively knows was not delegated MUST From 29427e033b7d79e947c3f47aaba715828bc41cc9 Mon Sep 17 00:00:00 2001 From: Tatsat Mishra Date: Wed, 18 Nov 2026 07:17:46 +1300 Subject: [PATCH 5/5] chore(harnesswire): satisfy lint and compatibility notes --- .../unreleased/harnesswire-terminal-compat.md | 11 +++++++---- controller/prepare_run_test.go | 2 +- harnesswire/harnesswire.go | 16 +++++++--------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/changelog/unreleased/harnesswire-terminal-compat.md b/changelog/unreleased/harnesswire-terminal-compat.md index 1fae8ca2..7ef0813d 100644 --- a/changelog/unreleased/harnesswire-terminal-compat.md +++ b/changelog/unreleased/harnesswire-terminal-compat.md @@ -3,7 +3,10 @@ category: Compatibility --- - `Harness.Connect` now fails an execution for four wire shapes the host previously accepted: clean - closure without `EVENT_END`, a host-owned or unsupported harness event, any frame after - `EVENT_END`, and a harness-owned non-OK final RPC status after `EVENT_END{COMPLETED}` while the - caller remains live. Caller cancellation or deadline during the bounded post-END telemetry flush - does not replace a completed harness result (`WIRE-12`). + closure without `EVENT_END`; harness-sent `EVENT_INPUT`, `EVENT_LIFECYCLE`, `EVENT_ERROR`, or an + unspecified/unknown kind; any frame after `EVENT_END`; and a harness-owned non-OK final RPC status + after `EVENT_END{COMPLETED}` while the caller remains live. Caller cancellation or deadline during + the bounded post-END telemetry flush does not replace a completed harness result (`WIRE-12`). +- A new `Exec` whose Connect preparation fails with `UNAVAILABLE` or `DEADLINE_EXCEEDED` while the + caller remains live now returns `UNAVAILABLE` with `agentsessions/HARNESS_UNAVAILABLE` and fixed + host text. Before this change it returned `INTERNAL` with the raw Connect diagnostic. diff --git a/controller/prepare_run_test.go b/controller/prepare_run_test.go index 4bd28469..88767182 100644 --- a/controller/prepare_run_test.go +++ b/controller/prepare_run_test.go @@ -48,7 +48,7 @@ func (r *testPreparedRun) Run(ctx context.Context, start *api.Start, sink api.Ev if r.h.runErr != nil { return r.h.runErr } - return r.h.executionConfigHarness.Run(ctx, start, sink) + return r.h.Run(ctx, start, sink) } func (r *testPreparedRun) Close() { diff --git a/harnesswire/harnesswire.go b/harnesswire/harnesswire.go index 3d119a50..895ed229 100644 --- a/harnesswire/harnesswire.go +++ b/harnesswire/harnesswire.go @@ -697,16 +697,14 @@ func awaitClose(ctx, streamCtx context.Context, stream v1.Harness_ConnectClient, cancel() }) defer t.Stop() - for { - ev, err := stream.Recv() - if err == nil { - return fmt.Errorf("harnesswire: harness emitted %s after EVENT_END", ev.GetKind()) - } - if err == io.EOF || timedOut.Load() || ctx.Err() != nil || streamCtx.Err() != nil { - return nil - } - return err + ev, err := stream.Recv() + if err == nil { + return fmt.Errorf("harnesswire: harness emitted %s after EVENT_END", ev.GetKind()) + } + if err == io.EOF || timedOut.Load() || ctx.Err() != nil || streamCtx.Err() != nil { + return nil } + return err } // hostFailed ends a turn whose sink call failed on the host side, such as a provider error. While