diff --git a/changelog/unreleased/fork-primary-failure-after-deadline.md b/changelog/unreleased/fork-primary-failure-after-deadline.md new file mode 100644 index 00000000..a3970a65 --- /dev/null +++ b/changelog/unreleased/fork-primary-failure-after-deadline.md @@ -0,0 +1,8 @@ +--- +category: Fixed +--- + +- A Substrate `Fork` whose child failed while the caller was still waiting keeps that failure's + status code and ErrorInfo reason when the caller's deadline or cancellation lapses during the + backend's teardown of the child. It was reported as `DEADLINE_EXCEEDED` or `CANCELED`. A caller + context that ended before the failure is still reported as before. diff --git a/internal/callerctx/callerctx.go b/internal/callerctx/callerctx.go new file mode 100644 index 00000000..5a0d1408 --- /dev/null +++ b/internal/callerctx/callerctx.go @@ -0,0 +1,30 @@ +// Package callerctx carries, inside this module, the caller's context state at the moment a +// runtime operation failed. +// +// A host classifies a failed operation as a caller interruption when the caller's context has +// ended. A runtime that runs its own teardown before it returns would let a deadline that lapses +// during that teardown replace the primary failure, so it marks a failure the caller was still +// waiting for before the teardown started. +package callerctx + +import "errors" + +type liveAtFailure struct{ err error } + +func (e *liveAtFailure) Error() string { return e.err.Error() } +func (e *liveAtFailure) Unwrap() error { return e.err } + +// MarkLive records that err ended an operation while the caller's context was still live. The +// text and errors.Is/As chain of err are unchanged. A nil err stays nil. +func MarkLive(err error) error { + if err == nil { + return nil + } + return &liveAtFailure{err: err} +} + +// LiveAtFailure reports whether err carries MarkLive's mark. +func LiveAtFailure(err error) bool { + var live *liveAtFailure + return errors.As(err, &live) +} diff --git a/placement/cleanup.go b/placement/cleanup.go index be3de1bc..147448bc 100644 --- a/placement/cleanup.go +++ b/placement/cleanup.go @@ -8,6 +8,7 @@ import ( "github.com/agent-primitives/sessions/api" "github.com/agent-primitives/sessions/eventlog" + "github.com/agent-primitives/sessions/internal/callerctx" ) // allocationRecorder is optional so standalone event logs retain their existing behavior. @@ -73,8 +74,13 @@ func callerContextError(ctx context.Context, err error) error { } // Only the caller's context during start is an admission interruption. Confirmation -// uses an independent deadline, and checkpoint supersession remains ABORTED. +// uses an independent deadline, and checkpoint supersession remains ABORTED. A runtime +// that tears down before returning marks a failure the caller was still live for, so a +// context that ended during that teardown does not replace the primary failure. func allocationStartError(ctx context.Context, err error) error { + if callerctx.LiveAtFailure(err) { + return err + } if err != nil && errors.Is(context.Cause(ctx), errSuperseded) { return fmt.Errorf("%w (compute start ended with: %v)", errSuperseded, err) } diff --git a/placement/cleanup_context_test.go b/placement/cleanup_context_test.go index b8df13e9..40ef2a69 100644 --- a/placement/cleanup_context_test.go +++ b/placement/cleanup_context_test.go @@ -7,6 +7,7 @@ import ( "github.com/agent-primitives/sessions/api" "github.com/agent-primitives/sessions/eventlog" + "github.com/agent-primitives/sessions/internal/callerctx" ) type contextAllocationLog struct { @@ -40,6 +41,8 @@ func TestAllocationContextFailureBoundaries(t *testing.T) { {name: "independent confirmation deadline", tracked: true, confirmErr: context.DeadlineExceeded}, {name: "caller ended but confirmation failed", tracked: true, cancelCause: context.Canceled, confirmErr: context.DeadlineExceeded}, {name: "checkpoint superseded start", cancelCause: errSuperseded, effectErr: context.Canceled, superseded: true}, + // The runtime's own teardown outlasted the caller after a failure the caller was live for. + {name: "caller ended during runtime teardown", cancelCause: context.Canceled, effectErr: callerctx.MarkLive(api.ErrComputeUnavailable)}, } { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithCancelCause(t.Context()) diff --git a/runtime/substrate/substrate.go b/runtime/substrate/substrate.go index 295f216e..414bd0e5 100644 --- a/runtime/substrate/substrate.go +++ b/runtime/substrate/substrate.go @@ -32,6 +32,7 @@ import ( "time" "github.com/agent-primitives/sessions/api" + "github.com/agent-primitives/sessions/internal/callerctx" "github.com/agent-primitives/sessions/observability" ) @@ -1327,6 +1328,16 @@ func (b *Backend) Fork(ctx context.Context, ref api.SnapshotRef, opts api.ForkOp defer func() { finish(err, "error_kind", substrateErrorKind(err), "incarnation_id", inc.ID, "runtime", inc.Runtime) }() + // The teardown below runs on detached contexts, so the caller's deadline can lapse during it. + // Report whether the caller was still waiting when it started, so the host keeps the primary + // failure instead of classifying it as an interruption. + failure := &forkFailure{} + ctx = context.WithValue(ctx, forkFailureKey{}, failure) + defer func() { + if err != nil && failure.sampled && failure.live { + err = callerctx.MarkLive(err) + } + }() child := opts.ChildSessionUID if child == "" { @@ -1574,10 +1585,24 @@ const cleanupTimeout = 30 * time.Second // cleanupContext detaches teardown from the caller's context. The most likely reason a fork fails // midway is that the caller's deadline expired or it went away, which is exactly when a cleanup // inheriting that context would fail instantly and strand everything it was written to reclaim. +// +// The first call in a Fork also records whether the caller was still live as its teardown started. func cleanupContext(ctx context.Context) (context.Context, context.CancelFunc) { + if f, ok := ctx.Value(forkFailureKey{}).(*forkFailure); ok && !f.sampled { + f.sampled, f.live = true, ctx.Err() == nil + } return context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) } +// forkFailureKey carries a Fork call's *forkFailure to cleanupContext. +type forkFailureKey struct{} + +// forkFailure is the caller's context state when a failed Fork started its teardown, just after the +// failing call returned. Fork tears down synchronously in its own goroutine, so it needs no lock. +type forkFailure struct { + sampled, live bool +} + // destroyChild best-effort undoes everything a failed Fork provisioned for one child. Stop suspends // before deleting (substrate rejects deleting a RUNNING actor) and then deletes the fork tag, so it // covers a child that failed at any point after CreateActor. It returns Stop's error: a suspend diff --git a/runtime/substrate/substrate_test.go b/runtime/substrate/substrate_test.go index c800f931..85b43d7e 100644 --- a/runtime/substrate/substrate_test.go +++ b/runtime/substrate/substrate_test.go @@ -14,6 +14,7 @@ import ( "github.com/agent-primitives/sessions/api" "github.com/agent-primitives/sessions/controller" + "github.com/agent-primitives/sessions/internal/callerctx" "github.com/agent-primitives/sessions/runtime/substrate" ) @@ -1480,6 +1481,56 @@ func TestForkCleanupSurvivesACancelledCaller(t *testing.T) { } } +// The host classifies a failed fork by the caller's context, but Fork tears down on detached +// contexts before it returns, so the caller can end during that teardown. Fork marks a failure the +// caller was still live for when the teardown started, so the host keeps the primary failure; one +// the caller had already ended, or that needed no teardown, is left unmarked. +func TestForkMarksAFailureItsCallerOutlived(t *testing.T) { + primary := fmt.Errorf("%w: worker pool drained", api.ErrComputeUnavailable) + for _, tc := range []struct { + name string + ref api.SnapshotRef + setup func(m *mockCloner, cancel context.CancelFunc) + live bool + }{ + {name: "caller ends during the clone's teardown", live: true, setup: func(m *mockCloner, cancel context.CancelFunc) { + m.resumeErr = primary + // discardClone reads the clone before it destroys it. + m.afterGet = func(name string) { + if name == "child" { + cancel() + } + } + }}, + {name: "caller ended before the clone failed", setup: func(m *mockCloner, cancel context.CancelFunc) { + m.cloneErr = primary + cancel() + }}, + {name: "no teardown", ref: api.SnapshotRef{Local: "parent", Memory: true}, setup: func(*mockCloner, context.CancelFunc) {}}, + } { + t.Run(tc.name, func(t *testing.T) { + m := newClonerWithSuspendedParent() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + tc.setup(m, cancel) + ref := tc.ref + if ref.Local == "" { + ref = api.SnapshotRef{Local: "parent", ExternalURI: "snap-parent-1", Memory: true} + } + _, err := newMemoryBackend(m).Fork(ctx, ref, api.ForkOpts{ChildSessionUID: "child"}) + if err == nil { + t.Fatal("expected the fork to fail") + } + if got := callerctx.LiveAtFailure(err); got != tc.live { + t.Fatalf("fork=%v: marked live=%v, want %v", err, got, tc.live) + } + if tc.live && !errors.Is(err, primary) { + t.Fatalf("fork=%v lost the primary failure", err) + } + }) + } +} + // A REQUIRES_MEMORY_SNAPSHOT session keeps state only in its actor's RAM. Once it has a started // turn, an actor that no longer exists cannot be replaced by a fresh one from the template: that would // continue the journal with a harness that remembers none of it. Create must refuse with diff --git a/session/error_fork_test.go b/session/error_fork_test.go index 94c52094..2a4777ce 100644 --- a/session/error_fork_test.go +++ b/session/error_fork_test.go @@ -3,12 +3,15 @@ package session_test import ( "context" "errors" + "fmt" "testing" + "time" "github.com/agent-primitives/sessions/api" v1 "github.com/agent-primitives/sessions/api/genpb" "github.com/agent-primitives/sessions/harness/echoagent" "github.com/agent-primitives/sessions/runtime/local" + "github.com/agent-primitives/sessions/runtime/substrate" "github.com/agent-primitives/sessions/session" "github.com/agent-primitives/sessions/sqlitelog" "google.golang.org/genproto/googleapis/rpc/errdetails" @@ -93,3 +96,72 @@ func TestForkFailureClassificationPrecedesRollback(t *testing.T) { }) } } + +// Substrate tears a failed fork child down on detached contexts before Fork returns, so the +// caller's deadline can lapse during that teardown. That must not turn the child's primary failure +// into DEADLINE_EXCEEDED; a deadline that lapsed during the failing call itself still does. +func TestForkKeepsPrimaryFailureWhenDeadlineLapsesInTeardown(t *testing.T) { + primary := fmt.Errorf("%w: worker pool drained", api.ErrComputeUnavailable) + for _, tc := range []struct { + name string + lapseInResume bool + code codes.Code + reason string + retainsPrimary bool + }{ + {name: "deadline lapses during teardown", code: codes.Unavailable, reason: "COMPUTE_UNAVAILABLE", retainsPrimary: true}, + {name: "deadline lapses during the child's resume", lapseInResume: true, code: codes.DeadlineExceeded}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 250*time.Millisecond) + defer cancel() + ctl := &actorControl{status: map[string]substrate.ActorStatus{}, snapshots: map[string]string{}} + var liveAtFailure bool + ctl.resumeErr = func(string) error { + if tc.lapseInResume { + <-ctx.Done() + // A gRPC ControlClient reports the caller's deadline as a status, not context.DeadlineExceeded. + return status.Error(codes.DeadlineExceeded, "context deadline exceeded") + } + liveAtFailure = ctx.Err() == nil + return primary + } + // The child's teardown suspends it first, and outlasts the caller's deadline. + ctl.onSuspend = func(string) { <-ctx.Done() } + desc, err := echoagent.Harness{}.Describe(t.Context()) + if err != nil { + t.Fatal(err) + } + desc.Capabilities.Resumability = api.ResumabilityStatelessReplay + backend := substrate.New(ctl, "space", substrate.ObjectRef{Atespace: "space", Name: "echo"}, desc) + store := openStore(t, ":memory:") + if err := store.PutSession(sqlitelog.SessionMeta{UID: "parent", Harness: "echo"}); err != nil { + t.Fatal(err) + } + + _, err = session.NewService(store, echoRegistry(t, backend)).Fork(ctx, &v1.ForkRequest{Session: "parent", Count: 1}) + if !tc.lapseInResume && !liveAtFailure { + t.Fatal("the child's resume ran after the caller's deadline; the test needs a longer one") + } + if ctx.Err() == nil { + t.Fatal("the caller's deadline did not lapse during the fork") + } + if status.Code(err) != tc.code || errors.Is(err, primary) != tc.retainsPrimary { + t.Fatalf("fork=%v, want %s (retaining the primary failure: %v)", err, tc.code, tc.retainsPrimary) + } + details := status.Convert(err).Details() + if tc.reason == "" { + if len(details) != 0 { + t.Fatalf("caller failure got a reason: %v", details) + } + return + } + if len(details) != 1 { + t.Fatalf("lost primary reason: %v", details) + } + if info, ok := details[0].(*errdetails.ErrorInfo); !ok || info.Domain != "agentsessions" || info.Reason != tc.reason { + t.Fatalf("details=%v, want agentsessions/%s", details, tc.reason) + } + }) + } +}