Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions changelog/unreleased/fork-primary-failure-after-deadline.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
category: Fixed
---

- A Substrate `Fork` whose child failed while the caller was still waiting keeps that failure's
status code and ErrorInfo reason when the caller's deadline or cancellation lapses during the
backend's teardown of the child. It was reported as `DEADLINE_EXCEEDED` or `CANCELED`. A caller
context that ended before the failure is still reported as before.
30 changes: 30 additions & 0 deletions internal/callerctx/callerctx.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// Package callerctx carries, inside this module, the caller's context state at the moment a
// runtime operation failed.
//
// A host classifies a failed operation as a caller interruption when the caller's context has
// ended. A runtime that runs its own teardown before it returns would let a deadline that lapses
// during that teardown replace the primary failure, so it marks a failure the caller was still
// waiting for before the teardown started.
package callerctx

import "errors"

type liveAtFailure struct{ err error }

func (e *liveAtFailure) Error() string { return e.err.Error() }
func (e *liveAtFailure) Unwrap() error { return e.err }

// MarkLive records that err ended an operation while the caller's context was still live. The
// text and errors.Is/As chain of err are unchanged. A nil err stays nil.
func MarkLive(err error) error {
if err == nil {
return nil
}
return &liveAtFailure{err: err}
}

// LiveAtFailure reports whether err carries MarkLive's mark.
func LiveAtFailure(err error) bool {
var live *liveAtFailure
return errors.As(err, &live)
}
8 changes: 7 additions & 1 deletion placement/cleanup.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/agent-primitives/sessions/api"
"github.com/agent-primitives/sessions/eventlog"
"github.com/agent-primitives/sessions/internal/callerctx"
)

// allocationRecorder is optional so standalone event logs retain their existing behavior.
Expand Down Expand Up @@ -73,8 +74,13 @@ func callerContextError(ctx context.Context, err error) error {
}

// Only the caller's context during start is an admission interruption. Confirmation
// uses an independent deadline, and checkpoint supersession remains ABORTED.
// uses an independent deadline, and checkpoint supersession remains ABORTED. A runtime
// that tears down before returning marks a failure the caller was still live for, so a
// context that ended during that teardown does not replace the primary failure.
func allocationStartError(ctx context.Context, err error) error {
if callerctx.LiveAtFailure(err) {
return err
}
if err != nil && errors.Is(context.Cause(ctx), errSuperseded) {
return fmt.Errorf("%w (compute start ended with: %v)", errSuperseded, err)
}
Expand Down
3 changes: 3 additions & 0 deletions placement/cleanup_context_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/agent-primitives/sessions/api"
"github.com/agent-primitives/sessions/eventlog"
"github.com/agent-primitives/sessions/internal/callerctx"
)

type contextAllocationLog struct {
Expand Down Expand Up @@ -40,6 +41,8 @@ func TestAllocationContextFailureBoundaries(t *testing.T) {
{name: "independent confirmation deadline", tracked: true, confirmErr: context.DeadlineExceeded},
{name: "caller ended but confirmation failed", tracked: true, cancelCause: context.Canceled, confirmErr: context.DeadlineExceeded},
{name: "checkpoint superseded start", cancelCause: errSuperseded, effectErr: context.Canceled, superseded: true},
// The runtime's own teardown outlasted the caller after a failure the caller was live for.
{name: "caller ended during runtime teardown", cancelCause: context.Canceled, effectErr: callerctx.MarkLive(api.ErrComputeUnavailable)},
} {
t.Run(tc.name, func(t *testing.T) {
ctx, cancel := context.WithCancelCause(t.Context())
Expand Down
25 changes: 25 additions & 0 deletions runtime/substrate/substrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"time"

"github.com/agent-primitives/sessions/api"
"github.com/agent-primitives/sessions/internal/callerctx"
"github.com/agent-primitives/sessions/observability"
)

Expand Down Expand Up @@ -1327,6 +1328,16 @@ func (b *Backend) Fork(ctx context.Context, ref api.SnapshotRef, opts api.ForkOp
defer func() {
finish(err, "error_kind", substrateErrorKind(err), "incarnation_id", inc.ID, "runtime", inc.Runtime)
}()
// The teardown below runs on detached contexts, so the caller's deadline can lapse during it.
// Report whether the caller was still waiting when it started, so the host keeps the primary
// failure instead of classifying it as an interruption.
failure := &forkFailure{}
ctx = context.WithValue(ctx, forkFailureKey{}, failure)
defer func() {
if err != nil && failure.sampled && failure.live {
err = callerctx.MarkLive(err)
}
}()

child := opts.ChildSessionUID
if child == "" {
Expand Down Expand Up @@ -1574,10 +1585,24 @@ const cleanupTimeout = 30 * time.Second
// cleanupContext detaches teardown from the caller's context. The most likely reason a fork fails
// midway is that the caller's deadline expired or it went away, which is exactly when a cleanup
// inheriting that context would fail instantly and strand everything it was written to reclaim.
//
// The first call in a Fork also records whether the caller was still live as its teardown started.
func cleanupContext(ctx context.Context) (context.Context, context.CancelFunc) {
if f, ok := ctx.Value(forkFailureKey{}).(*forkFailure); ok && !f.sampled {
f.sampled, f.live = true, ctx.Err() == nil
}
return context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout)
}

// forkFailureKey carries a Fork call's *forkFailure to cleanupContext.
type forkFailureKey struct{}

// forkFailure is the caller's context state when a failed Fork started its teardown, just after the
// failing call returned. Fork tears down synchronously in its own goroutine, so it needs no lock.
type forkFailure struct {
sampled, live bool
}

// destroyChild best-effort undoes everything a failed Fork provisioned for one child. Stop suspends
// before deleting (substrate rejects deleting a RUNNING actor) and then deletes the fork tag, so it
// covers a child that failed at any point after CreateActor. It returns Stop's error: a suspend
Expand Down
51 changes: 51 additions & 0 deletions runtime/substrate/substrate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

"github.com/agent-primitives/sessions/api"
"github.com/agent-primitives/sessions/controller"
"github.com/agent-primitives/sessions/internal/callerctx"
"github.com/agent-primitives/sessions/runtime/substrate"
)

Expand Down Expand Up @@ -1480,6 +1481,56 @@ func TestForkCleanupSurvivesACancelledCaller(t *testing.T) {
}
}

// The host classifies a failed fork by the caller's context, but Fork tears down on detached
// contexts before it returns, so the caller can end during that teardown. Fork marks a failure the
// caller was still live for when the teardown started, so the host keeps the primary failure; one
// the caller had already ended, or that needed no teardown, is left unmarked.
func TestForkMarksAFailureItsCallerOutlived(t *testing.T) {
primary := fmt.Errorf("%w: worker pool drained", api.ErrComputeUnavailable)
for _, tc := range []struct {
name string
ref api.SnapshotRef
setup func(m *mockCloner, cancel context.CancelFunc)
live bool
}{
{name: "caller ends during the clone's teardown", live: true, setup: func(m *mockCloner, cancel context.CancelFunc) {
m.resumeErr = primary
// discardClone reads the clone before it destroys it.
m.afterGet = func(name string) {
if name == "child" {
cancel()
}
}
}},
{name: "caller ended before the clone failed", setup: func(m *mockCloner, cancel context.CancelFunc) {
m.cloneErr = primary
cancel()
}},
{name: "no teardown", ref: api.SnapshotRef{Local: "parent", Memory: true}, setup: func(*mockCloner, context.CancelFunc) {}},
} {
t.Run(tc.name, func(t *testing.T) {
m := newClonerWithSuspendedParent()
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
tc.setup(m, cancel)
ref := tc.ref
if ref.Local == "" {
ref = api.SnapshotRef{Local: "parent", ExternalURI: "snap-parent-1", Memory: true}
}
_, err := newMemoryBackend(m).Fork(ctx, ref, api.ForkOpts{ChildSessionUID: "child"})
if err == nil {
t.Fatal("expected the fork to fail")
}
if got := callerctx.LiveAtFailure(err); got != tc.live {
t.Fatalf("fork=%v: marked live=%v, want %v", err, got, tc.live)
}
if tc.live && !errors.Is(err, primary) {
t.Fatalf("fork=%v lost the primary failure", err)
}
})
}
}

// A REQUIRES_MEMORY_SNAPSHOT session keeps state only in its actor's RAM. Once it has a started
// turn, an actor that no longer exists cannot be replaced by a fresh one from the template: that would
// continue the journal with a harness that remembers none of it. Create must refuse with
Expand Down
72 changes: 72 additions & 0 deletions session/error_fork_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,15 @@ package session_test
import (
"context"
"errors"
"fmt"
"testing"
"time"

"github.com/agent-primitives/sessions/api"
v1 "github.com/agent-primitives/sessions/api/genpb"
"github.com/agent-primitives/sessions/harness/echoagent"
"github.com/agent-primitives/sessions/runtime/local"
"github.com/agent-primitives/sessions/runtime/substrate"
"github.com/agent-primitives/sessions/session"
"github.com/agent-primitives/sessions/sqlitelog"
"google.golang.org/genproto/googleapis/rpc/errdetails"
Expand Down Expand Up @@ -93,3 +96,72 @@ func TestForkFailureClassificationPrecedesRollback(t *testing.T) {
})
}
}

// Substrate tears a failed fork child down on detached contexts before Fork returns, so the
// caller's deadline can lapse during that teardown. That must not turn the child's primary failure
// into DEADLINE_EXCEEDED; a deadline that lapsed during the failing call itself still does.
func TestForkKeepsPrimaryFailureWhenDeadlineLapsesInTeardown(t *testing.T) {
primary := fmt.Errorf("%w: worker pool drained", api.ErrComputeUnavailable)
for _, tc := range []struct {
name string
lapseInResume bool
code codes.Code
reason string
retainsPrimary bool
}{
{name: "deadline lapses during teardown", code: codes.Unavailable, reason: "COMPUTE_UNAVAILABLE", retainsPrimary: true},
{name: "deadline lapses during the child's resume", lapseInResume: true, code: codes.DeadlineExceeded},
} {
t.Run(tc.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), 250*time.Millisecond)
defer cancel()
ctl := &actorControl{status: map[string]substrate.ActorStatus{}, snapshots: map[string]string{}}
var liveAtFailure bool
ctl.resumeErr = func(string) error {
if tc.lapseInResume {
<-ctx.Done()
// A gRPC ControlClient reports the caller's deadline as a status, not context.DeadlineExceeded.
return status.Error(codes.DeadlineExceeded, "context deadline exceeded")
}
liveAtFailure = ctx.Err() == nil
return primary
}
// The child's teardown suspends it first, and outlasts the caller's deadline.
ctl.onSuspend = func(string) { <-ctx.Done() }
desc, err := echoagent.Harness{}.Describe(t.Context())
if err != nil {
t.Fatal(err)
}
desc.Capabilities.Resumability = api.ResumabilityStatelessReplay
backend := substrate.New(ctl, "space", substrate.ObjectRef{Atespace: "space", Name: "echo"}, desc)
store := openStore(t, ":memory:")
if err := store.PutSession(sqlitelog.SessionMeta{UID: "parent", Harness: "echo"}); err != nil {
t.Fatal(err)
}

_, err = session.NewService(store, echoRegistry(t, backend)).Fork(ctx, &v1.ForkRequest{Session: "parent", Count: 1})
if !tc.lapseInResume && !liveAtFailure {
t.Fatal("the child's resume ran after the caller's deadline; the test needs a longer one")
}
if ctx.Err() == nil {
t.Fatal("the caller's deadline did not lapse during the fork")
}
if status.Code(err) != tc.code || errors.Is(err, primary) != tc.retainsPrimary {
t.Fatalf("fork=%v, want %s (retaining the primary failure: %v)", err, tc.code, tc.retainsPrimary)
}
details := status.Convert(err).Details()
if tc.reason == "" {
if len(details) != 0 {
t.Fatalf("caller failure got a reason: %v", details)
}
return
}
if len(details) != 1 {
t.Fatalf("lost primary reason: %v", details)
}
if info, ok := details[0].(*errdetails.ErrorInfo); !ok || info.Domain != "agentsessions" || info.Reason != tc.reason {
t.Fatalf("details=%v, want agentsessions/%s", details, tc.reason)
}
})
}
}
Loading