diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile index c19c89ec21c..eb7cbad5a40 100644 --- a/core/chainlink.Dockerfile +++ b/core/chainlink.Dockerfile @@ -117,7 +117,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates=20260601~24.04.1 \ gnupg=2.4.4-2ubuntu17.6 \ lsb-release=12.0-2 \ - curl=8.5.0-2ubuntu10.13 \ + curl=8.5.0-2ubuntu10.15 \ && rm -rf /var/lib/apt/lists/* # Install Postgres for CLI tools, needed specifically for DB backups diff --git a/core/services/workflows/syncer/v2/shard_failover_manager.go b/core/services/workflows/syncer/v2/shard_failover_manager.go index fecb1e66b35..5fa7129ccc0 100644 --- a/core/services/workflows/syncer/v2/shard_failover_manager.go +++ b/core/services/workflows/syncer/v2/shard_failover_manager.go @@ -45,7 +45,7 @@ type ShardFailoverManager struct { services.Service eng *services.Engine - engine *v2.Engine + engine v2.WorkflowEngine cfg ShardFailoverManagerConfig mu sync.RWMutex @@ -99,7 +99,7 @@ func (m *ShardFailoverManager) WireHooks(cfg *v2.EngineConfig) { } // SetEngine injects the engine after it has been created. Required before Start. -func (m *ShardFailoverManager) SetEngine(engine *v2.Engine) { +func (m *ShardFailoverManager) SetEngine(engine v2.WorkflowEngine) { m.engine = engine } diff --git a/core/services/workflows/v2/base_engine.go b/core/services/workflows/v2/base_engine.go new file mode 100644 index 00000000000..f30bf49934c --- /dev/null +++ b/core/services/workflows/v2/base_engine.go @@ -0,0 +1,1054 @@ +package v2 + +import ( + "context" + "errors" + "fmt" + "maps" + "runtime" + "runtime/debug" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/shopspring/decimal" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/chainlink-common/pkg/aggregation" + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/config" + "github.com/smartcontractkit/chainlink-common/pkg/contexts" + "github.com/smartcontractkit/chainlink-common/pkg/custmsg" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/metrics" + "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" + "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink-common/pkg/workflows" + billing "github.com/smartcontractkit/chainlink-protos/billing/go" + sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" + protoevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events" + "github.com/smartcontractkit/chainlink/v2/core/platform" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/events" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/metering" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/monitoring" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/store" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" + "github.com/smartcontractkit/chainlink/v2/core/utils/safe" +) + +var executingWorkflows atomic.Int64 + +var ( + ErrDuplicateExecution = errors.New("duplicate execution") + ErrShardDeniedNotOwner = errors.New("shard ownership denied: not owner") + ErrShardDeniedOrchestrator = errors.New("shard ownership denied: orchestrator error") + ErrMeteringReserveFailed = errors.New("metering reserve failed") + ErrAdmissionCache = errors.New("admission: event cached for failover") + + ErrEngineDraining = errors.New("engine is draining") + ErrQueueFull = errors.New("trigger event queue is full") + ErrEnqueueFailed = errors.New("failed to enqueue trigger event") +) + +// Pin config version to 1 to avoid updating forwarder contracts on every single config update. +// Config Version set in CapabilitiesRegistry is included in every report but is irrelevant +// to validation on the forwarder side. What matters is DON ID and the set of signer public keys. +const pinnedWorkflowDonConfigVersion = 1 + +// baseEngine holds the execution machinery shared by every workflow engine: +// module execution, metering, secrets, labels, the heartbeat, drain state, and +// DON sync. +// +// It is never used directly. engine (legacy, trigger-owning) and coordinatedEngine +// (execution-only) each embed it and supply their own start/init/close. The +// single services.Engine for a workflow engine lives here — see initServiceEngine. +type baseEngine struct { + services.Service + srvcEng *services.Engine + + cfg *EngineConfig + + // lggr is the engine's logger. It is protected by lggrMu to allow safe updates when DON configuration changes. + // IMPORTANT: Do NOT access this field directly. Always use the logger() method to ensure thread-safety. + // Direct access will cause data races when the logger is updated in localNodeSync(). + lggr logger.SugaredLogger + + // lggrMu protects lggr during dynamic updates when DON configuration changes. + // Write access (Lock): Only in setLogger() called from localNodeSync() + // Read access (RLock): In logger() method called from everywhere else + lggrMu sync.RWMutex + + loggerLabels atomic.Pointer[map[string]string] + localNode atomic.Pointer[capabilities.Node] + + capCallsSemaphore limits.ResourcePoolLimiter[int] + + meterReports *metering.Reports + + metrics *monitoring.WorkflowsMetricLabeler + + // tracer is the OTel tracer for this engine. It's a noop tracer when DebugMode is false. + tracer trace.Tracer + + orgID string + // orgIDMissingReason records why orgID is empty ("resolver_nil", + // "resolver_error", or "empty_response"). It is set once in startWith() and + // read by startExecution() to label the org_id_missing counter. It is only + // meaningful when orgID == "". + orgIDMissingReason string + + draining atomic.Bool + activeExecutions atomic.Int32 + drainStartedAtNs atomic.Int64 +} + +// newBaseEngine builds the execution machinery shared by every workflow engine. +// It installs no service: the caller must call initServiceEngine before the engine +// is started. +// +// Start and Close are supplied by the outer type, because the lifecycle differs +// per engine, but srvcEng itself lives here — base methods spawn the execution +// goroutines on it, and Drain sets its health condition. +func newBaseEngine(cfg *EngineConfig) (*baseEngine, logger.SugaredLogger, error) { + if err := cfg.Validate(); err != nil { + return nil, nil, fmt.Errorf("invalid config: %w", err) + } + + em, err := monitoring.InitMonitoringResources() + if err != nil { + return nil, nil, fmt.Errorf("could not initialize monitoring resources: %w", err) + } + + // LocalNode() is expected to be non-blocking at this stage (i.e. the registry is already synced) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.LocalLimits.LocalNodeTimeoutMs)*time.Millisecond) + defer cancel() + localNode, err := cfg.CapRegistry.LocalNode(ctx) + if err != nil { + return nil, nil, fmt.Errorf("could not get local node state: %w", err) + } + + // Create engine first so we can use the buildLabels method + engine := &baseEngine{ + cfg: cfg, + capCallsSemaphore: cfg.LocalLimiters.CapabilityConcurrency, + } + + // Build labels using the helper method + labels := engine.buildLabels(&localNode) + + beholderLogger := logger.Sugared(custmsg.NewBeholderLogger(cfg.Lggr, cfg.BeholderEmitter).Named("WorkflowEngine").With(labels...)) + baseLabels := []string{ + platform.KeyWorkflowID, cfg.WorkflowID, + platform.KeyWorkflowOwner, cfg.WorkflowOwner, + platform.KeyWorkflowName, cfg.WorkflowName.String(), + platform.KeySDK, cfg.SdkName, + } + metricsLabeler := monitoring.NewWorkflowsMetricLabeler(metrics.NewLabeler(), em).With(baseLabels...) + labelsMap := make(map[string]string, len(labels)/2) + for i := 0; i < len(labels); i += 2 { + labelsMap[labels[i].(string)] = labels[i+1].(string) + } + + if cfg.DebugMode { + beholderLogger.Errorw("WARNING: Debug mode is enabled, this is not suitable for production") + engine.tracer = otel.Tracer("workflow_engine_v2") + } else { + engine.tracer = noop.NewTracerProvider().Tracer("") + } + + // Store logger and other fields + engine.setLogger(beholderLogger) + engine.meterReports = metering.NewReports(cfg.BillingClient, cfg.WorkflowOwner, cfg.WorkflowID, beholderLogger, labelsMap, metricsLabeler, cfg.WorkflowRegistryAddress, cfg.WorkflowRegistryChainSelector, metering.EngineVersionV2) + engine.metrics = metricsLabeler + engine.loggerLabels.Store(&labelsMap) + engine.localNode.Store(&localNode) + + return engine, beholderLogger, nil +} + +// initServiceEngine installs the single services.Engine for this workflow engine. +// start and close belong to the outer type that owns the lifecycle. +func (e *baseEngine) initServiceEngine(lggr logger.SugaredLogger, name string, start func(context.Context) error, closeFn func() error) { + e.Service, e.srvcEng = services.Config{ + Name: name, + Start: start, + Close: closeFn, + }.NewServiceEngine(lggr) +} + +// ExecuteTrigger is the engine's single execution entry point. It performs no admission control, the caller is responsible for those. +func (e *baseEngine) ExecuteTrigger(ctx context.Context, event RoutedTriggerEvent) error { + e.activeExecutions.Add(1) + defer e.activeExecutions.Add(-1) + + eventID := event.Event.Event.ID + e.logger().Debugw("Scheduling a trigger event for execution", "eventID", eventID) + creCtx := contexts.CREValue(ctx) + // Tracer is no-op if DebugMode is false + ctx, span := e.tracer.Start(ctx, "workflow_execution", + trace.WithAttributes( + attribute.String("workflow_name", e.cfg.WorkflowName.String()), + attribute.String("version", "v2"), + attribute.String("org_id", creCtx.Org), + attribute.String("owner_id", creCtx.Owner), + attribute.String("workflow_id", creCtx.Workflow), + )) + defer span.End() + + return e.startExecution(ctx, event) +} + +// Subscribe issues the WASM Subscribe request and returns the validated trigger subscriptions. +func (e *baseEngine) Subscribe(ctx context.Context) ([]*sdkpb.TriggerSubscription, error) { + // call into the workflow to get trigger subscriptions + subCtx, subCancel, err := e.cfg.LocalLimiters.TriggerSubscriptionTime.WithTimeout(ctx) + if err != nil { + return nil, err + } + defer subCancel() + + maxUserLogEventsPerExecution, err := e.cfg.LocalLimiters.LogEvent.Limit(ctx) + if err != nil { + if !limits.IsErrRecoverable(err) { + return nil, err + } + e.logger().Errorw("Failed to get log event limit; continuing with the value the limiter returned", "err", err) + e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.LogEventLimit.Key) + } + userLogChan := make(chan *protoevents.LogLine, maxUserLogEventsPerExecution) + defer close(userLogChan) + e.srvcEng.GoCtx(subCtx, func(ctx context.Context) { + e.emitUserLogs(ctx, userLogChan, e.cfg.WorkflowID, e.eventLabels()) + }) + + var timeProvider TimeProvider = &types.LocalTimeProvider{} + if !e.cfg.UseLocalTimeProvider { + timeProvider = NewDonTimeProvider(e.cfg.DonTimeStore, e.cfg.WorkflowID, e.donTimeRequestTimeout(subCtx, e.cfg.LocalLimiters.DONTimeRequestTimeout), e.logger(), e.metrics, e.cfg.Clock) + } + + moduleExecuteMaxResponseSizeBytes, err := e.cfg.LocalLimiters.ExecutionResponse.Limit(ctx) + if err != nil { + if !limits.IsErrRecoverable(err) { + return nil, err + } + e.logger().Errorw("Failed to get execution response size limit; continuing with the value the limiter returned", "err", err) + e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.ExecutionResponseLimit.Key) + } + if moduleExecuteMaxResponseSizeBytes < 0 { + return nil, fmt.Errorf("invalid moduleExecuteMaxResponseSizeBytes; must not be negative: %d", moduleExecuteMaxResponseSizeBytes) + } + result, err := e.cfg.Module.Execute(subCtx, &sdkpb.ExecuteRequest{ + Request: &sdkpb.ExecuteRequest_Subscribe{}, + MaxResponseSize: uint64(moduleExecuteMaxResponseSizeBytes), + Config: e.cfg.WorkflowConfig, + }, NewDisallowedExecutionHelper(e.logger(), userLogChan, timeProvider, e.secretsFetcher(e.cfg.WorkflowID))) + if err != nil { + return nil, fmt.Errorf("failed to execute subscribe: %w", err) + } + if result.GetError() != "" { + return nil, fmt.Errorf("failed to execute subscribe: %s", result.GetError()) + } + subs := result.GetTriggerSubscriptions() + if subs == nil { + return nil, errors.New("subscribe result is nil") + } + err = e.cfg.LocalLimiters.TriggerSubscription.Check(ctx, len(subs.Subscriptions)) + if err != nil { + return nil, err + } + + return subs.Subscriptions, nil +} + +// Tenant is the engine's tenant identity. Valid once resolveOrgID has run during +// init. Every field it reads is written before OnInitialized fires. +func (e *baseEngine) Tenant() contexts.CRE { + return contexts.CRE{Org: e.orgID, Owner: e.cfg.WorkflowOwner, Workflow: e.cfg.WorkflowID} +} + +// Drain marks the engine as draining and prevents new executions from starting. +// In-flight executions continue to run to completion. +// It returns true only on the first transition to draining. +func (e *baseEngine) Drain() bool { + started := e.draining.CompareAndSwap(false, true) + if started { + e.drainStartedAtNs.CompareAndSwap(0, time.Now().UnixNano()) + } + e.srvcEng.SetHealthCond("draining", errors.New("engine is draining, pending deletion")) + return started +} + +// Draining returns true if the engine has been marked for deletion and is no longer accepting new trigger events. +func (e *baseEngine) Draining() bool { + return e.draining.Load() +} + +func (e *baseEngine) ActiveExecutions() int32 { + return e.activeExecutions.Load() +} + +func (e *baseEngine) DrainStartedAt() (time.Time, bool) { + ns := e.drainStartedAtNs.Load() + if ns == 0 { + return time.Time{}, false + } + + return time.Unix(0, ns), true +} + +// startWith performs the startup shared by every engine and spawns initFn as the +// initialization goroutine. Each engine passes its own init. +// +// triggerLoopFn is an engine's queue-draining loop (e.g., handleAllTriggerEvents). +// Pass nil to skip loop draining. +func (e *baseEngine) startWith(ctx context.Context, initFn func(context.Context), triggerLoopFn func(context.Context)) error { + e.cfg.Module.Start() + ctx = context.WithoutCancel(ctx) + + // Resolve the workflow owner's org once at engine startup and treat it as stable + // for the lifetime of this engine instance. If org membership/linking changes, the + // workflow must be restarted to pick up the new org mapping. + resolved := resolveOrgID(ctx, e.cfg.OrgResolver, e.cfg.WorkflowOwner, e.logger()) + e.orgID = resolved.ID + e.orgIDMissingReason = resolved.Reason + e.storeLoggerLabels(e.eventLabels()) + + e.metrics = e.metrics.With(platform.KeyOrganizationID, e.orgID) + + ctx = contexts.WithCRE(ctx, e.Tenant()) + e.srvcEng.GoCtx(ctx, e.heartbeatLoop) + e.srvcEng.GoCtx(ctx, initFn) + if triggerLoopFn != nil { + e.srvcEng.GoCtx(ctx, triggerLoopFn) + } + return nil +} + +// initDONSubscribe subscribes to the DON notifier and starts the local-node sync +// loop. A returned error has already been logged; the caller passes it to OnInitialized. +func (e *baseEngine) initDONSubscribe(ctx context.Context) error { + donSubCh, cleanup, err := e.cfg.DonSubscriber.Subscribe(ctx) + if err != nil { + e.logger().Errorw("failed to subscribe to DON notifier", "error", err) + return fmt.Errorf("failed to subscribe to DON notifier: %w", err) + } + + // start loop to sync local node state each time a DON is received on the + // subscribed channel + e.srvcEng.GoCtx(context.WithoutCancel(ctx), func(ctx context.Context) { + defer cleanup() + for { + select { + case <-ctx.Done(): + return + case _, open := <-donSubCh: + if !open { + return + } + e.localNodeSync(ctx) + } + } + }) + return nil +} + +// initDone records a successful initialization and fires OnInitialized(nil). +func (e *baseEngine) initDone(ctx context.Context) { + e.logger().Info("Workflow Engine initialized") + e.metrics.IncrementWorkflowInitializationCounter(ctx) + e.cfg.Hooks.OnInitialized(nil) +} + +func (e *baseEngine) localNodeSync(ctx context.Context) { + to := time.Duration(e.cfg.LocalLimits.LocalNodeTimeoutMs) * time.Millisecond + ctx, cancel := context.WithTimeout(ctx, to) + defer cancel() + + localNode, err := e.cfg.CapRegistry.LocalNode(ctx) + if err != nil { + e.cfg.Lggr.Errorf("could not get local node state: %s", err) + e.cfg.Hooks.OnNodeSynced(localNode, err) + return + } + + // ignore any reads that do not update the config version + if e.localNode.Load().WorkflowDON.ConfigVersion == localNode.WorkflowDON.ConfigVersion { + return + } + + e.cfg.Lggr.Debugw("Setting local node state", + "Workflow DON ID", localNode.WorkflowDON.ID, + "Workflow DON Families", localNode.WorkflowDON.Families, + "Workflow DON Config Version (onchain)", localNode.WorkflowDON.ConfigVersion, + "Workflow DON Config Version (pinned)", pinnedWorkflowDonConfigVersion, + ) + + // Publish the new node before updating logger state so concurrent executions + // observe the synced DON while labels are rebuilt. + e.localNode.Store(&localNode) + + // Recreate the beholder logger with updated labels to reflect the new DON version. + labels := e.buildLabels(&localNode) + newLogger := logger.Sugared( + custmsg.NewBeholderLogger(e.cfg.Lggr, e.cfg.BeholderEmitter). + Named("WorkflowEngine"). + With(labels...), + ) + e.setLogger(newLogger) + + labelsMap := make(map[string]string, len(labels)/2) + for i := 0; i < len(labels); i += 2 { + labelsMap[labels[i].(string)] = labels[i+1].(string) + } + e.storeLoggerLabels(labelsMap) + + e.cfg.Hooks.OnNodeSynced(localNode, nil) +} + +// startExecution initiates a new workflow execution, blocking until completed +func (e *baseEngine) startExecution(ctx context.Context, event RoutedTriggerEvent) error { + triggerDrop := func(reason string) { + e.metrics.With(platform.KeyTriggerID, event.TriggerCapID).IncrementTriggerEventDroppedTotal(ctx, reason) + } + + executionID, err := workflows.GenerateExecutionIDWithTriggerIndex(e.cfg.WorkflowID, event.Event.Event.ID, event.TriggerIndex) + if err != nil { + e.logger().Errorw("Failed to generate execution ID", "err", err, "triggerID", event.TriggerCapID) + triggerDrop(monitoring.TriggerDropReasonExecutionIDGenerationFailed) + return err + } + e.metrics.IncrementExecutionIDFullCounter(ctx) + trace.SpanFromContext(ctx).SetAttributes(attribute.String("execution_id", executionID)) + + loggerLabels := e.eventLabels() + lggr := e.logger().With(platform.KeyOrganizationID, e.orgID) + + var executionTimestamp time.Time + var executionDonTimeProvider TimeProvider + if tsErr := e.cfg.LocalLimiters.ExecutionTimestampsEnabled.AllowErr(ctx); tsErr == nil { + executionDonTimeProvider = NewDonTimeProvider(e.cfg.DonTimeStore, executionID, e.donTimeRequestTimeout(ctx, e.cfg.LocalLimiters.DONTimeRequestTimeout), lggr, e.metrics, e.cfg.Clock) + donTime, dtErr := executionDonTimeProvider.GetDONTime() + if dtErr != nil { + executionTimestamp = e.cfg.Clock.Now() + lggr.Warnw("Failed to get DON time for execution timestamp, falling back to local time", "err", dtErr, "executionTimestamp", executionTimestamp) + e.metrics.IncrementExecutionTimestampFallbackCounter(ctx) + } else { + executionTimestamp = donTime + lggr.Debugw("Execution timestamp assigned", "executionTimestamp", executionTimestamp) + e.metrics.IncrementExecutionTimestampAssignedCounter(ctx) + } + } + + triggerEvent := event.Event.Event + + // disallow duplicate executions + _, addErr := e.cfg.ExecutionsStore.Add(ctx, nil, executionID, e.cfg.WorkflowID, store.StatusStarted) + if addErr != nil { + if errors.Is(addErr, store.ErrDuplicateExecution) { + lggr.Infow("Skipping duplicate execution", "executionID", executionID, "triggerID", event.TriggerCapID, "triggerIndex", event.TriggerIndex) + tm := e.metrics.With(platform.KeyTriggerID, event.TriggerCapID) + tm.IncrementTriggerExecutionDeduplicatedCounter(ctx) + tm.IncrementWorkflowTriggerEventErrorCounter(ctx) + tm.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonDuplicateExecution) + registrationID := TriggerRegistrationID(e.cfg.WorkflowID, event.TriggerIndex) + ackErr := e.cfg.TriggerAcknowledger.Ack(ctx, event.TriggerCapID, registrationID, triggerEvent.ID) + if ackErr != nil { + e.lggr.Errorw("failed to re-ACK trigger event", "eventID", triggerEvent.ID, "err", ackErr) + } + return ErrDuplicateExecution + } + lggr.Errorw("Failed to register execution in store, proceeding anyway", "executionID", executionID, "err", addErr) + } + + var executionStatus string + defer func() { + if executionStatus == "" { + executionStatus = store.StatusErrored + } + if _, finishErr := e.cfg.ExecutionsStore.FinishExecution(ctx, executionID, executionStatus); finishErr != nil { + lggr.Errorw("Failed to finish execution in store", "executionID", executionID, "status", executionStatus, "err", finishErr) + } + }() + + // emitDroppedExecution publishes the Started/Finished pair for an execution abandoned + // before the normal Started/Finished emit points below, so the failure reaches the UI + // instead of vanishing. + emitDroppedExecution := func(cause error, class events.ErrorClassification) { + executionStatus = store.StatusErrored + _ = events.EmitExecutionStartedEvent(ctx, loggerLabels, triggerEvent.ID, executionID) + _ = events.EmitExecutionFinishedEvent(ctx, loggerLabels, store.StatusErrored, executionID, cause, class, lggr) + e.metrics.IncrementWorkflowExecutionFinishedCounter(ctx, store.StatusErrored) + } + + e.metrics.UpdateTotalWorkflowsGauge(ctx, executingWorkflows.Add(1)) + defer e.metrics.UpdateTotalWorkflowsGauge(ctx, executingWorkflows.Add(-1)) + + // TODO(CAPPL-911): add rate-limiting + + meteringReport, meteringErr := e.meterReports.Start(ctx, executionID) + if meteringErr != nil { + lggr.Errorw("could start metering workflow execution. continuing without metering", "err", meteringErr) + } + + isMetering := meteringErr == nil + if isMetering { + mrErr := meteringReport.Reserve(ctx) + if mrErr != nil { + lggr.Errorw("could not reserve metering", "err", mrErr) + triggerDrop(monitoring.TriggerDropReasonMeteringReserveFailed) + return ErrMeteringReserveFailed + } + + e.deductStandardBalances(ctx, meteringReport) + } + + // WithTimeout returns a usable ctx/cancel even on a read failure; err is advisory. + execCtx, execCancel, err := e.cfg.LocalLimiters.ExecutionTime.WithTimeout(ctx) + if err != nil { + if !limits.IsErrRecoverable(err) { + lggr.Errorw("Failed to get execution time limit with no usable value", "err", err) + triggerDrop(monitoring.TriggerDropReasonExecutionTimeLimitReadFailed) + emitDroppedExecution(err, events.ErrorClassificationSystem) + if execCancel != nil { + execCancel() // WithTimeout may still have built a context + } + return err + } + lggr.Errorw("Failed to get execution time limit; continuing with the timeout the limiter returned", "err", err) + e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.ExecutionTimeout.Key) + if execCtx == nil { // only nil when the limiter is closed and no ctx was built + execCtx, execCancel = context.WithTimeout(ctx, cresettings.Default.PerWorkflow.ExecutionTimeout.DefaultValue) + } + } + defer execCancel() + triggerCapID := event.TriggerCapID + skewRec := &monitoring.TriggerSkewRecorder{ + EnqueueTime: event.ObservedAt, + Record: func(ctx context.Context, seconds float64, source string) { + e.metrics.With(platform.KeyTriggerID, triggerCapID).RecordTriggerQueueToExecutionStartSeconds(ctx, seconds, source) + }, + } + execCtx = monitoring.ContextWithTriggerSkewRecorder(execCtx, skewRec) + executionLogger := logger.With(lggr, "executionID", executionID, "triggerID", event.TriggerCapID, + "triggerIndex", event.TriggerIndex, "eventID", triggerEvent.ID) + + // This is only a peek to size the user-log channel's burst buffer; LogEvent.Check + // (called per log line in emitUserLogs) is what actually enforces the cap. + maxUserLogEventsPerExecution, err := e.cfg.LocalLimiters.LogEvent.Limit(ctx) + if err != nil { + if !limits.IsErrRecoverable(err) { + lggr.Errorw("Failed to get log event limit with no usable value", "err", err) + triggerDrop(monitoring.TriggerDropReasonLogEventLimitReadFailed) + emitDroppedExecution(err, events.ErrorClassificationSystem) + return err + } + lggr.Errorw("Failed to get log event limit; continuing with the value the limiter returned", "err", err) + e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.LogEventLimit.Key) + } + userLogChan := make(chan *protoevents.LogLine, maxUserLogEventsPerExecution) + defer close(userLogChan) + e.srvcEng.GoCtx(execCtx, func(ctx context.Context) { + e.emitUserLogs(ctx, userLogChan, executionID, loggerLabels) + }) + + tid, err := safe.IntToUint64(event.TriggerIndex) + if err != nil { + executionLogger.Errorw("Failed to convert trigger index to uint64", "err", err) + triggerDrop(monitoring.TriggerDropReasonTriggerIndexInvalid) + emitDroppedExecution(err, events.ErrorClassificationSystem) + return err + } + + startTime := e.cfg.Clock.Now() + executionLogger.Infow("Workflow execution starting ...") + if e.orgID == "" { + e.metrics.IncrementOrgIDMissingCounter(ctx, e.orgIDMissingReason) + } + _ = events.EmitExecutionStartedEvent(ctx, loggerLabels, triggerEvent.ID, executionID) + + registrationID := TriggerRegistrationID(e.cfg.WorkflowID, event.TriggerIndex) + err = e.cfg.TriggerAcknowledger.Ack(ctx, event.TriggerCapID, registrationID, triggerEvent.ID) + if err != nil { + e.lggr.Errorf("failed to ACK trigger event (eventID=%s): %v", triggerEvent.ID, err) + } + e.metrics.With("workflowID", e.cfg.WorkflowID, "workflowName", e.cfg.WorkflowName.String()).IncrementWorkflowExecutionStartedCounter(ctx) + + // Track execution error (and its user/system classification) for deferred + // event emission. Set alongside executionStatus at each outcome site below. + var execErr error + execErrClass := events.ErrorClassificationUnspecified + var execHelper *ExecutionHelper + defer func() { + _ = events.EmitExecutionFinishedEvent(ctx, loggerLabels, executionStatus, executionID, execErr, execErrClass, lggr) + if execHelper != nil { + endTime := e.cfg.Clock.Now() + profile, emitErr := events.EmitExecutionProfile( + ctx, + e.cfg.WorkflowID, + executionID, + startTime, + endTime, + executionStatus, + execHelper.executionProfile.stepInputs(), + ) + if emitErr != nil { + lggr.Errorw("Failed to emit execution profile", "err", emitErr) + } + if profile != nil { + profileJSON, jsonErr := protojson.Marshal(profile) + if jsonErr != nil { + lggr.Errorw("Failed to marshal execution profile to JSON", "err", jsonErr) + } else { + lggr.Infow("Workflow execution profile", "executionProfile", string(profileJSON)) + } + } + } + e.cfg.Hooks.OnExecutionFinished(executionID, executionStatus) + e.cfg.Hooks.OnExecutionStatusUpdate(e.cfg.WorkflowID, executionID, triggerEvent.ID, event.TriggerIndex, executionStatus, execErrClass) + if execErr != nil { + e.cfg.Hooks.OnExecutionError(execErr.Error()) + } + }() + + var timeProvider TimeProvider = &types.LocalTimeProvider{} + if !e.cfg.UseLocalTimeProvider { + if executionDonTimeProvider != nil { + timeProvider = executionDonTimeProvider + } else { + lggr.Warnw("ExecutionTimestampsEnabled is false - creating a new DON time provider") + timeProvider = NewDonTimeProvider(e.cfg.DonTimeStore, executionID, e.donTimeRequestTimeout(execCtx, e.cfg.LocalLimiters.DONTimeRequestTimeout), lggr, e.metrics, e.cfg.Clock) + } + } + + // Track time the guest spends blocked in DON-time host calls so it can be + // excluded from metered compute (see compute_metering.go). The wrapper is + // always installed; whether the recorded wait is subtracted is gated by + // e.cfg.MeterComputeExcludingHostWait below. + suspension := &suspensionTracker{} + timeProvider = newMeasuredTimeProvider(timeProvider, e.cfg.Clock, suspension) + + // Limit is always usable even on a read failure; err is advisory. + moduleExecuteMaxResponseSizeBytes, err := e.cfg.LocalLimiters.ExecutionResponse.Limit(ctx) + if err != nil { + if !limits.IsErrRecoverable(err) { + execErr = fmt.Errorf("failed to get execution response size limit with no usable value: %w", err) + lggr.Errorw(execErr.Error()) + executionStatus = store.StatusErrored + execErrClass = events.ErrorClassificationSystem + triggerDrop(monitoring.TriggerDropReasonExecutionResponseLimitReadFailed) + return execErr + } + lggr.Errorw("Failed to get execution response size limit; continuing with the value the limiter returned", "err", err) + e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.ExecutionResponseLimit.Key) + } + if moduleExecuteMaxResponseSizeBytes < 0 { + execErr = fmt.Errorf("invalid moduleExecuteMaxResponseSizeBytes; must not be negative: %d", moduleExecuteMaxResponseSizeBytes) + lggr.Errorw(execErr.Error()) + executionStatus = store.StatusErrored + execErrClass = events.ErrorClassificationSystem + triggerDrop(monitoring.TriggerDropReasonExecutionResponseLimitInvalid) + return execErr + } + execHelper = &ExecutionHelper{ + base: e, WorkflowExecutionID: executionID, ExecutionTimestamp: executionTimestamp, + UserLogChan: userLogChan, TimeProvider: timeProvider, SecretsFetcher: e.secretsFetcher(executionID), + executionProfile: newExecutionProfileCollector(), + suspension: suspension, + } + + execHelper.initLimiters(e.cfg.LocalLimiters) + e.metrics.With(platform.KeyTriggerID, event.TriggerCapID).RecordTriggerPayloadBytes(ctx, int64(proto.Size(triggerEvent.Payload))) + var result *sdkpb.ExecutionResult + result, execErr = e.cfg.Module.Execute(execCtx, &sdkpb.ExecuteRequest{ + Request: &sdkpb.ExecuteRequest_Trigger{ + Trigger: &sdkpb.Trigger{ + Id: tid, + Payload: triggerEvent.Payload, + }, + }, + MaxResponseSize: uint64(moduleExecuteMaxResponseSizeBytes), + Config: e.cfg.WorkflowConfig, + }, execHelper.PossiblyWithRawSecrets()) + // Non-evictable modules do not record skew; label those as direct. + skewRec.RecordReady(execCtx, monitoring.ModuleLoadSourceDirect) + + endTime := e.cfg.Clock.Now() + executionDuration := endTime.Sub(startTime) + + // computeDuration is what we meter as RESOURCE_TYPE_COMPUTE. We start from the + // end-to-end wall-clock and subtract time the guest spent blocked in DON-time + // host calls: that wait is not compute and is non-deterministic across nodes + // (a node whose DON-time request lands in the current consensus round returns + // in microseconds; one whose request slips to the next round blocks ~a full + // round). Operational latency metrics below intentionally keep using + // executionDuration. + computeDuration := executionDuration + if hostWait := suspension.total(); hostWait > 0 { + computeDuration -= hostWait + if computeDuration < 0 { + computeDuration = 0 + } + executionLogger.Debugw("Excluding DON-time wait from metered compute", + "wallClockMs", executionDuration.Milliseconds(), + "hostWaitMs", hostWait.Milliseconds(), + "computeMs", computeDuration.Milliseconds()) + } + + if isMetering { + computeUnit := billing.ResourceType_name[int32(billing.ResourceType_RESOURCE_TYPE_COMPUTE)] + mrErr := meteringReport.Settle(computeUnit, + capabilities.ResponseMetadata{ + Metering: []capabilities.MeteringNodeDetail{{ + Peer2PeerID: e.localNode.Load().PeerID.String(), + SpendUnit: computeUnit, + SpendValue: strconv.Itoa(int(computeDuration.Milliseconds())), + }}, + CapDON_N: 1, + }, + ) + if mrErr != nil { + lggr.Errorw("could not set metering for compute", "err", mrErr) + } + mrErr = e.meterReports.End(ctx, executionID) + if mrErr != nil { + lggr.Errorw("could not end metering report", "err", mrErr) + } + } + + if execErr != nil { + executionStatus = store.StatusErrored + // Module/host and timeout failures are platform errors by default, but a + // user-origin caperrors.Error propagating from a capability or the guest + // is attributed to the user. + execErrClass = events.ClassifyError(execErr, events.ErrorClassificationSystem) + if errors.Is(execErr, context.DeadlineExceeded) { + executionStatus = store.StatusTimeout + e.metrics.UpdateWorkflowTimeoutDurationHistogram(ctx, int64(executionDuration.Seconds())) + } else { + e.metrics.UpdateWorkflowErrorDurationHistogram(ctx, int64(executionDuration.Seconds())) + } + e.metrics.IncrementWorkflowExecutionFinishedCounter(ctx, executionStatus) + executionLogger.Errorw("Workflow execution failed with module execution error", "status", executionStatus, "durationMs", executionDuration.Milliseconds(), "err", execErr) + return nil // not an error from the caller's perspective. Execution ran, just failed. This is already captured by the deferred lifecycle hook e.cfg.Hooks.OnExecutionError(execErr.Error()) + } + + if e.cfg.DebugMode { + lggr.Debugw("User workflow execution result", "result", result.GetValue(), "err", result.GetError()) + } + + if len(result.GetError()) > 0 { + executionStatus = store.StatusErrored + execErr = errors.New(result.GetError()) + // The user's workflow ran and returned an error: a user failure. + execErrClass = events.ErrorClassificationUser + e.metrics.UpdateWorkflowErrorDurationHistogram(ctx, int64(executionDuration.Seconds())) + e.metrics.With("workflowID", e.cfg.WorkflowID, "workflowName", e.cfg.WorkflowName.String()).IncrementWorkflowExecutionFailedCounter(ctx) + e.metrics.IncrementWorkflowExecutionFinishedCounter(ctx, executionStatus) + executionLogger.Errorw("Workflow execution failed", "status", executionStatus, "durationMs", executionDuration.Milliseconds(), "error", result.GetError()) + return nil // not an error from the caller's perspective. Execution ran, just returned an error. This is already captured by the deferred lifecycle hook e.cfg.Hooks.OnExecutionError(execErr.Error()) + } + + executionStatus = store.StatusCompleted + executionLogger.Infow("Workflow execution finished successfully", "durationMs", executionDuration.Milliseconds()) + e.metrics.UpdateWorkflowCompletedDurationHistogram(ctx, int64(executionDuration.Seconds())) + e.metrics.With("workflowID", e.cfg.WorkflowID, "workflowName", e.cfg.WorkflowName.String()).IncrementWorkflowExecutionSucceededCounter(ctx) + e.metrics.IncrementWorkflowExecutionFinishedCounter(ctx, executionStatus) + e.cfg.Hooks.OnResultReceived(result) + return nil +} + +func (e *baseEngine) secretsFetcher(phaseID string) SecretsFetcher { + if e.cfg.SecretsFetcher != nil { + return e.cfg.SecretsFetcher + } + + return NewSecretsFetcher( + e.metrics, + e.cfg.CapRegistry, + e.logger(), + e.cfg.LocalLimiters.SecretsConcurrency, + e.cfg.LocalLimiters.SecretsCalls, + e.cfg.LocalLimiters.Settings, + e.orgID, + e.cfg.WorkflowOwner, + e.cfg.WorkflowName.String(), + e.cfg.WorkflowID, + // phaseID is the executionID if called during an execution, + // or the workflowID if called during trigger subscription + phaseID, + e.cfg.WorkflowEncryptionKey, + e.cfg.OverrideFetcher, + ) +} + +func (e *baseEngine) heartbeatLoop(ctx context.Context) { + ticker := time.NewTicker(time.Duration(e.cfg.LocalLimits.HeartbeatFrequencyMs) * time.Millisecond) + defer ticker.Stop() + e.logger().Info("Starting heartbeat loop") + e.metrics.EngineHeartbeatGauge(ctx, 1) + + for { + select { + case <-ctx.Done(): + e.metrics.EngineHeartbeatGauge(ctx, 0) + e.logger().Info("Shutting down heartbeat") + return + case <-ticker.C: + e.logger().Debugw("Engine heartbeat tick", "time", e.cfg.Clock.Now().Format(time.RFC3339)) + e.metrics.IncrementEngineHeartbeatCounter(ctx) + } + } +} + +func (e *baseEngine) deductStandardBalances(ctx context.Context, meteringReport *metering.Report) { + // V2Engine runs the entirety of a module's execution as compute. Ensure that the max execution time can run. + // Add an extra second of metering padding for context cancel propagation + ctxCancelPadding := (time.Millisecond * 1000).Milliseconds() + // Limit is always usable even on a read failure; err is advisory. + workflowExecutionTimeout, err := e.cfg.LocalLimiters.ExecutionTime.Limit(ctx) + if err != nil { + if !limits.IsErrRecoverable(err) { + e.logger().Errorw("Failed to get execution time limit with no usable value; skipping compute deduction", "err", err) + return + } + e.logger().Errorw("Failed to get execution time limit; continuing with the value the limiter returned", "err", err) + e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.ExecutionTimeout.Key) + } + compMs := decimal.NewFromInt(workflowExecutionTimeout.Milliseconds() + ctxCancelPadding) + computeUnit := billing.ResourceType_RESOURCE_TYPE_COMPUTE.String() + + if _, err := meteringReport.Deduct( + computeUnit, + metering.ByResource(computeUnit, "v2-standard-deduction-compute", compMs), + ); err != nil { + e.logger().Errorw("could not deduct balance for capability request", "capReq", "standard-deduction-compute", "err", err) + } +} + +const emitUserLogsTimeout = 30 * time.Second + +// separate call for each workflow execution +func (e *baseEngine) emitUserLogs(ctx context.Context, userLogChan chan *protoevents.LogLine, executionID string, executionLabels map[string]string) { + e.logger().Debugw("Listening for user logs ...") + count := 0 + defer func() { e.logger().Debugw("Listening for user logs done.", "processedLogLines", count) }() + + processLogLine := func(emitCtx context.Context, logLine *protoevents.LogLine) bool { + if e.cfg.DebugMode { + e.logger().Debugf("User log: <<<%s>>>, local node timestamp: %s", logLine.Message, logLine.NodeTimestamp) + } + if err := e.cfg.LocalLimiters.LogEvent.Check(emitCtx, count); err != nil { + if errBoundLimited, ok := errors.AsType[limits.ErrorBoundLimited[int]](err); ok { + e.logger().Warnw("Max user log events per execution reached, dropping event", "maxEvents", errBoundLimited.Limit, "err", err) + return false + } + // A settings read failure should not stop the drain. Fail open instead. + if limits.IsErrRecoverable(err) { + e.logger().Errorw("Failed to check user log event limit; emitting anyway", "err", err) + } else { + e.logger().Errorw("User log event limit could not be evaluated; emitting anyway", "err", err) + } + e.metrics.IncrementLimitCheckUnenforcedCounter(emitCtx, cresettings.Default.PerWorkflow.LogEventLimit.Key) + } + if err := e.cfg.LocalLimiters.LogLine.Check(emitCtx, config.Size(len(logLine.Message))); err != nil { + if errBoundLimited, ok := errors.AsType[limits.ErrorBoundLimited[config.Size]](err); ok { + logLine.Message = logLine.Message[:errBoundLimited.Limit] + " ...(truncated)" + } else { + if limits.IsErrRecoverable(err) { + e.logger().Errorw("Failed to check user log line limit; emitting untruncated", "err", err) + } else { + e.logger().Errorw("User log line limit could not be evaluated; emitting untruncated", "err", err) + } + e.metrics.IncrementLimitCheckUnenforcedCounter(emitCtx, cresettings.Default.PerWorkflow.LogLineLimit.Key) + } + } + + if err := events.EmitUserLogs(emitCtx, executionLabels, []*protoevents.LogLine{logLine}, executionID); err != nil { + e.logger().Errorw("Failed to emit user logs", "err", err) + } + count++ + return true + } + + for { + select { + case <-ctx.Done(): + // Ensure we don't hang shutdown if the user logs channel is not drained. + drainCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), emitUserLogsTimeout) + for { + select { + case <-drainCtx.Done(): + e.logger().Warnw("Timeout reached while draining user logs") + cancel() + return + case logLine, ok := <-userLogChan: + if !ok || !processLogLine(drainCtx, logLine) { + cancel() + return + } + } + } + case logLine, ok := <-userLogChan: + if !ok { + return + } + // The execution context is cancelled the moment the execution completes, + // but this goroutine outlives it and may still have buffered log lines. + // Emit on a context detached from the execution lifecycle. + emitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), emitUserLogsTimeout) + ok = processLogLine(emitCtx, logLine) + cancel() + if !ok { + return + } + } + } +} + +func (e *baseEngine) donTimeRequestTimeout(ctx context.Context, limiter limits.TimeLimiter) time.Duration { + if limiter == nil { + return cresettings.Default.PerWorkflow.DONTime.RequestTimeout.DefaultValue + } + // A zero timeout is a valid, explicitly configured limit and is honoured as-is. + limit, err := limiter.Limit(ctx) + if err != nil { + if !limits.IsErrRecoverable(err) { + e.logger().Errorw("Failed to get DON time request timeout with no usable value; using the compiled default", "err", err) + return cresettings.Default.PerWorkflow.DONTime.RequestTimeout.DefaultValue + } + e.logger().Errorw("Failed to get DON time request timeout; continuing with the value the limiter returned", "err", err) + e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.DONTime.RequestTimeout.Key) + } + return limit +} + +// shutdownCtx builds the close context: bounded by the shutdown timeout and +// carrying the workflow's tenant identity. +func (e *baseEngine) shutdownCtx() (context.Context, context.CancelFunc) { + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*time.Duration(e.cfg.LocalLimits.ShutdownTimeoutMs)) + return contexts.WithCRE(ctx, e.Tenant()), cancel +} + +// closeCommon is the teardown shared by every engine. +func (e *baseEngine) closeCommon(ctx context.Context) { + if err := e.cfg.ExecutionsStore.DeleteByWorkflowID(ctx, e.cfg.WorkflowID); err != nil { + e.logger().Errorw("Failed to purge executions on close", "err", err) + } + + e.cfg.Module.Close() + + if e.cfg.LocalLimiters != nil { + if err := e.cfg.LocalLimiters.EvictWorkflow(e.cfg.WorkflowID); err != nil { + e.logger().Errorw("Failed to evict workflow from scoped limiters", "err", err) + } + } + + // Encourage the Go runtime to release memory back to the OS after tearing + // down the WASM module and execution state. Without this, freed heap pages + // stay resident (MADV_FREE) and CGo/wasmtime freed pages remain in the C + // allocator's free-list, so RSS never drops even though the memory is unused. + runtime.GC() + debug.FreeOSMemory() + + // reset metering mode metric so that a positive value does not persist + e.metrics.UpdateWorkflowMeteringModeGauge(ctx, false) +} + +// buildLabels creates the label slice for the beholder logger based on config and localNode state. +// This is used both during engine creation and when updating labels after a DON configuration change. +func (e *baseEngine) buildLabels(localNode *capabilities.Node) []any { + return []any{ + platform.KeyWorkflowID, e.cfg.WorkflowID, + platform.KeyWorkflowOwner, e.cfg.WorkflowOwner, + platform.KeyWorkflowName, e.cfg.WorkflowName.String(), + platform.KeyWorkflowVersion, platform.ValueWorkflowVersionV2, + platform.KeyDonID, strconv.Itoa(int(localNode.WorkflowDON.ID)), + platform.KeyDonF, strconv.Itoa(int(localNode.WorkflowDON.F)), + platform.KeyDonN, strconv.Itoa(len(localNode.WorkflowDON.Members)), + platform.KeyDonQ, strconv.Itoa(aggregation.ByzantineQuorum( + len(localNode.WorkflowDON.Members), + int(localNode.WorkflowDON.F), + )), + platform.KeyP2PID, localNode.PeerID.String(), + platform.WorkflowRegistryAddress, e.cfg.WorkflowRegistryAddress, + platform.WorkflowRegistryChainSelector, e.cfg.WorkflowRegistryChainSelector, + platform.EngineVersion, platform.ValueWorkflowVersionV2, + platform.DonVersion, strconv.FormatUint(uint64(pinnedWorkflowDonConfigVersion), 10), + platform.KeySDK, e.cfg.SdkName, + } +} + +// eventLabels returns a copy of the current label map with org ID applied. +func (e *baseEngine) eventLabels() map[string]string { + return maps.Clone(*e.loggerLabels.Load()) +} + +// storeLoggerLabels persists base labels and always merges in the resolved org ID. +func (e *baseEngine) storeLoggerLabels(base map[string]string) { + labels := maps.Clone(base) + if e.orgID != "" { + labels[platform.KeyOrganizationID] = e.orgID + } + e.loggerLabels.Store(&labels) +} + +// logger returns the current logger in a thread-safe manner. +// This method should be used instead of accessing e.lggr directly to avoid race conditions +// when the logger is dynamically updated (e.g., when DON configuration changes). +func (e *baseEngine) logger() logger.SugaredLogger { + e.lggrMu.RLock() + defer e.lggrMu.RUnlock() + return e.lggr +} + +// setLogger updates the logger in a thread-safe manner. +// This is called when the DON configuration changes and we need to update the platform.DonVersion label. +func (e *baseEngine) setLogger(lggr logger.SugaredLogger) { + e.lggrMu.Lock() + defer e.lggrMu.Unlock() + e.lggr = lggr +} + +// resolvedOrg holds the result of an organization ID resolution attempt. +type resolvedOrg struct { + // ID is the resolved organization ID, or empty if resolution failed. + ID string + // Err is the error returned by the OrgResolver, or nil if resolution + // succeeded or the resolver was not configured. + Err error + // Reason explains why ID is empty: "resolver_nil" (OrgResolver not + // configured), "resolver_error" (Get returned an error), or + // "empty_response" (Get returned an empty string). Empty on success. + Reason string +} + +// resolveOrgID resolves the organization ID for the given workflow owner. +// If resolution fails, the returned ID is empty and Reason explains why. +// The original error from the resolver (if any) is preserved in Err and +// logged via the provided logger. +func resolveOrgID(ctx context.Context, resolver orgresolver.OrgResolver, workflowOwner string, lggr logger.SugaredLogger) resolvedOrg { + if resolver == nil { + return resolvedOrg{Reason: "resolver_nil"} + } + orgID, err := resolver.Get(ctx, workflowOwner) + if err != nil { + lggr.Warnw("Failed to resolve organization ID, continuing without it", "workflowOwner", workflowOwner, "err", err) + return resolvedOrg{Err: err, Reason: "resolver_error"} + } + if orgID == "" { + return resolvedOrg{Reason: "empty_response"} + } + return resolvedOrg{ID: orgID} +} + +func TriggerRegistrationID(workflowID string, triggerIndex int) string { + return fmt.Sprintf("trigger_reg_%s_%d", workflowID, triggerIndex) +} diff --git a/core/services/workflows/v2/capability_executor.go b/core/services/workflows/v2/capability_executor.go index 8fe7c60cff4..112141d03b5 100644 --- a/core/services/workflows/v2/capability_executor.go +++ b/core/services/workflows/v2/capability_executor.go @@ -31,7 +31,8 @@ import ( var _ host.ExecutionHelper = (*ExecutionHelper)(nil) type ExecutionHelper struct { - *Engine + base *baseEngine + WorkflowExecutionID string ExecutionTimestamp time.Time UserLogChan chan<- *protoevents.LogLine @@ -135,7 +136,7 @@ func (c *ExecutionHelper) CallCapability(ctx context.Context, request *sdkpb.Cap c.callCounts[limiter] = cnt c.mu.Unlock() - free, err := c.capCallsSemaphore.Wait(ctx, 1) + free, err := c.base.capCallsSemaphore.Wait(ctx, 1) if err != nil { return nil, err } @@ -145,9 +146,9 @@ func (c *ExecutionHelper) CallCapability(ctx context.Context, request *sdkpb.Cap } func (c *ExecutionHelper) callCapability(ctx context.Context, request *sdkpb.CapabilityRequest) (*sdkpb.CapabilityResponse, error) { - execLogger := c.logger().With("workflowExecutionID", c.WorkflowExecutionID, "capabilityID", request.Id, "callbackID", request.CallbackId, "method", request.Method) + execLogger := c.base.logger().With("workflowExecutionID", c.WorkflowExecutionID, "capabilityID", request.Id, "callbackID", request.CallbackId, "method", request.Method) // TODO (CAPPL-735): use request.Metadata.WorkflowExecutionId to associate the call with a specific execution - capability, err := c.cfg.CapRegistry.GetExecutable(ctx, request.Id) + capability, err := c.base.cfg.CapRegistry.GetExecutable(ctx, request.Id) if err != nil { return nil, fmt.Errorf("action capability not found: %w, ", err) } @@ -161,7 +162,7 @@ func (c *ExecutionHelper) callCapability(ctx context.Context, request *sdkpb.Cap return nil, fmt.Errorf("capability %q is system-only and cannot be called from a workflow", info.ID) } - localNode := c.localNode.Load() + localNode := c.base.localNode.Load() // If the capability info is missing a DON, then // the capability is local, and we should use the localNode's DON ID. @@ -175,14 +176,14 @@ func (c *ExecutionHelper) callCapability(ctx context.Context, request *sdkpb.Cap donID = localNode.WorkflowDON.ID } - config, err := c.cfg.CapRegistry.ConfigForCapability(ctx, info.ID, donID) + config, err := c.base.cfg.CapRegistry.ConfigForCapability(ctx, info.ID, donID) if err != nil { // not explicitly an error case and more relevant (helpful) logging occurs in the metering package // debug level should be sufficient here execLogger.Debugw("capability config not found", "err", err) } - meterReport, ok := c.meterReports.Get(c.WorkflowExecutionID) + meterReport, ok := c.base.meterReports.Get(c.WorkflowExecutionID) if !ok { execLogger.Error("no metering report found") } @@ -197,7 +198,7 @@ func (c *ExecutionHelper) callCapability(ctx context.Context, request *sdkpb.Cap userSpendLimit.Valid = false var openConcurrentCallSlots int - if openConcurrentCallSlots, err = c.cfg.LocalLimiters.CapabilityConcurrency.Available(ctx); err != nil { + if openConcurrentCallSlots, err = c.base.cfg.LocalLimiters.CapabilityConcurrency.Available(ctx); err != nil { return nil, err } @@ -210,7 +211,7 @@ func (c *ExecutionHelper) callCapability(ctx context.Context, request *sdkpb.Cap config.RestrictedConfig, ), ); err != nil { - c.cfg.Lggr.Errorw("could not deduct balance for capability request", "capReq", request.Id, "capReqCallbackID", request.CallbackId, "err", err) + c.base.cfg.Lggr.Errorw("could not deduct balance for capability request", "capReq", request.Id, "capReqCallbackID", request.CallbackId, "err", err) } } @@ -219,69 +220,69 @@ func (c *ExecutionHelper) callCapability(ctx context.Context, request *sdkpb.Cap Method: request.Method, CapabilityId: request.Id, Metadata: capabilities.RequestMetadata{ - WorkflowOwner: c.cfg.WorkflowOwner, - WorkflowID: c.cfg.WorkflowID, + WorkflowOwner: c.base.cfg.WorkflowOwner, + WorkflowID: c.base.cfg.WorkflowID, WorkflowExecutionID: c.WorkflowExecutionID, - WorkflowName: c.cfg.WorkflowName.Hex(), + WorkflowName: c.base.cfg.WorkflowName.Hex(), WorkflowDonID: localNode.WorkflowDON.ID, WorkflowDonConfigVersion: pinnedWorkflowDonConfigVersion, ReferenceID: strconv.Itoa(int(request.CallbackId)), - DecodedWorkflowName: c.cfg.WorkflowName.String(), + DecodedWorkflowName: c.base.cfg.WorkflowName.String(), SpendLimits: spendLimits, - WorkflowTag: c.cfg.WorkflowTag, + WorkflowTag: c.base.cfg.WorkflowTag, ExecutionTimestamp: c.ExecutionTimestamp, }, Config: values.EmptyMap(), } var creGetter settings.Getter - if c.cfg.LocalLimiters != nil { - creGetter = c.cfg.LocalLimiters.Settings + if c.base.cfg.LocalLimiters != nil { + creGetter = c.base.cfg.LocalLimiters.Settings } propagateOrgIDMeta, _ := cresettings.Default.PropagateOrgIDInRequestMetadata.GetOrDefault(ctx, creGetter) - if propagateOrgIDMeta && c.orgID != "" { - capReq.Metadata.OrgID = c.orgID + if propagateOrgIDMeta && c.base.orgID != "" { + capReq.Metadata.OrgID = c.base.orgID } execLogger.Debug("Executing capability ...") - c.metrics.With(platform.KeyCapabilityID, request.Id).IncrementCapabilityInvocationCounter(ctx) - loggerLabels := c.eventLabels() + c.base.metrics.With(platform.KeyCapabilityID, request.Id).IncrementCapabilityInvocationCounter(ctx) + loggerLabels := c.base.eventLabels() _ = events.EmitCapabilityStartedEvent(ctx, loggerLabels, c.WorkflowExecutionID, request.Id, meteringRef, request.Method) - execCtx, execCancel, err := c.cfg.LocalLimiters.CapabilityCallTime.WithTimeout(ctx) + execCtx, execCancel, err := c.base.cfg.LocalLimiters.CapabilityCallTime.WithTimeout(ctx) if err != nil { return nil, err } defer execCancel() - executionStart := c.cfg.Clock.Now() + executionStart := c.base.cfg.Clock.Now() c.executionProfile.recordStepStart(meteringRef, request.Id, executionStart) capResp, err := capability.Execute(execCtx, capReq) - executionEnd := c.cfg.Clock.Now() + executionEnd := c.base.cfg.Clock.Now() executionDuration := executionEnd.Sub(executionStart) c.executionProfile.recordStepEnd(meteringRef, executionEnd, err != nil) - c.metrics.With(platform.KeyCapabilityID, request.Id).UpdateCapabilityExecutionDurationHistogram(ctx, int64(executionDuration.Seconds())) + c.base.metrics.With(platform.KeyCapabilityID, request.Id).UpdateCapabilityExecutionDurationHistogram(ctx, int64(executionDuration.Seconds())) if err != nil { if capabilityError, ok := errors.AsType[caperrors.Error](err); ok { if capabilityError.Origin() == caperrors.OriginUser { execLogger.Debugw("Capability execution failed with user error", "userErr", err) _ = events.EmitCapabilityFinishedEvent(ctx, loggerLabels, c.WorkflowExecutionID, request.Id, meteringRef, store.StatusCompleted, request.Method, err) - c.metrics.With(platform.KeyCapabilityID, request.Id, platform.KeyCapabilityErrorCode, capabilityError.Code().String()).IncrementCapabilityUserErrorCounter(ctx) + c.base.metrics.With(platform.KeyCapabilityID, request.Id, platform.KeyCapabilityErrorCode, capabilityError.Code().String()).IncrementCapabilityUserErrorCounter(ctx) return nil, fmt.Errorf("capability execution failed with user error: %w", err) } execLogger.Debugw("Capability execution failed with system error", "systemErr", err) _ = events.EmitCapabilityFinishedEvent(ctx, loggerLabels, c.WorkflowExecutionID, request.Id, meteringRef, store.StatusErrored, request.Method, err) - c.metrics.With(platform.KeyCapabilityID, request.Id, platform.KeyCapabilityErrorCode, capabilityError.Code().String()).IncrementCapabilityFailureCounter(ctx) - c.metrics.IncrementTotalWorkflowStepErrorsCounter(ctx) + c.base.metrics.With(platform.KeyCapabilityID, request.Id, platform.KeyCapabilityErrorCode, capabilityError.Code().String()).IncrementCapabilityFailureCounter(ctx) + c.base.metrics.IncrementTotalWorkflowStepErrorsCounter(ctx) return nil, fmt.Errorf("failed to execute capability: %w", err) } execLogger.Debugw("Capability execution failed", "err", err) _ = events.EmitCapabilityFinishedEvent(ctx, loggerLabels, c.WorkflowExecutionID, request.Id, meteringRef, store.StatusErrored, request.Method, err) - c.metrics.With(platform.KeyCapabilityID, request.Id, platform.KeyCapabilityErrorCode, caperrors.Internal.String()).IncrementCapabilityFailureCounter(ctx) - c.metrics.IncrementTotalWorkflowStepErrorsCounter(ctx) + c.base.metrics.With(platform.KeyCapabilityID, request.Id, platform.KeyCapabilityErrorCode, caperrors.Internal.String()).IncrementCapabilityFailureCounter(ctx) + c.base.metrics.IncrementTotalWorkflowStepErrorsCounter(ctx) return nil, fmt.Errorf("failed to execute capability: %w", err) } @@ -313,7 +314,7 @@ func (c *ExecutionHelper) EmitUserLog(msg string) error { }: // Successfully sent to channel default: - c.logger().Warnw("Exceeded max allowed user log messages, dropping") + c.base.logger().Warnw("Exceeded max allowed user log messages, dropping") } return nil } @@ -337,7 +338,7 @@ func (c *ExecutionHelper) EmitUserMetric(ctx context.Context, metric *eventsv2.W return err } metric.Name = userMetricPrefix + metric.Name + suffix - return events.EmitUserMetric(ctx, c.eventLabels(), metric) + return events.EmitUserMetric(ctx, c.base.eventLabels(), metric) } // dontimeCapabilityID matches the capability ID registered by the OCR2 delegate diff --git a/core/services/workflows/v2/capability_executor_test.go b/core/services/workflows/v2/capability_executor_test.go index 815750dd39a..2ed9173b250 100644 --- a/core/services/workflows/v2/capability_executor_test.go +++ b/core/services/workflows/v2/capability_executor_test.go @@ -26,12 +26,12 @@ func TestExecutionHelper_SystemCapabilityBlocked(t *testing.T) { resolvedInfo := capabilities.CapabilityInfo{ID: confidentialWorkflowsCapabilityID} reg := stubRegistry{cap: stubExecutableCapability{CapabilityInfo: resolvedInfo}} - engine := &Engine{cfg: &EngineConfig{ + engine := &baseEngine{cfg: &EngineConfig{ Lggr: logger.TestLogger(t), CapRegistry: reg, }} engine.setLogger(commonlogger.Sugared(commonlogger.Test(t))) - exec := &ExecutionHelper{Engine: engine} + exec := &ExecutionHelper{base: engine} req := &sdk.CapabilityRequest{ Id: confidentialWorkflowsCapabilityID, @@ -82,12 +82,12 @@ func TestExecutionHelper_SystemCapabilityResolvedBypass(t *testing.T) { resolvedInfo := capabilities.CapabilityInfo{ID: confidentialWorkflowsCapabilityID} reg := stubRegistry{cap: stubExecutableCapability{CapabilityInfo: resolvedInfo}} - engine := &Engine{cfg: &EngineConfig{ + engine := &baseEngine{cfg: &EngineConfig{ Lggr: logger.TestLogger(t), CapRegistry: reg, }} engine.setLogger(commonlogger.Sugared(commonlogger.Test(t))) - exec := &ExecutionHelper{Engine: engine} + exec := &ExecutionHelper{base: engine} // Request a version strictly below the registered system-only one; the raw // ID is not in the deny map, so the fast-path check does not catch it. @@ -128,12 +128,12 @@ func TestExecutionHelper_VaultAndDonTimeSystemCapabilitiesBlocked(t *testing.T) resolvedInfo := capabilities.CapabilityInfo{ID: tt.registeredID} reg := stubRegistry{cap: stubExecutableCapability{CapabilityInfo: resolvedInfo}} - engine := &Engine{cfg: &EngineConfig{ + engine := &baseEngine{cfg: &EngineConfig{ Lggr: logger.TestLogger(t), CapRegistry: reg, }} engine.setLogger(commonlogger.Sugared(commonlogger.Test(t))) - exec := &ExecutionHelper{Engine: engine} + exec := &ExecutionHelper{base: engine} req := &sdk.CapabilityRequest{ Id: tt.requestID, diff --git a/core/services/workflows/v2/coordinated_engine.go b/core/services/workflows/v2/coordinated_engine.go new file mode 100644 index 00000000000..fefa34130cf --- /dev/null +++ b/core/services/workflows/v2/coordinated_engine.go @@ -0,0 +1,122 @@ +package v2 + +import ( + "context" + "errors" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + + "github.com/smartcontractkit/chainlink-common/pkg/contexts" + sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" +) + +var _ WorkflowEngine = (*coordinatedEngine)(nil) + +// coordinatedEngine is an execution-only workflow engine: it registers no +// triggers itself and instead relies on an external manager to Subscribe, +// register/unregister triggers, call its ExecuteTrigger method and handle trigger acknowledgement. +// coordinatedEngine makes no trigger admission decisions. OnTriggerAdmission is not called. + +// base is a named field rather than embedded so coordinatedEngine's own +// method set only contains what it explicitly delegates below +type coordinatedEngine struct { + base *baseEngine +} + +// NewCoordinatedEngine constructs the execution-only engine. cfg.TriggerAcknowledger +// is required: the engine holds no handles, so it cannot acknowledge by itself. +func NewCoordinatedEngine(cfg *EngineConfig) (WorkflowEngine, error) { + if cfg.TriggerAcknowledger == nil { + return nil, errors.New("trigger acknowledger not set") + } + + base, lggr, err := newBaseEngine(cfg) + if err != nil { + return nil, err + } + + e := &coordinatedEngine{base: base} + base.initServiceEngine(lggr, "WorkflowCoordinatedEngine", e.start, e.close) + return e, nil +} + +func (e *coordinatedEngine) Start(ctx context.Context) error { + return e.base.Start(ctx) +} + +func (e *coordinatedEngine) Close() error { + return e.base.Close() +} + +func (e *coordinatedEngine) Ready() error { + return e.base.Ready() +} + +func (e *coordinatedEngine) HealthReport() map[string]error { + return e.base.HealthReport() +} + +func (e *coordinatedEngine) Name() string { + return e.base.Name() +} + +func (e *coordinatedEngine) ExecuteTrigger(ctx context.Context, event RoutedTriggerEvent) error { + return e.base.ExecuteTrigger(ctx, event) +} + +func (e *coordinatedEngine) Drain() bool { + return e.base.Drain() +} + +func (e *coordinatedEngine) ActiveExecutions() int32 { + return e.base.ActiveExecutions() +} + +func (e *coordinatedEngine) DrainStartedAt() (time.Time, bool) { + return e.base.DrainStartedAt() +} + +func (e *coordinatedEngine) Subscribe(ctx context.Context) ([]*sdkpb.TriggerSubscription, error) { + return e.base.Subscribe(ctx) +} + +func (e *coordinatedEngine) Tenant() contexts.CRE { + return e.base.Tenant() +} + +// IsCoordinated reports that this engine makes no trigger admission decisions. +// An external coordinator must own this +func (e *coordinatedEngine) IsCoordinated() bool { + return true +} + +func (e *coordinatedEngine) start(ctx context.Context) error { + return e.base.startWith(ctx, e.init, nil) +} + +func (e *coordinatedEngine) init(ctx context.Context) { + // Tracer is no-op if DebugMode is false + ctx, span := e.base.tracer.Start(ctx, "workflow_engine_init", + trace.WithAttributes( + attribute.String("version", "v2"), + attribute.String("component", "workflow_engine"), + )) + defer span.End() + + if err := e.base.initDONSubscribe(ctx); err != nil { + e.base.cfg.Hooks.OnInitialized(err) + return + } + + e.base.initDone(ctx) +} + +func (e *coordinatedEngine) close() error { + ctx, cancel := e.base.shutdownCtx() + defer cancel() + + e.base.closeCommon(ctx) + return nil +} diff --git a/core/services/workflows/v2/coordinated_engine_test.go b/core/services/workflows/v2/coordinated_engine_test.go new file mode 100644 index 00000000000..68ed5816214 --- /dev/null +++ b/core/services/workflows/v2/coordinated_engine_test.go @@ -0,0 +1,79 @@ +package v2_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/contexts" + regmocks "github.com/smartcontractkit/chainlink-common/pkg/types/core/mocks" + modulemocks "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host/mocks" + sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" + v2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2" + "github.com/smartcontractkit/chainlink/v2/core/utils/matches" +) + +type noopAcknowledger struct{} + +func (noopAcknowledger) Ack(_ context.Context, _, _, _ string) error { return nil } + +func TestNewCoordinatedEngine_RequiresAcknowledger(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t, nil) + cfg.TriggerAcknowledger = nil + + _, err := v2.NewCoordinatedEngine(cfg) + require.EqualError(t, err, "trigger acknowledger not set") +} + +// TestCoordinatedEngine_ExecuteTrigger drives an external trigger event straight +// into ExecuteTrigger and asserts the execution runs to completion. Unlike the legacy Engine, +// a coordinatedEngine registers no triggers of its own, so this is the only way an event reaches it. +func TestCoordinatedEngine_ExecuteTrigger(t *testing.T) { + t.Parallel() + + capreg := regmocks.NewCapabilitiesRegistry(t) + capreg.EXPECT().LocalNode(matches.AnyContext).Return(newNode(t), nil) + + baseCfg := defaultTestConfig(t, nil) + baseCfg.CapRegistry = capreg + baseCfg.BillingClient = setupMockBillingClient(t) + baseCfg.TriggerAcknowledger = noopAcknowledger{} + + // The execution must reach WASM and return a value. + re := newTestEngine(t, baseCfg, v2.NewCoordinatedEngine, func(module *modulemocks.ModuleV2) { + module.EXPECT().Execute(matches.AnyContext, mock.Anything, mock.Anything). + Return(&sdkpb.ExecutionResult{ + Result: &sdkpb.ExecutionResult_Value{}, + }, nil). + Once() + }) + + ctx := contexts.WithCRE(t.Context(), contexts.CRE{ + Owner: baseCfg.WorkflowOwner, + Workflow: baseCfg.WorkflowID, + }) + event := v2.RoutedTriggerEvent{ + WorkflowID: baseCfg.WorkflowID, + TriggerCapID: "id_0", + TriggerIndex: 0, + ObservedAt: time.Now(), + Event: capabilities.TriggerResponse{ + Event: capabilities.TriggerEvent{ + TriggerType: "basic-trigger@1.0.0", + ID: "coordinated_happy_event", + }, + }, + } + + require.NoError(t, re.engine.ExecuteTrigger(ctx, event)) + + require.Equal(t, "completed", <-re.executionFinishedCh) + require.Equal(t, int32(0), re.errorCalls.Load(), "OnExecutionError should not fire on the happy path") + require.Equal(t, int32(0), re.engine.ActiveExecutions()) +} diff --git a/core/services/workflows/v2/emit_user_logs_test.go b/core/services/workflows/v2/emit_user_logs_test.go index 86aaa964731..316e8342ad9 100644 --- a/core/services/workflows/v2/emit_user_logs_test.go +++ b/core/services/workflows/v2/emit_user_logs_test.go @@ -30,7 +30,7 @@ func TestEngine_emitUserLogs_ProcessesLogsWhenContextCancelled(t *testing.T) { LogLine: &mockLimiter[config.Size]{limit: 1000}, } - e := &Engine{ + e := &baseEngine{ cfg: &EngineConfig{ DebugMode: true, LocalLimiters: limiters, @@ -85,7 +85,7 @@ func TestEngine_emitUserLogs_DrainsUntilTimeout(t *testing.T) { LogLine: &mockLimiter[config.Size]{limit: 1000}, } - e := &Engine{ + e := &baseEngine{ cfg: &EngineConfig{ DebugMode: true, LocalLimiters: limiters, @@ -139,7 +139,7 @@ func TestEngine_emitUserLogs_TeardownOnChannelClose(t *testing.T) { LogLine: &mockLimiter[config.Size]{limit: 1000}, } - e := &Engine{ + e := &baseEngine{ cfg: &EngineConfig{ DebugMode: true, LocalLimiters: limiters, diff --git a/core/services/workflows/v2/engine.go b/core/services/workflows/v2/engine.go index 14b01211478..43c9ff2ed7a 100644 --- a/core/services/workflows/v2/engine.go +++ b/core/services/workflows/v2/engine.go @@ -4,90 +4,39 @@ import ( "context" "errors" "fmt" - "maps" - "runtime" - "runtime/debug" - "strconv" "sync" "sync/atomic" "time" - "github.com/shopspring/decimal" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "go.opentelemetry.io/otel/trace/noop" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "github.com/smartcontractkit/chainlink-common/pkg/aggregation" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" - "github.com/smartcontractkit/chainlink-common/pkg/config" "github.com/smartcontractkit/chainlink-common/pkg/contexts" - "github.com/smartcontractkit/chainlink-common/pkg/custmsg" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/metrics" - "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-common/pkg/services/orgresolver" "github.com/smartcontractkit/chainlink-common/pkg/settings" "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" - "github.com/smartcontractkit/chainlink-common/pkg/workflows" - billing "github.com/smartcontractkit/chainlink-protos/billing/go" sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" - protoevents "github.com/smartcontractkit/chainlink-protos/workflows/go/events" "github.com/smartcontractkit/chainlink/v2/core/platform" - "github.com/smartcontractkit/chainlink/v2/core/services/workflows/events" - "github.com/smartcontractkit/chainlink/v2/core/services/workflows/metering" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/monitoring" - "github.com/smartcontractkit/chainlink/v2/core/services/workflows/store" "github.com/smartcontractkit/chainlink/v2/core/services/workflows/types" - "github.com/smartcontractkit/chainlink/v2/core/utils/safe" ) -var executingWorkflows atomic.Int64 +var _ Acknowledger = (*engine)(nil) +var _ EventSink = (*engine)(nil) +var _ WorkflowEngine = (*engine)(nil) -var ( - ErrDuplicateExecution = errors.New("duplicate execution") - ErrShardDeniedNotOwner = errors.New("shard ownership denied: not owner") - ErrShardDeniedOrchestrator = errors.New("shard ownership denied: orchestrator error") - ErrMeteringReserveFailed = errors.New("metering reserve failed") - ErrAdmissionCache = errors.New("admission: event cached for failover") - - ErrEngineDraining = errors.New("engine is draining") - ErrQueueFull = errors.New("trigger event queue is full") - ErrEnqueueFailed = errors.New("failed to enqueue trigger event") -) - -// Pin config version to 1 to avoid updating forwarder contracts on every single config update. -// Config Version set in CapabilitiesRegistry is included in every report but is irrelevant -// to validation on the forwarder side. What matters is DON ID and the set of signer public keys. -const pinnedWorkflowDonConfigVersion = 1 - -// TODO: remove acknowledger check after CRE-6002 is implemented. -var _ Acknowledger = (*Engine)(nil) -var _ EventSink = (*Engine)(nil) - -type Engine struct { - services.Service - srvcEng *services.Engine - - cfg *EngineConfig - - // lggr is the engine's logger. It is protected by lggrMu to allow safe updates when DON configuration changes. - // IMPORTANT: Do NOT access this field directly. Always use the logger() method to ensure thread-safety. - // Direct access will cause data races when the logger is updated in localNodeSync(). - lggr logger.SugaredLogger - - // lggrMu protects lggr during dynamic updates when DON configuration changes. - // Write access (Lock): Only in setLogger() called from localNodeSync() - // Read access (RLock): In logger() method called from everywhere else - lggrMu sync.RWMutex - - loggerLabels atomic.Pointer[map[string]string] - localNode atomic.Pointer[capabilities.Node] +// engine is the legacy trigger-owning workflow engine: it wraps the shared +// execution machinery and adds the extra responsibilities of trigger registration, +// handle ownership, acknowledgement, and the node's workflow-count limit. +// +// base is a named field rather than embedded so engine's own method set only +// contains what it explicitly delegates below. +type engine struct { + base *baseEngine workflowLimitUsed atomic.Bool // true if GlobalWorkflowLimit must be freed @@ -98,25 +47,6 @@ type Engine struct { allTriggerEventsQueueCh limits.QueueLimiter[RoutedTriggerEvent] executionsSemaphore limits.ResourcePoolLimiter[int] - capCallsSemaphore limits.ResourcePoolLimiter[int] - - meterReports *metering.Reports - - metrics *monitoring.WorkflowsMetricLabeler - - // tracer is the OTel tracer for this engine. It's a noop tracer when DebugMode is false. - tracer trace.Tracer - - orgID string - // orgIDMissingReason records why orgID is empty ("resolver_nil", - // "resolver_error", or "empty_response"). It is set once in start() and - // read by startExecution() to label the org_id_missing counter. It is only - // meaningful when orgID == "". - orgIDMissingReason string - - draining atomic.Bool - activeExecutions atomic.Int32 - drainStartedAtNs atomic.Int64 } type triggerCapability struct { @@ -125,535 +55,165 @@ type triggerCapability struct { method string } -func TriggerRegistrationID(workflowID string, triggerIndex int) string { - return fmt.Sprintf("trigger_reg_%s_%d", workflowID, triggerIndex) -} - -// buildLabels creates the label slice for the beholder logger based on config and localNode state. -// This is used both during engine creation and when updating labels after a DON configuration change. -func (e *Engine) buildLabels(localNode *capabilities.Node) []any { - return []any{ - platform.KeyWorkflowID, e.cfg.WorkflowID, - platform.KeyWorkflowOwner, e.cfg.WorkflowOwner, - platform.KeyWorkflowName, e.cfg.WorkflowName.String(), - platform.KeyWorkflowVersion, platform.ValueWorkflowVersionV2, - platform.KeyDonID, strconv.Itoa(int(localNode.WorkflowDON.ID)), - platform.KeyDonF, strconv.Itoa(int(localNode.WorkflowDON.F)), - platform.KeyDonN, strconv.Itoa(len(localNode.WorkflowDON.Members)), - platform.KeyDonQ, strconv.Itoa(aggregation.ByzantineQuorum( - len(localNode.WorkflowDON.Members), - int(localNode.WorkflowDON.F), - )), - platform.KeyP2PID, localNode.PeerID.String(), - platform.WorkflowRegistryAddress, e.cfg.WorkflowRegistryAddress, - platform.WorkflowRegistryChainSelector, e.cfg.WorkflowRegistryChainSelector, - platform.EngineVersion, platform.ValueWorkflowVersionV2, - platform.DonVersion, strconv.FormatUint(uint64(pinnedWorkflowDonConfigVersion), 10), - platform.KeySDK, e.cfg.SdkName, - } -} - -// eventLabels returns a copy of the current label map with org ID applied. -func (e *Engine) eventLabels() map[string]string { - return maps.Clone(*e.loggerLabels.Load()) -} - -// storeLoggerLabels persists base labels and always merges in the resolved org ID. -func (e *Engine) storeLoggerLabels(base map[string]string) { - labels := maps.Clone(base) - if e.orgID != "" { - labels[platform.KeyOrganizationID] = e.orgID - } - e.loggerLabels.Store(&labels) -} - -// logger returns the current logger in a thread-safe manner. -// This method should be used instead of accessing e.lggr directly to avoid race conditions -// when the logger is dynamically updated (e.g., when DON configuration changes). -func (e *Engine) logger() logger.SugaredLogger { - e.lggrMu.RLock() - defer e.lggrMu.RUnlock() - return e.lggr -} - -// setLogger updates the logger in a thread-safe manner. -// This is called when the DON configuration changes and we need to update the platform.DonVersion label. -func (e *Engine) setLogger(lggr logger.SugaredLogger) { - e.lggrMu.Lock() - defer e.lggrMu.Unlock() - e.lggr = lggr -} - -// Drain marks the engine as draining and prevents new executions from starting. -// In-flight executions continue to run to completion. -// It returns true only on the first transition to draining. -func (e *Engine) Drain() bool { - started := e.draining.CompareAndSwap(false, true) - if started { - e.drainStartedAtNs.CompareAndSwap(0, time.Now().UnixNano()) - } - e.srvcEng.SetHealthCond("draining", errors.New("engine is draining, pending deletion")) - return started -} - -func (e *Engine) ActiveExecutions() int32 { - return e.activeExecutions.Load() -} - -func (e *Engine) DrainStartedAt() (time.Time, bool) { - ns := e.drainStartedAtNs.Load() - if ns == 0 { - return time.Time{}, false - } - - return time.Unix(0, ns), true -} - -func NewEngine(cfg *EngineConfig) (*Engine, error) { - err := cfg.Validate() - if err != nil { - return nil, fmt.Errorf("invalid config: %w", err) - } - - em, err := monitoring.InitMonitoringResources() +// NewEngine constructs the legacy trigger-owning engine: it registers its own +// triggers, holds the handles, acknowledges through itself, and owns the +// workflow-count limit. +func NewEngine(cfg *EngineConfig) (WorkflowEngine, error) { + base, lggr, err := newBaseEngine(cfg) if err != nil { - return nil, fmt.Errorf("could not initialize monitoring resources: %w", err) - } - - // LocalNode() is expected to be non-blocking at this stage (i.e. the registry is already synced) - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.LocalLimits.LocalNodeTimeoutMs)*time.Millisecond) - defer cancel() - localNode, err := cfg.CapRegistry.LocalNode(ctx) - if err != nil { - return nil, fmt.Errorf("could not get local node state: %w", err) + return nil, err } - // Create engine first so we can use the buildLabels method - engine := &Engine{ - cfg: cfg, + e := &engine{ + base: base, triggers: make(map[string]*triggerCapability), allTriggerEventsQueueCh: cfg.LocalLimiters.TriggerEventQueue, executionsSemaphore: cfg.LocalLimiters.ExecutionConcurrency, - capCallsSemaphore: cfg.LocalLimiters.CapabilityConcurrency, } - // Self-inject: the engine is its own acknowledger in M1. - // In M2, the OCR reporting plugin implements the Acknowledger. + // Self-inject: the engine is its own acknowledger because it holds the + // trigger handles. if cfg.TriggerAcknowledger == nil { - cfg.TriggerAcknowledger = engine + cfg.TriggerAcknowledger = e } - // Build labels using the helper method - labels := engine.buildLabels(&localNode) - - beholderLogger := logger.Sugared(custmsg.NewBeholderLogger(cfg.Lggr, cfg.BeholderEmitter).Named("WorkflowEngine").With(labels...)) - baseLabels := []string{ - platform.KeyWorkflowID, cfg.WorkflowID, - platform.KeyWorkflowOwner, cfg.WorkflowOwner, - platform.KeyWorkflowName, cfg.WorkflowName.String(), - platform.KeySDK, cfg.SdkName, - } - metricsLabeler := monitoring.NewWorkflowsMetricLabeler(metrics.NewLabeler(), em).With(baseLabels...) - labelsMap := make(map[string]string, len(labels)/2) - for i := 0; i < len(labels); i += 2 { - labelsMap[labels[i].(string)] = labels[i+1].(string) - } - - if cfg.DebugMode { - beholderLogger.Errorw("WARNING: Debug mode is enabled, this is not suitable for production") - engine.tracer = otel.Tracer("workflow_engine_v2") - } else { - engine.tracer = noop.NewTracerProvider().Tracer("") - } - - // Store logger and other fields - engine.setLogger(beholderLogger) - engine.meterReports = metering.NewReports(cfg.BillingClient, cfg.WorkflowOwner, cfg.WorkflowID, beholderLogger, labelsMap, metricsLabeler, cfg.WorkflowRegistryAddress, cfg.WorkflowRegistryChainSelector, metering.EngineVersionV2) - engine.metrics = metricsLabeler - engine.loggerLabels.Store(&labelsMap) - engine.localNode.Store(&localNode) - engine.Service, engine.srvcEng = services.Config{ - Name: "WorkflowEngineV2", - Start: engine.start, - Close: engine.close, - }.NewServiceEngine(beholderLogger) - return engine, nil + base.initServiceEngine(lggr, "WorkflowEngineV2", e.start, e.close) + return e, nil } -// resolvedOrg holds the result of an organization ID resolution attempt. -type resolvedOrg struct { - // ID is the resolved organization ID, or empty if resolution failed. - ID string - // Err is the error returned by the OrgResolver, or nil if resolution - // succeeded or the resolver was not configured. - Err error - // Reason explains why ID is empty: "resolver_nil" (OrgResolver not - // configured), "resolver_error" (Get returned an error), or - // "empty_response" (Get returned an empty string). Empty on success. - Reason string +func (e *engine) Start(ctx context.Context) error { + return e.base.Start(ctx) } -// ExecuteTrigger is the engine's single execution entry point. It performs no admission control, the caller is responsible for those. -func (e *Engine) ExecuteTrigger(ctx context.Context, event RoutedTriggerEvent) error { - e.activeExecutions.Add(1) - defer e.activeExecutions.Add(-1) - - eventID := event.Event.Event.ID - e.logger().Debugw("Scheduling a trigger event for execution", "eventID", eventID) - creCtx := contexts.CREValue(ctx) - // Tracer is no-op if DebugMode is false - ctx, span := e.tracer.Start(ctx, "workflow_execution", - trace.WithAttributes( - attribute.String("workflow_name", e.cfg.WorkflowName.String()), - attribute.String("version", "v2"), - attribute.String("org_id", creCtx.Org), - attribute.String("owner_id", creCtx.Owner), - attribute.String("workflow_id", creCtx.Workflow), - )) - defer span.End() - - return e.startExecution(ctx, event) +func (e *engine) Close() error { + return e.base.Close() } -// Subscribe issues the WASM Subscribe request and returns the validated trigger subscriptions. -func (e *Engine) Subscribe(ctx context.Context) ([]*sdkpb.TriggerSubscription, error) { - // call into the workflow to get trigger subscriptions - subCtx, subCancel, err := e.cfg.LocalLimiters.TriggerSubscriptionTime.WithTimeout(ctx) - if err != nil { - return nil, err - } - defer subCancel() - - maxUserLogEventsPerExecution, err := e.cfg.LocalLimiters.LogEvent.Limit(ctx) - if err != nil { - if !limits.IsErrRecoverable(err) { - return nil, err - } - e.logger().Errorw("Failed to get log event limit; continuing with the value the limiter returned", "err", err) - e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.LogEventLimit.Key) - } - userLogChan := make(chan *protoevents.LogLine, maxUserLogEventsPerExecution) - defer close(userLogChan) - e.srvcEng.GoCtx(subCtx, func(ctx context.Context) { - e.emitUserLogs(ctx, userLogChan, e.cfg.WorkflowID, e.eventLabels()) - }) - - var timeProvider TimeProvider = &types.LocalTimeProvider{} - if !e.cfg.UseLocalTimeProvider { - timeProvider = NewDonTimeProvider(e.cfg.DonTimeStore, e.cfg.WorkflowID, e.donTimeRequestTimeout(subCtx, e.cfg.LocalLimiters.DONTimeRequestTimeout), e.logger(), e.metrics, e.cfg.Clock) - } - - moduleExecuteMaxResponseSizeBytes, err := e.cfg.LocalLimiters.ExecutionResponse.Limit(ctx) - if err != nil { - if !limits.IsErrRecoverable(err) { - return nil, err - } - e.logger().Errorw("Failed to get execution response size limit; continuing with the value the limiter returned", "err", err) - e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.ExecutionResponseLimit.Key) - } - if moduleExecuteMaxResponseSizeBytes < 0 { - return nil, fmt.Errorf("invalid moduleExecuteMaxResponseSizeBytes; must not be negative: %d", moduleExecuteMaxResponseSizeBytes) - } - result, err := e.cfg.Module.Execute(subCtx, &sdkpb.ExecuteRequest{ - Request: &sdkpb.ExecuteRequest_Subscribe{}, - MaxResponseSize: uint64(moduleExecuteMaxResponseSizeBytes), - Config: e.cfg.WorkflowConfig, - }, NewDisallowedExecutionHelper(e.logger(), userLogChan, timeProvider, e.secretsFetcher(e.cfg.WorkflowID))) - if err != nil { - return nil, fmt.Errorf("failed to execute subscribe: %w", err) - } - if result.GetError() != "" { - return nil, fmt.Errorf("failed to execute subscribe: %s", result.GetError()) - } - subs := result.GetTriggerSubscriptions() - if subs == nil { - return nil, errors.New("subscribe result is nil") - } - err = e.cfg.LocalLimiters.TriggerSubscription.Check(ctx, len(subs.Subscriptions)) - if err != nil { - return nil, err - } - - return subs.Subscriptions, nil +func (e *engine) Ready() error { + return e.base.Ready() } -// Draining returns true if the engine has been marked for deletion and is no longer accepting new trigger events. -func (e *Engine) Draining() bool { - return e.draining.Load() +func (e *engine) HealthReport() map[string]error { + return e.base.HealthReport() } -// Put enqueues a trigger event into the engine's internal queue. It is a transitional method that wraps the existing queue. -// It exists only until the dispatcher owns admission (CRE-6179). At that point the engine's queue is removed and the dispatcher calls HandleTriggerEvent directly. -func (e *Engine) Put(ctx context.Context, event RoutedTriggerEvent) error { // transitional - triggerID := event.TriggerCapID - eventID := event.Event.Event.ID - idx := event.TriggerIndex - - if e.Draining() { - e.logger().Infow("Engine is draining, dropping trigger event before enqueue", "triggerID", triggerID, "eventID", eventID) - tm := e.metrics.With(platform.KeyTriggerID, triggerID) - tm.IncrementTriggerEventEnqueueDroppedCounter(ctx) - tm.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonEnqueueDraining) - e.cfg.Hooks.OnTriggerEventDropped(triggerID, eventID, "draining") - return ErrEngineDraining - } - - // Admission check: an external management layer (e.g. ShardFailoverManager) - // decides whether this event should be processed by the engine. - if err := e.cfg.Hooks.OnTriggerAdmission(ctx, event); err != nil { - if errors.Is(err, ErrAdmissionCache) { - e.logger().Infow("Trigger event cached for failover, not enqueuing", "triggerID", triggerID, "eventID", eventID) - return err - } - // Denied: ACK and drop - registrationID := TriggerRegistrationID(e.cfg.WorkflowID, event.TriggerIndex) - if ackErr := e.cfg.TriggerAcknowledger.Ack(ctx, event.TriggerCapID, registrationID, eventID); ackErr != nil { - e.logger().Errorw("failed to ACK trigger after admission denial", "eventID", eventID, "err", ackErr) - } - e.metrics.With(platform.KeyTriggerID, triggerID).IncrementTriggerEventDroppedTotal(ctx, "admission_denied") - return err - } - - // Stamp the deadline once at dispatch: observedAt + queue timeout. - // If ObservedAt is not set, use the current time. - if event.ObservedAt.IsZero() { - event.ObservedAt = e.cfg.Clock.Now() - } - queueTimeout, err := e.cfg.LocalLimiters.TriggerEventQueueTimeout.Limit(ctx) - if err != nil { - if !limits.IsErrRecoverable(err) { - // No value was resolved, so the deadline below would already be expired. - e.logger().Errorw("Failed to get trigger event queue time limit with no usable value", "err", err) - e.metrics.With(platform.KeyTriggerID, triggerID). - IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonQueueAgeLimitReadFailed) - return ErrEnqueueFailed - } - // A settings read failure is not a reason to drop a customer's trigger event: - // the limiter still returns a usable timeout, so stamp the deadline and continue. - e.logger().Errorw("Failed to get trigger event queue time limit; continuing with the value the limiter returned", "err", err) - e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.TriggerEventQueueTimeout.Key) - } - event.Deadline = event.ObservedAt.Add(queueTimeout) - - if err := e.allTriggerEventsQueueCh.Put(ctx, event); err != nil { - tm := e.metrics.With(platform.KeyTriggerID, triggerID) - tm.IncrementTriggerEventEnqueueDroppedCounter(ctx) - retErr := ErrEnqueueFailed - if _, ok := errors.AsType[limits.ErrorQueueFull](err); ok { - // queue full, drop the event - e.logger().Errorw("Trigger event queue is full, dropping event", "triggerID", triggerID, "triggerIndex", idx, "err", err) - tm.IncrementWorkflowTriggerEventQueueFullCounter(ctx) - tm.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonEnqueueQueueFull) - retErr = ErrQueueFull - } else { - tm.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonEnqueueFailed) - } - e.logger().Errorw("Failed to enqueue trigger event", "triggerID", triggerID, "triggerIndex", idx, "err", err) - tm.IncrementWorkflowTriggerEventErrorCounter(ctx) - return retErr - } - e.metrics.With(platform.KeyTriggerID, triggerID).IncrementTriggerEventEnqueuedCounter(ctx) - e.logger().Debugw("Enqueued trigger event", "triggerID", triggerID, "eventID", eventID) - - return nil +func (e *engine) Name() string { + return e.base.Name() } -// Ack acknowledges a trigger event via the injected TriggerAcknowledger. -// In M1 this is the existing engine's internal acknowledger logic. In M2 the -// OCR reporting plugin implements this to ACK. -func (e *Engine) Ack(ctx context.Context, triggerCapID, triggerRegistrationID, eventID string) error { - e.logger().Infow("ACKing trigger event", "triggerRegistrationID", triggerRegistrationID, "eventID", eventID) - - tm := e.metrics.With(platform.KeyTriggerID, triggerCapID) +func (e *engine) ExecuteTrigger(ctx context.Context, event RoutedTriggerEvent) error { + return e.base.ExecuteTrigger(ctx, event) +} - e.triggersRegMu.Lock() - trigger, ok := e.triggers[triggerRegistrationID] - e.triggersRegMu.Unlock() +func (e *engine) Drain() bool { + return e.base.Drain() +} - if !ok { - tm.IncrementTriggerEventAckFailureCounter(ctx) - return fmt.Errorf("failed to find trigger %s", triggerRegistrationID) - } - err := trigger.AckEvent(ctx, triggerRegistrationID, eventID, trigger.method) - if err != nil { - tm.IncrementTriggerEventAckFailureCounter(ctx) - return err - } - tm.IncrementTriggerEventAckSuccessCounter(ctx) - return nil +func (e *engine) ActiveExecutions() int32 { + return e.base.ActiveExecutions() } -// resolveOrgID resolves the organization ID for the given workflow owner. -// If resolution fails, the returned ID is empty and Reason explains why. -// The original error from the resolver (if any) is preserved in Err and -// logged via the provided logger. -func resolveOrgID(ctx context.Context, resolver orgresolver.OrgResolver, workflowOwner string, lggr logger.SugaredLogger) resolvedOrg { - if resolver == nil { - return resolvedOrg{Reason: "resolver_nil"} - } - orgID, err := resolver.Get(ctx, workflowOwner) - if err != nil { - lggr.Warnw("Failed to resolve organization ID, continuing without it", "workflowOwner", workflowOwner, "err", err) - return resolvedOrg{Err: err, Reason: "resolver_error"} - } - if orgID == "" { - return resolvedOrg{Reason: "empty_response"} - } - return resolvedOrg{ID: orgID} +func (e *engine) DrainStartedAt() (time.Time, bool) { + return e.base.DrainStartedAt() } -func (e *Engine) start(ctx context.Context) error { - e.cfg.Module.Start() - ctx = context.WithoutCancel(ctx) +func (e *engine) Subscribe(ctx context.Context) ([]*sdkpb.TriggerSubscription, error) { + return e.base.Subscribe(ctx) +} - // Resolve the workflow owner's org once at engine startup and treat it as stable - // for the lifetime of this engine instance. If org membership/linking changes, the - // workflow must be restarted to pick up the new org mapping. - resolved := resolveOrgID(ctx, e.cfg.OrgResolver, e.cfg.WorkflowOwner, e.logger()) - e.orgID = resolved.ID - e.orgIDMissingReason = resolved.Reason - e.storeLoggerLabels(e.eventLabels()) +func (e *engine) Tenant() contexts.CRE { + return e.base.Tenant() +} - e.metrics = e.metrics.With(platform.KeyOrganizationID, e.orgID) +// IsCoordinated reports that this engine manages its own trigger registration and acknowledgement. +func (e *engine) IsCoordinated() bool { + return false +} - ctx = contexts.WithCRE(ctx, contexts.CRE{Org: e.orgID, Owner: e.cfg.WorkflowOwner, Workflow: e.cfg.WorkflowID}) - e.srvcEng.GoCtx(ctx, e.heartbeatLoop) - e.srvcEng.GoCtx(ctx, e.init) - e.srvcEng.GoCtx(ctx, e.handleAllTriggerEvents) - return nil +func (e *engine) start(ctx context.Context) error { + return e.base.startWith(ctx, e.init, e.handleAllTriggerEvents) } -func (e *Engine) init(ctx context.Context) { +// init is the legacy initialization: it acquires the workflow-count limit before +// anything else and registers the workflow's triggers before reporting success. +func (e *engine) init(ctx context.Context) { // Tracer is no-op if DebugMode is false - ctx, span := e.tracer.Start(ctx, "workflow_engine_init", + ctx, span := e.base.tracer.Start(ctx, "workflow_engine_init", trace.WithAttributes( attribute.String("version", "v2"), attribute.String("component", "workflow_engine"), )) defer span.End() - // apply global engine instance limits - // TODO(CAPPL-794): consider moving this outside of the engine, into the Syncer - err := e.cfg.GlobalWorkflowLimit.Use(ctx, 1) - if err != nil { - if errLimited, ok := errors.AsType[limits.ErrorResourceLimited[int]](err); ok { - switch errLimited.Scope { - case settings.ScopeOwner: - e.logger().Infow("Per owner workflow count limit reached", "err", err) - e.metrics.IncrementWorkflowLimitPerOwnerCounter(ctx) - e.cfg.Hooks.OnInitialized(types.ErrPerOwnerWorkflowCountLimitReached) - case settings.ScopeGlobal: - e.logger().Infow("Global workflow count limit reached", "err", err) - e.metrics.IncrementWorkflowLimitGlobalCounter(ctx) - e.cfg.Hooks.OnInitialized(types.ErrGlobalWorkflowCountLimitReached) - default: - e.logger().Errorw("Workflow count limit reached for unexpected scope", "scope", errLimited.Scope, "err", err) - e.cfg.Hooks.OnInitialized(err) - } - } else { - e.cfg.Hooks.OnInitialized(err) - } + if err := e.useWorkflowLimit(ctx); err != nil { + e.base.cfg.Hooks.OnInitialized(err) return } - e.workflowLimitUsed.Store(true) - - donSubCh, cleanup, err := e.cfg.DonSubscriber.Subscribe(ctx) - if err != nil { - e.logger().Errorw("failed to subscribe to DON notifier", "error", err) - e.cfg.Hooks.OnInitialized(fmt.Errorf("failed to subscribe to DON notifier: %w", err)) + if err := e.base.initDONSubscribe(ctx); err != nil { + e.base.cfg.Hooks.OnInitialized(err) return } - // start loop to sync local node state each time a DON is received on the - // subscribed channel - e.srvcEng.GoCtx(context.WithoutCancel(ctx), func(ctx context.Context) { - defer cleanup() - for { - select { - case <-ctx.Done(): - return - case _, open := <-donSubCh: - if !open { - return - } - e.localNodeSync(ctx) - } - } - }) - - subscriptions, err := e.Subscribe(ctx) + subscriptions, err := e.base.Subscribe(ctx) if err != nil { - e.logger().Errorw("failed to subscribe to triggers", "err", err) - e.cfg.Hooks.OnInitialized(err) + e.base.logger().Errorw("failed to subscribe to triggers", "err", err) + e.base.cfg.Hooks.OnInitialized(err) return } - - cre := contexts.CRE{Org: e.orgID, Owner: e.cfg.WorkflowOwner, Workflow: e.cfg.WorkflowID} - if err = e.cfg.Hooks.OnSubscriptionsReady(subscriptions, cre); err != nil { - e.logger().Errorw("OnSubscriptionsReady hook failed", "err", err) - e.cfg.Hooks.OnInitialized(err) + if err := e.runTriggerSubscriptionPhase(ctx, subscriptions); err != nil { + e.base.logger().Errorw("Workflow Engine initialization failed", "err", err) + e.base.cfg.Hooks.OnInitialized(err) return } + e.base.initDone(ctx) +} - err = e.runTriggerSubscriptionPhase(ctx, subscriptions) - if err != nil { - e.logger().Errorw("Workflow Engine initialization failed", "err", err) - e.cfg.Hooks.OnInitialized(err) - return +// useWorkflowLimit acquires one slot of the node's workflow-count limit. The +// returned error is the one that must reach OnInitialized: the scope-specific +// sentinel for a limit breach, the raw error otherwise. +func (e *engine) useWorkflowLimit(ctx context.Context) error { + if err := e.base.cfg.GlobalWorkflowLimit.Use(ctx, 1); err != nil { + errLimited, ok := errors.AsType[limits.ErrorResourceLimited[int]](err) + if !ok { + return err + } + switch errLimited.Scope { + case settings.ScopeOwner: + e.base.logger().Infow("Per owner workflow count limit reached", "err", err) + e.base.metrics.IncrementWorkflowLimitPerOwnerCounter(ctx) + return types.ErrPerOwnerWorkflowCountLimitReached + case settings.ScopeGlobal: + e.base.logger().Infow("Global workflow count limit reached", "err", err) + e.base.metrics.IncrementWorkflowLimitGlobalCounter(ctx) + return types.ErrGlobalWorkflowCountLimitReached + default: + e.base.logger().Errorw("Workflow count limit reached for unexpected scope", "scope", errLimited.Scope, "err", err) + return err + } } - e.logger().Info("Workflow Engine initialized") - e.metrics.IncrementWorkflowInitializationCounter(ctx) - e.cfg.Hooks.OnInitialized(nil) + e.workflowLimitUsed.Store(true) + return nil } -func (e *Engine) localNodeSync(ctx context.Context) { - to := time.Duration(e.cfg.LocalLimits.LocalNodeTimeoutMs) * time.Millisecond - ctx, cancel := context.WithTimeout(ctx, to) +func (e *engine) close() error { + ctx, cancel := e.base.shutdownCtx() defer cancel() - localNode, err := e.cfg.CapRegistry.LocalNode(ctx) - if err != nil { - e.cfg.Lggr.Errorf("could not get local node state: %s", err) - e.cfg.Hooks.OnNodeSynced(localNode, err) - return - } + e.triggersRegMu.Lock() + e.unregisterAllTriggers(ctx) + e.triggersRegMu.Unlock() + e.base.metrics.IncrementWorkflowUnregisteredCounter(ctx) - // ignore any reads that do not update the config version - if e.localNode.Load().WorkflowDON.ConfigVersion == localNode.WorkflowDON.ConfigVersion { - return - } + e.base.closeCommon(ctx) - e.cfg.Lggr.Debugw("Setting local node state", - "Workflow DON ID", localNode.WorkflowDON.ID, - "Workflow DON Families", localNode.WorkflowDON.Families, - "Workflow DON Config Version (onchain)", localNode.WorkflowDON.ConfigVersion, - "Workflow DON Config Version (pinned)", pinnedWorkflowDonConfigVersion, - ) - - // Publish the new node before updating logger state so concurrent executions - // observe the synced DON while labels are rebuilt. - e.localNode.Store(&localNode) - - // Recreate the beholder logger with updated labels to reflect the new DON version. - labels := e.buildLabels(&localNode) - newLogger := logger.Sugared( - custmsg.NewBeholderLogger(e.cfg.Lggr, e.cfg.BeholderEmitter). - Named("WorkflowEngine"). - With(labels...), - ) - e.setLogger(newLogger) - - labelsMap := make(map[string]string, len(labels)/2) - for i := 0; i < len(labels); i += 2 { - labelsMap[labels[i].(string)] = labels[i+1].(string) + if e.workflowLimitUsed.Load() { + return e.base.cfg.GlobalWorkflowLimit.Free(ctx, 1) } - e.storeLoggerLabels(labelsMap) - - e.cfg.Hooks.OnNodeSynced(localNode, nil) + return nil } -func (e *Engine) runTriggerSubscriptionPhase(ctx context.Context, subscriptions []*sdkpb.TriggerSubscription) error { +func (e *engine) runTriggerSubscriptionPhase(ctx context.Context, subscriptions []*sdkpb.TriggerSubscription) error { // check if all requested triggers exist in the registry triggers := make([]capabilities.TriggerCapability, 0, len(subscriptions)) for _, sub := range subscriptions { @@ -663,7 +223,7 @@ func (e *Engine) runTriggerSubscriptionPhase(ctx context.Context, subscriptions return fmt.Errorf("invalid chain selector for ID %s: %w", sub.Id, err2) } if chainSelector != nil { - err2 := e.cfg.LocalLimiters.ChainAllowed.AllowErr(contexts.WithChainSelector(ctx, *chainSelector)) + err2 := e.base.cfg.LocalLimiters.ChainAllowed.AllowErr(contexts.WithChainSelector(ctx, *chainSelector)) if err2 != nil { if errors.Is(err2, limits.ErrorNotAllowed{}) { return fmt.Errorf("unable to subscribe to capability %s: ChainSelector %d: %w", sub.Id, *chainSelector, err2) @@ -671,7 +231,7 @@ func (e *Engine) runTriggerSubscriptionPhase(ctx context.Context, subscriptions return fmt.Errorf("failed to check access for ChainSelector %d: %w", *chainSelector, err2) } } - triggerCap, triggerErr := e.cfg.CapRegistry.GetTrigger(ctx, sub.Id) + triggerCap, triggerErr := e.base.cfg.CapRegistry.GetTrigger(ctx, sub.Id) if triggerErr != nil { return fmt.Errorf("trigger capability not found: %w", triggerErr) } @@ -679,7 +239,7 @@ func (e *Engine) runTriggerSubscriptionPhase(ctx context.Context, subscriptions } // register to all triggers concurrently - regCtx, regCancel, err := e.cfg.LocalLimiters.TriggerRegistrationsTime.WithTimeout(ctx) + regCtx, regCancel, err := e.base.cfg.LocalLimiters.TriggerRegistrationsTime.WithTimeout(ctx) if err != nil { return err } @@ -703,33 +263,33 @@ func (e *Engine) runTriggerSubscriptionPhase(ctx context.Context, subscriptions for i, sub := range subscriptions { triggerCap := triggers[i] g.Go(func() error { - registrationID := TriggerRegistrationID(e.cfg.WorkflowID, i) + registrationID := TriggerRegistrationID(e.base.cfg.WorkflowID, i) args := []any{"triggerID", sub.Id, "method", sub.Method} if sub.Payload != nil { args = append(args, "payload", protojson.Format(sub.Payload)) } - e.logger().Infow("Registering trigger", args...) + e.base.logger().Infow("Registering trigger", args...) metadata := capabilities.RequestMetadata{ - WorkflowID: e.cfg.WorkflowID, - WorkflowOwner: e.cfg.WorkflowOwner, - WorkflowName: e.cfg.WorkflowName.Hex(), - WorkflowTag: e.cfg.WorkflowTag, - DecodedWorkflowName: e.cfg.WorkflowName.String(), - WorkflowDonID: e.localNode.Load().WorkflowDON.ID, + WorkflowID: e.base.cfg.WorkflowID, + WorkflowOwner: e.base.cfg.WorkflowOwner, + WorkflowName: e.base.cfg.WorkflowName.Hex(), + WorkflowTag: e.base.cfg.WorkflowTag, + DecodedWorkflowName: e.base.cfg.WorkflowName.String(), + WorkflowDonID: e.base.localNode.Load().WorkflowDON.ID, WorkflowDonConfigVersion: pinnedWorkflowDonConfigVersion, ReferenceID: fmt.Sprintf("trigger_%d", i), - WorkflowRegistryChainSelector: e.cfg.WorkflowRegistryChainSelector, - WorkflowRegistryAddress: e.cfg.WorkflowRegistryAddress, + WorkflowRegistryChainSelector: e.base.cfg.WorkflowRegistryChainSelector, + WorkflowRegistryAddress: e.base.cfg.WorkflowRegistryAddress, EngineVersion: platform.ValueWorkflowVersionV2, // no WorkflowExecutionID needed (or available at this stage) } var creGetter settings.Getter - if e.cfg.LocalLimiters != nil { - creGetter = e.cfg.LocalLimiters.Settings + if e.base.cfg.LocalLimiters != nil { + creGetter = e.base.cfg.LocalLimiters.Settings } propagateOrgIDMeta, _ := cresettings.Default.PropagateOrgIDInRequestMetadata.GetOrDefault(gCtx, creGetter) - if propagateOrgIDMeta && e.orgID != "" { - metadata.OrgID = e.orgID + if propagateOrgIDMeta && e.base.orgID != "" { + metadata.OrgID = e.base.orgID } triggerEventCh, regErr := triggerCap.RegisterTrigger(gCtx, capabilities.TriggerRegistrationRequest{ TriggerID: registrationID, @@ -739,8 +299,8 @@ func (e *Engine) runTriggerSubscriptionPhase(ctx context.Context, subscriptions // no Config needed - NoDAG uses Payload }) if regErr != nil { - e.logger().Errorw("Trigger registration failed", "triggerID", sub.Id, "err", regErr) - e.metrics.With(platform.KeyTriggerID, sub.Id).IncrementRegisterTriggerFailureCounter(gCtx) + e.base.logger().Errorw("Trigger registration failed", "triggerID", sub.Id, "err", regErr) + e.base.metrics.With(platform.KeyTriggerID, sub.Id).IncrementRegisterTriggerFailureCounter(gCtx) return fmt.Errorf("failed to register trigger %s: %w", sub.Id, regErr) } // Send successful result @@ -781,14 +341,14 @@ func (e *Engine) runTriggerSubscriptionPhase(ctx context.Context, subscriptions // If any registration failed, unregister successful ones and return error if registrationErr != nil { - e.logger().Errorw("One or more trigger registrations failed - reverting all", "err", registrationErr) + e.base.logger().Errorw("One or more trigger registrations failed - reverting all", "err", registrationErr) e.unregisterAllTriggers(ctx) // needs to be called under e.triggersRegMu lock return registrationErr } // start listening for trigger events only if all registrations succeeded for idx, triggerEventCh := range eventChans { - e.srvcEng.GoCtx(context.WithoutCancel(ctx), func(ctx context.Context) { + e.base.srvcEng.GoCtx(context.WithoutCancel(ctx), func(ctx context.Context) { for { select { case <-ctx.Done(): @@ -799,671 +359,218 @@ func (e *Engine) runTriggerSubscriptionPhase(ctx context.Context, subscriptions } triggerID := subscriptions[idx].Id eventID := event.Event.ID - e.metrics.With(platform.KeyTriggerID, triggerID).IncrementTriggerEventReceivedCounter(ctx) - e.logger().Debugw("Processing trigger event", "triggerID", triggerID, "eventID", eventID) + e.base.metrics.With(platform.KeyTriggerID, triggerID).IncrementTriggerEventReceivedCounter(ctx) + e.base.logger().Debugw("Processing trigger event", "triggerID", triggerID, "eventID", eventID) if event.Err != nil { - e.logger().Errorw("Received a trigger event with error, dropping", "triggerID", triggerID, "err", event.Err) - tm := e.metrics.With(platform.KeyTriggerID, triggerID) + e.base.logger().Errorw("Received a trigger event with error, dropping", "triggerID", triggerID, "err", event.Err) + tm := e.base.metrics.With(platform.KeyTriggerID, triggerID) tm.IncrementWorkflowTriggerEventErrorCounter(ctx) tm.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonTriggerResponseError) continue } routed := RoutedTriggerEvent{ - WorkflowID: e.cfg.WorkflowID, + WorkflowID: e.base.cfg.WorkflowID, TriggerCapID: triggerID, TriggerIndex: idx, - ObservedAt: e.cfg.Clock.Now(), + ObservedAt: e.base.cfg.Clock.Now(), SequenceNumber: 0, Event: event, } - if err := e.Put(ctx, routed); err != nil { + if err := e.put(ctx, routed); err != nil { // Draining is expected during workflow deletion, so it logs at info rather than error level. if errors.Is(err, ErrEngineDraining) { - e.logger().Infow("Dropping trigger event: engine draining", "triggerID", triggerID, "eventID", eventID) + e.base.logger().Infow("Dropping trigger event: engine draining", "triggerID", triggerID, "eventID", eventID) } else { - e.logger().Errorw("Failed to put routed trigger event", "triggerID", triggerID, "eventID", eventID, "err", err) + e.base.logger().Errorw("Failed to put routed trigger event", "triggerID", triggerID, "eventID", eventID, "err", err) } } } } }) } - e.logger().Infow("All triggers registered successfully", "numTriggers", len(subscriptions), "triggerIDs", triggerCapIDs) - e.metrics.IncrementWorkflowRegisteredCounter(ctx) - e.cfg.Hooks.OnSubscribedToTriggers(triggerCapIDs) + e.base.logger().Infow("All triggers registered successfully", "numTriggers", len(subscriptions), "triggerIDs", triggerCapIDs) + e.base.metrics.IncrementWorkflowRegisteredCounter(ctx) + e.base.cfg.Hooks.OnSubscribedToTriggers(triggerCapIDs) return nil } -func (e *Engine) handleAllTriggerEvents(ctx context.Context) { - for { - queueHead, err := e.allTriggerEventsQueueCh.Wait(ctx) +// NOTE: needs to be called under the triggersRegMu lock +func (e *engine) unregisterAllTriggers(ctx context.Context) { + failCount := 0 + for registrationID, trigger := range e.triggers { + err := trigger.UnregisterTrigger(ctx, capabilities.TriggerRegistrationRequest{ + TriggerID: registrationID, + Metadata: capabilities.RequestMetadata{ + WorkflowID: e.base.cfg.WorkflowID, + WorkflowDonID: e.base.localNode.Load().WorkflowDON.ID, + }, + Payload: trigger.payload, + Method: trigger.method, + }) if err != nil { - return - } - eventID := queueHead.Event.Event.ID - triggerMetricLabels := e.metrics.With(platform.KeyTriggerID, queueHead.TriggerCapID) - if e.Draining() { - triggerMetricLabels.IncrementTriggerEventDequeueDroppedCounter(ctx) - triggerMetricLabels.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonDequeueDraining) - e.cfg.Hooks.OnTriggerEventDropped(queueHead.TriggerCapID, eventID, "draining") - e.logger().Infow("Engine is draining, stopping trigger handling loop", "eventID", eventID, "triggerID", queueHead.TriggerCapID) - return - } - - now := e.cfg.Clock.Now() - eventAge := now.Sub(queueHead.ObservedAt) - e.logger().Debugw("Popped a trigger event from the queue", "eventID", eventID, "eventAgeMs", eventAge.Milliseconds()) - triggerMetricLabels.RecordTriggerEventQueueWaitSeconds(ctx, eventAge.Seconds()) - if now.After(queueHead.Deadline) { - e.logger().Warnw("Trigger event is too old, skipping execution", "triggerID", queueHead.TriggerCapID, "eventID", eventID, "eventAgeMs", eventAge.Milliseconds()) - triggerMetricLabels.IncrementTriggerEventExpiredCounter(ctx) - triggerMetricLabels.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonExpired) - continue + e.base.logger().Errorw("Failed to unregister trigger", "registrationId", registrationID, "err", err) + failCount++ } + } + e.base.logger().Infow("All triggers unregistered", "numTriggers", len(e.triggers), "failed", failCount) + e.triggers = make(map[string]*triggerCapability) +} - semWaitStart := e.cfg.Clock.Now() - free, err := e.executionsSemaphore.Wait(ctx, 1) // block if too many concurrent workflow executions - triggerMetricLabels.RecordExecutionSemaphoreWaitSeconds(ctx, e.cfg.Clock.Now().Sub(semWaitStart).Seconds()) - if err != nil { - e.logger().Errorw("Failed to acquire executions semaphore", "err", err) - triggerMetricLabels.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonExecutionSemaphoreWaitFailed) - continue - } +func (e *engine) Ack(ctx context.Context, triggerCapID, triggerRegistrationID, eventID string) error { + e.base.logger().Infow("ACKing trigger event", "triggerRegistrationID", triggerRegistrationID, "eventID", eventID) - e.srvcEng.GoCtx(context.WithoutCancel(ctx), func(ctx context.Context) { - defer free() + tm := e.base.metrics.With(platform.KeyTriggerID, triggerCapID) - // Legacy path: startExecution handles all errors internally (metrics, ACK, hooks). - // This logs eventID context at the call site; the future dispatcher admitter - // (CRE-6176) will use this error for admission decisions. - if err := e.ExecuteTrigger(ctx, queueHead); err != nil { - // Dedup and shard-denial are expected outcomes (the event is handled, - // just not executed here), so they log at info rather than error level. - if errors.Is(err, ErrDuplicateExecution) || errors.Is(err, ErrShardDeniedNotOwner) { - e.logger().Infow("Skipping trigger event execution", "triggerID", queueHead.TriggerCapID, "eventID", queueHead.Event.Event.ID, "err", err) - } else { - e.logger().Errorw("Failed to execute trigger event", "triggerID", queueHead.TriggerCapID, "eventID", queueHead.Event.Event.ID, "err", err) - } - } - }) - } -} + e.triggersRegMu.Lock() + trigger, ok := e.triggers[triggerRegistrationID] + e.triggersRegMu.Unlock() -// startExecution initiates a new workflow execution, blocking until completed -func (e *Engine) startExecution(ctx context.Context, event RoutedTriggerEvent) error { - triggerDrop := func(reason string) { - e.metrics.With(platform.KeyTriggerID, event.TriggerCapID).IncrementTriggerEventDroppedTotal(ctx, reason) + if !ok { + tm.IncrementTriggerEventAckFailureCounter(ctx) + return fmt.Errorf("failed to find trigger %s", triggerRegistrationID) } - - executionID, err := workflows.GenerateExecutionIDWithTriggerIndex(e.cfg.WorkflowID, event.Event.Event.ID, event.TriggerIndex) + err := trigger.AckEvent(ctx, triggerRegistrationID, eventID, trigger.method) if err != nil { - e.logger().Errorw("Failed to generate execution ID", "err", err, "triggerID", event.TriggerCapID) - triggerDrop(monitoring.TriggerDropReasonExecutionIDGenerationFailed) + tm.IncrementTriggerEventAckFailureCounter(ctx) return err } - e.metrics.IncrementExecutionIDFullCounter(ctx) - trace.SpanFromContext(ctx).SetAttributes(attribute.String("execution_id", executionID)) - - loggerLabels := e.eventLabels() - lggr := e.logger().With(platform.KeyOrganizationID, e.orgID) - - var executionTimestamp time.Time - var executionDonTimeProvider TimeProvider - if tsErr := e.cfg.LocalLimiters.ExecutionTimestampsEnabled.AllowErr(ctx); tsErr == nil { - executionDonTimeProvider = NewDonTimeProvider(e.cfg.DonTimeStore, executionID, e.donTimeRequestTimeout(ctx, e.cfg.LocalLimiters.DONTimeRequestTimeout), lggr, e.metrics, e.cfg.Clock) - donTime, dtErr := executionDonTimeProvider.GetDONTime() - if dtErr != nil { - executionTimestamp = e.cfg.Clock.Now() - lggr.Warnw("Failed to get DON time for execution timestamp, falling back to local time", "err", dtErr, "executionTimestamp", executionTimestamp) - e.metrics.IncrementExecutionTimestampFallbackCounter(ctx) - } else { - executionTimestamp = donTime - lggr.Debugw("Execution timestamp assigned", "executionTimestamp", executionTimestamp) - e.metrics.IncrementExecutionTimestampAssignedCounter(ctx) - } - } - - triggerEvent := event.Event.Event - - // disallow duplicate executions - _, addErr := e.cfg.ExecutionsStore.Add(ctx, nil, executionID, e.cfg.WorkflowID, store.StatusStarted) - if addErr != nil { - if errors.Is(addErr, store.ErrDuplicateExecution) { - lggr.Infow("Skipping duplicate execution", "executionID", executionID, "triggerID", event.TriggerCapID, "triggerIndex", event.TriggerIndex) - tm := e.metrics.With(platform.KeyTriggerID, event.TriggerCapID) - tm.IncrementTriggerExecutionDeduplicatedCounter(ctx) - tm.IncrementWorkflowTriggerEventErrorCounter(ctx) - tm.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonDuplicateExecution) - registrationID := TriggerRegistrationID(e.cfg.WorkflowID, event.TriggerIndex) - ackErr := e.cfg.TriggerAcknowledger.Ack(ctx, event.TriggerCapID, registrationID, triggerEvent.ID) - if ackErr != nil { - e.lggr.Errorw("failed to re-ACK trigger event", "eventID", triggerEvent.ID, "err", ackErr) - } - return ErrDuplicateExecution - } - lggr.Errorw("Failed to register execution in store, proceeding anyway", "executionID", executionID, "err", addErr) - } - - var executionStatus string - defer func() { - if executionStatus == "" { - executionStatus = store.StatusErrored - } - if _, finishErr := e.cfg.ExecutionsStore.FinishExecution(ctx, executionID, executionStatus); finishErr != nil { - lggr.Errorw("Failed to finish execution in store", "executionID", executionID, "status", executionStatus, "err", finishErr) - } - }() - - // emitDroppedExecution publishes the Started/Finished pair for an execution abandoned - // before the normal Started/Finished emit points below, so the failure reaches the UI - // instead of vanishing. - emitDroppedExecution := func(cause error, class events.ErrorClassification) { - executionStatus = store.StatusErrored - _ = events.EmitExecutionStartedEvent(ctx, loggerLabels, triggerEvent.ID, executionID) - _ = events.EmitExecutionFinishedEvent(ctx, loggerLabels, store.StatusErrored, executionID, cause, class, lggr) - e.metrics.IncrementWorkflowExecutionFinishedCounter(ctx, store.StatusErrored) - } - - e.metrics.UpdateTotalWorkflowsGauge(ctx, executingWorkflows.Add(1)) - defer e.metrics.UpdateTotalWorkflowsGauge(ctx, executingWorkflows.Add(-1)) - - // TODO(CAPPL-911): add rate-limiting + tm.IncrementTriggerEventAckSuccessCounter(ctx) + return nil +} - meteringReport, meteringErr := e.meterReports.Start(ctx, executionID) - if meteringErr != nil { - lggr.Errorw("could start metering workflow execution. continuing without metering", "err", meteringErr) - } +// errObservedAtMissing guards put's deadline derivation: put computes the queue +// deadline from ObservedAt, so an unstamped event is rejected rather than given +// a garbage deadline. +var errObservedAtMissing = errors.New("trigger event ObservedAt not set") - isMetering := meteringErr == nil - if isMetering { - mrErr := meteringReport.Reserve(ctx) - if mrErr != nil { - lggr.Errorw("could not reserve metering", "err", mrErr) - triggerDrop(monitoring.TriggerDropReasonMeteringReserveFailed) - return ErrMeteringReserveFailed - } +// put enqueues a trigger event into the engine's internal queue. +func (e *engine) put(ctx context.Context, event RoutedTriggerEvent) error { + triggerID := event.TriggerCapID + eventID := event.Event.Event.ID + idx := event.TriggerIndex - e.deductStandardBalances(ctx, meteringReport) + if e.base.Draining() { + e.base.logger().Infow("Engine is draining, dropping trigger event before enqueue", "triggerID", triggerID, "eventID", eventID) + tm := e.base.metrics.With(platform.KeyTriggerID, triggerID) + tm.IncrementTriggerEventEnqueueDroppedCounter(ctx) + tm.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonEnqueueDraining) + e.base.cfg.Hooks.OnTriggerEventDropped(triggerID, eventID, "draining") + return ErrEngineDraining } - // WithTimeout returns a usable ctx/cancel even on a read failure; err is advisory. - execCtx, execCancel, err := e.cfg.LocalLimiters.ExecutionTime.WithTimeout(ctx) - if err != nil { - if !limits.IsErrRecoverable(err) { - lggr.Errorw("Failed to get execution time limit with no usable value", "err", err) - triggerDrop(monitoring.TriggerDropReasonExecutionTimeLimitReadFailed) - emitDroppedExecution(err, events.ErrorClassificationSystem) - if execCancel != nil { - execCancel() // WithTimeout may still have built a context - } + // Admission check: an external management layer (e.g. ShardFailoverManager) + // decides whether this event should be processed by the engine. + if err := e.base.cfg.Hooks.OnTriggerAdmission(ctx, event); err != nil { + if errors.Is(err, ErrAdmissionCache) { + e.base.logger().Infow("Trigger event cached for failover, not enqueuing", "triggerID", triggerID, "eventID", eventID) return err } - lggr.Errorw("Failed to get execution time limit; continuing with the timeout the limiter returned", "err", err) - e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.ExecutionTimeout.Key) - if execCtx == nil { // only nil when the limiter is closed and no ctx was built - execCtx, execCancel = context.WithTimeout(ctx, cresettings.Default.PerWorkflow.ExecutionTimeout.DefaultValue) - } - } - defer execCancel() - triggerCapID := event.TriggerCapID - skewRec := &monitoring.TriggerSkewRecorder{ - EnqueueTime: event.ObservedAt, - Record: func(ctx context.Context, seconds float64, source string) { - e.metrics.With(platform.KeyTriggerID, triggerCapID).RecordTriggerQueueToExecutionStartSeconds(ctx, seconds, source) - }, - } - execCtx = monitoring.ContextWithTriggerSkewRecorder(execCtx, skewRec) - executionLogger := logger.With(lggr, "executionID", executionID, "triggerID", event.TriggerCapID, - "triggerIndex", event.TriggerIndex, "eventID", triggerEvent.ID) - - // This is only a peek to size the user-log channel's burst buffer; LogEvent.Check - // (called per log line in emitUserLogs) is what actually enforces the cap. - maxUserLogEventsPerExecution, err := e.cfg.LocalLimiters.LogEvent.Limit(ctx) - if err != nil { - if !limits.IsErrRecoverable(err) { - lggr.Errorw("Failed to get log event limit with no usable value", "err", err) - triggerDrop(monitoring.TriggerDropReasonLogEventLimitReadFailed) - emitDroppedExecution(err, events.ErrorClassificationSystem) - return err + // Denied: ACK and drop + registrationID := TriggerRegistrationID(e.base.cfg.WorkflowID, event.TriggerIndex) + if ackErr := e.base.cfg.TriggerAcknowledger.Ack(ctx, event.TriggerCapID, registrationID, eventID); ackErr != nil { + e.base.logger().Errorw("failed to ACK trigger after admission denial", "eventID", eventID, "err", ackErr) } - lggr.Errorw("Failed to get log event limit; continuing with the value the limiter returned", "err", err) - e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.LogEventLimit.Key) - } - userLogChan := make(chan *protoevents.LogLine, maxUserLogEventsPerExecution) - defer close(userLogChan) - e.srvcEng.GoCtx(execCtx, func(ctx context.Context) { - e.emitUserLogs(ctx, userLogChan, executionID, loggerLabels) - }) - - tid, err := safe.IntToUint64(event.TriggerIndex) - if err != nil { - executionLogger.Errorw("Failed to convert trigger index to uint64", "err", err) - triggerDrop(monitoring.TriggerDropReasonTriggerIndexInvalid) - emitDroppedExecution(err, events.ErrorClassificationSystem) + e.base.metrics.With(platform.KeyTriggerID, triggerID).IncrementTriggerEventDroppedTotal(ctx, "admission_denied") return err } - startTime := e.cfg.Clock.Now() - executionLogger.Infow("Workflow execution starting ...") - if e.orgID == "" { - e.metrics.IncrementOrgIDMissingCounter(ctx, e.orgIDMissingReason) - } - _ = events.EmitExecutionStartedEvent(ctx, loggerLabels, triggerEvent.ID, executionID) - - registrationID := TriggerRegistrationID(e.cfg.WorkflowID, event.TriggerIndex) - err = e.cfg.TriggerAcknowledger.Ack(ctx, event.TriggerCapID, registrationID, triggerEvent.ID) - if err != nil { - e.lggr.Errorf("failed to ACK trigger event (eventID=%s): %v", triggerEvent.ID, err) - } - e.metrics.With("workflowID", e.cfg.WorkflowID, "workflowName", e.cfg.WorkflowName.String()).IncrementWorkflowExecutionStartedCounter(ctx) - - // Track execution error (and its user/system classification) for deferred - // event emission. Set alongside executionStatus at each outcome site below. - var execErr error - execErrClass := events.ErrorClassificationUnspecified - var execHelper *ExecutionHelper - defer func() { - _ = events.EmitExecutionFinishedEvent(ctx, loggerLabels, executionStatus, executionID, execErr, execErrClass, lggr) - if execHelper != nil { - endTime := e.cfg.Clock.Now() - profile, emitErr := events.EmitExecutionProfile( - ctx, - e.cfg.WorkflowID, - executionID, - startTime, - endTime, - executionStatus, - execHelper.executionProfile.stepInputs(), - ) - if emitErr != nil { - lggr.Errorw("Failed to emit execution profile", "err", emitErr) - } - if profile != nil { - profileJSON, jsonErr := protojson.Marshal(profile) - if jsonErr != nil { - lggr.Errorw("Failed to marshal execution profile to JSON", "err", jsonErr) - } else { - lggr.Infow("Workflow execution profile", "executionProfile", string(profileJSON)) - } - } - } - e.cfg.Hooks.OnExecutionFinished(executionID, executionStatus) - e.cfg.Hooks.OnExecutionStatusUpdate(e.cfg.WorkflowID, executionID, triggerEvent.ID, event.TriggerIndex, executionStatus, execErrClass) - if execErr != nil { - e.cfg.Hooks.OnExecutionError(execErr.Error()) - } - }() - - var timeProvider TimeProvider = &types.LocalTimeProvider{} - if !e.cfg.UseLocalTimeProvider { - if executionDonTimeProvider != nil { - timeProvider = executionDonTimeProvider - } else { - lggr.Warnw("ExecutionTimestampsEnabled is false - creating a new DON time provider") - timeProvider = NewDonTimeProvider(e.cfg.DonTimeStore, executionID, e.donTimeRequestTimeout(execCtx, e.cfg.LocalLimiters.DONTimeRequestTimeout), lggr, e.metrics, e.cfg.Clock) - } + // The deadline below is derived from ObservedAt, so reject events that + // arrive without it rather than silently backfill a garbage deadline. + if event.ObservedAt.IsZero() { + e.base.logger().Errorw("Trigger event missing ObservedAt, dropping", "triggerID", triggerID, "eventID", eventID) + e.base.metrics.With(platform.KeyTriggerID, triggerID).IncrementTriggerEventDroppedTotal(ctx, "observed_at_missing") + return errObservedAtMissing } - - // Track time the guest spends blocked in DON-time host calls so it can be - // excluded from metered compute (see compute_metering.go). The wrapper is - // always installed; whether the recorded wait is subtracted is gated by - // e.cfg.MeterComputeExcludingHostWait below. - suspension := &suspensionTracker{} - timeProvider = newMeasuredTimeProvider(timeProvider, e.cfg.Clock, suspension) - - // Limit is always usable even on a read failure; err is advisory. - moduleExecuteMaxResponseSizeBytes, err := e.cfg.LocalLimiters.ExecutionResponse.Limit(ctx) + queueTimeout, err := e.base.cfg.LocalLimiters.TriggerEventQueueTimeout.Limit(ctx) if err != nil { if !limits.IsErrRecoverable(err) { - execErr = fmt.Errorf("failed to get execution response size limit with no usable value: %w", err) - lggr.Errorw(execErr.Error()) - executionStatus = store.StatusErrored - execErrClass = events.ErrorClassificationSystem - triggerDrop(monitoring.TriggerDropReasonExecutionResponseLimitReadFailed) - return execErr - } - lggr.Errorw("Failed to get execution response size limit; continuing with the value the limiter returned", "err", err) - e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.ExecutionResponseLimit.Key) - } - if moduleExecuteMaxResponseSizeBytes < 0 { - execErr = fmt.Errorf("invalid moduleExecuteMaxResponseSizeBytes; must not be negative: %d", moduleExecuteMaxResponseSizeBytes) - lggr.Errorw(execErr.Error()) - executionStatus = store.StatusErrored - execErrClass = events.ErrorClassificationSystem - triggerDrop(monitoring.TriggerDropReasonExecutionResponseLimitInvalid) - return execErr - } - execHelper = &ExecutionHelper{ - Engine: e, WorkflowExecutionID: executionID, ExecutionTimestamp: executionTimestamp, - UserLogChan: userLogChan, TimeProvider: timeProvider, SecretsFetcher: e.secretsFetcher(executionID), - executionProfile: newExecutionProfileCollector(), - suspension: suspension, - } - - execHelper.initLimiters(e.cfg.LocalLimiters) - e.metrics.With(platform.KeyTriggerID, event.TriggerCapID).RecordTriggerPayloadBytes(ctx, int64(proto.Size(triggerEvent.Payload))) - var result *sdkpb.ExecutionResult - result, execErr = e.cfg.Module.Execute(execCtx, &sdkpb.ExecuteRequest{ - Request: &sdkpb.ExecuteRequest_Trigger{ - Trigger: &sdkpb.Trigger{ - Id: tid, - Payload: triggerEvent.Payload, - }, - }, - MaxResponseSize: uint64(moduleExecuteMaxResponseSizeBytes), - Config: e.cfg.WorkflowConfig, - }, execHelper.PossiblyWithRawSecrets()) - // Non-evictable modules do not record skew; label those as direct. - skewRec.RecordReady(execCtx, monitoring.ModuleLoadSourceDirect) - - endTime := e.cfg.Clock.Now() - executionDuration := endTime.Sub(startTime) - - // computeDuration is what we meter as RESOURCE_TYPE_COMPUTE. We start from the - // end-to-end wall-clock and subtract time the guest spent blocked in DON-time - // host calls: that wait is not compute and is non-deterministic across nodes - // (a node whose DON-time request lands in the current consensus round returns - // in microseconds; one whose request slips to the next round blocks ~a full - // round). Operational latency metrics below intentionally keep using - // executionDuration. - computeDuration := executionDuration - if hostWait := suspension.total(); hostWait > 0 { - computeDuration -= hostWait - if computeDuration < 0 { - computeDuration = 0 - } - executionLogger.Debugw("Excluding DON-time wait from metered compute", - "wallClockMs", executionDuration.Milliseconds(), - "hostWaitMs", hostWait.Milliseconds(), - "computeMs", computeDuration.Milliseconds()) - } - - if isMetering { - computeUnit := billing.ResourceType_name[int32(billing.ResourceType_RESOURCE_TYPE_COMPUTE)] - mrErr := meteringReport.Settle(computeUnit, - capabilities.ResponseMetadata{ - Metering: []capabilities.MeteringNodeDetail{{ - Peer2PeerID: e.localNode.Load().PeerID.String(), - SpendUnit: computeUnit, - SpendValue: strconv.Itoa(int(computeDuration.Milliseconds())), - }}, - CapDON_N: 1, - }, - ) - if mrErr != nil { - lggr.Errorw("could not set metering for compute", "err", mrErr) - } - mrErr = e.meterReports.End(ctx, executionID) - if mrErr != nil { - lggr.Errorw("could not end metering report", "err", mrErr) + // No value was resolved, so the deadline below would already be expired. + e.base.logger().Errorw("Failed to get trigger event queue time limit with no usable value", "err", err) + e.base.metrics.With(platform.KeyTriggerID, triggerID). + IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonQueueAgeLimitReadFailed) + return ErrEnqueueFailed } + // A settings read failure is not a reason to drop a customer's trigger event: + // the limiter still returns a usable timeout, so stamp the deadline and continue. + e.base.logger().Errorw("Failed to get trigger event queue time limit; continuing with the value the limiter returned", "err", err) + e.base.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.TriggerEventQueueTimeout.Key) } + event.Deadline = event.ObservedAt.Add(queueTimeout) - if execErr != nil { - executionStatus = store.StatusErrored - // Module/host and timeout failures are platform errors by default, but a - // user-origin caperrors.Error propagating from a capability or the guest - // is attributed to the user. - execErrClass = events.ClassifyError(execErr, events.ErrorClassificationSystem) - if errors.Is(execErr, context.DeadlineExceeded) { - executionStatus = store.StatusTimeout - e.metrics.UpdateWorkflowTimeoutDurationHistogram(ctx, int64(executionDuration.Seconds())) + if err := e.allTriggerEventsQueueCh.Put(ctx, event); err != nil { + tm := e.base.metrics.With(platform.KeyTriggerID, triggerID) + tm.IncrementTriggerEventEnqueueDroppedCounter(ctx) + retErr := ErrEnqueueFailed + if _, ok := errors.AsType[limits.ErrorQueueFull](err); ok { + // queue full, drop the event + e.base.logger().Errorw("Trigger event queue is full, dropping event", "triggerID", triggerID, "triggerIndex", idx, "err", err) + tm.IncrementWorkflowTriggerEventQueueFullCounter(ctx) + tm.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonEnqueueQueueFull) + retErr = ErrQueueFull } else { - e.metrics.UpdateWorkflowErrorDurationHistogram(ctx, int64(executionDuration.Seconds())) + tm.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonEnqueueFailed) } - e.metrics.IncrementWorkflowExecutionFinishedCounter(ctx, executionStatus) - executionLogger.Errorw("Workflow execution failed with module execution error", "status", executionStatus, "durationMs", executionDuration.Milliseconds(), "err", execErr) - return nil // not an error from the caller's perspective. Execution ran, just failed. This is already captured by the deferred lifecycle hook e.cfg.Hooks.OnExecutionError(execErr.Error()) - } - - if e.cfg.DebugMode { - lggr.Debugw("User workflow execution result", "result", result.GetValue(), "err", result.GetError()) - } - - if len(result.GetError()) > 0 { - executionStatus = store.StatusErrored - execErr = errors.New(result.GetError()) - // The user's workflow ran and returned an error: a user failure. - execErrClass = events.ErrorClassificationUser - e.metrics.UpdateWorkflowErrorDurationHistogram(ctx, int64(executionDuration.Seconds())) - e.metrics.With("workflowID", e.cfg.WorkflowID, "workflowName", e.cfg.WorkflowName.String()).IncrementWorkflowExecutionFailedCounter(ctx) - e.metrics.IncrementWorkflowExecutionFinishedCounter(ctx, executionStatus) - executionLogger.Errorw("Workflow execution failed", "status", executionStatus, "durationMs", executionDuration.Milliseconds(), "error", result.GetError()) - return nil // not an error from the caller's perspective. Execution ran, just returned an error. This is already captured by the deferred lifecycle hook e.cfg.Hooks.OnExecutionError(execErr.Error()) + e.base.logger().Errorw("Failed to enqueue trigger event", "triggerID", triggerID, "triggerIndex", idx, "err", err) + tm.IncrementWorkflowTriggerEventErrorCounter(ctx) + return retErr } + e.base.metrics.With(platform.KeyTriggerID, triggerID).IncrementTriggerEventEnqueuedCounter(ctx) + e.base.logger().Debugw("Enqueued trigger event", "triggerID", triggerID, "eventID", eventID) - executionStatus = store.StatusCompleted - executionLogger.Infow("Workflow execution finished successfully", "durationMs", executionDuration.Milliseconds()) - e.metrics.UpdateWorkflowCompletedDurationHistogram(ctx, int64(executionDuration.Seconds())) - e.metrics.With("workflowID", e.cfg.WorkflowID, "workflowName", e.cfg.WorkflowName.String()).IncrementWorkflowExecutionSucceededCounter(ctx) - e.metrics.IncrementWorkflowExecutionFinishedCounter(ctx, executionStatus) - e.cfg.Hooks.OnResultReceived(result) return nil } -func (e *Engine) secretsFetcher(phaseID string) SecretsFetcher { - if e.cfg.SecretsFetcher != nil { - return e.cfg.SecretsFetcher - } - - return NewSecretsFetcher( - e.metrics, - e.cfg.CapRegistry, - e.logger(), - e.cfg.LocalLimiters.SecretsConcurrency, - e.cfg.LocalLimiters.SecretsCalls, - e.cfg.LocalLimiters.Settings, - e.orgID, - e.cfg.WorkflowOwner, - e.cfg.WorkflowName.String(), - e.cfg.WorkflowID, - // phaseID is the executionID if called during an execution, - // or the workflowID if called during trigger subscription - phaseID, - e.cfg.WorkflowEncryptionKey, - e.cfg.OverrideFetcher, - ) -} - -func (e *Engine) close() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*time.Duration(e.cfg.LocalLimits.ShutdownTimeoutMs)) - defer cancel() - - ctx = contexts.WithCRE(ctx, contexts.CRE{Org: e.orgID, Owner: e.cfg.WorkflowOwner, Workflow: e.cfg.WorkflowID}) - e.triggersRegMu.Lock() - e.unregisterAllTriggers(ctx) - e.triggersRegMu.Unlock() - e.metrics.IncrementWorkflowUnregisteredCounter(ctx) - - if err := e.cfg.ExecutionsStore.DeleteByWorkflowID(ctx, e.cfg.WorkflowID); err != nil { - e.logger().Errorw("Failed to purge executions on close", "err", err) - } - - e.cfg.Module.Close() - - if e.cfg.LocalLimiters != nil { - if err := e.cfg.LocalLimiters.EvictWorkflow(e.cfg.WorkflowID); err != nil { - e.logger().Errorw("Failed to evict workflow from scoped limiters", "err", err) - } - } - - // Encourage the Go runtime to release memory back to the OS after tearing - // down the WASM module and execution state. Without this, freed heap pages - // stay resident (MADV_FREE) and CGo/wasmtime freed pages remain in the C - // allocator's free-list, so RSS never drops even though the memory is unused. - runtime.GC() - debug.FreeOSMemory() - - // reset metering mode metric so that a positive value does not persist - e.metrics.UpdateWorkflowMeteringModeGauge(ctx, false) - var err error - if e.workflowLimitUsed.Load() { // init called Use - err = e.cfg.GlobalWorkflowLimit.Free(ctx, 1) - } - return err -} - -// NOTE: needs to be called under the triggersRegMu lock -func (e *Engine) unregisterAllTriggers(ctx context.Context) { - failCount := 0 - for registrationID, trigger := range e.triggers { - err := trigger.UnregisterTrigger(ctx, capabilities.TriggerRegistrationRequest{ - TriggerID: registrationID, - Metadata: capabilities.RequestMetadata{ - WorkflowID: e.cfg.WorkflowID, - WorkflowDonID: e.localNode.Load().WorkflowDON.ID, - }, - Payload: trigger.payload, - Method: trigger.method, - }) - if err != nil { - e.logger().Errorw("Failed to unregister trigger", "registrationId", registrationID, "err", err) - failCount++ - } - } - e.logger().Infow("All triggers unregistered", "numTriggers", len(e.triggers), "failed", failCount) - e.triggers = make(map[string]*triggerCapability) -} - -func (e *Engine) heartbeatLoop(ctx context.Context) { - ticker := time.NewTicker(time.Duration(e.cfg.LocalLimits.HeartbeatFrequencyMs) * time.Millisecond) - defer ticker.Stop() - e.logger().Info("Starting heartbeat loop") - e.metrics.EngineHeartbeatGauge(ctx, 1) - +// handleAllTriggerEvents drains the engine's trigger-event queue (populated by put method) +// and executes each event in turn. +func (e *engine) handleAllTriggerEvents(ctx context.Context) { for { - select { - case <-ctx.Done(): - e.metrics.EngineHeartbeatGauge(ctx, 0) - e.logger().Info("Shutting down heartbeat") + queueHead, err := e.allTriggerEventsQueueCh.Wait(ctx) + if err != nil { return - case <-ticker.C: - e.logger().Debugw("Engine heartbeat tick", "time", e.cfg.Clock.Now().Format(time.RFC3339)) - e.metrics.IncrementEngineHeartbeatCounter(ctx) } - } -} - -func (e *Engine) deductStandardBalances(ctx context.Context, meteringReport *metering.Report) { - // V2Engine runs the entirety of a module's execution as compute. Ensure that the max execution time can run. - // Add an extra second of metering padding for context cancel propagation - ctxCancelPadding := (time.Millisecond * 1000).Milliseconds() - // Limit is always usable even on a read failure; err is advisory. - workflowExecutionTimeout, err := e.cfg.LocalLimiters.ExecutionTime.Limit(ctx) - if err != nil { - if !limits.IsErrRecoverable(err) { - e.logger().Errorw("Failed to get execution time limit with no usable value; skipping compute deduction", "err", err) + eventID := queueHead.Event.Event.ID + triggerMetricLabels := e.base.metrics.With(platform.KeyTriggerID, queueHead.TriggerCapID) + if e.base.Draining() { + triggerMetricLabels.IncrementTriggerEventDequeueDroppedCounter(ctx) + triggerMetricLabels.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonDequeueDraining) + e.base.cfg.Hooks.OnTriggerEventDropped(queueHead.TriggerCapID, eventID, "draining") + e.base.logger().Infow("Engine is draining, stopping trigger handling loop", "eventID", eventID, "triggerID", queueHead.TriggerCapID) return } - e.logger().Errorw("Failed to get execution time limit; continuing with the value the limiter returned", "err", err) - e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.ExecutionTimeout.Key) - } - compMs := decimal.NewFromInt(workflowExecutionTimeout.Milliseconds() + ctxCancelPadding) - computeUnit := billing.ResourceType_RESOURCE_TYPE_COMPUTE.String() - - if _, err := meteringReport.Deduct( - computeUnit, - metering.ByResource(computeUnit, "v2-standard-deduction-compute", compMs), - ); err != nil { - e.logger().Errorw("could not deduct balance for capability request", "capReq", "standard-deduction-compute", "err", err) - } -} - -const emitUserLogsTimeout = 30 * time.Second - -// separate call for each workflow execution -func (e *Engine) emitUserLogs(ctx context.Context, userLogChan chan *protoevents.LogLine, executionID string, executionLabels map[string]string) { - e.logger().Debugw("Listening for user logs ...") - count := 0 - defer func() { e.logger().Debugw("Listening for user logs done.", "processedLogLines", count) }() - processLogLine := func(emitCtx context.Context, logLine *protoevents.LogLine) bool { - if e.cfg.DebugMode { - e.logger().Debugf("User log: <<<%s>>>, local node timestamp: %s", logLine.Message, logLine.NodeTimestamp) - } - if err := e.cfg.LocalLimiters.LogEvent.Check(emitCtx, count); err != nil { - if errBoundLimited, ok := errors.AsType[limits.ErrorBoundLimited[int]](err); ok { - e.logger().Warnw("Max user log events per execution reached, dropping event", "maxEvents", errBoundLimited.Limit, "err", err) - return false - } - // A settings read failure should not stop the drain. Fail open instead. - if limits.IsErrRecoverable(err) { - e.logger().Errorw("Failed to check user log event limit; emitting anyway", "err", err) - } else { - e.logger().Errorw("User log event limit could not be evaluated; emitting anyway", "err", err) - } - e.metrics.IncrementLimitCheckUnenforcedCounter(emitCtx, cresettings.Default.PerWorkflow.LogEventLimit.Key) - } - if err := e.cfg.LocalLimiters.LogLine.Check(emitCtx, config.Size(len(logLine.Message))); err != nil { - if errBoundLimited, ok := errors.AsType[limits.ErrorBoundLimited[config.Size]](err); ok { - logLine.Message = logLine.Message[:errBoundLimited.Limit] + " ...(truncated)" - } else { - if limits.IsErrRecoverable(err) { - e.logger().Errorw("Failed to check user log line limit; emitting untruncated", "err", err) - } else { - e.logger().Errorw("User log line limit could not be evaluated; emitting untruncated", "err", err) - } - e.metrics.IncrementLimitCheckUnenforcedCounter(emitCtx, cresettings.Default.PerWorkflow.LogLineLimit.Key) - } + now := e.base.cfg.Clock.Now() + eventAge := now.Sub(queueHead.ObservedAt) + e.base.logger().Debugw("Popped a trigger event from the queue", "eventID", eventID, "eventAgeMs", eventAge.Milliseconds()) + triggerMetricLabels.RecordTriggerEventQueueWaitSeconds(ctx, eventAge.Seconds()) + if now.After(queueHead.Deadline) { + e.base.logger().Warnw("Trigger event is too old, skipping execution", "triggerID", queueHead.TriggerCapID, "eventID", eventID, "eventAgeMs", eventAge.Milliseconds()) + triggerMetricLabels.IncrementTriggerEventExpiredCounter(ctx) + triggerMetricLabels.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonExpired) + continue } - if err := events.EmitUserLogs(emitCtx, executionLabels, []*protoevents.LogLine{logLine}, executionID); err != nil { - e.logger().Errorw("Failed to emit user logs", "err", err) + semWaitStart := e.base.cfg.Clock.Now() + free, err := e.executionsSemaphore.Wait(ctx, 1) // block if too many concurrent workflow executions + triggerMetricLabels.RecordExecutionSemaphoreWaitSeconds(ctx, e.base.cfg.Clock.Now().Sub(semWaitStart).Seconds()) + if err != nil { + e.base.logger().Errorw("Failed to acquire executions semaphore", "err", err) + triggerMetricLabels.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonExecutionSemaphoreWaitFailed) + continue } - count++ - return true - } - for { - select { - case <-ctx.Done(): - // Ensure we don't hang shutdown if the user logs channel is not drained. - drainCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), emitUserLogsTimeout) - for { - select { - case <-drainCtx.Done(): - e.logger().Warnw("Timeout reached while draining user logs") - cancel() - return - case logLine, ok := <-userLogChan: - if !ok || !processLogLine(drainCtx, logLine) { - cancel() - return - } + e.base.srvcEng.GoCtx(context.WithoutCancel(ctx), func(ctx context.Context) { + defer free() + + if err := e.ExecuteTrigger(ctx, queueHead); err != nil { + // Dedup is an expected outcome (the event is handled, just not executed here), so it's logged at info rather than error level. + if errors.Is(err, ErrDuplicateExecution) { + e.base.logger().Infow("Skipping trigger event execution", "triggerID", queueHead.TriggerCapID, "eventID", queueHead.Event.Event.ID, "err", err) + } else { + e.base.logger().Errorw("Failed to execute trigger event", "triggerID", queueHead.TriggerCapID, "eventID", queueHead.Event.Event.ID, "err", err) } } - case logLine, ok := <-userLogChan: - if !ok { - return - } - // The execution context is cancelled the moment the execution completes, - // but this goroutine outlives it and may still have buffered log lines. - // Emit on a context detached from the execution lifecycle. - emitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), emitUserLogsTimeout) - ok = processLogLine(emitCtx, logLine) - cancel() - if !ok { - return - } - } - } -} - -func (e *Engine) donTimeRequestTimeout(ctx context.Context, limiter limits.TimeLimiter) time.Duration { - if limiter == nil { - return cresettings.Default.PerWorkflow.DONTime.RequestTimeout.DefaultValue - } - // A zero timeout is a valid, explicitly configured limit and is honoured as-is. - limit, err := limiter.Limit(ctx) - if err != nil { - if !limits.IsErrRecoverable(err) { - e.logger().Errorw("Failed to get DON time request timeout with no usable value; using the compiled default", "err", err) - return cresettings.Default.PerWorkflow.DONTime.RequestTimeout.DefaultValue - } - e.logger().Errorw("Failed to get DON time request timeout; continuing with the value the limiter returned", "err", err) - e.metrics.IncrementLimitReadFallbackCounter(ctx, cresettings.Default.PerWorkflow.DONTime.RequestTimeout.Key) + }) } - return limit } diff --git a/core/services/workflows/v2/engine_labels_test.go b/core/services/workflows/v2/engine_labels_test.go index c678d40ae99..14622f19727 100644 --- a/core/services/workflows/v2/engine_labels_test.go +++ b/core/services/workflows/v2/engine_labels_test.go @@ -15,7 +15,7 @@ func TestEngine_EventLabelsPreservedAfterLabelRebuild(t *testing.T) { WorkflowID: "wf-1", WorkflowOwner: "owner-1", } - engine := &Engine{cfg: cfg, orgID: "test-org"} + engine := &baseEngine{cfg: cfg, orgID: "test-org"} base := map[string]string{platform.KeyWorkflowID: "wf-1"} engine.storeLoggerLabels(base) diff --git a/core/services/workflows/v2/engine_org_id_missing_test.go b/core/services/workflows/v2/engine_org_id_missing_test.go index 61b93efe3ea..d812ed0d54a 100644 --- a/core/services/workflows/v2/engine_org_id_missing_test.go +++ b/core/services/workflows/v2/engine_org_id_missing_test.go @@ -206,7 +206,7 @@ func TestEngine_OrgIDMissingCounter(t *testing.T) { require.NoError(t, err) labeler := monitoring.NewWorkflowsMetricLabeler(metrics.NewLabeler(), em) - engine := &Engine{ + engine := &baseEngine{ cfg: &EngineConfig{WorkflowID: "wf-1", WorkflowOwner: "owner-1"}, orgID: tt.orgID, orgIDMissingReason: tt.reason, diff --git a/core/services/workflows/v2/engine_test.go b/core/services/workflows/v2/engine_test.go index 4d7cdc41467..0eb775c70a6 100644 --- a/core/services/workflows/v2/engine_test.go +++ b/core/services/workflows/v2/engine_test.go @@ -10,6 +10,7 @@ import ( "maps" "strings" "sync" + "sync/atomic" "testing" "time" @@ -248,7 +249,7 @@ WorkflowLimit = "1" cfg.CapRegistry = capreg cfg.GlobalWorkflowLimit = sLimiter cfg.Hooks = hooks - var engine1, engine2, engine3, engine4 *v2.Engine + var engine1, engine2, engine3, engine4 v2.WorkflowEngine t.Run("engine 1 inits successfully", func(t *testing.T) { //nolint:paralleltest // subtests share setup engine1, err = v2.NewEngine(cfg) @@ -2514,51 +2515,9 @@ func TestEngine_ExecuteTrigger(t *testing.T) { }, } } - - // newTestEngine creates a fresh engine + hooks for each subtest. - // setupModule is called to set module expectations BEFORE NewEngine. - // Optional cfgFn overrides are applied to the per-subtest config copy - // (e.g. sharding, billing) before the engine is constructed. - type engineWithChans struct { - engine *v2.Engine - executionFinishedCh chan string // receives status - executionErrorCh chan string // receives error message - resultReceivedCh chan *sdkpb.ExecutionResult - } - newTestEngine := func(t *testing.T, setupModule func(module *modulemocks.ModuleV2), cfgFn ...func(*v2.EngineConfig)) engineWithChans { - t.Helper() - module := modulemocks.NewModuleV2(t) - setupModule(module) - - executionFinishedCh := make(chan string, 1) - executionErrorCh := make(chan string, 1) - resultReceivedCh := make(chan *sdkpb.ExecutionResult, 1) - - testCfg := *baseCfg - testCfg.Module = module - testCfg.Hooks = v2.LifecycleHooks{ - OnExecutionFinished: func(_ string, status string) { - executionFinishedCh <- status - }, - OnExecutionError: func(msg string) { - executionErrorCh <- msg - }, - OnResultReceived: func(res *sdkpb.ExecutionResult) { - resultReceivedCh <- res - }, - } - for _, fn := range cfgFn { - fn(&testCfg) - } - - engine, err := v2.NewEngine(&testCfg) - require.NoError(t, err) - return engineWithChans{engine, executionFinishedCh, executionErrorCh, resultReceivedCh} - } - t.Run("happy path completes with status completed", func(t *testing.T) { t.Parallel() - ew := newTestEngine(t, func(module *modulemocks.ModuleV2) { + ew := newTestEngine(t, baseCfg, v2.NewEngine, func(module *modulemocks.ModuleV2) { module.EXPECT().Execute(matches.AnyContext, mock.Anything, mock.Anything). Return(&sdkpb.ExecutionResult{ Result: &sdkpb.ExecutionResult_Value{}, @@ -2570,18 +2529,14 @@ func TestEngine_ExecuteTrigger(t *testing.T) { require.NoError(t, err) require.Equal(t, "completed", <-ew.executionFinishedCh) - select { - case msg := <-ew.executionErrorCh: - t.Fatalf("unexpected OnExecutionError: %s", msg) - default: - } + require.Equal(t, int32(0), ew.errorCalls.Load(), "OnExecutionError should not fire on the happy path") require.Equal(t, int32(0), ew.engine.ActiveExecutions()) }) t.Run("module execution error returns errored status", func(t *testing.T) { t.Parallel() execErr := errors.New("wasm panic: out of memory") - ew := newTestEngine(t, func(module *modulemocks.ModuleV2) { + ew := newTestEngine(t, baseCfg, v2.NewEngine, func(module *modulemocks.ModuleV2) { module.EXPECT().Execute(matches.AnyContext, mock.Anything, mock.Anything). Return(nil, execErr). Once() @@ -2594,17 +2549,13 @@ func TestEngine_ExecuteTrigger(t *testing.T) { require.Equal(t, "errored", <-ew.executionFinishedCh) require.Contains(t, <-ew.executionErrorCh, "out of memory") - select { - case res := <-ew.resultReceivedCh: - t.Fatalf("OnResultReceived should not fire on error, got: %v", res) - default: - } + require.Equal(t, int32(0), ew.resultCalls.Load(), "OnResultReceived should not fire on error") require.Equal(t, int32(0), ew.engine.ActiveExecutions()) }) t.Run("module result error returns errored status", func(t *testing.T) { t.Parallel() - ew := newTestEngine(t, func(module *modulemocks.ModuleV2) { + ew := newTestEngine(t, baseCfg, v2.NewEngine, func(module *modulemocks.ModuleV2) { module.EXPECT().Execute(matches.AnyContext, mock.Anything, mock.Anything). Return(&sdkpb.ExecutionResult{ Result: &sdkpb.ExecutionResult_Error{ @@ -2619,17 +2570,13 @@ func TestEngine_ExecuteTrigger(t *testing.T) { require.Equal(t, "errored", <-ew.executionFinishedCh) require.Contains(t, <-ew.executionErrorCh, "assertion failed") - select { - case res := <-ew.resultReceivedCh: - t.Fatalf("OnResultReceived should not fire on result error, got: %v", res) - default: - } + require.Equal(t, int32(0), ew.resultCalls.Load(), "OnResultReceived should not fire on result error") require.Equal(t, int32(0), ew.engine.ActiveExecutions()) }) t.Run("duplicate event ID is rejected with ErrDuplicateExecution", func(t *testing.T) { t.Parallel() - ew := newTestEngine(t, func(module *modulemocks.ModuleV2) { + ew := newTestEngine(t, baseCfg, v2.NewEngine, func(module *modulemocks.ModuleV2) { // Only ONE execution should reach Module.Execute. module.EXPECT().Execute(matches.AnyContext, mock.Anything, mock.Anything). Return(&sdkpb.ExecutionResult{ @@ -2649,109 +2596,119 @@ func TestEngine_ExecuteTrigger(t *testing.T) { err = ew.engine.ExecuteTrigger(ctx, event) require.ErrorIs(t, err, v2.ErrDuplicateExecution) - // No second execution should have fired. - select { - case status := <-ew.executionFinishedCh: - t.Fatalf("unexpected second execution with status: %s", status) - case <-time.After(200 * time.Millisecond): - // expected — no second execution - } + // ExecuteTrigger is synchronous, so after it returns the hook counts are + // final: the duplicate must not have produced a second execution. + require.Equal(t, int32(1), ew.finishedCalls.Load(), "duplicate event must not run a second execution") require.Equal(t, int32(0), ew.engine.ActiveExecutions()) }) - t.Run("shard denial not owner returns ErrShardDeniedNotOwner", func(t *testing.T) { + t.Run("metering reserve failure returns ErrMeteringReserveFailed", func(t *testing.T) { t.Parallel() ack := &recordingAcknowledger{} - ew := newTestEngine(t, func(module *modulemocks.ModuleV2) { + ew := newTestEngine(t, baseCfg, v2.NewEngine, func(module *modulemocks.ModuleV2) { // No Module.Execute expectation: the execution must never reach WASM. }, func(cfg *v2.EngineConfig) { + cfg.BillingClient = setupFailingReserveBillingClient(t) cfg.TriggerAcknowledger = ack - cfg.Hooks.OnTriggerAdmission = func(_ context.Context, _ v2.RoutedTriggerEvent) error { - return v2.ErrShardDeniedNotOwner - } }) - err := ew.engine.Put(ctx, makeEvent("shard_not_owner_event")) - require.ErrorIs(t, err, v2.ErrShardDeniedNotOwner) + err := ew.engine.ExecuteTrigger(ctx, makeEvent("metering_reserve_failed_event")) + require.ErrorIs(t, err, v2.ErrMeteringReserveFailed) - // The engine ACKs the skipped event before returning. - registrationID := v2.TriggerRegistrationID(baseCfg.WorkflowID, 0) - require.Equal(t, []string{registrationID + "/shard_not_owner_event"}, ack.ackCalls()) + // No ACK is sent on metering reserve failure; the caller may retry. + require.Empty(t, ack.ackCalls()) - // No execution lifecycle hooks should have fired. - select { - case status := <-ew.executionFinishedCh: - t.Fatalf("unexpected OnExecutionFinished: %s", status) - default: - } - select { - case msg := <-ew.executionErrorCh: - t.Fatalf("unexpected OnExecutionError: %s", msg) - default: - } + // The execution never started, so neither execution hook fired. + require.Equal(t, int32(0), ew.finishedCalls.Load(), "OnExecutionFinished should not fire when reserve fails") + require.Equal(t, int32(0), ew.errorCalls.Load(), "OnExecutionError should not fire when reserve fails") require.Equal(t, int32(0), ew.engine.ActiveExecutions()) }) +} - t.Run("shard resolver error returns ErrShardDeniedOrchestrator", func(t *testing.T) { - t.Parallel() - ack := &recordingAcknowledger{} - ew := newTestEngine(t, func(module *modulemocks.ModuleV2) { - // No Module.execute expectation: the execution must never reach WASM. - }, func(cfg *v2.EngineConfig) { - cfg.TriggerAcknowledger = ack - cfg.Hooks.OnTriggerAdmission = func(_ context.Context, _ v2.RoutedTriggerEvent) error { - return v2.ErrShardDeniedOrchestrator - } - }) +// TestEngine_ShardDenial verifies that when the OnTriggerAdmission lifecycle hook +// denies admission of a trigger, the engine correctly acknowledges and drops the trigger. +func TestEngine_ShardDenial(t *testing.T) { + t.Parallel() - err := ew.engine.Put(ctx, makeEvent("shard_orchestrator_error_event")) - require.ErrorIs(t, err, v2.ErrShardDeniedOrchestrator) + testCases := []struct { + name string + wantErr error + }{ + {name: "not owner", wantErr: v2.ErrShardDeniedNotOwner}, + {name: "orchestrator error", wantErr: v2.ErrShardDeniedOrchestrator}, + } - // The engine ACKs the skipped event before returning. - registrationID := v2.TriggerRegistrationID(baseCfg.WorkflowID, 0) - require.Equal(t, []string{registrationID + "/shard_orchestrator_error_event"}, ack.ackCalls()) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - select { - case status := <-ew.executionFinishedCh: - t.Fatalf("unexpected OnExecutionFinished: %s", status) - default: - } - select { - case msg := <-ew.executionErrorCh: - t.Fatalf("unexpected OnExecutionError: %s", msg) - default: - } - require.Equal(t, int32(0), ew.engine.ActiveExecutions()) - }) + capreg := regmocks.NewCapabilitiesRegistry(t) + capreg.EXPECT().LocalNode(matches.AnyContext).Return(newNode(t), nil) - t.Run("metering reserve failure returns ErrMeteringReserveFailed", func(t *testing.T) { - t.Parallel() - ack := &recordingAcknowledger{} - ew := newTestEngine(t, func(module *modulemocks.ModuleV2) { - // No Module.Execute expectation: the execution must never reach WASM. - }, func(cfg *v2.EngineConfig) { - cfg.BillingClient = setupFailingReserveBillingClient(t) - cfg.TriggerAcknowledger = ack - }) + trigger := capmocks.NewTriggerCapability(t) + capreg.EXPECT().GetTrigger(matches.AnyContext, "id_0").Return(trigger, nil).Once() + eventCh := make(chan capabilities.TriggerResponse) + trigger.EXPECT().RegisterTrigger(matches.AnyContext, mock.Anything).Return(eventCh, nil).Once() + trigger.EXPECT().UnregisterTrigger(matches.AnyContext, mock.Anything).Return(nil).Once() - err := ew.engine.ExecuteTrigger(ctx, makeEvent("metering_reserve_failed_event")) - require.ErrorIs(t, err, v2.ErrMeteringReserveFailed) + baseCfg := defaultTestConfig(t, nil) + baseCfg.CapRegistry = capreg - // No ACK is sent on metering reserve failure; the caller may retry. - require.Empty(t, ack.ackCalls()) + var admissionCalls atomic.Int32 - select { - case status := <-ew.executionFinishedCh: - t.Fatalf("unexpected OnExecutionFinished: %s", status) - default: - } - select { - case msg := <-ew.executionErrorCh: - t.Fatalf("unexpected OnExecutionError: %s", msg) - default: - } - require.Equal(t, int32(0), ew.engine.ActiveExecutions()) - }) + ew := newTestEngine(t, baseCfg, v2.NewEngine, func(module *modulemocks.ModuleV2) { + module.EXPECT().Start() + module.EXPECT().Close() + module.EXPECT().Execute(matches.AnyContext, mock.Anything, mock.Anything).Return(newTriggerSubs(1), nil).Once() + }, func(cfg *v2.EngineConfig) { + cfg.Hooks.OnTriggerAdmission = func(_ context.Context, _ v2.RoutedTriggerEvent) error { + admissionCalls.Add(1) + return tc.wantErr + } + }) + + registrationID := v2.TriggerRegistrationID(baseCfg.WorkflowID, 0) + ackedCh := make(chan struct{}, 1) + event := capabilities.TriggerEvent{ + TriggerType: "basic-trigger@1.0.0", + ID: "shard_denial_event", + } + trigger.EXPECT(). + AckEvent(matches.AnyContext, registrationID, event.ID, mock.Anything). + Run(func(context.Context, string, string, string) { close(ackedCh) }). + Return(nil). + Once() + + require.NoError(t, ew.engine.Start(t.Context())) + require.NoError(t, <-ew.initializedCh) + require.Equal(t, []string{"id_0"}, <-ew.subscribedToTriggersCh) + + // fire the trigger by writing to the channel returned by the mock + // trigger registration + eventCh <- capabilities.TriggerResponse{ + Event: event, + } + + select { + case <-ackedCh: + case <-time.After(5 * time.Second): + t.Fatal("expected the denied event to be ACKed") + } + + // shutdown the engine after acknowledgement + require.NoError(t, ew.engine.Close()) + + // The admission hook is the wiring under test: it must have been + // consulted exactly once for the denied event. + require.Equal(t, int32(1), admissionCalls.Load()) + + // Assert that no other hooks were called and no active executions + // remain + require.Equal(t, int32(0), ew.finishedCalls.Load(), "OnExecutionFinished should not fire for a denied event") + require.Equal(t, int32(0), ew.errorCalls.Load(), "OnExecutionError should not fire for a denied event") + require.Equal(t, int32(0), ew.engine.ActiveExecutions()) + }) + } } // setupMockBillingClient creates a mock billing client with default expectations. @@ -2857,6 +2814,103 @@ func (a *recordingAcknowledger) ackCalls() []string { return append([]string(nil), a.calls...) } +// testEngine wraps a WorkflowEngine with a default implementation of every +// lifecycle hook, recording how many times each fired, alongside channels +// carrying the payloads of the ones tests commonly need to wait on. +type testEngine struct { + engine v2.WorkflowEngine + + initializedCh chan error + subscribedToTriggersCh chan []string + executionFinishedCh chan string // receives status + executionErrorCh chan string // receives error message + resultReceivedCh chan *sdkpb.ExecutionResult + subscriptionsReadyCalls atomic.Int32 + triggerEventDroppedCalls atomic.Int32 + finishedCalls atomic.Int32 + errorCalls atomic.Int32 + executionStatusUpdateCalls atomic.Int32 + resultCalls atomic.Int32 + rateLimitedCalls atomic.Int32 + nodeSyncedCalls atomic.Int32 + triggerAdmissionCalls atomic.Int32 + requirementsSetCalls atomic.Int32 +} + +func newTestEngine( + t *testing.T, + baseCfg *v2.EngineConfig, + newEngine func(*v2.EngineConfig) (v2.WorkflowEngine, error), + setupModule func(module *modulemocks.ModuleV2), + cfgFn ...func(*v2.EngineConfig), +) *testEngine { + t.Helper() + module := modulemocks.NewModuleV2(t) + setupModule(module) + + e := &testEngine{ + initializedCh: make(chan error, 1), + subscribedToTriggersCh: make(chan []string, 1), + executionFinishedCh: make(chan string, 1), + executionErrorCh: make(chan string, 1), + resultReceivedCh: make(chan *sdkpb.ExecutionResult, 1), + } + + testCfg := *baseCfg + testCfg.Module = module + testCfg.Hooks = v2.LifecycleHooks{ + OnInitialized: func(err error) { + e.initializedCh <- err + }, + OnSubscriptionsReady: func(_ []*sdkpb.TriggerSubscription, _ contexts.CRE) error { + e.subscriptionsReadyCalls.Add(1) + return nil + }, + OnSubscribedToTriggers: func(triggerIDs []string) { + e.subscribedToTriggersCh <- triggerIDs + }, + OnTriggerEventDropped: func(_, _, _ string) { + e.triggerEventDroppedCalls.Add(1) + }, + OnExecutionFinished: func(_ string, status string) { + e.finishedCalls.Add(1) + e.executionFinishedCh <- status + }, + OnExecutionError: func(msg string) { + e.errorCalls.Add(1) + e.executionErrorCh <- msg + }, + OnExecutionStatusUpdate: func(_ string, _ string, _ string, _ int, _ string, _ workflowEvents.ErrorClassification) { + e.executionStatusUpdateCalls.Add(1) + }, + OnResultReceived: func(res *sdkpb.ExecutionResult) { + e.resultCalls.Add(1) + e.resultReceivedCh <- res + }, + OnRateLimited: func(_ string) { + e.rateLimitedCalls.Add(1) + }, + OnNodeSynced: func(_ capabilities.Node, _ error) { + e.nodeSyncedCalls.Add(1) + }, + OnTriggerAdmission: func(_ context.Context, _ v2.RoutedTriggerEvent) error { + e.triggerAdmissionCalls.Add(1) + return nil + }, + OnRequirementsSet: func(_ string, _ *sdkpb.Requirements) { + e.requirementsSetCalls.Add(1) + }, + } + for _, fn := range cfgFn { + fn(&testCfg) + } + + engine, err := newEngine(&testCfg) + require.NoError(t, err) + e.engine = engine + return e +} + type observedBaseMessage struct { Msg string Labels map[string]string @@ -3228,7 +3282,7 @@ func createTestEngineForDonVersionTest( registry *capreg.Registry, donNotifier coreCap.DonNotifyWaitSubscriber, emitter custmsg.MessageEmitter, -) (*v2.Engine, *v2.EngineConfig) { +) (v2.WorkflowEngine, *v2.EngineConfig) { lf := limits.Factory{Logger: lggr} name, err := types.NewWorkflowName("test-don-update-workflow") diff --git a/core/services/workflows/v2/types.go b/core/services/workflows/v2/types.go index 4c1560bd099..83339190735 100644 --- a/core/services/workflows/v2/types.go +++ b/core/services/workflows/v2/types.go @@ -5,6 +5,9 @@ import ( "time" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/contexts" + "github.com/smartcontractkit/chainlink-common/pkg/services" + sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" ) // EventSink is how trigger events are delivered to an engine for execution. @@ -23,10 +26,6 @@ import ( // Expected errors: // - ErrDuplicateExecution — the event was already executed (dedup gate). // The engine ACKs the duplicate internally before returning. -// - ErrShardDeniedNotOwner — this node is not the shard owner. -// The engine ACKs the event internally before returning. -// - ErrShardDeniedOrchestrator — shard ownership check failed due to -// orchestrator error. The engine ACKs the event internally before returning. // - ErrMeteringReserveFailed — metering report reservation failed. // No ACK is sent; the caller may retry. // @@ -47,7 +46,7 @@ type EventSink interface { // - Shard ownership denial — this node is not the shard owner; the engine // ACKs to signal the event was processed (skipped). // - Normal execution start — the engine ACKs after the execution begins -// (not shown in the current code path; reserved for M2 dispatcher). +// (not shown in the current code path; reserved for M2 coordinator). // // Ack is idempotent: calling it multiple times for the same event is safe. // The implementation is responsible for looking up the trigger handle by @@ -56,19 +55,52 @@ type Acknowledger interface { Ack(ctx context.Context, triggerCapID, triggerRegistrationID, eventID string) error } +// Subscriber is how a caller obtains an engine's trigger subscriptions on +// demand. Subscribe issues the WASM Subscribe call directly (no caching): the +// engine holds no subscription state of its own, so every call is a fresh +// WASM round trip and callers are responsible for calling it exactly once +// per registration. Tenant identifies the tenant the subscriptions belong to. +type Subscriber interface { + Subscribe(ctx context.Context) ([]*sdkpb.TriggerSubscription, error) + Tenant() contexts.CRE +} + +// Drainable is the graceful-shutdown contract. The syncer has a structurally +// identical local interface (syncer/v2.DrainableService); both are satisfied by +// the same methods, so no cross-package dependency is introduced. +type Drainable interface { + Drain() bool + ActiveExecutions() int32 + DrainStartedAt() (time.Time, bool) +} + +// WorkflowEngine is the contract every workflow engine implementation satisfies, +// independent of which component owns trigger registration and acknowledgement. +type WorkflowEngine interface { + services.Service + EventSink + Drainable + Subscriber + + // IsCoordinated is true if the engine does not manage its own + // trigger registration, trigger dequeuing, execution or acknowledgement. + // Fixed at construction. + IsCoordinated() bool +} + // RoutedTriggerEvent is the canonical trigger event type that flows -// through the dispatch path into the engine. +// through the coordinator path into the engine. type RoutedTriggerEvent struct { WorkflowID string TriggerCapID string TriggerIndex int // ObservedAt is the time the RoutedTriggerEvent was constructed by the - // dispatcher. It is used for skew metrics (queue wait time) + // coordinator. It is used for skew metrics (queue wait time) // and deadline enforcement. ObservedAt time.Time - // Deadline is the expiry of this event in the dispatch queue, + // Deadline is the expiry of this event in the coordinator queue, // stamped once at dispatch as ObservedAt + TriggerEventQueueTimeout. // A settings change after dispatch does not affect already-queued events Deadline time.Time diff --git a/plugins/chainlink.Dockerfile b/plugins/chainlink.Dockerfile index d953d0c392a..7e27d5bd06c 100644 --- a/plugins/chainlink.Dockerfile +++ b/plugins/chainlink.Dockerfile @@ -116,7 +116,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates=20260601~24.04.1 \ gnupg=2.4.4-2ubuntu17.6 \ lsb-release=12.0-2 \ - curl=8.5.0-2ubuntu10.13 \ + curl=8.5.0-2ubuntu10.15 \ && rm -rf /var/lib/apt/lists/* # Install Postgres for CLI tools, needed specifically for DB backups