diff --git a/core/scripts/go.mod b/core/scripts/go.mod index ae6189014b9..c394aed57b8 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -29,7 +29,7 @@ require ( github.com/shopspring/decimal v1.4.0 github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261112211734-e640e1c12078 diff --git a/core/scripts/go.sum b/core/scripts/go.sum index e72e4fadca2..a6f2331b734 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1686,8 +1686,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.16.0 h1:WhDQEEI2Kb5A4C0PNAa+nyGLDvG0H+uT6h4y1NwmkYg= github.com/smartcontractkit/chainlink-ccv v0.16.0/go.mod h1:oaSLVHiJVbJ9aCcUQnoRuVONACWzA/DWVqvUnLAe9bE= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad h1:2thuaxFkbEEnPz2mlUC+AOIp6m5IRduqxThR9mzozU4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 h1:QQpXxddXoJi7b200xLaTI8yO8iKOukebi8WctQotQmY= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb h1:9rVrSeVOAwZsVe2pOpeVRp9XjhtMyjndgeROFglvOhg= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb/go.mod h1:WcOKzF6HkTDrkQ96aBHhcJ2IIYE7iK7M2tdt58ea2qc= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261106154152-9426766b30a9 h1:rs2zmGoDw4AHaJ2Y76J46slPPE3iL801sWPgOTCyDZc= diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index 0f0f50fb310..855f6313efb 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -841,6 +841,8 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err OrgResolver: creServices.OrgResolver, LimitsFactory: limitsFactory, OCRConfigService: creServices.OCRConfigService, + TriggerCoordinator: creServices.TriggerCoordinator, + TriggerQueue: creServices.TriggerQueue, DefaultBootstrappers: safeDefaultBootstrappers(cfg), CapRegistryAddress: safeExternalRegistryAddress(cfg), CapRegistryChainID: safeExternalRegistryChainID(cfg), diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index 1a752ca7e6d..8bff553b1eb 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -135,6 +135,10 @@ type Services struct { WorkflowRegistrySyncer syncerV2.WorkflowRegistrySyncer + TriggerCoordinator triggers.Coordinator + + TriggerQueue triggers.Queue + OrgResolver orgresolver.OrgResolver OCRConfigService capregconfig.OCRConfigService @@ -297,7 +301,7 @@ func (s *Services) newSubservices( wfSyncerLggr.Infof("failed to create billing client: %s", err) } - wfSyncer, wfSyncerSrvcs, err := newWorkflowRegistrySyncerV2( + wfSyncer, triggerCoordinator, triggerQueue, wfSyncerSrvcs, err := newWorkflowRegistrySyncerV2( cfg, relayerChainInterops, billingClient, @@ -320,6 +324,8 @@ func (s *Services) newSubservices( } s.BillingClient = billingClient s.WorkflowRegistrySyncer = wfSyncer + s.TriggerCoordinator = triggerCoordinator + s.TriggerQueue = triggerQueue srvs = append(srvs, wfSyncerSrvcs...) return srvs, nil @@ -800,14 +806,14 @@ func newWorkflowRegistrySyncerV2( meterIdentity resourcemanager.ResourceIdentity, dispatcher remotetypes.Dispatcher, shardIndexMapper *shardownership.ShardIndexMapper, -) (syncerV2.WorkflowRegistrySyncer, []commonsrv.Service, error) { +) (syncerV2.WorkflowRegistrySyncer, triggers.Coordinator, triggers.Queue, []commonsrv.Service, error) { capCfg := cfg.Capabilities() wfReg := capCfg.WorkflowRegistry() key := opts.WorkflowKey fetcherFunc, retrieverFunc, srvcs, err := newFetcherServiceV2(opts, capCfg, lggr, gatewayConnectorWrapper) if err != nil { - return nil, nil, err + return nil, nil, nil, nil, err } artifactsStore, err := artifactsV2.NewStore( @@ -824,29 +830,29 @@ func newWorkflowRegistrySyncerV2( }), ) if err != nil { - return nil, nil, fmt.Errorf("unable to create artifact store: %w", err) + return nil, nil, nil, nil, fmt.Errorf("unable to create artifact store: %w", err) } engineRegistry := syncerV2.NewEngineRegistry() engineLimiters, err := v2.NewLimiters(lf, nil) if err != nil { - return nil, nil, fmt.Errorf("could not instantiate engine limiters: %w", err) + return nil, nil, nil, nil, fmt.Errorf("could not instantiate engine limiters: %w", err) } featureFlags, err := v2.NewFeatureFlags(lf, nil) if err != nil { - return nil, nil, fmt.Errorf("could not instantiate engine feature flags: %w", err) + return nil, nil, nil, nil, fmt.Errorf("could not instantiate engine feature flags: %w", err) } selector, err := chainSelector(wfReg.ChainID(), wfReg.NetworkID()) if err != nil { - return nil, nil, err + return nil, nil, nil, nil, err } crFactory, err := newContractReaderFactory(capCfg, relayerChainInterops) if err != nil { - return nil, nil, fmt.Errorf("failed to instantiate contract reader factory: %w", err) + return nil, nil, nil, nil, fmt.Errorf("failed to instantiate contract reader factory: %w", err) } var shardOrchestratorClient shardorchestrator.ClientInterface @@ -855,7 +861,7 @@ func newWorkflowRegistrySyncerV2( } else { c, clientErr := newShardOrchestratorClient(cfg, lggr) if clientErr != nil { - return nil, nil, clientErr + return nil, nil, nil, nil, clientErr } if c != nil { shardOrchestratorClient = c @@ -902,15 +908,15 @@ func newWorkflowRegistrySyncerV2( coordinatorMetrics, err := wfmonitoring.InitMonitoringResources() if err != nil { - return nil, nil, fmt.Errorf("unable to init trigger coordinator metrics: %w", err) + return nil, nil, nil, nil, fmt.Errorf("unable to init trigger coordinator metrics: %w", err) } centralQueueMetrics, err := wfmonitoring.InitCentralTriggerQueueMetrics() if err != nil { - return nil, nil, fmt.Errorf("unable to init central trigger queue metrics: %w", err) + return nil, nil, nil, nil, fmt.Errorf("unable to init central trigger queue metrics: %w", err) } drainerMetrics, err := wfmonitoring.InitTriggerDrainerMetrics() if err != nil { - return nil, nil, fmt.Errorf("unable to init central trigger queue drainer metrics: %w", err) + return nil, nil, nil, nil, fmt.Errorf("unable to init central trigger queue drainer metrics: %w", err) } coordinatorClock := clockwork.NewRealClock() @@ -971,7 +977,7 @@ func newWorkflowRegistrySyncerV2( }) specMeter, smErr := syncerV2.NewSpecMeter(lggr, rm, meterIdentity, artifactsStore, orgResolver) if smErr != nil { - return nil, nil, fmt.Errorf("unable to create workflow spec meter: %w", smErr) + return nil, nil, nil, nil, fmt.Errorf("unable to create workflow spec meter: %w", smErr) } handlerOpts = append(handlerOpts, syncerV2.WithSpecMeter(specMeter)) } @@ -992,7 +998,7 @@ func newWorkflowRegistrySyncerV2( if diskMonitorEnabled { fileStore, fsErr := artifactsV2.NewFileModuleStore(mc.CacheDir(), cacheEnabled) if fsErr != nil { - return nil, nil, fmt.Errorf("unable to create file module store: %w", fsErr) + return nil, nil, nil, nil, fmt.Errorf("unable to create file module store: %w", fsErr) } dm, dmErr := syncerV2.NewWorkflowModuleCacheDiskMonitor( @@ -1001,7 +1007,7 @@ func newWorkflowRegistrySyncerV2( syncerV2.WorkflowModuleCacheDiskMonitorTickInterval, ) if dmErr != nil { - return nil, nil, fmt.Errorf("unable to create module cache disk monitor: %w", dmErr) + return nil, nil, nil, nil, fmt.Errorf("unable to create module cache disk monitor: %w", dmErr) } srvcs = append(srvcs, dm) @@ -1010,7 +1016,7 @@ func newWorkflowRegistrySyncerV2( if cacheEnabled { cm, cmErr := syncerV2.NewCacheMetrics() if cmErr != nil { - return nil, nil, fmt.Errorf("unable to create module cache metrics: %w", cmErr) + return nil, nil, nil, nil, fmt.Errorf("unable to create module cache metrics: %w", cmErr) } var idleTimeout time.Duration @@ -1060,7 +1066,7 @@ func newWorkflowRegistrySyncerV2( handlerOpts..., ) if err != nil { - return nil, nil, fmt.Errorf("unable to create workflow registry event handler: %w", err) + return nil, nil, nil, nil, fmt.Errorf("unable to create workflow registry event handler: %w", err) } addSources := wfReg.AdditionalSources() @@ -1113,12 +1119,12 @@ func newWorkflowRegistrySyncerV2( registryOpts..., ) if err != nil { - return nil, nil, fmt.Errorf("unable to create workflow registry syncer: %w", err) + return nil, nil, nil, nil, fmt.Errorf("unable to create workflow registry syncer: %w", err) } srvcs = append(srvcs, workflowRegistrySyncerV2) lggr.Debugw("Created WorkflowRegistrySyncer V2") - return workflowRegistrySyncerV2, srvcs, nil + return workflowRegistrySyncerV2, triggerCoordinator, centralQueue, srvcs, nil } // NewServices creates and initializes all CRE services diff --git a/core/services/ocr2/delegate.go b/core/services/ocr2/delegate.go index cc9cacfb536..3f55eee07b8 100644 --- a/core/services/ocr2/delegate.go +++ b/core/services/ocr2/delegate.go @@ -83,6 +83,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" syncerV2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncer/v2" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2/triggers" "github.com/smartcontractkit/chainlink/v2/core/utils" "github.com/smartcontractkit/chainlink/v2/plugins" ) @@ -99,6 +100,7 @@ const ( vaultDiskMonitorTickInterval = time.Minute consensusQueueCapabilityID = internal.ConsensusQueueCapabilityID + consensusQueueOCRConfigKey = "consensus-queue" gaugeConsensusQueueBytes = "platform_consensus_queue_usage_bytes" consensusQueueDiskMonitorTickInterval = time.Minute ) @@ -157,6 +159,8 @@ type Delegate struct { OrgResolver orgresolver.OrgResolver limitsFactory limits.Factory ocrConfigService capregconfig.OCRConfigService + triggerCoordinator triggers.Coordinator + triggerQueue triggers.Queue // Registry-driven launch support: when the delegate is used via // LocalCapabilityManager (no job spec), these fields provide the @@ -310,6 +314,8 @@ type DelegateOpts struct { OrgResolver orgresolver.OrgResolver LimitsFactory limits.Factory OCRConfigService capregconfig.OCRConfigService + TriggerCoordinator triggers.Coordinator + TriggerQueue triggers.Queue // Registry-driven launch support: defaults for capabilities launched // via LocalCapabilityManager (no job spec). @@ -359,6 +365,8 @@ func NewDelegate( OrgResolver: opts.OrgResolver, limitsFactory: opts.LimitsFactory, ocrConfigService: opts.OCRConfigService, + triggerCoordinator: opts.TriggerCoordinator, + triggerQueue: opts.TriggerQueue, defaultBootstrappers: opts.DefaultBootstrappers, capRegistryAddress: opts.CapRegistryAddress, capRegistryChainID: opts.CapRegistryChainID, @@ -895,7 +903,7 @@ func (d *Delegate) newServicesConsensusQueue( BinaryNetworkEndpointFactory: d.peerWrapper.Peer3_1, V2Bootstrappers: bootstrapPeers, ContractConfigTracker: configTracker, - ContractTransmitter: queue.NewTransmitter(lggr, ocrtypes.Account(spec.TransmitterID.String)), + ContractTransmitter: queue.NewTransmitter(ocrtypes.Account(spec.TransmitterID.String)), Database: ocrDB, KeyValueDatabaseFactory: kvFactory, LocalConfig: lc, @@ -907,7 +915,7 @@ func (d *Delegate) newServicesConsensusQueue( MetricsRegisterer: prometheus.WrapRegistererWith(map[string]string{"job_name": jb.Name.ValueOrZero()}, prometheus.DefaultRegisterer), } - rpf, err := queue.NewConsensusQueuePluginFactory(lggr, cfg, d.limitsFactory) + rpf, err := queue.NewConsensusQueuePluginFactory(lggr, cfg, d.limitsFactory, d.triggerCoordinator, d.triggerQueue) if err != nil { return nil, err } diff --git a/core/services/ocr3_1/plugins/queue/blobs.go b/core/services/ocr3_1/plugins/queue/blobs.go new file mode 100644 index 00000000000..1bde88bc86c --- /dev/null +++ b/core/services/ocr3_1/plugins/queue/blobs.go @@ -0,0 +1,133 @@ +package queue + +import ( + "context" + "fmt" + "sync" + "time" + + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/proto" + + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3_1types" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" +) + +// blobCache is a short-lived cache for coordinating concurrent fetches of event blobs. +type blobCache struct { + fetcher ocr3_1types.BlobFetcher + + mu sync.RWMutex + cache map[string]func() ([]byte, error) // blob handle -> blob +} + +func (c *blobCache) fetchBlob(ctx context.Context, handle []byte) ([]byte, error) { + c.mu.RLock() + if fn, ok := c.cache[string(handle)]; ok { + c.mu.RUnlock() + return fn() + } + c.mu.RUnlock() + + c.mu.Lock() + if fn, ok := c.cache[string(handle)]; ok { + c.mu.Unlock() + return fn() + } + fn := sync.OnceValues(func() ([]byte, error) { + var blobHandle ocr3_1types.BlobHandle + if err := blobHandle.UnmarshalBinary(handle); err != nil { + return nil, fmt.Errorf("failed to unmarshal blob handle: %w", err) + } + return c.fetcher.FetchBlob(ctx, blobHandle) + }) + c.cache[string(handle)] = fn + c.mu.Unlock() + return fn() +} + +// fetchCoordinatedEventFromBlob fetches and extracts the idx-th event from the blob, without unmarshaling any other events. +func fetchCoordinatedEventFromBlob(ctx context.Context, fetcher *blobCache, handle []byte, idx int) (*pb.CoordinatedEvent, []byte, error) { + bb, err := fetcher.fetchBlob(ctx, handle) + if err != nil { + return nil, nil, fmt.Errorf("failed to fetch blob: %w", err) + } + + coordEvent, eb, err := extractEvent(bb, idx) + if err != nil { + return nil, nil, err + } + return coordEvent, eb, nil +} + +const coordinatedEventsFieldNumber = 1 // pb.CoordinatedEventList.Events + +// extractEvent extracts the idx-th event from the blob, without unmarshaling any other events. +func extractEvent(b []byte, idx int) (*pb.CoordinatedEvent, []byte, error) { + i := 0 + for len(b) > 0 { + num, typ, n := protowire.ConsumeTag(b) + if n < 0 { + return nil, nil, protowire.ParseError(n) + } + b = b[n:] + if num == coordinatedEventsFieldNumber { + if typ != protowire.BytesType { + return nil, nil, fmt.Errorf("expected bytes type for field %d, got %d", coordinatedEventsFieldNumber, typ) + } + v, m := protowire.ConsumeBytes(b) + if m < 0 { + return nil, nil, protowire.ParseError(m) + } + if i == idx { + var event pb.CoordinatedEvent + if err := proto.Unmarshal(v, &event); err != nil { + return nil, v, fmt.Errorf("failed to unmarshal event: %w", err) + } + return &event, v, nil + } + // advance + i++ + b = b[m:] + continue + } else { + n := protowire.ConsumeFieldValue(num, typ, b) + if n < 0 { + return nil, nil, protowire.ParseError(n) + } + b = b[n:] + } + } + return nil, nil, fmt.Errorf("index %d out of range", idx) +} + +// broadcastAsBlob broadcasts a blob with as many coordinated events as possible, returning the blob handle and count of events included. +func broadcastAsBlob(ctx context.Context, broadcaster ocr3_1types.BlobBroadcastFetcher, events []*pb.CoordinatedEvent, + seqNr uint64, estimatedRoundInterval time.Duration) (blobHandle []byte, n int, err error) { + protoOpts := proto.MarshalOptions{Deterministic: true} + for ; protoOpts.Size(&pb.CoordinatedEventList{Events: events}) > maxBlobPayloadBytes; events = events[:len(events)-1] { + //TODO opt: binary search for largest fit instead.... + } + n = len(events) + var maxDeadline time.Time + for _, event := range events { + if t := event.Deadline.AsTime(); t.After(maxDeadline) { + maxDeadline = t + } + } + roundDur := estimatedRoundInterval //TODO OK? pad? + //TODO pad extra time and/or rounds? + expSeqNr := seqNr + uint64(time.Until(maxDeadline)/roundDur) + + data, err := protoOpts.Marshal(&pb.CoordinatedEventList{Events: events}) + if err != nil { + return nil, -1, fmt.Errorf("failed to marshal blob: %w", err) + } + bh, err := broadcaster.BroadcastBlob(ctx, data, ocr3_1types.BlobExpirationHintSequenceNumber{SeqNr: expSeqNr}) + if err != nil { + return nil, -1, fmt.Errorf("failed to broadcast blob: %w", err) + } + blobHandle, err = bh.MarshalBinary() + return +} diff --git a/core/services/ocr3_1/plugins/queue/blobs_test.go b/core/services/ocr3_1/plugins/queue/blobs_test.go new file mode 100644 index 00000000000..eaa8e74fb50 --- /dev/null +++ b/core/services/ocr3_1/plugins/queue/blobs_test.go @@ -0,0 +1,87 @@ +package queue + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" +) + +func Test_extractEvent(t *testing.T) { + payload, err := anypb.New(timestamppb.New(time.Now())) + require.NoError(t, err) + event := &pb.CoordinatedEvent{ + WorkflowID: "want", + TriggerCapID: "capID", + TriggerIndex: 1, + ObservedAt: ×tamppb.Timestamp{Seconds: 10, Nanos: 42}, + Deadline: timestamppb.New(time.Now()), + Event: &pb.TriggerResponse{ + Event: &pb.TriggerEvent{ + TriggerType: "type", + Id: "id", + Outputs: nil, + Payload: payload, + }, + Error: "asdf", + }, + } + otherPayload, err := anypb.New(&pb.CapabilityResponse{}) + require.NoError(t, err) + others := &pb.CoordinatedEvent{ + WorkflowID: "foo", + TriggerCapID: "bar", + TriggerIndex: 99, + ObservedAt: ×tamppb.Timestamp{Nanos: 99}, + Deadline: ×tamppb.Timestamp{Seconds: 1000}, + Event: &pb.TriggerResponse{ + Event: &pb.TriggerEvent{ + TriggerType: "baz", + Id: "trigger-id", + Outputs: nil, + Payload: otherPayload, + }, + Error: "none", + }, + } + for _, tt := range []struct { + name string + len, idx int + }{ + {"first", 4, 0}, + {"last", 4, 3}, + {"middle", 4, 2}, + {"bounds", 4, 4}, + } { + t.Run(tt.name, func(t *testing.T) { + var events []*pb.CoordinatedEvent + for i := range tt.len { + if i == tt.idx { + events = append(events, event) + } else { + events = append(events, others) + } + } + b, err := proto.MarshalOptions{Deterministic: true}.Marshal(&pb.CoordinatedEventList{Events: events}) + require.NoError(t, err) + got, _, err := extractEvent(b, tt.idx) + if tt.idx < tt.len { + require.True(t, proto.Equal(event, got)) + } else { + require.Error(t, err) + } + }) + } +} + +func Test_coordinatedEventsFieldNumber(t *testing.T) { + fd := (&pb.CoordinatedEventList{}).ProtoReflect().Descriptor().Fields().ByName("events") + require.NotNil(t, fd) + require.Equal(t, protowire.Number(coordinatedEventsFieldNumber), fd.Number()) +} diff --git a/core/services/ocr3_1/plugins/queue/config.go b/core/services/ocr3_1/plugins/queue/config.go index 0cdf2ec0086..3e2d7501cfb 100644 --- a/core/services/ocr3_1/plugins/queue/config.go +++ b/core/services/ocr3_1/plugins/queue/config.go @@ -4,6 +4,17 @@ import ( "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3_1types" ) +const ( + maxObservedBlobHandles = 10 // sanity check on blobs in an observation + maxObservedEvents = 1000 // sanity check events in an observation + + //TODO reduce these? connect to cre settings? + maxObservationBytes = ocr3_1types.MaxMaxObservationBytes + maxKeyValueValueBytes = ocr3_1types.MaxMaxKeyValueKeyBytes + maxKeyValueModifiedKeysPlusValuesBytes = ocr3_1types.MaxMaxKeyValueModifiedKeysPlusValuesBytes + maxBlobPayloadBytes = ocr3_1types.MaxMaxBlobPayloadBytes +) + type PluginConfig struct { Limits ocr3_1types.ReportingPluginLimits `json:"limits,omitzero"` } diff --git a/core/services/ocr3_1/plugins/queue/kvstate.go b/core/services/ocr3_1/plugins/queue/kvstate.go new file mode 100644 index 00000000000..dbdea067ed4 --- /dev/null +++ b/core/services/ocr3_1/plugins/queue/kvstate.go @@ -0,0 +1,455 @@ +package queue + +import ( + "cmp" + "container/heap" + "fmt" + "maps" + "slices" + "time" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3_1types" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" +) + +// KV store keys +const ( + // observed events waiting for consensus (<2f+1 observers) + observedEventsKey = "observed_events::index" // pb.ConsensusQueueObservedEvents { map[string]*pb.ConsensusQueueObservedEvent; [][]byte } + // priority queue of events waiting for capacity + queuedEventsKey = "queued_events::index" // pb.ConsensusQueueEventList { []*pb.ConsensusQueueEvent } + // events the plugin committed to sending to nodes + consensusEventsKey = "consensus_events::index" // pb.ConsensusQueueEventList { []*pb.ConsensusQueueEvent } + // blob handles referenced from events in the PQ and consensus set + blobHandlesKey = "blob_handles::index" // pb.ConsensusQueueBlobHandleList{ [][]byte } +) + +type stateWriter struct { + kv ocr3_1types.KeyValueStateReadWriter +} + +func (s *stateWriter) writeValue[M proto.Message](key string, m M) error { + val, err := proto.MarshalOptions{Deterministic: true}.Marshal(m) + if err != nil { + return fmt.Errorf("failed to marshal %T: %w", m, err) + } + return s.kv.Write([]byte(key), val) +} + +var _ heap.Interface = (*priorityQueue)(nil) + +// priorityQueue is a heap of coordinated events, ordered by deadline and then trigger event ID. +type priorityQueue []*pb.ConsensusQueueEvent + +func (p *priorityQueue) Len() int { + if p == nil { + return 0 + } + return len(*p) +} + +func (p *priorityQueue) Less(i, j int) bool { + a, b := (*p)[i], (*p)[j] + return a.CompareDeadline(b) < 0 +} + +func (p *priorityQueue) Swap(i, j int) { + (*p)[i], (*p)[j] = (*p)[j], (*p)[i] +} + +func (p *priorityQueue) Push(x any) { + *p = append(*p, x.(*pb.ConsensusQueueEvent)) +} + +func (p *priorityQueue) Pop() any { + e := (*p)[len(*p)-1] + *p = (*p)[:len(*p)-1] + return e +} + +func (p *priorityQueue) push(x *pb.ConsensusQueueEvent) { heap.Push(p, x) } + +func (p *priorityQueue) pushObservations(timelessHash string, obs *pb.ConsensusQueueObservedEvent) *pb.ConsensusQueueEvent { + event := &pb.ConsensusQueueEvent{ + WorkflowID: obs.WorkflowID, + TriggerEventID: obs.TriggerEventID, + ObservedAt: medianTimestamp(obs.ObservedAts), + Deadline: medianTimestamp(obs.Deadlines), + TimelessHash: []byte(timelessHash), + Blobs: obs.Blobs, + } + p.push(event) + return event +} + +// mustPop pops the next event from the queue, or returns nil if the queue is empty. +// The value must be of type *pb.ConsensusQueueEvent. Use push to ensure this. +func (p *priorityQueue) mustPop() *pb.ConsensusQueueEvent { + if len(*p) == 0 { + return nil + } + return heap.Pop(p).(*pb.ConsensusQueueEvent) +} + +// popLimited pops up to capacity events from the queue, each of which must also have sufficient wf capacity. +// Note: capPerWF is mutated. +func (p *priorityQueue) popLimited(capPerWF map[string]uint32, expiration time.Time) (event *pb.ConsensusQueueEvent) { + var wfLimited []*pb.ConsensusQueueEvent // omitted due to workflow limit + for { + event = p.mustPop() + if event == nil { + break + } + if event.Deadline.AsTime().Before(expiration) { + // drop expired + continue + } + if capPerWF[event.WorkflowID] == 0 { + // ignore but restore after + wfLimited = append(wfLimited, event) + continue + } + capPerWF[event.WorkflowID]-- + break + } + // restore omitted for next time + for _, limited := range wfLimited { + p.push(limited) + } + return +} + +type revPriorityQueue struct { + *priorityQueue +} + +func (r *revPriorityQueue) Less(i, j int) bool { + a, b := (*r.priorityQueue)[i], (*r.priorityQueue)[j] + return b.CompareDeadline(a) < 0 +} + +type blobHandleMap map[string]map[string]uint32 // eventHash -> blobHandle -> idx + +func (m blobHandleMap) add(eventHash string, blobHandle string, element uint32) { + if _, ok := m[eventHash]; !ok { + m[eventHash] = make(map[string]uint32) + } + m[eventHash][blobHandle] = element +} + +func (m blobHandleMap) addAll(eventHash string, blobHandles map[string]uint32) { + if _, ok := m[eventHash]; !ok { + m[eventHash] = make(map[string]uint32) + } + for blobHandle, elementIndex := range blobHandles { + m[eventHash][blobHandle] = elementIndex + } +} + +type kvState struct { + // observedEventsKey + observedEvents map[string]*pb.ConsensusQueueObservedEvent // key: timeless hash + observedEventEntries []*pb.ConsensusQueueObservedEventEntry // only populated before write + observedBlobs blobHandleMap + observedBlobHandles [][]byte // only populated before write + + // priorityQueueKey + queuedEvents *priorityQueue + // consensusEventsKey + consensusEvents []*pb.ConsensusQueueEvent + // blobHandlesKey + queuedBlobs blobHandleMap + queuedBlobHandles [][]byte // only populated before write +} + +// loadFrom loads the kvState from the KV store, excluding consensus events, which are not required to transition. +func (s *kvState) loadFrom(kv ocr3_1types.KeyValueStateReadWriter) error { + var err error + s.observedEvents, s.observedBlobs, err = readObservedEvents(kv) + if err != nil { + return err + } + + v, err := kv.Read([]byte(queuedEventsKey)) + if err != nil { + return fmt.Errorf("failed to read queued events: %w", err) + } + if v != nil { + var queued pb.ConsensusQueueEventList + if err = proto.Unmarshal(v, &queued); err != nil { + return fmt.Errorf("failed to unmarshal queued events: %w", err) + } + s.queuedEvents = new(priorityQueue(queued.GetEvents())) + } + + s.queuedBlobs = make(blobHandleMap) + queuedBlobs, err := readBlobHandles(kv) + if err != nil { + return err + } + for _, event := range []*pb.ConsensusQueueEvent(*s.queuedEvents) { + for _, blob := range event.GetBlobs() { + s.queuedBlobs.add(string(event.TimelessHash), string(queuedBlobs[blob.Blob]), blob.Element) + } + } + + return nil +} + +func readObservedEvents(kv ocr3_1types.KeyValueStateReader) (observedEvents map[string]*pb.ConsensusQueueObservedEvent, observedBlobs blobHandleMap, err error) { + v, err := kv.Read([]byte(observedEventsKey)) + if err != nil { + return nil, nil, fmt.Errorf("failed to read observed events: %w", err) + } + observedBlobs = make(blobHandleMap) + if v != nil { + var observed pb.ConsensusQueueObservedEvents + if err = proto.Unmarshal(v, &observed); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal observed events: %w", err) + } + observedEvents = make(map[string]*pb.ConsensusQueueObservedEvent) + for _, e := range observed.GetObservations() { + observedEvents[string(e.TimelessHash)] = e.Event + } + blobHandles := observed.GetBlobHandles() + for eventHash, event := range observedEvents { + for _, b := range event.GetBlobs() { + observedBlobs.add(eventHash, string(blobHandles[b.Blob]), b.Element) + } + } + } else { + observedEvents = make(map[string]*pb.ConsensusQueueObservedEvent) + } + return +} + +// sizeOf returns the size of the state when serialized and written to the KV store via writeAll. +func (s *kvState) sizeOf() (size int) { + + size += len(observedEventsKey) + len(queuedEventsKey) + len(consensusEventsKey) + len(blobHandlesKey) + size += s.sizeOfObservedEvents() + s.sizeOfQueuedEvents() + s.sizeOfConsensusEvents() + s.sizeOfBlobHandles() + //TODO opt: proto.MarshalOptions.UseCachedSize + return +} + +func (s *kvState) sizeOfObservedEvents() int { + s.prepareWriteObservedEvents() + return proto.MarshalOptions{Deterministic: true}.Size(&pb.ConsensusQueueObservedEvents{Observations: s.observedEventEntries, BlobHandles: s.observedBlobHandles}) +} + +func (s *kvState) sizeOfQueuedEvents() int { + return proto.MarshalOptions{Deterministic: true}.Size(&pb.ConsensusQueueEventList{Events: *s.queuedEvents}) +} + +func (s *kvState) sizeOfConsensusEvents() int { + return proto.MarshalOptions{Deterministic: true}.Size(&pb.ConsensusQueueEventList{Events: s.consensusEvents}) +} + +func (s *kvState) sizeOfBlobHandles() int { + s.prepareWriteBlobHandles() + return proto.MarshalOptions{Deterministic: true}.Size(&pb.ConsensusQueueBlobHandleList{BlobHandles: s.queuedBlobHandles}) +} + +// prepareWrite initializes observed blob handles & entries, and updates the event references. +func (s *kvState) prepareWriteObservedEvents() { + // gather blob handles and update events + s.observedBlobHandles = nil + observedBlobIndexes := make(map[string]int) + s.observedEventEntries = nil + + // sort so that we can drop lowest priority efficiently + for _, eventHash := range slices.SortedFunc(maps.Keys(s.observedEvents), func(a string, b string) int { + ea, eb := s.observedEvents[a], s.observedEvents[b] + //TODO can we use deadline here? is median valid & best option? + return cmp.Or(cmp.Compare(len(ea.NodeIds), len(eb.NodeIds)), + cmp.Compare(ea.TriggerEventID, eb.TriggerEventID)) + }) { + event := s.observedEvents[eventHash] + event.Blobs = nil // replace all + for _, blobHandle := range slices.Sorted(maps.Keys(s.observedBlobs[eventHash])) { + ei := s.observedBlobs[eventHash][blobHandle] + idx, ok := observedBlobIndexes[blobHandle] + if !ok { + idx = len(s.observedBlobHandles) + observedBlobIndexes[blobHandle] = idx + s.observedBlobHandles = append(s.observedBlobHandles, []byte(blobHandle)) + } + event.Blobs = append(event.Blobs, &pb.CoordinatedEventBlobHandle{ + Blob: uint32(idx), + Element: ei, + }) + } + s.observedEventEntries = append(s.observedEventEntries, &pb.ConsensusQueueObservedEventEntry{ + TimelessHash: []byte(eventHash), Event: event}) + } +} + +// prepareWrite initializes queued blob handles and updates the event references. +func (s *kvState) prepareWriteBlobHandles() { + // gather blob handles and update events + s.queuedBlobHandles = nil + queuedBlobIndexes := make(map[string]int) + addBlobs := func(blobs map[string]uint32) []*pb.CoordinatedEventBlobHandle { + var blobHandles []*pb.CoordinatedEventBlobHandle + for _, blobHandle := range slices.Sorted(maps.Keys(blobs)) { + ei := blobs[blobHandle] + idx, ok := queuedBlobIndexes[blobHandle] + if !ok { + idx = len(s.queuedBlobHandles) + queuedBlobIndexes[blobHandle] = idx + s.queuedBlobHandles = append(s.queuedBlobHandles, []byte(blobHandle)) + } + blobHandles = append(blobHandles, &pb.CoordinatedEventBlobHandle{ + Blob: uint32(idx), + Element: ei, + }) + } + return blobHandles + } + for _, event := range []*pb.ConsensusQueueEvent(*s.queuedEvents) { + event.Blobs = addBlobs(s.queuedBlobs[string(event.TimelessHash)]) + } + for _, event := range s.consensusEvents { + event.Blobs = addBlobs(s.queuedBlobs[string(event.TimelessHash)]) + } +} + +// writeAll writes full state to the KV store, including consensus events. +// prepareWrite must be called to include blob handles. +func (s *kvState) writeTo(w *stateWriter) error { + //TODO MarshalOptions{UseCachedSize}? + + err := w.writeValue(observedEventsKey, &pb.ConsensusQueueObservedEvents{Observations: s.observedEventEntries, BlobHandles: s.observedBlobHandles}) + if err != nil { + return fmt.Errorf("failed to write observed events: %w", err) + } + + err = w.writeValue(queuedEventsKey, &pb.ConsensusQueueEventList{Events: *s.queuedEvents}) + if err != nil { + return fmt.Errorf("failed to write queued events: %w", err) + } + + err = w.writeValue(consensusEventsKey, &pb.ConsensusQueueEventList{Events: s.consensusEvents}) + if err != nil { + return fmt.Errorf("failed to write consensus events: %w", err) + } + + err = w.writeValue(blobHandlesKey, &pb.ConsensusQueueBlobHandleList{BlobHandles: s.queuedBlobHandles}) + if err != nil { + return fmt.Errorf("failed to write blob handles: %w", err) + } + + return nil +} + +func (s *kvState) pruneExpiredObservations(expiration time.Time) { + unexpiredFn := func(t *timestamppb.Timestamp) bool { return t.AsTime().After(expiration) } + maps.DeleteFunc(s.observedEvents, func(k string, v *pb.ConsensusQueueObservedEvent) bool { + if !slices.ContainsFunc(v.Deadlines, unexpiredFn) { + return false // no unexpired deadlines, so delete the whole thing + } + + // some expired, so prune them + var toDelete []int + for i, d := range v.Deadlines { + if !unexpiredFn(d) { + toDelete = append(toDelete, i) + } + } + for _, i := range toDelete { + v.NodeIds = slices.Delete(v.NodeIds, i, i+1) + v.ObservedAts = slices.Delete(v.ObservedAts, i, i+1) + v.Deadlines = slices.Delete(v.Deadlines, i, i+1) + v.Blobs = slices.Delete(v.Blobs, i, i+1) + } + + return false + }) +} + +// addObserved adds the observed events to the set, promoting to the priority queue on 2f+1 and calling onQueued. +func (s *kvState) addObservedEvents(f int, toAdd map[string]*pb.ConsensusQueueObservedEvent, observedBlobs blobHandleMap, onQueued func(event *pb.ConsensusQueueEvent)) { + for _, eventHash := range slices.Sorted(maps.Keys(toAdd)) { + eventToAdd := toAdd[eventHash] + obsEvent, ok := s.observedEvents[eventHash] + if !ok && len(eventToAdd.NodeIds) >= 2*f+1 { + // new proposal straight to priority queue + s.queuedBlobs.addAll(eventHash, observedBlobs[eventHash]) + event := s.queuedEvents.pushObservations(eventHash, eventToAdd) + onQueued(event) + } else if !ok { + // new proposal < f + 1 + s.observedBlobs.addAll(eventHash, observedBlobs[eventHash]) + s.observedEvents[eventHash] = eventToAdd + } else { + // update proposal + // TODO opt: build a map to avoid n^2? + for i, node := range eventToAdd.NodeIds { + if prev := slices.Index(obsEvent.NodeIds, node); prev == -1 { + obsEvent.NodeIds = append(obsEvent.NodeIds, node) + obsEvent.ObservedAts = append(obsEvent.ObservedAts, eventToAdd.ObservedAts[i]) + obsEvent.Deadlines = append(obsEvent.Deadlines, eventToAdd.Deadlines[i]) + obsEvent.Blobs = append(obsEvent.Blobs, eventToAdd.Blobs[i]) + } else { // overwrite previous observation + obsEvent.ObservedAts[prev] = eventToAdd.ObservedAts[i] + obsEvent.Deadlines[prev] = eventToAdd.Deadlines[i] + obsEvent.Blobs[prev] = eventToAdd.Blobs[i] + } + } + + if len(obsEvent.NodeIds) >= 2*f+1 { + // promote proposal to priority queue + delete(s.observedEvents, eventHash) + s.queuedBlobs[eventHash] = s.observedBlobs[eventHash] + delete(s.observedBlobs, eventHash) + event := s.queuedEvents.pushObservations(eventHash, obsEvent) + onQueued(event) + } + } + } +} + +// readConsensusState reads the consensus events and blob handles from the kvReader. +func readConsensusState(kvReader ocr3_1types.KeyValueStateReader) ([]*pb.ConsensusQueueEvent, [][]byte, error) { + v, err := kvReader.Read([]byte(consensusEventsKey)) + if err != nil { + return nil, nil, fmt.Errorf("failed to read consensus events: %w", err) + } + + var consensusEvents []*pb.ConsensusQueueEvent + if v != nil { + var pbs pb.ConsensusQueueEventList + if err := proto.Unmarshal(v, &pbs); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal consensus events: %w", err) + } + consensusEvents = pbs.GetEvents() + } + + blobHandles, err := readBlobHandles(kvReader) + if err != nil { + return nil, nil, err + } + return consensusEvents, blobHandles, nil +} + +// readBlobHandles reads the blob handles from the kvReader. +func readBlobHandles(kvReader ocr3_1types.KeyValueStateReader) ([][]byte, error) { + v, err := kvReader.Read([]byte(blobHandlesKey)) + if err != nil { + return nil, fmt.Errorf("failed to read blob handles: %w", err) + } + + var blobHandles [][]byte + if v != nil { + var bs pb.ConsensusQueueBlobHandleList + if err := proto.Unmarshal(v, &bs); err != nil { + return nil, fmt.Errorf("failed to unmarshal consensus blob handles: %w", err) + } + blobHandles = bs.GetBlobHandles() + } + return blobHandles, err +} diff --git a/core/services/ocr3_1/plugins/queue/math.go b/core/services/ocr3_1/plugins/queue/math.go new file mode 100644 index 00000000000..400276f25df --- /dev/null +++ b/core/services/ocr3_1/plugins/queue/math.go @@ -0,0 +1,32 @@ +package queue + +import ( + "slices" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +// medianTimestamp returns the median value of a slice of timestamps. +// len must be greater than 0. +func medianTimestamp(s []*timestamppb.Timestamp) *timestamppb.Timestamp { + switch len(s) { + case 0: + panic("median: len must be greater than 0") + case 1: + return s[0] + } + times := make([]time.Time, len(s)) + for i, t := range s { + times[i] = time.Unix(t.Seconds, int64(t.Nanos)) + } + slices.SortFunc(times, time.Time.Compare) + + if len(times)%2 == 1 { + return timestamppb.New(times[len(times)/2]) + } + // avg of two + a, b := times[len(times)/2], times[len(times)/2-1] + diff := a.Sub(b) + return timestamppb.New(b.Add(diff / 2)) +} diff --git a/core/services/ocr3_1/plugins/queue/queue.go b/core/services/ocr3_1/plugins/queue/queue.go index c5060f5fead..895c19d9a50 100644 --- a/core/services/ocr3_1/plugins/queue/queue.go +++ b/core/services/ocr3_1/plugins/queue/queue.go @@ -1,33 +1,82 @@ +// Package queue contains the consensus queue OCR 3.1 reporting plugin, for coordinating execution of triggered events. +// Each node has a local [triggers.Queue] of [triggers.CoordinatedEvent], which are observed (and broadcast) by the plugin. +// When consensus is reached, events are put in to the KeyValueState priority queue (and removed from the [triggers.Queue]). +// The priority queue is ordered by deadline (with nanosecond precision) and then by trigger event ID. +// Each round, as many events as possible are popped from the priority queue and then (fetched and) executed via [triggers.Coordinator.Deliver]. +// Capacity, both overall and per-workflow, limits what may execute. Capacities are observed and medianized. package queue import ( + "bytes" + "container/heap" "context" + "crypto/sha256" + "errors" + "fmt" + "iter" + "math" + "slices" + "sync" + "time" + + "github.com/smartcontractkit/libocr/commontypes" + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3_1types" "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" "github.com/smartcontractkit/libocr/quorumhelper" + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2/triggers" ) var _ ocr3_1types.ReportingPluginFactory[[]byte] = &consensusQueuePluginFactory{} type consensusQueuePluginFactory struct { - lggr logger.Logger - cfg *PluginConfig + lggr logger.Logger + cfg *PluginConfig + triggerCoordinator triggers.Coordinator + triggerQueue triggers.Queue + expirationDur func(context.Context) (time.Duration, error) } -func NewConsensusQueuePluginFactory(lggr logger.Logger, cfg *PluginConfig, limitsFactory limits.Factory) (ocr3_1types.ReportingPluginFactory[[]byte], error) { - // TODO pass CentralTriggerQueue, CapacityReporter, EngineRegistry - // TODO construct limiters - return &consensusQueuePluginFactory{lggr: lggr, cfg: cfg}, nil +func NewConsensusQueuePluginFactory( + lggr logger.Logger, + cfg *PluginConfig, + limitsFactory limits.Factory, + triggerCoordinator triggers.Coordinator, + triggerQueue triggers.Queue, +) (ocr3_1types.ReportingPluginFactory[[]byte], error) { + if cfg == nil { + return nil, errors.New("cfg cannot be nil") + } + if triggerCoordinator == nil { + return nil, errors.New("triggerCoordinator cannot be nil") + } + if triggerQueue == nil { + return nil, errors.New("triggerQueue cannot be nil") + } + return &consensusQueuePluginFactory{ + lggr: lggr, + cfg: cfg, + triggerCoordinator: triggerCoordinator, + triggerQueue: triggerQueue, + expirationDur: func(ctx context.Context) (time.Duration, error) { + //TODO also allow this grace period in the trigger coordinator + return cresettings.Default.ConsensusQueueExpirationInterval.GetOrDefault(ctx, limitsFactory.Settings) + }, + }, nil } -func (c *consensusQueuePluginFactory) NewReportingPlugin(ctx context.Context, config ocr3types.ReportingPluginConfig, fetcher ocr3_1types.BlobBroadcastFetcher) (ocr3_1types.ReportingPlugin[[]byte], ocr3_1types.ReportingPluginInfo, error) { - plugin := newConsensusQueuePlugin(c.lggr, config) +func (c *consensusQueuePluginFactory) NewReportingPlugin(ctx context.Context, config ocr3types.ReportingPluginConfig, blobs ocr3_1types.BlobBroadcastFetcher) (ocr3_1types.ReportingPlugin[[]byte], ocr3_1types.ReportingPluginInfo, error) { + plugin := newConsensusQueuePlugin(c.lggr, config, c.triggerCoordinator, c.triggerQueue, c.expirationDur, blobs) pluginInfo := ocr3_1types.ReportingPluginInfo1{ Name: "ConsensusQueueReportingPlugin", Limits: c.cfg.Limits, @@ -39,17 +88,33 @@ var _ ocr3_1types.ReportingPlugin[[]byte] = &consensusQueuePlugin{} type consensusQueuePlugin struct { services.Service - eng *services.Engine - lggr logger.Logger - onchainCfg ocr3types.ReportingPluginConfig + eng *services.Engine + lggr logger.Logger + onchainCfg ocr3types.ReportingPluginConfig + blobHandleMarshalledBytesUpperBound int + triggerCoordinator triggers.Coordinator + triggerQueue triggers.Queue + blobBroadcastFetcher ocr3_1types.BlobBroadcastFetcher + expirationDur func(context.Context) (time.Duration, error) } -func newConsensusQueuePlugin(lggr logger.Logger, onchainCfg ocr3types.ReportingPluginConfig) *consensusQueuePlugin { - c := &consensusQueuePlugin{lggr: logger.Named(lggr, "ConsensusQueuePlugin"), onchainCfg: onchainCfg} +func newConsensusQueuePlugin( + lggr logger.Logger, onchainCfg ocr3types.ReportingPluginConfig, triggerCoordinator triggers.Coordinator, + triggerQueue triggers.Queue, expirationGraceDur func(context.Context) (time.Duration, error), + blobs ocr3_1types.BlobBroadcastFetcher, +) *consensusQueuePlugin { + c := &consensusQueuePlugin{ + lggr: logger.Named(lggr, "ConsensusQueuePlugin"), + onchainCfg: onchainCfg, + blobHandleMarshalledBytesUpperBound: ocr3_1types.BlobHandleMarshalledBytesUpperBound(onchainCfg.N, onchainCfg.F), + triggerCoordinator: triggerCoordinator, + triggerQueue: triggerQueue, + blobBroadcastFetcher: blobs, + expirationDur: expirationGraceDur, + } c.Service, c.eng = services.Config{ Name: "ConsensusQueueReportingPlugin", }.NewServiceEngine(lggr) - return c } @@ -58,45 +123,491 @@ func (c *consensusQueuePlugin) Query(_ context.Context, _ uint64, _ ocr3_1types. } func (c *consensusQueuePlugin) Observation( - ctx context.Context, seqNr uint64, _ ocrtypes.AttributedQuery, kvState ocr3_1types.KeyValueStateReader, _ ocr3_1types.BlobBroadcastFetcher, + ctx context.Context, seqNr uint64, _ ocrtypes.AttributedQuery, kvReader ocr3_1types.KeyValueStateReader, + blobBroadcaster ocr3_1types.BlobBroadcastFetcher, ) (ocrtypes.Observation, error) { - return nil, nil + // Always attempt to start any executions from the KV Store in the event Committed was not called. + // tryStartExecutions is idempotent. + c.tryStartExecutions(kvReader) + + capacity, err := c.triggerCoordinator.AvailableCapacity(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get available capacity: %w", err) + } + + expirationGraceDur, err := c.expirationDur(ctx) + if err != nil { + c.lggr.Warnw("Failed to get expiration grace duration - using default", "err", err) + } + + emptyBlobHandle := make([]byte, c.blobHandleMarshalledBytesUpperBound) + obs := &pb.ConsensusQueueObservation{ + // placeholder for accurate size calculation + BlobHandles: slices.Repeat([][]byte{emptyBlobHandle}, maxObservedBlobHandles), + Expiration: timestamppb.New(time.Now().Add(-expirationGraceDur)), + Capacity: uint32(min(capacity.Global, math.MaxUint32)), + WorkflowCapacities: make(map[string]uint32), + } + for wf, wfCap := range capacity.PerWorkflow { + //TODO only for events proposed and queued in the KV store? + obs.WorkflowCapacities[wf] = uint32(min(wfCap, math.MaxUint32)) + } + + observedEvents, _, err := readObservedEvents(kvReader) + if err != nil { + return nil, err + } + + protoOpts := proto.MarshalOptions{Deterministic: true} + + // Assemble an observation up to maxObservationBytes + var coordEvents []*pb.CoordinatedEvent + for coord := range toProtoSeq(c.triggerQueue.Sorted()) { + observedAt, deadline := coord.ObservedAt, coord.Deadline + coord.ObservedAt, coord.Deadline = nil, nil // for consistent hashing + protoBytes, err2 := protoOpts.Marshal(coord) // TODO opt: avoid remarshalling + if err2 != nil { + c.lggr.Errorw("failed to marshal event", "eventID", coord.Event.Event.Id, "err", err2) + continue + } + timelessHash := sha256.Sum256(protoBytes) + + // skip if kv observation exists already + if obsEvent, ok := observedEvents[string(timelessHash[:])]; ok { + if slices.Contains(obsEvent.NodeIds, uint8(c.onchainCfg.OracleID)) { + continue + } + } + + obs.Events = append(obs.Events, &pb.ConsensusQueueEvent{ + WorkflowID: coord.WorkflowID, + TriggerEventID: coord.Event.Event.Id, + ObservedAt: observedAt, + Deadline: deadline, + TimelessHash: timelessHash[:], + Blobs: []*pb.CoordinatedEventBlobHandle{{ + Blob: 0, + Element: uint32(len(coordEvents)), + }}, + }) + coordEvents = append(coordEvents, coord) + + var addedCap bool + if _, ok := obs.WorkflowCapacities[coord.WorkflowID]; !ok { + addedCap = true + obs.WorkflowCapacities[coord.WorkflowID] = uint32(min(capacity.PerWorkflow[coord.WorkflowID], math.MaxUint32)) + } + + // Check size + if protoOpts.Size(obs) > maxObservationBytes { + // too large, remove this one + obs.Events = obs.Events[:len(obs.Events)-1] + if addedCap { + delete(obs.WorkflowCapacities, coord.WorkflowID) + } + break + } + } + + obs.BlobHandles = nil // remove the placeholders + if len(coordEvents) > 0 { + blobHandle, n, err2 := broadcastAsBlob(ctx, blobBroadcaster, coordEvents, seqNr, c.onchainCfg.EstimatedRoundInterval) + if err2 != nil { + return nil, err2 + } + obs.BlobHandles = append(obs.BlobHandles, blobHandle) + // note: elements already reference blob 0 and natural element index is aligned + + var offset int + //TODO stop after batch maxObservedBlobHandles + for batch := 1; n < len(coordEvents); batch++ { + offset += n + coordEvents = coordEvents[n:] + blobHandle, n, err2 = broadcastAsBlob(ctx, blobBroadcaster, coordEvents, seqNr, c.onchainCfg.EstimatedRoundInterval) + if err2 != nil { + return nil, err2 + } + obs.BlobHandles = append(obs.BlobHandles, blobHandle) + for i := offset; i < n; i++ { // update indices + obs.Events[offset+i].Blobs[0].Blob = uint32(batch) + obs.Events[offset+i].Blobs[0].Element = uint32(i) + } + } + } + + return protoOpts.Marshal(obs) +} + +func toProtoSeq(in iter.Seq[triggers.CoordinatedEvent]) iter.Seq[*pb.CoordinatedEvent] { + return func(yield func(*pb.CoordinatedEvent) bool) { + for event := range in { + if !yield(event.ToProto()) { + return + } + } + } } func (c *consensusQueuePlugin) ValidateObservation( - ctx context.Context, seqNr uint64, _ ocrtypes.AttributedQuery, ao ocrtypes.AttributedObservation, - _ ocr3_1types.KeyValueStateReader, _ ocr3_1types.BlobFetcher, + ctx context.Context, _ uint64, _ ocrtypes.AttributedQuery, ao ocrtypes.AttributedObservation, + _ ocr3_1types.KeyValueStateReader, blobFetcher ocr3_1types.BlobFetcher, ) error { - // TODO validate + var obs pb.ConsensusQueueObservation + err := proto.Unmarshal(ao.Observation, &obs) + if err != nil { + return fmt.Errorf("failed to unmarshal observation: %w", err) + } + if len(obs.BlobHandles) > maxObservedBlobHandles { + return fmt.Errorf("blob handles are limited to %d but got %d", maxObservedBlobHandles, len(obs.BlobHandles)) + } + if obs.Expiration == nil { + return errors.New("expiration is missing") + } + //TODO what do we expect from WF capacities? check KV state and insist 100% match? or subset - none extra? + + if len(obs.Events) > maxObservedEvents { + return fmt.Errorf("events are limited to %d but got %d", maxObservedEvents, len(obs.Events)) + } + var eg errgroup.Group + fetcher := blobCache{fetcher: blobFetcher, cache: make(map[string]func() ([]byte, error))} + for i, event := range obs.Events { + eg.Go(func() error { + if err := validateObservedEvent(ctx, &fetcher, event, obs.BlobHandles); err != nil { + return fmt.Errorf("event %d failed validation: %w", i, err) + } + return nil + }) + } + if err = eg.Wait(); err != nil { + return err + } + + // wait to do this since we depend on every event being valid + if !slices.IsSortedFunc(obs.Events, (*pb.ConsensusQueueEvent).CompareDeadline) { + return fmt.Errorf("events must be sorted") + } + + return nil +} + +func validateObservedEvent(ctx context.Context, fetcher *blobCache, event *pb.ConsensusQueueEvent, blobHandles [][]byte) error { + if event.WorkflowID == "" { + return errors.New("event workflow id is missing") + } ///TODO check not too long - exact length? + if event.TriggerEventID == "" { + return errors.New("event trigger event id is missing") + } ///TODO check not too long - exact length? + if event.ObservedAt == nil { + return errors.New("event observation ts is missing") + } + if event.Deadline == nil { + return errors.New("event deadline is missing") + } + if event.TimelessHash == nil { + return fmt.Errorf("event blob hash is missing") + } ///TODO check not too long - exact length? + + if len(event.Blobs) == 0 { + return fmt.Errorf("event blob is missing") + } + if len(event.Blobs) > 1 { + return fmt.Errorf("event has more than one blob") + } + + blobIdx := int(event.Blobs[0].Blob) + if blobIdx >= len(blobHandles) { + return fmt.Errorf("event blob index %d is out of range", blobIdx) + } + blobHandle := blobHandles[blobIdx] + + coordEvent, pbBytes, err := fetchCoordinatedEventFromBlob(ctx, fetcher, blobHandle, int(event.Blobs[0].Element)) + if err != nil { + return fmt.Errorf("failed to fetch event from blob: %w", err) + } + + if coordEvent.WorkflowID != event.WorkflowID { + return fmt.Errorf("blob workflow ID %s does not match event workflow ID %s", coordEvent.WorkflowID, event.WorkflowID) + } + if coordEvent.GetEvent().GetEvent().Id != event.TriggerEventID { + return fmt.Errorf("blob event ID %s does not match event trigger event ID %s", coordEvent.GetEvent().GetEvent().Id, event.TriggerEventID) + } + + if coordEvent.ObservedAt != nil { + return fmt.Errorf("blob observation timestamp is not nil: %v", coordEvent.ObservedAt) + } + if coordEvent.Deadline != nil { + return fmt.Errorf("blob deadline timestamp is not nil: %v", coordEvent.Deadline) + } + + //TODO opt: reduce allocations with a pool + if shaSum := sha256.Sum256(pbBytes); !bytes.Equal(shaSum[:], event.TimelessHash) { + return fmt.Errorf("blob hash %x does not match observation hash %x", shaSum, event.TimelessHash) + } + return nil } func (c *consensusQueuePlugin) ObservationQuorum( - ctx context.Context, seqNr uint64, _ ocrtypes.AttributedQuery, aos []ocrtypes.AttributedObservation, + _ context.Context, _ uint64, _ ocrtypes.AttributedQuery, aos []ocrtypes.AttributedObservation, _ ocr3_1types.KeyValueStateReader, _ ocr3_1types.BlobFetcher, ) (quorumReached bool, err error) { return quorumhelper.ObservationCountReachesObservationQuorum(quorumhelper.QuorumTwoFPlusOne, c.onchainCfg.N, c.onchainCfg.F, aos), nil } func (c *consensusQueuePlugin) StateTransition( - ctx context.Context, seqNr uint64, aq ocrtypes.AttributedQuery, aos []ocrtypes.AttributedObservation, - kvState ocr3_1types.KeyValueStateReadWriter, _ ocr3_1types.BlobFetcher, + ctx context.Context, _ uint64, _ ocrtypes.AttributedQuery, aos []ocrtypes.AttributedObservation, + kvReaderWriter ocr3_1types.KeyValueStateReadWriter, _ ocr3_1types.BlobFetcher, ) (ocr3_1types.ReportsPlusPrecursor, error) { - return ocr3_1types.ReportsPlusPrecursor{}, nil + done := make(chan struct{}) + var obs observations + + go func() { + defer close(done) + obs.aggregate(ctx, c.onchainCfg.F, aos) + }() + + var state kvState + err := state.loadFrom(kvReaderWriter) + if err != nil { + return nil, fmt.Errorf("failed to read state: %w", err) + } + + // wait for aggregation + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-done: + if err = ctx.Err(); err != nil { + return nil, err + } + } + if obs.err != nil { + return nil, fmt.Errorf("failed to aggregate observations: %w", obs.err) + } + + // update state + state.pruneExpiredObservations(obs.medianExpiration) + var queuedTriggerEventIDs []string // to be removed from the trigger.Queue + state.addObservedEvents(c.onchainCfg.F, obs.newObservedEvents, obs.observedBlobs, func(event *pb.ConsensusQueueEvent) { + queuedTriggerEventIDs = append(queuedTriggerEventIDs, event.TriggerEventID) + }) + // pop as many queuedEvents as possible in to consensusEvents (up to capacity or KV limits) + for len(*state.queuedEvents) > 0 && len(state.consensusEvents) < int(obs.medianCap) { + event := state.queuedEvents.popLimited(obs.medianCapPerWF, obs.medianExpiration) + if event == nil { + break + } + state.consensusEvents = append(state.consensusEvents, event) + // stop when either value or the overall size is too large + if events, blobs := state.sizeOfConsensusEvents(), state.sizeOfBlobHandles(); events > maxKeyValueValueBytes || + blobs > maxKeyValueValueBytes || events+blobs > maxKeyValueModifiedKeysPlusValuesBytes/2 { + state.consensusEvents = state.consensusEvents[:len(state.consensusEvents)-1] + break + } + } + + // prune observations if that value or the overall size is too large + for len(state.observedEventEntries) > 0 && + (state.sizeOfObservedEvents() > maxKeyValueValueBytes || + state.sizeOf() > maxKeyValueModifiedKeysPlusValuesBytes) { + // sorted by decreasing priority, so prune the last + toRemove := state.observedEventEntries[len(state.observedEventEntries)-1] + delete(state.observedEvents, toRemove.Event.TriggerEventID) + //TODO opt: drop more than one each time? + } + + // prune queuedEvents if that value, blob handles, or the overall size is too large + if len(*state.queuedEvents) > 0 && + (state.sizeOfQueuedEvents() > maxKeyValueValueBytes || + state.sizeOfBlobHandles() > maxKeyValueValueBytes || + state.sizeOf() > maxKeyValueModifiedKeysPlusValuesBytes) { + // prune lowest priority queuedEvents from newly added queuedTriggerEventIDs + queuedTriggerEventIDSet := map[string]struct{}{} + for _, id := range queuedTriggerEventIDs { + queuedTriggerEventIDSet[id] = struct{}{} + } + eligibleEvents := make([]*pb.ConsensusQueueEvent, 0, len(*state.queuedEvents)) + for _, e := range *state.queuedEvents { + if _, ok := queuedTriggerEventIDSet[e.TriggerEventID]; ok { + eligibleEvents = append(eligibleEvents, e) + } + } + backwardsQueuedEvents := &revPriorityQueue{priorityQueue: new(priorityQueue(eligibleEvents))} + heap.Init(backwardsQueuedEvents) + + // prune until we fit + for backwardsQueuedEvents.Len() > 0 && + (state.sizeOfQueuedEvents() > maxKeyValueValueBytes || + state.sizeOfBlobHandles() > maxKeyValueValueBytes || + state.sizeOf() > maxKeyValueModifiedKeysPlusValuesBytes) { + toRemove := heap.Pop(backwardsQueuedEvents).(*pb.ConsensusQueueEvent) + //TODO opt: faster delete...but need to maintain state.queuedEvents (though not order)...map indices? + *state.queuedEvents = slices.DeleteFunc(*state.queuedEvents, func(e *pb.ConsensusQueueEvent) bool { return e.TriggerEventID == toRemove.TriggerEventID }) + //TODO opt: drop more than one each time? + } + // rebuild the priority queue + heap.Init(state.queuedEvents) + } + + // note: no need to prune consensus events, since already limited what we collected + + err = state.writeTo(&stateWriter{kv: kvReaderWriter}) + if err != nil { + return nil, err + } + c.triggerQueue.Remove(ctx, queuedTriggerEventIDs) //TODO move to committed - do we need to write it through KV? or can we Remove the whole PQ every time? + + // no report to return, executions start when committed + return nil, nil +} + +type observations struct { + medianExpiration time.Time + medianCap uint32 + medianCapPerWF map[string]uint32 // wf ID -> capacity + newObservedEvents map[string]*pb.ConsensusQueueObservedEvent // key: timeless hash + observedBlobs blobHandleMap + + err error +} + +func (res *observations) aggregate(ctx context.Context, f int, aos []ocrtypes.AttributedObservation) { + res.medianCapPerWF = make(map[string]uint32) + res.newObservedEvents = make(map[string]*pb.ConsensusQueueObservedEvent) + + expirations := make([]*timestamppb.Timestamp, 0, len(aos)) + capacities := make([]uint32, 0, len(aos)) + wfCapacities := make(map[string][]uint32) + for _, ao := range aos { + var obs pb.ConsensusQueueObservation + err := proto.Unmarshal(ao.Observation, &obs) + if err != nil { + res.err = fmt.Errorf("failed to unmarshal observation from oracle %d: %w", ao.Observer, err) + return + } + expirations = append(expirations, obs.Expiration) + capacities = append(capacities, obs.Capacity) + for wf, c := range obs.WorkflowCapacities { + wfCapacities[wf] = append(wfCapacities[wf], c) + } + for _, event := range obs.Events { + if ctx.Err() != nil { + return + } + res.addEvent(ao.Observer, event, obs.BlobHandles) + } + } + res.medianExpiration = medianTimestamp(expirations).AsTime() + + // If we used median, then honest nodes reporting reduced capacity would not be able to keep up. + // Instead, we use the F+1th lowest value - the least honest value. + var medianCount int + res.medianCap = slices.Sorted(slices.Values(capacities))[f] + for wf, caps := range wfCapacities { + if len(caps) > 2*f+1 { + res.medianCapPerWF[wf] = slices.Sorted(slices.Values(caps))[f] + if medianCount < 2*f+1 { + res.medianCapPerWF[wf] = (res.medianCapPerWF[wf] * 9) / 10 + } + } + } + return +} + +// addEvent adds the observed event to the observation set, either merging it into an existing event or creating a new one. +func (res *observations) addEvent(obs commontypes.OracleID, event *pb.ConsensusQueueEvent, blobHandles [][]byte) { + res.observedBlobs.add(string(event.TimelessHash), string(blobHandles[event.Blobs[0].Blob]), event.Blobs[0].Element) + aggregated, ok := res.newObservedEvents[string(event.TimelessHash)] + if !ok { + res.newObservedEvents[string(event.TimelessHash)] = &pb.ConsensusQueueObservedEvent{ + TriggerEventID: event.TriggerEventID, + WorkflowID: event.WorkflowID, + NodeIds: []uint8{uint8(obs)}, + ObservedAts: []*timestamppb.Timestamp{event.ObservedAt}, + Deadlines: []*timestamppb.Timestamp{event.Deadline}, + Blobs: nil, // res.observedBlobs tracks these while in memory + } + } else { + // note: trigger event and workflow ID alignment is ensured by ValidateObservation + aggregated.NodeIds = append(aggregated.NodeIds, uint8(obs)) + aggregated.ObservedAts = append(aggregated.ObservedAts, event.ObservedAt) + aggregated.Deadlines = append(aggregated.Deadlines, event.Deadline) + // note: res.observedBlobs tracks blobs while in memory + } } -func (c *consensusQueuePlugin) Committed(ctx context.Context, seqNr uint64, keyValueStateReader ocr3_1types.KeyValueStateReader) error { +func (c *consensusQueuePlugin) Committed(ctx context.Context, _ uint64, kvReader ocr3_1types.KeyValueStateReader) error { + c.tryStartExecutions(kvReader) return nil } -func (c *consensusQueuePlugin) Reports(ctx context.Context, seqNr uint64, reportsPlusPrecursor ocr3_1types.ReportsPlusPrecursor) ([]ocr3types.ReportPlus[[]byte], error) { +// tryStartExecutions calls StartExecution asynchronously on the corresponding engine for each trigger event. +// It is idempotent, thus safe to call from Observation and Committed. +func (c *consensusQueuePlugin) tryStartExecutions(kvReader ocr3_1types.KeyValueStateReader) { + consensusEvents, blobHandles, err := readConsensusState(kvReader) + if err != nil { + c.lggr.Errorw("Failed to read consensus state. Unable to execute events", "err", err) + return + } + + fetcher := blobCache{fetcher: c.blobBroadcastFetcher, cache: make(map[string]func() ([]byte, error))} + var wg sync.WaitGroup + for _, consensusEvent := range consensusEvents { + wg.Add(1) + c.eng.Go(func(ctx context.Context) { + defer wg.Done() + + if err2 := c.tryStartExecution(ctx, &fetcher, consensusEvent, blobHandles); err2 != nil { + c.lggr.Errorw("Failed to execute event", "eventID", consensusEvent.TriggerEventID, "err", err2) + return + } + }) + } + wg.Wait() +} + +func (c *consensusQueuePlugin) tryStartExecution(ctx context.Context, fetcher *blobCache, consensusEvent *pb.ConsensusQueueEvent, blobHandles [][]byte) error { + // any blob will do, first that exists + var pbEvent *pb.CoordinatedEvent + var errs []error + for _, blob := range consensusEvent.Blobs { + var err error + if int(blob.Blob) > len(blobHandles) { + c.lggr.Errorw("Blob handle index out of bounds", "index", int(blob.Blob), "len", len(blobHandles)) + continue + } + blobHandle := blobHandles[int(blob.Blob)] + pbEvent, _, err = fetchCoordinatedEventFromBlob(ctx, fetcher, blobHandle, int(blob.Element)) + if err != nil { + errs = append(errs, fmt.Errorf("failed to fetch blob: %w", err)) + c.lggr.Errorw("Failed to fetch blob", "handle", blobHandle, "index", blob.Element, "err", err) + continue // try next blob + } + break + } + if pbEvent == nil { + return fmt.Errorf("failed to fetch blob from all %d handles: %w", len(consensusEvent.Blobs), errors.Join(errs...)) + } + // fill in medianized timestamps + pbEvent.ObservedAt = consensusEvent.ObservedAt + pbEvent.Deadline = consensusEvent.Deadline + + var event triggers.CoordinatedEvent + if err := event.FromProto(pbEvent); err != nil { + return fmt.Errorf("failed to convert CoordinatedEvent from proto: %w", err) + } + + return c.triggerCoordinator.StartExecution(ctx, event) +} + +func (c *consensusQueuePlugin) Reports(ctx context.Context, _ uint64, _ ocr3_1types.ReportsPlusPrecursor) ([]ocr3types.ReportPlus[[]byte], error) { return nil, nil } -func (c *consensusQueuePlugin) ShouldAcceptAttestedReport(ctx context.Context, seqNr uint64, reportWithInfo ocr3types.ReportWithInfo[[]byte]) (bool, error) { - return true, nil +func (c *consensusQueuePlugin) ShouldAcceptAttestedReport(ctx context.Context, _ uint64, _ ocr3types.ReportWithInfo[[]byte]) (bool, error) { + return false, nil } -func (c *consensusQueuePlugin) ShouldTransmitAcceptedReport(ctx context.Context, seqNr uint64, reportWithInfo ocr3types.ReportWithInfo[[]byte]) (bool, error) { - return true, nil +func (c *consensusQueuePlugin) ShouldTransmitAcceptedReport(ctx context.Context, _ uint64, _ ocr3types.ReportWithInfo[[]byte]) (bool, error) { + return false, nil } diff --git a/core/services/ocr3_1/plugins/queue/queue_test.go b/core/services/ocr3_1/plugins/queue/queue_test.go index f5b83690325..9ce9228d4fa 100644 --- a/core/services/ocr3_1/plugins/queue/queue_test.go +++ b/core/services/ocr3_1/plugins/queue/queue_test.go @@ -1,6 +1,8 @@ package queue import ( + "context" + "iter" "testing" "github.com/stretchr/testify/require" @@ -10,6 +12,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2/triggers" ) // libocr calls NewReportingPlugin with no recover, so a panic here takes down @@ -18,7 +21,7 @@ func TestConsensusQueuePluginFactory_NewReportingPlugin(t *testing.T) { t.Parallel() cfg := &PluginConfig{Limits: ocr3_1types.ReportingPluginLimits{MaxKeyValueModifiedKeys: 1}} - factory, err := NewConsensusQueuePluginFactory(logger.Test(t), cfg, limits.Factory{}) + factory, err := NewConsensusQueuePluginFactory(logger.Test(t), cfg, limits.Factory{}, noopCoordinator{}, noopQueue{}) require.NoError(t, err) plugin, info, err := factory.NewReportingPlugin(t.Context(), ocr3types.ReportingPluginConfig{N: 4, F: 1}, nil) @@ -32,3 +35,65 @@ func TestConsensusQueuePluginFactory_NewReportingPlugin(t *testing.T) { require.NoError(t, plugin.Close()) } + +type noopCoordinator struct{} + +func (n noopCoordinator) Start(ctx context.Context) error { + return nil +} + +func (n noopCoordinator) Close() error { + return nil +} + +func (n noopCoordinator) Ready() error { + return nil +} + +func (n noopCoordinator) HealthReport() map[string]error { + return map[string]error{} +} + +func (n noopCoordinator) Name() string { + return "NoopTriggerCoordinator" +} + +func (n noopCoordinator) Ack(ctx context.Context, triggerCapID, triggerRegistrationID, eventID string) error { + return nil +} + +func (n noopCoordinator) RegisterTriggers(ctx context.Context, subscriber triggers.Subscriber, params triggers.RegistrationParams) ([]string, error) { + return nil, nil +} + +func (n noopCoordinator) UnregisterTriggers(ctx context.Context, workflowID string) error { + return nil +} + +func (n noopCoordinator) AvailableCapacity(ctx context.Context) (triggers.Capacity, error) { + return triggers.Capacity{}, nil +} + +func (n noopCoordinator) StartExecution(ctx context.Context, event triggers.CoordinatedEvent) error { + return nil +} + +type noopQueue struct{} + +func (n noopQueue) Put(ctx context.Context, event triggers.CoordinatedEvent) error { + return nil +} + +func (n noopQueue) Drain(ctx context.Context) ([]triggers.CoordinatedEvent, error) { + return nil, nil +} + +func (n noopQueue) Stats(ctx context.Context) (triggers.Stats, error) { + return triggers.Stats{}, nil +} + +func (n noopQueue) Sorted() iter.Seq[triggers.CoordinatedEvent] { + return func(yield func(triggers.CoordinatedEvent) bool) {} +} + +func (n noopQueue) Remove(ctx context.Context, triggerEventIDs []string) {} diff --git a/core/services/ocr3_1/plugins/queue/transmitter.go b/core/services/ocr3_1/plugins/queue/transmitter.go index 4c9b94187c5..4c43d022f08 100644 --- a/core/services/ocr3_1/plugins/queue/transmitter.go +++ b/core/services/ocr3_1/plugins/queue/transmitter.go @@ -5,26 +5,20 @@ import ( "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" ) -type contractTransmitter struct { - lggr logger.Logger +type noopTransmitter struct { fromAccount ocrtypes.Account } -func NewTransmitter(lggr logger.Logger, fromAccount ocrtypes.Account) ocr3types.ContractTransmitter[[]byte] { - return &contractTransmitter{ - lggr: lggr, - fromAccount: fromAccount, - } +func NewTransmitter(fromAccount ocrtypes.Account) ocr3types.ContractTransmitter[[]byte] { + return &noopTransmitter{fromAccount: fromAccount} } -func (c *contractTransmitter) Transmit(ctx context.Context, digest ocrtypes.ConfigDigest, seqNum uint64, rpi ocr3types.ReportWithInfo[[]byte], signatures []ocrtypes.AttributedOnchainSignature) error { +func (c *noopTransmitter) Transmit(ctx context.Context, digest ocrtypes.ConfigDigest, seqNum uint64, rpi ocr3types.ReportWithInfo[[]byte], signatures []ocrtypes.AttributedOnchainSignature) error { return nil } -func (c *contractTransmitter) FromAccount(ctx context.Context) (ocrtypes.Account, error) { +func (c *noopTransmitter) FromAccount(ctx context.Context) (ocrtypes.Account, error) { return c.fromAccount, nil } diff --git a/core/services/workflows/v2/triggers/coordinator.go b/core/services/workflows/v2/triggers/coordinator.go index 02ecf62a528..f12ced66992 100644 --- a/core/services/workflows/v2/triggers/coordinator.go +++ b/core/services/workflows/v2/triggers/coordinator.go @@ -83,6 +83,9 @@ type Coordinator interface { // Returns ErrCapacityNotSupported if the shared ExecutionConcurrency // limiter does not break down by scope. AvailableCapacity(ctx context.Context) (Capacity, error) + + // StartExecution starts execution of the triggered event or returns an error. + StartExecution(context.Context, CoordinatedEvent) error } // RegisteredEngine is what EngineRegistry.Get returns: a sink the coordinator @@ -476,6 +479,35 @@ func (c *coordinator) Ack(ctx context.Context, triggerCapID, triggerRegistration return Ack(ctx, lggr, wfMetrics, triggerCapID, triggerRegistrationID, eventID, handle) } +func (c *coordinator) StartExecution(ctx context.Context, event CoordinatedEvent) error { + wfID, err := types.WorkflowIDFromHex(event.WorkflowID) + if err != nil { + return err + } + + //TODO share code with admitEvent? + + eng, ok := c.engines.Get(wfID) + if !ok { + return fmt.Errorf("workflow %s: engine not found", wfID) + } + if !eng.IsCoordinated() { + return fmt.Errorf("workflow %s: engine not coordinated", wfID) + } + + //TODO more admission checks? + // c.admission.ExecutionConcurrency.Wait(ctx, 1)? + + _ = time.Since(event.ObservedAt) //TODO emit latency metric + + if err = eng.ExecuteTrigger(ctx, event); err != nil { + //TODO handle errors, ack? + return fmt.Errorf("workflow %s: failed to execute trigger %s: %w", wfID, event.Event.Event.ID, err) + } + + return nil +} + // close runs after the reader and release goroutines exit. // It unregisters any triggers still in a pending state before returning. func (c *coordinator) close() error { @@ -632,7 +664,7 @@ func (c *coordinator) admit(ctx context.Context, cw *coordinatedWorkflow) { eventAge := now.Sub(event.ObservedAt) cw.lggr.Debugw("Popped a trigger event from the queue", "eventID", event.Event.Event.ID, "eventAgeMs", eventAge.Milliseconds()) tm.RecordTriggerEventQueueWaitSeconds(ctx, eventAge.Seconds()) - if now.After(event.Deadline) { + if now.After(event.Deadline) { //TODO grace period cw.lggr.Warnw("Trigger event is too old, skipping execution", "triggerID", event.TriggerCapID, "eventID", event.Event.Event.ID, "eventAgeMs", eventAge.Milliseconds()) tm.IncrementTriggerEventExpiredCounter(ctx) tm.IncrementTriggerEventDroppedTotal(ctx, monitoring.TriggerDropReasonExpired) diff --git a/core/services/workflows/v2/triggers/coordinator_shadow_test.go b/core/services/workflows/v2/triggers/coordinator_shadow_test.go index a05d1c804c7..960ec5e8e73 100644 --- a/core/services/workflows/v2/triggers/coordinator_shadow_test.go +++ b/core/services/workflows/v2/triggers/coordinator_shadow_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "iter" "strings" "sync" "sync/atomic" @@ -333,6 +334,12 @@ type failingQueue struct { puts atomic.Int32 } +func (q *failingQueue) Sorted() iter.Seq[CoordinatedEvent] { + return func(yield func(CoordinatedEvent) bool) {} +} + +func (q *failingQueue) Remove(ctx context.Context, triggerEventIDs []string) {} + func (q *failingQueue) Put(context.Context, CoordinatedEvent) error { q.puts.Add(1) return errors.New("boom") diff --git a/core/services/workflows/v2/triggers/drainer_test.go b/core/services/workflows/v2/triggers/drainer_test.go index c499af946b7..03c6f990094 100644 --- a/core/services/workflows/v2/triggers/drainer_test.go +++ b/core/services/workflows/v2/triggers/drainer_test.go @@ -1,8 +1,11 @@ package triggers import ( + "container/heap" "context" "errors" + "iter" + "slices" "sync" "sync/atomic" "testing" @@ -53,9 +56,42 @@ func findMetric(t *testing.T, reader *sdkmetric.ManualReader, name string) *metr // fakeQueue is a minimal Queue whose Observe pops a scripted sequence of // results and counts calls. Once the script is exhausted it returns empty. type fakeQueue struct { - mu sync.Mutex - results []observeResult - calls int + mu sync.Mutex + results []*observeResult + drainCalls int +} + +func (q *fakeQueue) Sorted() iter.Seq[CoordinatedEvent] { + q.mu.Lock() + events := slices.Clone(q.results[0].events) + q.mu.Unlock() + + eventHeap := new(coordinatedEventHeap(events)) + heap.Init(eventHeap) + return func(yield func(CoordinatedEvent) bool) { + for eventHeap.Len() > 0 { + e := heap.Pop(eventHeap).(CoordinatedEvent) + if !yield(e) { + return + } + } + } +} + +func (q *fakeQueue) Remove(ctx context.Context, triggerEventIDs []string) { + triggerEventIDSet := map[string]struct{}{} + for _, triggerEventID := range triggerEventIDs { + triggerEventIDSet[triggerEventID] = struct{}{} + } + q.mu.Lock() + defer q.mu.Unlock() + for _, r := range q.results { + r.events = slices.DeleteFunc(r.events, func(e CoordinatedEvent) bool { + _, ok := triggerEventIDSet[e.Event.Event.ID] + return ok + }) + // note: empty results remain + } } type observeResult struct { @@ -70,7 +106,7 @@ func (q *fakeQueue) Stats(context.Context) (Stats, error) { return Stats{}, nil func (q *fakeQueue) Drain(context.Context) ([]CoordinatedEvent, error) { q.mu.Lock() defer q.mu.Unlock() - q.calls++ + q.drainCalls++ if len(q.results) == 0 { return nil, nil } @@ -79,10 +115,10 @@ func (q *fakeQueue) Drain(context.Context) ([]CoordinatedEvent, error) { return r.events, r.err } -func (q *fakeQueue) observeCalls() int { +func (q *fakeQueue) getDrainCalls() int { q.mu.Lock() defer q.mu.Unlock() - return q.calls + return q.drainCalls } func (q *fakeQueue) remaining() int { @@ -115,25 +151,25 @@ func TestDrainer_Drain(t *testing.T) { tests := []struct { name string - results []observeResult // one per drain call + results []*observeResult // one per drain call want [][]CoordinatedEvent }{ { name: "queues an observed batch", - results: []observeResult{{events: batchOf("a")}}, + results: []*observeResult{{events: batchOf("a")}}, want: [][]CoordinatedEvent{batchOf("a")}, }, { name: "empty observe queues nothing", - results: []observeResult{{}}, + results: []*observeResult{{}}, }, { name: "observe error queues nothing, even with events", - results: []observeResult{{events: batchOf("ignored"), err: errors.New("boom")}}, + results: []*observeResult{{events: batchOf("ignored"), err: errors.New("boom")}}, }, { name: "full buffer drops the overflow and keeps the oldest", - results: []observeResult{{events: batchOf("a")}, {events: batchOf("b")}}, + results: []*observeResult{{events: batchOf("a")}, {events: batchOf("b")}}, want: [][]CoordinatedEvent{batchOf("a")}, }, } @@ -151,7 +187,7 @@ func TestDrainer_Drain(t *testing.T) { d.drain(t.Context()) // must return even when the buffer is full } - assert.Equal(t, len(tt.results), q.observeCalls(), "one Observe per drain") + assert.Equal(t, len(tt.results), q.getDrainCalls(), "one Observe per drain") assert.Equal(t, tt.want, queued(d)) assert.False(t, hookCalled.Load(), "drain never calls the hook itself; only the worker does") }) @@ -164,7 +200,7 @@ func TestDrainer_DroppedEventsMetric(t *testing.T) { t.Parallel() dm, reader := testDrainerMetrics(t) - q := &fakeQueue{results: []observeResult{ + q := &fakeQueue{results: []*observeResult{ {events: batchOf("a")}, {events: append(batchOf("b"), batchOf("c")...)}, // dropped: the buffer holds "a" }} @@ -188,7 +224,7 @@ func TestDrainer_HookDurationMetric(t *testing.T) { t.Parallel() dm, reader := testDrainerMetrics(t) - q := &fakeQueue{results: []observeResult{{events: batchOf("a")}}} + q := &fakeQueue{results: []*observeResult{{events: batchOf("a")}}} d := NewDrainer(q, limits.NewUpperBoundLimiter(minDrainInterval), drainerHooks{}, dm, clockwork.NewRealClock(), logger.Test(t)) require.NoError(t, d.Start(t.Context())) t.Cleanup(func() { assert.NoError(t, d.Close()) }) @@ -209,17 +245,17 @@ func TestDrainer_Running(t *testing.T) { tests := []struct { name string - results []observeResult + results []*observeResult want []string // workflow IDs the hook should see }{ { name: "hook receives an observed batch", - results: []observeResult{{events: batchOf("a")}}, + results: []*observeResult{{events: batchOf("a")}}, want: []string{"a"}, }, { name: "loop survives an observe error", - results: []observeResult{{err: errors.New("boom")}, {events: batchOf("a")}}, + results: []*observeResult{{err: errors.New("boom")}, {events: batchOf("a")}}, want: []string{"a"}, }, } @@ -269,9 +305,9 @@ func TestDrainer_BlockedHook(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - results := make([]observeResult, tt.batches) + results := make([]*observeResult, tt.batches) for i := range results { - results[i] = observeResult{events: batchOf("x")} + results[i] = &observeResult{events: batchOf("x")} } q := &fakeQueue{results: results} @@ -349,12 +385,12 @@ func TestDrainer_IntervalReRead(t *testing.T) { drainsAfter := func(wait time.Duration) { t.Helper() waiting() - before := q.observeCalls() + before := q.getDrainCalls() clock.Advance(wait - time.Millisecond) - assert.Never(t, func() bool { return q.observeCalls() > before }, 50*time.Millisecond, 5*time.Millisecond, + assert.Never(t, func() bool { return q.getDrainCalls() > before }, 50*time.Millisecond, 5*time.Millisecond, "drained before %s", wait) clock.Advance(time.Millisecond) - require.Eventually(t, func() bool { return q.observeCalls() == before+1 }, 5*time.Second, 5*time.Millisecond, + require.Eventually(t, func() bool { return q.getDrainCalls() == before+1 }, 5*time.Second, 5*time.Millisecond, "no drain at %s", wait) } @@ -390,7 +426,7 @@ func TestNewCentralQueueDrainer_LogsDrainedEvents(t *testing.T) { t.Parallel() clock := clockwork.NewFakeClock() batch := append(append(batchOf("wf-a"), batchOf("wf-b")...), batchOf("wf-a")...) - q := &fakeQueue{results: []observeResult{{events: batch}}} + q := &fakeQueue{results: []*observeResult{{events: batch}}} dm, _ := testDrainerMetrics(t) lggr, logs := logger.TestObserved(t, zapcore.DebugLevel) d := NewCentralQueueDrainer(q, limits.NewUpperBoundLimiter(minDrainInterval), dm, clock, lggr) diff --git a/core/services/workflows/v2/triggers/queue.go b/core/services/workflows/v2/triggers/queue.go index cfa3d0089cd..35643d60577 100644 --- a/core/services/workflows/v2/triggers/queue.go +++ b/core/services/workflows/v2/triggers/queue.go @@ -1,9 +1,12 @@ package triggers import ( + "container/heap" "context" "errors" + "iter" "maps" + "slices" "sync" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -29,6 +32,13 @@ type Queue interface { // Stats reports current length, capacity, and cumulative drops. Stats(context.Context) (Stats, error) + + // Sorted returns an iterator over all events, ordered by deadline and then trigger event ID. + // It does not drain the queue. Events remain but can be removed via Remove. + Sorted() iter.Seq[CoordinatedEvent] + + // Remove removes events from the queue. + Remove(ctx context.Context, triggerEventIDs []string) } // Stats is a point-in-time view of a Queue. @@ -113,6 +123,79 @@ func (q *CentralQueue) Drain(ctx context.Context) ([]CoordinatedEvent, error) { return events, nil } +func (q *CentralQueue) Remove(ctx context.Context, triggerEventIDs []string) { + triggerEventIDSet := map[string]struct{}{} + for _, triggerEventID := range triggerEventIDs { + triggerEventIDSet[triggerEventID] = struct{}{} + } + + capacity := q.capacity(ctx) + + q.mu.Lock() + defer q.mu.Unlock() + + var deleted int + q.events = slices.DeleteFunc(q.events, func(e CoordinatedEvent) bool { + _, ok := triggerEventIDSet[e.Event.Event.ID] + if ok { + deleted++ + q.depthByWorkflow[e.WorkflowID]-- + q.depthByTriggerType[e.TriggerCapID]-- + //TODO emit metric for time.Since(e.ObservedAt) + } + return ok + }) + q.metrics.IncrementDrainedEventsCounter(ctx, deleted) + q.metrics.RecordDepth(ctx, len(q.events)) + q.metrics.RecordCapacity(ctx, q.effectiveCapacity(capacity)) + return +} + +func (q *CentralQueue) Sorted() iter.Seq[CoordinatedEvent] { + q.mu.Lock() + events := slices.Clone(q.events) + q.mu.Unlock() + + eventHeap := new(coordinatedEventHeap(events)) + heap.Init(eventHeap) + + return func(yield func(CoordinatedEvent) bool) { + for eventHeap.Len() > 0 { + if !yield((*eventHeap)[0]) { + return + } + heap.Pop(eventHeap) + } + } +} + +type coordinatedEventHeap []CoordinatedEvent + +func (c *coordinatedEventHeap) Len() int { + if c == nil { + return 0 + } + return len(*c) +} + +func (c *coordinatedEventHeap) Less(i, j int) bool { + return (*c)[i].CompareDeadline(&(*c)[j]) < 0 +} + +func (c *coordinatedEventHeap) Swap(i, j int) { + (*c)[i], (*c)[j] = (*c)[j], (*c)[i] +} + +func (c *coordinatedEventHeap) Push(x any) { + *c = append(*c, x.(CoordinatedEvent)) +} + +func (c *coordinatedEventHeap) Pop() any { + e := (*c)[len(*c)-1] + *c = (*c)[:len(*c)-1] + return e +} + // Stats returns copies of the queue's maps, so callers may keep or change them. func (q *CentralQueue) Stats(ctx context.Context) (Stats, error) { capacity := q.capacity(ctx) diff --git a/core/services/workflows/v2/triggers/queue_test.go b/core/services/workflows/v2/triggers/queue_test.go index 3966a48396d..378696d62b8 100644 --- a/core/services/workflows/v2/triggers/queue_test.go +++ b/core/services/workflows/v2/triggers/queue_test.go @@ -71,13 +71,12 @@ func testQueueMetrics(t *testing.T) (*monitoring.CentralTriggerQueueMetricLabele func centralEvent(workflowID string, triggerIndex int, id string) CoordinatedEvent { observed := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) return CoordinatedEvent{ - WorkflowID: workflowID, - TriggerCapID: fmt.Sprintf("trigger-%d@1.0.0", triggerIndex), - TriggerIndex: triggerIndex, - ObservedAt: observed, - Deadline: observed.Add(time.Minute), - SequenceNumber: 42, - Event: capabilities.TriggerResponse{Event: capabilities.TriggerEvent{ID: id}}, + WorkflowID: workflowID, + TriggerCapID: fmt.Sprintf("trigger-%d@1.0.0", triggerIndex), + TriggerIndex: triggerIndex, + ObservedAt: observed, + Deadline: observed.Add(time.Minute), + Event: capabilities.TriggerResponse{Event: capabilities.TriggerEvent{ID: id}}, } } @@ -269,9 +268,8 @@ func TestQueue_ConcurrentPutDrain(t *testing.T) { var wg sync.WaitGroup for p := range producers { wg.Go(func() { - for seq := range perProducer { + for range perProducer { e := centralEvent(fmt.Sprintf("wf-%d", p), 0, "evt") - e.SequenceNumber = uint64(seq) err := q.Put(t.Context(), e) if err == nil { accepted[p]++ @@ -334,13 +332,13 @@ func TestQueue_ConcurrentPutDrain(t *testing.T) { assert.Positive(t, totalDropped, "the drop path never ran") } - last := map[string]uint64{} + last := map[string]time.Time{} for _, e := range drained { prev, seen := last[e.WorkflowID] if seen { - assert.Greater(t, e.SequenceNumber, prev, "order for %s", e.WorkflowID) + assert.Greater(t, e.ObservedAt, prev, "order for %s", e.WorkflowID) } - last[e.WorkflowID] = e.SequenceNumber + last[e.WorkflowID] = e.ObservedAt } stats, err := q.Stats(t.Context()) diff --git a/core/services/workflows/v2/triggers/triggers.go b/core/services/workflows/v2/triggers/triggers.go index 3d584f10079..b653de95539 100644 --- a/core/services/workflows/v2/triggers/triggers.go +++ b/core/services/workflows/v2/triggers/triggers.go @@ -1,6 +1,7 @@ package triggers import ( + "cmp" "context" "errors" "fmt" @@ -182,20 +183,21 @@ type CoordinatedEvent struct { // A settings change after dispatch does not affect already-queued events Deadline time.Time - // SequenceNumber determines the execution order of trigger events across the DON. - SequenceNumber uint64 - Event capabilities.TriggerResponse + Event capabilities.TriggerResponse +} + +func (e *CoordinatedEvent) CompareDeadline(o *CoordinatedEvent) int { + return cmp.Or(e.Deadline.Compare(o.Deadline), cmp.Compare(e.Event.Event.ID, o.Event.Event.ID)) } func (e *CoordinatedEvent) ToProto() *pb.CoordinatedEvent { return &pb.CoordinatedEvent{ - WorkflowID: e.WorkflowID, - TriggerCapID: e.TriggerCapID, - TriggerIndex: int64(e.TriggerIndex), - ObservedAt: timestamppb.New(e.ObservedAt), - Deadline: timestamppb.New(e.Deadline), - SequenceNumber: e.SequenceNumber, - Event: pb.TriggerResponseToProto(e.Event), + WorkflowID: e.WorkflowID, + TriggerCapID: e.TriggerCapID, + TriggerIndex: int64(e.TriggerIndex), + ObservedAt: timestamppb.New(e.ObservedAt), + Deadline: timestamppb.New(e.Deadline), + Event: pb.TriggerResponseToProto(e.Event), } } @@ -205,7 +207,6 @@ func (e *CoordinatedEvent) FromProto(p *pb.CoordinatedEvent) (err error) { e.TriggerIndex = int(p.TriggerIndex) e.ObservedAt = p.ObservedAt.AsTime() e.Deadline = p.Deadline.AsTime() - e.SequenceNumber = p.SequenceNumber e.Event, err = pb.TriggerResponseFromProto(p.Event) return } diff --git a/deployment/cre/jobs/propose_job_spec_test.go b/deployment/cre/jobs/propose_job_spec_test.go index 898fc65f1fd..684e20f69cc 100644 --- a/deployment/cre/jobs/propose_job_spec_test.go +++ b/deployment/cre/jobs/propose_job_spec_test.go @@ -31,6 +31,7 @@ import ( "github.com/smartcontractkit/chainlink/deployment/cre/test" ) +// TODO add a subtest for ConsensusQueue func TestProposeJobSpec_VerifyPreconditions(t *testing.T) { j := jobs.ProposeJobSpec{} var env cldf.Environment @@ -530,6 +531,7 @@ func TestProposeJobSpec_VerifyPreconditions_Aptos(t *testing.T) { } } +// TODO add a subtest for ConsensusQueue func TestProposeJobSpec_Apply(t *testing.T) { h := test.NewTestHarness(t) env := new(h.Runtime.Environment()) diff --git a/deployment/go.mod b/deployment/go.mod index 38a5014a057..52fcdcb2041 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -41,7 +41,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/deployment v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260922043016-517a12a4220e github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb github.com/smartcontractkit/chainlink-data-streams v1.1.3 github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 diff --git a/deployment/go.sum b/deployment/go.sum index 4d2ca4bf731..52da3f4f005 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1411,8 +1411,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.16.0 h1:WhDQEEI2Kb5A4C0PNAa+nyGLDvG0H+uT6h4y1NwmkYg= github.com/smartcontractkit/chainlink-ccv v0.16.0/go.mod h1:oaSLVHiJVbJ9aCcUQnoRuVONACWzA/DWVqvUnLAe9bE= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad h1:2thuaxFkbEEnPz2mlUC+AOIp6m5IRduqxThR9mzozU4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 h1:QQpXxddXoJi7b200xLaTI8yO8iKOukebi8WctQotQmY= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb h1:9rVrSeVOAwZsVe2pOpeVRp9XjhtMyjndgeROFglvOhg= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb/go.mod h1:WcOKzF6HkTDrkQ96aBHhcJ2IIYE7iK7M2tdt58ea2qc= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261106154152-9426766b30a9 h1:rs2zmGoDw4AHaJ2Y76J46slPPE3iL801sWPgOTCyDZc= diff --git a/go.mod b/go.mod index e7923153e80..f8de929717a 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62 github.com/smartcontractkit/chainlink-ccv v0.16.0 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261106154152-9426766b30a9 github.com/smartcontractkit/chainlink-data-streams v1.1.3 diff --git a/go.sum b/go.sum index ff1d531e709..e5ba383d77a 100644 --- a/go.sum +++ b/go.sum @@ -1102,8 +1102,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260908155618-50f521d70e62/go.mod h1:c++Ed0jeYn4w8WIfiAboRwN6ClRRVGglL0qwB/QjJvM= github.com/smartcontractkit/chainlink-ccv v0.16.0 h1:WhDQEEI2Kb5A4C0PNAa+nyGLDvG0H+uT6h4y1NwmkYg= github.com/smartcontractkit/chainlink-ccv v0.16.0/go.mod h1:oaSLVHiJVbJ9aCcUQnoRuVONACWzA/DWVqvUnLAe9bE= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad h1:2thuaxFkbEEnPz2mlUC+AOIp6m5IRduqxThR9mzozU4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 h1:QQpXxddXoJi7b200xLaTI8yO8iKOukebi8WctQotQmY= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb h1:9rVrSeVOAwZsVe2pOpeVRp9XjhtMyjndgeROFglvOhg= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb/go.mod h1:WcOKzF6HkTDrkQ96aBHhcJ2IIYE7iK7M2tdt58ea2qc= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261106154152-9426766b30a9 h1:rs2zmGoDw4AHaJ2Y76J46slPPE3iL801sWPgOTCyDZc= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index e9bae7403ba..404dddb6f4b 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -30,7 +30,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260922043016-517a12a4220e github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260922043016-517a12a4220e - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261112211734-e640e1c12078 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 069b6b58900..fb9330cb328 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1396,8 +1396,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.16.0 h1:WhDQEEI2Kb5A4C0PNAa+nyGLDvG0H+uT6h4y1NwmkYg= github.com/smartcontractkit/chainlink-ccv v0.16.0/go.mod h1:oaSLVHiJVbJ9aCcUQnoRuVONACWzA/DWVqvUnLAe9bE= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad h1:2thuaxFkbEEnPz2mlUC+AOIp6m5IRduqxThR9mzozU4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 h1:QQpXxddXoJi7b200xLaTI8yO8iKOukebi8WctQotQmY= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb h1:9rVrSeVOAwZsVe2pOpeVRp9XjhtMyjndgeROFglvOhg= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb/go.mod h1:WcOKzF6HkTDrkQ96aBHhcJ2IIYE7iK7M2tdt58ea2qc= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261106154152-9426766b30a9 h1:rs2zmGoDw4AHaJ2Y76J46slPPE3iL801sWPgOTCyDZc= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 0d85f379a3d..e6c25a18c78 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -20,7 +20,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260922043016-517a12a4220e github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260922043016-517a12a4220e - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261112211734-e640e1c12078 github.com/smartcontractkit/chainlink-testing-framework/framework v0.16.9 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 1f222fc34a9..591c6c77cb0 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1636,8 +1636,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.16.0 h1:WhDQEEI2Kb5A4C0PNAa+nyGLDvG0H+uT6h4y1NwmkYg= github.com/smartcontractkit/chainlink-ccv v0.16.0/go.mod h1:oaSLVHiJVbJ9aCcUQnoRuVONACWzA/DWVqvUnLAe9bE= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad h1:2thuaxFkbEEnPz2mlUC+AOIp6m5IRduqxThR9mzozU4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 h1:QQpXxddXoJi7b200xLaTI8yO8iKOukebi8WctQotQmY= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb h1:9rVrSeVOAwZsVe2pOpeVRp9XjhtMyjndgeROFglvOhg= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb/go.mod h1:WcOKzF6HkTDrkQ96aBHhcJ2IIYE7iK7M2tdt58ea2qc= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261106154152-9426766b30a9 h1:rs2zmGoDw4AHaJ2Y76J46slPPE3iL801sWPgOTCyDZc= diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index e5c51dfa545..2ab485d707f 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -32,7 +32,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260922043016-517a12a4220e - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-deployments-framework v0.126.0 diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 431fd93adda..bd08182f238 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1657,8 +1657,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.16.0 h1:WhDQEEI2Kb5A4C0PNAa+nyGLDvG0H+uT6h4y1NwmkYg= github.com/smartcontractkit/chainlink-ccv v0.16.0/go.mod h1:oaSLVHiJVbJ9aCcUQnoRuVONACWzA/DWVqvUnLAe9bE= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad h1:2thuaxFkbEEnPz2mlUC+AOIp6m5IRduqxThR9mzozU4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 h1:QQpXxddXoJi7b200xLaTI8yO8iKOukebi8WctQotQmY= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb h1:9rVrSeVOAwZsVe2pOpeVRp9XjhtMyjndgeROFglvOhg= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb/go.mod h1:WcOKzF6HkTDrkQ96aBHhcJ2IIYE7iK7M2tdt58ea2qc= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261106154152-9426766b30a9 h1:rs2zmGoDw4AHaJ2Y76J46slPPE3iL801sWPgOTCyDZc= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index 59c108595cb..f4cbd125a58 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -73,7 +73,7 @@ require ( github.com/smartcontractkit/chain-selectors v1.0.112 github.com/smartcontractkit/chainlink-aptos v0.0.0-20260923062809-ca886ade4d26 github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260922043016-517a12a4220e - github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad + github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261106154152-9426766b30a9 github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index 3e653142aa3..4616835de7d 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1836,8 +1836,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d6 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca/go.mod h1:Dm/Abz0vr7F2v9TygUX947q8HIwJEJikDfSdanX1Cic= github.com/smartcontractkit/chainlink-ccv v0.16.0 h1:WhDQEEI2Kb5A4C0PNAa+nyGLDvG0H+uT6h4y1NwmkYg= github.com/smartcontractkit/chainlink-ccv v0.16.0/go.mod h1:oaSLVHiJVbJ9aCcUQnoRuVONACWzA/DWVqvUnLAe9bE= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad h1:2thuaxFkbEEnPz2mlUC+AOIp6m5IRduqxThR9mzozU4= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087 h1:QQpXxddXoJi7b200xLaTI8yO8iKOukebi8WctQotQmY= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087/go.mod h1:81DIRjlxA+Nq4sugpyD0zofdXg46owE/N3eter3+EkI= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb h1:9rVrSeVOAwZsVe2pOpeVRp9XjhtMyjndgeROFglvOhg= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb/go.mod h1:WcOKzF6HkTDrkQ96aBHhcJ2IIYE7iK7M2tdt58ea2qc= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261106154152-9426766b30a9 h1:rs2zmGoDw4AHaJ2Y76J46slPPE3iL801sWPgOTCyDZc=