Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/scripts/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ require (
github.com/shopspring/decimal v1.4.0
github.com/smartcontractkit/chain-selectors v1.0.112
github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20261001213322-a2d686f610ca
github.com/smartcontractkit/chainlink-common v0.11.2-0.20261109191636-419bd72008ad
github.com/smartcontractkit/chainlink-common v0.11.2-0.20261118150912-05caa74bd087
github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20261111193935-c83f8e2996fb
github.com/smartcontractkit/chainlink-deployments-framework v0.126.0
github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261112211734-e640e1c12078
Expand Down
4 changes: 2 additions & 2 deletions core/scripts/go.sum

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions core/services/chainlink/application.go
Original file line number Diff line number Diff line change
Expand Up @@ -841,6 +841,8 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err
OrgResolver: creServices.OrgResolver,
LimitsFactory: limitsFactory,
OCRConfigService: creServices.OCRConfigService,
TriggerCoordinator: creServices.TriggerCoordinator,
TriggerQueue: creServices.TriggerQueue,
DefaultBootstrappers: safeDefaultBootstrappers(cfg),
CapRegistryAddress: safeExternalRegistryAddress(cfg),
CapRegistryChainID: safeExternalRegistryChainID(cfg),
Expand Down
44 changes: 25 additions & 19 deletions core/services/cre/cre.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,10 @@ type Services struct {

WorkflowRegistrySyncer syncerV2.WorkflowRegistrySyncer

TriggerCoordinator triggers.Coordinator

TriggerQueue triggers.Queue

OrgResolver orgresolver.OrgResolver

OCRConfigService capregconfig.OCRConfigService
Expand Down Expand Up @@ -297,7 +301,7 @@ func (s *Services) newSubservices(
wfSyncerLggr.Infof("failed to create billing client: %s", err)
}

wfSyncer, wfSyncerSrvcs, err := newWorkflowRegistrySyncerV2(
wfSyncer, triggerCoordinator, triggerQueue, wfSyncerSrvcs, err := newWorkflowRegistrySyncerV2(
cfg,
relayerChainInterops,
billingClient,
Expand All @@ -320,6 +324,8 @@ func (s *Services) newSubservices(
}
s.BillingClient = billingClient
s.WorkflowRegistrySyncer = wfSyncer
s.TriggerCoordinator = triggerCoordinator
s.TriggerQueue = triggerQueue
srvs = append(srvs, wfSyncerSrvcs...)

return srvs, nil
Expand Down Expand Up @@ -800,14 +806,14 @@ func newWorkflowRegistrySyncerV2(
meterIdentity resourcemanager.ResourceIdentity,
dispatcher remotetypes.Dispatcher,
shardIndexMapper *shardownership.ShardIndexMapper,
) (syncerV2.WorkflowRegistrySyncer, []commonsrv.Service, error) {
) (syncerV2.WorkflowRegistrySyncer, triggers.Coordinator, triggers.Queue, []commonsrv.Service, error) {
capCfg := cfg.Capabilities()
wfReg := capCfg.WorkflowRegistry()
key := opts.WorkflowKey

fetcherFunc, retrieverFunc, srvcs, err := newFetcherServiceV2(opts, capCfg, lggr, gatewayConnectorWrapper)
if err != nil {
return nil, nil, err
return nil, nil, nil, nil, err
}

artifactsStore, err := artifactsV2.NewStore(
Expand All @@ -824,29 +830,29 @@ func newWorkflowRegistrySyncerV2(
}),
)
if err != nil {
return nil, nil, fmt.Errorf("unable to create artifact store: %w", err)
return nil, nil, nil, nil, fmt.Errorf("unable to create artifact store: %w", err)
}

engineRegistry := syncerV2.NewEngineRegistry()

engineLimiters, err := v2.NewLimiters(lf, nil)
if err != nil {
return nil, nil, fmt.Errorf("could not instantiate engine limiters: %w", err)
return nil, nil, nil, nil, fmt.Errorf("could not instantiate engine limiters: %w", err)
}

featureFlags, err := v2.NewFeatureFlags(lf, nil)
if err != nil {
return nil, nil, fmt.Errorf("could not instantiate engine feature flags: %w", err)
return nil, nil, nil, nil, fmt.Errorf("could not instantiate engine feature flags: %w", err)
}

selector, err := chainSelector(wfReg.ChainID(), wfReg.NetworkID())
if err != nil {
return nil, nil, err
return nil, nil, nil, nil, err
}

crFactory, err := newContractReaderFactory(capCfg, relayerChainInterops)
if err != nil {
return nil, nil, fmt.Errorf("failed to instantiate contract reader factory: %w", err)
return nil, nil, nil, nil, fmt.Errorf("failed to instantiate contract reader factory: %w", err)
}

var shardOrchestratorClient shardorchestrator.ClientInterface
Expand All @@ -855,7 +861,7 @@ func newWorkflowRegistrySyncerV2(
} else {
c, clientErr := newShardOrchestratorClient(cfg, lggr)
if clientErr != nil {
return nil, nil, clientErr
return nil, nil, nil, nil, clientErr
}
if c != nil {
shardOrchestratorClient = c
Expand Down Expand Up @@ -902,15 +908,15 @@ func newWorkflowRegistrySyncerV2(

coordinatorMetrics, err := wfmonitoring.InitMonitoringResources()
if err != nil {
return nil, nil, fmt.Errorf("unable to init trigger coordinator metrics: %w", err)
return nil, nil, nil, nil, fmt.Errorf("unable to init trigger coordinator metrics: %w", err)
}
centralQueueMetrics, err := wfmonitoring.InitCentralTriggerQueueMetrics()
if err != nil {
return nil, nil, fmt.Errorf("unable to init central trigger queue metrics: %w", err)
return nil, nil, nil, nil, fmt.Errorf("unable to init central trigger queue metrics: %w", err)
}
drainerMetrics, err := wfmonitoring.InitTriggerDrainerMetrics()
if err != nil {
return nil, nil, fmt.Errorf("unable to init central trigger queue drainer metrics: %w", err)
return nil, nil, nil, nil, fmt.Errorf("unable to init central trigger queue drainer metrics: %w", err)
}
coordinatorClock := clockwork.NewRealClock()

Expand Down Expand Up @@ -971,7 +977,7 @@ func newWorkflowRegistrySyncerV2(
})
specMeter, smErr := syncerV2.NewSpecMeter(lggr, rm, meterIdentity, artifactsStore, orgResolver)
if smErr != nil {
return nil, nil, fmt.Errorf("unable to create workflow spec meter: %w", smErr)
return nil, nil, nil, nil, fmt.Errorf("unable to create workflow spec meter: %w", smErr)
}
handlerOpts = append(handlerOpts, syncerV2.WithSpecMeter(specMeter))
}
Expand All @@ -992,7 +998,7 @@ func newWorkflowRegistrySyncerV2(
if diskMonitorEnabled {
fileStore, fsErr := artifactsV2.NewFileModuleStore(mc.CacheDir(), cacheEnabled)
if fsErr != nil {
return nil, nil, fmt.Errorf("unable to create file module store: %w", fsErr)
return nil, nil, nil, nil, fmt.Errorf("unable to create file module store: %w", fsErr)
}

dm, dmErr := syncerV2.NewWorkflowModuleCacheDiskMonitor(
Expand All @@ -1001,7 +1007,7 @@ func newWorkflowRegistrySyncerV2(
syncerV2.WorkflowModuleCacheDiskMonitorTickInterval,
)
if dmErr != nil {
return nil, nil, fmt.Errorf("unable to create module cache disk monitor: %w", dmErr)
return nil, nil, nil, nil, fmt.Errorf("unable to create module cache disk monitor: %w", dmErr)
}
srvcs = append(srvcs, dm)

Expand All @@ -1010,7 +1016,7 @@ func newWorkflowRegistrySyncerV2(
if cacheEnabled {
cm, cmErr := syncerV2.NewCacheMetrics()
if cmErr != nil {
return nil, nil, fmt.Errorf("unable to create module cache metrics: %w", cmErr)
return nil, nil, nil, nil, fmt.Errorf("unable to create module cache metrics: %w", cmErr)
}

var idleTimeout time.Duration
Expand Down Expand Up @@ -1060,7 +1066,7 @@ func newWorkflowRegistrySyncerV2(
handlerOpts...,
)
if err != nil {
return nil, nil, fmt.Errorf("unable to create workflow registry event handler: %w", err)
return nil, nil, nil, nil, fmt.Errorf("unable to create workflow registry event handler: %w", err)
}

addSources := wfReg.AdditionalSources()
Expand Down Expand Up @@ -1113,12 +1119,12 @@ func newWorkflowRegistrySyncerV2(
registryOpts...,
)
if err != nil {
return nil, nil, fmt.Errorf("unable to create workflow registry syncer: %w", err)
return nil, nil, nil, nil, fmt.Errorf("unable to create workflow registry syncer: %w", err)
}

srvcs = append(srvcs, workflowRegistrySyncerV2)
lggr.Debugw("Created WorkflowRegistrySyncer V2")
return workflowRegistrySyncerV2, srvcs, nil
return workflowRegistrySyncerV2, triggerCoordinator, centralQueue, srvcs, nil
}

// NewServices creates and initializes all CRE services
Expand Down
12 changes: 10 additions & 2 deletions core/services/ocr2/delegate.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ import (
"github.com/smartcontractkit/chainlink/v2/core/services/synchronization"
"github.com/smartcontractkit/chainlink/v2/core/services/telemetry"
syncerV2 "github.com/smartcontractkit/chainlink/v2/core/services/workflows/syncer/v2"
"github.com/smartcontractkit/chainlink/v2/core/services/workflows/v2/triggers"
"github.com/smartcontractkit/chainlink/v2/core/utils"
"github.com/smartcontractkit/chainlink/v2/plugins"
)
Expand All @@ -99,6 +100,7 @@ const (
vaultDiskMonitorTickInterval = time.Minute

consensusQueueCapabilityID = internal.ConsensusQueueCapabilityID
consensusQueueOCRConfigKey = "consensus-queue"
gaugeConsensusQueueBytes = "platform_consensus_queue_usage_bytes"
consensusQueueDiskMonitorTickInterval = time.Minute
)
Expand Down Expand Up @@ -157,6 +159,8 @@ type Delegate struct {
OrgResolver orgresolver.OrgResolver
limitsFactory limits.Factory
ocrConfigService capregconfig.OCRConfigService
triggerCoordinator triggers.Coordinator
triggerQueue triggers.Queue

// Registry-driven launch support: when the delegate is used via
// LocalCapabilityManager (no job spec), these fields provide the
Expand Down Expand Up @@ -310,6 +314,8 @@ type DelegateOpts struct {
OrgResolver orgresolver.OrgResolver
LimitsFactory limits.Factory
OCRConfigService capregconfig.OCRConfigService
TriggerCoordinator triggers.Coordinator
TriggerQueue triggers.Queue

// Registry-driven launch support: defaults for capabilities launched
// via LocalCapabilityManager (no job spec).
Expand Down Expand Up @@ -359,6 +365,8 @@ func NewDelegate(
OrgResolver: opts.OrgResolver,
limitsFactory: opts.LimitsFactory,
ocrConfigService: opts.OCRConfigService,
triggerCoordinator: opts.TriggerCoordinator,
triggerQueue: opts.TriggerQueue,
defaultBootstrappers: opts.DefaultBootstrappers,
capRegistryAddress: opts.CapRegistryAddress,
capRegistryChainID: opts.CapRegistryChainID,
Expand Down Expand Up @@ -895,7 +903,7 @@ func (d *Delegate) newServicesConsensusQueue(
BinaryNetworkEndpointFactory: d.peerWrapper.Peer3_1,
V2Bootstrappers: bootstrapPeers,
ContractConfigTracker: configTracker,
ContractTransmitter: queue.NewTransmitter(lggr, ocrtypes.Account(spec.TransmitterID.String)),
ContractTransmitter: queue.NewTransmitter(ocrtypes.Account(spec.TransmitterID.String)),
Database: ocrDB,
KeyValueDatabaseFactory: kvFactory,
LocalConfig: lc,
Expand All @@ -907,7 +915,7 @@ func (d *Delegate) newServicesConsensusQueue(
MetricsRegisterer: prometheus.WrapRegistererWith(map[string]string{"job_name": jb.Name.ValueOrZero()}, prometheus.DefaultRegisterer),
}

rpf, err := queue.NewConsensusQueuePluginFactory(lggr, cfg, d.limitsFactory)
rpf, err := queue.NewConsensusQueuePluginFactory(lggr, cfg, d.limitsFactory, d.triggerCoordinator, d.triggerQueue)
if err != nil {
return nil, err
}
Expand Down
133 changes: 133 additions & 0 deletions core/services/ocr3_1/plugins/queue/blobs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
package queue

import (
"context"
"fmt"
"sync"
"time"

"google.golang.org/protobuf/encoding/protowire"
"google.golang.org/protobuf/proto"

"github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3_1types"

"github.com/smartcontractkit/chainlink-common/pkg/capabilities/pb"
)

// blobCache is a short-lived cache for coordinating concurrent fetches of event blobs.
type blobCache struct {
fetcher ocr3_1types.BlobFetcher

mu sync.RWMutex
cache map[string]func() ([]byte, error) // blob handle -> blob
}

func (c *blobCache) fetchBlob(ctx context.Context, handle []byte) ([]byte, error) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't the BlobFetcher already keep everything locally once it's fetched?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried to look for an implementation to reassure me that it would be cached efficiently, but IIUC we would at least be paying extra synchronization and allocation overhead.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

libocr caches on their end but you'd still be paying the network cost to fetch the blob from cache so the cache here is helpful

c.mu.RLock()
if fn, ok := c.cache[string(handle)]; ok {
c.mu.RUnlock()
return fn()
}
c.mu.RUnlock()

c.mu.Lock()
if fn, ok := c.cache[string(handle)]; ok {
c.mu.Unlock()
return fn()
}
fn := sync.OnceValues(func() ([]byte, error) {
var blobHandle ocr3_1types.BlobHandle
if err := blobHandle.UnmarshalBinary(handle); err != nil {
return nil, fmt.Errorf("failed to unmarshal blob handle: %w", err)
}
return c.fetcher.FetchBlob(ctx, blobHandle)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will cache all errors. Sometimes FetchBlob will fail because of latency or network issues which we'd want to retry the next time instead of immediately returning the cached error

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think that is the case for these particular usages. We just have short bursts of spawn/sink parallel execution that is simpler to self-organize around synchronization rather than do one fetch of each blob up front before giving out access. i.e. there is no next time in this cases - we just fetch once and then discard it. Unless you are saying we should retry within the same round?

})
c.cache[string(handle)] = fn
c.mu.Unlock()
return fn()
}

// fetchCoordinatedEventFromBlob fetches and extracts the idx-th event from the blob, without unmarshaling any other events.
func fetchCoordinatedEventFromBlob(ctx context.Context, fetcher *blobCache, handle []byte, idx int) (*pb.CoordinatedEvent, []byte, error) {
bb, err := fetcher.fetchBlob(ctx, handle)
if err != nil {
return nil, nil, fmt.Errorf("failed to fetch blob: %w", err)
}

coordEvent, eb, err := extractEvent(bb, idx)
if err != nil {
return nil, nil, err
}
return coordEvent, eb, nil
}

const coordinatedEventsFieldNumber = 1 // pb.CoordinatedEventList.Events

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it better to use something like this to get the field number?

ageFieldNumber := (&person.Person{}).
ProtoReflect().
Descriptor().
Fields().
ByName("age").
Number()

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a test set up to ensure it doesn't drift. I suppose we could derive it directly, though we have to give up having a const


// extractEvent extracts the idx-th event from the blob, without unmarshaling any other events.
func extractEvent(b []byte, idx int) (*pb.CoordinatedEvent, []byte, error) {
i := 0
for len(b) > 0 {
num, typ, n := protowire.ConsumeTag(b)
if n < 0 {
return nil, nil, protowire.ParseError(n)
}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Whitespace after the closing } (when it's not followed by another }) would make this a lot easier to read.

This issues is in many functions in the PR

b = b[n:]
if num == coordinatedEventsFieldNumber {
Comment thread
jmank88 marked this conversation as resolved.
Resolved
if typ != protowire.BytesType {
return nil, nil, fmt.Errorf("expected bytes type for field %d, got %d", coordinatedEventsFieldNumber, typ)
}
v, m := protowire.ConsumeBytes(b)
if m < 0 {
return nil, nil, protowire.ParseError(m)
}
if i == idx {
var event pb.CoordinatedEvent
if err := proto.Unmarshal(v, &event); err != nil {
return nil, v, fmt.Errorf("failed to unmarshal event: %w", err)
}
return &event, v, nil
}
// advance
i++
b = b[m:]
continue
} else {
n := protowire.ConsumeFieldValue(num, typ, b)
if n < 0 {
return nil, nil, protowire.ParseError(n)
}
b = b[n:]
}
}
return nil, nil, fmt.Errorf("index %d out of range", idx)
}

// broadcastAsBlob broadcasts a blob with as many coordinated events as possible, returning the blob handle and count of events included.
func broadcastAsBlob(ctx context.Context, broadcaster ocr3_1types.BlobBroadcastFetcher, events []*pb.CoordinatedEvent,
seqNr uint64, estimatedRoundInterval time.Duration) (blobHandle []byte, n int, err error) {
protoOpts := proto.MarshalOptions{Deterministic: true}
for ; protoOpts.Size(&pb.CoordinatedEventList{Events: events}) > maxBlobPayloadBytes; events = events[:len(events)-1] {
//TODO opt: binary search for largest fit instead....
}
n = len(events)
var maxDeadline time.Time
for _, event := range events {
if t := event.Deadline.AsTime(); t.After(maxDeadline) {
maxDeadline = t
}
}
roundDur := estimatedRoundInterval //TODO OK? pad?
//TODO pad extra time and/or rounds?
expSeqNr := seqNr + uint64(time.Until(maxDeadline)/roundDur)

data, err := protoOpts.Marshal(&pb.CoordinatedEventList{Events: events})
if err != nil {
return nil, -1, fmt.Errorf("failed to marshal blob: %w", err)
}
bh, err := broadcaster.BroadcastBlob(ctx, data, ocr3_1types.BlobExpirationHintSequenceNumber{SeqNr: expSeqNr})
if err != nil {
return nil, -1, fmt.Errorf("failed to broadcast blob: %w", err)
}
blobHandle, err = bh.MarshalBinary()
return
}
Loading