From 893c003e393628cc834a73d05a48918476fbb050 Mon Sep 17 00:00:00 2001 From: Bolek Kulbabinski <1416262+bolekk@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:23:53 -0800 Subject: [PATCH 1/2] Paginate CapabilitiesRegistry v2 reads in registry syncer getCapabilities, getDONs and getNodes were each read as a single page of 1024 entries, so one eth_call's gas grows with the registry size. Read in fixed-size pages instead to stay well below RPC eth_call gas caps. Co-Authored-By: Claude Opus 5.5 --- core/services/registrysyncer/v2/syncer.go | 43 ++++++---- .../services/registrysyncer/v2/syncer_test.go | 82 +++++++++++++++++++ 2 files changed, 110 insertions(+), 15 deletions(-) diff --git a/core/services/registrysyncer/v2/syncer.go b/core/services/registrysyncer/v2/syncer.go index b481f207e99..0c064377b05 100644 --- a/core/services/registrysyncer/v2/syncer.go +++ b/core/services/registrysyncer/v2/syncer.go @@ -226,16 +226,33 @@ func (s *registrySyncer) updateStateLoop() { } } +const ( + capabilitiesPageSize = 100 + donsPageSize = 20 + nodesPageSize = 30 +) + +// readAllPages reads a paginated CapabilitiesRegistry list until a page comes back short. +func readAllPages[T any](ctx context.Context, reader types.ContractReader, readIdentifier string, pageSize int64) ([]T, error) { + var all []T + for start := int64(0); ; start += pageSize { + params := struct { + Start *big.Int + Limit *big.Int + }{Start: big.NewInt(start), Limit: big.NewInt(pageSize)} + var page []T + if err := reader.GetLatestValue(ctx, readIdentifier, primitives.Unconfirmed, params, &page); err != nil { + return nil, err + } + all = append(all, page...) + if int64(len(page)) < pageSize { + return all, nil + } + } +} + func (s *registrySyncer) importOnchainRegistry(ctx context.Context) (*registry.RegistryMetadata, error) { - caps := []capabilities_registry_v2.CapabilitiesRegistryCapabilityInfo{} - // TODO support pagination if needed - // Using large limit for now to avoid pagination complexity - // since we don't expect to have that many capabilities - params := struct { - Start *big.Int - Limit *big.Int - }{Start: big.NewInt(0), Limit: big.NewInt(1024)} - err := s.reader.GetLatestValue(ctx, s.capabilitiesContract.ReadIdentifier("getCapabilities"), primitives.Unconfirmed, params, &caps) + caps, err := readAllPages[capabilities_registry_v2.CapabilitiesRegistryCapabilityInfo](ctx, s.reader, s.capabilitiesContract.ReadIdentifier("getCapabilities"), capabilitiesPageSize) if err != nil { return nil, fmt.Errorf("failed to get latest value for getCapabilities: %w", err) } @@ -253,9 +270,7 @@ func (s *registrySyncer) importOnchainRegistry(ctx context.Context) (*registry.R } } - dons := []capabilities_registry_v2.CapabilitiesRegistryDONInfo{} - - err = s.reader.GetLatestValue(ctx, s.capabilitiesContract.ReadIdentifier("getDONs"), primitives.Unconfirmed, params, &dons) + dons, err := readAllPages[capabilities_registry_v2.CapabilitiesRegistryDONInfo](ctx, s.reader, s.capabilitiesContract.ReadIdentifier("getDONs"), donsPageSize) if err != nil { return nil, fmt.Errorf("failed to get latest value for getDONs: %w", err) } @@ -275,9 +290,7 @@ func (s *registrySyncer) importOnchainRegistry(ctx context.Context) (*registry.R } } - nodes := []capabilities_registry_v2.INodeInfoProviderNodeInfo{} - - err = s.reader.GetLatestValue(ctx, s.capabilitiesContract.ReadIdentifier("getNodes"), primitives.Unconfirmed, params, &nodes) + nodes, err := readAllPages[capabilities_registry_v2.INodeInfoProviderNodeInfo](ctx, s.reader, s.capabilitiesContract.ReadIdentifier("getNodes"), nodesPageSize) if err != nil { return nil, fmt.Errorf("failed to get latest value for getNodes: %w", err) } diff --git a/core/services/registrysyncer/v2/syncer_test.go b/core/services/registrysyncer/v2/syncer_test.go index 359422f332f..8d724890ddd 100644 --- a/core/services/registrysyncer/v2/syncer_test.go +++ b/core/services/registrysyncer/v2/syncer_test.go @@ -5,7 +5,9 @@ import ( "crypto/rand" "encoding/json" "errors" + "fmt" "math/big" + "slices" "sync" "testing" "time" @@ -1095,3 +1097,83 @@ func TestSync_V2_PreservesNodeCapabilitiesDONIds(t *testing.T) { got := l.localRegistry.IDsToNodes[p2ptypes.PeerID(node.P2pId)] require.Equal(t, donIDs, got.CapabilitiesDONIds) } + +func TestSync_V2_Pagination(t *testing.T) { + const ( + numNodes = 55 // > nodesPageSize + numDONs = 45 // > 2 * donsPageSize + ) + + owner := evmtestutils.MustNewSimTransactor(t) + balance, _ := new(big.Int).SetString("100000000000000000000", 10) + simulatedBackend := simulated.NewBackend(gethtypes.GenesisAlloc{owner.From: {Balance: balance}}, + simulated.WithBlockGasLimit(ethconfig.Defaults.Miner.GasCeil*2)) + simulatedBackend.Commit() + + regAddress, _, reg, err := capabilities_registry_v2.DeployCapabilitiesRegistry(owner, simulatedBackend.Client(), capabilities_registry_v2.CapabilitiesRegistryConstructorParams{}) + require.NoError(t, err) + simulatedBackend.Commit() + + const cid = "write-chain@1.0.1" + _, err = reg.AddCapabilities(owner, []capabilities_registry_v2.CapabilitiesRegistryCapability{ + {CapabilityId: cid, Metadata: []byte(`{"capabilityType": 3, "responseType": 1}`)}, + }) + require.NoError(t, err) + _, err = reg.AddNodeOperators(owner, []capabilities_registry_v2.CapabilitiesRegistryNodeOperatorParams{{Admin: owner.From, Name: "nop"}}) + require.NoError(t, err) + simulatedBackend.Commit() + + nodes := make([]capabilities_registry_v2.CapabilitiesRegistryNodeParams, numNodes) + for i := range nodes { + nodes[i] = capabilities_registry_v2.CapabilitiesRegistryNodeParams{ + NodeOperatorId: 1, + Signer: randomWord(), + P2pId: randomWord(), + EncryptionPublicKey: randomWord(), + CsaKey: randomWord(), + CapabilityIds: []string{cid}, + } + } + for batch := range slices.Chunk(nodes, 20) { + _, err = reg.AddNodes(owner, batch) + require.NoError(t, err) + simulatedBackend.Commit() + } + + members := [][32]byte{nodes[0].P2pId, nodes[1].P2pId, nodes[2].P2pId, nodes[3].P2pId} + for batch := 0; batch < numDONs; batch += 15 { + var dons []capabilities_registry_v2.CapabilitiesRegistryNewDONParams + for i := batch; i < min(batch+15, numDONs); i++ { + dons = append(dons, capabilities_registry_v2.CapabilitiesRegistryNewDONParams{ + Name: fmt.Sprintf("don-%d", i), + DonFamilies: []string{"family"}, + CapabilityConfigurations: []capabilities_registry_v2.CapabilitiesRegistryCapabilityConfiguration{{CapabilityId: cid, Config: []byte{}}}, + Nodes: members, + F: 1, + IsPublic: true, + }) + } + _, err = reg.AddDONs(owner, dons) + require.NoError(t, err) + simulatedBackend.Commit() + } + + syncer, err := registrysyncer_v2.New(logger.Test(t), func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, + newContractReaderFactory(t, simulatedBackend), regAddress.Hex(), nil) + require.NoError(t, err) + l := &launcher{} + syncer.AddListener(l) + require.NoError(t, syncer.Sync(t.Context(), false)) + + l.mu.RLock() + defer l.mu.RUnlock() + s := l.localRegistry + require.NotNil(t, s) + assert.Len(t, s.IDsToCapabilities, 1) + assert.Len(t, s.IDsToNodes, numNodes) + require.Len(t, s.IDsToDONs, numDONs) + for id := 1; id <= numDONs; id++ { + assert.Equal(t, fmt.Sprintf("don-%d", id-1), s.IDsToDONs[registry.DonID(id)].Name) + } + assert.Len(t, s.IDsToNodes[p2ptypes.PeerID(members[0])].CapabilitiesDONIds, numDONs) +} From 37174b164d2a5c311edb4d97076351c4ca7c1b51 Mon Sep 17 00:00:00 2001 From: Bolek Kulbabinski <1416262+bolekk@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:28:22 -0800 Subject: [PATCH 2/2] Add registry entries gauge to v2 registry syncer Records how many capabilities, DONs and nodes were read on-chain per sync, so read counts can be compared against on-chain totals. Co-Authored-By: Claude Opus 5.5 --- core/services/cre/cre.go | 1 + core/services/registrysyncer/v2/monitoring.go | 30 +++++++++++++++---- .../registrysyncer/v2/monitoring_test.go | 5 ++-- core/services/registrysyncer/v2/syncer.go | 13 +++++++- .../services/registrysyncer/v2/syncer_test.go | 11 ++++--- 5 files changed, 48 insertions(+), 12 deletions(-) diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index 1a752ca7e6d..0a1f6ceb71b 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -484,6 +484,7 @@ func (s *Services) newRegistrySyncer( relayer, registryAddress, registrysyncerV1.NewORM(ds, lggr), + opts.Meter, ) if err != nil { return nil, nil, nil, fmt.Errorf("could not configure syncer: %w", err) diff --git a/core/services/registrysyncer/v2/monitoring.go b/core/services/registrysyncer/v2/monitoring.go index 8d56768d167..455421f1042 100644 --- a/core/services/registrysyncer/v2/monitoring.go +++ b/core/services/registrysyncer/v2/monitoring.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "github.com/smartcontractkit/chainlink-common/pkg/beholder" @@ -16,24 +17,36 @@ type syncerMetricLabeler struct { metrics.Labeler remoteRegistrySyncFailureCounter metric.Int64Counter launcherFailureCounter metric.Int64Counter + registryEntriesGauge metric.Int64Gauge } -func newSyncerMetricLabeler() (*syncerMetricLabeler, error) { - remoteRegistrySyncFailureCounter, err := beholder.GetMeter().Int64Counter("platform_registrysyncer_sync_failures") +func newSyncerMetricLabeler(meter metric.Meter) (*syncerMetricLabeler, error) { + remoteRegistrySyncFailureCounter, err := meter.Int64Counter("platform_registrysyncer_sync_failures") if err != nil { return nil, fmt.Errorf("failed to register sync failure counter: %w", err) } - launcherFailureCounter, err := beholder.GetMeter().Int64Counter("platform_registrysyncer_launch_failures") + launcherFailureCounter, err := meter.Int64Counter("platform_registrysyncer_launch_failures") if err != nil { return nil, fmt.Errorf("failed to register launcher failure counter: %w", err) } - return &syncerMetricLabeler{remoteRegistrySyncFailureCounter: remoteRegistrySyncFailureCounter, launcherFailureCounter: launcherFailureCounter}, nil + registryEntriesGauge, err := meter.Int64Gauge("platform_registrysyncer_registry_entries") + if err != nil { + return nil, fmt.Errorf("failed to register registry entries gauge: %w", err) + } + + return &syncerMetricLabeler{ + remoteRegistrySyncFailureCounter: remoteRegistrySyncFailureCounter, + launcherFailureCounter: launcherFailureCounter, + registryEntriesGauge: registryEntriesGauge, + }, nil } func (c *syncerMetricLabeler) with(keyValues ...string) syncerMetricLabeler { - return syncerMetricLabeler{c.With(keyValues...), c.remoteRegistrySyncFailureCounter, c.launcherFailureCounter} + labeled := *c + labeled.Labeler = c.With(keyValues...) + return labeled } func (c *syncerMetricLabeler) incrementRemoteRegistryFailureCounter(ctx context.Context) { @@ -45,3 +58,10 @@ func (c *syncerMetricLabeler) incrementLauncherFailureCounter(ctx context.Contex otelLabels := beholder.OtelAttributes(c.Labels).AsStringAttributes() c.launcherFailureCounter.Add(ctx, 1, metric.WithAttributes(otelLabels...)) } + +// recordRegistryEntries records how many entries of the given type ("capabilities", "dons", "nodes") were read on-chain. +func (c *syncerMetricLabeler) recordRegistryEntries(ctx context.Context, entryType string, count int) { + otelLabels := beholder.OtelAttributes(c.Labels).AsStringAttributes() + otelLabels = append(otelLabels, attribute.String("type", entryType)) + c.registryEntriesGauge.Record(ctx, int64(count), metric.WithAttributes(otelLabels...)) +} diff --git a/core/services/registrysyncer/v2/monitoring_test.go b/core/services/registrysyncer/v2/monitoring_test.go index b35901a4ba6..a800c09f8da 100644 --- a/core/services/registrysyncer/v2/monitoring_test.go +++ b/core/services/registrysyncer/v2/monitoring_test.go @@ -4,17 +4,18 @@ import ( "testing" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/metric/noop" "github.com/smartcontractkit/chainlink-common/pkg/metrics" ) func Test_InitMonitoringResources(t *testing.T) { - _, err := newSyncerMetricLabeler() + _, err := newSyncerMetricLabeler(noop.NewMeterProvider().Meter("")) require.NoError(t, err) } func Test_SyncerMetricsLabeler(t *testing.T) { - testSyncerMetricLabeler := syncerMetricLabeler{metrics.NewLabeler(), nil, nil} + testSyncerMetricLabeler := syncerMetricLabeler{Labeler: metrics.NewLabeler()} testSyncerMetricLabeler2 := testSyncerMetricLabeler.with("foo", "baz") require.Equal(t, "baz", testSyncerMetricLabeler2.Labels["foo"]) } diff --git a/core/services/registrysyncer/v2/syncer.go b/core/services/registrysyncer/v2/syncer.go index 0c064377b05..8be68da4dc2 100644 --- a/core/services/registrysyncer/v2/syncer.go +++ b/core/services/registrysyncer/v2/syncer.go @@ -10,6 +10,9 @@ import ( "sync" "time" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/metric/noop" + p2ptypes "github.com/smartcontractkit/libocr/ragep2p/types" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" @@ -78,8 +81,12 @@ func New( relayer ContractReaderFactory, registryAddress string, orm registrysyncer.ORM, + meter metric.Meter, ) (RegistrySyncer, error) { - metricLabeler, err := newSyncerMetricLabeler() + if meter == nil { + meter = noop.NewMeterProvider().Meter("") + } + metricLabeler, err := newSyncerMetricLabeler(meter) if err != nil { return nil, fmt.Errorf("failed to create syncer metric labeler: %w", err) } @@ -295,6 +302,10 @@ func (s *registrySyncer) importOnchainRegistry(ctx context.Context) (*registry.R return nil, fmt.Errorf("failed to get latest value for getNodes: %w", err) } + s.metrics.recordRegistryEntries(ctx, "capabilities", len(caps)) + s.metrics.recordRegistryEntries(ctx, "dons", len(dons)) + s.metrics.recordRegistryEntries(ctx, "nodes", len(nodes)) + idsToNodes := map[p2ptypes.PeerID]registry.NodeInfo{} for _, node := range nodes { nodeInfo := registry.NodeInfo{ diff --git a/core/services/registrysyncer/v2/syncer_test.go b/core/services/registrysyncer/v2/syncer_test.go index 8d724890ddd..482c6e998da 100644 --- a/core/services/registrysyncer/v2/syncer_test.go +++ b/core/services/registrysyncer/v2/syncer_test.go @@ -296,7 +296,7 @@ func TestReader_Integration(t *testing.T) { db := pgtest.NewSqlxDB(t) factory := newContractReaderFactory(t, simulatedBackend) syncerORM := registrysyncer.NewORM(db, lggr) - syncer, err := registrysyncer_v2.New(lggr, func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, factory, regAddress.Hex(), syncerORM) + syncer, err := registrysyncer_v2.New(lggr, func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, factory, regAddress.Hex(), syncerORM, nil) require.NoError(t, err) l := &launcher{} @@ -514,6 +514,7 @@ func TestSyncer_V2_DBIntegration(t *testing.T) { factory, regAddress.Hex(), syncerORM, + nil, ) require.NoError(t, err) l := &launcher{} @@ -843,7 +844,7 @@ func TestReader_V2_FamilyOperations(t *testing.T) { db := pgtest.NewSqlxDB(t) factory := newContractReaderFactory(t, simulatedBackend) syncerORM := registrysyncer.NewORM(db, lggr) - syncer, err := registrysyncer_v2.New(lggr, func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, factory, regAddress.Hex(), syncerORM) + syncer, err := registrysyncer_v2.New(lggr, func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, factory, regAddress.Hex(), syncerORM, nil) require.NoError(t, err) l := &launcher{} @@ -1084,7 +1085,7 @@ func TestSync_V2_PreservesNodeCapabilitiesDONIds(t *testing.T) { reader := &stubReader{nodes: []capabilities_registry_v2.INodeInfoProviderNodeInfo{node}} - syncer, err := registrysyncer_v2.New(logger.Test(t), func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, &stubReaderFactory{reader: reader}, "0x0", nil) + syncer, err := registrysyncer_v2.New(logger.Test(t), func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, &stubReaderFactory{reader: reader}, "0x0", nil, nil) require.NoError(t, err) l := &launcher{} @@ -1099,6 +1100,8 @@ func TestSync_V2_PreservesNodeCapabilitiesDONIds(t *testing.T) { } func TestSync_V2_Pagination(t *testing.T) { + t.Parallel() + const ( numNodes = 55 // > nodesPageSize numDONs = 45 // > 2 * donsPageSize @@ -1159,7 +1162,7 @@ func TestSync_V2_Pagination(t *testing.T) { } syncer, err := registrysyncer_v2.New(logger.Test(t), func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, - newContractReaderFactory(t, simulatedBackend), regAddress.Hex(), nil) + newContractReaderFactory(t, simulatedBackend), regAddress.Hex(), nil, nil) require.NoError(t, err) l := &launcher{} syncer.AddListener(l)