Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/services/cre/cre.go
Original file line number Diff line number Diff line change
Expand Up @@ -484,6 +484,7 @@ func (s *Services) newRegistrySyncer(
relayer,
registryAddress,
registrysyncerV1.NewORM(ds, lggr),
opts.Meter,
)
if err != nil {
return nil, nil, nil, fmt.Errorf("could not configure syncer: %w", err)
Expand Down
30 changes: 25 additions & 5 deletions core/services/registrysyncer/v2/monitoring.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"

"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"

"github.com/smartcontractkit/chainlink-common/pkg/beholder"
Expand All @@ -16,24 +17,36 @@ type syncerMetricLabeler struct {
metrics.Labeler
remoteRegistrySyncFailureCounter metric.Int64Counter
launcherFailureCounter metric.Int64Counter
registryEntriesGauge metric.Int64Gauge
}

func newSyncerMetricLabeler() (*syncerMetricLabeler, error) {
remoteRegistrySyncFailureCounter, err := beholder.GetMeter().Int64Counter("platform_registrysyncer_sync_failures")
func newSyncerMetricLabeler(meter metric.Meter) (*syncerMetricLabeler, error) {
remoteRegistrySyncFailureCounter, err := meter.Int64Counter("platform_registrysyncer_sync_failures")
if err != nil {
return nil, fmt.Errorf("failed to register sync failure counter: %w", err)
}

launcherFailureCounter, err := beholder.GetMeter().Int64Counter("platform_registrysyncer_launch_failures")
launcherFailureCounter, err := meter.Int64Counter("platform_registrysyncer_launch_failures")
if err != nil {
return nil, fmt.Errorf("failed to register launcher failure counter: %w", err)
}

return &syncerMetricLabeler{remoteRegistrySyncFailureCounter: remoteRegistrySyncFailureCounter, launcherFailureCounter: launcherFailureCounter}, nil
registryEntriesGauge, err := meter.Int64Gauge("platform_registrysyncer_registry_entries")
if err != nil {
return nil, fmt.Errorf("failed to register registry entries gauge: %w", err)
}

return &syncerMetricLabeler{
remoteRegistrySyncFailureCounter: remoteRegistrySyncFailureCounter,
launcherFailureCounter: launcherFailureCounter,
registryEntriesGauge: registryEntriesGauge,
}, nil
}

func (c *syncerMetricLabeler) with(keyValues ...string) syncerMetricLabeler {
return syncerMetricLabeler{c.With(keyValues...), c.remoteRegistrySyncFailureCounter, c.launcherFailureCounter}
labeled := *c
labeled.Labeler = c.With(keyValues...)
return labeled
}

func (c *syncerMetricLabeler) incrementRemoteRegistryFailureCounter(ctx context.Context) {
Expand All @@ -45,3 +58,10 @@ func (c *syncerMetricLabeler) incrementLauncherFailureCounter(ctx context.Contex
otelLabels := beholder.OtelAttributes(c.Labels).AsStringAttributes()
c.launcherFailureCounter.Add(ctx, 1, metric.WithAttributes(otelLabels...))
}

// recordRegistryEntries records how many entries of the given type ("capabilities", "dons", "nodes") were read on-chain.
func (c *syncerMetricLabeler) recordRegistryEntries(ctx context.Context, entryType string, count int) {
otelLabels := beholder.OtelAttributes(c.Labels).AsStringAttributes()
otelLabels = append(otelLabels, attribute.String("type", entryType))
c.registryEntriesGauge.Record(ctx, int64(count), metric.WithAttributes(otelLabels...))
}
5 changes: 3 additions & 2 deletions core/services/registrysyncer/v2/monitoring_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,18 @@ import (
"testing"

"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/metric/noop"

"github.com/smartcontractkit/chainlink-common/pkg/metrics"
)

func Test_InitMonitoringResources(t *testing.T) {
_, err := newSyncerMetricLabeler()
_, err := newSyncerMetricLabeler(noop.NewMeterProvider().Meter(""))
require.NoError(t, err)
}

func Test_SyncerMetricsLabeler(t *testing.T) {
testSyncerMetricLabeler := syncerMetricLabeler{metrics.NewLabeler(), nil, nil}
testSyncerMetricLabeler := syncerMetricLabeler{Labeler: metrics.NewLabeler()}
testSyncerMetricLabeler2 := testSyncerMetricLabeler.with("foo", "baz")
require.Equal(t, "baz", testSyncerMetricLabeler2.Labels["foo"])
}
56 changes: 40 additions & 16 deletions core/services/registrysyncer/v2/syncer.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ import (
"sync"
"time"

"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/metric/noop"

p2ptypes "github.com/smartcontractkit/libocr/ragep2p/types"

"github.com/smartcontractkit/chainlink-common/pkg/capabilities"
Expand Down Expand Up @@ -78,8 +81,12 @@ func New(
relayer ContractReaderFactory,
registryAddress string,
orm registrysyncer.ORM,
meter metric.Meter,
) (RegistrySyncer, error) {
metricLabeler, err := newSyncerMetricLabeler()
if meter == nil {
meter = noop.NewMeterProvider().Meter("")
}
metricLabeler, err := newSyncerMetricLabeler(meter)
if err != nil {
return nil, fmt.Errorf("failed to create syncer metric labeler: %w", err)
}
Expand Down Expand Up @@ -226,16 +233,33 @@ func (s *registrySyncer) updateStateLoop() {
}
}

const (
capabilitiesPageSize = 100
donsPageSize = 20
nodesPageSize = 30
)

// readAllPages reads a paginated CapabilitiesRegistry list until a page comes back short.
func readAllPages[T any](ctx context.Context, reader types.ContractReader, readIdentifier string, pageSize int64) ([]T, error) {
var all []T
for start := int64(0); ; start += pageSize {
params := struct {
Start *big.Int
Limit *big.Int
}{Start: big.NewInt(start), Limit: big.NewInt(pageSize)}
var page []T
if err := reader.GetLatestValue(ctx, readIdentifier, primitives.Unconfirmed, params, &page); err != nil {
return nil, err
}
all = append(all, page...)
if int64(len(page)) < pageSize {
return all, nil
}
}
}

func (s *registrySyncer) importOnchainRegistry(ctx context.Context) (*registry.RegistryMetadata, error) {
caps := []capabilities_registry_v2.CapabilitiesRegistryCapabilityInfo{}
// TODO support pagination if needed
// Using large limit for now to avoid pagination complexity
// since we don't expect to have that many capabilities
params := struct {
Start *big.Int
Limit *big.Int
}{Start: big.NewInt(0), Limit: big.NewInt(1024)}
err := s.reader.GetLatestValue(ctx, s.capabilitiesContract.ReadIdentifier("getCapabilities"), primitives.Unconfirmed, params, &caps)
caps, err := readAllPages[capabilities_registry_v2.CapabilitiesRegistryCapabilityInfo](ctx, s.reader, s.capabilitiesContract.ReadIdentifier("getCapabilities"), capabilitiesPageSize)
if err != nil {
return nil, fmt.Errorf("failed to get latest value for getCapabilities: %w", err)
}
Expand All @@ -253,9 +277,7 @@ func (s *registrySyncer) importOnchainRegistry(ctx context.Context) (*registry.R
}
}

dons := []capabilities_registry_v2.CapabilitiesRegistryDONInfo{}

err = s.reader.GetLatestValue(ctx, s.capabilitiesContract.ReadIdentifier("getDONs"), primitives.Unconfirmed, params, &dons)
dons, err := readAllPages[capabilities_registry_v2.CapabilitiesRegistryDONInfo](ctx, s.reader, s.capabilitiesContract.ReadIdentifier("getDONs"), donsPageSize)
if err != nil {
return nil, fmt.Errorf("failed to get latest value for getDONs: %w", err)
}
Expand All @@ -275,13 +297,15 @@ func (s *registrySyncer) importOnchainRegistry(ctx context.Context) (*registry.R
}
}

nodes := []capabilities_registry_v2.INodeInfoProviderNodeInfo{}

err = s.reader.GetLatestValue(ctx, s.capabilitiesContract.ReadIdentifier("getNodes"), primitives.Unconfirmed, params, &nodes)
nodes, err := readAllPages[capabilities_registry_v2.INodeInfoProviderNodeInfo](ctx, s.reader, s.capabilitiesContract.ReadIdentifier("getNodes"), nodesPageSize)
if err != nil {
return nil, fmt.Errorf("failed to get latest value for getNodes: %w", err)
}

s.metrics.recordRegistryEntries(ctx, "capabilities", len(caps))
s.metrics.recordRegistryEntries(ctx, "dons", len(dons))
s.metrics.recordRegistryEntries(ctx, "nodes", len(nodes))

idsToNodes := map[p2ptypes.PeerID]registry.NodeInfo{}
for _, node := range nodes {
nodeInfo := registry.NodeInfo{
Expand Down
91 changes: 88 additions & 3 deletions core/services/registrysyncer/v2/syncer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import (
"crypto/rand"
"encoding/json"
"errors"
"fmt"
"math/big"
"slices"
"sync"
"testing"
"time"
Expand Down Expand Up @@ -294,7 +296,7 @@ func TestReader_Integration(t *testing.T) {
db := pgtest.NewSqlxDB(t)
factory := newContractReaderFactory(t, simulatedBackend)
syncerORM := registrysyncer.NewORM(db, lggr)
syncer, err := registrysyncer_v2.New(lggr, func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, factory, regAddress.Hex(), syncerORM)
syncer, err := registrysyncer_v2.New(lggr, func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, factory, regAddress.Hex(), syncerORM, nil)
require.NoError(t, err)

l := &launcher{}
Expand Down Expand Up @@ -512,6 +514,7 @@ func TestSyncer_V2_DBIntegration(t *testing.T) {
factory,
regAddress.Hex(),
syncerORM,
nil,
)
require.NoError(t, err)
l := &launcher{}
Expand Down Expand Up @@ -841,7 +844,7 @@ func TestReader_V2_FamilyOperations(t *testing.T) {
db := pgtest.NewSqlxDB(t)
factory := newContractReaderFactory(t, simulatedBackend)
syncerORM := registrysyncer.NewORM(db, lggr)
syncer, err := registrysyncer_v2.New(lggr, func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, factory, regAddress.Hex(), syncerORM)
syncer, err := registrysyncer_v2.New(lggr, func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, factory, regAddress.Hex(), syncerORM, nil)
require.NoError(t, err)

l := &launcher{}
Expand Down Expand Up @@ -1082,7 +1085,7 @@ func TestSync_V2_PreservesNodeCapabilitiesDONIds(t *testing.T) {

reader := &stubReader{nodes: []capabilities_registry_v2.INodeInfoProviderNodeInfo{node}}

syncer, err := registrysyncer_v2.New(logger.Test(t), func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, &stubReaderFactory{reader: reader}, "0x0", nil)
syncer, err := registrysyncer_v2.New(logger.Test(t), func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil }, &stubReaderFactory{reader: reader}, "0x0", nil, nil)
require.NoError(t, err)

l := &launcher{}
Expand All @@ -1095,3 +1098,85 @@ func TestSync_V2_PreservesNodeCapabilitiesDONIds(t *testing.T) {
got := l.localRegistry.IDsToNodes[p2ptypes.PeerID(node.P2pId)]
require.Equal(t, donIDs, got.CapabilitiesDONIds)
}

func TestSync_V2_Pagination(t *testing.T) {
t.Parallel()

const (
numNodes = 55 // > nodesPageSize
numDONs = 45 // > 2 * donsPageSize
)

owner := evmtestutils.MustNewSimTransactor(t)
balance, _ := new(big.Int).SetString("100000000000000000000", 10)
simulatedBackend := simulated.NewBackend(gethtypes.GenesisAlloc{owner.From: {Balance: balance}},
simulated.WithBlockGasLimit(ethconfig.Defaults.Miner.GasCeil*2))
simulatedBackend.Commit()

regAddress, _, reg, err := capabilities_registry_v2.DeployCapabilitiesRegistry(owner, simulatedBackend.Client(), capabilities_registry_v2.CapabilitiesRegistryConstructorParams{})
require.NoError(t, err)
simulatedBackend.Commit()

const cid = "write-chain@1.0.1"
_, err = reg.AddCapabilities(owner, []capabilities_registry_v2.CapabilitiesRegistryCapability{
{CapabilityId: cid, Metadata: []byte(`{"capabilityType": 3, "responseType": 1}`)},
})
require.NoError(t, err)
_, err = reg.AddNodeOperators(owner, []capabilities_registry_v2.CapabilitiesRegistryNodeOperatorParams{{Admin: owner.From, Name: "nop"}})
require.NoError(t, err)
simulatedBackend.Commit()

nodes := make([]capabilities_registry_v2.CapabilitiesRegistryNodeParams, numNodes)
for i := range nodes {
nodes[i] = capabilities_registry_v2.CapabilitiesRegistryNodeParams{
NodeOperatorId: 1,
Signer: randomWord(),
P2pId: randomWord(),
EncryptionPublicKey: randomWord(),
CsaKey: randomWord(),
CapabilityIds: []string{cid},
}
}
for batch := range slices.Chunk(nodes, 20) {
_, err = reg.AddNodes(owner, batch)
require.NoError(t, err)
simulatedBackend.Commit()
}

members := [][32]byte{nodes[0].P2pId, nodes[1].P2pId, nodes[2].P2pId, nodes[3].P2pId}
for batch := 0; batch < numDONs; batch += 15 {
var dons []capabilities_registry_v2.CapabilitiesRegistryNewDONParams
for i := batch; i < min(batch+15, numDONs); i++ {
dons = append(dons, capabilities_registry_v2.CapabilitiesRegistryNewDONParams{
Name: fmt.Sprintf("don-%d", i),
DonFamilies: []string{"family"},
CapabilityConfigurations: []capabilities_registry_v2.CapabilitiesRegistryCapabilityConfiguration{{CapabilityId: cid, Config: []byte{}}},
Nodes: members,
F: 1,
IsPublic: true,
})
}
_, err = reg.AddDONs(owner, dons)
require.NoError(t, err)
simulatedBackend.Commit()
}

syncer, err := registrysyncer_v2.New(logger.Test(t), func() (p2ptypes.PeerID, error) { return p2ptypes.PeerID{}, nil },
newContractReaderFactory(t, simulatedBackend), regAddress.Hex(), nil, nil)
require.NoError(t, err)
l := &launcher{}
syncer.AddListener(l)
require.NoError(t, syncer.Sync(t.Context(), false))

l.mu.RLock()
defer l.mu.RUnlock()
s := l.localRegistry
require.NotNil(t, s)
assert.Len(t, s.IDsToCapabilities, 1)
assert.Len(t, s.IDsToNodes, numNodes)
require.Len(t, s.IDsToDONs, numDONs)
for id := 1; id <= numDONs; id++ {
assert.Equal(t, fmt.Sprintf("don-%d", id-1), s.IDsToDONs[registry.DonID(id)].Name)
}
assert.Len(t, s.IDsToNodes[p2ptypes.PeerID(members[0])].CapabilitiesDONIds, numDONs)
}
Loading