Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/infoplat-19823-llo-observed-value.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"chainlink": minor
---

#added llo_datasource_stream_observed_value gauge: latest observed value per LLO stream (value_type benchmark|bid|ask, config_digest). To forward it to Beholder, set `[Telemetry.PrometheusBridge] Enabled = true` and add `llo_datasource_stream_observed_value` to `Prefixes`; setting `Prefixes` replaces the default `["go_"]`, so keep `"go_"` and any prefixes already forwarded.
33 changes: 33 additions & 0 deletions core/services/llo/observation/data_source.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,10 @@ type dataSource struct {

// loopWakeCh coalesces plugin Observe hints (buffer 1): the loop may skip pacing when the plugin is active.
loopWakeCh chan struct{}

// observedValues exports the values that the production instance reads from the cache (see observe).
observedValues *observedValueGauges
observedValuesExpiryOnce sync.Once
}

var _ llodatasource.DataSource = &dataSource{}
Expand All @@ -189,6 +193,7 @@ func newDataSource(lggr logger.Logger, registry Registry, t Telemeter) *dataSour
cache: NewCache(time.Minute),
observationLoopCloseCh: make(chan struct{}),
loopWakeCh: make(chan struct{}, 1),
observedValues: newObservedValueGauges(promStreamObservedValue),
}
}

Expand Down Expand Up @@ -249,9 +254,36 @@ func (d *dataSource) observe(ctx context.Context, streamValues lloprotocol.Strea
// Update stream values with the cached observations for all streams.
d.cache.UpdateStreamValues(streamValues)

// Only the production instance records: a staging or retired instance reads the same cache and would
// duplicate every series under a second config digest. inProduction is false for nil opts.
if inProduction {
d.observedValues.record(opts.ConfigDigest(), streamValues, time.Now())
d.startObservedValuesExpiry()
}

return nil
}

// startObservedValuesExpiry starts (once) a goroutine that deletes observed value series of config digests that
// stopped calling Observe. record() alone cannot do this when no digest calls Observe at all (DON stall), and the
// observation loop exits when the stream set is empty, so expiry needs its own ticker. It stops on Close.
func (d *dataSource) startObservedValuesExpiry() {
d.observedValuesExpiryOnce.Do(func() {
d.wg.Go(func() {
ticker := time.NewTicker(observedValueDigestTTL / 2)
defer ticker.Stop()
for {
select {
case <-d.observationLoopCloseCh:
return
case now := <-ticker.C:
d.observedValues.expire(now)
}
}
})
})
}

// startObservationLoop continuously runs pipeline observations for this data source and caches results in memory,
// so the plugin Observe path stays fast regardless of adapter latency.
// Loop pacing (observationLoopPacing), per-round deadline (observationTimeout), cache TTL, and stale-refresh
Expand Down Expand Up @@ -526,6 +558,7 @@ func (d *dataSource) Close() error {
close(d.observationLoopCloseCh)
d.wg.Wait()
d.cache.Close()
d.observedValues.deleteAll()

return nil
}
Expand Down
69 changes: 69 additions & 0 deletions core/services/llo/observation/data_source_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1025,3 +1025,72 @@ func Test_DataSource_StagingDoesNotObserve(t *testing.T) {
require.Nil(t, vals[1], "staging instance must not populate stream values")
require.Zero(t, reg.pipelines[1].runCount.Load(), "staging instance must not run pipelines")
}

func Test_DataSource_ObservedValueGauge(t *testing.T) {
t.Parallel()
newReg := func() *mockRegistry {
reg := &mockRegistry{pipelines: make(map[streams.StreamID]*mockPipeline)}
multi := makePipelineWithMultipleStreamResults([]streams.StreamID{1, 2, 3}, []any{
decimal.NewFromInt(2181),
decimal.NewFromInt(40602),
decimal.NewFromInt(15),
})
reg.pipelines[1] = multi
reg.pipelines[2] = multi
reg.pipelines[3] = multi
return reg
}

t.Run("production records and Close deletes", func(t *testing.T) {
t.Parallel()
ds := newDataSource(logger.NullLogger, newReg(), telem.NullTelemeter)
vec := newTestObservedValueVec()
ds.observedValues = newObservedValueGauges(vec)
opts := &mockOpts{configDigest: ocr2types.ConfigDigest{6, 5, 4}}
digest := opts.configDigest.String()
want := map[[3]string]float64{
{"1", "benchmark", digest}: 2181,
{"2", "benchmark", digest}: 40602,
{"3", "benchmark", digest}: 15,
}

require.EventuallyWithT(t, func(c *assert.CollectT) {
ctx, cancel := context.WithTimeout(t.Context(), observationTimeout)
defer cancel()
assert.NoError(c, ds.Observe(ctx, makeStreamValues(), opts))
assert.Equal(c, want, observedSeries(c, vec))
}, 5*time.Second, 50*time.Millisecond)

require.NoError(t, ds.Close())
assert.Empty(t, observedSeries(t, vec))
})

t.Run("staging does not record", func(t *testing.T) {
t.Parallel()
ds := newDataSource(logger.NullLogger, newReg(), telem.NullTelemeter)
vec := newTestObservedValueVec()
ds.observedValues = newObservedValueGauges(vec)
defer ds.Close()
production := &mockOpts{configDigest: ocr2types.ConfigDigest{6, 5, 4}}
staging := &mockOpts{configDigest: ocr2types.ConfigDigest{9}, lifeCycleStage: lloprotocol.LifeCycleStageStaging}

// Populate the shared cache through the production instance first, so a staging Observe reads real values.
require.EventuallyWithT(t, func(c *assert.CollectT) {
ctx, cancel := context.WithTimeout(t.Context(), observationTimeout)
defer cancel()
assert.NoError(c, ds.Observe(ctx, makeStreamValues(), production))
assert.Len(c, observedSeries(c, vec), 3)
}, 5*time.Second, 50*time.Millisecond)

ctx, cancel := context.WithTimeout(t.Context(), observationTimeout)
defer cancel()
vals := makeStreamValues()
require.NoError(t, ds.Observe(ctx, vals, staging))
require.NotNil(t, vals[1], "staging Observe must read the cached value for this test to be meaningful")
series := observedSeries(t, vec)
assert.Len(t, series, 3, "staging must not add series under its own digest")
for k := range series {
assert.Equal(t, production.configDigest.String(), k[2])
}
})
}
254 changes: 254 additions & 0 deletions core/services/llo/observation/observed_value_gauge.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,254 @@
package observation

import (
"math"
"math/big"
"math/bits"
"strconv"
"sync"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/shopspring/decimal"

ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types"

llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo"
lloprotocol "github.com/smartcontractkit/chainlink-data-streams/llo/protocol"
)

// promStreamObservedValue uses snake_case labels (unlike the other metrics in this package) so config_digest joins
// directly with libocr's config_digest label, which carries the same ConfigDigest.String() value.
var promStreamObservedValue = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "llo",
Subsystem: "datasource",
Name: "stream_observed_value",
Help: "Latest value this node observed for an LLO stream, read from the observation cache when the " +
"production OCR instance calls Observe. value_type is benchmark for decimal streams (the stream's value, " +
"whatever it measures) and bid, benchmark or ask for quote streams. Values are float64, so digits beyond " +
"about 15 significant figures are lost. A series is deleted when its stream has no fresh cached value, " +
"leaves the plugin's stream set, or its config digest stops observing.",
}, []string{"stream_id", "value_type", "config_digest"})

const (
observedValueTypeBenchmark = "benchmark"
observedValueTypeBid = "bid"
observedValueTypeAsk = "ask"
// observedValueDigestTTL is how long a config digest's series survive after its last production Observe.
// A TTL (instead of deleting on every digest change) avoids churning every series while Blue and Green
// instances are both production during a handover.
observedValueDigestTTL = 10 * time.Second
)

// observedValueGauges owns the llo_datasource_stream_observed_value series that one dataSource sets,
// so it can delete them when a stream goes stale, a config digest stops observing, or the data source closes.
type observedValueGauges struct {
vec *prometheus.GaugeVec
mu sync.Mutex
digests map[ocr2types.ConfigDigest]*digestObservedValues
}

type digestObservedValues struct {
digestLabel string // ConfigDigest.String(), same format as libocr's config_digest label
lastRecorded time.Time // time of the last record() for this digest
round uint64 // incremented on each record() for this digest
streams map[llotypes.StreamID]*streamObservedValue
}

type streamObservedValue struct {
streamIDLabel string
last lloprotocol.StreamValue // value already written to the gauges; pointer compare skips Float64
round uint64 // digestObservedValues.round when last seen with a value
benchmark prometheus.Gauge
bid, ask prometheus.Gauge // nil for decimal streams
}

func newObservedValueGauges(vec *prometheus.GaugeVec) *observedValueGauges {
return &observedValueGauges{
vec: vec,
digests: make(map[ocr2types.ConfigDigest]*digestObservedValues),
}
}

// record sets one series per value component for every stream in streamValues that has a decimal or quote value,
// and deletes the series of streams that no longer have one. It also deletes the series of other config digests
// that have not recorded for observedValueDigestTTL.
func (g *observedValueGauges) record(digest ocr2types.ConfigDigest, streamValues lloprotocol.StreamValues, now time.Time) {
g.mu.Lock()
defer g.mu.Unlock()

dg, ok := g.digests[digest]
if !ok {
dg = &digestObservedValues{
digestLabel: digest.String(),
streams: make(map[llotypes.StreamID]*streamObservedValue, len(streamValues)),
}
g.digests[digest] = dg
}
dg.lastRecorded = now
dg.round++

seen := 0
for id, orig := range streamValues {
switch v := unwrapStreamValue(orig).(type) {
case *lloprotocol.Decimal:
if v == nil {
continue
}
entry := g.getOrCreateStream(dg, id, false)
if entry.last != orig {
entry.benchmark.Set(decimalToFloat64(v.Decimal()))
entry.last = orig
}
entry.round = dg.round
seen++
case *lloprotocol.Quote:
if v == nil {
continue
}
entry := g.getOrCreateStream(dg, id, true)
if entry.last != orig {
entry.bid.Set(decimalToFloat64(v.Bid))
entry.benchmark.Set(decimalToFloat64(v.Benchmark))
entry.ask.Set(decimalToFloat64(v.Ask))
entry.last = orig
}
entry.round = dg.round
seen++
}
}

if seen < len(dg.streams) {
for id, entry := range dg.streams {
if entry.round != dg.round {
g.deleteStream(dg.digestLabel, entry)
delete(dg.streams, id)
}
}
}

// dg.lastRecorded is now, so this only expires other digests (a retired Blue/Green instance).
g.expireLocked(now)
}

// expire deletes the series of every config digest that has not recorded for observedValueDigestTTL. It also
// covers the only production digest when Observe stops (for example a DON stall), which record cannot detect.
func (g *observedValueGauges) expire(now time.Time) {
g.mu.Lock()
defer g.mu.Unlock()
g.expireLocked(now)
}

func (g *observedValueGauges) expireLocked(now time.Time) {
for d, dg := range g.digests {
if now.Sub(dg.lastRecorded) > observedValueDigestTTL {
g.deleteDigest(dg)
delete(g.digests, d)
}
}
}

// deleteAll deletes every series this recorder set.
func (g *observedValueGauges) deleteAll() {
g.mu.Lock()
defer g.mu.Unlock()
for _, dg := range g.digests {
g.deleteDigest(dg)
}
g.digests = make(map[ocr2types.ConfigDigest]*digestObservedValues)
}

// getOrCreateStream returns the entry for id, replacing it when its kind (decimal or quote) differs from isQuote.
func (g *observedValueGauges) getOrCreateStream(dg *digestObservedValues, id llotypes.StreamID, isQuote bool) *streamObservedValue {
entry, ok := dg.streams[id]
if ok && (entry.bid != nil) == isQuote {
return entry
}
if ok {
g.deleteStream(dg.digestLabel, entry)
}
entry = &streamObservedValue{streamIDLabel: strconv.FormatUint(uint64(id), 10)}
entry.benchmark = g.vec.WithLabelValues(entry.streamIDLabel, observedValueTypeBenchmark, dg.digestLabel)
if isQuote {
entry.bid = g.vec.WithLabelValues(entry.streamIDLabel, observedValueTypeBid, dg.digestLabel)
entry.ask = g.vec.WithLabelValues(entry.streamIDLabel, observedValueTypeAsk, dg.digestLabel)
}
dg.streams[id] = entry
return entry
}

func (g *observedValueGauges) deleteDigest(dg *digestObservedValues) {
for _, entry := range dg.streams {
g.deleteStream(dg.digestLabel, entry)
}
}

func (g *observedValueGauges) deleteStream(digestLabel string, entry *streamObservedValue) {
g.vec.DeleteLabelValues(entry.streamIDLabel, observedValueTypeBenchmark, digestLabel)
if entry.bid != nil {
g.vec.DeleteLabelValues(entry.streamIDLabel, observedValueTypeBid, digestLabel)
g.vec.DeleteLabelValues(entry.streamIDLabel, observedValueTypeAsk, digestLabel)
}
}

// unwrapStreamValue returns the value inside any TimestampedStreamValue wrappers, or nil for a nil wrapper.
func unwrapStreamValue(v lloprotocol.StreamValue) lloprotocol.StreamValue {
for {
ts, ok := v.(*lloprotocol.TimestampedStreamValue)
if !ok {
return v
}
if ts == nil {
return nil
}
v = ts.StreamValue
}
}

// exactPow10 holds the powers of ten that float64 represents exactly.
var exactPow10 = [...]float64{
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11,
1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22,
}

// decimalToFloat64 converts d to float64 within 2 ulps of the nearest value. decimal.Float64 rounds exactly, but it
// builds a big.Rat and divides on every call (about 20x slower, with about 11 allocations); this runs on the OCR
// observation path for every changed stream, and the gauge does not need the last bit. It falls back to Float64 for
// exponents outside [-22, 22] and on platforms without 64-bit words.
func decimalToFloat64(d decimal.Decimal) float64 {
exp := d.Exponent()
if exp < -22 || exp > 22 || bits.UintSize != 64 {
f, _ := d.Float64()
return f
}
f := bigIntToFloat64(d.Coefficient()) // Coefficient copies: the only allocation
if exp < 0 {
return f / exactPow10[-exp]
}
return f * exactPow10[exp]
}

// bigIntToFloat64 converts c using its 64 most significant bits; the bits below them can only change the last bit.
// It requires 64-bit words.
func bigIntToFloat64(c *big.Int) float64 {
n := c.BitLen()
var f float64
if n <= 64 {
f = float64(c.Uint64()) // Uint64 is the low 64 bits of |c|, which is all of it here
} else {
w := c.Bits() // little-endian 64-bit words of |c|
hi := (n - 1) / 64
var top uint64
if s := uint(n % 64); s == 0 {
top = uint64(w[hi])
} else {
top = uint64(w[hi])<<(64-s) | uint64(w[hi-1])>>s
}
f = math.Ldexp(float64(top), n-64)
}
if c.Sign() < 0 {
f = -f
}
return f
}
Loading
Loading