From f0c57a9ff3f5933fdf0772563eeab9832679d5cd Mon Sep 17 00:00:00 2001 From: Chet Sharma Date: Tue, 17 Nov 2026 11:10:05 -0800 Subject: [PATCH 1/3] feat(llo): add per-stream observed value gauge (INFOPLAT-19823) --- .../infoplat-19823-llo-observed-value.md | 5 + core/services/llo/observation/data_source.go | 33 +++ .../llo/observation/data_source_test.go | 67 +++++ .../llo/observation/observed_value_gauge.go | 206 ++++++++++++++ .../observation/observed_value_gauge_test.go | 259 ++++++++++++++++++ 5 files changed, 570 insertions(+) create mode 100644 .changeset/infoplat-19823-llo-observed-value.md create mode 100644 core/services/llo/observation/observed_value_gauge.go create mode 100644 core/services/llo/observation/observed_value_gauge_test.go diff --git a/.changeset/infoplat-19823-llo-observed-value.md b/.changeset/infoplat-19823-llo-observed-value.md new file mode 100644 index 00000000000..c40117b7a1d --- /dev/null +++ b/.changeset/infoplat-19823-llo-observed-value.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +#added llo_datasource_stream_observed_value gauge: latest observed value per LLO stream (value_type benchmark|bid|ask, config_digest), forwardable via [Telemetry.PrometheusBridge] diff --git a/core/services/llo/observation/data_source.go b/core/services/llo/observation/data_source.go index 368b9621a6c..8d0918c237d 100644 --- a/core/services/llo/observation/data_source.go +++ b/core/services/llo/observation/data_source.go @@ -170,6 +170,10 @@ type dataSource struct { // loopWakeCh coalesces plugin Observe hints (buffer 1): the loop may skip pacing when the plugin is active. loopWakeCh chan struct{} + + // observedValues exports the values that the production instance reads from the cache (see observe). + observedValues *observedValueGauges + observedValuesExpiryOnce sync.Once } var _ llodatasource.DataSource = &dataSource{} @@ -189,6 +193,7 @@ func newDataSource(lggr logger.Logger, registry Registry, t Telemeter) *dataSour cache: NewCache(time.Minute), observationLoopCloseCh: make(chan struct{}), loopWakeCh: make(chan struct{}, 1), + observedValues: newObservedValueGauges(promStreamObservedValue), } } @@ -249,9 +254,36 @@ func (d *dataSource) observe(ctx context.Context, streamValues lloprotocol.Strea // Update stream values with the cached observations for all streams. d.cache.UpdateStreamValues(streamValues) + // Only the production instance records: a staging or retired instance reads the same cache and would + // duplicate every series under a second config digest. inProduction is false for nil opts. + if inProduction { + d.observedValues.record(opts.ConfigDigest(), streamValues, time.Now()) + d.startObservedValuesExpiry() + } + return nil } +// startObservedValuesExpiry starts (once) a goroutine that deletes observed value series of config digests that +// stopped calling Observe. record() alone cannot do this when no digest calls Observe at all (DON stall), and the +// observation loop exits when the stream set is empty, so expiry needs its own ticker. It stops on Close. +func (d *dataSource) startObservedValuesExpiry() { + d.observedValuesExpiryOnce.Do(func() { + d.wg.Go(func() { + ticker := time.NewTicker(observedValueDigestTTL / 2) + defer ticker.Stop() + for { + select { + case <-d.observationLoopCloseCh: + return + case now := <-ticker.C: + d.observedValues.expire(now) + } + } + }) + }) +} + // startObservationLoop continuously runs pipeline observations for this data source and caches results in memory, // so the plugin Observe path stays fast regardless of adapter latency. // Loop pacing (observationLoopPacing), per-round deadline (observationTimeout), cache TTL, and stale-refresh @@ -526,6 +558,7 @@ func (d *dataSource) Close() error { close(d.observationLoopCloseCh) d.wg.Wait() d.cache.Close() + d.observedValues.deleteAll() return nil } diff --git a/core/services/llo/observation/data_source_test.go b/core/services/llo/observation/data_source_test.go index f38d9f00466..c58e169df11 100644 --- a/core/services/llo/observation/data_source_test.go +++ b/core/services/llo/observation/data_source_test.go @@ -1025,3 +1025,70 @@ func Test_DataSource_StagingDoesNotObserve(t *testing.T) { require.Nil(t, vals[1], "staging instance must not populate stream values") require.Zero(t, reg.pipelines[1].runCount.Load(), "staging instance must not run pipelines") } + +func Test_DataSource_ObservedValueGauge(t *testing.T) { + t.Parallel() + newReg := func() *mockRegistry { + reg := &mockRegistry{pipelines: make(map[streams.StreamID]*mockPipeline)} + multi := makePipelineWithMultipleStreamResults([]streams.StreamID{1, 2, 3}, []any{ + decimal.NewFromInt(2181), + decimal.NewFromInt(40602), + decimal.NewFromInt(15), + }) + reg.pipelines[1] = multi + reg.pipelines[2] = multi + reg.pipelines[3] = multi + return reg + } + + t.Run("production records and Close deletes", func(t *testing.T) { + t.Parallel() + ds := newDataSource(logger.NullLogger, newReg(), telem.NullTelemeter) + vec := newTestObservedValueVec() + ds.observedValues = newObservedValueGauges(vec) + opts := &mockOpts{configDigest: ocr2types.ConfigDigest{6, 5, 4}} + digest := opts.configDigest.String() + want := map[[3]string]float64{ + {"1", "benchmark", digest}: 2181, + {"2", "benchmark", digest}: 40602, + {"3", "benchmark", digest}: 15, + } + + require.Eventually(t, func() bool { + ctx, cancel := context.WithTimeout(t.Context(), observationTimeout) + defer cancel() + require.NoError(t, ds.Observe(ctx, makeStreamValues(), opts)) + return assert.ObjectsAreEqual(want, observedSeries(t, vec)) + }, 5*time.Second, 50*time.Millisecond) + + require.NoError(t, ds.Close()) + assert.Empty(t, observedSeries(t, vec)) + }) + + t.Run("staging does not record", func(t *testing.T) { + t.Parallel() + ds := newDataSource(logger.NullLogger, newReg(), telem.NullTelemeter) + vec := newTestObservedValueVec() + ds.observedValues = newObservedValueGauges(vec) + defer ds.Close() + production := &mockOpts{configDigest: ocr2types.ConfigDigest{6, 5, 4}} + staging := &mockOpts{configDigest: ocr2types.ConfigDigest{9}, lifeCycleStage: lloprotocol.LifeCycleStageStaging} + + // Populate the shared cache through the production instance first, so a staging Observe reads real values. + require.Eventually(t, func() bool { + ctx, cancel := context.WithTimeout(t.Context(), observationTimeout) + defer cancel() + require.NoError(t, ds.Observe(ctx, makeStreamValues(), production)) + return len(observedSeries(t, vec)) == 3 + }, 5*time.Second, 50*time.Millisecond) + + ctx, cancel := context.WithTimeout(t.Context(), observationTimeout) + defer cancel() + vals := makeStreamValues() + require.NoError(t, ds.Observe(ctx, vals, staging)) + require.NotNil(t, vals[1], "staging Observe must read the cached value for this test to be meaningful") + for k := range observedSeries(t, vec) { + assert.Equal(t, production.configDigest.String(), k[2]) + } + }) +} diff --git a/core/services/llo/observation/observed_value_gauge.go b/core/services/llo/observation/observed_value_gauge.go new file mode 100644 index 00000000000..ed4a337d3bc --- /dev/null +++ b/core/services/llo/observation/observed_value_gauge.go @@ -0,0 +1,206 @@ +package observation + +import ( + "strconv" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" + lloprotocol "github.com/smartcontractkit/chainlink-data-streams/llo/protocol" +) + +// promStreamObservedValue uses snake_case labels (unlike the other metrics in this package) so config_digest joins +// directly with libocr's config_digest label, which carries the same ConfigDigest.String() value. +var promStreamObservedValue = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "llo", + Subsystem: "datasource", + Name: "stream_observed_value", + Help: "Latest value this node observed for an LLO stream, read from the observation cache when the " + + "production OCR instance calls Observe. value_type is benchmark for decimal streams and bid, " + + "benchmark or ask for quote streams. A series is deleted when its stream has no fresh cached value, " + + "leaves the plugin's stream set, or its config digest stops observing.", +}, []string{"stream_id", "value_type", "config_digest"}) + +const ( + observedValueTypeBenchmark = "benchmark" + observedValueTypeBid = "bid" + observedValueTypeAsk = "ask" + // observedValueDigestTTL is how long a config digest's series survive after its last production Observe. + // A TTL (instead of deleting on every digest change) avoids churning every series while Blue and Green + // instances are both production during a handover. + observedValueDigestTTL = 10 * time.Second +) + +// observedValueGauges owns the llo_datasource_stream_observed_value series that one dataSource sets, +// so it can delete them when a stream goes stale, a config digest stops observing, or the data source closes. +type observedValueGauges struct { + vec *prometheus.GaugeVec + mu sync.Mutex + digests map[ocr2types.ConfigDigest]*digestObservedValues +} + +type digestObservedValues struct { + digestLabel string // ConfigDigest.String(), same format as libocr's config_digest label + lastRecorded time.Time // time of the last record() for this digest + round uint64 // incremented on each record() for this digest + streams map[llotypes.StreamID]*streamObservedValue +} + +type streamObservedValue struct { + streamIDLabel string + last lloprotocol.StreamValue // value already written to the gauges; pointer compare skips Float64 + round uint64 // digestObservedValues.round when last seen with a value + benchmark prometheus.Gauge + bid, ask prometheus.Gauge // nil for decimal streams +} + +func newObservedValueGauges(vec *prometheus.GaugeVec) *observedValueGauges { + return &observedValueGauges{ + vec: vec, + digests: make(map[ocr2types.ConfigDigest]*digestObservedValues), + } +} + +// record sets one series per value component for every stream in streamValues that has a decimal or quote value, +// and deletes the series of streams that no longer have one. It also deletes the series of other config digests +// that have not recorded for observedValueDigestTTL. +func (g *observedValueGauges) record(digest ocr2types.ConfigDigest, streamValues lloprotocol.StreamValues, now time.Time) { + g.mu.Lock() + defer g.mu.Unlock() + + dg, ok := g.digests[digest] + if !ok { + dg = &digestObservedValues{ + digestLabel: digest.String(), + streams: make(map[llotypes.StreamID]*streamObservedValue, len(streamValues)), + } + g.digests[digest] = dg + } + dg.lastRecorded = now + dg.round++ + + seen := 0 + for id, orig := range streamValues { + switch v := unwrapStreamValue(orig).(type) { + case *lloprotocol.Decimal: + if v == nil { + continue + } + entry := g.getOrCreateStream(dg, id, false) + if entry.last != orig { + f, _ := v.Decimal().Float64() + entry.benchmark.Set(f) + entry.last = orig + } + entry.round = dg.round + seen++ + case *lloprotocol.Quote: + if v == nil { + continue + } + entry := g.getOrCreateStream(dg, id, true) + if entry.last != orig { + bid, _ := v.Bid.Float64() + benchmark, _ := v.Benchmark.Float64() + ask, _ := v.Ask.Float64() + entry.bid.Set(bid) + entry.benchmark.Set(benchmark) + entry.ask.Set(ask) + entry.last = orig + } + entry.round = dg.round + seen++ + } + } + + if seen < len(dg.streams) { + for id, entry := range dg.streams { + if entry.round != dg.round { + g.deleteStream(dg.digestLabel, entry) + delete(dg.streams, id) + } + } + } + + // dg.lastRecorded is now, so this only expires other digests (a retired Blue/Green instance). + g.expireLocked(now) +} + +// expire deletes the series of every config digest that has not recorded for observedValueDigestTTL. It also +// covers the only production digest when Observe stops (for example a DON stall), which record cannot detect. +func (g *observedValueGauges) expire(now time.Time) { + g.mu.Lock() + defer g.mu.Unlock() + g.expireLocked(now) +} + +func (g *observedValueGauges) expireLocked(now time.Time) { + for d, dg := range g.digests { + if now.Sub(dg.lastRecorded) > observedValueDigestTTL { + g.deleteDigest(dg) + delete(g.digests, d) + } + } +} + +// deleteAll deletes every series this recorder set. +func (g *observedValueGauges) deleteAll() { + g.mu.Lock() + defer g.mu.Unlock() + for _, dg := range g.digests { + g.deleteDigest(dg) + } + g.digests = make(map[ocr2types.ConfigDigest]*digestObservedValues) +} + +// getOrCreateStream returns the entry for id, replacing it when its kind (decimal or quote) differs from isQuote. +func (g *observedValueGauges) getOrCreateStream(dg *digestObservedValues, id llotypes.StreamID, isQuote bool) *streamObservedValue { + entry, ok := dg.streams[id] + if ok && (entry.bid != nil) == isQuote { + return entry + } + if ok { + g.deleteStream(dg.digestLabel, entry) + } + entry = &streamObservedValue{streamIDLabel: strconv.FormatUint(uint64(id), 10)} + entry.benchmark = g.vec.WithLabelValues(entry.streamIDLabel, observedValueTypeBenchmark, dg.digestLabel) + if isQuote { + entry.bid = g.vec.WithLabelValues(entry.streamIDLabel, observedValueTypeBid, dg.digestLabel) + entry.ask = g.vec.WithLabelValues(entry.streamIDLabel, observedValueTypeAsk, dg.digestLabel) + } + dg.streams[id] = entry + return entry +} + +func (g *observedValueGauges) deleteDigest(dg *digestObservedValues) { + for _, entry := range dg.streams { + g.deleteStream(dg.digestLabel, entry) + } +} + +func (g *observedValueGauges) deleteStream(digestLabel string, entry *streamObservedValue) { + g.vec.DeleteLabelValues(entry.streamIDLabel, observedValueTypeBenchmark, digestLabel) + if entry.bid != nil { + g.vec.DeleteLabelValues(entry.streamIDLabel, observedValueTypeBid, digestLabel) + g.vec.DeleteLabelValues(entry.streamIDLabel, observedValueTypeAsk, digestLabel) + } +} + +// unwrapStreamValue returns the value inside any TimestampedStreamValue wrappers, or nil for a nil wrapper. +func unwrapStreamValue(v lloprotocol.StreamValue) lloprotocol.StreamValue { + for { + ts, ok := v.(*lloprotocol.TimestampedStreamValue) + if !ok { + return v + } + if ts == nil { + return nil + } + v = ts.StreamValue + } +} diff --git a/core/services/llo/observation/observed_value_gauge_test.go b/core/services/llo/observation/observed_value_gauge_test.go new file mode 100644 index 00000000000..72a9d814da7 --- /dev/null +++ b/core/services/llo/observation/observed_value_gauge_test.go @@ -0,0 +1,259 @@ +package observation + +import ( + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + io_prometheus_client "github.com/prometheus/client_model/go" + "github.com/shopspring/decimal" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" + lloprotocol "github.com/smartcontractkit/chainlink-data-streams/llo/protocol" +) + +func newTestObservedValueVec() *prometheus.GaugeVec { + return prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_observed_value"}, []string{"stream_id", "value_type", "config_digest"}) +} + +// observedSeries returns every series in vec keyed by {stream_id, value_type, config_digest}. +func observedSeries(t testing.TB, vec *prometheus.GaugeVec) map[[3]string]float64 { + t.Helper() + ch := make(chan prometheus.Metric) + go func() { + vec.Collect(ch) + close(ch) + }() + out := make(map[[3]string]float64) + for m := range ch { + var pb io_prometheus_client.Metric + require.NoError(t, m.Write(&pb)) + labels := make(map[string]string, len(pb.GetLabel())) + for _, lp := range pb.GetLabel() { + labels[lp.GetName()] = lp.GetValue() + } + out[[3]string{labels["stream_id"], labels["value_type"], labels["config_digest"]}] = pb.GetGauge().GetValue() + } + return out +} + +func dec(f float64) *lloprotocol.Decimal { + return lloprotocol.ToDecimal(decimal.NewFromFloat(f)) +} + +func quote(bid, benchmark, ask float64) *lloprotocol.Quote { + return &lloprotocol.Quote{Bid: decimal.NewFromFloat(bid), Benchmark: decimal.NewFromFloat(benchmark), Ask: decimal.NewFromFloat(ask)} +} + +func Test_observedValueGauges(t *testing.T) { + t.Parallel() + dA := ocr2types.ConfigDigest{1} + dB := ocr2types.ConfigDigest{2} + t0 := time.Unix(1_700_000_000, 0) + + t.Run("decimal records benchmark", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(63421.07)}, t0) + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dA.String()}: 63421.07, + }, observedSeries(t, vec)) + }) + + t.Run("legacy quote records bid, benchmark and ask", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{2: quote(99, 100, 101)}, t0) + series := observedSeries(t, vec) + assert.Equal(t, map[[3]string]float64{ + {"2", "bid", dA.String()}: 99, + {"2", "benchmark", dA.String()}: 100, + {"2", "ask", dA.String()}: 101, + }, series) + for k := range series { + assert.Contains(t, []string{"benchmark", "bid", "ask"}, k[1]) + } + }) + + t.Run("timestamped values are unwrapped", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{ + 1: &lloprotocol.TimestampedStreamValue{ObservedAtNanoseconds: 1, StreamValue: dec(63421.07)}, + 2: &lloprotocol.TimestampedStreamValue{ObservedAtNanoseconds: 1, StreamValue: quote(99, 100, 101)}, + }, t0) + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dA.String()}: 63421.07, + {"2", "bid", dA.String()}: 99, + {"2", "benchmark", dA.String()}: 100, + {"2", "ask", dA.String()}: 101, + }, observedSeries(t, vec)) + }) + + t.Run("nil value deletes the stream's series", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(1), 2: dec(2)}, t0) + g.record(dA, lloprotocol.StreamValues{1: nil, 2: dec(2)}, t0.Add(time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"2", "benchmark", dA.String()}: 2, + }, observedSeries(t, vec)) + }) + + t.Run("stream removed from the set deletes its series", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(1), 2: dec(2)}, t0) + g.record(dA, lloprotocol.StreamValues{2: dec(2)}, t0.Add(time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"2", "benchmark", dA.String()}: 2, + }, observedSeries(t, vec)) + }) + + t.Run("new value updates the gauge", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(10)}, t0) + g.record(dA, lloprotocol.StreamValues{1: dec(11)}, t0.Add(time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dA.String()}: 11, + }, observedSeries(t, vec)) + }) + + t.Run("kind change replaces the series", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{3: dec(5)}, t0) + g.record(dA, lloprotocol.StreamValues{3: quote(4, 5, 6)}, t0.Add(time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"3", "bid", dA.String()}: 4, + {"3", "benchmark", dA.String()}: 5, + {"3", "ask", dA.String()}: 6, + }, observedSeries(t, vec)) + g.record(dA, lloprotocol.StreamValues{3: dec(7)}, t0.Add(2*time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"3", "benchmark", dA.String()}: 7, + }, observedSeries(t, vec)) + }) + + t.Run("config digest series expire after the TTL", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(1)}, t0) + g.record(dB, lloprotocol.StreamValues{1: dec(2)}, t0.Add(5*time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dA.String()}: 1, + {"1", "benchmark", dB.String()}: 2, + }, observedSeries(t, vec)) + g.record(dB, lloprotocol.StreamValues{1: dec(2)}, t0.Add(11*time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dB.String()}: 2, + }, observedSeries(t, vec)) + }) + + t.Run("deleteAll removes every series", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(1), 2: quote(1, 2, 3)}, t0) + g.record(dB, lloprotocol.StreamValues{1: dec(1)}, t0) + g.deleteAll() + assert.Empty(t, observedSeries(t, vec)) + }) + + t.Run("expire deletes the only digest after Observe stops", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(1), 2: quote(1, 2, 3)}, t0) + g.expire(t0.Add(observedValueDigestTTL)) + assert.Len(t, observedSeries(t, vec), 4, "series must survive up to the TTL") + g.expire(t0.Add(observedValueDigestTTL + time.Nanosecond)) + assert.Empty(t, observedSeries(t, vec)) + }) + + t.Run("concurrent Blue/Green records keep one series per stream and digest", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + var wg sync.WaitGroup + for _, d := range []ocr2types.ConfigDigest{dA, dB} { + wg.Go(func() { + for i := range 100 { + g.record(d, lloprotocol.StreamValues{1: dec(float64(i)), 2: quote(1, 2, 3)}, t0) + } + }) + } + wg.Go(func() { + for range 100 { + observedSeries(t, vec) + g.expire(t0) + } + }) + wg.Wait() + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dA.String()}: 99, + {"2", "bid", dA.String()}: 1, + {"2", "benchmark", dA.String()}: 2, + {"2", "ask", dA.String()}: 3, + {"1", "benchmark", dB.String()}: 99, + {"2", "bid", dB.String()}: 1, + {"2", "benchmark", dB.String()}: 2, + {"2", "ask", dB.String()}: 3, + }, observedSeries(t, vec)) + }) +} + +// BenchmarkObservedValueGaugesRecord measures record() on the DON 4 shape: 2,724 decimal and 154 quote streams +// (3,186 series). +func BenchmarkObservedValueGaugesRecord(b *testing.B) { + const decimalStreams, quoteStreams = 2724, 154 + newValues := func(seed float64) lloprotocol.StreamValues { + sv := make(lloprotocol.StreamValues, decimalStreams+quoteStreams) + for i := range decimalStreams { + sv[llotypes.StreamID(i)] = dec(seed + float64(i)) + } + for i := range quoteStreams { + f := seed + float64(i) + sv[llotypes.StreamID(decimalStreams+i)] = quote(f-1, f, f+1) + } + return sv + } + digest := ocr2types.ConfigDigest{1} + now := time.Unix(1_700_000_000, 0) + + b.Run("unchanged", func(b *testing.B) { + g := newObservedValueGauges(newTestObservedValueVec()) + sv := newValues(1) + g.record(digest, sv, now) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + g.record(digest, sv, now) + } + }) + + b.Run("all_changed", func(b *testing.B) { + g := newObservedValueGauges(newTestObservedValueVec()) + svs := [2]lloprotocol.StreamValues{newValues(1), newValues(2)} + g.record(digest, svs[1], now) + b.ReportAllocs() + b.ResetTimer() + for i := range b.N { + g.record(digest, svs[i%2], now) + } + }) +} From 82f5c731b3d0aec760f327c9283742d2ebb6d639 Mon Sep 17 00:00:00 2001 From: Chet Sharma Date: Wed, 18 Nov 2026 11:55:33 -0800 Subject: [PATCH 2/3] fix(llo): address review on observed value gauge (INFOPLAT-19823) - convert decimals without big.Rat (decimalToFloat64): all_changed record 900us -> 112us, 38k -> 3.2k allocs on realistic values - benchmark uses parsed-float and even-count-median coefficient shapes - Help text: benchmark is a decimal stream's value; float64 precision - tests: EventuallyWithT, no require off the test goroutine, staging test asserts 3 series - changeset: bridge Enabled and Prefixes replace semantics --- .../infoplat-19823-llo-observed-value.md | 2 +- .../llo/observation/data_source_test.go | 16 ++-- .../llo/observation/observed_value_gauge.go | 68 ++++++++++++--- .../observation/observed_value_gauge_test.go | 87 ++++++++++++++++--- 4 files changed, 145 insertions(+), 28 deletions(-) diff --git a/.changeset/infoplat-19823-llo-observed-value.md b/.changeset/infoplat-19823-llo-observed-value.md index c40117b7a1d..dc578a4fe22 100644 --- a/.changeset/infoplat-19823-llo-observed-value.md +++ b/.changeset/infoplat-19823-llo-observed-value.md @@ -2,4 +2,4 @@ "chainlink": minor --- -#added llo_datasource_stream_observed_value gauge: latest observed value per LLO stream (value_type benchmark|bid|ask, config_digest), forwardable via [Telemetry.PrometheusBridge] +#added llo_datasource_stream_observed_value gauge: latest observed value per LLO stream (value_type benchmark|bid|ask, config_digest). To forward it to Beholder, set `[Telemetry.PrometheusBridge] Enabled = true` and add `llo_datasource_stream_observed_value` to `Prefixes`; setting `Prefixes` replaces the default `["go_"]`, so keep `"go_"` and any prefixes already forwarded. diff --git a/core/services/llo/observation/data_source_test.go b/core/services/llo/observation/data_source_test.go index c58e169df11..d9082e460fc 100644 --- a/core/services/llo/observation/data_source_test.go +++ b/core/services/llo/observation/data_source_test.go @@ -1054,11 +1054,11 @@ func Test_DataSource_ObservedValueGauge(t *testing.T) { {"3", "benchmark", digest}: 15, } - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(c *assert.CollectT) { ctx, cancel := context.WithTimeout(t.Context(), observationTimeout) defer cancel() - require.NoError(t, ds.Observe(ctx, makeStreamValues(), opts)) - return assert.ObjectsAreEqual(want, observedSeries(t, vec)) + assert.NoError(c, ds.Observe(ctx, makeStreamValues(), opts)) + assert.Equal(c, want, observedSeries(c, vec)) }, 5*time.Second, 50*time.Millisecond) require.NoError(t, ds.Close()) @@ -1075,11 +1075,11 @@ func Test_DataSource_ObservedValueGauge(t *testing.T) { staging := &mockOpts{configDigest: ocr2types.ConfigDigest{9}, lifeCycleStage: lloprotocol.LifeCycleStageStaging} // Populate the shared cache through the production instance first, so a staging Observe reads real values. - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(c *assert.CollectT) { ctx, cancel := context.WithTimeout(t.Context(), observationTimeout) defer cancel() - require.NoError(t, ds.Observe(ctx, makeStreamValues(), production)) - return len(observedSeries(t, vec)) == 3 + assert.NoError(c, ds.Observe(ctx, makeStreamValues(), production)) + assert.Len(c, observedSeries(c, vec), 3) }, 5*time.Second, 50*time.Millisecond) ctx, cancel := context.WithTimeout(t.Context(), observationTimeout) @@ -1087,7 +1087,9 @@ func Test_DataSource_ObservedValueGauge(t *testing.T) { vals := makeStreamValues() require.NoError(t, ds.Observe(ctx, vals, staging)) require.NotNil(t, vals[1], "staging Observe must read the cached value for this test to be meaningful") - for k := range observedSeries(t, vec) { + series := observedSeries(t, vec) + assert.Len(t, series, 3, "staging must not add series under its own digest") + for k := range series { assert.Equal(t, production.configDigest.String(), k[2]) } }) diff --git a/core/services/llo/observation/observed_value_gauge.go b/core/services/llo/observation/observed_value_gauge.go index ed4a337d3bc..07b035b7d39 100644 --- a/core/services/llo/observation/observed_value_gauge.go +++ b/core/services/llo/observation/observed_value_gauge.go @@ -1,12 +1,16 @@ package observation import ( + "math" + "math/big" + "math/bits" "strconv" "sync" "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/shopspring/decimal" ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" @@ -21,8 +25,9 @@ var promStreamObservedValue = promauto.NewGaugeVec(prometheus.GaugeOpts{ Subsystem: "datasource", Name: "stream_observed_value", Help: "Latest value this node observed for an LLO stream, read from the observation cache when the " + - "production OCR instance calls Observe. value_type is benchmark for decimal streams and bid, " + - "benchmark or ask for quote streams. A series is deleted when its stream has no fresh cached value, " + + "production OCR instance calls Observe. value_type is benchmark for decimal streams (the stream's value, " + + "whatever it measures) and bid, benchmark or ask for quote streams. Values are float64, so digits beyond " + + "about 15 significant figures are lost. A series is deleted when its stream has no fresh cached value, " + "leaves the plugin's stream set, or its config digest stops observing.", }, []string{"stream_id", "value_type", "config_digest"}) @@ -93,8 +98,7 @@ func (g *observedValueGauges) record(digest ocr2types.ConfigDigest, streamValues } entry := g.getOrCreateStream(dg, id, false) if entry.last != orig { - f, _ := v.Decimal().Float64() - entry.benchmark.Set(f) + entry.benchmark.Set(decimalToFloat64(v.Decimal())) entry.last = orig } entry.round = dg.round @@ -105,12 +109,9 @@ func (g *observedValueGauges) record(digest ocr2types.ConfigDigest, streamValues } entry := g.getOrCreateStream(dg, id, true) if entry.last != orig { - bid, _ := v.Bid.Float64() - benchmark, _ := v.Benchmark.Float64() - ask, _ := v.Ask.Float64() - entry.bid.Set(bid) - entry.benchmark.Set(benchmark) - entry.ask.Set(ask) + entry.bid.Set(decimalToFloat64(v.Bid)) + entry.benchmark.Set(decimalToFloat64(v.Benchmark)) + entry.ask.Set(decimalToFloat64(v.Ask)) entry.last = orig } entry.round = dg.round @@ -204,3 +205,50 @@ func unwrapStreamValue(v lloprotocol.StreamValue) lloprotocol.StreamValue { v = ts.StreamValue } } + +// exactPow10 holds the powers of ten that float64 represents exactly. +var exactPow10 = [...]float64{ + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, +} + +// decimalToFloat64 converts d to float64 within 2 ulps of the nearest value. decimal.Float64 rounds exactly, but it +// builds a big.Rat and divides on every call (about 20x slower, with about 11 allocations); this runs on the OCR +// observation path for every changed stream, and the gauge does not need the last bit. It falls back to Float64 for +// exponents outside [-22, 22] and on platforms without 64-bit words. +func decimalToFloat64(d decimal.Decimal) float64 { + exp := d.Exponent() + if exp < -22 || exp > 22 || bits.UintSize != 64 { + f, _ := d.Float64() + return f + } + f := bigIntToFloat64(d.Coefficient()) // Coefficient copies: the only allocation + if exp < 0 { + return f / exactPow10[-exp] + } + return f * exactPow10[exp] +} + +// bigIntToFloat64 converts c using its 64 most significant bits; the bits below them can only change the last bit. +// It requires 64-bit words. +func bigIntToFloat64(c *big.Int) float64 { + n := c.BitLen() + var f float64 + if n <= 64 { + f = float64(c.Uint64()) // Uint64 is the low 64 bits of |c|, which is all of it here + } else { + w := c.Bits() // little-endian 64-bit words of |c| + hi := (n - 1) / 64 + var top uint64 + if s := uint(n % 64); s == 0 { + top = uint64(w[hi]) + } else { + top = uint64(w[hi])<<(64-s) | uint64(w[hi-1])>>s + } + f = math.Ldexp(float64(top), n-64) + } + if c.Sign() < 0 { + f = -f + } + return f +} diff --git a/core/services/llo/observation/observed_value_gauge_test.go b/core/services/llo/observation/observed_value_gauge_test.go index 72a9d814da7..cb166a8da8c 100644 --- a/core/services/llo/observation/observed_value_gauge_test.go +++ b/core/services/llo/observation/observed_value_gauge_test.go @@ -1,6 +1,8 @@ package observation import ( + "math/big" + "math/rand/v2" "sync" "testing" "time" @@ -9,7 +11,6 @@ import ( io_prometheus_client "github.com/prometheus/client_model/go" "github.com/shopspring/decimal" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" @@ -21,9 +22,12 @@ func newTestObservedValueVec() *prometheus.GaugeVec { return prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_observed_value"}, []string{"stream_id", "value_type", "config_digest"}) } -// observedSeries returns every series in vec keyed by {stream_id, value_type, config_digest}. -func observedSeries(t testing.TB, vec *prometheus.GaugeVec) map[[3]string]float64 { - t.Helper() +// observedSeries returns every series in vec keyed by {stream_id, value_type, config_digest}. It reports errors with +// assert, not require, so it is safe in goroutines and in EventuallyWithT conditions. +func observedSeries(t assert.TestingT, vec *prometheus.GaugeVec) map[[3]string]float64 { + if h, ok := t.(interface{ Helper() }); ok { + h.Helper() + } ch := make(chan prometheus.Metric) go func() { vec.Collect(ch) @@ -32,7 +36,9 @@ func observedSeries(t testing.TB, vec *prometheus.GaugeVec) map[[3]string]float6 out := make(map[[3]string]float64) for m := range ch { var pb io_prometheus_client.Metric - require.NoError(t, m.Write(&pb)) + if !assert.NoError(t, m.Write(&pb)) { + continue + } labels := make(map[string]string, len(pb.GetLabel())) for _, lp := range pb.GetLabel() { labels[lp.GetName()] = lp.GetValue() @@ -217,18 +223,79 @@ func Test_observedValueGauges(t *testing.T) { }) } +func Test_decimalToFloat64(t *testing.T) { + t.Parallel() + // maxRelErr is 2 ulps: one rounding to 53 bits plus one for the division or multiplication by a power of ten. + const maxRelErr = 2 * 0x1p-52 + check := func(t *testing.T, d decimal.Decimal) { + t.Helper() + want, _ := d.Float64() + got := decimalToFloat64(d) + if want == 0 { + assert.Zero(t, got, "%s", d) + return + } + assert.InDelta(t, 0, (got-want)/want, maxRelErr, "%s: got %v want %v", d, got, want) + } + + cases := map[string]decimal.Decimal{ + "zero": decimal.Zero, + "zero value": {}, + "one": decimal.NewFromInt(1), + "negative": decimal.RequireFromString("-63421.07"), + "price": decimal.RequireFromString("63421.07"), + "small fraction": decimal.RequireFromString("0.000000081907646999640001"), + "exponent -22": decimal.New(123456789, -22), + "exponent +22": decimal.New(123456789, 22), + "exponent -23 (fallback)": decimal.New(123456789, -23), + "exponent +23 (fallback)": decimal.New(123456789, 23), + "2^53+1 coefficient": decimal.New(1<<53+1, 0), + "max int64 coefficient": decimal.New(1<<63-1, -4), + "nanosecond timestamp": decimal.NewFromInt(1794948135870123456), + "65-bit coefficient": decimal.NewFromBigInt(new(big.Int).Lsh(big.NewInt(1), 64), -10), + "128-bit word boundary": decimal.NewFromBigInt(new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 128), big.NewInt(1)), -20), + "negative 100-bit": decimal.NewFromBigInt(new(big.Int).Neg(new(big.Int).Lsh(big.NewInt(12345), 86)), -16), + "median of two (21 digit)": decimal.RequireFromString("63421.07").Add(decimal.RequireFromString("63421.08")).Div(decimal.NewFromInt(2)), + } + for name, d := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + check(t, d) + }) + } + + t.Run("random parsed floats and medians", func(t *testing.T) { + t.Parallel() + r := rand.New(rand.NewPCG(1, 2)) + for range 10_000 { + a := decimal.NewFromFloat((r.Float64() - 0.5) * 1e6) + check(t, a) + check(t, a.Add(decimal.NewFromFloat(r.Float64()*1e6)).Div(decimal.NewFromInt(2))) + } + }) +} + // BenchmarkObservedValueGaugesRecord measures record() on the DON 4 shape: 2,724 decimal and 154 quote streams -// (3,186 series). +// (3,186 series). Values mix the two coefficient shapes pipelines produce: a parsed float (about 17 digits) and an +// even-count median, whose Div(2) rounds to 16 decimals (about 21 digits). func BenchmarkObservedValueGaugesRecord(b *testing.B) { const decimalStreams, quoteStreams = 2724, 154 - newValues := func(seed float64) lloprotocol.StreamValues { + newValues := func(seed uint64) lloprotocol.StreamValues { + r := rand.New(rand.NewPCG(seed, seed)) + price := func(i int) decimal.Decimal { + d := decimal.NewFromFloat(r.Float64() * 100_000) + if i%2 == 1 { + d = d.Add(decimal.NewFromFloat(r.Float64() * 100_000)).Div(decimal.NewFromInt(2)) + } + return d + } sv := make(lloprotocol.StreamValues, decimalStreams+quoteStreams) for i := range decimalStreams { - sv[llotypes.StreamID(i)] = dec(seed + float64(i)) + sv[llotypes.StreamID(i)] = lloprotocol.ToDecimal(price(i)) } for i := range quoteStreams { - f := seed + float64(i) - sv[llotypes.StreamID(decimalStreams+i)] = quote(f-1, f, f+1) + p := price(i) + sv[llotypes.StreamID(decimalStreams+i)] = &lloprotocol.Quote{Bid: p.Sub(decimal.NewFromInt(1)), Benchmark: p, Ask: p.Add(decimal.NewFromInt(1))} } return sv } From 2636ffcec0bceaf99fab5108ca8ab7f692e82b0b Mon Sep 17 00:00:00 2001 From: Chet Sharma Date: Wed, 18 Nov 2026 12:12:50 -0800 Subject: [PATCH 3/3] test(llo): satisfy testifylint in observed value gauge tests (INFOPLAT-19823) observedSeries uses require again (CollectT supports FailNow in EventuallyWithT conditions); the concurrent reader goroutine uses testutil.CollectAndCount, which makes no assertions. --- .../llo/observation/observed_value_gauge_test.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/services/llo/observation/observed_value_gauge_test.go b/core/services/llo/observation/observed_value_gauge_test.go index cb166a8da8c..559b4b7ff56 100644 --- a/core/services/llo/observation/observed_value_gauge_test.go +++ b/core/services/llo/observation/observed_value_gauge_test.go @@ -8,9 +8,11 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" io_prometheus_client "github.com/prometheus/client_model/go" "github.com/shopspring/decimal" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" @@ -22,9 +24,10 @@ func newTestObservedValueVec() *prometheus.GaugeVec { return prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_observed_value"}, []string{"stream_id", "value_type", "config_digest"}) } -// observedSeries returns every series in vec keyed by {stream_id, value_type, config_digest}. It reports errors with -// assert, not require, so it is safe in goroutines and in EventuallyWithT conditions. -func observedSeries(t assert.TestingT, vec *prometheus.GaugeVec) map[[3]string]float64 { +// observedSeries returns every series in vec keyed by {stream_id, value_type, config_digest}. t can be an +// *assert.CollectT: its FailNow ends only the EventuallyWithT condition. Do not call it off the test goroutine with a +// *testing.T. +func observedSeries(t require.TestingT, vec *prometheus.GaugeVec) map[[3]string]float64 { if h, ok := t.(interface{ Helper() }); ok { h.Helper() } @@ -36,9 +39,7 @@ func observedSeries(t assert.TestingT, vec *prometheus.GaugeVec) map[[3]string]f out := make(map[[3]string]float64) for m := range ch { var pb io_prometheus_client.Metric - if !assert.NoError(t, m.Write(&pb)) { - continue - } + require.NoError(t, m.Write(&pb)) labels := make(map[string]string, len(pb.GetLabel())) for _, lp := range pb.GetLabel() { labels[lp.GetName()] = lp.GetValue() @@ -205,7 +206,7 @@ func Test_observedValueGauges(t *testing.T) { } wg.Go(func() { for range 100 { - observedSeries(t, vec) + testutil.CollectAndCount(vec) // concurrent reader; no assertions off the test goroutine g.expire(t0) } })