diff --git a/.changeset/infoplat-19823-llo-observed-value.md b/.changeset/infoplat-19823-llo-observed-value.md new file mode 100644 index 00000000000..dc578a4fe22 --- /dev/null +++ b/.changeset/infoplat-19823-llo-observed-value.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +#added llo_datasource_stream_observed_value gauge: latest observed value per LLO stream (value_type benchmark|bid|ask, config_digest). To forward it to Beholder, set `[Telemetry.PrometheusBridge] Enabled = true` and add `llo_datasource_stream_observed_value` to `Prefixes`; setting `Prefixes` replaces the default `["go_"]`, so keep `"go_"` and any prefixes already forwarded. diff --git a/core/services/llo/observation/data_source.go b/core/services/llo/observation/data_source.go index 368b9621a6c..8d0918c237d 100644 --- a/core/services/llo/observation/data_source.go +++ b/core/services/llo/observation/data_source.go @@ -170,6 +170,10 @@ type dataSource struct { // loopWakeCh coalesces plugin Observe hints (buffer 1): the loop may skip pacing when the plugin is active. loopWakeCh chan struct{} + + // observedValues exports the values that the production instance reads from the cache (see observe). + observedValues *observedValueGauges + observedValuesExpiryOnce sync.Once } var _ llodatasource.DataSource = &dataSource{} @@ -189,6 +193,7 @@ func newDataSource(lggr logger.Logger, registry Registry, t Telemeter) *dataSour cache: NewCache(time.Minute), observationLoopCloseCh: make(chan struct{}), loopWakeCh: make(chan struct{}, 1), + observedValues: newObservedValueGauges(promStreamObservedValue), } } @@ -249,9 +254,36 @@ func (d *dataSource) observe(ctx context.Context, streamValues lloprotocol.Strea // Update stream values with the cached observations for all streams. d.cache.UpdateStreamValues(streamValues) + // Only the production instance records: a staging or retired instance reads the same cache and would + // duplicate every series under a second config digest. inProduction is false for nil opts. + if inProduction { + d.observedValues.record(opts.ConfigDigest(), streamValues, time.Now()) + d.startObservedValuesExpiry() + } + return nil } +// startObservedValuesExpiry starts (once) a goroutine that deletes observed value series of config digests that +// stopped calling Observe. record() alone cannot do this when no digest calls Observe at all (DON stall), and the +// observation loop exits when the stream set is empty, so expiry needs its own ticker. It stops on Close. +func (d *dataSource) startObservedValuesExpiry() { + d.observedValuesExpiryOnce.Do(func() { + d.wg.Go(func() { + ticker := time.NewTicker(observedValueDigestTTL / 2) + defer ticker.Stop() + for { + select { + case <-d.observationLoopCloseCh: + return + case now := <-ticker.C: + d.observedValues.expire(now) + } + } + }) + }) +} + // startObservationLoop continuously runs pipeline observations for this data source and caches results in memory, // so the plugin Observe path stays fast regardless of adapter latency. // Loop pacing (observationLoopPacing), per-round deadline (observationTimeout), cache TTL, and stale-refresh @@ -526,6 +558,7 @@ func (d *dataSource) Close() error { close(d.observationLoopCloseCh) d.wg.Wait() d.cache.Close() + d.observedValues.deleteAll() return nil } diff --git a/core/services/llo/observation/data_source_test.go b/core/services/llo/observation/data_source_test.go index f38d9f00466..d9082e460fc 100644 --- a/core/services/llo/observation/data_source_test.go +++ b/core/services/llo/observation/data_source_test.go @@ -1025,3 +1025,72 @@ func Test_DataSource_StagingDoesNotObserve(t *testing.T) { require.Nil(t, vals[1], "staging instance must not populate stream values") require.Zero(t, reg.pipelines[1].runCount.Load(), "staging instance must not run pipelines") } + +func Test_DataSource_ObservedValueGauge(t *testing.T) { + t.Parallel() + newReg := func() *mockRegistry { + reg := &mockRegistry{pipelines: make(map[streams.StreamID]*mockPipeline)} + multi := makePipelineWithMultipleStreamResults([]streams.StreamID{1, 2, 3}, []any{ + decimal.NewFromInt(2181), + decimal.NewFromInt(40602), + decimal.NewFromInt(15), + }) + reg.pipelines[1] = multi + reg.pipelines[2] = multi + reg.pipelines[3] = multi + return reg + } + + t.Run("production records and Close deletes", func(t *testing.T) { + t.Parallel() + ds := newDataSource(logger.NullLogger, newReg(), telem.NullTelemeter) + vec := newTestObservedValueVec() + ds.observedValues = newObservedValueGauges(vec) + opts := &mockOpts{configDigest: ocr2types.ConfigDigest{6, 5, 4}} + digest := opts.configDigest.String() + want := map[[3]string]float64{ + {"1", "benchmark", digest}: 2181, + {"2", "benchmark", digest}: 40602, + {"3", "benchmark", digest}: 15, + } + + require.EventuallyWithT(t, func(c *assert.CollectT) { + ctx, cancel := context.WithTimeout(t.Context(), observationTimeout) + defer cancel() + assert.NoError(c, ds.Observe(ctx, makeStreamValues(), opts)) + assert.Equal(c, want, observedSeries(c, vec)) + }, 5*time.Second, 50*time.Millisecond) + + require.NoError(t, ds.Close()) + assert.Empty(t, observedSeries(t, vec)) + }) + + t.Run("staging does not record", func(t *testing.T) { + t.Parallel() + ds := newDataSource(logger.NullLogger, newReg(), telem.NullTelemeter) + vec := newTestObservedValueVec() + ds.observedValues = newObservedValueGauges(vec) + defer ds.Close() + production := &mockOpts{configDigest: ocr2types.ConfigDigest{6, 5, 4}} + staging := &mockOpts{configDigest: ocr2types.ConfigDigest{9}, lifeCycleStage: lloprotocol.LifeCycleStageStaging} + + // Populate the shared cache through the production instance first, so a staging Observe reads real values. + require.EventuallyWithT(t, func(c *assert.CollectT) { + ctx, cancel := context.WithTimeout(t.Context(), observationTimeout) + defer cancel() + assert.NoError(c, ds.Observe(ctx, makeStreamValues(), production)) + assert.Len(c, observedSeries(c, vec), 3) + }, 5*time.Second, 50*time.Millisecond) + + ctx, cancel := context.WithTimeout(t.Context(), observationTimeout) + defer cancel() + vals := makeStreamValues() + require.NoError(t, ds.Observe(ctx, vals, staging)) + require.NotNil(t, vals[1], "staging Observe must read the cached value for this test to be meaningful") + series := observedSeries(t, vec) + assert.Len(t, series, 3, "staging must not add series under its own digest") + for k := range series { + assert.Equal(t, production.configDigest.String(), k[2]) + } + }) +} diff --git a/core/services/llo/observation/observed_value_gauge.go b/core/services/llo/observation/observed_value_gauge.go new file mode 100644 index 00000000000..07b035b7d39 --- /dev/null +++ b/core/services/llo/observation/observed_value_gauge.go @@ -0,0 +1,254 @@ +package observation + +import ( + "math" + "math/big" + "math/bits" + "strconv" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/shopspring/decimal" + + ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" + lloprotocol "github.com/smartcontractkit/chainlink-data-streams/llo/protocol" +) + +// promStreamObservedValue uses snake_case labels (unlike the other metrics in this package) so config_digest joins +// directly with libocr's config_digest label, which carries the same ConfigDigest.String() value. +var promStreamObservedValue = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "llo", + Subsystem: "datasource", + Name: "stream_observed_value", + Help: "Latest value this node observed for an LLO stream, read from the observation cache when the " + + "production OCR instance calls Observe. value_type is benchmark for decimal streams (the stream's value, " + + "whatever it measures) and bid, benchmark or ask for quote streams. Values are float64, so digits beyond " + + "about 15 significant figures are lost. A series is deleted when its stream has no fresh cached value, " + + "leaves the plugin's stream set, or its config digest stops observing.", +}, []string{"stream_id", "value_type", "config_digest"}) + +const ( + observedValueTypeBenchmark = "benchmark" + observedValueTypeBid = "bid" + observedValueTypeAsk = "ask" + // observedValueDigestTTL is how long a config digest's series survive after its last production Observe. + // A TTL (instead of deleting on every digest change) avoids churning every series while Blue and Green + // instances are both production during a handover. + observedValueDigestTTL = 10 * time.Second +) + +// observedValueGauges owns the llo_datasource_stream_observed_value series that one dataSource sets, +// so it can delete them when a stream goes stale, a config digest stops observing, or the data source closes. +type observedValueGauges struct { + vec *prometheus.GaugeVec + mu sync.Mutex + digests map[ocr2types.ConfigDigest]*digestObservedValues +} + +type digestObservedValues struct { + digestLabel string // ConfigDigest.String(), same format as libocr's config_digest label + lastRecorded time.Time // time of the last record() for this digest + round uint64 // incremented on each record() for this digest + streams map[llotypes.StreamID]*streamObservedValue +} + +type streamObservedValue struct { + streamIDLabel string + last lloprotocol.StreamValue // value already written to the gauges; pointer compare skips Float64 + round uint64 // digestObservedValues.round when last seen with a value + benchmark prometheus.Gauge + bid, ask prometheus.Gauge // nil for decimal streams +} + +func newObservedValueGauges(vec *prometheus.GaugeVec) *observedValueGauges { + return &observedValueGauges{ + vec: vec, + digests: make(map[ocr2types.ConfigDigest]*digestObservedValues), + } +} + +// record sets one series per value component for every stream in streamValues that has a decimal or quote value, +// and deletes the series of streams that no longer have one. It also deletes the series of other config digests +// that have not recorded for observedValueDigestTTL. +func (g *observedValueGauges) record(digest ocr2types.ConfigDigest, streamValues lloprotocol.StreamValues, now time.Time) { + g.mu.Lock() + defer g.mu.Unlock() + + dg, ok := g.digests[digest] + if !ok { + dg = &digestObservedValues{ + digestLabel: digest.String(), + streams: make(map[llotypes.StreamID]*streamObservedValue, len(streamValues)), + } + g.digests[digest] = dg + } + dg.lastRecorded = now + dg.round++ + + seen := 0 + for id, orig := range streamValues { + switch v := unwrapStreamValue(orig).(type) { + case *lloprotocol.Decimal: + if v == nil { + continue + } + entry := g.getOrCreateStream(dg, id, false) + if entry.last != orig { + entry.benchmark.Set(decimalToFloat64(v.Decimal())) + entry.last = orig + } + entry.round = dg.round + seen++ + case *lloprotocol.Quote: + if v == nil { + continue + } + entry := g.getOrCreateStream(dg, id, true) + if entry.last != orig { + entry.bid.Set(decimalToFloat64(v.Bid)) + entry.benchmark.Set(decimalToFloat64(v.Benchmark)) + entry.ask.Set(decimalToFloat64(v.Ask)) + entry.last = orig + } + entry.round = dg.round + seen++ + } + } + + if seen < len(dg.streams) { + for id, entry := range dg.streams { + if entry.round != dg.round { + g.deleteStream(dg.digestLabel, entry) + delete(dg.streams, id) + } + } + } + + // dg.lastRecorded is now, so this only expires other digests (a retired Blue/Green instance). + g.expireLocked(now) +} + +// expire deletes the series of every config digest that has not recorded for observedValueDigestTTL. It also +// covers the only production digest when Observe stops (for example a DON stall), which record cannot detect. +func (g *observedValueGauges) expire(now time.Time) { + g.mu.Lock() + defer g.mu.Unlock() + g.expireLocked(now) +} + +func (g *observedValueGauges) expireLocked(now time.Time) { + for d, dg := range g.digests { + if now.Sub(dg.lastRecorded) > observedValueDigestTTL { + g.deleteDigest(dg) + delete(g.digests, d) + } + } +} + +// deleteAll deletes every series this recorder set. +func (g *observedValueGauges) deleteAll() { + g.mu.Lock() + defer g.mu.Unlock() + for _, dg := range g.digests { + g.deleteDigest(dg) + } + g.digests = make(map[ocr2types.ConfigDigest]*digestObservedValues) +} + +// getOrCreateStream returns the entry for id, replacing it when its kind (decimal or quote) differs from isQuote. +func (g *observedValueGauges) getOrCreateStream(dg *digestObservedValues, id llotypes.StreamID, isQuote bool) *streamObservedValue { + entry, ok := dg.streams[id] + if ok && (entry.bid != nil) == isQuote { + return entry + } + if ok { + g.deleteStream(dg.digestLabel, entry) + } + entry = &streamObservedValue{streamIDLabel: strconv.FormatUint(uint64(id), 10)} + entry.benchmark = g.vec.WithLabelValues(entry.streamIDLabel, observedValueTypeBenchmark, dg.digestLabel) + if isQuote { + entry.bid = g.vec.WithLabelValues(entry.streamIDLabel, observedValueTypeBid, dg.digestLabel) + entry.ask = g.vec.WithLabelValues(entry.streamIDLabel, observedValueTypeAsk, dg.digestLabel) + } + dg.streams[id] = entry + return entry +} + +func (g *observedValueGauges) deleteDigest(dg *digestObservedValues) { + for _, entry := range dg.streams { + g.deleteStream(dg.digestLabel, entry) + } +} + +func (g *observedValueGauges) deleteStream(digestLabel string, entry *streamObservedValue) { + g.vec.DeleteLabelValues(entry.streamIDLabel, observedValueTypeBenchmark, digestLabel) + if entry.bid != nil { + g.vec.DeleteLabelValues(entry.streamIDLabel, observedValueTypeBid, digestLabel) + g.vec.DeleteLabelValues(entry.streamIDLabel, observedValueTypeAsk, digestLabel) + } +} + +// unwrapStreamValue returns the value inside any TimestampedStreamValue wrappers, or nil for a nil wrapper. +func unwrapStreamValue(v lloprotocol.StreamValue) lloprotocol.StreamValue { + for { + ts, ok := v.(*lloprotocol.TimestampedStreamValue) + if !ok { + return v + } + if ts == nil { + return nil + } + v = ts.StreamValue + } +} + +// exactPow10 holds the powers of ten that float64 represents exactly. +var exactPow10 = [...]float64{ + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, +} + +// decimalToFloat64 converts d to float64 within 2 ulps of the nearest value. decimal.Float64 rounds exactly, but it +// builds a big.Rat and divides on every call (about 20x slower, with about 11 allocations); this runs on the OCR +// observation path for every changed stream, and the gauge does not need the last bit. It falls back to Float64 for +// exponents outside [-22, 22] and on platforms without 64-bit words. +func decimalToFloat64(d decimal.Decimal) float64 { + exp := d.Exponent() + if exp < -22 || exp > 22 || bits.UintSize != 64 { + f, _ := d.Float64() + return f + } + f := bigIntToFloat64(d.Coefficient()) // Coefficient copies: the only allocation + if exp < 0 { + return f / exactPow10[-exp] + } + return f * exactPow10[exp] +} + +// bigIntToFloat64 converts c using its 64 most significant bits; the bits below them can only change the last bit. +// It requires 64-bit words. +func bigIntToFloat64(c *big.Int) float64 { + n := c.BitLen() + var f float64 + if n <= 64 { + f = float64(c.Uint64()) // Uint64 is the low 64 bits of |c|, which is all of it here + } else { + w := c.Bits() // little-endian 64-bit words of |c| + hi := (n - 1) / 64 + var top uint64 + if s := uint(n % 64); s == 0 { + top = uint64(w[hi]) + } else { + top = uint64(w[hi])<<(64-s) | uint64(w[hi-1])>>s + } + f = math.Ldexp(float64(top), n-64) + } + if c.Sign() < 0 { + f = -f + } + return f +} diff --git a/core/services/llo/observation/observed_value_gauge_test.go b/core/services/llo/observation/observed_value_gauge_test.go new file mode 100644 index 00000000000..559b4b7ff56 --- /dev/null +++ b/core/services/llo/observation/observed_value_gauge_test.go @@ -0,0 +1,327 @@ +package observation + +import ( + "math/big" + "math/rand/v2" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + io_prometheus_client "github.com/prometheus/client_model/go" + "github.com/shopspring/decimal" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" + lloprotocol "github.com/smartcontractkit/chainlink-data-streams/llo/protocol" +) + +func newTestObservedValueVec() *prometheus.GaugeVec { + return prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_observed_value"}, []string{"stream_id", "value_type", "config_digest"}) +} + +// observedSeries returns every series in vec keyed by {stream_id, value_type, config_digest}. t can be an +// *assert.CollectT: its FailNow ends only the EventuallyWithT condition. Do not call it off the test goroutine with a +// *testing.T. +func observedSeries(t require.TestingT, vec *prometheus.GaugeVec) map[[3]string]float64 { + if h, ok := t.(interface{ Helper() }); ok { + h.Helper() + } + ch := make(chan prometheus.Metric) + go func() { + vec.Collect(ch) + close(ch) + }() + out := make(map[[3]string]float64) + for m := range ch { + var pb io_prometheus_client.Metric + require.NoError(t, m.Write(&pb)) + labels := make(map[string]string, len(pb.GetLabel())) + for _, lp := range pb.GetLabel() { + labels[lp.GetName()] = lp.GetValue() + } + out[[3]string{labels["stream_id"], labels["value_type"], labels["config_digest"]}] = pb.GetGauge().GetValue() + } + return out +} + +func dec(f float64) *lloprotocol.Decimal { + return lloprotocol.ToDecimal(decimal.NewFromFloat(f)) +} + +func quote(bid, benchmark, ask float64) *lloprotocol.Quote { + return &lloprotocol.Quote{Bid: decimal.NewFromFloat(bid), Benchmark: decimal.NewFromFloat(benchmark), Ask: decimal.NewFromFloat(ask)} +} + +func Test_observedValueGauges(t *testing.T) { + t.Parallel() + dA := ocr2types.ConfigDigest{1} + dB := ocr2types.ConfigDigest{2} + t0 := time.Unix(1_700_000_000, 0) + + t.Run("decimal records benchmark", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(63421.07)}, t0) + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dA.String()}: 63421.07, + }, observedSeries(t, vec)) + }) + + t.Run("legacy quote records bid, benchmark and ask", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{2: quote(99, 100, 101)}, t0) + series := observedSeries(t, vec) + assert.Equal(t, map[[3]string]float64{ + {"2", "bid", dA.String()}: 99, + {"2", "benchmark", dA.String()}: 100, + {"2", "ask", dA.String()}: 101, + }, series) + for k := range series { + assert.Contains(t, []string{"benchmark", "bid", "ask"}, k[1]) + } + }) + + t.Run("timestamped values are unwrapped", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{ + 1: &lloprotocol.TimestampedStreamValue{ObservedAtNanoseconds: 1, StreamValue: dec(63421.07)}, + 2: &lloprotocol.TimestampedStreamValue{ObservedAtNanoseconds: 1, StreamValue: quote(99, 100, 101)}, + }, t0) + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dA.String()}: 63421.07, + {"2", "bid", dA.String()}: 99, + {"2", "benchmark", dA.String()}: 100, + {"2", "ask", dA.String()}: 101, + }, observedSeries(t, vec)) + }) + + t.Run("nil value deletes the stream's series", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(1), 2: dec(2)}, t0) + g.record(dA, lloprotocol.StreamValues{1: nil, 2: dec(2)}, t0.Add(time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"2", "benchmark", dA.String()}: 2, + }, observedSeries(t, vec)) + }) + + t.Run("stream removed from the set deletes its series", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(1), 2: dec(2)}, t0) + g.record(dA, lloprotocol.StreamValues{2: dec(2)}, t0.Add(time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"2", "benchmark", dA.String()}: 2, + }, observedSeries(t, vec)) + }) + + t.Run("new value updates the gauge", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(10)}, t0) + g.record(dA, lloprotocol.StreamValues{1: dec(11)}, t0.Add(time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dA.String()}: 11, + }, observedSeries(t, vec)) + }) + + t.Run("kind change replaces the series", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{3: dec(5)}, t0) + g.record(dA, lloprotocol.StreamValues{3: quote(4, 5, 6)}, t0.Add(time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"3", "bid", dA.String()}: 4, + {"3", "benchmark", dA.String()}: 5, + {"3", "ask", dA.String()}: 6, + }, observedSeries(t, vec)) + g.record(dA, lloprotocol.StreamValues{3: dec(7)}, t0.Add(2*time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"3", "benchmark", dA.String()}: 7, + }, observedSeries(t, vec)) + }) + + t.Run("config digest series expire after the TTL", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(1)}, t0) + g.record(dB, lloprotocol.StreamValues{1: dec(2)}, t0.Add(5*time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dA.String()}: 1, + {"1", "benchmark", dB.String()}: 2, + }, observedSeries(t, vec)) + g.record(dB, lloprotocol.StreamValues{1: dec(2)}, t0.Add(11*time.Second)) + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dB.String()}: 2, + }, observedSeries(t, vec)) + }) + + t.Run("deleteAll removes every series", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(1), 2: quote(1, 2, 3)}, t0) + g.record(dB, lloprotocol.StreamValues{1: dec(1)}, t0) + g.deleteAll() + assert.Empty(t, observedSeries(t, vec)) + }) + + t.Run("expire deletes the only digest after Observe stops", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + g.record(dA, lloprotocol.StreamValues{1: dec(1), 2: quote(1, 2, 3)}, t0) + g.expire(t0.Add(observedValueDigestTTL)) + assert.Len(t, observedSeries(t, vec), 4, "series must survive up to the TTL") + g.expire(t0.Add(observedValueDigestTTL + time.Nanosecond)) + assert.Empty(t, observedSeries(t, vec)) + }) + + t.Run("concurrent Blue/Green records keep one series per stream and digest", func(t *testing.T) { + t.Parallel() + vec := newTestObservedValueVec() + g := newObservedValueGauges(vec) + var wg sync.WaitGroup + for _, d := range []ocr2types.ConfigDigest{dA, dB} { + wg.Go(func() { + for i := range 100 { + g.record(d, lloprotocol.StreamValues{1: dec(float64(i)), 2: quote(1, 2, 3)}, t0) + } + }) + } + wg.Go(func() { + for range 100 { + testutil.CollectAndCount(vec) // concurrent reader; no assertions off the test goroutine + g.expire(t0) + } + }) + wg.Wait() + assert.Equal(t, map[[3]string]float64{ + {"1", "benchmark", dA.String()}: 99, + {"2", "bid", dA.String()}: 1, + {"2", "benchmark", dA.String()}: 2, + {"2", "ask", dA.String()}: 3, + {"1", "benchmark", dB.String()}: 99, + {"2", "bid", dB.String()}: 1, + {"2", "benchmark", dB.String()}: 2, + {"2", "ask", dB.String()}: 3, + }, observedSeries(t, vec)) + }) +} + +func Test_decimalToFloat64(t *testing.T) { + t.Parallel() + // maxRelErr is 2 ulps: one rounding to 53 bits plus one for the division or multiplication by a power of ten. + const maxRelErr = 2 * 0x1p-52 + check := func(t *testing.T, d decimal.Decimal) { + t.Helper() + want, _ := d.Float64() + got := decimalToFloat64(d) + if want == 0 { + assert.Zero(t, got, "%s", d) + return + } + assert.InDelta(t, 0, (got-want)/want, maxRelErr, "%s: got %v want %v", d, got, want) + } + + cases := map[string]decimal.Decimal{ + "zero": decimal.Zero, + "zero value": {}, + "one": decimal.NewFromInt(1), + "negative": decimal.RequireFromString("-63421.07"), + "price": decimal.RequireFromString("63421.07"), + "small fraction": decimal.RequireFromString("0.000000081907646999640001"), + "exponent -22": decimal.New(123456789, -22), + "exponent +22": decimal.New(123456789, 22), + "exponent -23 (fallback)": decimal.New(123456789, -23), + "exponent +23 (fallback)": decimal.New(123456789, 23), + "2^53+1 coefficient": decimal.New(1<<53+1, 0), + "max int64 coefficient": decimal.New(1<<63-1, -4), + "nanosecond timestamp": decimal.NewFromInt(1794948135870123456), + "65-bit coefficient": decimal.NewFromBigInt(new(big.Int).Lsh(big.NewInt(1), 64), -10), + "128-bit word boundary": decimal.NewFromBigInt(new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 128), big.NewInt(1)), -20), + "negative 100-bit": decimal.NewFromBigInt(new(big.Int).Neg(new(big.Int).Lsh(big.NewInt(12345), 86)), -16), + "median of two (21 digit)": decimal.RequireFromString("63421.07").Add(decimal.RequireFromString("63421.08")).Div(decimal.NewFromInt(2)), + } + for name, d := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + check(t, d) + }) + } + + t.Run("random parsed floats and medians", func(t *testing.T) { + t.Parallel() + r := rand.New(rand.NewPCG(1, 2)) + for range 10_000 { + a := decimal.NewFromFloat((r.Float64() - 0.5) * 1e6) + check(t, a) + check(t, a.Add(decimal.NewFromFloat(r.Float64()*1e6)).Div(decimal.NewFromInt(2))) + } + }) +} + +// BenchmarkObservedValueGaugesRecord measures record() on the DON 4 shape: 2,724 decimal and 154 quote streams +// (3,186 series). Values mix the two coefficient shapes pipelines produce: a parsed float (about 17 digits) and an +// even-count median, whose Div(2) rounds to 16 decimals (about 21 digits). +func BenchmarkObservedValueGaugesRecord(b *testing.B) { + const decimalStreams, quoteStreams = 2724, 154 + newValues := func(seed uint64) lloprotocol.StreamValues { + r := rand.New(rand.NewPCG(seed, seed)) + price := func(i int) decimal.Decimal { + d := decimal.NewFromFloat(r.Float64() * 100_000) + if i%2 == 1 { + d = d.Add(decimal.NewFromFloat(r.Float64() * 100_000)).Div(decimal.NewFromInt(2)) + } + return d + } + sv := make(lloprotocol.StreamValues, decimalStreams+quoteStreams) + for i := range decimalStreams { + sv[llotypes.StreamID(i)] = lloprotocol.ToDecimal(price(i)) + } + for i := range quoteStreams { + p := price(i) + sv[llotypes.StreamID(decimalStreams+i)] = &lloprotocol.Quote{Bid: p.Sub(decimal.NewFromInt(1)), Benchmark: p, Ask: p.Add(decimal.NewFromInt(1))} + } + return sv + } + digest := ocr2types.ConfigDigest{1} + now := time.Unix(1_700_000_000, 0) + + b.Run("unchanged", func(b *testing.B) { + g := newObservedValueGauges(newTestObservedValueVec()) + sv := newValues(1) + g.record(digest, sv, now) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + g.record(digest, sv, now) + } + }) + + b.Run("all_changed", func(b *testing.B) { + g := newObservedValueGauges(newTestObservedValueVec()) + svs := [2]lloprotocol.StreamValues{newValues(1), newValues(2)} + g.record(digest, svs[1], now) + b.ReportAllocs() + b.ResetTimer() + for i := range b.N { + g.record(digest, svs[i%2], now) + } + }) +}