diff --git a/chanevents/analyzer.go b/chanevents/analyzer.go index fac591a..f30626d 100644 --- a/chanevents/analyzer.go +++ b/chanevents/analyzer.go @@ -52,6 +52,11 @@ type EventsSource interface { // ScidToPeerMap returns the historically recorded scid→peer index, // including closed channels. ScidToPeerMap(ctx context.Context) (map[uint64]string, error) + + // UnobservedIntervals returns the sorted, disjoint unobserved + // intervals that overlap [start, end), clipped to that range. + UnobservedIntervals(ctx context.Context, start, + end time.Time) ([]UnobservedInterval, error) } // ForwardingAnalyzer computes forwarding velocity and effective uptime for @@ -72,7 +77,7 @@ type channelEventSeq = iter.Seq2[*ChannelEvent, error] // uptime) from EffectiveUptime and ForwardedMsat. type ForwardingAbility struct { // EffectiveUptime is the time the pair held at least the liquidity floor - // of directional forwardable liquidity over the window. + // of directional forwardable liquidity over the observed window. EffectiveUptime time.Duration // ForwardedMsat is the total successfully forwarded amount over the @@ -130,13 +135,16 @@ func NewForwardingAnalyzer(store EventsSource, } // EffectiveUptime returns a ForwardingAbility for every (peerIn, peerOut) pair -// over [startTime, endTime). Closed channels are folded into the considered set -// so survivorship bias does not skew the uptime denominator. A single -// liquidityFloor is applied uniformly to every pair, so effective uptime is the -// time each pair held at least that much directional forwardable liquidity. +// over [startTime, endTime), and the unobserved time within that window. +// Effective uptime only covers observed time, so a client divides it by the +// window minus the unobserved time. Closed channels are folded into the +// considered set so survivorship bias does not skew the uptime denominator. A +// single liquidityFloor is applied uniformly to every pair, so effective uptime +// is the time each pair held at least that much directional forwardable +// liquidity. func (a *ForwardingAnalyzer) EffectiveUptime(ctx context.Context, startTime, endTime time.Time, liquidityFloor btcutil.Amount) ( - map[PeerPair]ForwardingAbility, error) { + map[PeerPair]ForwardingAbility, time.Duration, error) { log.DebugS( ctx, "Calculating effective uptime", @@ -147,7 +155,7 @@ func (a *ForwardingAnalyzer) EffectiveUptime(ctx context.Context, startTime, scidToPeer, err := a.store.ScidToPeerMap(ctx) if err != nil { - return nil, err + return nil, 0, err } log.DebugS( ctx, "Found historical channels", @@ -158,7 +166,7 @@ func (a *ForwardingAnalyzer) EffectiveUptime(ctx context.Context, startTime, ctx, startTime, endTime, scidToPeer, ) if err != nil { - return nil, err + return nil, 0, err } log.DebugS( ctx, "Found peer pairs with successful forwards", @@ -167,24 +175,44 @@ func (a *ForwardingAnalyzer) EffectiveUptime(ctx context.Context, startTime, err = a.addActiveChannels(ctx, channelPeersConsidered) if err != nil { - return nil, err + return nil, 0, err } peerChannels, initialStates, err := a.getPeerChannelData( ctx, startTime, channelPeersConsidered, ) if err != nil { - return nil, err + return nil, 0, err } log.DebugS( ctx, "Identified channels for peers", slog.Int("count", len(peerChannels)), ) - return calculateAllPairsUptime( - ctx, a.store, startTime, endTime, liquidityFloor, + unobserved, err := a.store.UnobservedIntervals(ctx, startTime, endTime) + if err != nil { + return nil, 0, err + } + + var unobservedTotal time.Duration + for _, interval := range unobserved { + unobservedTotal += interval.End.Sub(interval.Start) + } + log.DebugS( + ctx, "Found unobserved intervals", + slog.Int("count", len(unobserved)), + slog.Duration("total", unobservedTotal), + ) + + abilities, err := calculateAllPairsUptime( + ctx, a.store, startTime, endTime, unobserved, liquidityFloor, successfulForwards, initialStates, peerChannels, ) + if err != nil { + return nil, 0, err + } + + return abilities, unobservedTotal, nil } // getForwardingData queries lnd's forwarding history sequentially in paginated @@ -456,7 +484,8 @@ func (a *ForwardingAnalyzer) getInitialChannelState(ctx context.Context, // calculateAllPairsUptime returns forwarding abilities for every peer pair, // computing both directions (A→B and B→A) in a single pass. func calculateAllPairsUptime(ctx context.Context, store EventsSource, startTime, - endTime time.Time, liquidityFloor btcutil.Amount, + endTime time.Time, unobserved []UnobservedInterval, + liquidityFloor btcutil.Amount, successfulForwards map[PeerPair]*pairForwards, initialStates map[string]map[int64]*channelState, peerChannels map[string][]int64) ( @@ -554,7 +583,7 @@ func calculateAllPairsUptime(ctx context.Context, store EventsSource, startTime, abilityAB, abilityBA, err := calculateBothDirectionsUptime( ctx, startTime, endTime, - liquidityFloor, + unobserved, liquidityFloor, statesA, statesB, sumsA.remote, sumsA.local, sumsB.remote, sumsB.local, @@ -643,12 +672,14 @@ func pairForwardTotals(successfulForwards map[PeerPair]*pairForwards, // calculateBothDirectionsUptime computes the effective forwarding uptime for // both directions of a peer pair in a single chronological walk of the merged // event stream. Only the liquidity-direction roles differ between the two -// accumulators, as both share the same uniform liquidityFloor. totalsAB and +// accumulators, as both share the same uniform liquidityFloor. Time inside the +// sorted, disjoint unobserved intervals does not count as uptime. totalsAB and // totalsBA carry each direction's forwarded volume, fees and count through to // the returned abilities. For self-pair calls both returned abilities are // equal. func calculateBothDirectionsUptime(ctx context.Context, startTime, - endTime time.Time, liquidityFloor btcutil.Amount, statesA, + endTime time.Time, unobserved []UnobservedInterval, + liquidityFloor btcutil.Amount, statesA, statesB map[int64]*channelState, sumARemote, sumALocal, sumBRemote, sumBLocal btcutil.Amount, mergedEvents channelEventSeq, totalsAB, totalsBA pairForwards) ( @@ -703,8 +734,12 @@ func calculateBothDirectionsUptime(ctx context.Context, startTime, var uptimeAB, uptimeBA time.Duration lastTimestamp := startTime + var cursor int - accumulate := func(intervalDuration time.Duration) { + accumulate := func(from, to time.Time) { + intervalDuration := observedDuration( + from, to, unobserved, &cursor, + ) if intervalDuration <= 0 { return } @@ -759,7 +794,7 @@ func calculateBothDirectionsUptime(ctx context.Context, startTime, // event, based on the state of the channels during that // interval. The events are ordered chronologically so the state // is consistent with the entire interval. - accumulate(event.Timestamp.Sub(lastTimestamp)) + accumulate(lastTimestamp, event.Timestamp) // Update the state for each peer if the event affects one of // their channels. Before applying the event, we remove the @@ -767,7 +802,7 @@ func calculateBothDirectionsUptime(ctx context.Context, startTime, // because the event may change the channel's online status or // balances in a way that affects the sums. if state, ok := statesA[event.ChannelID]; ok { - // We would have inlcuded the channel's balances in the + // We would have included the channel's balances in the // sums if it was online, so we need to remove them // before applying the event. if state.online { @@ -806,7 +841,7 @@ func calculateBothDirectionsUptime(ctx context.Context, startTime, // Account for the final interval between the last event and the end // time. - accumulate(endTime.Sub(lastTimestamp)) + accumulate(lastTimestamp, endTime) if traceOn { log.TraceS( @@ -825,6 +860,41 @@ func calculateBothDirectionsUptime(ctx context.Context, startTime, return abilityAB, abilityBA, nil } +// observedDuration returns the part of [from, to) that lies outside every +// unobserved interval. The intervals must be sorted and disjoint, and the +// caller must pass a non-decreasing from on each call, because cursor skips the +// intervals that end at or before from. +func observedDuration(from, to time.Time, unobserved []UnobservedInterval, + cursor *int) time.Duration { + + observed := to.Sub(from) + if observed <= 0 { + return 0 + } + + for *cursor < len(unobserved) && !unobserved[*cursor].End.After(from) { + *cursor++ + } + + for _, interval := range unobserved[*cursor:] { + if !interval.Start.Before(to) { + break + } + + overlapStart := interval.Start + if overlapStart.Before(from) { + overlapStart = from + } + overlapEnd := interval.End + if overlapEnd.After(to) { + overlapEnd = to + } + observed -= overlapEnd.Sub(overlapStart) + } + + return observed +} + // mergeEventSlices interleaves two sorted event streams into a single // chronological iter.Seq2. Equal-timestamp events from sliceA are yielded // first. Self-pair calls (sliceA == sliceB) yield each event twice. Callers diff --git a/chanevents/analyzer_test.go b/chanevents/analyzer_test.go index e579c9f..8a74199 100644 --- a/chanevents/analyzer_test.go +++ b/chanevents/analyzer_test.go @@ -229,6 +229,10 @@ func TestCalculateBothDirectionsUptime(t *testing.T) { successAmts []btcutil.Amount liquidityFloor btcutil.Amount + // unobserved lists unobserved intervals as [start, end) + // pairs of Unix seconds. + unobserved [][2]int64 + expectedUptime time.Duration }{ { @@ -589,6 +593,69 @@ func TestCalculateBothDirectionsUptime(t *testing.T) { }, expectedUptime: 0, }, + { + name: "Unobserved interval in the middle", + inStates: map[int64]*channelState{ + chanInID: { + online: true, + remoteBalance: 1000, + }, + }, + outStates: map[int64]*channelState{ + chanOutID: { + online: true, + localBalance: 800, + }, + }, + unobserved: [][2]int64{{130, 160}}, + expectedUptime: 70 * time.Second, + }, + { + // The interval covers the event at t=150 that takes + // the in channel offline. Only t=100..120 is observed + // and qualifying; t=180..200 is observed but offline. + name: "Unobserved interval covers an event", + inStates: map[int64]*channelState{ + chanInID: { + online: true, + remoteBalance: 1000, + }, + }, + outStates: map[int64]*channelState{ + chanOutID: { + online: true, + localBalance: 800, + }, + }, + inEvents: []*ChannelEvent{ + newStatusEvent(chanInID, 150, EventTypeOffline), + }, + unobserved: [][2]int64{{120, 180}}, + expectedUptime: 20 * time.Second, + }, + { + name: "Several unobserved intervals", + inStates: map[int64]*channelState{ + chanInID: { + online: true, + remoteBalance: 1000, + }, + }, + outStates: map[int64]*channelState{ + chanOutID: { + online: true, + localBalance: 800, + }, + }, + inEvents: []*ChannelEvent{ + newEvent( + chanInID, 140, EventTypeUpdate, 0, + 900, + ), + }, + unobserved: [][2]int64{{110, 120}, {130, 150}}, + expectedUptime: 70 * time.Second, + }, } for _, tc := range testCases { @@ -618,6 +685,20 @@ func TestCalculateBothDirectionsUptime(t *testing.T) { tc.inEvents, tc.outEvents, ) + var unobserved []UnobservedInterval + for _, iv := range tc.unobserved { + unobserved = append( + unobserved, UnobservedInterval{ + Start: time.Unix( + iv[0], 0, + ), + End: time.Unix( + iv[1], 0, + ), + }, + ) + } + // Precompute the starting balances. var sumARemote, sumALocal, sumBRemote, sumBLocal btcutil.Amount @@ -642,6 +723,7 @@ func TestCalculateBothDirectionsUptime(t *testing.T) { calculateBothDirectionsUptime( context.Background(), startTime, endTime, + unobserved, tc.liquidityFloor, tc.inStates, tc.outStates, sumARemote, sumALocal, @@ -713,7 +795,7 @@ func TestCalculateBothDirectionsUptimeAsymmetric(t *testing.T) { } abilityAB, abilityBA, err := calculateBothDirectionsUptime( - context.Background(), startTime, endTime, + context.Background(), startTime, endTime, nil, liquidityFloor, statesA, statesB, sumARemote, sumALocal, sumBRemote, sumBLocal, mergeEventSlices(nil, nil), @@ -937,7 +1019,7 @@ func TestEffectiveUptimeIncludesClosedChannels(t *testing.T) { startTime := seedTime.Add(time.Second) endTime := startTime.Add(time.Minute) - abilities, err := a.EffectiveUptime(ctx, startTime, endTime, 0) + abilities, _, err := a.EffectiveUptime(ctx, startTime, endTime, 0) require.NoError(t, err) // Cross-pair entries in both directions are the cleanest assertion @@ -955,6 +1037,88 @@ func TestEffectiveUptimeIncludesClosedChannels(t *testing.T) { ) } +// TestEffectiveUptimeUnobserved pins that a restart with a remote lnd does not +// count as uptime. Both channels stay online with the same balances across the +// restart, so only the unobserved interval tells the analyzer that nobody +// watched the node in that time. +func TestEffectiveUptimeUnobserved(t *testing.T) { + t.Parallel() + + store := NewTestDB(t, clock.NewTestClock(testTime)) + ctx := context.Background() + + peerA, err := store.AddPeer(ctx, validPubKey1) + require.NoError(t, err) + peerB, err := store.AddPeer(ctx, validPubKey2) + require.NoError(t, err) + + chanA, err := store.AddChannel( + ctx, testChanPoint1, testShortChanID1, peerA, + ) + require.NoError(t, err) + chanB, err := store.AddChannel( + ctx, testChanPoint2, testShortChanID2, peerB, + ) + require.NoError(t, err) + + startTime := testTime.Add(time.Second) + endTime := startTime.Add(3 * time.Hour) + lastLive := startTime.Add(10 * time.Minute) + syncTime := lastLive.Add(time.Hour) + + addEvents := func(ts time.Time, isSync bool) { + for _, chanID := range []int64{chanA, chanB} { + err := store.AddChannelEvent(ctx, &ChannelEvent{ + ChannelID: chanID, + EventType: EventTypeOnline, + Timestamp: ts, + IsSync: isSync, + }) + require.NoError(t, err) + + err = store.AddChannelEvent(ctx, &ChannelEvent{ + ChannelID: chanID, + EventType: EventTypeUpdate, + Timestamp: ts, + LocalBalance: fn.Some(btcutil.Amount(1000)), + RemoteBalance: fn.Some(btcutil.Amount(1000)), + IsSync: isSync, + }) + require.NoError(t, err) + } + } + + // Live events before and inside the window, then one hour without + // events, then the sync batch of the restart with unchanged state. + addEvents(testTime, false) + addEvents(lastLive, false) + addEvents(syncTime, true) + + vertexA, err := route.NewVertexFromStr(validPubKey1) + require.NoError(t, err) + vertexB, err := route.NewVertexFromStr(validPubKey2) + require.NoError(t, err) + + stub := &stubLndChannelClient{ + openChannels: []lndclient.ChannelInfo{ + {ChannelID: testShortChanID1, PubKeyBytes: vertexA}, + {ChannelID: testShortChanID2, PubKeyBytes: vertexB}, + }, + } + a := NewForwardingAnalyzer(store, lndclient.LndServices{Client: stub}) + + pairAB := PeerPair{PeerIn: validPubKey1, PeerOut: validPubKey2} + pairBA := PeerPair{PeerIn: validPubKey2, PeerOut: validPubKey1} + + abilities, unobserved, err := a.EffectiveUptime( + ctx, startTime, endTime, 0, + ) + require.NoError(t, err) + require.Equal(t, time.Hour, unobserved) + require.Equal(t, 2*time.Hour, abilities[pairAB].EffectiveUptime) + require.Equal(t, 2*time.Hour, abilities[pairBA].EffectiveUptime) +} + // TestEffectiveUptimeArgs exercises the liquidityFloor, startTime, and endTime // arguments of EffectiveUptime, verifying that the single uniform floor governs // effective uptime and that forwarded volume is reported regardless of uptime. @@ -1043,7 +1207,7 @@ func TestEffectiveUptimeArgs(t *testing.T) { // Case 1: liquidityFloor = 50k. Liquidity of 1M exceeds the floor for // the whole 60s window, so effective uptime is the full minute and the // forwarded volume is the 400M msat total, earning 400k msat. - abilities, err := a.EffectiveUptime(ctx, startTime, endTime, 50_000) + abilities, _, err := a.EffectiveUptime(ctx, startTime, endTime, 50_000) require.NoError(t, err) pair := PeerPair{PeerIn: validPubKey1, PeerOut: validPubKey2} @@ -1061,7 +1225,9 @@ func TestEffectiveUptimeArgs(t *testing.T) { // Case 2: liquidityFloor = 1.5M, above the 1M liquidity, so the floor is // never crossed and effective uptime is zero. The forwarded volume and // fees are still reported so the consumer keeps the signal. - abilities, err = a.EffectiveUptime(ctx, startTime, endTime, 1_500_000) + abilities, _, err = a.EffectiveUptime( + ctx, startTime, endTime, 1_500_000, + ) require.NoError(t, err) require.Contains(t, abilities, pair) diff --git a/chanevents/monitor.go b/chanevents/monitor.go index 87f9cb2..07f1aef 100644 --- a/chanevents/monitor.go +++ b/chanevents/monitor.go @@ -19,6 +19,11 @@ const ( // transient error or while waiting for lnd to become ready. retryInterval = 5 * time.Second + // maxReconnectInterval caps the reconnect backoff. Each reconnect + // writes a full sync batch, so a subscription that fails at once must + // not reconnect at retryInterval forever. + maxReconnectInterval = 5 * time.Minute + // pruneInterval is how often the monitor enforces the channel event // storage limits while consuming live events. pruneInterval = time.Hour @@ -108,7 +113,8 @@ func (m *Monitor) Stop() error { // monitorLoop is the main loop of the channel events monitor. It waits for lnd // to be fully synced, performs an initial state sync, and then subscribes to // channel events. If the subscription fails or the stream breaks, it retries -// from the beginning. +// from the beginning, so every reconnect writes a new sync batch that ends the +// unobserved interval of the break. func (m *Monitor) monitorLoop(ctx context.Context) { defer m.wg.Done() @@ -136,46 +142,83 @@ func (m *Monitor) monitorLoop(ctx context.Context) { pruneChan = pruneTicker.C } - var synced bool - + var reconnectDelay time.Duration for { // Wait for lnd to be synced to chain, retrying on RPC errors. if !m.waitForReady(ctx) { return } - // Initial state sync, only performed once. - if !synced { - if err := m.initialSync(ctx); err != nil { - log.Errorf("Error during initial sync: %v", err) - } else { - synced = true - - // The initial sync can insert a sizeable number - // of events, so prune once it completes. - m.pruneEvents(ctx) + // Re-baseline the channel state, as events may have been missed + // since the last subscription broke. Do not subscribe without a + // sync batch: a live event would then end the unobserved + // interval, and the outage would count as observed time. + if err := m.initialSync(ctx); err != nil { + reconnectDelay = nextReconnectDelay( + reconnectDelay, false, + ) + log.Errorf("Error during initial sync, retrying in "+ + "%v: %v", reconnectDelay, err) + + if !m.wait(ctx, reconnectDelay) { + return } + + continue } + // The initial sync can insert a sizeable number of events, so + // prune once it completes. + m.pruneEvents(ctx) + // Subscribe and consume events until the stream breaks or an // error occurs. - if !m.subscribe(ctx, pruneChan) { + retry, handledEvent := m.subscribe(ctx, pruneChan) + if !retry { return } // Stream broke, wait before reconnecting. - log.Infof("Reconnecting channel event subscription...") + reconnectDelay = nextReconnectDelay( + reconnectDelay, handledEvent, + ) + log.Infof("Reconnecting channel event subscription in %v...", + reconnectDelay) - select { - case <-time.After(retryInterval): - case <-m.quit: - return - case <-ctx.Done(): + if !m.wait(ctx, reconnectDelay) { return } } } +// nextReconnectDelay returns the wait before the next reconnect, given the +// last wait. It returns retryInterval after a subscription that handled an +// event. It also returns retryInterval when the last wait is below +// retryInterval. Otherwise it doubles the last wait, up to +// maxReconnectInterval. +func nextReconnectDelay(last time.Duration, + handledEvent bool) time.Duration { + + if handledEvent || last < retryInterval { + return retryInterval + } + + return min(2*last, maxReconnectInterval) +} + +// wait blocks for the given duration. It returns false if the monitor shuts +// down first. +func (m *Monitor) wait(ctx context.Context, d time.Duration) bool { + select { + case <-time.After(d): + return true + case <-m.quit: + return false + case <-ctx.Done(): + return false + } +} + // pruneEvents enforces the configured channel event storage limits, logging // how many events were deleted. func (m *Monitor) pruneEvents(ctx context.Context) { @@ -231,17 +274,18 @@ func (m *Monitor) waitForReady(ctx context.Context) bool { } // subscribe subscribes to lnd channel events and processes them until the -// stream breaks or an error occurs. It returns true on transient failures -// (caller should retry) or false if the monitor is shutting down. +// stream breaks or an error occurs. retry is true on transient failures +// (caller should retry) and false if the monitor is shutting down. +// handledEvent reports whether the subscription delivered at least one event. func (m *Monitor) subscribe(ctx context.Context, - pruneChan <-chan time.Time) bool { + pruneChan <-chan time.Time) (retry, handledEvent bool) { eventChan, errChan, err := m.lnd.SubscribeChannelEvents(ctx) if err != nil { log.Errorf("Error subscribing to channel events: %v", err) // Return true to signal the caller to retry. - return true + return true, false } for { @@ -249,8 +293,9 @@ func (m *Monitor) subscribe(ctx context.Context, case event, ok := <-eventChan: if !ok { log.Warn("Channel event stream closed") - return true + return true, handledEvent } + handledEvent = true if err := m.handleChannelEvent(ctx, event); err != nil { log.Errorf("Error handling channel event: %v", err) @@ -266,31 +311,33 @@ func (m *Monitor) subscribe(ctx context.Context, log.Warn("Channel event error stream " + "closed") - return true + return true, handledEvent } log.Errorf("Error from channel event "+ "subscription: %v", err) - return true + return true, handledEvent case <-m.quit: log.Info("Channel events monitor stopping") - return false + return false, handledEvent case <-ctx.Done(): log.Info("Channel events monitor stopping") - return false + return false, handledEvent } } } -// initialSync performs an initial sync of the channel state. It queries lnd for -// all known channels (open and closed) and records their current state in the -// database. This ensures that any channel events that occurred while faraday -// was offline are accounted for: even though individual events are lost, the +// initialSync performs an initial sync of the channel state. It runs on start +// and on every reconnect of the subscription. It queries lnd for all known +// channels (open and closed) and records their current state in the database. +// This ensures that any channel events that occurred while faraday did not +// observe lnd are accounted for: even though individual events are lost, the // latest state is captured as a baseline. Events recorded during initial sync // are marked with IsSync=true to distinguish them from real-time events -// received via the subscription. +// received via the subscription. The analyzer treats the time from the last +// live event before such a sync batch to the end of that batch as unobserved. func (m *Monitor) initialSync(ctx context.Context) error { log.Info("Performing initial sync of channel state") diff --git a/chanevents/monitor_test.go b/chanevents/monitor_test.go new file mode 100644 index 0000000..6850e43 --- /dev/null +++ b/chanevents/monitor_test.go @@ -0,0 +1,142 @@ +package chanevents + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/lightninglabs/lndclient" + "github.com/lightningnetwork/lnd/clock" + "github.com/stretchr/testify/require" +) + +// TestNextReconnectDelay pins the reconnect backoff. Each reconnect writes a +// full sync batch, so a subscription that fails at once must back off to the +// cap, and a subscription that handled an event must reset the wait. +func TestNextReconnectDelay(t *testing.T) { + t.Parallel() + + var delay time.Duration + var got []time.Duration + for range 8 { + delay = nextReconnectDelay(delay, false) + got = append(got, delay) + } + require.Equal(t, []time.Duration{ + 5 * time.Second, 10 * time.Second, 20 * time.Second, + 40 * time.Second, 80 * time.Second, 160 * time.Second, + maxReconnectInterval, maxReconnectInterval, + }, got) + + require.Equal(t, retryInterval, nextReconnectDelay(delay, true)) +} + +// syncLnd is an lnd stub that scripts the result of each initial sync and +// records a subscription that does not follow a successful sync. The embedded +// interface is nil, so any method it does not override panics. +type syncLnd struct { + lndclient.LightningClient + + // syncErrs holds the result of each initial sync. done closes once + // the monitor attempted all of them. + syncErrs []error + done chan struct{} + + mu sync.Mutex + syncs int + synced bool + unsynced bool +} + +func (l *syncLnd) GetInfo(_ context.Context) (*lndclient.Info, error) { + return &lndclient.Info{SyncedToChain: true}, nil +} + +func (l *syncLnd) ClosedChannels(_ context.Context) ( + []lndclient.ClosedChannel, error) { + + l.mu.Lock() + defer l.mu.Unlock() + + if l.syncs == len(l.syncErrs) { + return nil, errors.New("sync not scripted") + } + + err := l.syncErrs[l.syncs] + l.syncs++ + if l.syncs == len(l.syncErrs) { + close(l.done) + } + + return nil, err +} + +// ListChannels is the last lnd call of a successful initial sync. +func (l *syncLnd) ListChannels(_ context.Context, _, _ bool, + _ ...lndclient.ListChannelsOption) ([]lndclient.ChannelInfo, error) { + + l.mu.Lock() + defer l.mu.Unlock() + + l.synced = true + + return nil, nil +} + +func (l *syncLnd) SubscribeChannelEvents(_ context.Context) ( + <-chan *lndclient.ChannelEventUpdate, <-chan error, error) { + + l.mu.Lock() + defer l.mu.Unlock() + + if !l.synced { + l.unsynced = true + } + l.synced = false + + return nil, nil, errors.New("subscription unavailable") +} + +// TestMonitorSyncsBeforeSubscribe pins that the monitor writes a sync batch +// before each subscription. Without it, a live event would end the unobserved +// interval, and the outage would count as observed time. Each case waits one +// reconnect interval. +func TestMonitorSyncsBeforeSubscribe(t *testing.T) { + t.Parallel() + + syncErr := errors.New("closed channels unavailable") + tests := []struct { + name string + syncErrs []error + }{ + {"failed sync", []error{syncErr, syncErr}}, + {"reconnect", []error{nil, nil}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + lnd := &syncLnd{ + syncErrs: tc.syncErrs, + done: make(chan struct{}), + } + store := NewTestDB(t, clock.NewTestClock(testTime)) + monitor := NewMonitor(lnd, store, Config{}) + require.NoError(t, monitor.Start(t.Context())) + + select { + case <-lnd.done: + case <-time.After(30 * time.Second): + t.Fatal("initial sync not retried") + } + require.NoError(t, monitor.Stop()) + + lnd.mu.Lock() + defer lnd.mu.Unlock() + require.False(t, lnd.unsynced) + }) + } +} diff --git a/chanevents/store.go b/chanevents/store.go index ae26b21..78bdffa 100644 --- a/chanevents/store.go +++ b/chanevents/store.go @@ -58,6 +58,10 @@ type Queries interface { PruneChannelEventsByAge(ctx context.Context, timestamp time.Time) ( int64, error) + + GetUnobservedIntervals(ctx context.Context, + arg sqlc.GetUnobservedIntervalsParams) ( + []sqlc.GetUnobservedIntervalsRow, error) } // Store provides access to the db for channel events. @@ -342,6 +346,58 @@ func (s *Store) GetLatestChannelUpdateBefore(ctx context.Context, return marshalChannelEvent(dbEvent), nil } +// UnobservedInterval is a time range in which faraday did not observe lnd. It +// starts at the last live event before a sync batch and ends at the last +// event of that batch. +type UnobservedInterval struct { + Start time.Time + End time.Time +} + +// UnobservedIntervals returns the unobserved intervals that overlap [start, +// end), clipped to that range, sorted and disjoint. An interval with no live +// event before its sync batch starts at start. The method relies on ids and +// timestamps growing together, which holds while writers leave Timestamp zero +// so the store stamps clock.Now(). +func (s *Store) UnobservedIntervals(ctx context.Context, start, + end time.Time) ([]UnobservedInterval, error) { + + start, end = start.UTC(), end.UTC() + rows, err := s.db.GetUnobservedIntervals( + ctx, sqlc.GetUnobservedIntervalsParams{ + StartTime: start, + EndTime: end, + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to get unobserved intervals: %w", + err) + } + + intervals := make([]UnobservedInterval, 0, len(rows)) + for _, row := range rows { + interval := UnobservedInterval{ + Start: start, + End: row.SyncTimestamp.UTC(), + } + if row.LiveTimestamp.Valid && + row.LiveTimestamp.Time.After(start) { + + interval.Start = row.LiveTimestamp.Time.UTC() + } + if interval.End.After(end) { + interval.End = end + } + if !interval.End.After(interval.Start) { + continue + } + + intervals = append(intervals, interval) + } + + return intervals, nil +} + // PruneEvents enforces the size and age storage limits independently, // returning the number of events deleted. A zero maxEvents or retention // disables the corresponding limit, and zero for both disables pruning. diff --git a/chanevents/store_test.go b/chanevents/store_test.go index 52f6b3b..4c82a15 100644 --- a/chanevents/store_test.go +++ b/chanevents/store_test.go @@ -473,3 +473,132 @@ func TestPagination(t *testing.T) { require.Equal(t, btcutil.Amount(3), page2[0].LocalBalance.UnwrapOr(0)) require.Equal(t, btcutil.Amount(4), page2[1].LocalBalance.UnwrapOr(0)) } + +// TestUnobservedIntervals verifies that every sync batch yields one interval +// from the last live event of any channel before the batch to the last event of +// the batch, clipped to the query range. +func TestUnobservedIntervals(t *testing.T) { + t.Parallel() + + ctx := context.Background() + at := func(s int64) time.Time { + return testTime.Add(time.Duration(s) * time.Second).UTC() + } + + // newStore returns a store with two channels on one peer. + newStore := func(t *testing.T) (*Store, int64, int64) { + store := NewTestDB(t, clock.NewTestClock(testTime)) + + peerID, err := store.AddPeer(ctx, testPubKey) + require.NoError(t, err) + + chan1, err := store.AddChannel( + ctx, testChanPoint1, testShortChanID1, peerID, + ) + require.NoError(t, err) + + chan2, err := store.AddChannel( + ctx, testChanPoint2, testShortChanID2, peerID, + ) + require.NoError(t, err) + + return store, chan1, chan2 + } + + add := func(t *testing.T, store *Store, channelID, s int64, + isSync bool) { + + err := store.AddChannelEvent(ctx, &ChannelEvent{ + ChannelID: channelID, + EventType: EventTypeOnline, + Timestamp: at(s), + IsSync: isSync, + }) + require.NoError(t, err) + } + + t.Run("batches", func(t *testing.T) { + t.Parallel() + + store, chan1, chan2 := newStore(t) + + // The first batch follows a live event on another channel, so + // the interval starts at the live event of any channel. + add(t, store, chan1, 0, false) + add(t, store, chan2, 10, false) + add(t, store, chan1, 100, true) + add(t, store, chan2, 101, true) + + add(t, store, chan1, 200, false) + add(t, store, chan1, 300, true) + add(t, store, chan2, 301, true) + + add(t, store, chan2, 350, false) + add(t, store, chan1, 400, true) + + testCases := []struct { + name string + start, end int64 + expected []UnobservedInterval + }{{ + name: "full range", + start: 0, + end: 1000, + expected: []UnobservedInterval{ + {Start: at(10), End: at(101)}, + {Start: at(200), End: at(301)}, + {Start: at(350), End: at(400)}, + }, + }, { + // The second batch ends after the end, but it starts + // before it, so it is clipped and returned. The + // third batch starts after the end and is dropped. + name: "clipped", + start: 50, + end: 250, + expected: []UnobservedInterval{ + {Start: at(50), End: at(101)}, + {Start: at(200), End: at(250)}, + }, + }, { + name: "batch before start", + start: 150, + end: 1000, + expected: []UnobservedInterval{ + {Start: at(200), End: at(301)}, + {Start: at(350), End: at(400)}, + }, + }, { + name: "no batch in range", + start: 401, + end: 1000, + expected: []UnobservedInterval{}, + }} + + for _, tc := range testCases { + intervals, err := store.UnobservedIntervals( + ctx, at(tc.start), at(tc.end), + ) + require.NoError(t, err, tc.name) + require.Equal(t, tc.expected, intervals, tc.name) + } + }) + + t.Run("no live event before batch", func(t *testing.T) { + t.Parallel() + + store, chan1, chan2 := newStore(t) + add(t, store, chan1, 100, true) + add(t, store, chan2, 101, true) + add(t, store, chan1, 200, false) + + intervals, err := store.UnobservedIntervals( + ctx, at(20), at(1000), + ) + require.NoError(t, err) + require.Equal( + t, []UnobservedInterval{{Start: at(20), End: at(101)}}, + intervals, + ) + }) +} diff --git a/cmd/frcli/forwarding_ability.go b/cmd/frcli/forwarding_ability.go index 7ca8875..6bbce65 100644 --- a/cmd/frcli/forwarding_ability.go +++ b/cmd/frcli/forwarding_ability.go @@ -33,9 +33,10 @@ var forwardingAbilityCommand = cli.Command{ }, cli.Float64Flag{ Name: "uptime_threshold", - Usage: "the uptime fraction in [0,1] at or above " + - "which a non-forwarding pair is reported as " + - "up but idle; zero uses the server default", + Usage: "the uptime fraction in [0,1] of the observed " + + "window at or above which a non-forwarding " + + "pair is reported as up but idle; zero uses " + + "the server default", }, }, Action: queryForwardingAbility, @@ -53,6 +54,14 @@ type pairView struct { FeeRate float64 `json:"fee_rate"` } +// forwardingAbilityView is the JSON output of the command. UnobservedS is the +// seconds of the window that faraday did not observe lnd. Uptime fractions use +// the observed window. +type forwardingAbilityView struct { + UnobservedS uint64 `json:"unobserved_s"` + Pairs []pairView `json:"pairs"` +} + func queryForwardingAbility(ctx *cli.Context) error { client, cleanup := getClient(ctx) defer cleanup() @@ -76,8 +85,10 @@ func queryForwardingAbility(ctx *cli.Context) error { } // The metrics are raw, so derive uptime fraction, velocity and fee rate - // here from the window the server reported. - windowSeconds := resp.EndTime - resp.StartTime + // here from the observed window the server reported. + windowSeconds := frdrpc.ObservedWindow( + resp.StartTime, resp.EndTime, resp.UnobservedS, + ) var views []pairView for inPeer, outMap := range abilities { @@ -134,7 +145,10 @@ func queryForwardingAbility(ctx *cli.Context) error { }, ) - printJSON(views) + printJSON(forwardingAbilityView{ + UnobservedS: resp.UnobservedS, + Pairs: views, + }) return nil } diff --git a/db/migrations.go b/db/migrations.go index 3365074..e6325b8 100644 --- a/db/migrations.go +++ b/db/migrations.go @@ -10,11 +10,11 @@ const ( // daemon. // // NOTE: This MUST be updated when a new migration is added. - LatestMigrationVersion = 2 + LatestMigrationVersion = 3 ) // MakeMigrationDescriptors returns the ordered migration descriptors for -// faraday's SQL schema. Both migrations are schema-only, so neither carries a +// faraday's SQL schema. All migrations are schema-only, so none carries a // programmatic migration step. func MakeMigrationDescriptors() []sqldb.MigrationDescriptor { return []sqldb.MigrationDescriptor{ @@ -28,5 +28,10 @@ func MakeMigrationDescriptors() []sqldb.MigrationDescriptor { Version: 2, SchemaVersion: 2, }, + { + Name: "chanevents_sync_idx", + Version: 3, + SchemaVersion: 3, + }, } } diff --git a/db/sqlc/chanevents.sql.go b/db/sqlc/chanevents.sql.go index 77a05c7..7cb521b 100644 --- a/db/sqlc/chanevents.sql.go +++ b/db/sqlc/chanevents.sql.go @@ -172,6 +172,69 @@ func (q *Queries) GetPeerByPubKey(ctx context.Context, pubkey string) (Peer, err return i, err } +const getUnobservedIntervals = `-- name: GetUnobservedIntervals :many +WITH sync_events AS ( + SELECT s.id, ( + SELECT l.id FROM channel_events l + WHERE l.is_sync = FALSE AND l.id < s.id + ORDER BY l.id DESC + LIMIT 1 + ) AS live_id + FROM channel_events s + WHERE s.is_sync = TRUE AND s.timestamp >= $2 +), batches AS ( + SELECT live_id, MAX(id) AS last_sync_id + FROM sync_events + GROUP BY live_id +) +SELECT live.timestamp AS live_timestamp, sync.timestamp AS sync_timestamp +FROM batches b +JOIN channel_events sync ON sync.id = b.last_sync_id +LEFT JOIN channel_events live ON live.id = b.live_id +WHERE live.id IS NULL OR live.timestamp < $1 +ORDER BY sync.id ASC +` + +type GetUnobservedIntervalsParams struct { + EndTime time.Time + StartTime time.Time +} + +type GetUnobservedIntervalsRow struct { + LiveTimestamp sql.NullTime + SyncTimestamp time.Time +} + +// GetUnobservedIntervals returns one row per sync batch whose last sync event +// lies at or after start_time. A batch is the set of sync events that share +// the same preceding live event. The row carries the timestamp of that live +// event (NULL if none exists) and the timestamp of the last sync event of the +// batch. Rows whose live event lies at or after end_time are left out. The +// query aggregates over ids, not timestamps, because SQLite cannot scan an +// aggregated timestamp back into a time value. +func (q *Queries) GetUnobservedIntervals(ctx context.Context, arg GetUnobservedIntervalsParams) ([]GetUnobservedIntervalsRow, error) { + rows, err := q.db.QueryContext(ctx, getUnobservedIntervals, arg.EndTime, arg.StartTime) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetUnobservedIntervalsRow + for rows.Next() { + var i GetUnobservedIntervalsRow + if err := rows.Scan(&i.LiveTimestamp, &i.SyncTimestamp); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertChannel = `-- name: InsertChannel :one INSERT INTO channels (channel_point, short_channel_id, peer_id) VALUES ($1, $2, $3) RETURNING id ` diff --git a/db/sqlc/migrations/000003_chanevents_sync_idx.down.sql b/db/sqlc/migrations/000003_chanevents_sync_idx.down.sql new file mode 100644 index 0000000..221c6a4 --- /dev/null +++ b/db/sqlc/migrations/000003_chanevents_sync_idx.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS channel_events_sync_id_idx; diff --git a/db/sqlc/migrations/000003_chanevents_sync_idx.up.sql b/db/sqlc/migrations/000003_chanevents_sync_idx.up.sql new file mode 100644 index 0000000..cde9bda --- /dev/null +++ b/db/sqlc/migrations/000003_chanevents_sync_idx.up.sql @@ -0,0 +1,5 @@ +-- This composite index supports GetUnobservedIntervals, which finds, for each +-- sync event, the newest live event with a smaller id. Without it, the lookup +-- scans back over every sync event of a run, so the query cost grows with the +-- square of the run length. +CREATE INDEX IF NOT EXISTS channel_events_sync_id_idx ON channel_events (is_sync, id); diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 5870af6..0bf5b2d 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -16,6 +16,14 @@ type Querier interface { GetChannels(ctx context.Context) ([]GetChannelsRow, error) GetLatestChannelEventBefore(ctx context.Context, arg GetLatestChannelEventBeforeParams) (ChannelEvent, error) GetPeerByPubKey(ctx context.Context, pubkey string) (Peer, error) + // GetUnobservedIntervals returns one row per sync batch whose last sync event + // lies at or after start_time. A batch is the set of sync events that share + // the same preceding live event. The row carries the timestamp of that live + // event (NULL if none exists) and the timestamp of the last sync event of the + // batch. Rows whose live event lies at or after end_time are left out. The + // query aggregates over ids, not timestamps, because SQLite cannot scan an + // aggregated timestamp back into a time value. + GetUnobservedIntervals(ctx context.Context, arg GetUnobservedIntervalsParams) ([]GetUnobservedIntervalsRow, error) InsertChannel(ctx context.Context, arg InsertChannelParams) (int64, error) InsertChannelEvent(ctx context.Context, arg InsertChannelEventParams) error InsertPeer(ctx context.Context, pubkey string) (int64, error) diff --git a/db/sqlc/queries/chanevents.sql b/db/sqlc/queries/chanevents.sql index d32498d..f4ff4ce 100644 --- a/db/sqlc/queries/chanevents.sql +++ b/db/sqlc/queries/chanevents.sql @@ -58,3 +58,32 @@ WHERE channel_events.id < COALESCE(( -- timestamp predates the given cutoff. DELETE FROM channel_events WHERE channel_events.timestamp < $1; + +-- name: GetUnobservedIntervals :many +-- GetUnobservedIntervals returns one row per sync batch whose last sync event +-- lies at or after start_time. A batch is the set of sync events that share +-- the same preceding live event. The row carries the timestamp of that live +-- event (NULL if none exists) and the timestamp of the last sync event of the +-- batch. Rows whose live event lies at or after end_time are left out. The +-- query aggregates over ids, not timestamps, because SQLite cannot scan an +-- aggregated timestamp back into a time value. +WITH sync_events AS ( + SELECT s.id, ( + SELECT l.id FROM channel_events l + WHERE l.is_sync = FALSE AND l.id < s.id + ORDER BY l.id DESC + LIMIT 1 + ) AS live_id + FROM channel_events s + WHERE s.is_sync = TRUE AND s.timestamp >= sqlc.arg(start_time) +), batches AS ( + SELECT live_id, MAX(id) AS last_sync_id + FROM sync_events + GROUP BY live_id +) +SELECT live.timestamp AS live_timestamp, sync.timestamp AS sync_timestamp +FROM batches b +JOIN channel_events sync ON sync.id = b.last_sync_id +LEFT JOIN channel_events live ON live.id = b.live_id +WHERE live.id IS NULL OR live.timestamp < sqlc.arg(end_time) +ORDER BY sync.id ASC; diff --git a/docs/release-notes/release-notes-next.md b/docs/release-notes/release-notes-next.md index d321588..ebd37e9 100644 --- a/docs/release-notes/release-notes-next.md +++ b/docs/release-notes/release-notes-next.md @@ -5,6 +5,25 @@ * Migrate exported Bitcoin types and dependencies to the btcd v2 modules. Source builds require Go 1.26.8. +* `frcli forwardingability` prints a JSON object with the fields + `unobserved_s` and `pairs`. The `pairs` field holds the array the command + printed before. The `uptime_fraction` of a pair uses the observed window. + +## RPC Changes + +* `ForwardingAbility` reports `unobserved_s`, the seconds of the window in + which faraday did not observe lnd, for example while faraday or lnd was + down. Effective uptime excludes this time, and the uptime threshold applies + to the observed window. Divide effective uptime by + `end_time - start_time - unobserved_s`. An outage that still goes on counts + as observed time until the next initial sync marks it as unobserved. + +## Improvements + +* The channel events monitor runs the initial sync again each time the + subscription to lnd reconnects. This re-baselines the channel balances and + marks the break as unobserved time. + ## Maintenance * Retain the lnd v0.21.4 recovery fixes while migrating to btcd v2. diff --git a/frdrpc/faraday.pb.go b/frdrpc/faraday.pb.go index 9191f3f..ab23aa9 100644 --- a/frdrpc/faraday.pb.go +++ b/frdrpc/faraday.pb.go @@ -2225,9 +2225,10 @@ type ForwardingAbilityRequest struct { // economically forwardable. A value of 0 selects the server default. Hold // this constant across calls for comparable series. LiquidityFloorSat uint64 `protobuf:"varint,3,opt,name=liquidity_floor_sat,json=liquidityFloorSat,proto3" json:"liquidity_floor_sat,omitempty"` - // The uptime fraction in [0, 1] at or above which a peer pair that did not - // forward is reported compactly as a single bit in up_but_idle_bitmask - // rather than as a full entry. A value of 0 selects the server default. + // The uptime fraction in [0, 1] of the observed window at or above which a + // peer pair that did not forward is reported compactly as a single bit in + // up_but_idle_bitmask rather than as a full entry. The observed window is + // the window minus unobserved_s. A value of 0 selects the server default. // Pairs below this threshold that also did not forward are omitted // entirely. If no pair meets the threshold the server returns a // FailedPrecondition error, since that indicates the node itself was down @@ -2314,14 +2315,22 @@ type ForwardingAbilityResponse struct { EndTime int64 `protobuf:"varint,4,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` // A packed bitmask over the n*n ordered peer pairs, where n is the length // of peers. The bit at index in*n+out is set when that pair held at least - // the uptime_threshold fraction of effective uptime over the window but - // carried no forward: the dense "up but idle" population. A pair with an - // entry is never flagged here. A pair that is neither listed in entries nor - // flagged here had sub-threshold uptime and no forwards: treat it as zero. + // the uptime_threshold fraction of effective uptime over the observed + // window but carried no forward: the dense "up but idle" population. A pair + // with an entry is never flagged here. A pair that is neither listed in + // entries nor flagged here had sub-threshold uptime and no forwards: treat + // it as zero. UpButIdleBitmask []byte `protobuf:"bytes,5,opt,name=up_but_idle_bitmask,json=upButIdleBitmask,proto3" json:"up_but_idle_bitmask,omitempty"` - // The uptime fraction in [0, 1] used to populate up_but_idle_bitmask, - // echoed back so consumers know the threshold the server applied. + // The uptime fraction in [0, 1] of the observed window used to populate + // up_but_idle_bitmask, echoed back so consumers know the threshold the + // server applied. UptimeThreshold float64 `protobuf:"fixed64,6,opt,name=uptime_threshold,json=uptimeThreshold,proto3" json:"uptime_threshold,omitempty"` + // The seconds of the window in which faraday did not observe lnd, for + // example while faraday or lnd was down. Effective uptime excludes this + // time, so clients divide effective uptime by + // end_time - start_time - unobserved_s. A server that does not report it + // leaves it at 0. + UnobservedS uint64 `protobuf:"varint,7,opt,name=unobserved_s,json=unobservedS,proto3" json:"unobserved_s,omitempty"` } func (x *ForwardingAbilityResponse) Reset() { @@ -2398,6 +2407,13 @@ func (x *ForwardingAbilityResponse) GetUptimeThreshold() float64 { return 0 } +func (x *ForwardingAbilityResponse) GetUnobservedS() uint64 { + if x != nil { + return x.UnobservedS + } + return 0 +} + type ForwardingAbilityEntry struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2408,7 +2424,7 @@ type ForwardingAbilityEntry struct { // 65535 peers per direction. PackedIdx uint32 `protobuf:"varint,1,opt,name=packed_idx,json=packedIdx,proto3" json:"packed_idx,omitempty"` // Seconds the peer pair held at least the requested liquidity floor of - // directional forwardable liquidity over the window. + // directional forwardable liquidity over the observed window. EffectiveUptimeS uint64 `protobuf:"varint,2,opt,name=effective_uptime_s,json=effectiveUptimeS,proto3" json:"effective_uptime_s,omitempty"` // Total successfully forwarded amount over the window, in millisatoshis. ForwardedMsat int64 `protobuf:"varint,3,opt,name=forwarded_msat,json=forwardedMsat,proto3" json:"forwarded_msat,omitempty"` @@ -2764,7 +2780,7 @@ var file_faraday_proto_rawDesc = []byte{ 0x6c, 0x69, 0x71, 0x75, 0x69, 0x64, 0x69, 0x74, 0x79, 0x46, 0x6c, 0x6f, 0x6f, 0x72, 0x53, 0x61, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0f, 0x75, 0x70, 0x74, - 0x69, 0x6d, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x22, 0xff, 0x01, 0x0a, + 0x69, 0x6d, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x22, 0xa2, 0x02, 0x0a, 0x19, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, @@ -2780,114 +2796,116 @@ var file_faraday_proto_rawDesc = []byte{ 0x0c, 0x52, 0x10, 0x75, 0x70, 0x42, 0x75, 0x74, 0x49, 0x64, 0x6c, 0x65, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x73, 0x6b, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0f, 0x75, - 0x70, 0x74, 0x69, 0x6d, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x22, 0xc3, - 0x01, 0x0a, 0x16, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, - 0x6c, 0x69, 0x74, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x63, - 0x6b, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x70, - 0x61, 0x63, 0x6b, 0x65, 0x64, 0x49, 0x64, 0x78, 0x12, 0x2c, 0x0a, 0x12, 0x65, 0x66, 0x66, 0x65, - 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x55, - 0x70, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, - 0x64, 0x65, 0x64, 0x5f, 0x6d, 0x73, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, - 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x4d, 0x73, 0x61, 0x74, 0x12, 0x19, 0x0a, - 0x08, 0x66, 0x65, 0x65, 0x5f, 0x6d, 0x73, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x07, 0x66, 0x65, 0x65, 0x4d, 0x73, 0x61, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x6f, 0x72, 0x77, - 0x61, 0x72, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x66, 0x6f, 0x72, 0x77, - 0x61, 0x72, 0x64, 0x73, 0x2a, 0xa1, 0x01, 0x0a, 0x0b, 0x47, 0x72, 0x61, 0x6e, 0x75, 0x6c, 0x61, - 0x72, 0x69, 0x74, 0x79, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, - 0x47, 0x52, 0x41, 0x4e, 0x55, 0x4c, 0x41, 0x52, 0x49, 0x54, 0x59, 0x10, 0x00, 0x12, 0x0a, 0x0a, - 0x06, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x46, 0x49, 0x56, - 0x45, 0x5f, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, 0x53, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x46, - 0x49, 0x46, 0x54, 0x45, 0x45, 0x4e, 0x5f, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, 0x53, 0x10, 0x03, - 0x12, 0x12, 0x0a, 0x0e, 0x54, 0x48, 0x49, 0x52, 0x54, 0x59, 0x5f, 0x4d, 0x49, 0x4e, 0x55, 0x54, - 0x45, 0x53, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x4f, 0x55, 0x52, 0x10, 0x05, 0x12, 0x0d, - 0x0a, 0x09, 0x53, 0x49, 0x58, 0x5f, 0x48, 0x4f, 0x55, 0x52, 0x53, 0x10, 0x06, 0x12, 0x10, 0x0a, - 0x0c, 0x54, 0x57, 0x45, 0x4c, 0x56, 0x45, 0x5f, 0x48, 0x4f, 0x55, 0x52, 0x53, 0x10, 0x07, 0x12, - 0x07, 0x0a, 0x03, 0x44, 0x41, 0x59, 0x10, 0x08, 0x2a, 0x6a, 0x0a, 0x0b, 0x46, 0x69, 0x61, 0x74, - 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, - 0x57, 0x4e, 0x5f, 0x46, 0x49, 0x41, 0x54, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44, 0x10, 0x00, - 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x49, 0x4e, 0x43, 0x41, 0x50, 0x10, 0x01, 0x12, 0x0c, 0x0a, - 0x08, 0x43, 0x4f, 0x49, 0x4e, 0x44, 0x45, 0x53, 0x4b, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x43, - 0x55, 0x53, 0x54, 0x4f, 0x4d, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x49, 0x4e, 0x47, - 0x45, 0x43, 0x4b, 0x4f, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x49, 0x54, 0x46, 0x49, 0x4e, - 0x45, 0x58, 0x10, 0x05, 0x2a, 0xa2, 0x02, 0x0a, 0x09, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, - 0x16, 0x0a, 0x12, 0x4c, 0x4f, 0x43, 0x41, 0x4c, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, - 0x5f, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x52, 0x45, 0x4d, 0x4f, 0x54, - 0x45, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x02, - 0x12, 0x14, 0x0a, 0x10, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x4f, 0x50, 0x45, 0x4e, - 0x5f, 0x46, 0x45, 0x45, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, - 0x4c, 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x04, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x43, - 0x45, 0x49, 0x50, 0x54, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x41, 0x59, 0x4d, 0x45, 0x4e, - 0x54, 0x10, 0x06, 0x12, 0x07, 0x0a, 0x03, 0x46, 0x45, 0x45, 0x10, 0x07, 0x12, 0x14, 0x0a, 0x10, - 0x43, 0x49, 0x52, 0x43, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x52, 0x45, 0x43, 0x45, 0x49, 0x50, 0x54, - 0x10, 0x08, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x4f, 0x52, 0x57, 0x41, 0x52, 0x44, 0x10, 0x09, 0x12, - 0x0f, 0x0a, 0x0b, 0x46, 0x4f, 0x52, 0x57, 0x41, 0x52, 0x44, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x0a, - 0x12, 0x14, 0x0a, 0x10, 0x43, 0x49, 0x52, 0x43, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x50, 0x41, 0x59, - 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x0b, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x49, 0x52, 0x43, 0x55, 0x4c, - 0x41, 0x52, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x0c, 0x12, 0x09, 0x0a, 0x05, 0x53, 0x57, 0x45, 0x45, - 0x50, 0x10, 0x0d, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x57, 0x45, 0x45, 0x50, 0x5f, 0x46, 0x45, 0x45, - 0x10, 0x0e, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x43, 0x4c, - 0x4f, 0x53, 0x45, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x0f, 0x2a, 0x70, 0x0a, 0x10, 0x43, 0x68, 0x61, - 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, - 0x12, 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, - 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, - 0x45, 0x4e, 0x54, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, - 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4f, 0x46, 0x46, 0x4c, 0x49, - 0x4e, 0x45, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, 0x45, - 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x03, 0x32, 0x83, 0x06, 0x0a, 0x0d, - 0x46, 0x61, 0x72, 0x61, 0x64, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x65, 0x0a, - 0x16, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, - 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x25, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, - 0x2e, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, - 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, - 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x63, - 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x69, 0x0a, 0x18, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, - 0x64, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x27, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, - 0x6f, 0x6c, 0x64, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x66, 0x72, 0x64, 0x72, - 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, - 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4c, 0x0a, 0x0d, 0x52, 0x65, 0x76, 0x65, 0x6e, 0x75, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, - 0x12, 0x1c, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x76, 0x65, 0x6e, 0x75, - 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, - 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x76, 0x65, 0x6e, 0x75, 0x65, 0x52, - 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x52, 0x0a, - 0x0f, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, - 0x12, 0x1e, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, - 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1f, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, - 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x49, 0x0a, 0x0c, 0x45, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x61, 0x74, - 0x65, 0x12, 0x1b, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x63, 0x68, 0x61, - 0x6e, 0x67, 0x65, 0x52, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, - 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, - 0x52, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x09, - 0x4e, 0x6f, 0x64, 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x12, 0x18, 0x2e, 0x66, 0x72, 0x64, 0x72, - 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, - 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, - 0x0a, 0x0b, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x2e, - 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x70, 0x6f, - 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x66, 0x72, 0x64, 0x72, - 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1c, 0x2e, 0x66, 0x72, 0x64, - 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, - 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x11, 0x46, 0x6f, 0x72, 0x77, 0x61, - 0x72, 0x64, 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x20, 0x2e, 0x66, - 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, - 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, - 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, - 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x6c, 0x69, 0x67, 0x68, 0x74, 0x6e, 0x69, 0x6e, 0x67, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x66, 0x61, - 0x72, 0x61, 0x64, 0x61, 0x79, 0x2f, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x70, 0x74, 0x69, 0x6d, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x21, + 0x0a, 0x0c, 0x75, 0x6e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x5f, 0x73, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x75, 0x6e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, + 0x53, 0x22, 0xc3, 0x01, 0x0a, 0x16, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, + 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1d, 0x0a, 0x0a, + 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x49, 0x64, 0x78, 0x12, 0x2c, 0x0a, 0x12, 0x65, + 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x5f, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, + 0x76, 0x65, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x6f, 0x72, + 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x5f, 0x6d, 0x73, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0d, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x4d, 0x73, 0x61, 0x74, + 0x12, 0x19, 0x0a, 0x08, 0x66, 0x65, 0x65, 0x5f, 0x6d, 0x73, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x07, 0x66, 0x65, 0x65, 0x4d, 0x73, 0x61, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x66, + 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x66, + 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x73, 0x2a, 0xa1, 0x01, 0x0a, 0x0b, 0x47, 0x72, 0x61, 0x6e, + 0x75, 0x6c, 0x61, 0x72, 0x69, 0x74, 0x79, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, + 0x57, 0x4e, 0x5f, 0x47, 0x52, 0x41, 0x4e, 0x55, 0x4c, 0x41, 0x52, 0x49, 0x54, 0x59, 0x10, 0x00, + 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, + 0x46, 0x49, 0x56, 0x45, 0x5f, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, 0x53, 0x10, 0x02, 0x12, 0x13, + 0x0a, 0x0f, 0x46, 0x49, 0x46, 0x54, 0x45, 0x45, 0x4e, 0x5f, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, + 0x53, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x54, 0x48, 0x49, 0x52, 0x54, 0x59, 0x5f, 0x4d, 0x49, + 0x4e, 0x55, 0x54, 0x45, 0x53, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x4f, 0x55, 0x52, 0x10, + 0x05, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x49, 0x58, 0x5f, 0x48, 0x4f, 0x55, 0x52, 0x53, 0x10, 0x06, + 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x57, 0x45, 0x4c, 0x56, 0x45, 0x5f, 0x48, 0x4f, 0x55, 0x52, 0x53, + 0x10, 0x07, 0x12, 0x07, 0x0a, 0x03, 0x44, 0x41, 0x59, 0x10, 0x08, 0x2a, 0x6a, 0x0a, 0x0b, 0x46, + 0x69, 0x61, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x4e, + 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x46, 0x49, 0x41, 0x54, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, + 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x49, 0x4e, 0x43, 0x41, 0x50, 0x10, 0x01, + 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x4f, 0x49, 0x4e, 0x44, 0x45, 0x53, 0x4b, 0x10, 0x02, 0x12, 0x0a, + 0x0a, 0x06, 0x43, 0x55, 0x53, 0x54, 0x4f, 0x4d, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, + 0x49, 0x4e, 0x47, 0x45, 0x43, 0x4b, 0x4f, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x49, 0x54, + 0x46, 0x49, 0x4e, 0x45, 0x58, 0x10, 0x05, 0x2a, 0xa2, 0x02, 0x0a, 0x09, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, + 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4c, 0x4f, 0x43, 0x41, 0x4c, 0x5f, 0x43, 0x48, 0x41, 0x4e, + 0x4e, 0x45, 0x4c, 0x5f, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x52, 0x45, + 0x4d, 0x4f, 0x54, 0x45, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x4f, 0x50, 0x45, + 0x4e, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x4f, + 0x50, 0x45, 0x4e, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x48, 0x41, + 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x04, 0x12, 0x0b, 0x0a, 0x07, + 0x52, 0x45, 0x43, 0x45, 0x49, 0x50, 0x54, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x41, 0x59, + 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x06, 0x12, 0x07, 0x0a, 0x03, 0x46, 0x45, 0x45, 0x10, 0x07, 0x12, + 0x14, 0x0a, 0x10, 0x43, 0x49, 0x52, 0x43, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x52, 0x45, 0x43, 0x45, + 0x49, 0x50, 0x54, 0x10, 0x08, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x4f, 0x52, 0x57, 0x41, 0x52, 0x44, + 0x10, 0x09, 0x12, 0x0f, 0x0a, 0x0b, 0x46, 0x4f, 0x52, 0x57, 0x41, 0x52, 0x44, 0x5f, 0x46, 0x45, + 0x45, 0x10, 0x0a, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x49, 0x52, 0x43, 0x55, 0x4c, 0x41, 0x52, 0x5f, + 0x50, 0x41, 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x0b, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x49, 0x52, + 0x43, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x0c, 0x12, 0x09, 0x0a, 0x05, 0x53, + 0x57, 0x45, 0x45, 0x50, 0x10, 0x0d, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x57, 0x45, 0x45, 0x50, 0x5f, + 0x46, 0x45, 0x45, 0x10, 0x0e, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, + 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x0f, 0x2a, 0x70, 0x0a, 0x10, + 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x16, 0x0a, 0x12, 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, + 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, + 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, + 0x16, 0x0a, 0x12, 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4f, 0x46, + 0x46, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, 0x5f, + 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x03, 0x32, 0x83, + 0x06, 0x0a, 0x0d, 0x46, 0x61, 0x72, 0x61, 0x64, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x12, 0x65, 0x0a, 0x16, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x6d, + 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x25, 0x2e, 0x66, 0x72, 0x64, + 0x72, 0x70, 0x63, 0x2e, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x6d, + 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x24, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, + 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x69, 0x0a, 0x18, 0x54, 0x68, 0x72, 0x65, 0x73, + 0x68, 0x6f, 0x6c, 0x64, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x27, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x54, 0x68, 0x72, + 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x66, + 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x6d, + 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0d, 0x52, 0x65, 0x76, 0x65, 0x6e, 0x75, 0x65, 0x52, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x12, 0x1c, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x76, + 0x65, 0x6e, 0x75, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x76, 0x65, 0x6e, + 0x75, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x52, 0x0a, 0x0f, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, + 0x68, 0x74, 0x73, 0x12, 0x1e, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x0c, 0x45, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, + 0x52, 0x61, 0x74, 0x65, 0x12, 0x1b, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, + 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1c, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x63, 0x68, 0x61, + 0x6e, 0x67, 0x65, 0x52, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x40, 0x0a, 0x09, 0x4e, 0x6f, 0x64, 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x12, 0x18, 0x2e, 0x66, + 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, + 0x4e, 0x6f, 0x64, 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x46, 0x0a, 0x0b, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x12, 0x1a, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, + 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x66, + 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, + 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x10, 0x47, 0x65, 0x74, + 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1c, 0x2e, + 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x72, + 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x11, 0x46, 0x6f, + 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, + 0x20, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, + 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, + 0x72, 0x64, 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x6e, 0x69, 0x6e, 0x67, 0x6c, 0x61, 0x62, 0x73, + 0x2f, 0x66, 0x61, 0x72, 0x61, 0x64, 0x61, 0x79, 0x2f, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/frdrpc/faraday.proto b/frdrpc/faraday.proto index fc67cb4..8b75b0e 100644 --- a/frdrpc/faraday.proto +++ b/frdrpc/faraday.proto @@ -696,9 +696,10 @@ message ForwardingAbilityRequest { // this constant across calls for comparable series. uint64 liquidity_floor_sat = 3; - // The uptime fraction in [0, 1] at or above which a peer pair that did not - // forward is reported compactly as a single bit in up_but_idle_bitmask - // rather than as a full entry. A value of 0 selects the server default. + // The uptime fraction in [0, 1] of the observed window at or above which a + // peer pair that did not forward is reported compactly as a single bit in + // up_but_idle_bitmask rather than as a full entry. The observed window is + // the window minus unobserved_s. A value of 0 selects the server default. // Pairs below this threshold that also did not forward are omitted // entirely. If no pair meets the threshold the server returns a // FailedPrecondition error, since that indicates the node itself was down @@ -725,15 +726,24 @@ message ForwardingAbilityResponse { // A packed bitmask over the n*n ordered peer pairs, where n is the length // of peers. The bit at index in*n+out is set when that pair held at least - // the uptime_threshold fraction of effective uptime over the window but - // carried no forward: the dense "up but idle" population. A pair with an - // entry is never flagged here. A pair that is neither listed in entries nor - // flagged here had sub-threshold uptime and no forwards: treat it as zero. + // the uptime_threshold fraction of effective uptime over the observed + // window but carried no forward: the dense "up but idle" population. A pair + // with an entry is never flagged here. A pair that is neither listed in + // entries nor flagged here had sub-threshold uptime and no forwards: treat + // it as zero. bytes up_but_idle_bitmask = 5; - // The uptime fraction in [0, 1] used to populate up_but_idle_bitmask, - // echoed back so consumers know the threshold the server applied. + // The uptime fraction in [0, 1] of the observed window used to populate + // up_but_idle_bitmask, echoed back so consumers know the threshold the + // server applied. double uptime_threshold = 6; + + // The seconds of the window in which faraday did not observe lnd, for + // example while faraday or lnd was down. Effective uptime excludes this + // time, so clients divide effective uptime by + // end_time - start_time - unobserved_s. A server that does not report it + // leaves it at 0. + uint64 unobserved_s = 7; } message ForwardingAbilityEntry { @@ -743,7 +753,7 @@ message ForwardingAbilityEntry { uint32 packed_idx = 1; // Seconds the peer pair held at least the requested liquidity floor of - // directional forwardable liquidity over the window. + // directional forwardable liquidity over the observed window. uint64 effective_uptime_s = 2; // Total successfully forwarded amount over the window, in millisatoshis. diff --git a/frdrpc/faraday.swagger.json b/frdrpc/faraday.swagger.json index 9b4a4e2..7ec67aa 100644 --- a/frdrpc/faraday.swagger.json +++ b/frdrpc/faraday.swagger.json @@ -1036,7 +1036,7 @@ "effective_uptime_s": { "type": "string", "format": "uint64", - "description": "Seconds the peer pair held at least the requested liquidity floor of\ndirectional forwardable liquidity over the window." + "description": "Seconds the peer pair held at least the requested liquidity floor of\ndirectional forwardable liquidity over the observed window." }, "forwarded_msat": { "type": "string", @@ -1087,12 +1087,17 @@ "up_but_idle_bitmask": { "type": "string", "format": "byte", - "description": "A packed bitmask over the n*n ordered peer pairs, where n is the length\nof peers. The bit at index in*n+out is set when that pair held at least\nthe uptime_threshold fraction of effective uptime over the window but\ncarried no forward: the dense \"up but idle\" population. A pair with an\nentry is never flagged here. A pair that is neither listed in entries nor\nflagged here had sub-threshold uptime and no forwards: treat it as zero." + "description": "A packed bitmask over the n*n ordered peer pairs, where n is the length\nof peers. The bit at index in*n+out is set when that pair held at least\nthe uptime_threshold fraction of effective uptime over the observed\nwindow but carried no forward: the dense \"up but idle\" population. A pair\nwith an entry is never flagged here. A pair that is neither listed in\nentries nor flagged here had sub-threshold uptime and no forwards: treat\nit as zero." }, "uptime_threshold": { "type": "number", "format": "double", - "description": "The uptime fraction in [0, 1] used to populate up_but_idle_bitmask,\nechoed back so consumers know the threshold the server applied." + "description": "The uptime fraction in [0, 1] of the observed window used to populate\nup_but_idle_bitmask, echoed back so consumers know the threshold the\nserver applied." + }, + "unobserved_s": { + "type": "string", + "format": "uint64", + "description": "The seconds of the window in which faraday did not observe lnd, for\nexample while faraday or lnd was down. Effective uptime excludes this\ntime, so clients divide effective uptime by\nend_time - start_time - unobserved_s. A server that does not report it\nleaves it at 0." } } }, diff --git a/frdrpc/forwarding_ability_codec.go b/frdrpc/forwarding_ability_codec.go index 7f21aff..773340b 100644 --- a/frdrpc/forwarding_ability_codec.go +++ b/frdrpc/forwarding_ability_codec.go @@ -20,10 +20,11 @@ const maxPackedPeers = 1<<16 - 1 // categories are left to the consumer. type ForwardingAbility struct { // EffectiveUptimeS is the seconds the pair held at least the requested - // liquidity floor of directional forwardable liquidity over the window. - // The value is whole seconds: sub-second uptime floors to zero, so a - // pair that forwarded volume over a fleeting qualifying window can - // report a zero uptime alongside a non-zero ForwardedMsat. + // liquidity floor of directional forwardable liquidity over the + // observed window. The value is whole seconds: sub-second uptime + // floors to zero, so a pair that forwarded volume over a fleeting + // qualifying window can report a zero uptime alongside a non-zero + // ForwardedMsat. EffectiveUptimeS uint64 // ForwardedMsat is the total successfully forwarded amount over the @@ -104,6 +105,18 @@ func MinQualifyingUptime(threshold float64, windowSeconds int64) uint64 { return v } +// ObservedWindow returns the seconds of [startTime, endTime) in which faraday +// observed lnd. It is zero when the unobserved time covers the whole window or +// the window is empty. +func ObservedWindow(startTime, endTime int64, unobservedS uint64) int64 { + window := endTime - startTime + if window <= 0 || unobservedS >= uint64(window) { + return 0 + } + + return window - int64(unobservedS) +} + // setBit sets the bit at the given index in a packed bitmask. The index is an // int64 because an n*n bitmask over the full peer set overflows a 32-bit int. func setBit(mask []byte, index int64) { @@ -121,14 +134,16 @@ func getBit(mask []byte, index int64) bool { // into a memory-efficient sparse gRPC response over [startTime, endTime]. To // optimize payload size it tiers each pair: pairs that carried at least one // forward keep a full entry, pairs that were up at least uptimeThreshold of the -// window but carried none collapse to a single bit in the up-but-idle bitmask, -// and pairs below the threshold that carried none are omitted entirely. Public -// keys are deduplicated and peer pairs packed into 32-bit indices. +// observed window but carried none collapse to a single bit in the up-but-idle +// bitmask, and pairs below the threshold that carried none are omitted +// entirely. The observed window is the window minus unobservedS. Public keys +// are deduplicated and peer pairs packed into 32-bit indices. func EncodeForwardingAbility(abilities map[string]map[string]ForwardingAbility, - startTime, endTime int64, + startTime, endTime int64, unobservedS uint64, uptimeThreshold float64) (*ForwardingAbilityResponse, error) { - minUptimeS := MinQualifyingUptime(uptimeThreshold, endTime-startTime) + observedWindow := ObservedWindow(startTime, endTime, unobservedS) + minUptimeS := MinQualifyingUptime(uptimeThreshold, observedWindow) // First, find all unique peers involved in pairs that warrant either an // entry or a bit. Keys are normalized to lower-case hex so a peer that @@ -265,15 +280,16 @@ func EncodeForwardingAbility(abilities map[string]map[string]ForwardingAbility, EndTime: endTime, UpButIdleBitmask: bitmask, UptimeThreshold: uptimeThreshold, + UnobservedS: unobservedS, }, nil } // DecodeForwardingAbility reconstructs the nested map of peer forwarding // abilities from a sparse packed gRPC response. Forwarded pairs come back with -// their exact facts; up-but-idle pairs flagged in the bitmask come back at full -// window uptime with no forwards, and therefore no volume and no fees. It -// validates packed indices and the bitmask length against the decoded peer list -// to prevent out-of-bounds errors. +// their exact facts; up-but-idle pairs flagged in the bitmask come back at +// full observed window uptime with no forwards, and therefore no volume and no +// fees. It validates packed indices and the bitmask length against the decoded +// peer list to prevent out-of-bounds errors. func DecodeForwardingAbility(resp *ForwardingAbilityResponse) ( map[string]map[string]ForwardingAbility, error) { @@ -342,19 +358,15 @@ func DecodeForwardingAbility(resp *ForwardingAbilityResponse) ( numPeers) } - // Up-but-idle pairs were up the whole window by definition of the - // threshold bucket, so reconstruct them at full window uptime with no + // Up-but-idle pairs were up the whole observed window by definition of + // the threshold bucket, so reconstruct them at that uptime with no // forwards, and therefore no volume and no fees. Iterate over the // bitmask bytes directly, skipping zero bytes, so cost scales with the // number of set bits rather than the O(n*n) pair space; padding bits // beyond n*n are ignored. - // The window is a difference of signed unix timestamps, so an end - // before the start yields zero uptime rather than wrapping into a - // near-infinite one. - var windowSeconds uint64 - if d := resp.EndTime - resp.StartTime; d > 0 { - windowSeconds = uint64(d) - } + windowSeconds := uint64(ObservedWindow( + resp.StartTime, resp.EndTime, resp.UnobservedS, + )) for i, b := range bitmask { if b == 0 { continue diff --git a/frdrpc/forwarding_ability_codec_test.go b/frdrpc/forwarding_ability_codec_test.go index d247093..5c5f2e5 100644 --- a/frdrpc/forwarding_ability_codec_test.go +++ b/frdrpc/forwarding_ability_codec_test.go @@ -215,7 +215,7 @@ func TestForwardingAbilityCodecRoundTrip(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { resp, err := EncodeForwardingAbility( - tc.abilities, startTime, endTime, threshold, + tc.abilities, startTime, endTime, 0, threshold, ) require.NoError(t, err) require.Equal(t, startTime, resp.StartTime) @@ -282,7 +282,7 @@ func TestForwardingAbilityCodecZeroValueForward(t *testing.T) { } resp, err := EncodeForwardingAbility( - abilities, startTime, endTime, threshold, + abilities, startTime, endTime, 0, threshold, ) require.NoError(t, err) @@ -306,6 +306,46 @@ func TestForwardingAbilityCodecZeroValueForward(t *testing.T) { ) } +// TestForwardingAbilityCodecObservedWindow verifies that the threshold applies +// to the observed window, which is the window minus the unobserved time, and +// that a decoded bit carries the observed window as its uptime. The window is +// [0, 100) with 40 unobserved seconds and the threshold 0.5, so the minimum +// qualifying uptime is 30 seconds instead of 50. +func TestForwardingAbilityCodecObservedWindow(t *testing.T) { + const ( + startTime, endTime int64 = 0, 100 + unobservedS uint64 = 40 + threshold float64 = 0.5 + ) + + abilities := map[string]map[string]ForwardingAbility{ + fwdKey(1): { + // Meets the threshold of the observed window only. + fwdKey(2): {EffectiveUptimeS: 40}, + }, + fwdKey(2): { + // Misses the threshold of the observed window. + fwdKey(1): {EffectiveUptimeS: 20}, + }, + } + + resp, err := EncodeForwardingAbility( + abilities, startTime, endTime, unobservedS, threshold, + ) + require.NoError(t, err) + require.Equal(t, unobservedS, resp.UnobservedS) + require.Empty(t, resp.Entries) + require.NotEmpty(t, resp.UpButIdleBitmask) + + decoded, err := DecodeForwardingAbility(resp) + require.NoError(t, err) + require.Equal( + t, map[string]map[string]ForwardingAbility{ + fwdKey(1): {fwdKey(2): {EffectiveUptimeS: 60}}, + }, decoded, + ) +} + // TestMinQualifyingUptime verifies the threshold-to-seconds conversion shared // by the encoder and the server guard, including its boundary behavior. func TestMinQualifyingUptime(t *testing.T) { @@ -361,6 +401,29 @@ func TestMinQualifyingUptime(t *testing.T) { } } +// TestObservedWindow verifies that the observed window is the window minus the +// unobserved time, and that it never goes below zero. +func TestObservedWindow(t *testing.T) { + tests := []struct { + name string + startTime, endTime int64 + unobservedS uint64 + want int64 + }{ + {"subtracts unobserved time", 0, 100, 40, 60}, + {"unobserved time exceeds window", 0, 100, 150, 0}, + {"end before start", 100, 0, 0, 0}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, ObservedWindow( + tc.startTime, tc.endTime, tc.unobservedS, + )) + }) + } +} + // TestBitmaskHelpers verifies that setBit and getBit address the same bit. func TestBitmaskHelpers(t *testing.T) { mask := make([]byte, 2) @@ -483,7 +546,7 @@ func TestForwardingAbilityEncodePeerCap(t *testing.T) { fwdKey(0): outMap, } - _, err := EncodeForwardingAbility(abilities, 0, 1, 0.5) + _, err := EncodeForwardingAbility(abilities, 0, 1, 0, 0.5) require.ErrorContains(t, err, "exceeds") } @@ -510,7 +573,7 @@ func TestForwardingAbilityEncodeNormalizesCase(t *testing.T) { }, } - resp, err := EncodeForwardingAbility(abilities, 0, 1, 0.5) + resp, err := EncodeForwardingAbility(abilities, 0, 1, 0, 0.5) require.NoError(t, err) // The upper- and lower-case forms of the shared peer must dedup to one @@ -559,7 +622,7 @@ func TestForwardingAbilityEncodeRejectsCaseCollision(t *testing.T) { }, } - _, err := EncodeForwardingAbility(abilities, 0, 1, 0.5) + _, err := EncodeForwardingAbility(abilities, 0, 1, 0, 0.5) require.Error(t, err) } @@ -607,7 +670,7 @@ func TestForwardingAbilityCodecRoundTripHighIndices(t *testing.T) { } resp, err := EncodeForwardingAbility( - abilities, startTime, endTime, threshold, + abilities, startTime, endTime, 0, threshold, ) require.NoError(t, err) require.Len(t, resp.Peers, numPeers) diff --git a/frdrpcserver/forwarding_ability.go b/frdrpcserver/forwarding_ability.go index 4cb4dff..cb07b0e 100644 --- a/frdrpcserver/forwarding_ability.go +++ b/frdrpcserver/forwarding_ability.go @@ -96,7 +96,7 @@ func (s *RPCServer) ForwardingAbility(ctx context.Context, ) } - abilities, err := s.cfg.ForwardingAnalyzer.EffectiveUptime( + abilities, unobserved, err := s.cfg.ForwardingAnalyzer.EffectiveUptime( ctx, startTime, endTime, btcutil.Amount(liquidityFloor), ) if err != nil { @@ -111,6 +111,10 @@ func (s *RPCServer) ForwardingAbility(ctx context.Context, "calculate effective uptime: %v", err) } + // Round up, because uptime rounds down: a pair that was up for the + // whole observed window then still reaches its full length. + unobservedS := uint64((unobserved + time.Second - 1) / time.Second) + // Convert the flat map into the nested map the codec expects, carrying // the raw facts through unchanged. EffectiveUptime is truncated to // whole seconds to match the second-granularity wire field. A pair with @@ -134,11 +138,13 @@ func (s *RPCServer) ForwardingAbility(ctx context.Context, } // Guard against returning data when the node itself was down for the - // window. If no pair held at least the threshold fraction of uptime, - // the response carries no signal and lowering the threshold to surface - // something would only inflate it, so fail loudly instead. + // observed window. If no pair held at least the threshold fraction of + // uptime, the response carries no signal and lowering the threshold to + // surface something would only inflate it, so fail loudly instead. minUptimeS := frdrpc.MinQualifyingUptime( - uptimeThreshold, endTime.Unix()-startTime.Unix(), + uptimeThreshold, frdrpc.ObservedWindow( + startTime.Unix(), endTime.Unix(), unobservedS, + ), ) var qualifying int for _, outMap := range nested { @@ -155,7 +161,8 @@ func (s *RPCServer) ForwardingAbility(ctx context.Context, } resp, err := frdrpc.EncodeForwardingAbility( - nested, startTime.Unix(), endTime.Unix(), uptimeThreshold, + nested, startTime.Unix(), endTime.Unix(), unobservedS, + uptimeThreshold, ) if err != nil { log.ErrorS( diff --git a/frdrpcserver/forwarding_ability_test.go b/frdrpcserver/forwarding_ability_test.go index 5e30ded..e48b0ba 100644 --- a/frdrpcserver/forwarding_ability_test.go +++ b/frdrpcserver/forwarding_ability_test.go @@ -17,12 +17,14 @@ import ( type mockForwardingAnalyzer struct { effectiveUptimeFunc func(ctx context.Context, startTime, endTime time.Time, liquidityFloor btcutil.Amount) ( - map[chanevents.PeerPair]chanevents.ForwardingAbility, error) + map[chanevents.PeerPair]chanevents.ForwardingAbility, + time.Duration, error) } func (m *mockForwardingAnalyzer) EffectiveUptime(ctx context.Context, startTime, endTime time.Time, liquidityFloor btcutil.Amount) ( - map[chanevents.PeerPair]chanevents.ForwardingAbility, error) { + map[chanevents.PeerPair]chanevents.ForwardingAbility, time.Duration, + error) { return m.effectiveUptimeFunc(ctx, startTime, endTime, liquidityFloor) } @@ -45,6 +47,9 @@ func TestForwardingAbility(t *testing.T) { analyzer analyzerResult req *frdrpc.ForwardingAbilityRequest + // unobserved is the unobserved time the analyzer reports. + unobserved time.Duration + // wantCode is the expected gRPC status; codes.OK denotes // success. wantCode codes.Code @@ -212,6 +217,39 @@ func TestForwardingAbility(t *testing.T) { require.NotEmpty(t, resp.UpButIdleBitmask) }, }, + { + // The pair was up for the whole observed window. The + // unobserved time rounds up to 60s, leaving an observed + // window of 40s. The uptime rounds down to 40s, so the + // pair clears a threshold of 1. + name: "unobserved time shrinks guard window", + unobserved: 59500 * time.Millisecond, + analyzer: func() ( + map[chanevents.PeerPair]chanevents.ForwardingAbility, + error) { + + return map[chanevents.PeerPair]chanevents.ForwardingAbility{ + {PeerIn: peerIn, PeerOut: peerOut}: { + EffectiveUptime: 40500 * + time.Millisecond, + }, + }, nil + }, + req: &frdrpc.ForwardingAbilityRequest{ + StartTime: 100, + EndTime: 200, + UptimeThreshold: 1, + }, + wantCode: codes.OK, + check: func(t *testing.T, + resp *frdrpc.ForwardingAbilityResponse, + _ btcutil.Amount) { + + require.EqualValues(t, 60, resp.UnobservedS) + require.Empty(t, resp.Entries) + require.NotEmpty(t, resp.UpButIdleBitmask) + }, + }, { name: "out of range threshold is rejected", analyzer: func() ( @@ -275,11 +313,14 @@ func TestForwardingAbility(t *testing.T) { _ time.Time, floor btcutil.Amount) ( map[chanevents.PeerPair]chanevents.ForwardingAbility, - error) { + time.Duration, error) { gotFloor = floor + abilities, err := + tc.analyzer() - return tc.analyzer() + return abilities, + tc.unobserved, err }, } } diff --git a/frdrpcserver/rpcserver.go b/frdrpcserver/rpcserver.go index 2115170..72c2af6 100644 --- a/frdrpcserver/rpcserver.go +++ b/frdrpcserver/rpcserver.go @@ -56,12 +56,13 @@ type RPCServer struct { } // ForwardingAnalyzer is the seam the RPC server uses to compute per-peer-pair -// forwarding facts. It is an interface so the handler can be exercised against -// a mock analyzer in tests. +// forwarding facts and the unobserved time of the window. It is an interface +// so the handler can be exercised against a mock analyzer in tests. type ForwardingAnalyzer interface { EffectiveUptime(ctx context.Context, startTime, endTime time.Time, liquidityFloor btcutil.Amount) ( - map[chanevents.PeerPair]chanevents.ForwardingAbility, error) + map[chanevents.PeerPair]chanevents.ForwardingAbility, + time.Duration, error) } // Config provides closures and settings required to run the rpc server.