From f381a78c10ea8ed8dc5e0dadf8baaebe6864e72b Mon Sep 17 00:00:00 2001 From: bitromortac Date: Fri, 30 Oct 2026 09:32:14 +0000 Subject: [PATCH 1/5] chanevents: add query for unobserved intervals An unobserved interval is the time faraday did not observe lnd. It ends at the last event of a sync batch and starts at the last live event of any channel before that batch. Sync events that share the same preceding live event form one batch, so one restart gives one interval. The store returns the intervals clipped to the query range. The query aggregates over ids, because SQLite cannot scan an aggregated timestamp back into a time value. --- chanevents/store.go | 56 ++++++++ chanevents/store_test.go | 129 ++++++++++++++++++ db/migrations.go | 9 +- db/sqlc/chanevents.sql.go | 63 +++++++++ .../000003_chanevents_sync_idx.down.sql | 1 + .../000003_chanevents_sync_idx.up.sql | 5 + db/sqlc/querier.go | 8 ++ db/sqlc/queries/chanevents.sql | 29 ++++ 8 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 db/sqlc/migrations/000003_chanevents_sync_idx.down.sql create mode 100644 db/sqlc/migrations/000003_chanevents_sync_idx.up.sql diff --git a/chanevents/store.go b/chanevents/store.go index ae26b21..78bdffa 100644 --- a/chanevents/store.go +++ b/chanevents/store.go @@ -58,6 +58,10 @@ type Queries interface { PruneChannelEventsByAge(ctx context.Context, timestamp time.Time) ( int64, error) + + GetUnobservedIntervals(ctx context.Context, + arg sqlc.GetUnobservedIntervalsParams) ( + []sqlc.GetUnobservedIntervalsRow, error) } // Store provides access to the db for channel events. @@ -342,6 +346,58 @@ func (s *Store) GetLatestChannelUpdateBefore(ctx context.Context, return marshalChannelEvent(dbEvent), nil } +// UnobservedInterval is a time range in which faraday did not observe lnd. It +// starts at the last live event before a sync batch and ends at the last +// event of that batch. +type UnobservedInterval struct { + Start time.Time + End time.Time +} + +// UnobservedIntervals returns the unobserved intervals that overlap [start, +// end), clipped to that range, sorted and disjoint. An interval with no live +// event before its sync batch starts at start. The method relies on ids and +// timestamps growing together, which holds while writers leave Timestamp zero +// so the store stamps clock.Now(). +func (s *Store) UnobservedIntervals(ctx context.Context, start, + end time.Time) ([]UnobservedInterval, error) { + + start, end = start.UTC(), end.UTC() + rows, err := s.db.GetUnobservedIntervals( + ctx, sqlc.GetUnobservedIntervalsParams{ + StartTime: start, + EndTime: end, + }, + ) + if err != nil { + return nil, fmt.Errorf("failed to get unobserved intervals: %w", + err) + } + + intervals := make([]UnobservedInterval, 0, len(rows)) + for _, row := range rows { + interval := UnobservedInterval{ + Start: start, + End: row.SyncTimestamp.UTC(), + } + if row.LiveTimestamp.Valid && + row.LiveTimestamp.Time.After(start) { + + interval.Start = row.LiveTimestamp.Time.UTC() + } + if interval.End.After(end) { + interval.End = end + } + if !interval.End.After(interval.Start) { + continue + } + + intervals = append(intervals, interval) + } + + return intervals, nil +} + // PruneEvents enforces the size and age storage limits independently, // returning the number of events deleted. A zero maxEvents or retention // disables the corresponding limit, and zero for both disables pruning. diff --git a/chanevents/store_test.go b/chanevents/store_test.go index 52f6b3b..4c82a15 100644 --- a/chanevents/store_test.go +++ b/chanevents/store_test.go @@ -473,3 +473,132 @@ func TestPagination(t *testing.T) { require.Equal(t, btcutil.Amount(3), page2[0].LocalBalance.UnwrapOr(0)) require.Equal(t, btcutil.Amount(4), page2[1].LocalBalance.UnwrapOr(0)) } + +// TestUnobservedIntervals verifies that every sync batch yields one interval +// from the last live event of any channel before the batch to the last event of +// the batch, clipped to the query range. +func TestUnobservedIntervals(t *testing.T) { + t.Parallel() + + ctx := context.Background() + at := func(s int64) time.Time { + return testTime.Add(time.Duration(s) * time.Second).UTC() + } + + // newStore returns a store with two channels on one peer. + newStore := func(t *testing.T) (*Store, int64, int64) { + store := NewTestDB(t, clock.NewTestClock(testTime)) + + peerID, err := store.AddPeer(ctx, testPubKey) + require.NoError(t, err) + + chan1, err := store.AddChannel( + ctx, testChanPoint1, testShortChanID1, peerID, + ) + require.NoError(t, err) + + chan2, err := store.AddChannel( + ctx, testChanPoint2, testShortChanID2, peerID, + ) + require.NoError(t, err) + + return store, chan1, chan2 + } + + add := func(t *testing.T, store *Store, channelID, s int64, + isSync bool) { + + err := store.AddChannelEvent(ctx, &ChannelEvent{ + ChannelID: channelID, + EventType: EventTypeOnline, + Timestamp: at(s), + IsSync: isSync, + }) + require.NoError(t, err) + } + + t.Run("batches", func(t *testing.T) { + t.Parallel() + + store, chan1, chan2 := newStore(t) + + // The first batch follows a live event on another channel, so + // the interval starts at the live event of any channel. + add(t, store, chan1, 0, false) + add(t, store, chan2, 10, false) + add(t, store, chan1, 100, true) + add(t, store, chan2, 101, true) + + add(t, store, chan1, 200, false) + add(t, store, chan1, 300, true) + add(t, store, chan2, 301, true) + + add(t, store, chan2, 350, false) + add(t, store, chan1, 400, true) + + testCases := []struct { + name string + start, end int64 + expected []UnobservedInterval + }{{ + name: "full range", + start: 0, + end: 1000, + expected: []UnobservedInterval{ + {Start: at(10), End: at(101)}, + {Start: at(200), End: at(301)}, + {Start: at(350), End: at(400)}, + }, + }, { + // The second batch ends after the end, but it starts + // before it, so it is clipped and returned. The + // third batch starts after the end and is dropped. + name: "clipped", + start: 50, + end: 250, + expected: []UnobservedInterval{ + {Start: at(50), End: at(101)}, + {Start: at(200), End: at(250)}, + }, + }, { + name: "batch before start", + start: 150, + end: 1000, + expected: []UnobservedInterval{ + {Start: at(200), End: at(301)}, + {Start: at(350), End: at(400)}, + }, + }, { + name: "no batch in range", + start: 401, + end: 1000, + expected: []UnobservedInterval{}, + }} + + for _, tc := range testCases { + intervals, err := store.UnobservedIntervals( + ctx, at(tc.start), at(tc.end), + ) + require.NoError(t, err, tc.name) + require.Equal(t, tc.expected, intervals, tc.name) + } + }) + + t.Run("no live event before batch", func(t *testing.T) { + t.Parallel() + + store, chan1, chan2 := newStore(t) + add(t, store, chan1, 100, true) + add(t, store, chan2, 101, true) + add(t, store, chan1, 200, false) + + intervals, err := store.UnobservedIntervals( + ctx, at(20), at(1000), + ) + require.NoError(t, err) + require.Equal( + t, []UnobservedInterval{{Start: at(20), End: at(101)}}, + intervals, + ) + }) +} diff --git a/db/migrations.go b/db/migrations.go index 3365074..e6325b8 100644 --- a/db/migrations.go +++ b/db/migrations.go @@ -10,11 +10,11 @@ const ( // daemon. // // NOTE: This MUST be updated when a new migration is added. - LatestMigrationVersion = 2 + LatestMigrationVersion = 3 ) // MakeMigrationDescriptors returns the ordered migration descriptors for -// faraday's SQL schema. Both migrations are schema-only, so neither carries a +// faraday's SQL schema. All migrations are schema-only, so none carries a // programmatic migration step. func MakeMigrationDescriptors() []sqldb.MigrationDescriptor { return []sqldb.MigrationDescriptor{ @@ -28,5 +28,10 @@ func MakeMigrationDescriptors() []sqldb.MigrationDescriptor { Version: 2, SchemaVersion: 2, }, + { + Name: "chanevents_sync_idx", + Version: 3, + SchemaVersion: 3, + }, } } diff --git a/db/sqlc/chanevents.sql.go b/db/sqlc/chanevents.sql.go index 77a05c7..7cb521b 100644 --- a/db/sqlc/chanevents.sql.go +++ b/db/sqlc/chanevents.sql.go @@ -172,6 +172,69 @@ func (q *Queries) GetPeerByPubKey(ctx context.Context, pubkey string) (Peer, err return i, err } +const getUnobservedIntervals = `-- name: GetUnobservedIntervals :many +WITH sync_events AS ( + SELECT s.id, ( + SELECT l.id FROM channel_events l + WHERE l.is_sync = FALSE AND l.id < s.id + ORDER BY l.id DESC + LIMIT 1 + ) AS live_id + FROM channel_events s + WHERE s.is_sync = TRUE AND s.timestamp >= $2 +), batches AS ( + SELECT live_id, MAX(id) AS last_sync_id + FROM sync_events + GROUP BY live_id +) +SELECT live.timestamp AS live_timestamp, sync.timestamp AS sync_timestamp +FROM batches b +JOIN channel_events sync ON sync.id = b.last_sync_id +LEFT JOIN channel_events live ON live.id = b.live_id +WHERE live.id IS NULL OR live.timestamp < $1 +ORDER BY sync.id ASC +` + +type GetUnobservedIntervalsParams struct { + EndTime time.Time + StartTime time.Time +} + +type GetUnobservedIntervalsRow struct { + LiveTimestamp sql.NullTime + SyncTimestamp time.Time +} + +// GetUnobservedIntervals returns one row per sync batch whose last sync event +// lies at or after start_time. A batch is the set of sync events that share +// the same preceding live event. The row carries the timestamp of that live +// event (NULL if none exists) and the timestamp of the last sync event of the +// batch. Rows whose live event lies at or after end_time are left out. The +// query aggregates over ids, not timestamps, because SQLite cannot scan an +// aggregated timestamp back into a time value. +func (q *Queries) GetUnobservedIntervals(ctx context.Context, arg GetUnobservedIntervalsParams) ([]GetUnobservedIntervalsRow, error) { + rows, err := q.db.QueryContext(ctx, getUnobservedIntervals, arg.EndTime, arg.StartTime) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetUnobservedIntervalsRow + for rows.Next() { + var i GetUnobservedIntervalsRow + if err := rows.Scan(&i.LiveTimestamp, &i.SyncTimestamp); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertChannel = `-- name: InsertChannel :one INSERT INTO channels (channel_point, short_channel_id, peer_id) VALUES ($1, $2, $3) RETURNING id ` diff --git a/db/sqlc/migrations/000003_chanevents_sync_idx.down.sql b/db/sqlc/migrations/000003_chanevents_sync_idx.down.sql new file mode 100644 index 0000000..221c6a4 --- /dev/null +++ b/db/sqlc/migrations/000003_chanevents_sync_idx.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS channel_events_sync_id_idx; diff --git a/db/sqlc/migrations/000003_chanevents_sync_idx.up.sql b/db/sqlc/migrations/000003_chanevents_sync_idx.up.sql new file mode 100644 index 0000000..cde9bda --- /dev/null +++ b/db/sqlc/migrations/000003_chanevents_sync_idx.up.sql @@ -0,0 +1,5 @@ +-- This composite index supports GetUnobservedIntervals, which finds, for each +-- sync event, the newest live event with a smaller id. Without it, the lookup +-- scans back over every sync event of a run, so the query cost grows with the +-- square of the run length. +CREATE INDEX IF NOT EXISTS channel_events_sync_id_idx ON channel_events (is_sync, id); diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 5870af6..0bf5b2d 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -16,6 +16,14 @@ type Querier interface { GetChannels(ctx context.Context) ([]GetChannelsRow, error) GetLatestChannelEventBefore(ctx context.Context, arg GetLatestChannelEventBeforeParams) (ChannelEvent, error) GetPeerByPubKey(ctx context.Context, pubkey string) (Peer, error) + // GetUnobservedIntervals returns one row per sync batch whose last sync event + // lies at or after start_time. A batch is the set of sync events that share + // the same preceding live event. The row carries the timestamp of that live + // event (NULL if none exists) and the timestamp of the last sync event of the + // batch. Rows whose live event lies at or after end_time are left out. The + // query aggregates over ids, not timestamps, because SQLite cannot scan an + // aggregated timestamp back into a time value. + GetUnobservedIntervals(ctx context.Context, arg GetUnobservedIntervalsParams) ([]GetUnobservedIntervalsRow, error) InsertChannel(ctx context.Context, arg InsertChannelParams) (int64, error) InsertChannelEvent(ctx context.Context, arg InsertChannelEventParams) error InsertPeer(ctx context.Context, pubkey string) (int64, error) diff --git a/db/sqlc/queries/chanevents.sql b/db/sqlc/queries/chanevents.sql index d32498d..f4ff4ce 100644 --- a/db/sqlc/queries/chanevents.sql +++ b/db/sqlc/queries/chanevents.sql @@ -58,3 +58,32 @@ WHERE channel_events.id < COALESCE(( -- timestamp predates the given cutoff. DELETE FROM channel_events WHERE channel_events.timestamp < $1; + +-- name: GetUnobservedIntervals :many +-- GetUnobservedIntervals returns one row per sync batch whose last sync event +-- lies at or after start_time. A batch is the set of sync events that share +-- the same preceding live event. The row carries the timestamp of that live +-- event (NULL if none exists) and the timestamp of the last sync event of the +-- batch. Rows whose live event lies at or after end_time are left out. The +-- query aggregates over ids, not timestamps, because SQLite cannot scan an +-- aggregated timestamp back into a time value. +WITH sync_events AS ( + SELECT s.id, ( + SELECT l.id FROM channel_events l + WHERE l.is_sync = FALSE AND l.id < s.id + ORDER BY l.id DESC + LIMIT 1 + ) AS live_id + FROM channel_events s + WHERE s.is_sync = TRUE AND s.timestamp >= sqlc.arg(start_time) +), batches AS ( + SELECT live_id, MAX(id) AS last_sync_id + FROM sync_events + GROUP BY live_id +) +SELECT live.timestamp AS live_timestamp, sync.timestamp AS sync_timestamp +FROM batches b +JOIN channel_events sync ON sync.id = b.last_sync_id +LEFT JOIN channel_events live ON live.id = b.live_id +WHERE live.id IS NULL OR live.timestamp < sqlc.arg(end_time) +ORDER BY sync.id ASC; From 471c9f681e300ec9b1ecdf33716814b05da48663 Mon Sep 17 00:00:00 2001 From: bitromortac Date: Fri, 30 Oct 2026 11:23:44 +0000 Subject: [PATCH 2/5] frdrpc: add unobserved_s to forwarding ability Add the unobserved_s field to the forwarding ability response. It holds the seconds of the window in which faraday did not observe lnd. The codec applies the uptime threshold to the observed window, which is the window minus unobserved_s, and decodes up-but-idle pairs at the full observed window. frcli prints the field once and derives the uptime fraction from the observed window. The server reports zero until the analyzer computes the unobserved time. --- cmd/frcli/forwarding_ability.go | 26 ++- frdrpc/faraday.pb.go | 256 +++++++++++++----------- frdrpc/faraday.proto | 30 ++- frdrpc/faraday.swagger.json | 11 +- frdrpc/forwarding_ability_codec.go | 56 ++++-- frdrpc/forwarding_ability_codec_test.go | 75 ++++++- frdrpcserver/forwarding_ability.go | 2 +- 7 files changed, 289 insertions(+), 167 deletions(-) diff --git a/cmd/frcli/forwarding_ability.go b/cmd/frcli/forwarding_ability.go index 7ca8875..6bbce65 100644 --- a/cmd/frcli/forwarding_ability.go +++ b/cmd/frcli/forwarding_ability.go @@ -33,9 +33,10 @@ var forwardingAbilityCommand = cli.Command{ }, cli.Float64Flag{ Name: "uptime_threshold", - Usage: "the uptime fraction in [0,1] at or above " + - "which a non-forwarding pair is reported as " + - "up but idle; zero uses the server default", + Usage: "the uptime fraction in [0,1] of the observed " + + "window at or above which a non-forwarding " + + "pair is reported as up but idle; zero uses " + + "the server default", }, }, Action: queryForwardingAbility, @@ -53,6 +54,14 @@ type pairView struct { FeeRate float64 `json:"fee_rate"` } +// forwardingAbilityView is the JSON output of the command. UnobservedS is the +// seconds of the window that faraday did not observe lnd. Uptime fractions use +// the observed window. +type forwardingAbilityView struct { + UnobservedS uint64 `json:"unobserved_s"` + Pairs []pairView `json:"pairs"` +} + func queryForwardingAbility(ctx *cli.Context) error { client, cleanup := getClient(ctx) defer cleanup() @@ -76,8 +85,10 @@ func queryForwardingAbility(ctx *cli.Context) error { } // The metrics are raw, so derive uptime fraction, velocity and fee rate - // here from the window the server reported. - windowSeconds := resp.EndTime - resp.StartTime + // here from the observed window the server reported. + windowSeconds := frdrpc.ObservedWindow( + resp.StartTime, resp.EndTime, resp.UnobservedS, + ) var views []pairView for inPeer, outMap := range abilities { @@ -134,7 +145,10 @@ func queryForwardingAbility(ctx *cli.Context) error { }, ) - printJSON(views) + printJSON(forwardingAbilityView{ + UnobservedS: resp.UnobservedS, + Pairs: views, + }) return nil } diff --git a/frdrpc/faraday.pb.go b/frdrpc/faraday.pb.go index 9191f3f..ab23aa9 100644 --- a/frdrpc/faraday.pb.go +++ b/frdrpc/faraday.pb.go @@ -2225,9 +2225,10 @@ type ForwardingAbilityRequest struct { // economically forwardable. A value of 0 selects the server default. Hold // this constant across calls for comparable series. LiquidityFloorSat uint64 `protobuf:"varint,3,opt,name=liquidity_floor_sat,json=liquidityFloorSat,proto3" json:"liquidity_floor_sat,omitempty"` - // The uptime fraction in [0, 1] at or above which a peer pair that did not - // forward is reported compactly as a single bit in up_but_idle_bitmask - // rather than as a full entry. A value of 0 selects the server default. + // The uptime fraction in [0, 1] of the observed window at or above which a + // peer pair that did not forward is reported compactly as a single bit in + // up_but_idle_bitmask rather than as a full entry. The observed window is + // the window minus unobserved_s. A value of 0 selects the server default. // Pairs below this threshold that also did not forward are omitted // entirely. If no pair meets the threshold the server returns a // FailedPrecondition error, since that indicates the node itself was down @@ -2314,14 +2315,22 @@ type ForwardingAbilityResponse struct { EndTime int64 `protobuf:"varint,4,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` // A packed bitmask over the n*n ordered peer pairs, where n is the length // of peers. The bit at index in*n+out is set when that pair held at least - // the uptime_threshold fraction of effective uptime over the window but - // carried no forward: the dense "up but idle" population. A pair with an - // entry is never flagged here. A pair that is neither listed in entries nor - // flagged here had sub-threshold uptime and no forwards: treat it as zero. + // the uptime_threshold fraction of effective uptime over the observed + // window but carried no forward: the dense "up but idle" population. A pair + // with an entry is never flagged here. A pair that is neither listed in + // entries nor flagged here had sub-threshold uptime and no forwards: treat + // it as zero. UpButIdleBitmask []byte `protobuf:"bytes,5,opt,name=up_but_idle_bitmask,json=upButIdleBitmask,proto3" json:"up_but_idle_bitmask,omitempty"` - // The uptime fraction in [0, 1] used to populate up_but_idle_bitmask, - // echoed back so consumers know the threshold the server applied. + // The uptime fraction in [0, 1] of the observed window used to populate + // up_but_idle_bitmask, echoed back so consumers know the threshold the + // server applied. UptimeThreshold float64 `protobuf:"fixed64,6,opt,name=uptime_threshold,json=uptimeThreshold,proto3" json:"uptime_threshold,omitempty"` + // The seconds of the window in which faraday did not observe lnd, for + // example while faraday or lnd was down. Effective uptime excludes this + // time, so clients divide effective uptime by + // end_time - start_time - unobserved_s. A server that does not report it + // leaves it at 0. + UnobservedS uint64 `protobuf:"varint,7,opt,name=unobserved_s,json=unobservedS,proto3" json:"unobserved_s,omitempty"` } func (x *ForwardingAbilityResponse) Reset() { @@ -2398,6 +2407,13 @@ func (x *ForwardingAbilityResponse) GetUptimeThreshold() float64 { return 0 } +func (x *ForwardingAbilityResponse) GetUnobservedS() uint64 { + if x != nil { + return x.UnobservedS + } + return 0 +} + type ForwardingAbilityEntry struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2408,7 +2424,7 @@ type ForwardingAbilityEntry struct { // 65535 peers per direction. PackedIdx uint32 `protobuf:"varint,1,opt,name=packed_idx,json=packedIdx,proto3" json:"packed_idx,omitempty"` // Seconds the peer pair held at least the requested liquidity floor of - // directional forwardable liquidity over the window. + // directional forwardable liquidity over the observed window. EffectiveUptimeS uint64 `protobuf:"varint,2,opt,name=effective_uptime_s,json=effectiveUptimeS,proto3" json:"effective_uptime_s,omitempty"` // Total successfully forwarded amount over the window, in millisatoshis. ForwardedMsat int64 `protobuf:"varint,3,opt,name=forwarded_msat,json=forwardedMsat,proto3" json:"forwarded_msat,omitempty"` @@ -2764,7 +2780,7 @@ var file_faraday_proto_rawDesc = []byte{ 0x6c, 0x69, 0x71, 0x75, 0x69, 0x64, 0x69, 0x74, 0x79, 0x46, 0x6c, 0x6f, 0x6f, 0x72, 0x53, 0x61, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0f, 0x75, 0x70, 0x74, - 0x69, 0x6d, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x22, 0xff, 0x01, 0x0a, + 0x69, 0x6d, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x22, 0xa2, 0x02, 0x0a, 0x19, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, @@ -2780,114 +2796,116 @@ var file_faraday_proto_rawDesc = []byte{ 0x0c, 0x52, 0x10, 0x75, 0x70, 0x42, 0x75, 0x74, 0x49, 0x64, 0x6c, 0x65, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x73, 0x6b, 0x12, 0x29, 0x0a, 0x10, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0f, 0x75, - 0x70, 0x74, 0x69, 0x6d, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x22, 0xc3, - 0x01, 0x0a, 0x16, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, - 0x6c, 0x69, 0x74, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x63, - 0x6b, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x70, - 0x61, 0x63, 0x6b, 0x65, 0x64, 0x49, 0x64, 0x78, 0x12, 0x2c, 0x0a, 0x12, 0x65, 0x66, 0x66, 0x65, - 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x55, - 0x70, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, - 0x64, 0x65, 0x64, 0x5f, 0x6d, 0x73, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, - 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x4d, 0x73, 0x61, 0x74, 0x12, 0x19, 0x0a, - 0x08, 0x66, 0x65, 0x65, 0x5f, 0x6d, 0x73, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x07, 0x66, 0x65, 0x65, 0x4d, 0x73, 0x61, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x6f, 0x72, 0x77, - 0x61, 0x72, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x66, 0x6f, 0x72, 0x77, - 0x61, 0x72, 0x64, 0x73, 0x2a, 0xa1, 0x01, 0x0a, 0x0b, 0x47, 0x72, 0x61, 0x6e, 0x75, 0x6c, 0x61, - 0x72, 0x69, 0x74, 0x79, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, - 0x47, 0x52, 0x41, 0x4e, 0x55, 0x4c, 0x41, 0x52, 0x49, 0x54, 0x59, 0x10, 0x00, 0x12, 0x0a, 0x0a, - 0x06, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x46, 0x49, 0x56, - 0x45, 0x5f, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, 0x53, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x46, - 0x49, 0x46, 0x54, 0x45, 0x45, 0x4e, 0x5f, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, 0x53, 0x10, 0x03, - 0x12, 0x12, 0x0a, 0x0e, 0x54, 0x48, 0x49, 0x52, 0x54, 0x59, 0x5f, 0x4d, 0x49, 0x4e, 0x55, 0x54, - 0x45, 0x53, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x4f, 0x55, 0x52, 0x10, 0x05, 0x12, 0x0d, - 0x0a, 0x09, 0x53, 0x49, 0x58, 0x5f, 0x48, 0x4f, 0x55, 0x52, 0x53, 0x10, 0x06, 0x12, 0x10, 0x0a, - 0x0c, 0x54, 0x57, 0x45, 0x4c, 0x56, 0x45, 0x5f, 0x48, 0x4f, 0x55, 0x52, 0x53, 0x10, 0x07, 0x12, - 0x07, 0x0a, 0x03, 0x44, 0x41, 0x59, 0x10, 0x08, 0x2a, 0x6a, 0x0a, 0x0b, 0x46, 0x69, 0x61, 0x74, - 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, - 0x57, 0x4e, 0x5f, 0x46, 0x49, 0x41, 0x54, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44, 0x10, 0x00, - 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x49, 0x4e, 0x43, 0x41, 0x50, 0x10, 0x01, 0x12, 0x0c, 0x0a, - 0x08, 0x43, 0x4f, 0x49, 0x4e, 0x44, 0x45, 0x53, 0x4b, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x43, - 0x55, 0x53, 0x54, 0x4f, 0x4d, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x49, 0x4e, 0x47, - 0x45, 0x43, 0x4b, 0x4f, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x49, 0x54, 0x46, 0x49, 0x4e, - 0x45, 0x58, 0x10, 0x05, 0x2a, 0xa2, 0x02, 0x0a, 0x09, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, - 0x16, 0x0a, 0x12, 0x4c, 0x4f, 0x43, 0x41, 0x4c, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, - 0x5f, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x52, 0x45, 0x4d, 0x4f, 0x54, - 0x45, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x02, - 0x12, 0x14, 0x0a, 0x10, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x4f, 0x50, 0x45, 0x4e, - 0x5f, 0x46, 0x45, 0x45, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, - 0x4c, 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x04, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x45, 0x43, - 0x45, 0x49, 0x50, 0x54, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x41, 0x59, 0x4d, 0x45, 0x4e, - 0x54, 0x10, 0x06, 0x12, 0x07, 0x0a, 0x03, 0x46, 0x45, 0x45, 0x10, 0x07, 0x12, 0x14, 0x0a, 0x10, - 0x43, 0x49, 0x52, 0x43, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x52, 0x45, 0x43, 0x45, 0x49, 0x50, 0x54, - 0x10, 0x08, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x4f, 0x52, 0x57, 0x41, 0x52, 0x44, 0x10, 0x09, 0x12, - 0x0f, 0x0a, 0x0b, 0x46, 0x4f, 0x52, 0x57, 0x41, 0x52, 0x44, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x0a, - 0x12, 0x14, 0x0a, 0x10, 0x43, 0x49, 0x52, 0x43, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x50, 0x41, 0x59, - 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x0b, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x49, 0x52, 0x43, 0x55, 0x4c, - 0x41, 0x52, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x0c, 0x12, 0x09, 0x0a, 0x05, 0x53, 0x57, 0x45, 0x45, - 0x50, 0x10, 0x0d, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x57, 0x45, 0x45, 0x50, 0x5f, 0x46, 0x45, 0x45, - 0x10, 0x0e, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x43, 0x4c, - 0x4f, 0x53, 0x45, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x0f, 0x2a, 0x70, 0x0a, 0x10, 0x43, 0x68, 0x61, - 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, - 0x12, 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, - 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, - 0x45, 0x4e, 0x54, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, - 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4f, 0x46, 0x46, 0x4c, 0x49, - 0x4e, 0x45, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, 0x45, - 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x03, 0x32, 0x83, 0x06, 0x0a, 0x0d, - 0x46, 0x61, 0x72, 0x61, 0x64, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x65, 0x0a, - 0x16, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, - 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x25, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, - 0x2e, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, - 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, - 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x63, - 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x69, 0x0a, 0x18, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, - 0x64, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x27, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, - 0x6f, 0x6c, 0x64, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x66, 0x72, 0x64, 0x72, - 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, - 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4c, 0x0a, 0x0d, 0x52, 0x65, 0x76, 0x65, 0x6e, 0x75, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, - 0x12, 0x1c, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x76, 0x65, 0x6e, 0x75, - 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, - 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x76, 0x65, 0x6e, 0x75, 0x65, 0x52, - 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x52, 0x0a, - 0x0f, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, - 0x12, 0x1e, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, - 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1f, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, - 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x49, 0x0a, 0x0c, 0x45, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x61, 0x74, - 0x65, 0x12, 0x1b, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x63, 0x68, 0x61, - 0x6e, 0x67, 0x65, 0x52, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, - 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, - 0x52, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x09, - 0x4e, 0x6f, 0x64, 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x12, 0x18, 0x2e, 0x66, 0x72, 0x64, 0x72, - 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, - 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, - 0x0a, 0x0b, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x1a, 0x2e, - 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x70, 0x6f, - 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x66, 0x72, 0x64, 0x72, - 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, - 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1c, 0x2e, 0x66, 0x72, 0x64, - 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, - 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x11, 0x46, 0x6f, 0x72, 0x77, 0x61, - 0x72, 0x64, 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x20, 0x2e, 0x66, - 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, - 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, - 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, - 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x6c, 0x69, 0x67, 0x68, 0x74, 0x6e, 0x69, 0x6e, 0x67, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x66, 0x61, - 0x72, 0x61, 0x64, 0x61, 0x79, 0x2f, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x70, 0x74, 0x69, 0x6d, 0x65, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x21, + 0x0a, 0x0c, 0x75, 0x6e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x5f, 0x73, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x75, 0x6e, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, + 0x53, 0x22, 0xc3, 0x01, 0x0a, 0x16, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, + 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x1d, 0x0a, 0x0a, + 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x5f, 0x69, 0x64, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x49, 0x64, 0x78, 0x12, 0x2c, 0x0a, 0x12, 0x65, + 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x5f, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, + 0x76, 0x65, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x6f, 0x72, + 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x5f, 0x6d, 0x73, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0d, 0x66, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x4d, 0x73, 0x61, 0x74, + 0x12, 0x19, 0x0a, 0x08, 0x66, 0x65, 0x65, 0x5f, 0x6d, 0x73, 0x61, 0x74, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x07, 0x66, 0x65, 0x65, 0x4d, 0x73, 0x61, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x66, + 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x66, + 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x73, 0x2a, 0xa1, 0x01, 0x0a, 0x0b, 0x47, 0x72, 0x61, 0x6e, + 0x75, 0x6c, 0x61, 0x72, 0x69, 0x74, 0x79, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, + 0x57, 0x4e, 0x5f, 0x47, 0x52, 0x41, 0x4e, 0x55, 0x4c, 0x41, 0x52, 0x49, 0x54, 0x59, 0x10, 0x00, + 0x12, 0x0a, 0x0a, 0x06, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, + 0x46, 0x49, 0x56, 0x45, 0x5f, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, 0x53, 0x10, 0x02, 0x12, 0x13, + 0x0a, 0x0f, 0x46, 0x49, 0x46, 0x54, 0x45, 0x45, 0x4e, 0x5f, 0x4d, 0x49, 0x4e, 0x55, 0x54, 0x45, + 0x53, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x54, 0x48, 0x49, 0x52, 0x54, 0x59, 0x5f, 0x4d, 0x49, + 0x4e, 0x55, 0x54, 0x45, 0x53, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x4f, 0x55, 0x52, 0x10, + 0x05, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x49, 0x58, 0x5f, 0x48, 0x4f, 0x55, 0x52, 0x53, 0x10, 0x06, + 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x57, 0x45, 0x4c, 0x56, 0x45, 0x5f, 0x48, 0x4f, 0x55, 0x52, 0x53, + 0x10, 0x07, 0x12, 0x07, 0x0a, 0x03, 0x44, 0x41, 0x59, 0x10, 0x08, 0x2a, 0x6a, 0x0a, 0x0b, 0x46, + 0x69, 0x61, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x4e, + 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x46, 0x49, 0x41, 0x54, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, + 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x49, 0x4e, 0x43, 0x41, 0x50, 0x10, 0x01, + 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x4f, 0x49, 0x4e, 0x44, 0x45, 0x53, 0x4b, 0x10, 0x02, 0x12, 0x0a, + 0x0a, 0x06, 0x43, 0x55, 0x53, 0x54, 0x4f, 0x4d, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, + 0x49, 0x4e, 0x47, 0x45, 0x43, 0x4b, 0x4f, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x49, 0x54, + 0x46, 0x49, 0x4e, 0x45, 0x58, 0x10, 0x05, 0x2a, 0xa2, 0x02, 0x0a, 0x09, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, + 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4c, 0x4f, 0x43, 0x41, 0x4c, 0x5f, 0x43, 0x48, 0x41, 0x4e, + 0x4e, 0x45, 0x4c, 0x5f, 0x4f, 0x50, 0x45, 0x4e, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x52, 0x45, + 0x4d, 0x4f, 0x54, 0x45, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x4f, 0x50, 0x45, + 0x4e, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x4f, + 0x50, 0x45, 0x4e, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x48, 0x41, + 0x4e, 0x4e, 0x45, 0x4c, 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x04, 0x12, 0x0b, 0x0a, 0x07, + 0x52, 0x45, 0x43, 0x45, 0x49, 0x50, 0x54, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x41, 0x59, + 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x06, 0x12, 0x07, 0x0a, 0x03, 0x46, 0x45, 0x45, 0x10, 0x07, 0x12, + 0x14, 0x0a, 0x10, 0x43, 0x49, 0x52, 0x43, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x52, 0x45, 0x43, 0x45, + 0x49, 0x50, 0x54, 0x10, 0x08, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x4f, 0x52, 0x57, 0x41, 0x52, 0x44, + 0x10, 0x09, 0x12, 0x0f, 0x0a, 0x0b, 0x46, 0x4f, 0x52, 0x57, 0x41, 0x52, 0x44, 0x5f, 0x46, 0x45, + 0x45, 0x10, 0x0a, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x49, 0x52, 0x43, 0x55, 0x4c, 0x41, 0x52, 0x5f, + 0x50, 0x41, 0x59, 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x0b, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x49, 0x52, + 0x43, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x0c, 0x12, 0x09, 0x0a, 0x05, 0x53, + 0x57, 0x45, 0x45, 0x50, 0x10, 0x0d, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x57, 0x45, 0x45, 0x50, 0x5f, + 0x46, 0x45, 0x45, 0x10, 0x0e, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, 0x4e, 0x45, 0x4c, + 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x5f, 0x46, 0x45, 0x45, 0x10, 0x0f, 0x2a, 0x70, 0x0a, 0x10, + 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x16, 0x0a, 0x12, 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, + 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, + 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, + 0x16, 0x0a, 0x12, 0x43, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4f, 0x46, + 0x46, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x48, 0x41, 0x4e, 0x5f, + 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x03, 0x32, 0x83, + 0x06, 0x0a, 0x0d, 0x46, 0x61, 0x72, 0x61, 0x64, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x12, 0x65, 0x0a, 0x16, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x6d, + 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x25, 0x2e, 0x66, 0x72, 0x64, + 0x72, 0x70, 0x63, 0x2e, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x6d, + 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x24, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, + 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x69, 0x0a, 0x18, 0x54, 0x68, 0x72, 0x65, 0x73, + 0x68, 0x6f, 0x6c, 0x64, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x27, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x54, 0x68, 0x72, + 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x66, + 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x6d, + 0x6d, 0x65, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0d, 0x52, 0x65, 0x76, 0x65, 0x6e, 0x75, 0x65, 0x52, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x12, 0x1c, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x76, + 0x65, 0x6e, 0x75, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x76, 0x65, 0x6e, + 0x75, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x52, 0x0a, 0x0f, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, + 0x68, 0x74, 0x73, 0x12, 0x1e, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x73, 0x69, 0x67, 0x68, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x0c, 0x45, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, + 0x52, 0x61, 0x74, 0x65, 0x12, 0x1b, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, + 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1c, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x63, 0x68, 0x61, + 0x6e, 0x67, 0x65, 0x52, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x40, 0x0a, 0x09, 0x4e, 0x6f, 0x64, 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x12, 0x18, 0x2e, 0x66, + 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, + 0x4e, 0x6f, 0x64, 0x65, 0x41, 0x75, 0x64, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x46, 0x0a, 0x0b, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x12, 0x1a, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, + 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x66, + 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x72, + 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x10, 0x47, 0x65, 0x74, + 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1c, 0x2e, + 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x72, + 0x64, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x11, 0x46, 0x6f, + 0x72, 0x77, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, + 0x20, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, + 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, + 0x72, 0x64, 0x69, 0x6e, 0x67, 0x41, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6c, 0x69, 0x67, 0x68, 0x74, 0x6e, 0x69, 0x6e, 0x67, 0x6c, 0x61, 0x62, 0x73, + 0x2f, 0x66, 0x61, 0x72, 0x61, 0x64, 0x61, 0x79, 0x2f, 0x66, 0x72, 0x64, 0x72, 0x70, 0x63, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/frdrpc/faraday.proto b/frdrpc/faraday.proto index fc67cb4..8b75b0e 100644 --- a/frdrpc/faraday.proto +++ b/frdrpc/faraday.proto @@ -696,9 +696,10 @@ message ForwardingAbilityRequest { // this constant across calls for comparable series. uint64 liquidity_floor_sat = 3; - // The uptime fraction in [0, 1] at or above which a peer pair that did not - // forward is reported compactly as a single bit in up_but_idle_bitmask - // rather than as a full entry. A value of 0 selects the server default. + // The uptime fraction in [0, 1] of the observed window at or above which a + // peer pair that did not forward is reported compactly as a single bit in + // up_but_idle_bitmask rather than as a full entry. The observed window is + // the window minus unobserved_s. A value of 0 selects the server default. // Pairs below this threshold that also did not forward are omitted // entirely. If no pair meets the threshold the server returns a // FailedPrecondition error, since that indicates the node itself was down @@ -725,15 +726,24 @@ message ForwardingAbilityResponse { // A packed bitmask over the n*n ordered peer pairs, where n is the length // of peers. The bit at index in*n+out is set when that pair held at least - // the uptime_threshold fraction of effective uptime over the window but - // carried no forward: the dense "up but idle" population. A pair with an - // entry is never flagged here. A pair that is neither listed in entries nor - // flagged here had sub-threshold uptime and no forwards: treat it as zero. + // the uptime_threshold fraction of effective uptime over the observed + // window but carried no forward: the dense "up but idle" population. A pair + // with an entry is never flagged here. A pair that is neither listed in + // entries nor flagged here had sub-threshold uptime and no forwards: treat + // it as zero. bytes up_but_idle_bitmask = 5; - // The uptime fraction in [0, 1] used to populate up_but_idle_bitmask, - // echoed back so consumers know the threshold the server applied. + // The uptime fraction in [0, 1] of the observed window used to populate + // up_but_idle_bitmask, echoed back so consumers know the threshold the + // server applied. double uptime_threshold = 6; + + // The seconds of the window in which faraday did not observe lnd, for + // example while faraday or lnd was down. Effective uptime excludes this + // time, so clients divide effective uptime by + // end_time - start_time - unobserved_s. A server that does not report it + // leaves it at 0. + uint64 unobserved_s = 7; } message ForwardingAbilityEntry { @@ -743,7 +753,7 @@ message ForwardingAbilityEntry { uint32 packed_idx = 1; // Seconds the peer pair held at least the requested liquidity floor of - // directional forwardable liquidity over the window. + // directional forwardable liquidity over the observed window. uint64 effective_uptime_s = 2; // Total successfully forwarded amount over the window, in millisatoshis. diff --git a/frdrpc/faraday.swagger.json b/frdrpc/faraday.swagger.json index 9b4a4e2..7ec67aa 100644 --- a/frdrpc/faraday.swagger.json +++ b/frdrpc/faraday.swagger.json @@ -1036,7 +1036,7 @@ "effective_uptime_s": { "type": "string", "format": "uint64", - "description": "Seconds the peer pair held at least the requested liquidity floor of\ndirectional forwardable liquidity over the window." + "description": "Seconds the peer pair held at least the requested liquidity floor of\ndirectional forwardable liquidity over the observed window." }, "forwarded_msat": { "type": "string", @@ -1087,12 +1087,17 @@ "up_but_idle_bitmask": { "type": "string", "format": "byte", - "description": "A packed bitmask over the n*n ordered peer pairs, where n is the length\nof peers. The bit at index in*n+out is set when that pair held at least\nthe uptime_threshold fraction of effective uptime over the window but\ncarried no forward: the dense \"up but idle\" population. A pair with an\nentry is never flagged here. A pair that is neither listed in entries nor\nflagged here had sub-threshold uptime and no forwards: treat it as zero." + "description": "A packed bitmask over the n*n ordered peer pairs, where n is the length\nof peers. The bit at index in*n+out is set when that pair held at least\nthe uptime_threshold fraction of effective uptime over the observed\nwindow but carried no forward: the dense \"up but idle\" population. A pair\nwith an entry is never flagged here. A pair that is neither listed in\nentries nor flagged here had sub-threshold uptime and no forwards: treat\nit as zero." }, "uptime_threshold": { "type": "number", "format": "double", - "description": "The uptime fraction in [0, 1] used to populate up_but_idle_bitmask,\nechoed back so consumers know the threshold the server applied." + "description": "The uptime fraction in [0, 1] of the observed window used to populate\nup_but_idle_bitmask, echoed back so consumers know the threshold the\nserver applied." + }, + "unobserved_s": { + "type": "string", + "format": "uint64", + "description": "The seconds of the window in which faraday did not observe lnd, for\nexample while faraday or lnd was down. Effective uptime excludes this\ntime, so clients divide effective uptime by\nend_time - start_time - unobserved_s. A server that does not report it\nleaves it at 0." } } }, diff --git a/frdrpc/forwarding_ability_codec.go b/frdrpc/forwarding_ability_codec.go index 7f21aff..773340b 100644 --- a/frdrpc/forwarding_ability_codec.go +++ b/frdrpc/forwarding_ability_codec.go @@ -20,10 +20,11 @@ const maxPackedPeers = 1<<16 - 1 // categories are left to the consumer. type ForwardingAbility struct { // EffectiveUptimeS is the seconds the pair held at least the requested - // liquidity floor of directional forwardable liquidity over the window. - // The value is whole seconds: sub-second uptime floors to zero, so a - // pair that forwarded volume over a fleeting qualifying window can - // report a zero uptime alongside a non-zero ForwardedMsat. + // liquidity floor of directional forwardable liquidity over the + // observed window. The value is whole seconds: sub-second uptime + // floors to zero, so a pair that forwarded volume over a fleeting + // qualifying window can report a zero uptime alongside a non-zero + // ForwardedMsat. EffectiveUptimeS uint64 // ForwardedMsat is the total successfully forwarded amount over the @@ -104,6 +105,18 @@ func MinQualifyingUptime(threshold float64, windowSeconds int64) uint64 { return v } +// ObservedWindow returns the seconds of [startTime, endTime) in which faraday +// observed lnd. It is zero when the unobserved time covers the whole window or +// the window is empty. +func ObservedWindow(startTime, endTime int64, unobservedS uint64) int64 { + window := endTime - startTime + if window <= 0 || unobservedS >= uint64(window) { + return 0 + } + + return window - int64(unobservedS) +} + // setBit sets the bit at the given index in a packed bitmask. The index is an // int64 because an n*n bitmask over the full peer set overflows a 32-bit int. func setBit(mask []byte, index int64) { @@ -121,14 +134,16 @@ func getBit(mask []byte, index int64) bool { // into a memory-efficient sparse gRPC response over [startTime, endTime]. To // optimize payload size it tiers each pair: pairs that carried at least one // forward keep a full entry, pairs that were up at least uptimeThreshold of the -// window but carried none collapse to a single bit in the up-but-idle bitmask, -// and pairs below the threshold that carried none are omitted entirely. Public -// keys are deduplicated and peer pairs packed into 32-bit indices. +// observed window but carried none collapse to a single bit in the up-but-idle +// bitmask, and pairs below the threshold that carried none are omitted +// entirely. The observed window is the window minus unobservedS. Public keys +// are deduplicated and peer pairs packed into 32-bit indices. func EncodeForwardingAbility(abilities map[string]map[string]ForwardingAbility, - startTime, endTime int64, + startTime, endTime int64, unobservedS uint64, uptimeThreshold float64) (*ForwardingAbilityResponse, error) { - minUptimeS := MinQualifyingUptime(uptimeThreshold, endTime-startTime) + observedWindow := ObservedWindow(startTime, endTime, unobservedS) + minUptimeS := MinQualifyingUptime(uptimeThreshold, observedWindow) // First, find all unique peers involved in pairs that warrant either an // entry or a bit. Keys are normalized to lower-case hex so a peer that @@ -265,15 +280,16 @@ func EncodeForwardingAbility(abilities map[string]map[string]ForwardingAbility, EndTime: endTime, UpButIdleBitmask: bitmask, UptimeThreshold: uptimeThreshold, + UnobservedS: unobservedS, }, nil } // DecodeForwardingAbility reconstructs the nested map of peer forwarding // abilities from a sparse packed gRPC response. Forwarded pairs come back with -// their exact facts; up-but-idle pairs flagged in the bitmask come back at full -// window uptime with no forwards, and therefore no volume and no fees. It -// validates packed indices and the bitmask length against the decoded peer list -// to prevent out-of-bounds errors. +// their exact facts; up-but-idle pairs flagged in the bitmask come back at +// full observed window uptime with no forwards, and therefore no volume and no +// fees. It validates packed indices and the bitmask length against the decoded +// peer list to prevent out-of-bounds errors. func DecodeForwardingAbility(resp *ForwardingAbilityResponse) ( map[string]map[string]ForwardingAbility, error) { @@ -342,19 +358,15 @@ func DecodeForwardingAbility(resp *ForwardingAbilityResponse) ( numPeers) } - // Up-but-idle pairs were up the whole window by definition of the - // threshold bucket, so reconstruct them at full window uptime with no + // Up-but-idle pairs were up the whole observed window by definition of + // the threshold bucket, so reconstruct them at that uptime with no // forwards, and therefore no volume and no fees. Iterate over the // bitmask bytes directly, skipping zero bytes, so cost scales with the // number of set bits rather than the O(n*n) pair space; padding bits // beyond n*n are ignored. - // The window is a difference of signed unix timestamps, so an end - // before the start yields zero uptime rather than wrapping into a - // near-infinite one. - var windowSeconds uint64 - if d := resp.EndTime - resp.StartTime; d > 0 { - windowSeconds = uint64(d) - } + windowSeconds := uint64(ObservedWindow( + resp.StartTime, resp.EndTime, resp.UnobservedS, + )) for i, b := range bitmask { if b == 0 { continue diff --git a/frdrpc/forwarding_ability_codec_test.go b/frdrpc/forwarding_ability_codec_test.go index d247093..5c5f2e5 100644 --- a/frdrpc/forwarding_ability_codec_test.go +++ b/frdrpc/forwarding_ability_codec_test.go @@ -215,7 +215,7 @@ func TestForwardingAbilityCodecRoundTrip(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { resp, err := EncodeForwardingAbility( - tc.abilities, startTime, endTime, threshold, + tc.abilities, startTime, endTime, 0, threshold, ) require.NoError(t, err) require.Equal(t, startTime, resp.StartTime) @@ -282,7 +282,7 @@ func TestForwardingAbilityCodecZeroValueForward(t *testing.T) { } resp, err := EncodeForwardingAbility( - abilities, startTime, endTime, threshold, + abilities, startTime, endTime, 0, threshold, ) require.NoError(t, err) @@ -306,6 +306,46 @@ func TestForwardingAbilityCodecZeroValueForward(t *testing.T) { ) } +// TestForwardingAbilityCodecObservedWindow verifies that the threshold applies +// to the observed window, which is the window minus the unobserved time, and +// that a decoded bit carries the observed window as its uptime. The window is +// [0, 100) with 40 unobserved seconds and the threshold 0.5, so the minimum +// qualifying uptime is 30 seconds instead of 50. +func TestForwardingAbilityCodecObservedWindow(t *testing.T) { + const ( + startTime, endTime int64 = 0, 100 + unobservedS uint64 = 40 + threshold float64 = 0.5 + ) + + abilities := map[string]map[string]ForwardingAbility{ + fwdKey(1): { + // Meets the threshold of the observed window only. + fwdKey(2): {EffectiveUptimeS: 40}, + }, + fwdKey(2): { + // Misses the threshold of the observed window. + fwdKey(1): {EffectiveUptimeS: 20}, + }, + } + + resp, err := EncodeForwardingAbility( + abilities, startTime, endTime, unobservedS, threshold, + ) + require.NoError(t, err) + require.Equal(t, unobservedS, resp.UnobservedS) + require.Empty(t, resp.Entries) + require.NotEmpty(t, resp.UpButIdleBitmask) + + decoded, err := DecodeForwardingAbility(resp) + require.NoError(t, err) + require.Equal( + t, map[string]map[string]ForwardingAbility{ + fwdKey(1): {fwdKey(2): {EffectiveUptimeS: 60}}, + }, decoded, + ) +} + // TestMinQualifyingUptime verifies the threshold-to-seconds conversion shared // by the encoder and the server guard, including its boundary behavior. func TestMinQualifyingUptime(t *testing.T) { @@ -361,6 +401,29 @@ func TestMinQualifyingUptime(t *testing.T) { } } +// TestObservedWindow verifies that the observed window is the window minus the +// unobserved time, and that it never goes below zero. +func TestObservedWindow(t *testing.T) { + tests := []struct { + name string + startTime, endTime int64 + unobservedS uint64 + want int64 + }{ + {"subtracts unobserved time", 0, 100, 40, 60}, + {"unobserved time exceeds window", 0, 100, 150, 0}, + {"end before start", 100, 0, 0, 0}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, ObservedWindow( + tc.startTime, tc.endTime, tc.unobservedS, + )) + }) + } +} + // TestBitmaskHelpers verifies that setBit and getBit address the same bit. func TestBitmaskHelpers(t *testing.T) { mask := make([]byte, 2) @@ -483,7 +546,7 @@ func TestForwardingAbilityEncodePeerCap(t *testing.T) { fwdKey(0): outMap, } - _, err := EncodeForwardingAbility(abilities, 0, 1, 0.5) + _, err := EncodeForwardingAbility(abilities, 0, 1, 0, 0.5) require.ErrorContains(t, err, "exceeds") } @@ -510,7 +573,7 @@ func TestForwardingAbilityEncodeNormalizesCase(t *testing.T) { }, } - resp, err := EncodeForwardingAbility(abilities, 0, 1, 0.5) + resp, err := EncodeForwardingAbility(abilities, 0, 1, 0, 0.5) require.NoError(t, err) // The upper- and lower-case forms of the shared peer must dedup to one @@ -559,7 +622,7 @@ func TestForwardingAbilityEncodeRejectsCaseCollision(t *testing.T) { }, } - _, err := EncodeForwardingAbility(abilities, 0, 1, 0.5) + _, err := EncodeForwardingAbility(abilities, 0, 1, 0, 0.5) require.Error(t, err) } @@ -607,7 +670,7 @@ func TestForwardingAbilityCodecRoundTripHighIndices(t *testing.T) { } resp, err := EncodeForwardingAbility( - abilities, startTime, endTime, threshold, + abilities, startTime, endTime, 0, threshold, ) require.NoError(t, err) require.Len(t, resp.Peers, numPeers) diff --git a/frdrpcserver/forwarding_ability.go b/frdrpcserver/forwarding_ability.go index 4cb4dff..453f28e 100644 --- a/frdrpcserver/forwarding_ability.go +++ b/frdrpcserver/forwarding_ability.go @@ -155,7 +155,7 @@ func (s *RPCServer) ForwardingAbility(ctx context.Context, } resp, err := frdrpc.EncodeForwardingAbility( - nested, startTime.Unix(), endTime.Unix(), uptimeThreshold, + nested, startTime.Unix(), endTime.Unix(), 0, uptimeThreshold, ) if err != nil { log.ErrorS( From b50a55bad05b84ec8e5c004c7637e1db288f2e20 Mon Sep 17 00:00:00 2001 From: bitromortac Date: Fri, 30 Oct 2026 11:23:44 +0000 Subject: [PATCH 3/5] multi: drop unobserved time from forwarding uptime The analyzer replayed the last known channel state across node outages, so an outage counted as uptime for every pair that was routable before it. The walk now skips unobserved intervals, and EffectiveUptime returns their total. The server reports the total as unobserved_s, and the node-down guard uses the observed window. The server rounds the unobserved time up, because uptime rounds down. --- chanevents/analyzer.go | 110 ++++++++++++--- chanevents/analyzer_test.go | 174 +++++++++++++++++++++++- frdrpcserver/forwarding_ability.go | 19 ++- frdrpcserver/forwarding_ability_test.go | 49 ++++++- frdrpcserver/rpcserver.go | 7 +- 5 files changed, 322 insertions(+), 37 deletions(-) diff --git a/chanevents/analyzer.go b/chanevents/analyzer.go index fac591a..f30626d 100644 --- a/chanevents/analyzer.go +++ b/chanevents/analyzer.go @@ -52,6 +52,11 @@ type EventsSource interface { // ScidToPeerMap returns the historically recorded scid→peer index, // including closed channels. ScidToPeerMap(ctx context.Context) (map[uint64]string, error) + + // UnobservedIntervals returns the sorted, disjoint unobserved + // intervals that overlap [start, end), clipped to that range. + UnobservedIntervals(ctx context.Context, start, + end time.Time) ([]UnobservedInterval, error) } // ForwardingAnalyzer computes forwarding velocity and effective uptime for @@ -72,7 +77,7 @@ type channelEventSeq = iter.Seq2[*ChannelEvent, error] // uptime) from EffectiveUptime and ForwardedMsat. type ForwardingAbility struct { // EffectiveUptime is the time the pair held at least the liquidity floor - // of directional forwardable liquidity over the window. + // of directional forwardable liquidity over the observed window. EffectiveUptime time.Duration // ForwardedMsat is the total successfully forwarded amount over the @@ -130,13 +135,16 @@ func NewForwardingAnalyzer(store EventsSource, } // EffectiveUptime returns a ForwardingAbility for every (peerIn, peerOut) pair -// over [startTime, endTime). Closed channels are folded into the considered set -// so survivorship bias does not skew the uptime denominator. A single -// liquidityFloor is applied uniformly to every pair, so effective uptime is the -// time each pair held at least that much directional forwardable liquidity. +// over [startTime, endTime), and the unobserved time within that window. +// Effective uptime only covers observed time, so a client divides it by the +// window minus the unobserved time. Closed channels are folded into the +// considered set so survivorship bias does not skew the uptime denominator. A +// single liquidityFloor is applied uniformly to every pair, so effective uptime +// is the time each pair held at least that much directional forwardable +// liquidity. func (a *ForwardingAnalyzer) EffectiveUptime(ctx context.Context, startTime, endTime time.Time, liquidityFloor btcutil.Amount) ( - map[PeerPair]ForwardingAbility, error) { + map[PeerPair]ForwardingAbility, time.Duration, error) { log.DebugS( ctx, "Calculating effective uptime", @@ -147,7 +155,7 @@ func (a *ForwardingAnalyzer) EffectiveUptime(ctx context.Context, startTime, scidToPeer, err := a.store.ScidToPeerMap(ctx) if err != nil { - return nil, err + return nil, 0, err } log.DebugS( ctx, "Found historical channels", @@ -158,7 +166,7 @@ func (a *ForwardingAnalyzer) EffectiveUptime(ctx context.Context, startTime, ctx, startTime, endTime, scidToPeer, ) if err != nil { - return nil, err + return nil, 0, err } log.DebugS( ctx, "Found peer pairs with successful forwards", @@ -167,24 +175,44 @@ func (a *ForwardingAnalyzer) EffectiveUptime(ctx context.Context, startTime, err = a.addActiveChannels(ctx, channelPeersConsidered) if err != nil { - return nil, err + return nil, 0, err } peerChannels, initialStates, err := a.getPeerChannelData( ctx, startTime, channelPeersConsidered, ) if err != nil { - return nil, err + return nil, 0, err } log.DebugS( ctx, "Identified channels for peers", slog.Int("count", len(peerChannels)), ) - return calculateAllPairsUptime( - ctx, a.store, startTime, endTime, liquidityFloor, + unobserved, err := a.store.UnobservedIntervals(ctx, startTime, endTime) + if err != nil { + return nil, 0, err + } + + var unobservedTotal time.Duration + for _, interval := range unobserved { + unobservedTotal += interval.End.Sub(interval.Start) + } + log.DebugS( + ctx, "Found unobserved intervals", + slog.Int("count", len(unobserved)), + slog.Duration("total", unobservedTotal), + ) + + abilities, err := calculateAllPairsUptime( + ctx, a.store, startTime, endTime, unobserved, liquidityFloor, successfulForwards, initialStates, peerChannels, ) + if err != nil { + return nil, 0, err + } + + return abilities, unobservedTotal, nil } // getForwardingData queries lnd's forwarding history sequentially in paginated @@ -456,7 +484,8 @@ func (a *ForwardingAnalyzer) getInitialChannelState(ctx context.Context, // calculateAllPairsUptime returns forwarding abilities for every peer pair, // computing both directions (A→B and B→A) in a single pass. func calculateAllPairsUptime(ctx context.Context, store EventsSource, startTime, - endTime time.Time, liquidityFloor btcutil.Amount, + endTime time.Time, unobserved []UnobservedInterval, + liquidityFloor btcutil.Amount, successfulForwards map[PeerPair]*pairForwards, initialStates map[string]map[int64]*channelState, peerChannels map[string][]int64) ( @@ -554,7 +583,7 @@ func calculateAllPairsUptime(ctx context.Context, store EventsSource, startTime, abilityAB, abilityBA, err := calculateBothDirectionsUptime( ctx, startTime, endTime, - liquidityFloor, + unobserved, liquidityFloor, statesA, statesB, sumsA.remote, sumsA.local, sumsB.remote, sumsB.local, @@ -643,12 +672,14 @@ func pairForwardTotals(successfulForwards map[PeerPair]*pairForwards, // calculateBothDirectionsUptime computes the effective forwarding uptime for // both directions of a peer pair in a single chronological walk of the merged // event stream. Only the liquidity-direction roles differ between the two -// accumulators, as both share the same uniform liquidityFloor. totalsAB and +// accumulators, as both share the same uniform liquidityFloor. Time inside the +// sorted, disjoint unobserved intervals does not count as uptime. totalsAB and // totalsBA carry each direction's forwarded volume, fees and count through to // the returned abilities. For self-pair calls both returned abilities are // equal. func calculateBothDirectionsUptime(ctx context.Context, startTime, - endTime time.Time, liquidityFloor btcutil.Amount, statesA, + endTime time.Time, unobserved []UnobservedInterval, + liquidityFloor btcutil.Amount, statesA, statesB map[int64]*channelState, sumARemote, sumALocal, sumBRemote, sumBLocal btcutil.Amount, mergedEvents channelEventSeq, totalsAB, totalsBA pairForwards) ( @@ -703,8 +734,12 @@ func calculateBothDirectionsUptime(ctx context.Context, startTime, var uptimeAB, uptimeBA time.Duration lastTimestamp := startTime + var cursor int - accumulate := func(intervalDuration time.Duration) { + accumulate := func(from, to time.Time) { + intervalDuration := observedDuration( + from, to, unobserved, &cursor, + ) if intervalDuration <= 0 { return } @@ -759,7 +794,7 @@ func calculateBothDirectionsUptime(ctx context.Context, startTime, // event, based on the state of the channels during that // interval. The events are ordered chronologically so the state // is consistent with the entire interval. - accumulate(event.Timestamp.Sub(lastTimestamp)) + accumulate(lastTimestamp, event.Timestamp) // Update the state for each peer if the event affects one of // their channels. Before applying the event, we remove the @@ -767,7 +802,7 @@ func calculateBothDirectionsUptime(ctx context.Context, startTime, // because the event may change the channel's online status or // balances in a way that affects the sums. if state, ok := statesA[event.ChannelID]; ok { - // We would have inlcuded the channel's balances in the + // We would have included the channel's balances in the // sums if it was online, so we need to remove them // before applying the event. if state.online { @@ -806,7 +841,7 @@ func calculateBothDirectionsUptime(ctx context.Context, startTime, // Account for the final interval between the last event and the end // time. - accumulate(endTime.Sub(lastTimestamp)) + accumulate(lastTimestamp, endTime) if traceOn { log.TraceS( @@ -825,6 +860,41 @@ func calculateBothDirectionsUptime(ctx context.Context, startTime, return abilityAB, abilityBA, nil } +// observedDuration returns the part of [from, to) that lies outside every +// unobserved interval. The intervals must be sorted and disjoint, and the +// caller must pass a non-decreasing from on each call, because cursor skips the +// intervals that end at or before from. +func observedDuration(from, to time.Time, unobserved []UnobservedInterval, + cursor *int) time.Duration { + + observed := to.Sub(from) + if observed <= 0 { + return 0 + } + + for *cursor < len(unobserved) && !unobserved[*cursor].End.After(from) { + *cursor++ + } + + for _, interval := range unobserved[*cursor:] { + if !interval.Start.Before(to) { + break + } + + overlapStart := interval.Start + if overlapStart.Before(from) { + overlapStart = from + } + overlapEnd := interval.End + if overlapEnd.After(to) { + overlapEnd = to + } + observed -= overlapEnd.Sub(overlapStart) + } + + return observed +} + // mergeEventSlices interleaves two sorted event streams into a single // chronological iter.Seq2. Equal-timestamp events from sliceA are yielded // first. Self-pair calls (sliceA == sliceB) yield each event twice. Callers diff --git a/chanevents/analyzer_test.go b/chanevents/analyzer_test.go index e579c9f..8a74199 100644 --- a/chanevents/analyzer_test.go +++ b/chanevents/analyzer_test.go @@ -229,6 +229,10 @@ func TestCalculateBothDirectionsUptime(t *testing.T) { successAmts []btcutil.Amount liquidityFloor btcutil.Amount + // unobserved lists unobserved intervals as [start, end) + // pairs of Unix seconds. + unobserved [][2]int64 + expectedUptime time.Duration }{ { @@ -589,6 +593,69 @@ func TestCalculateBothDirectionsUptime(t *testing.T) { }, expectedUptime: 0, }, + { + name: "Unobserved interval in the middle", + inStates: map[int64]*channelState{ + chanInID: { + online: true, + remoteBalance: 1000, + }, + }, + outStates: map[int64]*channelState{ + chanOutID: { + online: true, + localBalance: 800, + }, + }, + unobserved: [][2]int64{{130, 160}}, + expectedUptime: 70 * time.Second, + }, + { + // The interval covers the event at t=150 that takes + // the in channel offline. Only t=100..120 is observed + // and qualifying; t=180..200 is observed but offline. + name: "Unobserved interval covers an event", + inStates: map[int64]*channelState{ + chanInID: { + online: true, + remoteBalance: 1000, + }, + }, + outStates: map[int64]*channelState{ + chanOutID: { + online: true, + localBalance: 800, + }, + }, + inEvents: []*ChannelEvent{ + newStatusEvent(chanInID, 150, EventTypeOffline), + }, + unobserved: [][2]int64{{120, 180}}, + expectedUptime: 20 * time.Second, + }, + { + name: "Several unobserved intervals", + inStates: map[int64]*channelState{ + chanInID: { + online: true, + remoteBalance: 1000, + }, + }, + outStates: map[int64]*channelState{ + chanOutID: { + online: true, + localBalance: 800, + }, + }, + inEvents: []*ChannelEvent{ + newEvent( + chanInID, 140, EventTypeUpdate, 0, + 900, + ), + }, + unobserved: [][2]int64{{110, 120}, {130, 150}}, + expectedUptime: 70 * time.Second, + }, } for _, tc := range testCases { @@ -618,6 +685,20 @@ func TestCalculateBothDirectionsUptime(t *testing.T) { tc.inEvents, tc.outEvents, ) + var unobserved []UnobservedInterval + for _, iv := range tc.unobserved { + unobserved = append( + unobserved, UnobservedInterval{ + Start: time.Unix( + iv[0], 0, + ), + End: time.Unix( + iv[1], 0, + ), + }, + ) + } + // Precompute the starting balances. var sumARemote, sumALocal, sumBRemote, sumBLocal btcutil.Amount @@ -642,6 +723,7 @@ func TestCalculateBothDirectionsUptime(t *testing.T) { calculateBothDirectionsUptime( context.Background(), startTime, endTime, + unobserved, tc.liquidityFloor, tc.inStates, tc.outStates, sumARemote, sumALocal, @@ -713,7 +795,7 @@ func TestCalculateBothDirectionsUptimeAsymmetric(t *testing.T) { } abilityAB, abilityBA, err := calculateBothDirectionsUptime( - context.Background(), startTime, endTime, + context.Background(), startTime, endTime, nil, liquidityFloor, statesA, statesB, sumARemote, sumALocal, sumBRemote, sumBLocal, mergeEventSlices(nil, nil), @@ -937,7 +1019,7 @@ func TestEffectiveUptimeIncludesClosedChannels(t *testing.T) { startTime := seedTime.Add(time.Second) endTime := startTime.Add(time.Minute) - abilities, err := a.EffectiveUptime(ctx, startTime, endTime, 0) + abilities, _, err := a.EffectiveUptime(ctx, startTime, endTime, 0) require.NoError(t, err) // Cross-pair entries in both directions are the cleanest assertion @@ -955,6 +1037,88 @@ func TestEffectiveUptimeIncludesClosedChannels(t *testing.T) { ) } +// TestEffectiveUptimeUnobserved pins that a restart with a remote lnd does not +// count as uptime. Both channels stay online with the same balances across the +// restart, so only the unobserved interval tells the analyzer that nobody +// watched the node in that time. +func TestEffectiveUptimeUnobserved(t *testing.T) { + t.Parallel() + + store := NewTestDB(t, clock.NewTestClock(testTime)) + ctx := context.Background() + + peerA, err := store.AddPeer(ctx, validPubKey1) + require.NoError(t, err) + peerB, err := store.AddPeer(ctx, validPubKey2) + require.NoError(t, err) + + chanA, err := store.AddChannel( + ctx, testChanPoint1, testShortChanID1, peerA, + ) + require.NoError(t, err) + chanB, err := store.AddChannel( + ctx, testChanPoint2, testShortChanID2, peerB, + ) + require.NoError(t, err) + + startTime := testTime.Add(time.Second) + endTime := startTime.Add(3 * time.Hour) + lastLive := startTime.Add(10 * time.Minute) + syncTime := lastLive.Add(time.Hour) + + addEvents := func(ts time.Time, isSync bool) { + for _, chanID := range []int64{chanA, chanB} { + err := store.AddChannelEvent(ctx, &ChannelEvent{ + ChannelID: chanID, + EventType: EventTypeOnline, + Timestamp: ts, + IsSync: isSync, + }) + require.NoError(t, err) + + err = store.AddChannelEvent(ctx, &ChannelEvent{ + ChannelID: chanID, + EventType: EventTypeUpdate, + Timestamp: ts, + LocalBalance: fn.Some(btcutil.Amount(1000)), + RemoteBalance: fn.Some(btcutil.Amount(1000)), + IsSync: isSync, + }) + require.NoError(t, err) + } + } + + // Live events before and inside the window, then one hour without + // events, then the sync batch of the restart with unchanged state. + addEvents(testTime, false) + addEvents(lastLive, false) + addEvents(syncTime, true) + + vertexA, err := route.NewVertexFromStr(validPubKey1) + require.NoError(t, err) + vertexB, err := route.NewVertexFromStr(validPubKey2) + require.NoError(t, err) + + stub := &stubLndChannelClient{ + openChannels: []lndclient.ChannelInfo{ + {ChannelID: testShortChanID1, PubKeyBytes: vertexA}, + {ChannelID: testShortChanID2, PubKeyBytes: vertexB}, + }, + } + a := NewForwardingAnalyzer(store, lndclient.LndServices{Client: stub}) + + pairAB := PeerPair{PeerIn: validPubKey1, PeerOut: validPubKey2} + pairBA := PeerPair{PeerIn: validPubKey2, PeerOut: validPubKey1} + + abilities, unobserved, err := a.EffectiveUptime( + ctx, startTime, endTime, 0, + ) + require.NoError(t, err) + require.Equal(t, time.Hour, unobserved) + require.Equal(t, 2*time.Hour, abilities[pairAB].EffectiveUptime) + require.Equal(t, 2*time.Hour, abilities[pairBA].EffectiveUptime) +} + // TestEffectiveUptimeArgs exercises the liquidityFloor, startTime, and endTime // arguments of EffectiveUptime, verifying that the single uniform floor governs // effective uptime and that forwarded volume is reported regardless of uptime. @@ -1043,7 +1207,7 @@ func TestEffectiveUptimeArgs(t *testing.T) { // Case 1: liquidityFloor = 50k. Liquidity of 1M exceeds the floor for // the whole 60s window, so effective uptime is the full minute and the // forwarded volume is the 400M msat total, earning 400k msat. - abilities, err := a.EffectiveUptime(ctx, startTime, endTime, 50_000) + abilities, _, err := a.EffectiveUptime(ctx, startTime, endTime, 50_000) require.NoError(t, err) pair := PeerPair{PeerIn: validPubKey1, PeerOut: validPubKey2} @@ -1061,7 +1225,9 @@ func TestEffectiveUptimeArgs(t *testing.T) { // Case 2: liquidityFloor = 1.5M, above the 1M liquidity, so the floor is // never crossed and effective uptime is zero. The forwarded volume and // fees are still reported so the consumer keeps the signal. - abilities, err = a.EffectiveUptime(ctx, startTime, endTime, 1_500_000) + abilities, _, err = a.EffectiveUptime( + ctx, startTime, endTime, 1_500_000, + ) require.NoError(t, err) require.Contains(t, abilities, pair) diff --git a/frdrpcserver/forwarding_ability.go b/frdrpcserver/forwarding_ability.go index 453f28e..cb07b0e 100644 --- a/frdrpcserver/forwarding_ability.go +++ b/frdrpcserver/forwarding_ability.go @@ -96,7 +96,7 @@ func (s *RPCServer) ForwardingAbility(ctx context.Context, ) } - abilities, err := s.cfg.ForwardingAnalyzer.EffectiveUptime( + abilities, unobserved, err := s.cfg.ForwardingAnalyzer.EffectiveUptime( ctx, startTime, endTime, btcutil.Amount(liquidityFloor), ) if err != nil { @@ -111,6 +111,10 @@ func (s *RPCServer) ForwardingAbility(ctx context.Context, "calculate effective uptime: %v", err) } + // Round up, because uptime rounds down: a pair that was up for the + // whole observed window then still reaches its full length. + unobservedS := uint64((unobserved + time.Second - 1) / time.Second) + // Convert the flat map into the nested map the codec expects, carrying // the raw facts through unchanged. EffectiveUptime is truncated to // whole seconds to match the second-granularity wire field. A pair with @@ -134,11 +138,13 @@ func (s *RPCServer) ForwardingAbility(ctx context.Context, } // Guard against returning data when the node itself was down for the - // window. If no pair held at least the threshold fraction of uptime, - // the response carries no signal and lowering the threshold to surface - // something would only inflate it, so fail loudly instead. + // observed window. If no pair held at least the threshold fraction of + // uptime, the response carries no signal and lowering the threshold to + // surface something would only inflate it, so fail loudly instead. minUptimeS := frdrpc.MinQualifyingUptime( - uptimeThreshold, endTime.Unix()-startTime.Unix(), + uptimeThreshold, frdrpc.ObservedWindow( + startTime.Unix(), endTime.Unix(), unobservedS, + ), ) var qualifying int for _, outMap := range nested { @@ -155,7 +161,8 @@ func (s *RPCServer) ForwardingAbility(ctx context.Context, } resp, err := frdrpc.EncodeForwardingAbility( - nested, startTime.Unix(), endTime.Unix(), 0, uptimeThreshold, + nested, startTime.Unix(), endTime.Unix(), unobservedS, + uptimeThreshold, ) if err != nil { log.ErrorS( diff --git a/frdrpcserver/forwarding_ability_test.go b/frdrpcserver/forwarding_ability_test.go index 5e30ded..e48b0ba 100644 --- a/frdrpcserver/forwarding_ability_test.go +++ b/frdrpcserver/forwarding_ability_test.go @@ -17,12 +17,14 @@ import ( type mockForwardingAnalyzer struct { effectiveUptimeFunc func(ctx context.Context, startTime, endTime time.Time, liquidityFloor btcutil.Amount) ( - map[chanevents.PeerPair]chanevents.ForwardingAbility, error) + map[chanevents.PeerPair]chanevents.ForwardingAbility, + time.Duration, error) } func (m *mockForwardingAnalyzer) EffectiveUptime(ctx context.Context, startTime, endTime time.Time, liquidityFloor btcutil.Amount) ( - map[chanevents.PeerPair]chanevents.ForwardingAbility, error) { + map[chanevents.PeerPair]chanevents.ForwardingAbility, time.Duration, + error) { return m.effectiveUptimeFunc(ctx, startTime, endTime, liquidityFloor) } @@ -45,6 +47,9 @@ func TestForwardingAbility(t *testing.T) { analyzer analyzerResult req *frdrpc.ForwardingAbilityRequest + // unobserved is the unobserved time the analyzer reports. + unobserved time.Duration + // wantCode is the expected gRPC status; codes.OK denotes // success. wantCode codes.Code @@ -212,6 +217,39 @@ func TestForwardingAbility(t *testing.T) { require.NotEmpty(t, resp.UpButIdleBitmask) }, }, + { + // The pair was up for the whole observed window. The + // unobserved time rounds up to 60s, leaving an observed + // window of 40s. The uptime rounds down to 40s, so the + // pair clears a threshold of 1. + name: "unobserved time shrinks guard window", + unobserved: 59500 * time.Millisecond, + analyzer: func() ( + map[chanevents.PeerPair]chanevents.ForwardingAbility, + error) { + + return map[chanevents.PeerPair]chanevents.ForwardingAbility{ + {PeerIn: peerIn, PeerOut: peerOut}: { + EffectiveUptime: 40500 * + time.Millisecond, + }, + }, nil + }, + req: &frdrpc.ForwardingAbilityRequest{ + StartTime: 100, + EndTime: 200, + UptimeThreshold: 1, + }, + wantCode: codes.OK, + check: func(t *testing.T, + resp *frdrpc.ForwardingAbilityResponse, + _ btcutil.Amount) { + + require.EqualValues(t, 60, resp.UnobservedS) + require.Empty(t, resp.Entries) + require.NotEmpty(t, resp.UpButIdleBitmask) + }, + }, { name: "out of range threshold is rejected", analyzer: func() ( @@ -275,11 +313,14 @@ func TestForwardingAbility(t *testing.T) { _ time.Time, floor btcutil.Amount) ( map[chanevents.PeerPair]chanevents.ForwardingAbility, - error) { + time.Duration, error) { gotFloor = floor + abilities, err := + tc.analyzer() - return tc.analyzer() + return abilities, + tc.unobserved, err }, } } diff --git a/frdrpcserver/rpcserver.go b/frdrpcserver/rpcserver.go index 2115170..72c2af6 100644 --- a/frdrpcserver/rpcserver.go +++ b/frdrpcserver/rpcserver.go @@ -56,12 +56,13 @@ type RPCServer struct { } // ForwardingAnalyzer is the seam the RPC server uses to compute per-peer-pair -// forwarding facts. It is an interface so the handler can be exercised against -// a mock analyzer in tests. +// forwarding facts and the unobserved time of the window. It is an interface +// so the handler can be exercised against a mock analyzer in tests. type ForwardingAnalyzer interface { EffectiveUptime(ctx context.Context, startTime, endTime time.Time, liquidityFloor btcutil.Amount) ( - map[chanevents.PeerPair]chanevents.ForwardingAbility, error) + map[chanevents.PeerPair]chanevents.ForwardingAbility, + time.Duration, error) } // Config provides closures and settings required to run the rpc server. From 0ca1f4eeb8266bc009ad51f093216c71d712e3cf Mon Sep 17 00:00:00 2001 From: bitromortac Date: Fri, 30 Oct 2026 09:32:16 +0000 Subject: [PATCH 4/5] chanevents: run initial sync on every reconnect A broken subscription loses events, and without a sync the break was not marked as unobserved time. The monitor now runs the initial sync each time it reconnects. The sync batch re-baselines the balances and ends the unobserved interval of the break. --- chanevents/monitor.go | 115 +++++++++++++++++++++--------- chanevents/monitor_test.go | 142 +++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 34 deletions(-) create mode 100644 chanevents/monitor_test.go diff --git a/chanevents/monitor.go b/chanevents/monitor.go index 87f9cb2..07f1aef 100644 --- a/chanevents/monitor.go +++ b/chanevents/monitor.go @@ -19,6 +19,11 @@ const ( // transient error or while waiting for lnd to become ready. retryInterval = 5 * time.Second + // maxReconnectInterval caps the reconnect backoff. Each reconnect + // writes a full sync batch, so a subscription that fails at once must + // not reconnect at retryInterval forever. + maxReconnectInterval = 5 * time.Minute + // pruneInterval is how often the monitor enforces the channel event // storage limits while consuming live events. pruneInterval = time.Hour @@ -108,7 +113,8 @@ func (m *Monitor) Stop() error { // monitorLoop is the main loop of the channel events monitor. It waits for lnd // to be fully synced, performs an initial state sync, and then subscribes to // channel events. If the subscription fails or the stream breaks, it retries -// from the beginning. +// from the beginning, so every reconnect writes a new sync batch that ends the +// unobserved interval of the break. func (m *Monitor) monitorLoop(ctx context.Context) { defer m.wg.Done() @@ -136,46 +142,83 @@ func (m *Monitor) monitorLoop(ctx context.Context) { pruneChan = pruneTicker.C } - var synced bool - + var reconnectDelay time.Duration for { // Wait for lnd to be synced to chain, retrying on RPC errors. if !m.waitForReady(ctx) { return } - // Initial state sync, only performed once. - if !synced { - if err := m.initialSync(ctx); err != nil { - log.Errorf("Error during initial sync: %v", err) - } else { - synced = true - - // The initial sync can insert a sizeable number - // of events, so prune once it completes. - m.pruneEvents(ctx) + // Re-baseline the channel state, as events may have been missed + // since the last subscription broke. Do not subscribe without a + // sync batch: a live event would then end the unobserved + // interval, and the outage would count as observed time. + if err := m.initialSync(ctx); err != nil { + reconnectDelay = nextReconnectDelay( + reconnectDelay, false, + ) + log.Errorf("Error during initial sync, retrying in "+ + "%v: %v", reconnectDelay, err) + + if !m.wait(ctx, reconnectDelay) { + return } + + continue } + // The initial sync can insert a sizeable number of events, so + // prune once it completes. + m.pruneEvents(ctx) + // Subscribe and consume events until the stream breaks or an // error occurs. - if !m.subscribe(ctx, pruneChan) { + retry, handledEvent := m.subscribe(ctx, pruneChan) + if !retry { return } // Stream broke, wait before reconnecting. - log.Infof("Reconnecting channel event subscription...") + reconnectDelay = nextReconnectDelay( + reconnectDelay, handledEvent, + ) + log.Infof("Reconnecting channel event subscription in %v...", + reconnectDelay) - select { - case <-time.After(retryInterval): - case <-m.quit: - return - case <-ctx.Done(): + if !m.wait(ctx, reconnectDelay) { return } } } +// nextReconnectDelay returns the wait before the next reconnect, given the +// last wait. It returns retryInterval after a subscription that handled an +// event. It also returns retryInterval when the last wait is below +// retryInterval. Otherwise it doubles the last wait, up to +// maxReconnectInterval. +func nextReconnectDelay(last time.Duration, + handledEvent bool) time.Duration { + + if handledEvent || last < retryInterval { + return retryInterval + } + + return min(2*last, maxReconnectInterval) +} + +// wait blocks for the given duration. It returns false if the monitor shuts +// down first. +func (m *Monitor) wait(ctx context.Context, d time.Duration) bool { + select { + case <-time.After(d): + return true + case <-m.quit: + return false + case <-ctx.Done(): + return false + } +} + // pruneEvents enforces the configured channel event storage limits, logging // how many events were deleted. func (m *Monitor) pruneEvents(ctx context.Context) { @@ -231,17 +274,18 @@ func (m *Monitor) waitForReady(ctx context.Context) bool { } // subscribe subscribes to lnd channel events and processes them until the -// stream breaks or an error occurs. It returns true on transient failures -// (caller should retry) or false if the monitor is shutting down. +// stream breaks or an error occurs. retry is true on transient failures +// (caller should retry) and false if the monitor is shutting down. +// handledEvent reports whether the subscription delivered at least one event. func (m *Monitor) subscribe(ctx context.Context, - pruneChan <-chan time.Time) bool { + pruneChan <-chan time.Time) (retry, handledEvent bool) { eventChan, errChan, err := m.lnd.SubscribeChannelEvents(ctx) if err != nil { log.Errorf("Error subscribing to channel events: %v", err) // Return true to signal the caller to retry. - return true + return true, false } for { @@ -249,8 +293,9 @@ func (m *Monitor) subscribe(ctx context.Context, case event, ok := <-eventChan: if !ok { log.Warn("Channel event stream closed") - return true + return true, handledEvent } + handledEvent = true if err := m.handleChannelEvent(ctx, event); err != nil { log.Errorf("Error handling channel event: %v", err) @@ -266,31 +311,33 @@ func (m *Monitor) subscribe(ctx context.Context, log.Warn("Channel event error stream " + "closed") - return true + return true, handledEvent } log.Errorf("Error from channel event "+ "subscription: %v", err) - return true + return true, handledEvent case <-m.quit: log.Info("Channel events monitor stopping") - return false + return false, handledEvent case <-ctx.Done(): log.Info("Channel events monitor stopping") - return false + return false, handledEvent } } } -// initialSync performs an initial sync of the channel state. It queries lnd for -// all known channels (open and closed) and records their current state in the -// database. This ensures that any channel events that occurred while faraday -// was offline are accounted for: even though individual events are lost, the +// initialSync performs an initial sync of the channel state. It runs on start +// and on every reconnect of the subscription. It queries lnd for all known +// channels (open and closed) and records their current state in the database. +// This ensures that any channel events that occurred while faraday did not +// observe lnd are accounted for: even though individual events are lost, the // latest state is captured as a baseline. Events recorded during initial sync // are marked with IsSync=true to distinguish them from real-time events -// received via the subscription. +// received via the subscription. The analyzer treats the time from the last +// live event before such a sync batch to the end of that batch as unobserved. func (m *Monitor) initialSync(ctx context.Context) error { log.Info("Performing initial sync of channel state") diff --git a/chanevents/monitor_test.go b/chanevents/monitor_test.go new file mode 100644 index 0000000..6850e43 --- /dev/null +++ b/chanevents/monitor_test.go @@ -0,0 +1,142 @@ +package chanevents + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/lightninglabs/lndclient" + "github.com/lightningnetwork/lnd/clock" + "github.com/stretchr/testify/require" +) + +// TestNextReconnectDelay pins the reconnect backoff. Each reconnect writes a +// full sync batch, so a subscription that fails at once must back off to the +// cap, and a subscription that handled an event must reset the wait. +func TestNextReconnectDelay(t *testing.T) { + t.Parallel() + + var delay time.Duration + var got []time.Duration + for range 8 { + delay = nextReconnectDelay(delay, false) + got = append(got, delay) + } + require.Equal(t, []time.Duration{ + 5 * time.Second, 10 * time.Second, 20 * time.Second, + 40 * time.Second, 80 * time.Second, 160 * time.Second, + maxReconnectInterval, maxReconnectInterval, + }, got) + + require.Equal(t, retryInterval, nextReconnectDelay(delay, true)) +} + +// syncLnd is an lnd stub that scripts the result of each initial sync and +// records a subscription that does not follow a successful sync. The embedded +// interface is nil, so any method it does not override panics. +type syncLnd struct { + lndclient.LightningClient + + // syncErrs holds the result of each initial sync. done closes once + // the monitor attempted all of them. + syncErrs []error + done chan struct{} + + mu sync.Mutex + syncs int + synced bool + unsynced bool +} + +func (l *syncLnd) GetInfo(_ context.Context) (*lndclient.Info, error) { + return &lndclient.Info{SyncedToChain: true}, nil +} + +func (l *syncLnd) ClosedChannels(_ context.Context) ( + []lndclient.ClosedChannel, error) { + + l.mu.Lock() + defer l.mu.Unlock() + + if l.syncs == len(l.syncErrs) { + return nil, errors.New("sync not scripted") + } + + err := l.syncErrs[l.syncs] + l.syncs++ + if l.syncs == len(l.syncErrs) { + close(l.done) + } + + return nil, err +} + +// ListChannels is the last lnd call of a successful initial sync. +func (l *syncLnd) ListChannels(_ context.Context, _, _ bool, + _ ...lndclient.ListChannelsOption) ([]lndclient.ChannelInfo, error) { + + l.mu.Lock() + defer l.mu.Unlock() + + l.synced = true + + return nil, nil +} + +func (l *syncLnd) SubscribeChannelEvents(_ context.Context) ( + <-chan *lndclient.ChannelEventUpdate, <-chan error, error) { + + l.mu.Lock() + defer l.mu.Unlock() + + if !l.synced { + l.unsynced = true + } + l.synced = false + + return nil, nil, errors.New("subscription unavailable") +} + +// TestMonitorSyncsBeforeSubscribe pins that the monitor writes a sync batch +// before each subscription. Without it, a live event would end the unobserved +// interval, and the outage would count as observed time. Each case waits one +// reconnect interval. +func TestMonitorSyncsBeforeSubscribe(t *testing.T) { + t.Parallel() + + syncErr := errors.New("closed channels unavailable") + tests := []struct { + name string + syncErrs []error + }{ + {"failed sync", []error{syncErr, syncErr}}, + {"reconnect", []error{nil, nil}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + lnd := &syncLnd{ + syncErrs: tc.syncErrs, + done: make(chan struct{}), + } + store := NewTestDB(t, clock.NewTestClock(testTime)) + monitor := NewMonitor(lnd, store, Config{}) + require.NoError(t, monitor.Start(t.Context())) + + select { + case <-lnd.done: + case <-time.After(30 * time.Second): + t.Fatal("initial sync not retried") + } + require.NoError(t, monitor.Stop()) + + lnd.mu.Lock() + defer lnd.mu.Unlock() + require.False(t, lnd.unsynced) + }) + } +} From a0e8558cf45df78440081e671e9952d78ee8ef7c Mon Sep 17 00:00:00 2001 From: bitromortac Date: Fri, 30 Oct 2026 09:32:16 +0000 Subject: [PATCH 5/5] docs: add release notes for unobserved time --- docs/release-notes/release-notes-next.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/release-notes/release-notes-next.md b/docs/release-notes/release-notes-next.md index d321588..ebd37e9 100644 --- a/docs/release-notes/release-notes-next.md +++ b/docs/release-notes/release-notes-next.md @@ -5,6 +5,25 @@ * Migrate exported Bitcoin types and dependencies to the btcd v2 modules. Source builds require Go 1.26.8. +* `frcli forwardingability` prints a JSON object with the fields + `unobserved_s` and `pairs`. The `pairs` field holds the array the command + printed before. The `uptime_fraction` of a pair uses the observed window. + +## RPC Changes + +* `ForwardingAbility` reports `unobserved_s`, the seconds of the window in + which faraday did not observe lnd, for example while faraday or lnd was + down. Effective uptime excludes this time, and the uptime threshold applies + to the observed window. Divide effective uptime by + `end_time - start_time - unobserved_s`. An outage that still goes on counts + as observed time until the next initial sync marks it as unobserved. + +## Improvements + +* The channel events monitor runs the initial sync again each time the + subscription to lnd reconnects. This re-baselines the channel balances and + marks the break as unobserved time. + ## Maintenance * Retain the lnd v0.21.4 recovery fixes while migrating to btcd v2.