From 74bd6c602d745241cfb41e3710cca03d7b42106c Mon Sep 17 00:00:00 2001 From: Krish-vemula Date: Mon, 16 Nov 2026 16:31:08 -0700 Subject: [PATCH 1/3] multinode: random pick within tier for PriorityLevel selection --- multinode/node_selector_priority_level.go | 36 ++-------- .../node_selector_priority_level_test.go | 70 ++++++++++++++----- 2 files changed, 61 insertions(+), 45 deletions(-) diff --git a/multinode/node_selector_priority_level.go b/multinode/node_selector_priority_level.go index a7cdad6a..c3a5dc96 100644 --- a/multinode/node_selector_priority_level.go +++ b/multinode/node_selector_priority_level.go @@ -2,16 +2,15 @@ package multinode import ( "math" + "math/rand/v2" "sort" - "sync/atomic" ) type priorityLevelNodeSelector[ CHAIN_ID ID, RPC any, ] struct { - nodes []Node[CHAIN_ID, RPC] - roundRobinCount []atomic.Uint32 + nodes []Node[CHAIN_ID, RPC] } type nodeWithPriority[ @@ -27,25 +26,19 @@ func NewPriorityLevelNodeSelector[ RPC any, ](nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { return &priorityLevelNodeSelector[CHAIN_ID, RPC]{ - nodes: nodes, - roundRobinCount: make([]atomic.Uint32, nrOfPriorityTiers(nodes)), + nodes: nodes, } } +// Select picks a random alive node from the highest priority tier. Random rather than round-robin, so that pods sharing +// the same RPC set spread their load instead of all starting from the same node. func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { nodes := s.getHighestPriorityAliveTier() - if len(nodes) == 0 { return nil } - priorityLevel := nodes[len(nodes)-1].priority - - // NOTE: Inc returns the number after addition, so we must -1 to get the "current" counter - count := s.roundRobinCount[priorityLevel].Add(1) - 1 - // #nosec G115 - idx := int(count % uint32(len(nodes))) - - return nodes[idx].node + // #nosec G404 + return nodes[rand.IntN(len(nodes))].node } func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Name() string { @@ -90,21 +83,6 @@ func removeLowerTiers[ return nodes2 } -// nrOfPriorityTiers calculates the total number of priority tiers -func nrOfPriorityTiers[ - CHAIN_ID ID, - RPC any, -](nodes []Node[CHAIN_ID, RPC]) int32 { - highestPriority := int32(0) - for _, n := range nodes { - priority := n.Order() - if highestPriority < priority { - highestPriority = priority - } - } - return highestPriority + 1 -} - // firstOrHighestPriority takes a list of nodes and returns the first one with the highest priority func firstOrHighestPriority[ CHAIN_ID ID, diff --git a/multinode/node_selector_priority_level_test.go b/multinode/node_selector_priority_level_test.go index 00662569..434c72ba 100644 --- a/multinode/node_selector_priority_level_test.go +++ b/multinode/node_selector_priority_level_test.go @@ -20,9 +20,9 @@ func TestPriorityLevelNodeSelector(t *testing.T) { state nodeState } type testCase struct { - name string - nodes []testNode - expect []int // indexes of the nodes expected to be returned by Select + name string + nodes []testNode + allowed []int // indexes of the nodes Select may return; every one must be returned at least once } testCases := []testCase{ @@ -32,7 +32,7 @@ func TestPriorityLevelNodeSelector(t *testing.T) { {order: 1, state: nodeStateAlive}, {order: 1, state: nodeStateAlive}, }, - expect: []int{0, 1, 0, 1, 0, 1}, + allowed: []int{0, 1}, }, { name: "TwoNodesSameOrder: Lowest Allowed Order", @@ -40,7 +40,7 @@ func TestPriorityLevelNodeSelector(t *testing.T) { {order: 100, state: nodeStateAlive}, {order: 100, state: nodeStateAlive}, }, - expect: []int{0, 1, 0, 1, 0, 1}, + allowed: []int{0, 1}, }, { name: "NoneAvailable", @@ -49,7 +49,6 @@ func TestPriorityLevelNodeSelector(t *testing.T) { {order: 1, state: nodeStateUnreachable}, {order: 1, state: nodeStateUnreachable}, }, - expect: []int{}, // no nodes should be selected }, { name: "DifferentOrder", @@ -58,7 +57,19 @@ func TestPriorityLevelNodeSelector(t *testing.T) { {order: 2, state: nodeStateAlive}, {order: 3, state: nodeStateAlive}, }, - expect: []int{0, 0}, // only the highest order node should be selected + allowed: []int{0}, // only the highest priority tier should be selected + }, + { + name: "HighestTierPartiallyAlive", + nodes: []testNode{ + {order: 1, state: nodeStateUnreachable}, + {order: 2, state: nodeStateAlive}, + {order: 2, state: nodeStateOutOfSync}, + {order: 2, state: nodeStateAlive}, + {order: 2, state: nodeStateAlive}, + {order: 3, state: nodeStateAlive}, + }, + allowed: []int{1, 3, 4}, // alive nodes of the best tier that has any alive node }, } @@ -68,22 +79,49 @@ func TestPriorityLevelNodeSelector(t *testing.T) { for _, tn := range tc.nodes { node := newMockNode[ID, nodeClient](t) node.On("State").Return(tn.state) - node.On("Order").Return(tn.order) + node.On("Order").Return(tn.order).Maybe() nodes = append(nodes, node) } selector := newNodeSelector(NodeSelectionModePriorityLevel, nodes) - for _, idx := range tc.expect { - if idx >= len(nodes) { - t.Fatalf("Invalid node index %d in test case '%s'", idx, tc.name) - } - assert.Same(t, nodes[idx], selector.Select()) + if len(tc.allowed) == 0 { + assert.Nil(t, selector.Select()) + return } - // Check for nil selection if expected slice is empty - if len(tc.expect) == 0 { - assert.Nil(t, selector.Select()) + // With 1000 draws, the chance of never picking a given allowed node is negligible (< 0.67^1000). + picked := map[Node[ID, nodeClient]]int{} + for range 1000 { + picked[selector.Select()]++ } + for _, idx := range tc.allowed { + assert.Positive(t, picked[nodes[idx]], "node %d was never selected", idx) + delete(picked, nodes[idx]) + } + assert.Empty(t, picked, "selected nodes outside the allowed set") }) } } + +// Pods share the same RPC set; each builds its own selector. Their first picks must spread across the tier instead of +// all landing on the same node. +func TestPriorityLevelNodeSelector_FirstPickSpreadsAcrossSelectors(t *testing.T) { + t.Parallel() + + type nodeClient RPCClient[ID, Head] + nodes := make([]Node[ID, nodeClient], 0, 3) + for range 3 { + node := newMockNode[ID, nodeClient](t) + node.On("State").Return(nodeStateAlive) + node.On("Order").Return(int32(1)) + nodes = append(nodes, node) + } + + firstPicks := map[Node[ID, nodeClient]]int{} + for range 300 { + firstPicks[newNodeSelector(NodeSelectionModePriorityLevel, nodes).Select()]++ + } + for i, n := range nodes { + assert.Positive(t, firstPicks[n], "no selector picked node %d first", i) + } +} From 122039c700be8d1ffa38f3e9721d974ba09c169c Mon Sep 17 00:00:00 2001 From: Krish-vemula Date: Mon, 16 Nov 2026 16:58:38 -0700 Subject: [PATCH 2/3] multinode: build PriorityLevel tier in a single pass --- multinode/node_selector_priority_level.go | 56 +++++-------------- .../node_selector_priority_level_test.go | 11 ++++ 2 files changed, 25 insertions(+), 42 deletions(-) diff --git a/multinode/node_selector_priority_level.go b/multinode/node_selector_priority_level.go index c3a5dc96..ad03938f 100644 --- a/multinode/node_selector_priority_level.go +++ b/multinode/node_selector_priority_level.go @@ -3,7 +3,6 @@ package multinode import ( "math" "math/rand/v2" - "sort" ) type priorityLevelNodeSelector[ @@ -13,14 +12,6 @@ type priorityLevelNodeSelector[ nodes []Node[CHAIN_ID, RPC] } -type nodeWithPriority[ - CHAIN_ID ID, - RPC any, -] struct { - node Node[CHAIN_ID, RPC] - priority int32 -} - func NewPriorityLevelNodeSelector[ CHAIN_ID ID, RPC any, @@ -38,49 +29,30 @@ func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { return nil } // #nosec G404 - return nodes[rand.IntN(len(nodes))].node + return nodes[rand.IntN(len(nodes))] } func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Name() string { return NodeSelectionModePriorityLevel } -// getHighestPriorityAliveTier filters nodes that are not in state nodeStateAlive and -// returns only the highest tier of alive nodes -func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) getHighestPriorityAliveTier() []nodeWithPriority[CHAIN_ID, RPC] { - var nodes []nodeWithPriority[CHAIN_ID, RPC] +// getHighestPriorityAliveTier returns the alive nodes with the lowest Order (highest priority). +func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) getHighestPriorityAliveTier() []Node[CHAIN_ID, RPC] { + var tier []Node[CHAIN_ID, RPC] + bestOrder := int32(math.MaxInt32) for _, n := range s.nodes { - if n.State() == nodeStateAlive { - nodes = append(nodes, nodeWithPriority[CHAIN_ID, RPC]{n, n.Order()}) + if n.State() != nodeStateAlive { + continue } - } - - if len(nodes) == 0 { - return nil - } - - return removeLowerTiers(nodes) -} - -// removeLowerTiers take a slice of nodeWithPriority[CHAIN_ID, BLOCK_HASH, HEAD, RPC] and keeps only the highest tier -func removeLowerTiers[ - CHAIN_ID ID, - RPC any, -](nodes []nodeWithPriority[CHAIN_ID, RPC]) []nodeWithPriority[CHAIN_ID, RPC] { - sort.SliceStable(nodes, func(i, j int) bool { - return nodes[i].priority > nodes[j].priority - }) - - var nodes2 []nodeWithPriority[CHAIN_ID, RPC] - currentPriority := nodes[len(nodes)-1].priority - - for _, n := range nodes { - if n.priority == currentPriority { - nodes2 = append(nodes2, n) + switch order := n.Order(); { + case order < bestOrder: + bestOrder = order + tier = append(tier[:0], n) + case order == bestOrder: + tier = append(tier, n) } } - - return nodes2 + return tier } // firstOrHighestPriority takes a list of nodes and returns the first one with the highest priority diff --git a/multinode/node_selector_priority_level_test.go b/multinode/node_selector_priority_level_test.go index 434c72ba..7f627d34 100644 --- a/multinode/node_selector_priority_level_test.go +++ b/multinode/node_selector_priority_level_test.go @@ -59,6 +59,17 @@ func TestPriorityLevelNodeSelector(t *testing.T) { }, allowed: []int{0}, // only the highest priority tier should be selected }, + { + name: "UnsortedOrder", + nodes: []testNode{ + {order: 3, state: nodeStateAlive}, + {order: 2, state: nodeStateAlive}, + {order: 1, state: nodeStateAlive}, + {order: 2, state: nodeStateAlive}, + {order: 1, state: nodeStateAlive}, + }, + allowed: []int{2, 4}, // a better tier found later must replace the one collected so far + }, { name: "HighestTierPartiallyAlive", nodes: []testNode{ From 34a43f81fb3e3ceb88e1245c4d65db03f1484256 Mon Sep 17 00:00:00 2001 From: Krish-vemula Date: Tue, 17 Nov 2026 09:45:41 -0700 Subject: [PATCH 3/3] multinode: revert PriorityLevel tier refactor --- multinode/node_selector_priority_level.go | 56 +++++++++++++++++------ 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/multinode/node_selector_priority_level.go b/multinode/node_selector_priority_level.go index ad03938f..c3a5dc96 100644 --- a/multinode/node_selector_priority_level.go +++ b/multinode/node_selector_priority_level.go @@ -3,6 +3,7 @@ package multinode import ( "math" "math/rand/v2" + "sort" ) type priorityLevelNodeSelector[ @@ -12,6 +13,14 @@ type priorityLevelNodeSelector[ nodes []Node[CHAIN_ID, RPC] } +type nodeWithPriority[ + CHAIN_ID ID, + RPC any, +] struct { + node Node[CHAIN_ID, RPC] + priority int32 +} + func NewPriorityLevelNodeSelector[ CHAIN_ID ID, RPC any, @@ -29,30 +38,49 @@ func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { return nil } // #nosec G404 - return nodes[rand.IntN(len(nodes))] + return nodes[rand.IntN(len(nodes))].node } func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Name() string { return NodeSelectionModePriorityLevel } -// getHighestPriorityAliveTier returns the alive nodes with the lowest Order (highest priority). -func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) getHighestPriorityAliveTier() []Node[CHAIN_ID, RPC] { - var tier []Node[CHAIN_ID, RPC] - bestOrder := int32(math.MaxInt32) +// getHighestPriorityAliveTier filters nodes that are not in state nodeStateAlive and +// returns only the highest tier of alive nodes +func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) getHighestPriorityAliveTier() []nodeWithPriority[CHAIN_ID, RPC] { + var nodes []nodeWithPriority[CHAIN_ID, RPC] for _, n := range s.nodes { - if n.State() != nodeStateAlive { - continue + if n.State() == nodeStateAlive { + nodes = append(nodes, nodeWithPriority[CHAIN_ID, RPC]{n, n.Order()}) } - switch order := n.Order(); { - case order < bestOrder: - bestOrder = order - tier = append(tier[:0], n) - case order == bestOrder: - tier = append(tier, n) + } + + if len(nodes) == 0 { + return nil + } + + return removeLowerTiers(nodes) +} + +// removeLowerTiers take a slice of nodeWithPriority[CHAIN_ID, BLOCK_HASH, HEAD, RPC] and keeps only the highest tier +func removeLowerTiers[ + CHAIN_ID ID, + RPC any, +](nodes []nodeWithPriority[CHAIN_ID, RPC]) []nodeWithPriority[CHAIN_ID, RPC] { + sort.SliceStable(nodes, func(i, j int) bool { + return nodes[i].priority > nodes[j].priority + }) + + var nodes2 []nodeWithPriority[CHAIN_ID, RPC] + currentPriority := nodes[len(nodes)-1].priority + + for _, n := range nodes { + if n.priority == currentPriority { + nodes2 = append(nodes2, n) } } - return tier + + return nodes2 } // firstOrHighestPriority takes a list of nodes and returns the first one with the highest priority