diff --git a/multinode/node_selector_priority_level.go b/multinode/node_selector_priority_level.go index a7cdad6..c3a5dc9 100644 --- a/multinode/node_selector_priority_level.go +++ b/multinode/node_selector_priority_level.go @@ -2,16 +2,15 @@ package multinode import ( "math" + "math/rand/v2" "sort" - "sync/atomic" ) type priorityLevelNodeSelector[ CHAIN_ID ID, RPC any, ] struct { - nodes []Node[CHAIN_ID, RPC] - roundRobinCount []atomic.Uint32 + nodes []Node[CHAIN_ID, RPC] } type nodeWithPriority[ @@ -27,25 +26,19 @@ func NewPriorityLevelNodeSelector[ RPC any, ](nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] { return &priorityLevelNodeSelector[CHAIN_ID, RPC]{ - nodes: nodes, - roundRobinCount: make([]atomic.Uint32, nrOfPriorityTiers(nodes)), + nodes: nodes, } } +// Select picks a random alive node from the highest priority tier. Random rather than round-robin, so that pods sharing +// the same RPC set spread their load instead of all starting from the same node. func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] { nodes := s.getHighestPriorityAliveTier() - if len(nodes) == 0 { return nil } - priorityLevel := nodes[len(nodes)-1].priority - - // NOTE: Inc returns the number after addition, so we must -1 to get the "current" counter - count := s.roundRobinCount[priorityLevel].Add(1) - 1 - // #nosec G115 - idx := int(count % uint32(len(nodes))) - - return nodes[idx].node + // #nosec G404 + return nodes[rand.IntN(len(nodes))].node } func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Name() string { @@ -90,21 +83,6 @@ func removeLowerTiers[ return nodes2 } -// nrOfPriorityTiers calculates the total number of priority tiers -func nrOfPriorityTiers[ - CHAIN_ID ID, - RPC any, -](nodes []Node[CHAIN_ID, RPC]) int32 { - highestPriority := int32(0) - for _, n := range nodes { - priority := n.Order() - if highestPriority < priority { - highestPriority = priority - } - } - return highestPriority + 1 -} - // firstOrHighestPriority takes a list of nodes and returns the first one with the highest priority func firstOrHighestPriority[ CHAIN_ID ID, diff --git a/multinode/node_selector_priority_level_test.go b/multinode/node_selector_priority_level_test.go index 0066256..7f627d3 100644 --- a/multinode/node_selector_priority_level_test.go +++ b/multinode/node_selector_priority_level_test.go @@ -20,9 +20,9 @@ func TestPriorityLevelNodeSelector(t *testing.T) { state nodeState } type testCase struct { - name string - nodes []testNode - expect []int // indexes of the nodes expected to be returned by Select + name string + nodes []testNode + allowed []int // indexes of the nodes Select may return; every one must be returned at least once } testCases := []testCase{ @@ -32,7 +32,7 @@ func TestPriorityLevelNodeSelector(t *testing.T) { {order: 1, state: nodeStateAlive}, {order: 1, state: nodeStateAlive}, }, - expect: []int{0, 1, 0, 1, 0, 1}, + allowed: []int{0, 1}, }, { name: "TwoNodesSameOrder: Lowest Allowed Order", @@ -40,7 +40,7 @@ func TestPriorityLevelNodeSelector(t *testing.T) { {order: 100, state: nodeStateAlive}, {order: 100, state: nodeStateAlive}, }, - expect: []int{0, 1, 0, 1, 0, 1}, + allowed: []int{0, 1}, }, { name: "NoneAvailable", @@ -49,7 +49,6 @@ func TestPriorityLevelNodeSelector(t *testing.T) { {order: 1, state: nodeStateUnreachable}, {order: 1, state: nodeStateUnreachable}, }, - expect: []int{}, // no nodes should be selected }, { name: "DifferentOrder", @@ -58,7 +57,30 @@ func TestPriorityLevelNodeSelector(t *testing.T) { {order: 2, state: nodeStateAlive}, {order: 3, state: nodeStateAlive}, }, - expect: []int{0, 0}, // only the highest order node should be selected + allowed: []int{0}, // only the highest priority tier should be selected + }, + { + name: "UnsortedOrder", + nodes: []testNode{ + {order: 3, state: nodeStateAlive}, + {order: 2, state: nodeStateAlive}, + {order: 1, state: nodeStateAlive}, + {order: 2, state: nodeStateAlive}, + {order: 1, state: nodeStateAlive}, + }, + allowed: []int{2, 4}, // a better tier found later must replace the one collected so far + }, + { + name: "HighestTierPartiallyAlive", + nodes: []testNode{ + {order: 1, state: nodeStateUnreachable}, + {order: 2, state: nodeStateAlive}, + {order: 2, state: nodeStateOutOfSync}, + {order: 2, state: nodeStateAlive}, + {order: 2, state: nodeStateAlive}, + {order: 3, state: nodeStateAlive}, + }, + allowed: []int{1, 3, 4}, // alive nodes of the best tier that has any alive node }, } @@ -68,22 +90,49 @@ func TestPriorityLevelNodeSelector(t *testing.T) { for _, tn := range tc.nodes { node := newMockNode[ID, nodeClient](t) node.On("State").Return(tn.state) - node.On("Order").Return(tn.order) + node.On("Order").Return(tn.order).Maybe() nodes = append(nodes, node) } selector := newNodeSelector(NodeSelectionModePriorityLevel, nodes) - for _, idx := range tc.expect { - if idx >= len(nodes) { - t.Fatalf("Invalid node index %d in test case '%s'", idx, tc.name) - } - assert.Same(t, nodes[idx], selector.Select()) + if len(tc.allowed) == 0 { + assert.Nil(t, selector.Select()) + return } - // Check for nil selection if expected slice is empty - if len(tc.expect) == 0 { - assert.Nil(t, selector.Select()) + // With 1000 draws, the chance of never picking a given allowed node is negligible (< 0.67^1000). + picked := map[Node[ID, nodeClient]]int{} + for range 1000 { + picked[selector.Select()]++ + } + for _, idx := range tc.allowed { + assert.Positive(t, picked[nodes[idx]], "node %d was never selected", idx) + delete(picked, nodes[idx]) } + assert.Empty(t, picked, "selected nodes outside the allowed set") }) } } + +// Pods share the same RPC set; each builds its own selector. Their first picks must spread across the tier instead of +// all landing on the same node. +func TestPriorityLevelNodeSelector_FirstPickSpreadsAcrossSelectors(t *testing.T) { + t.Parallel() + + type nodeClient RPCClient[ID, Head] + nodes := make([]Node[ID, nodeClient], 0, 3) + for range 3 { + node := newMockNode[ID, nodeClient](t) + node.On("State").Return(nodeStateAlive) + node.On("Order").Return(int32(1)) + nodes = append(nodes, node) + } + + firstPicks := map[Node[ID, nodeClient]]int{} + for range 300 { + firstPicks[newNodeSelector(NodeSelectionModePriorityLevel, nodes).Select()]++ + } + for i, n := range nodes { + assert.Positive(t, firstPicks[n], "no selector picked node %d first", i) + } +}