Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 7 additions & 29 deletions multinode/node_selector_priority_level.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,15 @@ package multinode

import (
"math"
"math/rand/v2"
"sort"
"sync/atomic"
)

type priorityLevelNodeSelector[
CHAIN_ID ID,
RPC any,
] struct {
nodes []Node[CHAIN_ID, RPC]
roundRobinCount []atomic.Uint32
nodes []Node[CHAIN_ID, RPC]
}

type nodeWithPriority[
Expand All @@ -27,25 +26,19 @@ func NewPriorityLevelNodeSelector[
RPC any,
](nodes []Node[CHAIN_ID, RPC]) NodeSelector[CHAIN_ID, RPC] {
return &priorityLevelNodeSelector[CHAIN_ID, RPC]{
nodes: nodes,
roundRobinCount: make([]atomic.Uint32, nrOfPriorityTiers(nodes)),
nodes: nodes,
}
}

// Select picks a random alive node from the highest priority tier. Random rather than round-robin, so that pods sharing
// the same RPC set spread their load instead of all starting from the same node.
func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Select() Node[CHAIN_ID, RPC] {
nodes := s.getHighestPriorityAliveTier()

if len(nodes) == 0 {
return nil
}
priorityLevel := nodes[len(nodes)-1].priority

// NOTE: Inc returns the number after addition, so we must -1 to get the "current" counter
count := s.roundRobinCount[priorityLevel].Add(1) - 1
// #nosec G115
idx := int(count % uint32(len(nodes)))

return nodes[idx].node
// #nosec G404
return nodes[rand.IntN(len(nodes))].node
}

func (s priorityLevelNodeSelector[CHAIN_ID, RPC]) Name() string {
Expand Down Expand Up @@ -90,21 +83,6 @@ func removeLowerTiers[
return nodes2
}

// nrOfPriorityTiers calculates the total number of priority tiers
func nrOfPriorityTiers[
CHAIN_ID ID,
RPC any,
](nodes []Node[CHAIN_ID, RPC]) int32 {
highestPriority := int32(0)
for _, n := range nodes {
priority := n.Order()
if highestPriority < priority {
highestPriority = priority
}
}
return highestPriority + 1
}

// firstOrHighestPriority takes a list of nodes and returns the first one with the highest priority
func firstOrHighestPriority[
CHAIN_ID ID,
Expand Down
81 changes: 65 additions & 16 deletions multinode/node_selector_priority_level_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ func TestPriorityLevelNodeSelector(t *testing.T) {
state nodeState
}
type testCase struct {
name string
nodes []testNode
expect []int // indexes of the nodes expected to be returned by Select
name string
nodes []testNode
allowed []int // indexes of the nodes Select may return; every one must be returned at least once
}

testCases := []testCase{
Expand All @@ -32,15 +32,15 @@ func TestPriorityLevelNodeSelector(t *testing.T) {
{order: 1, state: nodeStateAlive},
{order: 1, state: nodeStateAlive},
},
expect: []int{0, 1, 0, 1, 0, 1},
allowed: []int{0, 1},
},
{
name: "TwoNodesSameOrder: Lowest Allowed Order",
nodes: []testNode{
{order: 100, state: nodeStateAlive},
{order: 100, state: nodeStateAlive},
},
expect: []int{0, 1, 0, 1, 0, 1},
allowed: []int{0, 1},
},
{
name: "NoneAvailable",
Expand All @@ -49,7 +49,6 @@ func TestPriorityLevelNodeSelector(t *testing.T) {
{order: 1, state: nodeStateUnreachable},
{order: 1, state: nodeStateUnreachable},
},
expect: []int{}, // no nodes should be selected
},
{
name: "DifferentOrder",
Expand All @@ -58,7 +57,30 @@ func TestPriorityLevelNodeSelector(t *testing.T) {
{order: 2, state: nodeStateAlive},
{order: 3, state: nodeStateAlive},
},
expect: []int{0, 0}, // only the highest order node should be selected
allowed: []int{0}, // only the highest priority tier should be selected
},
{
name: "UnsortedOrder",
nodes: []testNode{
{order: 3, state: nodeStateAlive},
{order: 2, state: nodeStateAlive},
{order: 1, state: nodeStateAlive},
{order: 2, state: nodeStateAlive},
{order: 1, state: nodeStateAlive},
},
allowed: []int{2, 4}, // a better tier found later must replace the one collected so far
},
{
name: "HighestTierPartiallyAlive",
nodes: []testNode{
{order: 1, state: nodeStateUnreachable},
{order: 2, state: nodeStateAlive},
{order: 2, state: nodeStateOutOfSync},
{order: 2, state: nodeStateAlive},
{order: 2, state: nodeStateAlive},
{order: 3, state: nodeStateAlive},
},
allowed: []int{1, 3, 4}, // alive nodes of the best tier that has any alive node
},
}

Expand All @@ -68,22 +90,49 @@ func TestPriorityLevelNodeSelector(t *testing.T) {
for _, tn := range tc.nodes {
node := newMockNode[ID, nodeClient](t)
node.On("State").Return(tn.state)
node.On("Order").Return(tn.order)
node.On("Order").Return(tn.order).Maybe()
nodes = append(nodes, node)
}

selector := newNodeSelector(NodeSelectionModePriorityLevel, nodes)
for _, idx := range tc.expect {
if idx >= len(nodes) {
t.Fatalf("Invalid node index %d in test case '%s'", idx, tc.name)
}
assert.Same(t, nodes[idx], selector.Select())
if len(tc.allowed) == 0 {
assert.Nil(t, selector.Select())
return
}

// Check for nil selection if expected slice is empty
if len(tc.expect) == 0 {
assert.Nil(t, selector.Select())
// With 1000 draws, the chance of never picking a given allowed node is negligible (< 0.67^1000).
picked := map[Node[ID, nodeClient]]int{}
for range 1000 {
picked[selector.Select()]++
}
for _, idx := range tc.allowed {
assert.Positive(t, picked[nodes[idx]], "node %d was never selected", idx)
delete(picked, nodes[idx])
}
assert.Empty(t, picked, "selected nodes outside the allowed set")
})
}
}

// Pods share the same RPC set; each builds its own selector. Their first picks must spread across the tier instead of
// all landing on the same node.
func TestPriorityLevelNodeSelector_FirstPickSpreadsAcrossSelectors(t *testing.T) {
t.Parallel()

type nodeClient RPCClient[ID, Head]
nodes := make([]Node[ID, nodeClient], 0, 3)
for range 3 {
node := newMockNode[ID, nodeClient](t)
node.On("State").Return(nodeStateAlive)
node.On("Order").Return(int32(1))
nodes = append(nodes, node)
}

firstPicks := map[Node[ID, nodeClient]]int{}
for range 300 {
firstPicks[newNodeSelector(NodeSelectionModePriorityLevel, nodes).Select()]++
}
for i, n := range nodes {
assert.Positive(t, firstPicks[n], "no selector picked node %d first", i)
}
}
Loading