From 812b3f5fe29e564474db226575846ff9937139b8 Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Tue, 28 Jul 2026 14:10:40 +0100 Subject: [PATCH 01/13] KRN-1117: Simplify stack_depot_trie_insert_locked() control flow The main loop mixed child insertion, splitting existing children on a diverging nibble, and promoting an internal child to a leaf, which made the ownership of the walk state hard to follow. Split those three transitions into dedicated helpers so the loop reads as a decision tree over the child slot's current state. Also skip the x86_64 frame-compression KUnit case under CONFIG_UML, since the UML build pulls in the x86_64 arch bits but can't exercise the direct-map addresses the test asserts against. --- lib/stackdepot.c | 341 ++++++++++++++++++++--------------- lib/tests/stackdepot_kunit.c | 6 +- 2 files changed, 202 insertions(+), 145 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 6bb63890d0bdc..1f3e0152076c8 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2099,173 +2099,230 @@ static unsigned int trie_size_append_chain(const unsigned long *entries, } static u32 -stack_depot_trie_insert_locked(const unsigned long *entries, - unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) +trie_insert_child_locked(const struct stack_depot_trie_child_array __rcu **slot, + struct stack_depot_trie_node *parent, + const struct stack_depot_trie_child_array *children, + unsigned int pos, const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; struct stack_depot_trie_child_array *slot_array; - struct stack_depot_trie_child_array *prefix_children; - const struct stack_depot_trie_child_array *children; - const struct stack_depot_trie_child_array __rcu **slot = - &stack_depot_trie_root; - const struct stack_depot_trie_node *child; + struct stack_depot_trie_child_array *tail_array; const struct stack_depot_trie_node *chain_head; const struct stack_depot_trie_node *chain_leaf; + unsigned int capacity = 1; + unsigned int nr_nodes; + unsigned long flags; + u32 new_leaf_id; + size_t slot_array_size; + bool tail_append = false; + + if (children) { + capacity = roundup_pow_of_two(children->nr_children + 1); + tail_append = pos == children->nr_children && + children->nr_children < children->capacity; + slot_array_size = tail_append ? 0 : trie_child_array_bytes(capacity); + } else { + slot_array_size = trie_child_array_bytes(capacity); + } + if (slot_array_size > DEPOT_POOL_SIZE) + return 0; + + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); + if (!new_leaf_id) + return 0; + nr_nodes = trie_size_append_chain(entries, nr_entries, alloc->node_sizes); + if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, nr_nodes - 1, + 0, slot_array_size)) + return 0; + trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, + alloc->nodes, alloc->chain_arrays, + &chain_head, &chain_leaf); + trie_side_table_publish_new_leaf(new_leaf_id, chain_leaf); + + if (!children) { + slot_array = alloc->slot_array; + slot_array->nr_children = 1; + slot_array->capacity = 1; + RCU_INIT_POINTER(slot_array->children[0], chain_head); + rcu_assign_pointer(*slot, slot_array); + return new_leaf_id; + } + + if (tail_append) { + tail_array = (struct stack_depot_trie_child_array *)children; + trie_publish_tail_append(tail_array, pos, chain_head); + } else { + slot_array = alloc->slot_array; + trie_child_array_insert_at(children, pos, chain_head, + slot_array, capacity); + rcu_assign_pointer(*slot, slot_array); + raw_spin_lock_irqsave(&pool_lock, flags); + trie_retire_child_array_locked(children); + raw_spin_unlock_irqrestore(&pool_lock, flags); + } + + return new_leaf_id; +} + +static u32 +trie_split_child_locked(const struct stack_depot_trie_child_array __rcu **slot, + const struct stack_depot_trie_child_array *children, + const struct stack_depot_trie_node *child, + unsigned int pos, unsigned int matched, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) +{ + struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; + struct stack_depot_trie_child_array *prefix_children; + struct stack_depot_trie_child_array *slot_array; const struct stack_depot_trie_node *split_leaf; + struct stack_depot_frame_run run; struct stack_depot_trie_node *split_prefix; struct stack_depot_trie_node *old_tail; + unsigned int new_nodes = 0; + u32 new_leaf_id; + bool has_new_tail; + + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); + if (!new_leaf_id) + return 0; + + run = child->run; + run.nr_entries = matched; + alloc->node_sizes[0] = trie_node_bytes(&run); + run.nr_entries = child->run.nr_entries - matched; + alloc->node_sizes[1] = trie_node_bytes(&run); + has_new_tail = matched < nr_entries; + if (has_new_tail) + new_nodes = trie_size_append_chain(&entries[matched], + nr_entries - matched, + &alloc->node_sizes[2]); + if (trie_pool_alloc_insert(alloc, pool_prealloc, 2 + new_nodes, + new_nodes ? new_nodes - 1 : 0, + trie_child_array_bytes(has_new_tail ? 2 : 1), + trie_child_array_bytes(children->capacity))) + return 0; + + slot_array = alloc->slot_array; + prefix_children = alloc->prefix_children; + split_prefix = alloc->nodes[0]; + old_tail = alloc->nodes[1]; + trie_build_split(child, matched, new_leaf_id, entries, nr_entries, + split_prefix, old_tail, &alloc->nodes[2], + alloc->chain_arrays, prefix_children, &split_leaf); + trie_side_table_publish_split_leaves(child->leaf_id, old_tail, + new_leaf_id, split_leaf); + trie_child_array_replace_at(children, split_prefix, slot_array, pos); + trie_reparent_children(old_tail); + rcu_assign_pointer(*slot, slot_array); + trie_retire_child_array_with_node(children, child); + + return new_leaf_id; +} + +static u32 +trie_promote_child_locked(const struct stack_depot_trie_child_array __rcu **slot, + const struct stack_depot_trie_child_array *children, + const struct stack_depot_trie_node *child, + unsigned int pos, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) +{ + struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; + struct stack_depot_trie_child_array *slot_array; struct stack_depot_trie_node *promoted_node; + u32 new_leaf_id; + size_t slot_array_size; + + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); + if (!new_leaf_id) + return 0; + alloc->node_sizes[0] = trie_node_bytes(&child->run); + slot_array_size = trie_child_array_bytes(children->capacity); + if (trie_pool_alloc_insert(alloc, pool_prealloc, 1, 0, 0, + slot_array_size)) + return 0; + + promoted_node = alloc->nodes[0]; + slot_array = alloc->slot_array; + memcpy(promoted_node, child, alloc->node_sizes[0]); + promoted_node->leaf_id = new_leaf_id; + trie_side_table_publish_new_leaf(new_leaf_id, promoted_node); + trie_child_array_replace_at(children, promoted_node, slot_array, pos); + trie_reparent_children(promoted_node); + rcu_assign_pointer(*slot, slot_array); + trie_retire_child_array_with_node(children, child); + + return new_leaf_id; +} + +static u32 trie_finish_insert(u32 leaf_id) +{ + if (leaf_id) + trie_side_table_last_leaf_id = leaf_id; + return leaf_id; +} + +static u32 +stack_depot_trie_insert_locked(const unsigned long *entries, + unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) +{ + const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_child_array __rcu **slot = + &stack_depot_trie_root; + const struct stack_depot_trie_node *child; struct stack_depot_trie_node *parent = NULL; unsigned int matched; unsigned int pos; - unsigned long flags; - u32 new_leaf_id; - size_t slot_array_size; + u32 leaf_id; for (;;) { children = trie_load_children_slot(slot); if (!children) { - unsigned int nr_nodes; - - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - slot_array_size = trie_child_array_bytes(1); - nr_nodes = trie_size_append_chain(entries, nr_entries, - alloc->node_sizes); - if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, - nr_nodes - 1, 0, - slot_array_size)) - return 0; - slot_array = alloc->slot_array; - trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - alloc->nodes, alloc->chain_arrays, - &chain_head, &chain_leaf); - trie_side_table_publish_new_leaf(new_leaf_id, chain_leaf); - slot_array->nr_children = 1; - slot_array->capacity = 1; - RCU_INIT_POINTER(slot_array->children[0], chain_head); - rcu_assign_pointer(*slot, slot_array); - goto out_success; + leaf_id = trie_insert_child_locked(slot, parent, NULL, 0, + entries, nr_entries, + pool_prealloc, + side_prealloc); + return trie_finish_insert(leaf_id); } + if (!trie_child_array_find_slot(children, entries[0], &pos)) { - struct stack_depot_trie_child_array *tail_array; - unsigned int capacity; - unsigned int nr_nodes; - bool tail_append; - - capacity = roundup_pow_of_two(children->nr_children + 1); - tail_append = pos == children->nr_children && - children->nr_children < children->capacity; - slot_array_size = tail_append ? 0 : - trie_child_array_bytes(capacity); - if (slot_array_size > DEPOT_POOL_SIZE) - return 0; - - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - nr_nodes = trie_size_append_chain(entries, nr_entries, - alloc->node_sizes); - if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, - nr_nodes - 1, 0, - slot_array_size)) - return 0; - trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - alloc->nodes, alloc->chain_arrays, - &chain_head, &chain_leaf); - trie_side_table_publish_new_leaf(new_leaf_id, chain_leaf); - if (tail_append) { - tail_array = (struct stack_depot_trie_child_array *)children; - trie_publish_tail_append(tail_array, pos, chain_head); - } else { - slot_array = alloc->slot_array; - trie_child_array_insert_at(children, pos, chain_head, - slot_array, capacity); - rcu_assign_pointer(*slot, slot_array); - raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_child_array_locked(children); - raw_spin_unlock_irqrestore(&pool_lock, flags); - } - goto out_success; + leaf_id = trie_insert_child_locked(slot, parent, children, pos, + entries, nr_entries, + pool_prealloc, + side_prealloc); + return trie_finish_insert(leaf_id); } child = trie_child_array_load_child(children, pos); matched = __stack_depot_trie_node_match(child, entries, nr_entries); - if (matched < child->run.nr_entries) { - struct stack_depot_frame_run run; - unsigned int new_nodes = 0; - bool has_new_tail; - - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - - run = child->run; - run.nr_entries = matched; - alloc->node_sizes[0] = trie_node_bytes(&run); - run.nr_entries = child->run.nr_entries - matched; - alloc->node_sizes[1] = trie_node_bytes(&run); - has_new_tail = matched < nr_entries; - if (has_new_tail) - new_nodes = trie_size_append_chain(&entries[matched], - nr_entries - matched, - &alloc->node_sizes[2]); - if (trie_pool_alloc_insert(alloc, pool_prealloc, - 2 + new_nodes, - new_nodes ? new_nodes - 1 : 0, - trie_child_array_bytes(has_new_tail ? 2 : 1), - trie_child_array_bytes(children->capacity))) - return 0; - slot_array = alloc->slot_array; - prefix_children = alloc->prefix_children; - split_prefix = alloc->nodes[0]; - old_tail = alloc->nodes[1]; - trie_build_split(child, matched, new_leaf_id, entries, nr_entries, - split_prefix, old_tail, &alloc->nodes[2], - alloc->chain_arrays, prefix_children, &split_leaf); - trie_side_table_publish_split_leaves(child->leaf_id, old_tail, - new_leaf_id, split_leaf); - trie_child_array_replace_at(children, split_prefix, slot_array, pos); - trie_reparent_children(old_tail); - rcu_assign_pointer(*slot, slot_array); - trie_retire_child_array_with_node(children, child); - goto out_success; - } - if (matched == nr_entries) { - if (child->leaf_id) - return child->leaf_id; - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - alloc->node_sizes[0] = trie_node_bytes(&child->run); - slot_array_size = trie_child_array_bytes(children->capacity); - if (trie_pool_alloc_insert(alloc, pool_prealloc, 1, 0, 0, - slot_array_size)) - return 0; - promoted_node = alloc->nodes[0]; - slot_array = alloc->slot_array; - memcpy(promoted_node, child, alloc->node_sizes[0]); - promoted_node->leaf_id = new_leaf_id; - trie_side_table_publish_new_leaf(new_leaf_id, promoted_node); - trie_child_array_replace_at(children, promoted_node, slot_array, pos); - trie_reparent_children(promoted_node); - rcu_assign_pointer(*slot, slot_array); - trie_retire_child_array_with_node(children, child); - goto out_success; + if (matched == child->run.nr_entries) { + if (matched == nr_entries) { + if (child->leaf_id) + return child->leaf_id; + leaf_id = trie_promote_child_locked(slot, children, + child, pos, + pool_prealloc, + side_prealloc); + return trie_finish_insert(leaf_id); + } + + parent = (struct stack_depot_trie_node *)child; + slot = &parent->children; + entries += matched; + nr_entries -= matched; + continue; } - parent = (struct stack_depot_trie_node *)child; - slot = &parent->children; - entries += matched; - nr_entries -= matched; + leaf_id = trie_split_child_locked(slot, children, child, pos, + matched, entries, nr_entries, + pool_prealloc, side_prealloc); + return trie_finish_insert(leaf_id); } - -out_success: - trie_side_table_last_leaf_id = new_leaf_id; - return new_leaf_id; } static unsigned int diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 1623a644f6e62..4fba72e283312 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -336,7 +336,7 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) KUNIT_EXPECT_FALSE(test, compressed); } -#ifdef CONFIG_X86_64 +#if defined(CONFIG_X86_64) && !defined(CONFIG_UML) static void stackdepot_frame_x86_64(struct kunit *test) { unsigned long direct_map = 0xffff888000001000UL; @@ -354,7 +354,7 @@ static void stackdepot_frame_x86_64(struct kunit *test) compressed = arch_stack_depot_frame_try_compress(direct_map, &low); KUNIT_EXPECT_FALSE(test, compressed); } -#endif /* CONFIG_X86_64 */ +#endif /* CONFIG_X86_64 && !CONFIG_UML */ #ifdef CONFIG_ARM64 static void stackdepot_frame_arm64(struct kunit *test) @@ -398,7 +398,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_topology_roundtrip), KUNIT_CASE(stackdepot_frame_storage_roundtrip), KUNIT_CASE(stackdepot_frame_raw_fallback), -#ifdef CONFIG_X86_64 +#if defined(CONFIG_X86_64) && !defined(CONFIG_UML) KUNIT_CASE(stackdepot_frame_x86_64), #endif #ifdef CONFIG_ARM64 From d53678a9ca3a99beb3ce4d44a2c26dfd59362f45 Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Tue, 28 Jul 2026 14:27:53 +0100 Subject: [PATCH 02/13] KRN-1117: Collapse stack_depot_trie_insert_locked() insert branches Follow-up to 812b3f5fe29e ("Simplify stack_depot_trie_insert_locked() control flow"). The missing-children and no-matching-child cases both end up calling the same helper to append the remaining path, so fold them into one branch and rename trie_insert_child_locked() to trie_insert_path_locked() to match what it actually does. Teach trie_child_array_find_slot() to accept a NULL array (treated as empty, returning pos = 0) so the caller doesn't need a separate NULL check or a conditional pos argument. Document the helper's return contract while we're there. While here, funnel every insertion case through a single trie_finish_insert() return via break, and drop a one-line comment on each of the four cases in the loop so the shape is obvious at a glance. --- lib/stackdepot.c | 81 ++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 1f3e0152076c8..f5d31552dfb43 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1831,6 +1831,13 @@ trie_child_array_load_child(const struct stack_depot_trie_child_array *array, rcu_read_lock_sched_held()); } +/* + * Find the child slot for @frame. + * + * Return true and set @pos to the matching child index when @array contains a + * matching child. Return false and set @pos to the insertion index otherwise. + * A NULL @array is treated as empty and returns @pos = 0. + */ static bool trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, unsigned long frame, unsigned int *pos) @@ -1838,6 +1845,11 @@ trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, unsigned int left = 0; unsigned int right; + if (!array) { + *pos = 0; + return false; + } + right = READ_ONCE(array->nr_children); while (left < right) { unsigned int mid = left + (right - left) / 2; @@ -2001,8 +2013,6 @@ stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) unsigned int matched; unsigned int slot; - if (!children) - return NULL; if (!trie_child_array_find_slot(children, entries[pos], &slot)) return NULL; @@ -2099,7 +2109,7 @@ static unsigned int trie_size_append_chain(const unsigned long *entries, } static u32 -trie_insert_child_locked(const struct stack_depot_trie_child_array __rcu **slot, +trie_insert_path_locked(const struct stack_depot_trie_child_array __rcu **slot, struct stack_depot_trie_node *parent, const struct stack_depot_trie_child_array *children, unsigned int pos, const unsigned long *entries, @@ -2278,51 +2288,48 @@ stack_depot_trie_insert_locked(const unsigned long *entries, struct stack_depot_trie_node *parent = NULL; unsigned int matched; unsigned int pos; - u32 leaf_id; + u32 leaf_id = 0; for (;;) { children = trie_load_children_slot(slot); - if (!children) { - leaf_id = trie_insert_child_locked(slot, parent, NULL, 0, - entries, nr_entries, - pool_prealloc, - side_prealloc); - return trie_finish_insert(leaf_id); - } - + /* Case 1: no matching child, so append the remaining stack path. */ if (!trie_child_array_find_slot(children, entries[0], &pos)) { - leaf_id = trie_insert_child_locked(slot, parent, children, pos, - entries, nr_entries, - pool_prealloc, - side_prealloc); - return trie_finish_insert(leaf_id); + leaf_id = trie_insert_path_locked(slot, parent, children, pos, + entries, nr_entries, + pool_prealloc, + side_prealloc); + break; } child = trie_child_array_load_child(children, pos); matched = __stack_depot_trie_node_match(child, entries, nr_entries); - if (matched == child->run.nr_entries) { - if (matched == nr_entries) { - if (child->leaf_id) - return child->leaf_id; - leaf_id = trie_promote_child_locked(slot, children, - child, pos, - pool_prealloc, - side_prealloc); - return trie_finish_insert(leaf_id); - } - - parent = (struct stack_depot_trie_node *)child; - slot = &parent->children; - entries += matched; - nr_entries -= matched; - continue; + + /* Case 2: the match ends inside the child run, so split it. */ + if (matched < child->run.nr_entries) { + leaf_id = trie_split_child_locked(slot, children, child, pos, + matched, entries, nr_entries, + pool_prealloc, side_prealloc); + break; } - leaf_id = trie_split_child_locked(slot, children, child, pos, - matched, entries, nr_entries, - pool_prealloc, side_prealloc); - return trie_finish_insert(leaf_id); + /* Case 3: the input ends here, so reuse or promote this child. */ + if (matched == nr_entries) { + if (child->leaf_id) + return child->leaf_id; + leaf_id = trie_promote_child_locked(slot, children, child, pos, + pool_prealloc, + side_prealloc); + break; + } + + /* Case 4: the child matched fully; descend with remaining frames. */ + parent = (struct stack_depot_trie_node *)child; + slot = &parent->children; + entries += matched; + nr_entries -= matched; } + + return trie_finish_insert(leaf_id); } static unsigned int From 483aa9a6b8b98c36eb87428fe667c12f71c2246c Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Tue, 28 Jul 2026 15:45:03 +0100 Subject: [PATCH 03/13] KRN-1117: Drop _locked suffix from private stackdepot trie helpers These helpers are all file-static and only ever run under the trie writer lock or pool_lock, so the _locked tag doesn't tell the reader anything the surrounding code doesn't already make obvious. Rename them to the shorter forms; no behaviour change. --- lib/stackdepot.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index f5d31552dfb43..33af77acf0192 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -739,7 +739,7 @@ static void trie_drain_pending_arrays(void) } } -static void trie_retire_child_array_locked(const void *ptr) +static void trie_retire_child_array(const void *ptr) { struct stack_depot_trie_retired_array *retired; @@ -759,7 +759,7 @@ trie_retire_child_array_with_node(const void *ptr, unsigned long flags; raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_child_array_locked(ptr); + trie_retire_child_array(ptr); retired = trie_retired_array(ptr); retired->pending_node = (struct stack_depot_trie_node *)node; raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -1488,7 +1488,7 @@ static inline struct stack_record *find_stack(struct list_head *bucket, } static u32 -stack_depot_trie_insert_locked(const unsigned long *entries, +stack_depot_trie_insert(const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc); @@ -1516,7 +1516,7 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, goto out_free; raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); - leaf_id = stack_depot_trie_insert_locked(entries, nr_entries, + leaf_id = stack_depot_trie_insert(entries, nr_entries, &pool_prealloc, &side_prealloc); if (leaf_id) handle = __stack_depot_trie_handle(leaf_id); @@ -2109,7 +2109,7 @@ static unsigned int trie_size_append_chain(const unsigned long *entries, } static u32 -trie_insert_path_locked(const struct stack_depot_trie_child_array __rcu **slot, +trie_insert_path(const struct stack_depot_trie_child_array __rcu **slot, struct stack_depot_trie_node *parent, const struct stack_depot_trie_child_array *children, unsigned int pos, const unsigned long *entries, @@ -2169,7 +2169,7 @@ trie_insert_path_locked(const struct stack_depot_trie_child_array __rcu **slot, slot_array, capacity); rcu_assign_pointer(*slot, slot_array); raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_child_array_locked(children); + trie_retire_child_array(children); raw_spin_unlock_irqrestore(&pool_lock, flags); } @@ -2177,7 +2177,7 @@ trie_insert_path_locked(const struct stack_depot_trie_child_array __rcu **slot, } static u32 -trie_split_child_locked(const struct stack_depot_trie_child_array __rcu **slot, +trie_split_child(const struct stack_depot_trie_child_array __rcu **slot, const struct stack_depot_trie_child_array *children, const struct stack_depot_trie_node *child, unsigned int pos, unsigned int matched, @@ -2234,7 +2234,7 @@ trie_split_child_locked(const struct stack_depot_trie_child_array __rcu **slot, } static u32 -trie_promote_child_locked(const struct stack_depot_trie_child_array __rcu **slot, +trie_promote_child(const struct stack_depot_trie_child_array __rcu **slot, const struct stack_depot_trie_child_array *children, const struct stack_depot_trie_node *child, unsigned int pos, void **pool_prealloc, @@ -2276,7 +2276,7 @@ static u32 trie_finish_insert(u32 leaf_id) } static u32 -stack_depot_trie_insert_locked(const unsigned long *entries, +stack_depot_trie_insert(const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) @@ -2294,7 +2294,7 @@ stack_depot_trie_insert_locked(const unsigned long *entries, children = trie_load_children_slot(slot); /* Case 1: no matching child, so append the remaining stack path. */ if (!trie_child_array_find_slot(children, entries[0], &pos)) { - leaf_id = trie_insert_path_locked(slot, parent, children, pos, + leaf_id = trie_insert_path(slot, parent, children, pos, entries, nr_entries, pool_prealloc, side_prealloc); @@ -2306,7 +2306,7 @@ stack_depot_trie_insert_locked(const unsigned long *entries, /* Case 2: the match ends inside the child run, so split it. */ if (matched < child->run.nr_entries) { - leaf_id = trie_split_child_locked(slot, children, child, pos, + leaf_id = trie_split_child(slot, children, child, pos, matched, entries, nr_entries, pool_prealloc, side_prealloc); break; @@ -2316,7 +2316,7 @@ stack_depot_trie_insert_locked(const unsigned long *entries, if (matched == nr_entries) { if (child->leaf_id) return child->leaf_id; - leaf_id = trie_promote_child_locked(slot, children, child, pos, + leaf_id = trie_promote_child(slot, children, child, pos, pool_prealloc, side_prealloc); break; From f3083422f28a391b181b25b7d8893dac56f5b657 Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Tue, 28 Jul 2026 16:43:34 +0100 Subject: [PATCH 04/13] KRN-1117: Inline stackdepot trie preallocation and drop retry The retry path in stack_depot_trie_save() only re-ran the same preallocate-then-insert sequence after releasing the leftover pool page and side-table prealloc, which the out_free path already handles. Drop it and inline __stack_depot_trie_alloc_prealloc() so the allocation and locking sit in one place. --- lib/stackdepot.c | 74 +++++++++--------------------------------------- 1 file changed, 14 insertions(+), 60 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 33af77acf0192..88490caeb377a 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -765,44 +765,6 @@ trie_retire_child_array_with_node(const void *ptr, raw_spin_unlock_irqrestore(&pool_lock, flags); } -/* - * Preallocate resources that cannot be allocated while trie writers hold raw - * spinlocks. Side-table growth is mandatory before a new leaf ID can be - * used, so side-table preallocation failure disables insertion for this - * save. Pool preallocation is opportunistic: reusable trie slots may still - * satisfy the insertion, and trie_pool_alloc_insert() reports -ENOSPC if they - * do not. - */ -static int -__stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) -{ - unsigned long flags; - bool need_pool; - int ret; - - raw_spin_lock_irqsave(&pool_lock, flags); - need_pool = !new_pool; - raw_spin_unlock_irqrestore(&pool_lock, flags); - if (need_pool) { - struct page *page; - - page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); - if (page) - *pool_prealloc = page_address(page); - } - ret = trie_side_table_get_prealloc(alloc_flags, side_prealloc); - - if (ret) { - if (*pool_prealloc) { - free_pages((unsigned long)*pool_prealloc, DEPOT_POOL_ORDER); - *pool_prealloc = NULL; - } - return -ENOSPC; - } - return 0; -} - /* * Reserve fixed-size pool slots for one trie insertion. If any reservation * fails, all slots reserved by this attempt are released locally. @@ -1499,42 +1461,34 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, { struct stack_depot_trie_side_prealloc side_prealloc = {}; void *pool_prealloc = NULL; - depot_stack_handle_t handle; + depot_stack_handle_t handle = 0; unsigned long flags; - bool retried = false; + bool need_pool; u32 leaf_id; - int ret; -retry: handle = trie_find_handle(entries, nr_entries); if (handle) return handle; - ret = __stack_depot_trie_alloc_prealloc(alloc_flags, &pool_prealloc, - &side_prealloc); - if (ret) + raw_spin_lock_irqsave(&pool_lock, flags); + need_pool = !new_pool; + raw_spin_unlock_irqrestore(&pool_lock, flags); + if (need_pool) { + struct page *page; + + page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); + if (page) + pool_prealloc = page_address(page); + } + if (trie_side_table_get_prealloc(alloc_flags, &side_prealloc)) goto out_free; raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); leaf_id = stack_depot_trie_insert(entries, nr_entries, - &pool_prealloc, &side_prealloc); + &pool_prealloc, &side_prealloc); if (leaf_id) handle = __stack_depot_trie_handle(leaf_id); raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); - if (!handle && !retried) { - retried = true; - if (pool_prealloc) { - raw_spin_lock_irqsave(&pool_lock, flags); - depot_keep_new_pool(&pool_prealloc); - raw_spin_unlock_irqrestore(&pool_lock, flags); - } - if (pool_prealloc) { - free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); - pool_prealloc = NULL; - } - trie_side_table_put_prealloc(&side_prealloc); - goto retry; - } out_free: if (pool_prealloc) { From b4633f0ea5d056f4a7c12fe0aaa2797fce58ba86 Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Tue, 28 Jul 2026 16:49:38 +0100 Subject: [PATCH 05/13] KRN-1117: Use READ_ONCE(new_pool) for trie preallocation hint Taking pool_lock just to read new_pool is overkill. new_pool can change between this check and the actual insertion anyway, so the test is only an opportunistic hint about whether we need to preallocate a page. Drop the lock and READ_ONCE() the pointer, matching what the hash stackdepot save path already does. --- lib/stackdepot.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 88490caeb377a..8f2f99bcdd0d6 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1463,17 +1463,13 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, void *pool_prealloc = NULL; depot_stack_handle_t handle = 0; unsigned long flags; - bool need_pool; u32 leaf_id; handle = trie_find_handle(entries, nr_entries); if (handle) return handle; - raw_spin_lock_irqsave(&pool_lock, flags); - need_pool = !new_pool; - raw_spin_unlock_irqrestore(&pool_lock, flags); - if (need_pool) { + if (!READ_ONCE(new_pool)) { struct page *page; page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); From 9066cec4a326beac3997190a3bb96a81914a08cd Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Tue, 28 Jul 2026 17:35:48 +0100 Subject: [PATCH 06/13] KRN-1117: Simplify stack_depot_trie_save() preallocation The new_pool hint was racy. Another writer could publish a pool between our READ_ONCE() and the trie insert, leaving us without a preallocated page when we needed one. Drop the hint and always try the pool-page prealloc once the side-table prealloc has succeeded. If the insert doesn't consume the page we still hand it to depot_keep_new_pool() or free it, so keep/free semantics are unchanged. The out_free label goes away too now that side-table prealloc failure has nothing to clean up. --- lib/stackdepot.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 8f2f99bcdd0d6..34c81c80439b6 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1463,21 +1463,20 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, void *pool_prealloc = NULL; depot_stack_handle_t handle = 0; unsigned long flags; + struct page *page; u32 leaf_id; handle = trie_find_handle(entries, nr_entries); if (handle) return handle; - if (!READ_ONCE(new_pool)) { - struct page *page; - - page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); - if (page) - pool_prealloc = page_address(page); - } if (trie_side_table_get_prealloc(alloc_flags, &side_prealloc)) - goto out_free; + return 0; + + /* If trie insertion does not consume this page, keep it for the next save. */ + page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); + if (page) + pool_prealloc = page_address(page); raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); leaf_id = stack_depot_trie_insert(entries, nr_entries, @@ -1486,14 +1485,14 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, handle = __stack_depot_trie_handle(leaf_id); raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); -out_free: if (pool_prealloc) { raw_spin_lock_irqsave(&pool_lock, flags); depot_keep_new_pool(&pool_prealloc); raw_spin_unlock_irqrestore(&pool_lock, flags); + /* depot_keep_new_pool() clears pool_prealloc if it consumes it. */ + if (pool_prealloc) + free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); } - if (pool_prealloc) - free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); trie_side_table_put_prealloc(&side_prealloc); return handle; } From 196aae03653503e02ca7a7ee5edeb1fd15d91c85 Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Wed, 29 Jul 2026 10:53:04 +0100 Subject: [PATCH 07/13] KRN-1117: Consistently name trie things The old code mixed child/node/children/array in various ways that made following it a bit confusing. Rename the container to struct stack_depot_trie_children with a nodes[] flexible member, and update the helpers, locals, and pending list to match. A trie node now has a parent and children, and a children container owns nodes; each individual child is a stack_depot_trie_node. No behaviour change. --- lib/stackdepot.c | 468 +++++++++++++++++++++++------------------------ 1 file changed, 234 insertions(+), 234 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 34c81c80439b6..a170c5a35c15e 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -6,7 +6,7 @@ * callers that request STACK_DEPOT_FLAG_COUNTABLE use the legacy hash table with * contiguous stack records in stack pools. Persistent non-refcounted entries * can use trie storage when enabled; trie nodes share common frame prefixes and - * are published through RCU/COW child arrays. + * are published through RCU/COW children containers. * * Author: Alexander Potapenko * Copyright (C) 2016 Google, Inc. @@ -124,38 +124,38 @@ struct stack_depot_frame_run { static_assert(CONFIG_STACKDEPOT_MAX_FRAMES <= U16_MAX); -struct stack_depot_trie_child_array; +struct stack_depot_trie_children; struct stack_depot_trie_node { /* Parent links let fetch rebuild a full stack from a leaf to the root. */ const struct stack_depot_trie_node __rcu *parent; - /* Child arrays are separate RCU/COW generations. */ - const struct stack_depot_trie_child_array __rcu *children; + /* Children are separate RCU/COW generations. */ + const struct stack_depot_trie_children __rcu *children; u32 leaf_id; struct stack_depot_frame_run run; unsigned char data[]; }; /* - * Children are sorted by first frame and searched by insertion slot. + * Children are sorted by first frame and searched by insertion position. * Writers may append to spare capacity at the sorted tail, but never change * existing child pointers. Other updates build and publish a replacement array. */ -struct stack_depot_trie_child_array { +struct stack_depot_trie_children { unsigned int nr_children; unsigned int capacity; - const struct stack_depot_trie_node __rcu *children[]; + const struct stack_depot_trie_node __rcu *nodes[]; }; -/* A retired child array carries an optional node through its RCU grace period. */ -struct stack_depot_trie_retired_array { +/* Retired children carry an optional node through their RCU grace period. */ +struct stack_depot_trie_retired_children { struct list_head list; unsigned long rcu_state; struct stack_depot_trie_node *pending_node; unsigned char data[]; }; -static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_retired_array, data), +static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_retired_children, data), 1UL << DEPOT_STACK_ALIGN)); #define STACK_DEPOT_TRIE_MAX_NODES (CONFIG_STACKDEPOT_MAX_FRAMES + 1) @@ -180,13 +180,13 @@ static_assert(STACK_DEPOT_TRIE_POOL_FIRST_SLOT < STACK_DEPOT_TRIE_POOL_SLOTS); struct stack_depot_trie_insert_alloc { struct stack_depot_trie_node *nodes[STACK_DEPOT_TRIE_MAX_NODES]; size_t node_sizes[STACK_DEPOT_TRIE_MAX_NODES]; - struct stack_depot_trie_child_array *chain_arrays[STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS]; - struct stack_depot_trie_child_array *prefix_children; - struct stack_depot_trie_child_array *slot_array; + struct stack_depot_trie_children *path_children[STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS]; + struct stack_depot_trie_children *prefix_children; + struct stack_depot_trie_children *new_children; }; static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); -static const struct stack_depot_trie_child_array __rcu *stack_depot_trie_root; +static const struct stack_depot_trie_children __rcu *stack_depot_trie_root; static struct stack_depot_trie_insert_alloc *stack_depot_trie_alloc; static DEFINE_RAW_SPINLOCK(stack_depot_trie_writer_lock); static bool stack_depot_trie_requested; @@ -199,7 +199,7 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); /* Retired fixed-size slots remain reserved until their RCU grace period ends. */ static LIST_HEAD(stack_depot_trie_pools); -static LIST_HEAD(pending_trie_arrays); +static LIST_HEAD(pending_trie_children); /* * stack_max_pools is the split point between hash and trie handle encodings. @@ -304,28 +304,28 @@ static inline size_t trie_node_bytes(const struct stack_depot_frame_run *run) stack_depot_frame_run_bytes(run), sizeof(unsigned long)); } -static size_t trie_child_array_bytes(unsigned int capacity) +static size_t trie_children_bytes(unsigned int capacity) { size_t size; - size = struct_size_t(struct stack_depot_trie_child_array, children, + size = struct_size_t(struct stack_depot_trie_children, nodes, capacity); return ALIGN(size, sizeof(unsigned long)); } -static void trie_child_array_init(struct stack_depot_trie_child_array *array, - unsigned int capacity, - const struct stack_depot_trie_node * const *nodes, - unsigned int nr_children) +static void trie_children_init(struct stack_depot_trie_children *children, + unsigned int capacity, + const struct stack_depot_trie_node * const *nodes, + unsigned int nr_children) { unsigned int i; - array->nr_children = nr_children; - array->capacity = capacity; + children->nr_children = nr_children; + children->capacity = capacity; for (i = 0; i < nr_children; i++) - RCU_INIT_POINTER(array->children[i], nodes[i]); + RCU_INIT_POINTER(children->nodes[i], nodes[i]); for (i = nr_children; i < capacity; i++) - RCU_INIT_POINTER(array->children[i], NULL); + RCU_INIT_POINTER(children->nodes[i], NULL); } static inline unsigned int trie_side_table_root_index(u32 id) @@ -609,15 +609,15 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( rcu_read_lock_sched_held()); } -static inline size_t trie_array_alloc_size(size_t size) +static inline size_t trie_children_alloc_size(size_t size) { - return offsetof(struct stack_depot_trie_retired_array, data) + size; + return offsetof(struct stack_depot_trie_retired_children, data) + size; } -static inline struct stack_depot_trie_retired_array * -trie_retired_array(const void *ptr) +static inline struct stack_depot_trie_retired_children * +trie_retired_children(const void *ptr) { - return container_of(ptr, struct stack_depot_trie_retired_array, data); + return container_of(ptr, struct stack_depot_trie_retired_children, data); } static bool depot_init_pool(void **prealloc); @@ -703,64 +703,64 @@ static void trie_pool_release(const void *ptr, size_t size) pool->free_slots += nr_slots; } -static struct stack_depot_trie_child_array * -trie_pool_alloc_array(size_t size, void **prealloc) +static struct stack_depot_trie_children * +trie_pool_alloc_children(size_t size, void **prealloc) { - struct stack_depot_trie_retired_array *retired; + struct stack_depot_trie_retired_children *retired; - retired = trie_pool_alloc(trie_array_alloc_size(size), prealloc); + retired = trie_pool_alloc(trie_children_alloc_size(size), prealloc); return retired ? (void *)retired->data : NULL; } -static void trie_pool_release_array(const void *ptr, size_t size) +static void trie_pool_release_children(const void *ptr, size_t size) { - trie_pool_release(trie_retired_array(ptr), trie_array_alloc_size(size)); + trie_pool_release(trie_retired_children(ptr), trie_children_alloc_size(size)); } -static void trie_drain_pending_arrays(void) +static void trie_drain_pending_children(void) { - struct stack_depot_trie_retired_array *retired; - struct stack_depot_trie_retired_array *tmp; - struct stack_depot_trie_child_array *array; + struct stack_depot_trie_retired_children *retired; + struct stack_depot_trie_retired_children *tmp; + struct stack_depot_trie_children *children; lockdep_assert_held(&pool_lock); - list_for_each_entry_safe(retired, tmp, &pending_trie_arrays, list) { - /* Pending arrays are FIFO; later entries cannot be ready yet. */ + list_for_each_entry_safe(retired, tmp, &pending_trie_children, list) { + /* Pending children are FIFO; later entries cannot be ready yet. */ if (!poll_state_synchronize_rcu(retired->rcu_state)) break; - array = (void *)retired->data; + children = (void *)retired->data; list_del(&retired->list); if (retired->pending_node) trie_pool_release(retired->pending_node, trie_node_bytes(&retired->pending_node->run)); - trie_pool_release_array(array, - trie_child_array_bytes(array->capacity)); + trie_pool_release_children(children, + trie_children_bytes(children->capacity)); } } -static void trie_retire_child_array(const void *ptr) +static void trie_retire_children(const void *ptr) { - struct stack_depot_trie_retired_array *retired; + struct stack_depot_trie_retired_children *retired; lockdep_assert_held(&pool_lock); - retired = trie_retired_array(ptr); + retired = trie_retired_children(ptr); retired->pending_node = NULL; retired->rcu_state = get_state_synchronize_rcu(); - list_add_tail(&retired->list, &pending_trie_arrays); + list_add_tail(&retired->list, &pending_trie_children); } static void -trie_retire_child_array_with_node(const void *ptr, - const struct stack_depot_trie_node *node) +trie_retire_children_with_node(const void *ptr, + const struct stack_depot_trie_node *node) { - struct stack_depot_trie_retired_array *retired; + struct stack_depot_trie_retired_children *retired; unsigned long flags; raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_child_array(ptr); - retired = trie_retired_array(ptr); + trie_retire_children(ptr); + retired = trie_retired_children(ptr); retired->pending_node = (struct stack_depot_trie_node *)node; raw_spin_unlock_irqrestore(&pool_lock, flags); } @@ -774,9 +774,9 @@ trie_retire_child_array_with_node(const void *ptr, static int trie_pool_alloc_insert(struct stack_depot_trie_insert_alloc *alloc, void **pool_prealloc, unsigned int nr_nodes, - unsigned int nr_chain_arrays, + unsigned int nr_path_children, size_t prefix_children_size, - size_t slot_array_size) + size_t new_children_size) { unsigned long flags; size_t one_child_size; @@ -784,37 +784,37 @@ static int trie_pool_alloc_insert(struct stack_depot_trie_insert_alloc *alloc, int ret = -ENOSPC; memset(alloc->nodes, 0, nr_nodes * sizeof(*alloc->nodes)); - memset(alloc->chain_arrays, 0, - nr_chain_arrays * sizeof(*alloc->chain_arrays)); + memset(alloc->path_children, 0, + nr_path_children * sizeof(*alloc->path_children)); alloc->prefix_children = NULL; - alloc->slot_array = NULL; + alloc->new_children = NULL; raw_spin_lock_irqsave(&pool_lock, flags); printk_deferred_enter(); - trie_drain_pending_arrays(); - one_child_size = trie_child_array_bytes(1); + trie_drain_pending_children(); + one_child_size = trie_children_bytes(1); for (i = 0; i < nr_nodes; i++) { alloc->nodes[i] = trie_pool_alloc(alloc->node_sizes[i], pool_prealloc); if (!alloc->nodes[i]) goto out_discard; } - for (i = 0; i < nr_chain_arrays; i++) { - alloc->chain_arrays[i] = - trie_pool_alloc_array(one_child_size, pool_prealloc); - if (!alloc->chain_arrays[i]) + for (i = 0; i < nr_path_children; i++) { + alloc->path_children[i] = + trie_pool_alloc_children(one_child_size, pool_prealloc); + if (!alloc->path_children[i]) goto out_discard; } if (prefix_children_size) { alloc->prefix_children = - trie_pool_alloc_array(prefix_children_size, pool_prealloc); + trie_pool_alloc_children(prefix_children_size, pool_prealloc); if (!alloc->prefix_children) goto out_discard; } - if (slot_array_size) { - alloc->slot_array = - trie_pool_alloc_array(slot_array_size, pool_prealloc); - if (!alloc->slot_array) + if (new_children_size) { + alloc->new_children = + trie_pool_alloc_children(new_children_size, pool_prealloc); + if (!alloc->new_children) goto out_discard; } ret = 0; @@ -826,18 +826,18 @@ static int trie_pool_alloc_insert(struct stack_depot_trie_insert_alloc *alloc, if (node) trie_pool_release(node, alloc->node_sizes[i]); } - for (i = 0; i < nr_chain_arrays; i++) { - struct stack_depot_trie_child_array *array = alloc->chain_arrays[i]; + for (i = 0; i < nr_path_children; i++) { + struct stack_depot_trie_children *children = alloc->path_children[i]; - if (!array) + if (!children) continue; - trie_pool_release_array(array, one_child_size); + trie_pool_release_children(children, one_child_size); } if (alloc->prefix_children) - trie_pool_release_array(alloc->prefix_children, - prefix_children_size); - if (alloc->slot_array) - trie_pool_release_array(alloc->slot_array, slot_array_size); + trie_pool_release_children(alloc->prefix_children, + prefix_children_size); + if (alloc->new_children) + trie_pool_release_children(alloc->new_children, new_children_size); out: printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -1763,8 +1763,8 @@ trie_load_parent(const struct stack_depot_trie_node *node) rcu_read_lock_sched_held()); } -static inline const struct stack_depot_trie_child_array * -trie_load_children_slot(const struct stack_depot_trie_child_array __rcu * const *slot) +static inline const struct stack_depot_trie_children * +trie_load_children(const struct stack_depot_trie_children __rcu * const *slot) { return rcu_dereference_check(*slot, lockdep_is_held(&stack_depot_trie_writer_lock) || @@ -1772,40 +1772,40 @@ trie_load_children_slot(const struct stack_depot_trie_child_array __rcu * const } static inline const struct stack_depot_trie_node * -trie_child_array_load_child(const struct stack_depot_trie_child_array *array, - unsigned int pos) +trie_children_load_child(const struct stack_depot_trie_children *children, + unsigned int pos) { - return rcu_dereference_check(array->children[pos], + return rcu_dereference_check(children->nodes[pos], lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } /* - * Find the child slot for @frame. + * Find the child position for @frame. * - * Return true and set @pos to the matching child index when @array contains a - * matching child. Return false and set @pos to the insertion index otherwise. - * A NULL @array is treated as empty and returns @pos = 0. + * Return true and set @pos to the matching child index when @children contains + * a matching child. Return false and set @pos to the insertion index otherwise. + * A NULL @children is treated as empty and returns @pos = 0. */ static bool -trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, - unsigned long frame, unsigned int *pos) +trie_children_find_slot(const struct stack_depot_trie_children *children, + unsigned long frame, unsigned int *pos) { unsigned int left = 0; unsigned int right; - if (!array) { + if (!children) { *pos = 0; return false; } - right = READ_ONCE(array->nr_children); + right = READ_ONCE(children->nr_children); while (left < right) { unsigned int mid = left + (right - left) / 2; const struct stack_depot_trie_node *node; unsigned long mid_frame; - node = trie_child_array_load_child(array, mid); + node = trie_children_load_child(children, mid); if (!node) { /* Tail append may produce a transient lockless lookup miss. */ right = mid; @@ -1827,54 +1827,54 @@ trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, } static void -trie_child_array_insert_at(const struct stack_depot_trie_child_array *old, - unsigned int pos, - const struct stack_depot_trie_node *node, - struct stack_depot_trie_child_array *new_array, - unsigned int new_capacity) +trie_children_insert_at(const struct stack_depot_trie_children *old_children, + unsigned int pos, + const struct stack_depot_trie_node *node, + struct stack_depot_trie_children *new_children, + unsigned int new_capacity) { unsigned int nr_old; unsigned int i; - nr_old = old->nr_children; + nr_old = old_children->nr_children; - new_array->nr_children = nr_old + 1; - new_array->capacity = new_capacity; + new_children->nr_children = nr_old + 1; + new_children->capacity = new_capacity; for (i = 0; i < pos; i++) - RCU_INIT_POINTER(new_array->children[i], - trie_child_array_load_child(old, i)); - RCU_INIT_POINTER(new_array->children[pos], node); + RCU_INIT_POINTER(new_children->nodes[i], + trie_children_load_child(old_children, i)); + RCU_INIT_POINTER(new_children->nodes[pos], node); for (i = pos; i < nr_old; i++) - RCU_INIT_POINTER(new_array->children[i + 1], - trie_child_array_load_child(old, i)); - for (i = nr_old + 1; i < new_array->capacity; i++) - RCU_INIT_POINTER(new_array->children[i], NULL); + RCU_INIT_POINTER(new_children->nodes[i + 1], + trie_children_load_child(old_children, i)); + for (i = nr_old + 1; i < new_children->capacity; i++) + RCU_INIT_POINTER(new_children->nodes[i], NULL); } static void -trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array, - const struct stack_depot_trie_node *new_child, - struct stack_depot_trie_child_array *new_array, - unsigned int pos) +trie_children_replace_at(const struct stack_depot_trie_children *old_children, + const struct stack_depot_trie_node *new_child, + struct stack_depot_trie_children *new_children, + unsigned int pos) { unsigned int i; - new_array->nr_children = old_array->nr_children; - new_array->capacity = old_array->capacity; - for (i = 0; i < old_array->nr_children; i++) - RCU_INIT_POINTER(new_array->children[i], - trie_child_array_load_child(old_array, i)); - RCU_INIT_POINTER(new_array->children[pos], new_child); - for (i = old_array->nr_children; i < new_array->capacity; i++) - RCU_INIT_POINTER(new_array->children[i], NULL); + new_children->nr_children = old_children->nr_children; + new_children->capacity = old_children->capacity; + for (i = 0; i < old_children->nr_children; i++) + RCU_INIT_POINTER(new_children->nodes[i], + trie_children_load_child(old_children, i)); + RCU_INIT_POINTER(new_children->nodes[pos], new_child); + for (i = old_children->nr_children; i < new_children->capacity; i++) + RCU_INIT_POINTER(new_children->nodes[i], NULL); } static void trie_reparent_children(struct stack_depot_trie_node *parent) { - const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_children *children; unsigned int i; - children = trie_load_children_slot(&parent->children); + children = trie_load_children(&parent->children); if (!children) return; /* @@ -1886,7 +1886,7 @@ static void trie_reparent_children(struct stack_depot_trie_node *parent) for (i = 0; i < children->nr_children; i++) { struct stack_depot_trie_node *child; - child = (struct stack_depot_trie_node *)trie_child_array_load_child(children, i); + child = (struct stack_depot_trie_node *)trie_children_load_child(children, i); rcu_assign_pointer(child->parent, parent); } } @@ -1895,9 +1895,9 @@ static void trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_node * const *nodes, - struct stack_depot_trie_child_array * const *chain_arrays, - const struct stack_depot_trie_node **head, - const struct stack_depot_trie_node **tail) + struct stack_depot_trie_children * const *path_children, + const struct stack_depot_trie_node **path_head, + const struct stack_depot_trie_node **path_leaf) { const struct stack_depot_trie_node *prev = parent; unsigned int pos = 0; @@ -1923,19 +1923,19 @@ trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, for (i = 0; i + 1 < used; i++) { const struct stack_depot_trie_node *next = nodes[i + 1]; struct stack_depot_trie_node *node = nodes[i]; - struct stack_depot_trie_child_array *array = chain_arrays[i]; + struct stack_depot_trie_children *children = path_children[i]; - trie_child_array_init(array, 1, &next, 1); - RCU_INIT_POINTER(node->children, array); + trie_children_init(children, 1, &next, 1); + RCU_INIT_POINTER(node->children, children); } - *head = nodes[0]; - *tail = nodes[used - 1]; + *path_head = nodes[0]; + *path_leaf = nodes[used - 1]; } -static void trie_publish_tail_append(struct stack_depot_trie_child_array *array, +static void trie_publish_tail_append(struct stack_depot_trie_children *children, unsigned int pos, - const struct stack_depot_trie_node *head) + const struct stack_depot_trie_node *child) { /* * Writers hold stack_depot_trie_writer_lock. Existing children are @@ -1944,36 +1944,36 @@ static void trie_publish_tail_append(struct stack_depot_trie_child_array *array, * see the new count either load the initialized child or treat a transient * NULL as a miss. The writer-lock recheck prevents permanent duplicates. */ - rcu_assign_pointer(array->children[pos], head); - WRITE_ONCE(array->nr_children, pos + 1); + rcu_assign_pointer(children->nodes[pos], child); + WRITE_ONCE(children->nr_children, pos + 1); } static const struct stack_depot_trie_node * stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) { - const struct stack_depot_trie_child_array *children; - unsigned int pos = 0; + const struct stack_depot_trie_children *children; + unsigned int entry = 0; - children = trie_load_children_slot(&stack_depot_trie_root); + children = trie_load_children(&stack_depot_trie_root); - while (pos < nr_entries) { + while (entry < nr_entries) { const struct stack_depot_trie_node *node; - unsigned int remaining = nr_entries - pos; + unsigned int remaining = nr_entries - entry; unsigned int matched; - unsigned int slot; + unsigned int pos; - if (!trie_child_array_find_slot(children, entries[pos], &slot)) + if (!trie_children_find_slot(children, entries[entry], &pos)) return NULL; - node = trie_child_array_load_child(children, slot); - matched = __stack_depot_trie_node_match(node, &entries[pos], remaining); + node = trie_children_load_child(children, pos); + matched = __stack_depot_trie_node_match(node, &entries[entry], remaining); if (matched < node->run.nr_entries) return NULL; - pos += matched; - if (pos == nr_entries) + entry += matched; + if (entry == nr_entries) return node->leaf_id ? node : NULL; - children = trie_load_children_slot(&node->children); + children = trie_load_children(&node->children); } return NULL; @@ -1985,28 +1985,28 @@ static void trie_build_split(const struct stack_depot_trie_node *child, struct stack_depot_trie_node *prefix, struct stack_depot_trie_node *old_tail, struct stack_depot_trie_node * const *new_nodes, - struct stack_depot_trie_child_array * const *chain_arrays, - struct stack_depot_trie_child_array *prefix_children, + struct stack_depot_trie_children * const *path_children, + struct stack_depot_trie_children *prefix_children, const struct stack_depot_trie_node **new_leaf) { - const struct stack_depot_trie_child_array *child_children; - const struct stack_depot_trie_node *child_parent; + const struct stack_depot_trie_children *old_children; + const struct stack_depot_trie_node *old_parent; const unsigned long *tail_entries; - const struct stack_depot_trie_node *split_children[2]; - const struct stack_depot_trie_node *new_head = NULL; - const struct stack_depot_trie_node *new_tail = NULL; + const struct stack_depot_trie_node *split_nodes[2]; + const struct stack_depot_trie_node *new_tail_head = NULL; + const struct stack_depot_trie_node *new_tail_leaf = NULL; unsigned long new_frame; unsigned long old_frame; u32 prefix_leaf_id; unsigned int tail_len; bool has_new_tail; - child_children = trie_load_children_slot(&child->children); - child_parent = trie_load_parent(child); + old_children = trie_load_children(&child->children); + old_parent = trie_load_parent(child); has_new_tail = matched < nr_entries; prefix_leaf_id = has_new_tail ? 0 : leaf_id; - trie_node_init_slice(prefix, child_parent, prefix_leaf_id, child, 0, + trie_node_init_slice(prefix, old_parent, prefix_leaf_id, child, 0, matched); tail_len = child->run.nr_entries - matched; trie_node_init_slice(old_tail, prefix, child->leaf_id, child, matched, @@ -2016,26 +2016,26 @@ static void trie_build_split(const struct stack_depot_trie_node *child, tail_entries = &entries[matched]; tail_len = nr_entries - matched; trie_build_append_chain(prefix, leaf_id, tail_entries, tail_len, - new_nodes, chain_arrays, - &new_head, &new_tail); + new_nodes, path_children, + &new_tail_head, &new_tail_leaf); stack_depot_trie_node_frame(old_tail, 0, &old_frame); - stack_depot_trie_node_frame(new_head, 0, &new_frame); + stack_depot_trie_node_frame(new_tail_head, 0, &new_frame); if (old_frame < new_frame) { - split_children[0] = old_tail; - split_children[1] = new_head; + split_nodes[0] = old_tail; + split_nodes[1] = new_tail_head; } else { - split_children[0] = new_head; - split_children[1] = old_tail; + split_nodes[0] = new_tail_head; + split_nodes[1] = old_tail; } - trie_child_array_init(prefix_children, ARRAY_SIZE(split_children), - split_children, ARRAY_SIZE(split_children)); + trie_children_init(prefix_children, ARRAY_SIZE(split_nodes), + split_nodes, ARRAY_SIZE(split_nodes)); } else { - split_children[0] = old_tail; - trie_child_array_init(prefix_children, 1, split_children, 1); + split_nodes[0] = old_tail; + trie_children_init(prefix_children, 1, split_nodes, 1); } - RCU_INIT_POINTER(old_tail->children, child_children); + RCU_INIT_POINTER(old_tail->children, old_children); RCU_INIT_POINTER(prefix->children, prefix_children); - *new_leaf = has_new_tail ? new_tail : prefix; + *new_leaf = has_new_tail ? new_tail_leaf : prefix; } static unsigned int trie_size_append_chain(const unsigned long *entries, @@ -2058,34 +2058,34 @@ static unsigned int trie_size_append_chain(const unsigned long *entries, } static u32 -trie_insert_path(const struct stack_depot_trie_child_array __rcu **slot, - struct stack_depot_trie_node *parent, - const struct stack_depot_trie_child_array *children, - unsigned int pos, const unsigned long *entries, - unsigned int nr_entries, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) +trie_insert_path(const struct stack_depot_trie_children __rcu **slot, + struct stack_depot_trie_node *parent, + const struct stack_depot_trie_children *children, + unsigned int pos, const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_child_array *slot_array; - struct stack_depot_trie_child_array *tail_array; - const struct stack_depot_trie_node *chain_head; - const struct stack_depot_trie_node *chain_leaf; + struct stack_depot_trie_children *new_children; + struct stack_depot_trie_children *tail_children; + const struct stack_depot_trie_node *path_head; + const struct stack_depot_trie_node *path_leaf; unsigned int capacity = 1; unsigned int nr_nodes; unsigned long flags; u32 new_leaf_id; - size_t slot_array_size; + size_t new_children_size; bool tail_append = false; if (children) { capacity = roundup_pow_of_two(children->nr_children + 1); tail_append = pos == children->nr_children && children->nr_children < children->capacity; - slot_array_size = tail_append ? 0 : trie_child_array_bytes(capacity); + new_children_size = tail_append ? 0 : trie_children_bytes(capacity); } else { - slot_array_size = trie_child_array_bytes(capacity); + new_children_size = trie_children_bytes(capacity); } - if (slot_array_size > DEPOT_POOL_SIZE) + if (new_children_size > DEPOT_POOL_SIZE) return 0; new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); @@ -2093,32 +2093,32 @@ trie_insert_path(const struct stack_depot_trie_child_array __rcu **slot, return 0; nr_nodes = trie_size_append_chain(entries, nr_entries, alloc->node_sizes); if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, nr_nodes - 1, - 0, slot_array_size)) + 0, new_children_size)) return 0; trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - alloc->nodes, alloc->chain_arrays, - &chain_head, &chain_leaf); - trie_side_table_publish_new_leaf(new_leaf_id, chain_leaf); + alloc->nodes, alloc->path_children, + &path_head, &path_leaf); + trie_side_table_publish_new_leaf(new_leaf_id, path_leaf); if (!children) { - slot_array = alloc->slot_array; - slot_array->nr_children = 1; - slot_array->capacity = 1; - RCU_INIT_POINTER(slot_array->children[0], chain_head); - rcu_assign_pointer(*slot, slot_array); + new_children = alloc->new_children; + new_children->nr_children = 1; + new_children->capacity = 1; + RCU_INIT_POINTER(new_children->nodes[0], path_head); + rcu_assign_pointer(*slot, new_children); return new_leaf_id; } if (tail_append) { - tail_array = (struct stack_depot_trie_child_array *)children; - trie_publish_tail_append(tail_array, pos, chain_head); + tail_children = (struct stack_depot_trie_children *)children; + trie_publish_tail_append(tail_children, pos, path_head); } else { - slot_array = alloc->slot_array; - trie_child_array_insert_at(children, pos, chain_head, - slot_array, capacity); - rcu_assign_pointer(*slot, slot_array); + new_children = alloc->new_children; + trie_children_insert_at(children, pos, path_head, new_children, + capacity); + rcu_assign_pointer(*slot, new_children); raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_child_array(children); + trie_retire_children(children); raw_spin_unlock_irqrestore(&pool_lock, flags); } @@ -2126,17 +2126,17 @@ trie_insert_path(const struct stack_depot_trie_child_array __rcu **slot, } static u32 -trie_split_child(const struct stack_depot_trie_child_array __rcu **slot, - const struct stack_depot_trie_child_array *children, - const struct stack_depot_trie_node *child, - unsigned int pos, unsigned int matched, - const unsigned long *entries, unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) +trie_split_child(const struct stack_depot_trie_children __rcu **slot, + const struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *child, + unsigned int pos, unsigned int matched, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_child_array *prefix_children; - struct stack_depot_trie_child_array *slot_array; + struct stack_depot_trie_children *prefix_children; + struct stack_depot_trie_children *new_children; const struct stack_depot_trie_node *split_leaf; struct stack_depot_frame_run run; struct stack_depot_trie_node *split_prefix; @@ -2160,59 +2160,59 @@ trie_split_child(const struct stack_depot_trie_child_array __rcu **slot, nr_entries - matched, &alloc->node_sizes[2]); if (trie_pool_alloc_insert(alloc, pool_prealloc, 2 + new_nodes, - new_nodes ? new_nodes - 1 : 0, - trie_child_array_bytes(has_new_tail ? 2 : 1), - trie_child_array_bytes(children->capacity))) + new_nodes ? new_nodes - 1 : 0, + trie_children_bytes(has_new_tail ? 2 : 1), + trie_children_bytes(children->capacity))) return 0; - slot_array = alloc->slot_array; + new_children = alloc->new_children; prefix_children = alloc->prefix_children; split_prefix = alloc->nodes[0]; old_tail = alloc->nodes[1]; trie_build_split(child, matched, new_leaf_id, entries, nr_entries, split_prefix, old_tail, &alloc->nodes[2], - alloc->chain_arrays, prefix_children, &split_leaf); + alloc->path_children, prefix_children, &split_leaf); trie_side_table_publish_split_leaves(child->leaf_id, old_tail, new_leaf_id, split_leaf); - trie_child_array_replace_at(children, split_prefix, slot_array, pos); + trie_children_replace_at(children, split_prefix, new_children, pos); trie_reparent_children(old_tail); - rcu_assign_pointer(*slot, slot_array); - trie_retire_child_array_with_node(children, child); + rcu_assign_pointer(*slot, new_children); + trie_retire_children_with_node(children, child); return new_leaf_id; } static u32 -trie_promote_child(const struct stack_depot_trie_child_array __rcu **slot, - const struct stack_depot_trie_child_array *children, - const struct stack_depot_trie_node *child, - unsigned int pos, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) +trie_promote_child(const struct stack_depot_trie_children __rcu **slot, + const struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *child, + unsigned int pos, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_child_array *slot_array; + struct stack_depot_trie_children *new_children; struct stack_depot_trie_node *promoted_node; u32 new_leaf_id; - size_t slot_array_size; + size_t new_children_size; new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; alloc->node_sizes[0] = trie_node_bytes(&child->run); - slot_array_size = trie_child_array_bytes(children->capacity); + new_children_size = trie_children_bytes(children->capacity); if (trie_pool_alloc_insert(alloc, pool_prealloc, 1, 0, 0, - slot_array_size)) + new_children_size)) return 0; promoted_node = alloc->nodes[0]; - slot_array = alloc->slot_array; + new_children = alloc->new_children; memcpy(promoted_node, child, alloc->node_sizes[0]); promoted_node->leaf_id = new_leaf_id; trie_side_table_publish_new_leaf(new_leaf_id, promoted_node); - trie_child_array_replace_at(children, promoted_node, slot_array, pos); + trie_children_replace_at(children, promoted_node, new_children, pos); trie_reparent_children(promoted_node); - rcu_assign_pointer(*slot, slot_array); - trie_retire_child_array_with_node(children, child); + rcu_assign_pointer(*slot, new_children); + trie_retire_children_with_node(children, child); return new_leaf_id; } @@ -2230,8 +2230,8 @@ stack_depot_trie_insert(const unsigned long *entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { - const struct stack_depot_trie_child_array *children; - const struct stack_depot_trie_child_array __rcu **slot = + const struct stack_depot_trie_children *children; + const struct stack_depot_trie_children __rcu **slot = &stack_depot_trie_root; const struct stack_depot_trie_node *child; struct stack_depot_trie_node *parent = NULL; @@ -2240,9 +2240,9 @@ stack_depot_trie_insert(const unsigned long *entries, u32 leaf_id = 0; for (;;) { - children = trie_load_children_slot(slot); + children = trie_load_children(slot); /* Case 1: no matching child, so append the remaining stack path. */ - if (!trie_child_array_find_slot(children, entries[0], &pos)) { + if (!trie_children_find_slot(children, entries[0], &pos)) { leaf_id = trie_insert_path(slot, parent, children, pos, entries, nr_entries, pool_prealloc, @@ -2250,7 +2250,7 @@ stack_depot_trie_insert(const unsigned long *entries, break; } - child = trie_child_array_load_child(children, pos); + child = trie_children_load_child(children, pos); matched = __stack_depot_trie_node_match(child, entries, nr_entries); /* Case 2: the match ends inside the child run, so split it. */ From 8d43f65150ac9bd33785203e36171b83a41b0bcb Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Wed, 29 Jul 2026 15:01:43 +0100 Subject: [PATCH 08/13] KRN-1117: Inline trie insertion allocation and simplify split The generic trie_pool_alloc_insert() and trie_build_split() helpers grew several unrelated parameters and workspace slots to cover all three insertion cases at once, which made each caller harder to follow than the work it was actually doing. Inline the pool reservation and rollback into trie_insert_path(), trie_split_child(), and trie_promote_child() so each path allocates only what it needs and releases the exact set it took on failure. Make trie_pool_alloc_children() capacity-based and self-sizing, and let trie_pool_release_children() derive the size from the object it frees. Zero the insertion workspace once up front and drop the size-carrying release variant. Split handling now has a common setup, a single divergence branch that builds the new tail and orders the two suffixes, and shared finalisation. Node sizing is expressed via trie_node_bytes_for(mode, nr_entries) and local names spell out prefix/tail intent. No behaviour change intended. Validated with git diff --check, strict checkpatch on the diff, kbuild of lib/stackdepot.o and lib/tests/stackdepot_kunit.o, and stackdepot KUnit under UML. --- lib/stackdepot.c | 428 +++++++++++++++++++++++++---------------------- 1 file changed, 229 insertions(+), 199 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index a170c5a35c15e..5b32c6f0061e9 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -298,10 +298,17 @@ static inline size_t stack_depot_frame_run_bytes(const struct stack_depot_frame_ return run->nr_entries * stack_depot_frame_run_entry_bytes(run->mode); } -static inline size_t trie_node_bytes(const struct stack_depot_frame_run *run) +static inline size_t trie_node_bytes_for(enum stack_depot_frame_mode mode, + unsigned int nr_entries) { return ALIGN(offsetof(struct stack_depot_trie_node, data) + - stack_depot_frame_run_bytes(run), sizeof(unsigned long)); + nr_entries * stack_depot_frame_run_entry_bytes(mode), + sizeof(unsigned long)); +} + +static inline size_t trie_node_bytes(const struct stack_depot_frame_run *run) +{ + return trie_node_bytes_for(run->mode, run->nr_entries); } static size_t trie_children_bytes(unsigned int capacity) @@ -704,17 +711,31 @@ static void trie_pool_release(const void *ptr, size_t size) } static struct stack_depot_trie_children * -trie_pool_alloc_children(size_t size, void **prealloc) +trie_pool_alloc_children(unsigned int capacity, void **prealloc) { struct stack_depot_trie_retired_children *retired; + struct stack_depot_trie_children *children; + size_t size; + size = trie_children_bytes(capacity); + if (size > DEPOT_POOL_SIZE) + return NULL; retired = trie_pool_alloc(trie_children_alloc_size(size), prealloc); - return retired ? (void *)retired->data : NULL; + if (!retired) + return NULL; + + children = (void *)retired->data; + children->nr_children = 0; + children->capacity = capacity; + return children; } -static void trie_pool_release_children(const void *ptr, size_t size) +static void trie_pool_release_children(const struct stack_depot_trie_children *children) { - trie_pool_release(trie_retired_children(ptr), trie_children_alloc_size(size)); + size_t size = trie_children_bytes(children->capacity); + + trie_pool_release(trie_retired_children(children), + trie_children_alloc_size(size)); } static void trie_drain_pending_children(void) @@ -734,8 +755,7 @@ static void trie_drain_pending_children(void) if (retired->pending_node) trie_pool_release(retired->pending_node, trie_node_bytes(&retired->pending_node->run)); - trie_pool_release_children(children, - trie_children_bytes(children->capacity)); + trie_pool_release_children(children); } } @@ -765,85 +785,6 @@ trie_retire_children_with_node(const void *ptr, raw_spin_unlock_irqrestore(&pool_lock, flags); } -/* - * Reserve fixed-size pool slots for one trie insertion. If any reservation - * fails, all slots reserved by this attempt are released locally. - * The caller must not publish any returned storage before side-table and trie - * publication succeeds. - */ -static int trie_pool_alloc_insert(struct stack_depot_trie_insert_alloc *alloc, - void **pool_prealloc, - unsigned int nr_nodes, - unsigned int nr_path_children, - size_t prefix_children_size, - size_t new_children_size) -{ - unsigned long flags; - size_t one_child_size; - unsigned int i; - int ret = -ENOSPC; - - memset(alloc->nodes, 0, nr_nodes * sizeof(*alloc->nodes)); - memset(alloc->path_children, 0, - nr_path_children * sizeof(*alloc->path_children)); - alloc->prefix_children = NULL; - alloc->new_children = NULL; - - raw_spin_lock_irqsave(&pool_lock, flags); - printk_deferred_enter(); - trie_drain_pending_children(); - one_child_size = trie_children_bytes(1); - for (i = 0; i < nr_nodes; i++) { - alloc->nodes[i] = trie_pool_alloc(alloc->node_sizes[i], - pool_prealloc); - if (!alloc->nodes[i]) - goto out_discard; - } - for (i = 0; i < nr_path_children; i++) { - alloc->path_children[i] = - trie_pool_alloc_children(one_child_size, pool_prealloc); - if (!alloc->path_children[i]) - goto out_discard; - } - if (prefix_children_size) { - alloc->prefix_children = - trie_pool_alloc_children(prefix_children_size, pool_prealloc); - if (!alloc->prefix_children) - goto out_discard; - } - if (new_children_size) { - alloc->new_children = - trie_pool_alloc_children(new_children_size, pool_prealloc); - if (!alloc->new_children) - goto out_discard; - } - ret = 0; - goto out; -out_discard: - for (i = 0; i < nr_nodes; i++) { - struct stack_depot_trie_node *node = alloc->nodes[i]; - - if (node) - trie_pool_release(node, alloc->node_sizes[i]); - } - for (i = 0; i < nr_path_children; i++) { - struct stack_depot_trie_children *children = alloc->path_children[i]; - - if (!children) - continue; - trie_pool_release_children(children, one_child_size); - } - if (alloc->prefix_children) - trie_pool_release_children(alloc->prefix_children, - prefix_children_size); - if (alloc->new_children) - trie_pool_release_children(alloc->new_children, new_children_size); -out: - printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); - return ret; -} - static const struct stack_depot_trie_node * stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries); @@ -1979,65 +1920,6 @@ stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) return NULL; } -static void trie_build_split(const struct stack_depot_trie_node *child, - unsigned int matched, u32 leaf_id, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_node *prefix, - struct stack_depot_trie_node *old_tail, - struct stack_depot_trie_node * const *new_nodes, - struct stack_depot_trie_children * const *path_children, - struct stack_depot_trie_children *prefix_children, - const struct stack_depot_trie_node **new_leaf) -{ - const struct stack_depot_trie_children *old_children; - const struct stack_depot_trie_node *old_parent; - const unsigned long *tail_entries; - const struct stack_depot_trie_node *split_nodes[2]; - const struct stack_depot_trie_node *new_tail_head = NULL; - const struct stack_depot_trie_node *new_tail_leaf = NULL; - unsigned long new_frame; - unsigned long old_frame; - u32 prefix_leaf_id; - unsigned int tail_len; - bool has_new_tail; - - old_children = trie_load_children(&child->children); - old_parent = trie_load_parent(child); - - has_new_tail = matched < nr_entries; - prefix_leaf_id = has_new_tail ? 0 : leaf_id; - trie_node_init_slice(prefix, old_parent, prefix_leaf_id, child, 0, - matched); - tail_len = child->run.nr_entries - matched; - trie_node_init_slice(old_tail, prefix, child->leaf_id, child, matched, - tail_len); - - if (has_new_tail) { - tail_entries = &entries[matched]; - tail_len = nr_entries - matched; - trie_build_append_chain(prefix, leaf_id, tail_entries, tail_len, - new_nodes, path_children, - &new_tail_head, &new_tail_leaf); - stack_depot_trie_node_frame(old_tail, 0, &old_frame); - stack_depot_trie_node_frame(new_tail_head, 0, &new_frame); - if (old_frame < new_frame) { - split_nodes[0] = old_tail; - split_nodes[1] = new_tail_head; - } else { - split_nodes[0] = new_tail_head; - split_nodes[1] = old_tail; - } - trie_children_init(prefix_children, ARRAY_SIZE(split_nodes), - split_nodes, ARRAY_SIZE(split_nodes)); - } else { - split_nodes[0] = old_tail; - trie_children_init(prefix_children, 1, split_nodes, 1); - } - RCU_INIT_POINTER(old_tail->children, old_children); - RCU_INIT_POINTER(prefix->children, prefix_children); - *new_leaf = has_new_tail ? new_tail_leaf : prefix; -} - static unsigned int trie_size_append_chain(const unsigned long *entries, unsigned int nr_entries, size_t *node_sizes) @@ -2071,30 +1953,55 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, const struct stack_depot_trie_node *path_head; const struct stack_depot_trie_node *path_leaf; unsigned int capacity = 1; - unsigned int nr_nodes; + unsigned int new_children_capacity = 1; + unsigned int i, nr_nodes; unsigned long flags; u32 new_leaf_id; - size_t new_children_size; bool tail_append = false; if (children) { capacity = roundup_pow_of_two(children->nr_children + 1); tail_append = pos == children->nr_children && children->nr_children < children->capacity; - new_children_size = tail_append ? 0 : trie_children_bytes(capacity); - } else { - new_children_size = trie_children_bytes(capacity); + if (tail_append) + new_children_capacity = 0; + else + new_children_capacity = capacity; } - if (new_children_size > DEPOT_POOL_SIZE) - return 0; - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; + + memset(alloc, 0, sizeof(*alloc)); nr_nodes = trie_size_append_chain(entries, nr_entries, alloc->node_sizes); - if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, nr_nodes - 1, - 0, new_children_size)) - return 0; + + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + trie_drain_pending_children(); + + for (i = 0; i < nr_nodes; i++) { + alloc->nodes[i] = trie_pool_alloc(alloc->node_sizes[i], pool_prealloc); + if (!alloc->nodes[i]) + goto err_release; + } + + for (i = 0; i < nr_nodes - 1; i++) { + alloc->path_children[i] = + trie_pool_alloc_children(1, pool_prealloc); + if (!alloc->path_children[i]) + goto err_release; + } + + if (new_children_capacity) { + alloc->new_children = trie_pool_alloc_children(new_children_capacity, + pool_prealloc); + if (!alloc->new_children) + goto err_release; + } + + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, alloc->nodes, alloc->path_children, &path_head, &path_leaf); @@ -2123,6 +2030,21 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, } return new_leaf_id; + +err_release: + for (i = 0; i < ARRAY_SIZE(alloc->nodes); i++) { + if (alloc->nodes[i]) + trie_pool_release(alloc->nodes[i], alloc->node_sizes[i]); + } + for (i = 0; i < ARRAY_SIZE(alloc->path_children); i++) { + if (alloc->path_children[i]) + trie_pool_release_children(alloc->path_children[i]); + } + if (alloc->new_children) + trie_pool_release_children(alloc->new_children); + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return 0; } static u32 @@ -2135,51 +2057,143 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, struct stack_depot_trie_side_prealloc *side_prealloc) { struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_children *prefix_children; - struct stack_depot_trie_children *new_children; + struct stack_depot_trie_children *prefix_children, *new_children; const struct stack_depot_trie_node *split_leaf; - struct stack_depot_frame_run run; - struct stack_depot_trie_node *split_prefix; - struct stack_depot_trie_node *old_tail; - unsigned int new_nodes = 0; - u32 new_leaf_id; - bool has_new_tail; + const struct stack_depot_trie_node *split_nodes[2]; + struct stack_depot_trie_node *split_prefix, *old_tail; + unsigned int nr_split_nodes; + unsigned int i, old_tail_len; + unsigned long flags; + u32 new_leaf_id, prefix_leaf_id; new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - run = child->run; - run.nr_entries = matched; - alloc->node_sizes[0] = trie_node_bytes(&run); - run.nr_entries = child->run.nr_entries - matched; - alloc->node_sizes[1] = trie_node_bytes(&run); - has_new_tail = matched < nr_entries; - if (has_new_tail) - new_nodes = trie_size_append_chain(&entries[matched], - nr_entries - matched, - &alloc->node_sizes[2]); - if (trie_pool_alloc_insert(alloc, pool_prealloc, 2 + new_nodes, - new_nodes ? new_nodes - 1 : 0, - trie_children_bytes(has_new_tail ? 2 : 1), - trie_children_bytes(children->capacity))) - return 0; + memset(alloc, 0, sizeof(*alloc)); + + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + trie_drain_pending_children(); + + alloc->node_sizes[0] = trie_node_bytes_for(child->run.mode, matched); + split_prefix = trie_pool_alloc(alloc->node_sizes[0], pool_prealloc); + if (!split_prefix) + goto err_release; + alloc->nodes[0] = split_prefix; + + old_tail_len = child->run.nr_entries - matched; + alloc->node_sizes[1] = trie_node_bytes_for(child->run.mode, old_tail_len); + old_tail = trie_pool_alloc(alloc->node_sizes[1], pool_prealloc); + if (!old_tail) + goto err_release; + alloc->nodes[1] = old_tail; + + new_children = trie_pool_alloc_children(children->capacity, pool_prealloc); + if (!new_children) + goto err_release; + alloc->new_children = new_children; + + if (matched < nr_entries) { + /* + * The new and existing stacks diverge inside this child. Keep the + * existing suffix as old_tail and build a second tail for the new + * stack. + */ + const struct stack_depot_trie_node *new_tail; + unsigned int new_tail_len; + unsigned int nr_tail_nodes; + unsigned long frame; + + prefix_leaf_id = 0; + nr_split_nodes = 2; + new_tail_len = nr_entries - matched; + nr_tail_nodes = trie_size_append_chain(&entries[matched], + new_tail_len, + &alloc->node_sizes[2]); - new_children = alloc->new_children; - prefix_children = alloc->prefix_children; - split_prefix = alloc->nodes[0]; - old_tail = alloc->nodes[1]; - trie_build_split(child, matched, new_leaf_id, entries, nr_entries, - split_prefix, old_tail, &alloc->nodes[2], - alloc->path_children, prefix_children, &split_leaf); - trie_side_table_publish_split_leaves(child->leaf_id, old_tail, - new_leaf_id, split_leaf); + for (i = 0; i < nr_tail_nodes; i++) { + struct stack_depot_trie_node *node; + + node = trie_pool_alloc(alloc->node_sizes[2 + i], pool_prealloc); + if (!node) + goto err_release; + alloc->nodes[2 + i] = node; + } + + for (i = 0; i < nr_tail_nodes - 1; i++) { + struct stack_depot_trie_children *children; + + children = trie_pool_alloc_children(1, pool_prealloc); + if (!children) + goto err_release; + alloc->path_children[i] = children; + } + + trie_build_append_chain(split_prefix, new_leaf_id, &entries[matched], + new_tail_len, &alloc->nodes[2], + alloc->path_children, &new_tail, + &split_leaf); + stack_depot_trie_node_frame(child, matched, &frame); + if (frame < entries[matched]) { + split_nodes[0] = old_tail; + split_nodes[1] = new_tail; + } else { + split_nodes[0] = new_tail; + split_nodes[1] = old_tail; + } + } else { + /* The new stack ends at the shared prefix. */ + prefix_leaf_id = new_leaf_id; + nr_split_nodes = 1; + split_leaf = split_prefix; + split_nodes[0] = old_tail; + } + + prefix_children = trie_pool_alloc_children(nr_split_nodes, pool_prealloc); + if (!prefix_children) + goto err_release; + alloc->prefix_children = prefix_children; + + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + + /* Keep the shared prefix under the old parent. */ + trie_node_init_slice(split_prefix, trie_load_parent(child), prefix_leaf_id, + child, 0, matched); + trie_node_init_slice(old_tail, split_prefix, child->leaf_id, child, matched, + old_tail_len); + trie_children_init(prefix_children, nr_split_nodes, split_nodes, + nr_split_nodes); + + /* Keep the old children under the old tail. */ + RCU_INIT_POINTER(old_tail->children, trie_load_children(&child->children)); + RCU_INIT_POINTER(split_prefix->children, prefix_children); + + trie_side_table_publish_split_leaves(child->leaf_id, old_tail, new_leaf_id, split_leaf); trie_children_replace_at(children, split_prefix, new_children, pos); trie_reparent_children(old_tail); rcu_assign_pointer(*slot, new_children); trie_retire_children_with_node(children, child); return new_leaf_id; + +err_release: + for (i = 0; i < ARRAY_SIZE(alloc->nodes); i++) { + if (alloc->nodes[i]) + trie_pool_release(alloc->nodes[i], alloc->node_sizes[i]); + } + for (i = 0; i < ARRAY_SIZE(alloc->path_children); i++) { + if (alloc->path_children[i]) + trie_pool_release_children(alloc->path_children[i]); + } + if (alloc->prefix_children) + trie_pool_release_children(alloc->prefix_children); + if (alloc->new_children) + trie_pool_release_children(alloc->new_children); + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return 0; } static u32 @@ -2189,24 +2203,33 @@ trie_promote_child(const struct stack_depot_trie_children __rcu **slot, unsigned int pos, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { - struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; struct stack_depot_trie_children *new_children; struct stack_depot_trie_node *promoted_node; + unsigned long flags; + size_t node_size; u32 new_leaf_id; - size_t new_children_size; new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - alloc->node_sizes[0] = trie_node_bytes(&child->run); - new_children_size = trie_children_bytes(children->capacity); - if (trie_pool_alloc_insert(alloc, pool_prealloc, 1, 0, 0, - new_children_size)) - return 0; - promoted_node = alloc->nodes[0]; - new_children = alloc->new_children; - memcpy(promoted_node, child, alloc->node_sizes[0]); + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + trie_drain_pending_children(); + + node_size = trie_node_bytes(&child->run); + promoted_node = trie_pool_alloc(node_size, pool_prealloc); + if (!promoted_node) + goto out_unlock; + + new_children = trie_pool_alloc_children(children->capacity, pool_prealloc); + if (!new_children) + goto out_release_node; + + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + + memcpy(promoted_node, child, node_size); promoted_node->leaf_id = new_leaf_id; trie_side_table_publish_new_leaf(new_leaf_id, promoted_node); trie_children_replace_at(children, promoted_node, new_children, pos); @@ -2215,6 +2238,13 @@ trie_promote_child(const struct stack_depot_trie_children __rcu **slot, trie_retire_children_with_node(children, child); return new_leaf_id; + +out_release_node: + trie_pool_release(promoted_node, node_size); +out_unlock: + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return 0; } static u32 trie_finish_insert(u32 leaf_id) From 89a315ff7716154bb774d393d16d65075332caf0 Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Thu, 30 Jul 2026 09:02:04 +0100 Subject: [PATCH 09/13] KRN-1117: Simplify trie_insert_path() path build and rollback Drop the in-place tail append so published children are immutable and readers no longer need to tolerate transient NULL slots during a lockless search. trie_children_copy() now accepts a NULL source so the first-child case shares the copy-and-publish path. Build the new node chain incrementally with local pointers instead of the stack_depot_trie_alloc scratch arrays, and roll back on allocation failure by walking parent links from the leaf. --- lib/stackdepot.c | 181 +++++++++++++++++++++-------------------------- 1 file changed, 82 insertions(+), 99 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 5b32c6f0061e9..052bfc03c4c84 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -138,8 +138,7 @@ struct stack_depot_trie_node { /* * Children are sorted by first frame and searched by insertion position. - * Writers may append to spare capacity at the sorted tail, but never change - * existing child pointers. Other updates build and publish a replacement array. + * Published children are immutable; updates build and publish a replacement. */ struct stack_depot_trie_children { unsigned int nr_children; @@ -1747,11 +1746,6 @@ trie_children_find_slot(const struct stack_depot_trie_children *children, unsigned long mid_frame; node = trie_children_load_child(children, mid); - if (!node) { - /* Tail append may produce a transient lockless lookup miss. */ - right = mid; - continue; - } stack_depot_trie_node_frame(node, 0, &mid_frame); if (mid_frame < frame) { left = mid + 1; @@ -1767,17 +1761,28 @@ trie_children_find_slot(const struct stack_depot_trie_children *children, return false; } +/** + * trie_children_copy() - Copy children into a new container + * @new_children: Unpublished destination children container + * @new_capacity: Capacity of @new_children + * @old_children: Immutable source children container, or %NULL if empty + * @pos: Position at which to insert @node + * @node: Node to insert + * + * Copy @old_children into @new_children and insert @node without modifying the + * published source generation. + */ static void -trie_children_insert_at(const struct stack_depot_trie_children *old_children, - unsigned int pos, - const struct stack_depot_trie_node *node, - struct stack_depot_trie_children *new_children, - unsigned int new_capacity) +trie_children_copy(struct stack_depot_trie_children *new_children, + unsigned int new_capacity, + const struct stack_depot_trie_children *old_children, + unsigned int pos, + const struct stack_depot_trie_node *node) { unsigned int nr_old; unsigned int i; - nr_old = old_children->nr_children; + nr_old = old_children ? old_children->nr_children : 0; new_children->nr_children = nr_old + 1; new_children->capacity = new_capacity; @@ -1874,21 +1879,6 @@ trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, *path_leaf = nodes[used - 1]; } -static void trie_publish_tail_append(struct stack_depot_trie_children *children, - unsigned int pos, - const struct stack_depot_trie_node *child) -{ - /* - * Writers hold stack_depot_trie_writer_lock. Existing children are - * immutable, so tail append publishes the new child before increasing the - * visible count. Lockless readers that see the old count miss; readers that - * see the new count either load the initialized child or treat a transient - * NULL as a miss. The writer-lock recheck prevents permanent duplicates. - */ - rcu_assign_pointer(children->nodes[pos], child); - WRITE_ONCE(children->nr_children, pos + 1); -} - static const struct stack_depot_trie_node * stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) { @@ -1947,101 +1937,94 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { - struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_children *new_children; - struct stack_depot_trie_children *tail_children; - const struct stack_depot_trie_node *path_head; - const struct stack_depot_trie_node *path_leaf; - unsigned int capacity = 1; - unsigned int new_children_capacity = 1; - unsigned int i, nr_nodes; + struct stack_depot_trie_children *new_children = NULL; + struct stack_depot_trie_children *path_children = NULL; + const struct stack_depot_trie_node *path_head = NULL; + const struct stack_depot_trie_node *path_leaf = NULL; + const struct stack_depot_trie_node *prev = parent; + unsigned int cap = 1; + unsigned int entry = 0; unsigned long flags; u32 new_leaf_id; - bool tail_append = false; - - if (children) { - capacity = roundup_pow_of_two(children->nr_children + 1); - tail_append = pos == children->nr_children && - children->nr_children < children->capacity; - if (tail_append) - new_children_capacity = 0; - else - new_children_capacity = capacity; - } + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - memset(alloc, 0, sizeof(*alloc)); - nr_nodes = trie_size_append_chain(entries, nr_entries, alloc->node_sizes); - raw_spin_lock_irqsave(&pool_lock, flags); printk_deferred_enter(); trie_drain_pending_children(); - for (i = 0; i < nr_nodes; i++) { - alloc->nodes[i] = trie_pool_alloc(alloc->node_sizes[i], pool_prealloc); - if (!alloc->nodes[i]) - goto err_release; - } + /* + * Build the path from parent to leaf. Each iteration links + * the previous node's path children to the new node, then reserves path + * children for the following node when more frames remain. + */ + while (entry < nr_entries) { + struct stack_depot_frame_run run; + struct stack_depot_trie_node *node; + unsigned int start = entry; + u32 leaf_id = 0; - for (i = 0; i < nr_nodes - 1; i++) { - alloc->path_children[i] = - trie_pool_alloc_children(1, pool_prealloc); - if (!alloc->path_children[i]) + frame_run_init(&entries[start], nr_entries - start, &run); + node = trie_pool_alloc(trie_node_bytes(&run), pool_prealloc); + if (!node) goto err_release; - } - if (new_children_capacity) { - alloc->new_children = trie_pool_alloc_children(new_children_capacity, - pool_prealloc); - if (!alloc->new_children) - goto err_release; - } + entry += run.nr_entries; + if (entry == nr_entries) + leaf_id = new_leaf_id; - printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); + trie_node_init(node, prev, leaf_id, &entries[start], &run); + path_leaf = node; + if (!path_head) + path_head = node; - trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - alloc->nodes, alloc->path_children, - &path_head, &path_leaf); - trie_side_table_publish_new_leaf(new_leaf_id, path_leaf); + /* Complete the link from the previous path node. */ + if (path_children) { + trie_children_init(path_children, 1, &path_leaf, 1); + path_children = NULL; + } - if (!children) { - new_children = alloc->new_children; - new_children->nr_children = 1; - new_children->capacity = 1; - RCU_INIT_POINTER(new_children->nodes[0], path_head); - rcu_assign_pointer(*slot, new_children); - return new_leaf_id; + /* Reserve path children for the next node. */ + if (entry < nr_entries) { + path_children = trie_pool_alloc_children(1, pool_prealloc); + if (!path_children) + goto err_release; + RCU_INIT_POINTER(node->children, path_children); + } + prev = node; } - if (tail_append) { - tail_children = (struct stack_depot_trie_children *)children; - trie_publish_tail_append(tail_children, pos, path_head); - } else { - new_children = alloc->new_children; - trie_children_insert_at(children, pos, path_head, new_children, - capacity); - rcu_assign_pointer(*slot, new_children); - raw_spin_lock_irqsave(&pool_lock, flags); + if (children) + cap = roundup_pow_of_two(children->nr_children + 1); + new_children = trie_pool_alloc_children(cap, pool_prealloc); + if (!new_children) + goto err_release; + + trie_side_table_publish_new_leaf(new_leaf_id, path_leaf); + trie_children_copy(new_children, cap, children, pos, path_head); + rcu_assign_pointer(*slot, new_children); + if (children) trie_retire_children(children); - raw_spin_unlock_irqrestore(&pool_lock, flags); - } + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); return new_leaf_id; err_release: - for (i = 0; i < ARRAY_SIZE(alloc->nodes); i++) { - if (alloc->nodes[i]) - trie_pool_release(alloc->nodes[i], alloc->node_sizes[i]); + if (new_children) + trie_pool_release_children(new_children); + while (path_leaf && path_leaf != parent) { + const struct stack_depot_trie_children *node_children; + const struct stack_depot_trie_node *node = path_leaf; + + path_leaf = trie_load_parent(node); + node_children = trie_load_children(&node->children); + if (node_children) + trie_pool_release_children(node_children); + trie_pool_release(node, trie_node_bytes(&node->run)); } - for (i = 0; i < ARRAY_SIZE(alloc->path_children); i++) { - if (alloc->path_children[i]) - trie_pool_release_children(alloc->path_children[i]); - } - if (alloc->new_children) - trie_pool_release_children(alloc->new_children); printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); return 0; From 0c6fd050d39ea80cc22c3e521d225ea5d66c7c07 Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Thu, 30 Jul 2026 11:00:39 +0100 Subject: [PATCH 10/13] KRN-1117: Make the stackdepot trie insert path directly readable The trie insert and split paths were hard to follow. Insertion sized the path in one pass, allocated in another via a global stack_depot_trie_insert_alloc workspace whose arrays were sized to CONFIG_STACKDEPOT_MAX_FRAMES, then built the chain in a third pass. Split reused the same workspace and its bulk rollback loop. The workspace also demanded init-time allocation from memblock or kvzalloc. Fold all of that into a single trie_path_alloc() shared by insert and split. It walks @entries once, allocates each node and its path children as it goes, and rolls back only what it allocated on failure. The workspace, its init-time allocation, the sizing pass, and the bulk rollback all go away. trie_children_copy() is split into trie_children_init(), which initialises an unpublished container from an immutable source, and trie_children_insert(), which shifts and inserts into an unpublished container. Callers now express "copy then insert" directly, and the immutable/unpublished invariants are visible at each step. Validation: git diff --check, strict checkpatch, kernel-doc, kbuild of lib/stackdepot.o and lib/tests/stackdepot_kunit.o, and the UML stackdepot KUnit suite. --- lib/stackdepot.c | 412 +++++++++++++++++++---------------------------- 1 file changed, 166 insertions(+), 246 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 052bfc03c4c84..b357b9516dccc 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -157,8 +157,6 @@ struct stack_depot_trie_retired_children { static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_retired_children, data), 1UL << DEPOT_STACK_ALIGN)); -#define STACK_DEPOT_TRIE_MAX_NODES (CONFIG_STACKDEPOT_MAX_FRAMES + 1) -#define STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS CONFIG_STACKDEPOT_MAX_FRAMES #define STACK_DEPOT_TRIE_SLOT_SIZE BIT(DEPOT_STACK_ALIGN) #define STACK_DEPOT_TRIE_POOL_SLOTS \ (DEPOT_POOL_SIZE / STACK_DEPOT_TRIE_SLOT_SIZE) @@ -175,18 +173,8 @@ struct stack_depot_trie_pool { static_assert(STACK_DEPOT_TRIE_POOL_FIRST_SLOT < STACK_DEPOT_TRIE_POOL_SLOTS); -/* Writer-owned storage for one unpublished trie insertion. */ -struct stack_depot_trie_insert_alloc { - struct stack_depot_trie_node *nodes[STACK_DEPOT_TRIE_MAX_NODES]; - size_t node_sizes[STACK_DEPOT_TRIE_MAX_NODES]; - struct stack_depot_trie_children *path_children[STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS]; - struct stack_depot_trie_children *prefix_children; - struct stack_depot_trie_children *new_children; -}; - static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); static const struct stack_depot_trie_children __rcu *stack_depot_trie_root; -static struct stack_depot_trie_insert_alloc *stack_depot_trie_alloc; static DEFINE_RAW_SPINLOCK(stack_depot_trie_writer_lock); static bool stack_depot_trie_requested; @@ -319,21 +307,6 @@ static size_t trie_children_bytes(unsigned int capacity) return ALIGN(size, sizeof(unsigned long)); } -static void trie_children_init(struct stack_depot_trie_children *children, - unsigned int capacity, - const struct stack_depot_trie_node * const *nodes, - unsigned int nr_children) -{ - unsigned int i; - - children->nr_children = nr_children; - children->capacity = capacity; - for (i = 0; i < nr_children; i++) - RCU_INIT_POINTER(children->nodes[i], nodes[i]); - for (i = nr_children; i < capacity; i++) - RCU_INIT_POINTER(children->nodes[i], NULL); -} - static inline unsigned int trie_side_table_root_index(u32 id) { return ((id - 1) / STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE) / @@ -494,21 +467,11 @@ static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) static int __init stack_depot_trie_init_memblock(void) { - struct stack_depot_trie_insert_alloc *alloc; - size_t size; int ret; - size = sizeof(*stack_depot_trie_alloc); - alloc = memblock_alloc(size, __alignof__(*alloc)); - if (!alloc) - return -ENOMEM; - ret = __stack_depot_trie_side_table_init_memblock(); - if (ret) { - memblock_free(alloc, size); + if (ret) return ret; - } - stack_depot_trie_alloc = alloc; static_branch_enable(&stack_depot_trie_enabled); return 0; @@ -516,19 +479,11 @@ static int __init stack_depot_trie_init_memblock(void) static int stack_depot_trie_init(gfp_t gfp_flags) { - struct stack_depot_trie_insert_alloc *alloc; int ret; - alloc = kvzalloc(sizeof(*stack_depot_trie_alloc), gfp_flags); - if (!alloc) - return -ENOMEM; - ret = __stack_depot_trie_side_table_init(gfp_flags); - if (ret) { - kvfree(alloc); + if (ret) return ret; - } - stack_depot_trie_alloc = alloc; static_branch_enable(&stack_depot_trie_enabled); return 0; @@ -1762,39 +1717,51 @@ trie_children_find_slot(const struct stack_depot_trie_children *children, } /** - * trie_children_copy() - Copy children into a new container - * @new_children: Unpublished destination children container - * @new_capacity: Capacity of @new_children - * @old_children: Immutable source children container, or %NULL if empty - * @pos: Position at which to insert @node - * @node: Node to insert + * trie_children_init() - Initialize new children from an old container + * @old: Immutable source children container, or %NULL if empty + * @new: Unpublished destination children container * - * Copy @old_children into @new_children and insert @node without modifying the - * published source generation. + * Initialize @new by copying child pointers from @old without modifying + * the published source container. Unused entries in @new are + * initialized to %NULL. The allocator must already have set the + * capacity of @new. */ static void -trie_children_copy(struct stack_depot_trie_children *new_children, - unsigned int new_capacity, - const struct stack_depot_trie_children *old_children, - unsigned int pos, - const struct stack_depot_trie_node *node) +trie_children_init(const struct stack_depot_trie_children *old, + struct stack_depot_trie_children *new) { unsigned int nr_old; unsigned int i; - nr_old = old_children ? old_children->nr_children : 0; + nr_old = old ? old->nr_children : 0; + new->nr_children = nr_old; + for (i = 0; i < nr_old; i++) + RCU_INIT_POINTER(new->nodes[i], trie_children_load_child(old, i)); + for (i = nr_old; i < new->capacity; i++) + RCU_INIT_POINTER(new->nodes[i], NULL); +} - new_children->nr_children = nr_old + 1; - new_children->capacity = new_capacity; - for (i = 0; i < pos; i++) - RCU_INIT_POINTER(new_children->nodes[i], - trie_children_load_child(old_children, i)); - RCU_INIT_POINTER(new_children->nodes[pos], node); - for (i = pos; i < nr_old; i++) - RCU_INIT_POINTER(new_children->nodes[i + 1], - trie_children_load_child(old_children, i)); - for (i = nr_old + 1; i < new_children->capacity; i++) - RCU_INIT_POINTER(new_children->nodes[i], NULL); +/** + * trie_children_insert() - Insert a node into unpublished children + * @children: Unpublished children container to update + * @node: Node to insert + * @pos: Position at which to insert @node + * + * @children must not be reachable from the trie. Shift existing child + * pointers at @pos and later one position to the right, then store + * @node at @pos. + */ +static void trie_children_insert(struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *node, + unsigned int pos) +{ + unsigned int i; + + for (i = children->nr_children; i > pos; i--) + RCU_INIT_POINTER(children->nodes[i], + trie_children_load_child(children, i - 1)); + RCU_INIT_POINTER(children->nodes[pos], node); + children->nr_children++; } static void @@ -1803,16 +1770,8 @@ trie_children_replace_at(const struct stack_depot_trie_children *old_children, struct stack_depot_trie_children *new_children, unsigned int pos) { - unsigned int i; - - new_children->nr_children = old_children->nr_children; - new_children->capacity = old_children->capacity; - for (i = 0; i < old_children->nr_children; i++) - RCU_INIT_POINTER(new_children->nodes[i], - trie_children_load_child(old_children, i)); + trie_children_init(old_children, new_children); RCU_INIT_POINTER(new_children->nodes[pos], new_child); - for (i = old_children->nr_children; i < new_children->capacity; i++) - RCU_INIT_POINTER(new_children->nodes[i], NULL); } static void trie_reparent_children(struct stack_depot_trie_node *parent) @@ -1837,46 +1796,85 @@ static void trie_reparent_children(struct stack_depot_trie_node *parent) } } -static void -trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_node * const *nodes, - struct stack_depot_trie_children * const *path_children, - const struct stack_depot_trie_node **path_head, - const struct stack_depot_trie_node **path_leaf) +/** + * trie_path_alloc() - Allocate and initialize a linear trie path + * @parent: Parent of the first path node, or %NULL at the trie root + * @leaf_id: Leaf ID to assign to the final path node + * @entries: Frames to store in the path + * @nr_entries: Number of frames in @entries + * @pool_prealloc: Preallocated pool storage available to the allocator + * @leaf_out: Returns the leaf on success + * + * Split @entries into frame runs, allocate one node per run, and link adjacent + * nodes through path children. The final node is always the leaf and is + * initialized with @leaf_id. The caller must hold pool_lock. A partially + * allocated path is released before returning an error. + * + * Return: The path root on success, or %NULL on allocation failure. + */ +static const struct stack_depot_trie_node * +trie_path_alloc(const struct stack_depot_trie_node *parent, u32 leaf_id, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + const struct stack_depot_trie_node **leaf_out) { + struct stack_depot_trie_children *path_children = NULL; + const struct stack_depot_trie_node *path_root = NULL; + const struct stack_depot_trie_node *leaf = NULL; const struct stack_depot_trie_node *prev = parent; - unsigned int pos = 0; - unsigned int used = 0; - unsigned int i; + unsigned int entry = 0; + + lockdep_assert_held(&pool_lock); + *leaf_out = NULL; - while (pos < nr_entries) { + while (entry < nr_entries) { struct stack_depot_frame_run run; struct stack_depot_trie_node *node; - u32 id; + unsigned int start = entry; + u32 node_leaf_id = 0; + + frame_run_init(&entries[start], nr_entries - start, &run); + node = trie_pool_alloc(trie_node_bytes(&run), pool_prealloc); + if (!node) + goto err_release; - node = nodes[used]; - frame_run_init(&entries[pos], nr_entries - pos, &run); + entry += run.nr_entries; + if (entry == nr_entries) + node_leaf_id = leaf_id; - id = pos + run.nr_entries == nr_entries ? leaf_id : 0; - trie_node_init(node, prev, id, &entries[pos], &run); + trie_node_init(node, prev, node_leaf_id, &entries[start], &run); + leaf = node; + if (!path_root) + path_root = node; + if (path_children) { + trie_children_insert(path_children, leaf, 0); + path_children = NULL; + } + if (entry < nr_entries) { + path_children = trie_pool_alloc_children(1, pool_prealloc); + if (!path_children) + goto err_release; + RCU_INIT_POINTER(node->children, path_children); + } prev = node; - pos += run.nr_entries; - used++; } - for (i = 0; i + 1 < used; i++) { - const struct stack_depot_trie_node *next = nodes[i + 1]; - struct stack_depot_trie_node *node = nodes[i]; - struct stack_depot_trie_children *children = path_children[i]; + *leaf_out = leaf; + return path_root; - trie_children_init(children, 1, &next, 1); - RCU_INIT_POINTER(node->children, children); - } +err_release: + while (leaf && leaf != parent) { + const struct stack_depot_trie_children *node_children; + const struct stack_depot_trie_node *node = leaf; - *path_head = nodes[0]; - *path_leaf = nodes[used - 1]; + leaf = trie_load_parent(node); + node_children = trie_load_children(&node->children); + if (node_children) + trie_pool_release_children(node_children); + trie_pool_release(node, trie_node_bytes(&node->run)); + } + return NULL; } static const struct stack_depot_trie_node * @@ -1910,25 +1908,6 @@ stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) return NULL; } -static unsigned int trie_size_append_chain(const unsigned long *entries, - unsigned int nr_entries, - size_t *node_sizes) -{ - unsigned int pos = 0; - unsigned int nr_nodes = 0; - - while (pos < nr_entries) { - struct stack_depot_frame_run run; - - frame_run_init(&entries[pos], nr_entries - pos, &run); - node_sizes[nr_nodes] = trie_node_bytes(&run); - pos += run.nr_entries; - nr_nodes++; - } - - return nr_nodes; -} - static u32 trie_insert_path(const struct stack_depot_trie_children __rcu **slot, struct stack_depot_trie_node *parent, @@ -1937,13 +1916,10 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { - struct stack_depot_trie_children *new_children = NULL; - struct stack_depot_trie_children *path_children = NULL; - const struct stack_depot_trie_node *path_head = NULL; - const struct stack_depot_trie_node *path_leaf = NULL; - const struct stack_depot_trie_node *prev = parent; + struct stack_depot_trie_children *new_children; + const struct stack_depot_trie_node *path_root; + const struct stack_depot_trie_node *leaf; unsigned int cap = 1; - unsigned int entry = 0; unsigned long flags; u32 new_leaf_id; @@ -1955,56 +1931,34 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, printk_deferred_enter(); trie_drain_pending_children(); - /* - * Build the path from parent to leaf. Each iteration links - * the previous node's path children to the new node, then reserves path - * children for the following node when more frames remain. - */ - while (entry < nr_entries) { - struct stack_depot_frame_run run; - struct stack_depot_trie_node *node; - unsigned int start = entry; - u32 leaf_id = 0; - - frame_run_init(&entries[start], nr_entries - start, &run); - node = trie_pool_alloc(trie_node_bytes(&run), pool_prealloc); - if (!node) - goto err_release; - - entry += run.nr_entries; - if (entry == nr_entries) - leaf_id = new_leaf_id; - - trie_node_init(node, prev, leaf_id, &entries[start], &run); - path_leaf = node; - if (!path_head) - path_head = node; - - /* Complete the link from the previous path node. */ - if (path_children) { - trie_children_init(path_children, 1, &path_leaf, 1); - path_children = NULL; - } - - /* Reserve path children for the next node. */ - if (entry < nr_entries) { - path_children = trie_pool_alloc_children(1, pool_prealloc); - if (!path_children) - goto err_release; - RCU_INIT_POINTER(node->children, path_children); - } - prev = node; - } - if (children) cap = roundup_pow_of_two(children->nr_children + 1); + + /* + * Published children are immutable, so adding a path requires a new + * children container. trie_children_init() copies existing child + * pointers, if any, and trie_children_insert() attaches the path root. + */ new_children = trie_pool_alloc_children(cap, pool_prealloc); if (!new_children) goto err_release; - trie_side_table_publish_new_leaf(new_leaf_id, path_leaf); - trie_children_copy(new_children, cap, children, pos, path_head); + path_root = trie_path_alloc(parent, new_leaf_id, entries, nr_entries, + pool_prealloc, &leaf); + if (!path_root) + goto err_release; + + trie_side_table_publish_new_leaf(new_leaf_id, leaf); + + trie_children_init(children, new_children); + trie_children_insert(new_children, path_root, pos); + + /* + * Publish the completed children container. Readers can now reach the + * new path, whose leaf is already visible in the side table. + */ rcu_assign_pointer(*slot, new_children); + if (children) trie_retire_children(children); @@ -2015,16 +1969,6 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, err_release: if (new_children) trie_pool_release_children(new_children); - while (path_leaf && path_leaf != parent) { - const struct stack_depot_trie_children *node_children; - const struct stack_depot_trie_node *node = path_leaf; - - path_leaf = trie_load_parent(node); - node_children = trie_load_children(&node->children); - if (node_children) - trie_pool_release_children(node_children); - trie_pool_release(node, trie_node_bytes(&node->run)); - } printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); return 0; @@ -2039,43 +1983,41 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { - struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_children *prefix_children, *new_children; + struct stack_depot_trie_children *prefix_children = NULL; + struct stack_depot_trie_children *new_children = NULL; const struct stack_depot_trie_node *split_leaf; const struct stack_depot_trie_node *split_nodes[2]; - struct stack_depot_trie_node *split_prefix, *old_tail; - unsigned int nr_split_nodes; - unsigned int i, old_tail_len; + struct stack_depot_trie_node *split_prefix = NULL; + struct stack_depot_trie_node *old_tail = NULL; + unsigned int i, nr_split_nodes; + unsigned int old_tail_len; unsigned long flags; + size_t split_prefix_size; + size_t old_tail_size; u32 new_leaf_id, prefix_leaf_id; new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - memset(alloc, 0, sizeof(*alloc)); - raw_spin_lock_irqsave(&pool_lock, flags); printk_deferred_enter(); trie_drain_pending_children(); - alloc->node_sizes[0] = trie_node_bytes_for(child->run.mode, matched); - split_prefix = trie_pool_alloc(alloc->node_sizes[0], pool_prealloc); + split_prefix_size = trie_node_bytes_for(child->run.mode, matched); + split_prefix = trie_pool_alloc(split_prefix_size, pool_prealloc); if (!split_prefix) goto err_release; - alloc->nodes[0] = split_prefix; old_tail_len = child->run.nr_entries - matched; - alloc->node_sizes[1] = trie_node_bytes_for(child->run.mode, old_tail_len); - old_tail = trie_pool_alloc(alloc->node_sizes[1], pool_prealloc); + old_tail_size = trie_node_bytes_for(child->run.mode, old_tail_len); + old_tail = trie_pool_alloc(old_tail_size, pool_prealloc); if (!old_tail) goto err_release; - alloc->nodes[1] = old_tail; new_children = trie_pool_alloc_children(children->capacity, pool_prealloc); if (!new_children) goto err_release; - alloc->new_children = new_children; if (matched < nr_entries) { /* @@ -2084,39 +2026,22 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, * stack. */ const struct stack_depot_trie_node *new_tail; - unsigned int new_tail_len; - unsigned int nr_tail_nodes; + unsigned int new_tail_len = nr_entries - matched; unsigned long frame; prefix_leaf_id = 0; nr_split_nodes = 2; - new_tail_len = nr_entries - matched; - nr_tail_nodes = trie_size_append_chain(&entries[matched], - new_tail_len, - &alloc->node_sizes[2]); - - for (i = 0; i < nr_tail_nodes; i++) { - struct stack_depot_trie_node *node; - - node = trie_pool_alloc(alloc->node_sizes[2 + i], pool_prealloc); - if (!node) - goto err_release; - alloc->nodes[2 + i] = node; - } - - for (i = 0; i < nr_tail_nodes - 1; i++) { - struct stack_depot_trie_children *children; + prefix_children = trie_pool_alloc_children(nr_split_nodes, + pool_prealloc); + if (!prefix_children) + goto err_release; - children = trie_pool_alloc_children(1, pool_prealloc); - if (!children) - goto err_release; - alloc->path_children[i] = children; - } + new_tail = trie_path_alloc(split_prefix, new_leaf_id, + &entries[matched], new_tail_len, + pool_prealloc, &split_leaf); + if (!new_tail) + goto err_release; - trie_build_append_chain(split_prefix, new_leaf_id, &entries[matched], - new_tail_len, &alloc->nodes[2], - alloc->path_children, &new_tail, - &split_leaf); stack_depot_trie_node_frame(child, matched, &frame); if (frame < entries[matched]) { split_nodes[0] = old_tail; @@ -2131,13 +2056,12 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, nr_split_nodes = 1; split_leaf = split_prefix; split_nodes[0] = old_tail; + prefix_children = trie_pool_alloc_children(nr_split_nodes, + pool_prealloc); + if (!prefix_children) + goto err_release; } - prefix_children = trie_pool_alloc_children(nr_split_nodes, pool_prealloc); - if (!prefix_children) - goto err_release; - alloc->prefix_children = prefix_children; - printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -2146,8 +2070,8 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, child, 0, matched); trie_node_init_slice(old_tail, split_prefix, child->leaf_id, child, matched, old_tail_len); - trie_children_init(prefix_children, nr_split_nodes, split_nodes, - nr_split_nodes); + for (i = 0; i < nr_split_nodes; i++) + trie_children_insert(prefix_children, split_nodes[i], i); /* Keep the old children under the old tail. */ RCU_INIT_POINTER(old_tail->children, trie_load_children(&child->children)); @@ -2162,18 +2086,14 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, return new_leaf_id; err_release: - for (i = 0; i < ARRAY_SIZE(alloc->nodes); i++) { - if (alloc->nodes[i]) - trie_pool_release(alloc->nodes[i], alloc->node_sizes[i]); - } - for (i = 0; i < ARRAY_SIZE(alloc->path_children); i++) { - if (alloc->path_children[i]) - trie_pool_release_children(alloc->path_children[i]); - } - if (alloc->prefix_children) - trie_pool_release_children(alloc->prefix_children); - if (alloc->new_children) - trie_pool_release_children(alloc->new_children); + if (split_prefix) + trie_pool_release(split_prefix, split_prefix_size); + if (old_tail) + trie_pool_release(old_tail, old_tail_size); + if (prefix_children) + trie_pool_release_children(prefix_children); + if (new_children) + trie_pool_release_children(new_children); printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); return 0; From 59ea22ffe5bfb5fb3e95ae838d75d72d3f4a895f Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Thu, 30 Jul 2026 11:35:08 +0100 Subject: [PATCH 11/13] KRN-1117: Unify stackdepot trie terminology around one glossary The trie code carried several overlapping vocabularies -- leaf, COW/generation, tail -- for the same underlying concepts. Reading the insert and split paths meant translating between them. Settle on one glossary so the algorithm reads directly. node one frame run in the trie children immutable container of child-node pointers child one node inside a children container path linear sequence of nodes for a stack path_root first node of that sequence stack_id optional non-zero ID set only where a stored stack ends prefix shared portion of a split child suffix divergent portion(s) of a split child pos index within a children container slot root-or-child pointer location the writer updates leaf_id and the "leaf" node concept go away; ordinary nodes carry an optional stack_id and there is no distinct leaf type. The RCU/COW and "generation" comments on children go away in favour of describing the replacement children container as immutable once published. Tail becomes suffix in the split path. While here, drop the __stack_depot_ prefix from private static trie helpers. We were using __ merely to mean "private", which conflicts with the kernel convention of reserving __ for genuine lower-level or public-interface pairings. These helpers are all file-local, so name them consistently with the trie_ prefix instead. No behaviour change. Validated with git diff --check, strict checkpatch, kernel-doc, kbuild objects, and UML stackdepot KUnit (8/8). --- lib/stackdepot.c | 421 ++++++++++++++++++++++++----------------------- 1 file changed, 214 insertions(+), 207 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index b357b9516dccc..03b4ac5b8d5f3 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -6,7 +6,7 @@ * callers that request STACK_DEPOT_FLAG_COUNTABLE use the legacy hash table with * contiguous stack records in stack pools. Persistent non-refcounted entries * can use trie storage when enabled; trie nodes share common frame prefixes and - * are published through RCU/COW children containers. + * are published through immutable RCU children containers. * * Author: Alexander Potapenko * Copyright (C) 2016 Google, Inc. @@ -127,11 +127,12 @@ static_assert(CONFIG_STACKDEPOT_MAX_FRAMES <= U16_MAX); struct stack_depot_trie_children; struct stack_depot_trie_node { - /* Parent links let fetch rebuild a full stack from a leaf to the root. */ + /* Parent links let fetch rebuild a full stack from a node to the root. */ const struct stack_depot_trie_node __rcu *parent; - /* Children are separate RCU/COW generations. */ + /* Children are immutable RCU-published containers. */ const struct stack_depot_trie_children __rcu *children; - u32 leaf_id; + /* Non-zero when a stored stack ends at this node. */ + u32 stack_id; struct stack_depot_frame_run run; unsigned char data[]; }; @@ -192,23 +193,23 @@ static LIST_HEAD(pending_trie_children); * stack_max_pools is the split point between hash and trie handle encodings. * A handle with pool_index_plus_1 in 1..stack_max_pools names a hash-backed * stack pool. Larger pool-index values cannot refer to hash pools, so trie - * storage uses that handle space to encode a dense leaf_id. The side table - * maps each leaf_id to its trie leaf node. + * storage uses that handle space to encode a dense stack ID. The side + * table maps each stack ID to its trie node. */ -static inline u32 __stack_depot_trie_max_leaf_id(void) +static inline u32 trie_max_stack_id(void) { return (DEPOT_POOL_INDEX_MASK - stack_max_pools) << DEPOT_OFFSET_BITS; } -static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) +static depot_stack_handle_t trie_handle(u32 stack_id) { union handle_parts parts = {}; u64 pool_index_plus_1; u32 pool_delta; u32 index; - index = leaf_id - 1; + index = stack_id - 1; pool_delta = index >> DEPOT_OFFSET_BITS; pool_index_plus_1 = (u64)stack_max_pools + 1 + pool_delta; @@ -224,7 +225,7 @@ static inline bool stack_depot_handle_is_trie(depot_stack_handle_t handle) return parts.pool_index_plus_1 > stack_max_pools; } -static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) +static u32 trie_stack_id(depot_stack_handle_t handle) { union handle_parts parts = { .handle = handle }; u32 pool_delta; @@ -234,13 +235,16 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) } /* - * Trie handles encode a dense leaf ID. The side table maps that ID to a leaf - * pointer for lockless fetch/print paths, which can run from diagnostic - * contexts where taking a lock would be unsafe. Initialization - * installs the root; early initialization also installs the first directory and - * chunk. Additional directories and chunks are preallocated and published - * lazily as leaf IDs grow. RCU pointer publication makes fully initialized - * directories, chunks, and leaves visible to those lockless readers. + * Trie handles encode a dense stack ID. The side table maps that ID + * to a node pointer for lockless fetch and print operations. Those + * operations can run from diagnostic contexts where taking a lock would + * be unsafe. + * + * Initialization installs the side-table root. Early initialization + * also installs the first directory and chunk. Additional directories + * and chunks are preallocated and published lazily as stack IDs grow. + * RCU publication makes initialized directories, chunks, and nodes + * visible to those lockless readers. */ #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ (PAGE_SIZE / sizeof(struct stack_depot_trie_node *)) @@ -248,7 +252,7 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) (PAGE_SIZE / sizeof(struct stack_depot_trie_node **)) struct stack_depot_trie_side_dir { - /* Both the chunk pointer and each leaf pointer in it are RCU-published. */ + /* The chunk and each node pointer are RCU-published. */ const struct stack_depot_trie_node __rcu * __rcu * chunks[STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE]; }; @@ -261,7 +265,7 @@ struct stack_depot_trie_side_root { struct stack_depot_trie_side_prealloc { /* Preallocated side-table directory page for sparse growth. */ struct stack_depot_trie_side_dir *dir; - /* Preallocated side-table leaf chunk for sparse growth. */ + /* Preallocated side-table pointer chunk for sparse growth. */ const struct stack_depot_trie_node __rcu **chunk; }; @@ -269,7 +273,7 @@ static struct stack_depot_trie_side_root *trie_side_table_root; static DEFINE_RAW_SPINLOCK(trie_side_table_cache_lock); /* Zeroed unpublished pages; get/put transfer ownership under the cache lock. */ static struct stack_depot_trie_side_prealloc trie_side_table_cache; -static u32 trie_side_table_last_leaf_id; +static u32 trie_side_table_last_stack_id; /* Lock order: writer_lock -> pool_lock. The cache lock is never nested. */ @@ -341,14 +345,14 @@ static inline const struct stack_depot_trie_node __rcu ** trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, unsigned int idx) { - /* Pairs with the chunk rcu_assign_pointer() in leaf ID preparation. */ + /* Pairs with the chunk rcu_assign_pointer() in stack ID preparation. */ return rcu_dereference_check(dir->chunks[idx], lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } static u32 -trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *prealloc) +trie_side_table_prepare_stack_slot(struct stack_depot_trie_side_prealloc *prealloc) { const struct stack_depot_trie_node __rcu **chunk; struct stack_depot_trie_side_dir *dir; @@ -359,8 +363,8 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo lockdep_assert_held(&stack_depot_trie_writer_lock); - id = trie_side_table_last_leaf_id + 1; - if (id > __stack_depot_trie_max_leaf_id()) + id = trie_side_table_last_stack_id + 1; + if (id > trie_max_stack_id()) return 0; root_vec = trie_side_table_root; @@ -392,30 +396,30 @@ static void trie_side_table_root_init(struct stack_depot_trie_side_root *root_ve unsigned int root_size) { root_vec->dir_capacity = root_size; - trie_side_table_last_leaf_id = 0; + trie_side_table_last_stack_id = 0; } -static inline unsigned int trie_side_table_root_size_for_max_id(u32 max_leaf_id) +static inline unsigned int trie_side_table_root_size_for_max_id(u32 max_stack_id) { unsigned int top_size; - top_size = DIV_ROUND_UP(max_leaf_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); + top_size = DIV_ROUND_UP(max_stack_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); return DIV_ROUND_UP(top_size, STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE); } -static int __init __stack_depot_trie_side_table_init_memblock(void) +static int __init trie_side_table_init_memblock(void) { struct stack_depot_trie_side_root *root_vec; struct stack_depot_trie_side_dir *first_dir; const struct stack_depot_trie_node __rcu **first_chunk; size_t root_bytes; - u32 max_leaf_id; + u32 max_stack_id; unsigned int root_size; - max_leaf_id = __stack_depot_trie_max_leaf_id(); - if (!max_leaf_id) + max_stack_id = trie_max_stack_id(); + if (!max_stack_id) return -EINVAL; - root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_size = trie_side_table_root_size_for_max_id(max_stack_id); root_bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); root_vec = memblock_alloc(root_bytes, __alignof__(*root_vec)); @@ -443,18 +447,18 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) return 0; } -static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) +static int trie_side_table_init(gfp_t gfp_flags) { struct stack_depot_trie_side_root *root_vec; unsigned int root_size; size_t root_bytes; - u32 max_leaf_id; + u32 max_stack_id; - max_leaf_id = __stack_depot_trie_max_leaf_id(); - if (!max_leaf_id) + max_stack_id = trie_max_stack_id(); + if (!max_stack_id) return -EINVAL; - root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_size = trie_side_table_root_size_for_max_id(max_stack_id); root_bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); root_vec = kvzalloc(root_bytes, gfp_flags); if (!root_vec) @@ -469,7 +473,7 @@ static int __init stack_depot_trie_init_memblock(void) { int ret; - ret = __stack_depot_trie_side_table_init_memblock(); + ret = trie_side_table_init_memblock(); if (ret) return ret; @@ -481,7 +485,7 @@ static int stack_depot_trie_init(gfp_t gfp_flags) { int ret; - ret = __stack_depot_trie_side_table_init(gfp_flags); + ret = trie_side_table_init(gfp_flags); if (ret) return ret; @@ -537,7 +541,7 @@ static void trie_side_table_put_prealloc(struct stack_depot_trie_side_prealloc * free_page((unsigned long)prealloc->chunk); } -static const struct stack_depot_trie_node __rcu **trie_side_table_leaf_slot(u32 id) +static const struct stack_depot_trie_node __rcu **trie_side_table_stack_slot(u32 id) { const struct stack_depot_trie_node __rcu **chunk; struct stack_depot_trie_side_dir *dir; @@ -550,7 +554,7 @@ static const struct stack_depot_trie_node __rcu **trie_side_table_leaf_slot(u32 return &chunk[trie_side_table_slot_index(id)]; } -static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup(u32 id) +static const struct stack_depot_trie_node *trie_side_table_lookup(u32 id) { const struct stack_depot_trie_node __rcu **chunk; struct stack_depot_trie_side_dir *dir; @@ -564,7 +568,7 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( if (!chunk) return NULL; - /* Pairs with side-table leaf rcu_assign_pointer(). */ + /* Pairs with side-table node publication. */ return rcu_dereference_check(chunk[trie_side_table_slot_index(id)], lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); @@ -746,50 +750,50 @@ static depot_stack_handle_t trie_find_handle(const unsigned long *entries, unsigned int nr_entries) { depot_stack_handle_t handle = 0; - const struct stack_depot_trie_node *leaf; + const struct stack_depot_trie_node *node; rcu_read_lock_sched_notrace(); - leaf = stack_depot_trie_lookup(entries, nr_entries); - if (leaf) - handle = __stack_depot_trie_handle(leaf->leaf_id); + node = stack_depot_trie_lookup(entries, nr_entries); + if (node) + handle = trie_handle(node->stack_id); rcu_read_unlock_sched_notrace(); return handle; } -static void trie_side_table_publish_new_leaf(u32 leaf_id, - const struct stack_depot_trie_node *leaf) +static void trie_side_table_publish_new_node(u32 stack_id, + const struct stack_depot_trie_node *node) { const struct stack_depot_trie_node __rcu **slot; lockdep_assert_held(&stack_depot_trie_writer_lock); - slot = trie_side_table_leaf_slot(leaf_id); - /* Pairs with __stack_depot_trie_side_table_lookup(). */ - rcu_assign_pointer(*slot, leaf); + slot = trie_side_table_stack_slot(stack_id); + /* Pairs with trie_side_table_lookup(). */ + rcu_assign_pointer(*slot, node); } -static void trie_side_table_publish_split_leaves(u32 old_leaf_id, - const struct stack_depot_trie_node *old_leaf, - u32 new_leaf_id, - const struct stack_depot_trie_node *new_leaf) +static void trie_side_table_publish_split_nodes(u32 old_stack_id, + const struct stack_depot_trie_node *old_node, + u32 new_stack_id, + const struct stack_depot_trie_node *new_node) { const struct stack_depot_trie_node __rcu **new_slot; const struct stack_depot_trie_node __rcu **old_slot = NULL; lockdep_assert_held(&stack_depot_trie_writer_lock); - if (old_leaf_id) - old_slot = trie_side_table_leaf_slot(old_leaf_id); + if (old_stack_id) + old_slot = trie_side_table_stack_slot(old_stack_id); - new_slot = trie_side_table_leaf_slot(new_leaf_id); + new_slot = trie_side_table_stack_slot(new_stack_id); if (old_slot) { - /* Pairs with __stack_depot_trie_side_table_lookup(). */ - rcu_assign_pointer(*old_slot, old_leaf); + /* Pairs with trie_side_table_lookup(). */ + rcu_assign_pointer(*old_slot, old_node); } - /* Pairs with __stack_depot_trie_side_table_lookup(). */ - rcu_assign_pointer(*new_slot, new_leaf); + /* Pairs with trie_side_table_lookup(). */ + rcu_assign_pointer(*new_slot, new_node); } static int __init disable_stack_depot(char *str) @@ -1359,7 +1363,7 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, depot_stack_handle_t handle = 0; unsigned long flags; struct page *page; - u32 leaf_id; + u32 stack_id; handle = trie_find_handle(entries, nr_entries); if (handle) @@ -1374,10 +1378,10 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, pool_prealloc = page_address(page); raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); - leaf_id = stack_depot_trie_insert(entries, nr_entries, - &pool_prealloc, &side_prealloc); - if (leaf_id) - handle = __stack_depot_trie_handle(leaf_id); + stack_id = stack_depot_trie_insert(entries, nr_entries, + &pool_prealloc, &side_prealloc); + if (stack_id) + handle = trie_handle(stack_id); raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); if (pool_prealloc) { @@ -1574,7 +1578,7 @@ stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, } static void trie_node_init(struct stack_depot_trie_node *node, - const struct stack_depot_trie_node *parent, u32 leaf_id, + const struct stack_depot_trie_node *parent, u32 stack_id, const unsigned long *entries, const struct stack_depot_frame_run *run) { @@ -1594,12 +1598,12 @@ static void trie_node_init(struct stack_depot_trie_node *node, RCU_INIT_POINTER(node->parent, parent); RCU_INIT_POINTER(node->children, NULL); - node->leaf_id = leaf_id; + node->stack_id = stack_id; node->run = *run; } static void trie_node_init_slice(struct stack_depot_trie_node *node, - const struct stack_depot_trie_node *parent, u32 leaf_id, + const struct stack_depot_trie_node *parent, u32 stack_id, const struct stack_depot_trie_node *src_node, unsigned int start, unsigned int nr_entries) { @@ -1614,14 +1618,14 @@ static void trie_node_init_slice(struct stack_depot_trie_node *node, stack_depot_frame_run_bytes(&run)); RCU_INIT_POINTER(node->parent, parent); RCU_INIT_POINTER(node->children, NULL); - node->leaf_id = leaf_id; + node->stack_id = stack_id; node->run = run; } static unsigned int -__stack_depot_trie_node_match(const struct stack_depot_trie_node *node, - const unsigned long *entries, - unsigned int nr_entries) +trie_node_match(const struct stack_depot_trie_node *node, + const unsigned long *entries, + unsigned int nr_entries) { unsigned int limit; unsigned int i; @@ -1783,10 +1787,11 @@ static void trie_reparent_children(struct stack_depot_trie_node *parent) if (!children) return; /* - * COW updates reuse unchanged descendant subtrees. Repoint their parent - * links before retiring the old parent so fetch never follows a freed node. - * Lockless fetches may see the new parent before structural publication, but - * the old and new parent chains contain the same frames and remain RCU-live. + * Replacement children reuse unchanged descendant subtrees. Repoint + * their parent links before retiring the old parent so fetch never + * follows a freed node. Lockless fetches may see the new parent before + * structural publication. The old and new parent chains contain the + * same frames and remain RCU-live. */ for (i = 0; i < children->nr_children; i++) { struct stack_depot_trie_node *child; @@ -1799,39 +1804,40 @@ static void trie_reparent_children(struct stack_depot_trie_node *parent) /** * trie_path_alloc() - Allocate and initialize a linear trie path * @parent: Parent of the first path node, or %NULL at the trie root - * @leaf_id: Leaf ID to assign to the final path node + * @stack_id: Stack ID to assign to the final path node * @entries: Frames to store in the path * @nr_entries: Number of frames in @entries * @pool_prealloc: Preallocated pool storage available to the allocator - * @leaf_out: Returns the leaf on success + * @node_out: Returns the final node on success * - * Split @entries into frame runs, allocate one node per run, and link adjacent - * nodes through path children. The final node is always the leaf and is - * initialized with @leaf_id. The caller must hold pool_lock. A partially + * Split @entries into frame runs, allocate one node per run, and link + * adjacent nodes through path children. Initialize the final node with + * @stack_id. + * The caller must hold pool_lock. A partially * allocated path is released before returning an error. * * Return: The path root on success, or %NULL on allocation failure. */ static const struct stack_depot_trie_node * -trie_path_alloc(const struct stack_depot_trie_node *parent, u32 leaf_id, +trie_path_alloc(const struct stack_depot_trie_node *parent, u32 stack_id, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, - const struct stack_depot_trie_node **leaf_out) + const struct stack_depot_trie_node **node_out) { struct stack_depot_trie_children *path_children = NULL; const struct stack_depot_trie_node *path_root = NULL; - const struct stack_depot_trie_node *leaf = NULL; + const struct stack_depot_trie_node *last_node = NULL; const struct stack_depot_trie_node *prev = parent; unsigned int entry = 0; lockdep_assert_held(&pool_lock); - *leaf_out = NULL; + *node_out = NULL; while (entry < nr_entries) { struct stack_depot_frame_run run; struct stack_depot_trie_node *node; unsigned int start = entry; - u32 node_leaf_id = 0; + u32 node_stack_id = 0; frame_run_init(&entries[start], nr_entries - start, &run); node = trie_pool_alloc(trie_node_bytes(&run), pool_prealloc); @@ -1840,15 +1846,15 @@ trie_path_alloc(const struct stack_depot_trie_node *parent, u32 leaf_id, entry += run.nr_entries; if (entry == nr_entries) - node_leaf_id = leaf_id; + node_stack_id = stack_id; - trie_node_init(node, prev, node_leaf_id, &entries[start], &run); - leaf = node; + trie_node_init(node, prev, node_stack_id, &entries[start], &run); + last_node = node; if (!path_root) path_root = node; if (path_children) { - trie_children_insert(path_children, leaf, 0); + trie_children_insert(path_children, last_node, 0); path_children = NULL; } if (entry < nr_entries) { @@ -1860,15 +1866,15 @@ trie_path_alloc(const struct stack_depot_trie_node *parent, u32 leaf_id, prev = node; } - *leaf_out = leaf; + *node_out = last_node; return path_root; err_release: - while (leaf && leaf != parent) { + while (last_node && last_node != parent) { const struct stack_depot_trie_children *node_children; - const struct stack_depot_trie_node *node = leaf; + const struct stack_depot_trie_node *node = last_node; - leaf = trie_load_parent(node); + last_node = trie_load_parent(node); node_children = trie_load_children(&node->children); if (node_children) trie_pool_release_children(node_children); @@ -1895,12 +1901,12 @@ stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) return NULL; node = trie_children_load_child(children, pos); - matched = __stack_depot_trie_node_match(node, &entries[entry], remaining); + matched = trie_node_match(node, &entries[entry], remaining); if (matched < node->run.nr_entries) return NULL; entry += matched; if (entry == nr_entries) - return node->leaf_id ? node : NULL; + return node->stack_id ? node : NULL; children = trie_load_children(&node->children); } @@ -1918,13 +1924,13 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, { struct stack_depot_trie_children *new_children; const struct stack_depot_trie_node *path_root; - const struct stack_depot_trie_node *leaf; + const struct stack_depot_trie_node *node; unsigned int cap = 1; unsigned long flags; - u32 new_leaf_id; + u32 new_stack_id; - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) + new_stack_id = trie_side_table_prepare_stack_slot(side_prealloc); + if (!new_stack_id) return 0; raw_spin_lock_irqsave(&pool_lock, flags); @@ -1943,19 +1949,19 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, if (!new_children) goto err_release; - path_root = trie_path_alloc(parent, new_leaf_id, entries, nr_entries, - pool_prealloc, &leaf); + path_root = trie_path_alloc(parent, new_stack_id, entries, nr_entries, + pool_prealloc, &node); if (!path_root) goto err_release; - trie_side_table_publish_new_leaf(new_leaf_id, leaf); + trie_side_table_publish_new_node(new_stack_id, node); trie_children_init(children, new_children); trie_children_insert(new_children, path_root, pos); /* * Publish the completed children container. Readers can now reach the - * new path, whose leaf is already visible in the side table. + * new path, whose final node is already visible in the side table. */ rcu_assign_pointer(*slot, new_children); @@ -1964,7 +1970,7 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); - return new_leaf_id; + return new_stack_id; err_release: if (new_children) @@ -1985,19 +1991,19 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, { struct stack_depot_trie_children *prefix_children = NULL; struct stack_depot_trie_children *new_children = NULL; - const struct stack_depot_trie_node *split_leaf; - const struct stack_depot_trie_node *split_nodes[2]; + const struct stack_depot_trie_node *new_node; + const struct stack_depot_trie_node *suffix_roots[2]; struct stack_depot_trie_node *split_prefix = NULL; - struct stack_depot_trie_node *old_tail = NULL; - unsigned int i, nr_split_nodes; - unsigned int old_tail_len; + struct stack_depot_trie_node *old_suffix = NULL; + unsigned int i, nr_suffix_roots; + unsigned int old_suffix_len; unsigned long flags; size_t split_prefix_size; - size_t old_tail_size; - u32 new_leaf_id, prefix_leaf_id; + size_t old_suffix_size; + u32 new_stack_id, prefix_stack_id; - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) + new_stack_id = trie_side_table_prepare_stack_slot(side_prealloc); + if (!new_stack_id) return 0; raw_spin_lock_irqsave(&pool_lock, flags); @@ -2009,10 +2015,10 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, if (!split_prefix) goto err_release; - old_tail_len = child->run.nr_entries - matched; - old_tail_size = trie_node_bytes_for(child->run.mode, old_tail_len); - old_tail = trie_pool_alloc(old_tail_size, pool_prealloc); - if (!old_tail) + old_suffix_len = child->run.nr_entries - matched; + old_suffix_size = trie_node_bytes_for(child->run.mode, old_suffix_len); + old_suffix = trie_pool_alloc(old_suffix_size, pool_prealloc); + if (!old_suffix) goto err_release; new_children = trie_pool_alloc_children(children->capacity, pool_prealloc); @@ -2022,41 +2028,41 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, if (matched < nr_entries) { /* * The new and existing stacks diverge inside this child. Keep the - * existing suffix as old_tail and build a second tail for the new + * existing suffix as old_suffix and build a new suffix for the new * stack. */ - const struct stack_depot_trie_node *new_tail; - unsigned int new_tail_len = nr_entries - matched; + const struct stack_depot_trie_node *new_suffix; + unsigned int new_suffix_len = nr_entries - matched; unsigned long frame; - prefix_leaf_id = 0; - nr_split_nodes = 2; - prefix_children = trie_pool_alloc_children(nr_split_nodes, + prefix_stack_id = 0; + nr_suffix_roots = 2; + prefix_children = trie_pool_alloc_children(nr_suffix_roots, pool_prealloc); if (!prefix_children) goto err_release; - new_tail = trie_path_alloc(split_prefix, new_leaf_id, - &entries[matched], new_tail_len, - pool_prealloc, &split_leaf); - if (!new_tail) + new_suffix = trie_path_alloc(split_prefix, new_stack_id, + &entries[matched], new_suffix_len, + pool_prealloc, &new_node); + if (!new_suffix) goto err_release; stack_depot_trie_node_frame(child, matched, &frame); if (frame < entries[matched]) { - split_nodes[0] = old_tail; - split_nodes[1] = new_tail; + suffix_roots[0] = old_suffix; + suffix_roots[1] = new_suffix; } else { - split_nodes[0] = new_tail; - split_nodes[1] = old_tail; + suffix_roots[0] = new_suffix; + suffix_roots[1] = old_suffix; } } else { /* The new stack ends at the shared prefix. */ - prefix_leaf_id = new_leaf_id; - nr_split_nodes = 1; - split_leaf = split_prefix; - split_nodes[0] = old_tail; - prefix_children = trie_pool_alloc_children(nr_split_nodes, + prefix_stack_id = new_stack_id; + nr_suffix_roots = 1; + new_node = split_prefix; + suffix_roots[0] = old_suffix; + prefix_children = trie_pool_alloc_children(nr_suffix_roots, pool_prealloc); if (!prefix_children) goto err_release; @@ -2066,30 +2072,31 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, raw_spin_unlock_irqrestore(&pool_lock, flags); /* Keep the shared prefix under the old parent. */ - trie_node_init_slice(split_prefix, trie_load_parent(child), prefix_leaf_id, + trie_node_init_slice(split_prefix, trie_load_parent(child), prefix_stack_id, child, 0, matched); - trie_node_init_slice(old_tail, split_prefix, child->leaf_id, child, matched, - old_tail_len); - for (i = 0; i < nr_split_nodes; i++) - trie_children_insert(prefix_children, split_nodes[i], i); + trie_node_init_slice(old_suffix, split_prefix, child->stack_id, child, matched, + old_suffix_len); + for (i = 0; i < nr_suffix_roots; i++) + trie_children_insert(prefix_children, suffix_roots[i], i); - /* Keep the old children under the old tail. */ - RCU_INIT_POINTER(old_tail->children, trie_load_children(&child->children)); + /* Keep the old children under the old suffix. */ + RCU_INIT_POINTER(old_suffix->children, trie_load_children(&child->children)); RCU_INIT_POINTER(split_prefix->children, prefix_children); - trie_side_table_publish_split_leaves(child->leaf_id, old_tail, new_leaf_id, split_leaf); + trie_side_table_publish_split_nodes(child->stack_id, old_suffix, + new_stack_id, new_node); trie_children_replace_at(children, split_prefix, new_children, pos); - trie_reparent_children(old_tail); + trie_reparent_children(old_suffix); rcu_assign_pointer(*slot, new_children); trie_retire_children_with_node(children, child); - return new_leaf_id; + return new_stack_id; err_release: if (split_prefix) trie_pool_release(split_prefix, split_prefix_size); - if (old_tail) - trie_pool_release(old_tail, old_tail_size); + if (old_suffix) + trie_pool_release(old_suffix, old_suffix_size); if (prefix_children) trie_pool_release_children(prefix_children); if (new_children) @@ -2101,19 +2108,19 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, static u32 trie_promote_child(const struct stack_depot_trie_children __rcu **slot, - const struct stack_depot_trie_children *children, - const struct stack_depot_trie_node *child, - unsigned int pos, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) + const struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *child, + unsigned int pos, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { struct stack_depot_trie_children *new_children; struct stack_depot_trie_node *promoted_node; unsigned long flags; size_t node_size; - u32 new_leaf_id; + u32 new_stack_id; - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) + new_stack_id = trie_side_table_prepare_stack_slot(side_prealloc); + if (!new_stack_id) return 0; raw_spin_lock_irqsave(&pool_lock, flags); @@ -2133,14 +2140,14 @@ trie_promote_child(const struct stack_depot_trie_children __rcu **slot, raw_spin_unlock_irqrestore(&pool_lock, flags); memcpy(promoted_node, child, node_size); - promoted_node->leaf_id = new_leaf_id; - trie_side_table_publish_new_leaf(new_leaf_id, promoted_node); + promoted_node->stack_id = new_stack_id; + trie_side_table_publish_new_node(new_stack_id, promoted_node); trie_children_replace_at(children, promoted_node, new_children, pos); trie_reparent_children(promoted_node); rcu_assign_pointer(*slot, new_children); trie_retire_children_with_node(children, child); - return new_leaf_id; + return new_stack_id; out_release_node: trie_pool_release(promoted_node, node_size); @@ -2150,11 +2157,11 @@ trie_promote_child(const struct stack_depot_trie_children __rcu **slot, return 0; } -static u32 trie_finish_insert(u32 leaf_id) +static u32 trie_finish_insert(u32 stack_id) { - if (leaf_id) - trie_side_table_last_leaf_id = leaf_id; - return leaf_id; + if (stack_id) + trie_side_table_last_stack_id = stack_id; + return stack_id; } static u32 @@ -2170,37 +2177,37 @@ stack_depot_trie_insert(const unsigned long *entries, struct stack_depot_trie_node *parent = NULL; unsigned int matched; unsigned int pos; - u32 leaf_id = 0; + u32 stack_id = 0; for (;;) { children = trie_load_children(slot); - /* Case 1: no matching child, so append the remaining stack path. */ + /* Case 1: no matching child, so add the remaining path. */ if (!trie_children_find_slot(children, entries[0], &pos)) { - leaf_id = trie_insert_path(slot, parent, children, pos, - entries, nr_entries, - pool_prealloc, - side_prealloc); + stack_id = trie_insert_path(slot, parent, children, pos, + entries, nr_entries, + pool_prealloc, + side_prealloc); break; } child = trie_children_load_child(children, pos); - matched = __stack_depot_trie_node_match(child, entries, nr_entries); + matched = trie_node_match(child, entries, nr_entries); /* Case 2: the match ends inside the child run, so split it. */ if (matched < child->run.nr_entries) { - leaf_id = trie_split_child(slot, children, child, pos, - matched, entries, nr_entries, - pool_prealloc, side_prealloc); + stack_id = trie_split_child(slot, children, child, pos, + matched, entries, nr_entries, + pool_prealloc, side_prealloc); break; } /* Case 3: the input ends here, so reuse or promote this child. */ if (matched == nr_entries) { - if (child->leaf_id) - return child->leaf_id; - leaf_id = trie_promote_child(slot, children, child, pos, - pool_prealloc, - side_prealloc); + if (child->stack_id) + return child->stack_id; + stack_id = trie_promote_child(slot, children, child, pos, + pool_prealloc, + side_prealloc); break; } @@ -2211,54 +2218,54 @@ stack_depot_trie_insert(const unsigned long *entries, nr_entries -= matched; } - return trie_finish_insert(leaf_id); + return trie_finish_insert(stack_id); } static unsigned int -__stack_depot_trie_fetch_into(const struct stack_depot_trie_node *leaf, - unsigned long *entries, - unsigned int max_entries) +trie_fetch_into(const struct stack_depot_trie_node *node, + unsigned long *entries, + unsigned int max_entries) { - const struct stack_depot_trie_node *node; + const struct stack_depot_trie_node *cur; unsigned int total; unsigned int pos; unsigned int i; total = 0; - for (node = leaf; node; node = trie_load_parent(node)) - total += node->run.nr_entries; + for (cur = node; cur; cur = trie_load_parent(cur)) + total += cur->run.nr_entries; if (WARN_ON_ONCE(!total)) return 0; if (max_entries < total) return 0; pos = total; - for (node = leaf; node; node = trie_load_parent(node)) { - pos -= node->run.nr_entries; - for (i = 0; i < node->run.nr_entries; i++) - stack_depot_trie_node_frame(node, i, &entries[pos + i]); + for (cur = node; cur; cur = trie_load_parent(cur)) { + pos -= cur->run.nr_entries; + for (i = 0; i < cur->run.nr_entries; i++) + stack_depot_trie_node_frame(cur, i, &entries[pos + i]); } return total; } static unsigned int -__stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, - unsigned long *entries, - unsigned int max_entries) +trie_fetch_handle_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries) { - const struct stack_depot_trie_node *leaf; - u32 leaf_id; + const struct stack_depot_trie_node *node; + u32 stack_id; unsigned int nr_entries; - leaf_id = __stack_depot_trie_leaf_id(handle); + stack_id = trie_stack_id(handle); rcu_read_lock_sched_notrace(); - leaf = __stack_depot_trie_side_table_lookup(leaf_id); - if (WARN_ONCE(!leaf, "corrupt trie handle %08x\n", handle)) { + node = trie_side_table_lookup(stack_id); + if (WARN_ONCE(!node, "corrupt trie handle %08x\n", handle)) { rcu_read_unlock_sched_notrace(); return 0; } - nr_entries = __stack_depot_trie_fetch_into(leaf, entries, max_entries); + nr_entries = trie_fetch_into(node, entries, max_entries); rcu_read_unlock_sched_notrace(); if (nr_entries) kmsan_unpoison_memory(entries, nr_entries * sizeof(*entries)); @@ -2309,7 +2316,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, return 0; WARN_ON_ONCE(!entries || !max_entries); if (stack_depot_handle_is_trie(handle)) - return __stack_depot_trie_fetch_handle_into(handle, entries, + return trie_fetch_handle_into(handle, entries, max_entries); stack = depot_fetch_stack(handle); @@ -2359,9 +2366,9 @@ void stack_depot_print(depot_stack_handle_t stack) if (stack_depot_handle_is_trie(stack)) { unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; - nr_entries = __stack_depot_trie_fetch_handle_into(stack, - trie_entries, - ARRAY_SIZE(trie_entries)); + nr_entries = trie_fetch_handle_into(stack, + trie_entries, + ARRAY_SIZE(trie_entries)); stack_trace_print(trie_entries, nr_entries, 0); return; } @@ -2381,9 +2388,9 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, if (stack_depot_handle_is_trie(handle)) { unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; - nr_entries = __stack_depot_trie_fetch_handle_into(handle, - trie_entries, - ARRAY_SIZE(trie_entries)); + nr_entries = trie_fetch_handle_into(handle, + trie_entries, + ARRAY_SIZE(trie_entries)); return stack_trace_snprint(buf, size, trie_entries, nr_entries, spaces); } From 8c4b290018e29e06d60371aa9d929f27032d6899 Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Thu, 30 Jul 2026 11:52:26 +0100 Subject: [PATCH 12/13] KRN-1117: Inline the one-caller trie_finish_insert() helper trie_finish_insert() is three lines and has a single caller. Folding it into stack_depot_trie_insert() makes the successful stack-ID update and return visible at the call site. --- lib/stackdepot.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 03b4ac5b8d5f3..9ff92bab10df4 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2157,13 +2157,6 @@ trie_promote_child(const struct stack_depot_trie_children __rcu **slot, return 0; } -static u32 trie_finish_insert(u32 stack_id) -{ - if (stack_id) - trie_side_table_last_stack_id = stack_id; - return stack_id; -} - static u32 stack_depot_trie_insert(const unsigned long *entries, unsigned int nr_entries, @@ -2218,7 +2211,10 @@ stack_depot_trie_insert(const unsigned long *entries, nr_entries -= matched; } - return trie_finish_insert(stack_id); + if (stack_id) + trie_side_table_last_stack_id = stack_id; + + return stack_id; } static unsigned int From 460ee168c0fdbe3311787bb78e042e49781f643c Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Thu, 30 Jul 2026 12:34:36 +0100 Subject: [PATCH 13/13] KRN-1117: Make trie update and lifetime mechanics explicit Rename trie_side_table_publish_new_node() to trie_side_table_publish() and reorder its arguments to (node, stack_id). Document that the path must be fully initialised before the call, that publication commits the path with no rollback, and that the side-table mapping must be installed before the trie slot that makes @node reachable. The two-node split variant had a single caller, so inline it into trie_split_child() and drop trie_side_table_publish_split_nodes(). Similarly drop trie_children_replace_at() and express the init plus pointer store at its callers, where the intent reads directly. Add kernel-doc for trie_pool_alloc()/trie_pool_release() clarifying that @size is a byte count, and for trie_pool_alloc_children()/ trie_pool_release_children() clarifying that @capacity counts child-pointer entries rather than bytes. Document trie_drain_pending_children() -- when it must run relative to pool_lock and allocation, the FIFO stop condition, and the paired node+children reclaim -- and document trie_reparent_children() as a reparent-before-retirement step, adding a lockdep assertion for the writer lock. Normalize trie function prototypes to keep the return type on the same line as the name where the signature fits in 100 columns. Validation: diff review, strict checkpatch, and kernel-doc all clean. UML stackdepot KUnit reports 8/8 passing on the behaviour-changing revision; the drain-comment refinement afterwards is comment-only. --- lib/stackdepot.c | 267 +++++++++++++++++++++++++++-------------------- 1 file changed, 155 insertions(+), 112 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 9ff92bab10df4..52755a56dd16b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -351,8 +351,7 @@ trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, rcu_read_lock_sched_held()); } -static u32 -trie_side_table_prepare_stack_slot(struct stack_depot_trie_side_prealloc *prealloc) +static u32 trie_side_table_prepare_stack_slot(struct stack_depot_trie_side_prealloc *prealloc) { const struct stack_depot_trie_node __rcu **chunk; struct stack_depot_trie_side_dir *dir; @@ -579,8 +578,7 @@ static inline size_t trie_children_alloc_size(size_t size) return offsetof(struct stack_depot_trie_retired_children, data) + size; } -static inline struct stack_depot_trie_retired_children * -trie_retired_children(const void *ptr) +static inline struct stack_depot_trie_retired_children *trie_retired_children(const void *ptr) { return container_of(ptr, struct stack_depot_trie_retired_children, data); } @@ -619,6 +617,17 @@ static unsigned int trie_pool_reserve_slots(struct stack_depot_trie_pool *pool, return STACK_DEPOT_TRIE_POOL_SLOTS; } +/** + * trie_pool_alloc() - Allocate byte-addressed storage from trie pools + * @size: Number of bytes to allocate + * @prealloc: Preallocated pool storage that may be consumed + * + * Round @size up to trie-pool slots and reserve one contiguous run. If + * no current pool can satisfy the request, @prealloc may be consumed to + * create a new pool. + * + * Return: A pointer to at least @size bytes, or %NULL on failure. + */ static void *trie_pool_alloc(size_t size, void **prealloc) { struct stack_depot_trie_pool *pool; @@ -648,6 +657,14 @@ static void *trie_pool_alloc(size_t size, void **prealloc) return (char *)pool + slot * STACK_DEPOT_TRIE_SLOT_SIZE; } +/** + * trie_pool_release() - Release byte-addressed trie-pool storage + * @ptr: Pointer returned by trie_pool_alloc() + * @size: Byte count originally passed to trie_pool_alloc() + * + * Release the trie-pool slots covering @size. The byte count must match + * the allocation, so the same rounded slot range returns to the pool. + */ static void trie_pool_release(const void *ptr, size_t size) { struct stack_depot_trie_pool *pool; @@ -668,6 +685,17 @@ static void trie_pool_release(const void *ptr, size_t size) pool->free_slots += nr_slots; } +/** + * trie_pool_alloc_children() - Allocate a children container + * @capacity: Number of child-node pointer entries, not a byte count + * @prealloc: Preallocated pool storage that may be consumed + * + * Allocate space for @capacity pointers plus hidden RCU-retirement + * metadata. The returned container is empty and has its capacity + * initialized. + * + * Return: An unpublished children container, or %NULL on failure. + */ static struct stack_depot_trie_children * trie_pool_alloc_children(unsigned int capacity, void **prealloc) { @@ -688,6 +716,13 @@ trie_pool_alloc_children(unsigned int capacity, void **prealloc) return children; } +/** + * trie_pool_release_children() - Release a children container + * @children: Container returned by trie_pool_alloc_children() + * + * Derive the allocation byte size from @children->capacity and release + * the container together with its hidden RCU-retirement metadata. + */ static void trie_pool_release_children(const struct stack_depot_trie_children *children) { size_t size = trie_children_bytes(children->capacity); @@ -696,6 +731,22 @@ static void trie_pool_release_children(const struct stack_depot_trie_children *c trie_children_alloc_size(size)); } +/** + * trie_drain_pending_children() - Reclaim retired trie-pool storage + * + * After publishing replacement children, trie_retire_children() records + * an RCU grace-period cookie in the old container's hidden metadata and + * appends it to pending_trie_children. When a node is replaced as well, + * the same metadata carries pending_node, so both allocations are + * reclaimed together. + * + * Call this after taking pool_lock and before allocating trie-pool + * storage, so completed grace periods return slots before searching for + * new space. Poll retired containers in FIFO order and stop when the + * first grace period is pending, because later entries cannot be ready + * yet. For each ready entry, release its optional node followed by the + * children container. + */ static void trie_drain_pending_children(void) { struct stack_depot_trie_retired_children *retired; @@ -729,9 +780,8 @@ static void trie_retire_children(const void *ptr) list_add_tail(&retired->list, &pending_trie_children); } -static void -trie_retire_children_with_node(const void *ptr, - const struct stack_depot_trie_node *node) +static void trie_retire_children_with_node(const void *ptr, + const struct stack_depot_trie_node *node) { struct stack_depot_trie_retired_children *retired; unsigned long flags; @@ -746,8 +796,7 @@ trie_retire_children_with_node(const void *ptr, static const struct stack_depot_trie_node * stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries); -static depot_stack_handle_t -trie_find_handle(const unsigned long *entries, unsigned int nr_entries) +static depot_stack_handle_t trie_find_handle(const unsigned long *entries, unsigned int nr_entries) { depot_stack_handle_t handle = 0; const struct stack_depot_trie_node *node; @@ -761,8 +810,27 @@ trie_find_handle(const unsigned long *entries, unsigned int nr_entries) return handle; } -static void trie_side_table_publish_new_node(u32 stack_id, - const struct stack_depot_trie_node *node) +/** + * trie_side_table_publish() - Publish a stack ID to lockless readers + * @node: Initialized node associated with @stack_id + * @stack_id: Non-zero stack ID to publish + * + * The node and its path to the root must be initialized before this + * call. All fallible allocation must also be complete. Publishing the + * stack-ID mapping commits the path, which cannot then be rolled back. + * + * This is the first half of the side-table/trie publication pair. The + * caller must publish the side-table mapping before publishing the + * children slot that makes the path reachable from the trie. Once that + * slot is visible, lookup may return @stack_id, so the side table must + * already resolve it to @node. + * + * Published storage must remain valid until retired through RCU. Frame + * data and children are immutable after publication, although parent + * links may still be updated through RCU while reparenting descendants. + */ +static void trie_side_table_publish(const struct stack_depot_trie_node *node, + u32 stack_id) { const struct stack_depot_trie_node __rcu **slot; @@ -773,29 +841,6 @@ static void trie_side_table_publish_new_node(u32 stack_id, rcu_assign_pointer(*slot, node); } -static void trie_side_table_publish_split_nodes(u32 old_stack_id, - const struct stack_depot_trie_node *old_node, - u32 new_stack_id, - const struct stack_depot_trie_node *new_node) -{ - const struct stack_depot_trie_node __rcu **new_slot; - const struct stack_depot_trie_node __rcu **old_slot = NULL; - - lockdep_assert_held(&stack_depot_trie_writer_lock); - - if (old_stack_id) - old_slot = trie_side_table_stack_slot(old_stack_id); - - new_slot = trie_side_table_stack_slot(new_stack_id); - if (old_slot) { - /* Pairs with trie_side_table_lookup(). */ - rcu_assign_pointer(*old_slot, old_node); - } - - /* Pairs with trie_side_table_lookup(). */ - rcu_assign_pointer(*new_slot, new_node); -} - static int __init disable_stack_depot(char *str) { return kstrtobool(str, &stack_depot_disabled); @@ -1348,15 +1393,13 @@ static inline struct stack_record *find_stack(struct list_head *bucket, return ret; } -static u32 -stack_depot_trie_insert(const unsigned long *entries, - unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc); +static u32 stack_depot_trie_insert(const unsigned long *entries, + unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc); -static depot_stack_handle_t -stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, - gfp_t alloc_flags) +static depot_stack_handle_t stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags) { struct stack_depot_trie_side_prealloc side_prealloc = {}; void *pool_prealloc = NULL; @@ -1561,9 +1604,8 @@ static void frame_run_init(const unsigned long *entries, run->nr_entries = i; } -static void -stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, - unsigned int index, unsigned long *frame) +static void stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, + unsigned int index, unsigned long *frame) { u32 payload; @@ -1622,10 +1664,9 @@ static void trie_node_init_slice(struct stack_depot_trie_node *node, node->run = run; } -static unsigned int -trie_node_match(const struct stack_depot_trie_node *node, - const unsigned long *entries, - unsigned int nr_entries) +static unsigned int trie_node_match(const struct stack_depot_trie_node *node, + const unsigned long *entries, + unsigned int nr_entries) { unsigned int limit; unsigned int i; @@ -1686,9 +1727,8 @@ trie_children_load_child(const struct stack_depot_trie_children *children, * a matching child. Return false and set @pos to the insertion index otherwise. * A NULL @children is treated as empty and returns @pos = 0. */ -static bool -trie_children_find_slot(const struct stack_depot_trie_children *children, - unsigned long frame, unsigned int *pos) +static bool trie_children_find_slot(const struct stack_depot_trie_children *children, + unsigned long frame, unsigned int *pos) { unsigned int left = 0; unsigned int right; @@ -1730,9 +1770,8 @@ trie_children_find_slot(const struct stack_depot_trie_children *children, * initialized to %NULL. The allocator must already have set the * capacity of @new. */ -static void -trie_children_init(const struct stack_depot_trie_children *old, - struct stack_depot_trie_children *new) +static void trie_children_init(const struct stack_depot_trie_children *old, + struct stack_depot_trie_children *new) { unsigned int nr_old; unsigned int i; @@ -1768,31 +1807,30 @@ static void trie_children_insert(struct stack_depot_trie_children *children, children->nr_children++; } -static void -trie_children_replace_at(const struct stack_depot_trie_children *old_children, - const struct stack_depot_trie_node *new_child, - struct stack_depot_trie_children *new_children, - unsigned int pos) -{ - trie_children_init(old_children, new_children); - RCU_INIT_POINTER(new_children->nodes[pos], new_child); -} - +/** + * trie_reparent_children() - Reparent children before node retirement + * @parent: Replacement node that references the reused children + * + * Use this when replacing a node while retaining its children and + * descendant subtrees. The direct children still point back to the node + * being replaced, so update their parent links before publishing the + * replacement topology and queuing the old node for RCU retirement. + * + * Lockless fetches may observe the new parent before the replacement is + * published. This is safe because the old and new parent chains contain + * the same frames and both remain RCU-live during the transition. + */ static void trie_reparent_children(struct stack_depot_trie_node *parent) { const struct stack_depot_trie_children *children; unsigned int i; + lockdep_assert_held(&stack_depot_trie_writer_lock); + children = trie_load_children(&parent->children); if (!children) return; - /* - * Replacement children reuse unchanged descendant subtrees. Repoint - * their parent links before retiring the old parent so fetch never - * follows a freed node. Lockless fetches may see the new parent before - * structural publication. The old and new parent chains contain the - * same frames and remain RCU-live. - */ + for (i = 0; i < children->nr_children; i++) { struct stack_depot_trie_node *child; @@ -1914,13 +1952,12 @@ stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) return NULL; } -static u32 -trie_insert_path(const struct stack_depot_trie_children __rcu **slot, - struct stack_depot_trie_node *parent, - const struct stack_depot_trie_children *children, - unsigned int pos, const unsigned long *entries, - unsigned int nr_entries, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) +static u32 trie_insert_path(const struct stack_depot_trie_children __rcu **slot, + struct stack_depot_trie_node *parent, + const struct stack_depot_trie_children *children, + unsigned int pos, const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { struct stack_depot_trie_children *new_children; const struct stack_depot_trie_node *path_root; @@ -1954,7 +1991,7 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, if (!path_root) goto err_release; - trie_side_table_publish_new_node(new_stack_id, node); + trie_side_table_publish(node, new_stack_id); trie_children_init(children, new_children); trie_children_insert(new_children, path_root, pos); @@ -1980,14 +2017,13 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, return 0; } -static u32 -trie_split_child(const struct stack_depot_trie_children __rcu **slot, - const struct stack_depot_trie_children *children, - const struct stack_depot_trie_node *child, - unsigned int pos, unsigned int matched, - const unsigned long *entries, unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) +static u32 trie_split_child(const struct stack_depot_trie_children __rcu **slot, + const struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *child, + unsigned int pos, unsigned int matched, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { struct stack_depot_trie_children *prefix_children = NULL; struct stack_depot_trie_children *new_children = NULL; @@ -2083,9 +2119,17 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, RCU_INIT_POINTER(old_suffix->children, trie_load_children(&child->children)); RCU_INIT_POINTER(split_prefix->children, prefix_children); - trie_side_table_publish_split_nodes(child->stack_id, old_suffix, - new_stack_id, new_node); - trie_children_replace_at(children, split_prefix, new_children, pos); + if (child->stack_id) + trie_side_table_publish(old_suffix, child->stack_id); + trie_side_table_publish(new_node, new_stack_id); + + /* + * Copy the current children into their replacement container, then + * store split_prefix at the position previously occupied by child. + */ + trie_children_init(children, new_children); + RCU_INIT_POINTER(new_children->nodes[pos], split_prefix); + trie_reparent_children(old_suffix); rcu_assign_pointer(*slot, new_children); trie_retire_children_with_node(children, child); @@ -2106,12 +2150,11 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, return 0; } -static u32 -trie_promote_child(const struct stack_depot_trie_children __rcu **slot, - const struct stack_depot_trie_children *children, - const struct stack_depot_trie_node *child, - unsigned int pos, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) +static u32 trie_promote_child(const struct stack_depot_trie_children __rcu **slot, + const struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *child, + unsigned int pos, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { struct stack_depot_trie_children *new_children; struct stack_depot_trie_node *promoted_node; @@ -2141,8 +2184,11 @@ trie_promote_child(const struct stack_depot_trie_children __rcu **slot, memcpy(promoted_node, child, node_size); promoted_node->stack_id = new_stack_id; - trie_side_table_publish_new_node(new_stack_id, promoted_node); - trie_children_replace_at(children, promoted_node, new_children, pos); + trie_side_table_publish(promoted_node, new_stack_id); + + trie_children_init(children, new_children); + RCU_INIT_POINTER(new_children->nodes[pos], promoted_node); + trie_reparent_children(promoted_node); rcu_assign_pointer(*slot, new_children); trie_retire_children_with_node(children, child); @@ -2157,11 +2203,10 @@ trie_promote_child(const struct stack_depot_trie_children __rcu **slot, return 0; } -static u32 -stack_depot_trie_insert(const unsigned long *entries, - unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) +static u32 stack_depot_trie_insert(const unsigned long *entries, + unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { const struct stack_depot_trie_children *children; const struct stack_depot_trie_children __rcu **slot = @@ -2217,10 +2262,9 @@ stack_depot_trie_insert(const unsigned long *entries, return stack_id; } -static unsigned int -trie_fetch_into(const struct stack_depot_trie_node *node, - unsigned long *entries, - unsigned int max_entries) +static unsigned int trie_fetch_into(const struct stack_depot_trie_node *node, + unsigned long *entries, + unsigned int max_entries) { const struct stack_depot_trie_node *cur; unsigned int total; @@ -2245,10 +2289,9 @@ trie_fetch_into(const struct stack_depot_trie_node *node, return total; } -static unsigned int -trie_fetch_handle_into(depot_stack_handle_t handle, - unsigned long *entries, - unsigned int max_entries) +static unsigned int trie_fetch_handle_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries) { const struct stack_depot_trie_node *node; u32 stack_id;