diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 6bb63890d0bdc..52755a56dd16b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -6,7 +6,7 @@ * callers that request STACK_DEPOT_FLAG_COUNTABLE use the legacy hash table with * contiguous stack records in stack pools. Persistent non-refcounted entries * can use trie storage when enabled; trie nodes share common frame prefixes and - * are published through RCU/COW child arrays. + * are published through immutable RCU children containers. * * Author: Alexander Potapenko * Copyright (C) 2016 Google, Inc. @@ -124,42 +124,40 @@ struct stack_depot_frame_run { static_assert(CONFIG_STACKDEPOT_MAX_FRAMES <= U16_MAX); -struct stack_depot_trie_child_array; +struct stack_depot_trie_children; struct stack_depot_trie_node { - /* Parent links let fetch rebuild a full stack from a leaf to the root. */ + /* Parent links let fetch rebuild a full stack from a node to the root. */ const struct stack_depot_trie_node __rcu *parent; - /* Child arrays are separate RCU/COW generations. */ - const struct stack_depot_trie_child_array __rcu *children; - u32 leaf_id; + /* Children are immutable RCU-published containers. */ + const struct stack_depot_trie_children __rcu *children; + /* Non-zero when a stored stack ends at this node. */ + u32 stack_id; struct stack_depot_frame_run run; unsigned char data[]; }; /* - * Children are sorted by first frame and searched by insertion slot. - * Writers may append to spare capacity at the sorted tail, but never change - * existing child pointers. Other updates build and publish a replacement array. + * Children are sorted by first frame and searched by insertion position. + * Published children are immutable; updates build and publish a replacement. */ -struct stack_depot_trie_child_array { +struct stack_depot_trie_children { unsigned int nr_children; unsigned int capacity; - const struct stack_depot_trie_node __rcu *children[]; + const struct stack_depot_trie_node __rcu *nodes[]; }; -/* A retired child array carries an optional node through its RCU grace period. */ -struct stack_depot_trie_retired_array { +/* Retired children carry an optional node through their RCU grace period. */ +struct stack_depot_trie_retired_children { struct list_head list; unsigned long rcu_state; struct stack_depot_trie_node *pending_node; unsigned char data[]; }; -static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_retired_array, data), +static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_retired_children, data), 1UL << DEPOT_STACK_ALIGN)); -#define STACK_DEPOT_TRIE_MAX_NODES (CONFIG_STACKDEPOT_MAX_FRAMES + 1) -#define STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS CONFIG_STACKDEPOT_MAX_FRAMES #define STACK_DEPOT_TRIE_SLOT_SIZE BIT(DEPOT_STACK_ALIGN) #define STACK_DEPOT_TRIE_POOL_SLOTS \ (DEPOT_POOL_SIZE / STACK_DEPOT_TRIE_SLOT_SIZE) @@ -176,18 +174,8 @@ struct stack_depot_trie_pool { static_assert(STACK_DEPOT_TRIE_POOL_FIRST_SLOT < STACK_DEPOT_TRIE_POOL_SLOTS); -/* Writer-owned storage for one unpublished trie insertion. */ -struct stack_depot_trie_insert_alloc { - struct stack_depot_trie_node *nodes[STACK_DEPOT_TRIE_MAX_NODES]; - size_t node_sizes[STACK_DEPOT_TRIE_MAX_NODES]; - struct stack_depot_trie_child_array *chain_arrays[STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS]; - struct stack_depot_trie_child_array *prefix_children; - struct stack_depot_trie_child_array *slot_array; -}; - static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); -static const struct stack_depot_trie_child_array __rcu *stack_depot_trie_root; -static struct stack_depot_trie_insert_alloc *stack_depot_trie_alloc; +static const struct stack_depot_trie_children __rcu *stack_depot_trie_root; static DEFINE_RAW_SPINLOCK(stack_depot_trie_writer_lock); static bool stack_depot_trie_requested; @@ -199,29 +187,29 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); /* Retired fixed-size slots remain reserved until their RCU grace period ends. */ static LIST_HEAD(stack_depot_trie_pools); -static LIST_HEAD(pending_trie_arrays); +static LIST_HEAD(pending_trie_children); /* * stack_max_pools is the split point between hash and trie handle encodings. * A handle with pool_index_plus_1 in 1..stack_max_pools names a hash-backed * stack pool. Larger pool-index values cannot refer to hash pools, so trie - * storage uses that handle space to encode a dense leaf_id. The side table - * maps each leaf_id to its trie leaf node. + * storage uses that handle space to encode a dense stack ID. The side + * table maps each stack ID to its trie node. */ -static inline u32 __stack_depot_trie_max_leaf_id(void) +static inline u32 trie_max_stack_id(void) { return (DEPOT_POOL_INDEX_MASK - stack_max_pools) << DEPOT_OFFSET_BITS; } -static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) +static depot_stack_handle_t trie_handle(u32 stack_id) { union handle_parts parts = {}; u64 pool_index_plus_1; u32 pool_delta; u32 index; - index = leaf_id - 1; + index = stack_id - 1; pool_delta = index >> DEPOT_OFFSET_BITS; pool_index_plus_1 = (u64)stack_max_pools + 1 + pool_delta; @@ -237,7 +225,7 @@ static inline bool stack_depot_handle_is_trie(depot_stack_handle_t handle) return parts.pool_index_plus_1 > stack_max_pools; } -static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) +static u32 trie_stack_id(depot_stack_handle_t handle) { union handle_parts parts = { .handle = handle }; u32 pool_delta; @@ -247,13 +235,16 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) } /* - * Trie handles encode a dense leaf ID. The side table maps that ID to a leaf - * pointer for lockless fetch/print paths, which can run from diagnostic - * contexts where taking a lock would be unsafe. Initialization - * installs the root; early initialization also installs the first directory and - * chunk. Additional directories and chunks are preallocated and published - * lazily as leaf IDs grow. RCU pointer publication makes fully initialized - * directories, chunks, and leaves visible to those lockless readers. + * Trie handles encode a dense stack ID. The side table maps that ID + * to a node pointer for lockless fetch and print operations. Those + * operations can run from diagnostic contexts where taking a lock would + * be unsafe. + * + * Initialization installs the side-table root. Early initialization + * also installs the first directory and chunk. Additional directories + * and chunks are preallocated and published lazily as stack IDs grow. + * RCU publication makes initialized directories, chunks, and nodes + * visible to those lockless readers. */ #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ (PAGE_SIZE / sizeof(struct stack_depot_trie_node *)) @@ -261,7 +252,7 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) (PAGE_SIZE / sizeof(struct stack_depot_trie_node **)) struct stack_depot_trie_side_dir { - /* Both the chunk pointer and each leaf pointer in it are RCU-published. */ + /* The chunk and each node pointer are RCU-published. */ const struct stack_depot_trie_node __rcu * __rcu * chunks[STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE]; }; @@ -274,7 +265,7 @@ struct stack_depot_trie_side_root { struct stack_depot_trie_side_prealloc { /* Preallocated side-table directory page for sparse growth. */ struct stack_depot_trie_side_dir *dir; - /* Preallocated side-table leaf chunk for sparse growth. */ + /* Preallocated side-table pointer chunk for sparse growth. */ const struct stack_depot_trie_node __rcu **chunk; }; @@ -282,7 +273,7 @@ static struct stack_depot_trie_side_root *trie_side_table_root; static DEFINE_RAW_SPINLOCK(trie_side_table_cache_lock); /* Zeroed unpublished pages; get/put transfer ownership under the cache lock. */ static struct stack_depot_trie_side_prealloc trie_side_table_cache; -static u32 trie_side_table_last_leaf_id; +static u32 trie_side_table_last_stack_id; /* Lock order: writer_lock -> pool_lock. The cache lock is never nested. */ @@ -298,34 +289,26 @@ static inline size_t stack_depot_frame_run_bytes(const struct stack_depot_frame_ return run->nr_entries * stack_depot_frame_run_entry_bytes(run->mode); } -static inline size_t trie_node_bytes(const struct stack_depot_frame_run *run) +static inline size_t trie_node_bytes_for(enum stack_depot_frame_mode mode, + unsigned int nr_entries) { return ALIGN(offsetof(struct stack_depot_trie_node, data) + - stack_depot_frame_run_bytes(run), sizeof(unsigned long)); + nr_entries * stack_depot_frame_run_entry_bytes(mode), + sizeof(unsigned long)); } -static size_t trie_child_array_bytes(unsigned int capacity) +static inline size_t trie_node_bytes(const struct stack_depot_frame_run *run) { - size_t size; - - size = struct_size_t(struct stack_depot_trie_child_array, children, - capacity); - return ALIGN(size, sizeof(unsigned long)); + return trie_node_bytes_for(run->mode, run->nr_entries); } -static void trie_child_array_init(struct stack_depot_trie_child_array *array, - unsigned int capacity, - const struct stack_depot_trie_node * const *nodes, - unsigned int nr_children) +static size_t trie_children_bytes(unsigned int capacity) { - unsigned int i; + size_t size; - array->nr_children = nr_children; - array->capacity = capacity; - for (i = 0; i < nr_children; i++) - RCU_INIT_POINTER(array->children[i], nodes[i]); - for (i = nr_children; i < capacity; i++) - RCU_INIT_POINTER(array->children[i], NULL); + size = struct_size_t(struct stack_depot_trie_children, nodes, + capacity); + return ALIGN(size, sizeof(unsigned long)); } static inline unsigned int trie_side_table_root_index(u32 id) @@ -362,14 +345,13 @@ static inline const struct stack_depot_trie_node __rcu ** trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, unsigned int idx) { - /* Pairs with the chunk rcu_assign_pointer() in leaf ID preparation. */ + /* Pairs with the chunk rcu_assign_pointer() in stack ID preparation. */ return rcu_dereference_check(dir->chunks[idx], lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } -static u32 -trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *prealloc) +static u32 trie_side_table_prepare_stack_slot(struct stack_depot_trie_side_prealloc *prealloc) { const struct stack_depot_trie_node __rcu **chunk; struct stack_depot_trie_side_dir *dir; @@ -380,8 +362,8 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo lockdep_assert_held(&stack_depot_trie_writer_lock); - id = trie_side_table_last_leaf_id + 1; - if (id > __stack_depot_trie_max_leaf_id()) + id = trie_side_table_last_stack_id + 1; + if (id > trie_max_stack_id()) return 0; root_vec = trie_side_table_root; @@ -413,30 +395,30 @@ static void trie_side_table_root_init(struct stack_depot_trie_side_root *root_ve unsigned int root_size) { root_vec->dir_capacity = root_size; - trie_side_table_last_leaf_id = 0; + trie_side_table_last_stack_id = 0; } -static inline unsigned int trie_side_table_root_size_for_max_id(u32 max_leaf_id) +static inline unsigned int trie_side_table_root_size_for_max_id(u32 max_stack_id) { unsigned int top_size; - top_size = DIV_ROUND_UP(max_leaf_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); + top_size = DIV_ROUND_UP(max_stack_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); return DIV_ROUND_UP(top_size, STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE); } -static int __init __stack_depot_trie_side_table_init_memblock(void) +static int __init trie_side_table_init_memblock(void) { struct stack_depot_trie_side_root *root_vec; struct stack_depot_trie_side_dir *first_dir; const struct stack_depot_trie_node __rcu **first_chunk; size_t root_bytes; - u32 max_leaf_id; + u32 max_stack_id; unsigned int root_size; - max_leaf_id = __stack_depot_trie_max_leaf_id(); - if (!max_leaf_id) + max_stack_id = trie_max_stack_id(); + if (!max_stack_id) return -EINVAL; - root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_size = trie_side_table_root_size_for_max_id(max_stack_id); root_bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); root_vec = memblock_alloc(root_bytes, __alignof__(*root_vec)); @@ -464,18 +446,18 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) return 0; } -static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) +static int trie_side_table_init(gfp_t gfp_flags) { struct stack_depot_trie_side_root *root_vec; unsigned int root_size; size_t root_bytes; - u32 max_leaf_id; + u32 max_stack_id; - max_leaf_id = __stack_depot_trie_max_leaf_id(); - if (!max_leaf_id) + max_stack_id = trie_max_stack_id(); + if (!max_stack_id) return -EINVAL; - root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_size = trie_side_table_root_size_for_max_id(max_stack_id); root_bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); root_vec = kvzalloc(root_bytes, gfp_flags); if (!root_vec) @@ -488,21 +470,11 @@ static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) static int __init stack_depot_trie_init_memblock(void) { - struct stack_depot_trie_insert_alloc *alloc; - size_t size; int ret; - size = sizeof(*stack_depot_trie_alloc); - alloc = memblock_alloc(size, __alignof__(*alloc)); - if (!alloc) - return -ENOMEM; - - ret = __stack_depot_trie_side_table_init_memblock(); - if (ret) { - memblock_free(alloc, size); + ret = trie_side_table_init_memblock(); + if (ret) return ret; - } - stack_depot_trie_alloc = alloc; static_branch_enable(&stack_depot_trie_enabled); return 0; @@ -510,19 +482,11 @@ static int __init stack_depot_trie_init_memblock(void) static int stack_depot_trie_init(gfp_t gfp_flags) { - struct stack_depot_trie_insert_alloc *alloc; int ret; - alloc = kvzalloc(sizeof(*stack_depot_trie_alloc), gfp_flags); - if (!alloc) - return -ENOMEM; - - ret = __stack_depot_trie_side_table_init(gfp_flags); - if (ret) { - kvfree(alloc); + ret = trie_side_table_init(gfp_flags); + if (ret) return ret; - } - stack_depot_trie_alloc = alloc; static_branch_enable(&stack_depot_trie_enabled); return 0; @@ -576,7 +540,7 @@ static void trie_side_table_put_prealloc(struct stack_depot_trie_side_prealloc * free_page((unsigned long)prealloc->chunk); } -static const struct stack_depot_trie_node __rcu **trie_side_table_leaf_slot(u32 id) +static const struct stack_depot_trie_node __rcu **trie_side_table_stack_slot(u32 id) { const struct stack_depot_trie_node __rcu **chunk; struct stack_depot_trie_side_dir *dir; @@ -589,7 +553,7 @@ static const struct stack_depot_trie_node __rcu **trie_side_table_leaf_slot(u32 return &chunk[trie_side_table_slot_index(id)]; } -static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup(u32 id) +static const struct stack_depot_trie_node *trie_side_table_lookup(u32 id) { const struct stack_depot_trie_node __rcu **chunk; struct stack_depot_trie_side_dir *dir; @@ -603,21 +567,20 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( if (!chunk) return NULL; - /* Pairs with side-table leaf rcu_assign_pointer(). */ + /* Pairs with side-table node publication. */ return rcu_dereference_check(chunk[trie_side_table_slot_index(id)], lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } -static inline size_t trie_array_alloc_size(size_t size) +static inline size_t trie_children_alloc_size(size_t size) { - return offsetof(struct stack_depot_trie_retired_array, data) + size; + return offsetof(struct stack_depot_trie_retired_children, data) + size; } -static inline struct stack_depot_trie_retired_array * -trie_retired_array(const void *ptr) +static inline struct stack_depot_trie_retired_children *trie_retired_children(const void *ptr) { - return container_of(ptr, struct stack_depot_trie_retired_array, data); + return container_of(ptr, struct stack_depot_trie_retired_children, data); } static bool depot_init_pool(void **prealloc); @@ -654,6 +617,17 @@ static unsigned int trie_pool_reserve_slots(struct stack_depot_trie_pool *pool, return STACK_DEPOT_TRIE_POOL_SLOTS; } +/** + * trie_pool_alloc() - Allocate byte-addressed storage from trie pools + * @size: Number of bytes to allocate + * @prealloc: Preallocated pool storage that may be consumed + * + * Round @size up to trie-pool slots and reserve one contiguous run. If + * no current pool can satisfy the request, @prealloc may be consumed to + * create a new pool. + * + * Return: A pointer to at least @size bytes, or %NULL on failure. + */ static void *trie_pool_alloc(size_t size, void **prealloc) { struct stack_depot_trie_pool *pool; @@ -683,6 +657,14 @@ static void *trie_pool_alloc(size_t size, void **prealloc) return (char *)pool + slot * STACK_DEPOT_TRIE_SLOT_SIZE; } +/** + * trie_pool_release() - Release byte-addressed trie-pool storage + * @ptr: Pointer returned by trie_pool_alloc() + * @size: Byte count originally passed to trie_pool_alloc() + * + * Release the trie-pool slots covering @size. The byte count must match + * the allocation, so the same rounded slot range returns to the pool. + */ static void trie_pool_release(const void *ptr, size_t size) { struct stack_depot_trie_pool *pool; @@ -703,236 +685,160 @@ static void trie_pool_release(const void *ptr, size_t size) pool->free_slots += nr_slots; } -static struct stack_depot_trie_child_array * -trie_pool_alloc_array(size_t size, void **prealloc) +/** + * trie_pool_alloc_children() - Allocate a children container + * @capacity: Number of child-node pointer entries, not a byte count + * @prealloc: Preallocated pool storage that may be consumed + * + * Allocate space for @capacity pointers plus hidden RCU-retirement + * metadata. The returned container is empty and has its capacity + * initialized. + * + * Return: An unpublished children container, or %NULL on failure. + */ +static struct stack_depot_trie_children * +trie_pool_alloc_children(unsigned int capacity, void **prealloc) { - struct stack_depot_trie_retired_array *retired; + struct stack_depot_trie_retired_children *retired; + struct stack_depot_trie_children *children; + size_t size; + + size = trie_children_bytes(capacity); + if (size > DEPOT_POOL_SIZE) + return NULL; + retired = trie_pool_alloc(trie_children_alloc_size(size), prealloc); + if (!retired) + return NULL; - retired = trie_pool_alloc(trie_array_alloc_size(size), prealloc); - return retired ? (void *)retired->data : NULL; + children = (void *)retired->data; + children->nr_children = 0; + children->capacity = capacity; + return children; } -static void trie_pool_release_array(const void *ptr, size_t size) +/** + * trie_pool_release_children() - Release a children container + * @children: Container returned by trie_pool_alloc_children() + * + * Derive the allocation byte size from @children->capacity and release + * the container together with its hidden RCU-retirement metadata. + */ +static void trie_pool_release_children(const struct stack_depot_trie_children *children) { - trie_pool_release(trie_retired_array(ptr), trie_array_alloc_size(size)); + size_t size = trie_children_bytes(children->capacity); + + trie_pool_release(trie_retired_children(children), + trie_children_alloc_size(size)); } -static void trie_drain_pending_arrays(void) +/** + * trie_drain_pending_children() - Reclaim retired trie-pool storage + * + * After publishing replacement children, trie_retire_children() records + * an RCU grace-period cookie in the old container's hidden metadata and + * appends it to pending_trie_children. When a node is replaced as well, + * the same metadata carries pending_node, so both allocations are + * reclaimed together. + * + * Call this after taking pool_lock and before allocating trie-pool + * storage, so completed grace periods return slots before searching for + * new space. Poll retired containers in FIFO order and stop when the + * first grace period is pending, because later entries cannot be ready + * yet. For each ready entry, release its optional node followed by the + * children container. + */ +static void trie_drain_pending_children(void) { - struct stack_depot_trie_retired_array *retired; - struct stack_depot_trie_retired_array *tmp; - struct stack_depot_trie_child_array *array; + struct stack_depot_trie_retired_children *retired; + struct stack_depot_trie_retired_children *tmp; + struct stack_depot_trie_children *children; lockdep_assert_held(&pool_lock); - list_for_each_entry_safe(retired, tmp, &pending_trie_arrays, list) { - /* Pending arrays are FIFO; later entries cannot be ready yet. */ + list_for_each_entry_safe(retired, tmp, &pending_trie_children, list) { + /* Pending children are FIFO; later entries cannot be ready yet. */ if (!poll_state_synchronize_rcu(retired->rcu_state)) break; - array = (void *)retired->data; + children = (void *)retired->data; list_del(&retired->list); if (retired->pending_node) trie_pool_release(retired->pending_node, trie_node_bytes(&retired->pending_node->run)); - trie_pool_release_array(array, - trie_child_array_bytes(array->capacity)); + trie_pool_release_children(children); } } -static void trie_retire_child_array_locked(const void *ptr) +static void trie_retire_children(const void *ptr) { - struct stack_depot_trie_retired_array *retired; + struct stack_depot_trie_retired_children *retired; lockdep_assert_held(&pool_lock); - retired = trie_retired_array(ptr); + retired = trie_retired_children(ptr); retired->pending_node = NULL; retired->rcu_state = get_state_synchronize_rcu(); - list_add_tail(&retired->list, &pending_trie_arrays); + list_add_tail(&retired->list, &pending_trie_children); } -static void -trie_retire_child_array_with_node(const void *ptr, - const struct stack_depot_trie_node *node) +static void trie_retire_children_with_node(const void *ptr, + const struct stack_depot_trie_node *node) { - struct stack_depot_trie_retired_array *retired; + struct stack_depot_trie_retired_children *retired; unsigned long flags; raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_child_array_locked(ptr); - retired = trie_retired_array(ptr); + trie_retire_children(ptr); + retired = trie_retired_children(ptr); retired->pending_node = (struct stack_depot_trie_node *)node; raw_spin_unlock_irqrestore(&pool_lock, flags); } -/* - * Preallocate resources that cannot be allocated while trie writers hold raw - * spinlocks. Side-table growth is mandatory before a new leaf ID can be - * used, so side-table preallocation failure disables insertion for this - * save. Pool preallocation is opportunistic: reusable trie slots may still - * satisfy the insertion, and trie_pool_alloc_insert() reports -ENOSPC if they - * do not. - */ -static int -__stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) -{ - unsigned long flags; - bool need_pool; - int ret; - - raw_spin_lock_irqsave(&pool_lock, flags); - need_pool = !new_pool; - raw_spin_unlock_irqrestore(&pool_lock, flags); - if (need_pool) { - struct page *page; - - page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); - if (page) - *pool_prealloc = page_address(page); - } - ret = trie_side_table_get_prealloc(alloc_flags, side_prealloc); - - if (ret) { - if (*pool_prealloc) { - free_pages((unsigned long)*pool_prealloc, DEPOT_POOL_ORDER); - *pool_prealloc = NULL; - } - return -ENOSPC; - } - return 0; -} - -/* - * Reserve fixed-size pool slots for one trie insertion. If any reservation - * fails, all slots reserved by this attempt are released locally. - * The caller must not publish any returned storage before side-table and trie - * publication succeeds. - */ -static int trie_pool_alloc_insert(struct stack_depot_trie_insert_alloc *alloc, - void **pool_prealloc, - unsigned int nr_nodes, - unsigned int nr_chain_arrays, - size_t prefix_children_size, - size_t slot_array_size) -{ - unsigned long flags; - size_t one_child_size; - unsigned int i; - int ret = -ENOSPC; - - memset(alloc->nodes, 0, nr_nodes * sizeof(*alloc->nodes)); - memset(alloc->chain_arrays, 0, - nr_chain_arrays * sizeof(*alloc->chain_arrays)); - alloc->prefix_children = NULL; - alloc->slot_array = NULL; - - raw_spin_lock_irqsave(&pool_lock, flags); - printk_deferred_enter(); - trie_drain_pending_arrays(); - one_child_size = trie_child_array_bytes(1); - for (i = 0; i < nr_nodes; i++) { - alloc->nodes[i] = trie_pool_alloc(alloc->node_sizes[i], - pool_prealloc); - if (!alloc->nodes[i]) - goto out_discard; - } - for (i = 0; i < nr_chain_arrays; i++) { - alloc->chain_arrays[i] = - trie_pool_alloc_array(one_child_size, pool_prealloc); - if (!alloc->chain_arrays[i]) - goto out_discard; - } - if (prefix_children_size) { - alloc->prefix_children = - trie_pool_alloc_array(prefix_children_size, pool_prealloc); - if (!alloc->prefix_children) - goto out_discard; - } - if (slot_array_size) { - alloc->slot_array = - trie_pool_alloc_array(slot_array_size, pool_prealloc); - if (!alloc->slot_array) - goto out_discard; - } - ret = 0; - goto out; -out_discard: - for (i = 0; i < nr_nodes; i++) { - struct stack_depot_trie_node *node = alloc->nodes[i]; - - if (node) - trie_pool_release(node, alloc->node_sizes[i]); - } - for (i = 0; i < nr_chain_arrays; i++) { - struct stack_depot_trie_child_array *array = alloc->chain_arrays[i]; - - if (!array) - continue; - trie_pool_release_array(array, one_child_size); - } - if (alloc->prefix_children) - trie_pool_release_array(alloc->prefix_children, - prefix_children_size); - if (alloc->slot_array) - trie_pool_release_array(alloc->slot_array, slot_array_size); -out: - printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); - return ret; -} - static const struct stack_depot_trie_node * stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries); -static depot_stack_handle_t -trie_find_handle(const unsigned long *entries, unsigned int nr_entries) +static depot_stack_handle_t trie_find_handle(const unsigned long *entries, unsigned int nr_entries) { depot_stack_handle_t handle = 0; - const struct stack_depot_trie_node *leaf; + const struct stack_depot_trie_node *node; rcu_read_lock_sched_notrace(); - leaf = stack_depot_trie_lookup(entries, nr_entries); - if (leaf) - handle = __stack_depot_trie_handle(leaf->leaf_id); + node = stack_depot_trie_lookup(entries, nr_entries); + if (node) + handle = trie_handle(node->stack_id); rcu_read_unlock_sched_notrace(); return handle; } -static void trie_side_table_publish_new_leaf(u32 leaf_id, - const struct stack_depot_trie_node *leaf) +/** + * trie_side_table_publish() - Publish a stack ID to lockless readers + * @node: Initialized node associated with @stack_id + * @stack_id: Non-zero stack ID to publish + * + * The node and its path to the root must be initialized before this + * call. All fallible allocation must also be complete. Publishing the + * stack-ID mapping commits the path, which cannot then be rolled back. + * + * This is the first half of the side-table/trie publication pair. The + * caller must publish the side-table mapping before publishing the + * children slot that makes the path reachable from the trie. Once that + * slot is visible, lookup may return @stack_id, so the side table must + * already resolve it to @node. + * + * Published storage must remain valid until retired through RCU. Frame + * data and children are immutable after publication, although parent + * links may still be updated through RCU while reparenting descendants. + */ +static void trie_side_table_publish(const struct stack_depot_trie_node *node, + u32 stack_id) { const struct stack_depot_trie_node __rcu **slot; lockdep_assert_held(&stack_depot_trie_writer_lock); - slot = trie_side_table_leaf_slot(leaf_id); - /* Pairs with __stack_depot_trie_side_table_lookup(). */ - rcu_assign_pointer(*slot, leaf); -} - -static void trie_side_table_publish_split_leaves(u32 old_leaf_id, - const struct stack_depot_trie_node *old_leaf, - u32 new_leaf_id, - const struct stack_depot_trie_node *new_leaf) -{ - const struct stack_depot_trie_node __rcu **new_slot; - const struct stack_depot_trie_node __rcu **old_slot = NULL; - - lockdep_assert_held(&stack_depot_trie_writer_lock); - - if (old_leaf_id) - old_slot = trie_side_table_leaf_slot(old_leaf_id); - - new_slot = trie_side_table_leaf_slot(new_leaf_id); - if (old_slot) { - /* Pairs with __stack_depot_trie_side_table_lookup(). */ - rcu_assign_pointer(*old_slot, old_leaf); - } - - /* Pairs with __stack_depot_trie_side_table_lookup(). */ - rcu_assign_pointer(*new_slot, new_leaf); + slot = trie_side_table_stack_slot(stack_id); + /* Pairs with trie_side_table_lookup(). */ + rcu_assign_pointer(*slot, node); } static int __init disable_stack_depot(char *str) @@ -1487,63 +1393,48 @@ static inline struct stack_record *find_stack(struct list_head *bucket, return ret; } -static u32 -stack_depot_trie_insert_locked(const unsigned long *entries, - unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc); +static u32 stack_depot_trie_insert(const unsigned long *entries, + unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc); -static depot_stack_handle_t -stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, - gfp_t alloc_flags) +static depot_stack_handle_t stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags) { struct stack_depot_trie_side_prealloc side_prealloc = {}; void *pool_prealloc = NULL; - depot_stack_handle_t handle; + depot_stack_handle_t handle = 0; unsigned long flags; - bool retried = false; - u32 leaf_id; - int ret; + struct page *page; + u32 stack_id; -retry: handle = trie_find_handle(entries, nr_entries); if (handle) return handle; - ret = __stack_depot_trie_alloc_prealloc(alloc_flags, &pool_prealloc, - &side_prealloc); - if (ret) - goto out_free; + if (trie_side_table_get_prealloc(alloc_flags, &side_prealloc)) + return 0; + + /* If trie insertion does not consume this page, keep it for the next save. */ + page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); + if (page) + pool_prealloc = page_address(page); raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); - leaf_id = stack_depot_trie_insert_locked(entries, nr_entries, - &pool_prealloc, &side_prealloc); - if (leaf_id) - handle = __stack_depot_trie_handle(leaf_id); + stack_id = stack_depot_trie_insert(entries, nr_entries, + &pool_prealloc, &side_prealloc); + if (stack_id) + handle = trie_handle(stack_id); raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); - if (!handle && !retried) { - retried = true; - if (pool_prealloc) { - raw_spin_lock_irqsave(&pool_lock, flags); - depot_keep_new_pool(&pool_prealloc); - raw_spin_unlock_irqrestore(&pool_lock, flags); - } - if (pool_prealloc) { - free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); - pool_prealloc = NULL; - } - trie_side_table_put_prealloc(&side_prealloc); - goto retry; - } -out_free: if (pool_prealloc) { raw_spin_lock_irqsave(&pool_lock, flags); depot_keep_new_pool(&pool_prealloc); raw_spin_unlock_irqrestore(&pool_lock, flags); + /* depot_keep_new_pool() clears pool_prealloc if it consumes it. */ + if (pool_prealloc) + free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); } - if (pool_prealloc) - free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); trie_side_table_put_prealloc(&side_prealloc); return handle; } @@ -1713,9 +1604,8 @@ static void frame_run_init(const unsigned long *entries, run->nr_entries = i; } -static void -stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, - unsigned int index, unsigned long *frame) +static void stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, + unsigned int index, unsigned long *frame) { u32 payload; @@ -1730,7 +1620,7 @@ stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, } static void trie_node_init(struct stack_depot_trie_node *node, - const struct stack_depot_trie_node *parent, u32 leaf_id, + const struct stack_depot_trie_node *parent, u32 stack_id, const unsigned long *entries, const struct stack_depot_frame_run *run) { @@ -1750,12 +1640,12 @@ static void trie_node_init(struct stack_depot_trie_node *node, RCU_INIT_POINTER(node->parent, parent); RCU_INIT_POINTER(node->children, NULL); - node->leaf_id = leaf_id; + node->stack_id = stack_id; node->run = *run; } static void trie_node_init_slice(struct stack_depot_trie_node *node, - const struct stack_depot_trie_node *parent, u32 leaf_id, + const struct stack_depot_trie_node *parent, u32 stack_id, const struct stack_depot_trie_node *src_node, unsigned int start, unsigned int nr_entries) { @@ -1770,14 +1660,13 @@ static void trie_node_init_slice(struct stack_depot_trie_node *node, stack_depot_frame_run_bytes(&run)); RCU_INIT_POINTER(node->parent, parent); RCU_INIT_POINTER(node->children, NULL); - node->leaf_id = leaf_id; + node->stack_id = stack_id; node->run = run; } -static unsigned int -__stack_depot_trie_node_match(const struct stack_depot_trie_node *node, - const unsigned long *entries, - unsigned int nr_entries) +static unsigned int trie_node_match(const struct stack_depot_trie_node *node, + const unsigned long *entries, + unsigned int nr_entries) { unsigned int limit; unsigned int i; @@ -1814,8 +1703,8 @@ trie_load_parent(const struct stack_depot_trie_node *node) rcu_read_lock_sched_held()); } -static inline const struct stack_depot_trie_child_array * -trie_load_children_slot(const struct stack_depot_trie_child_array __rcu * const *slot) +static inline const struct stack_depot_trie_children * +trie_load_children(const struct stack_depot_trie_children __rcu * const *slot) { return rcu_dereference_check(*slot, lockdep_is_held(&stack_depot_trie_writer_lock) || @@ -1823,33 +1712,39 @@ trie_load_children_slot(const struct stack_depot_trie_child_array __rcu * const } static inline const struct stack_depot_trie_node * -trie_child_array_load_child(const struct stack_depot_trie_child_array *array, - unsigned int pos) +trie_children_load_child(const struct stack_depot_trie_children *children, + unsigned int pos) { - return rcu_dereference_check(array->children[pos], + return rcu_dereference_check(children->nodes[pos], lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } -static bool -trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, - unsigned long frame, unsigned int *pos) +/* + * Find the child position for @frame. + * + * Return true and set @pos to the matching child index when @children contains + * a matching child. Return false and set @pos to the insertion index otherwise. + * A NULL @children is treated as empty and returns @pos = 0. + */ +static bool trie_children_find_slot(const struct stack_depot_trie_children *children, + unsigned long frame, unsigned int *pos) { unsigned int left = 0; unsigned int right; - right = READ_ONCE(array->nr_children); + if (!children) { + *pos = 0; + return false; + } + + right = READ_ONCE(children->nr_children); while (left < right) { unsigned int mid = left + (right - left) / 2; const struct stack_depot_trie_node *node; unsigned long mid_frame; - node = trie_child_array_load_child(array, mid); - if (!node) { - /* Tail append may produce a transient lockless lookup miss. */ - right = mid; - continue; - } + node = trie_children_load_child(children, mid); stack_depot_trie_node_frame(node, 0, &mid_frame); if (mid_frame < frame) { left = mid + 1; @@ -1865,454 +1760,551 @@ trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, return false; } -static void -trie_child_array_insert_at(const struct stack_depot_trie_child_array *old, - unsigned int pos, - const struct stack_depot_trie_node *node, - struct stack_depot_trie_child_array *new_array, - unsigned int new_capacity) +/** + * trie_children_init() - Initialize new children from an old container + * @old: Immutable source children container, or %NULL if empty + * @new: Unpublished destination children container + * + * Initialize @new by copying child pointers from @old without modifying + * the published source container. Unused entries in @new are + * initialized to %NULL. The allocator must already have set the + * capacity of @new. + */ +static void trie_children_init(const struct stack_depot_trie_children *old, + struct stack_depot_trie_children *new) { unsigned int nr_old; unsigned int i; - nr_old = old->nr_children; - - new_array->nr_children = nr_old + 1; - new_array->capacity = new_capacity; - for (i = 0; i < pos; i++) - RCU_INIT_POINTER(new_array->children[i], - trie_child_array_load_child(old, i)); - RCU_INIT_POINTER(new_array->children[pos], node); - for (i = pos; i < nr_old; i++) - RCU_INIT_POINTER(new_array->children[i + 1], - trie_child_array_load_child(old, i)); - for (i = nr_old + 1; i < new_array->capacity; i++) - RCU_INIT_POINTER(new_array->children[i], NULL); + nr_old = old ? old->nr_children : 0; + new->nr_children = nr_old; + for (i = 0; i < nr_old; i++) + RCU_INIT_POINTER(new->nodes[i], trie_children_load_child(old, i)); + for (i = nr_old; i < new->capacity; i++) + RCU_INIT_POINTER(new->nodes[i], NULL); } -static void -trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array, - const struct stack_depot_trie_node *new_child, - struct stack_depot_trie_child_array *new_array, - unsigned int pos) +/** + * trie_children_insert() - Insert a node into unpublished children + * @children: Unpublished children container to update + * @node: Node to insert + * @pos: Position at which to insert @node + * + * @children must not be reachable from the trie. Shift existing child + * pointers at @pos and later one position to the right, then store + * @node at @pos. + */ +static void trie_children_insert(struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *node, + unsigned int pos) { unsigned int i; - new_array->nr_children = old_array->nr_children; - new_array->capacity = old_array->capacity; - for (i = 0; i < old_array->nr_children; i++) - RCU_INIT_POINTER(new_array->children[i], - trie_child_array_load_child(old_array, i)); - RCU_INIT_POINTER(new_array->children[pos], new_child); - for (i = old_array->nr_children; i < new_array->capacity; i++) - RCU_INIT_POINTER(new_array->children[i], NULL); + for (i = children->nr_children; i > pos; i--) + RCU_INIT_POINTER(children->nodes[i], + trie_children_load_child(children, i - 1)); + RCU_INIT_POINTER(children->nodes[pos], node); + children->nr_children++; } +/** + * trie_reparent_children() - Reparent children before node retirement + * @parent: Replacement node that references the reused children + * + * Use this when replacing a node while retaining its children and + * descendant subtrees. The direct children still point back to the node + * being replaced, so update their parent links before publishing the + * replacement topology and queuing the old node for RCU retirement. + * + * Lockless fetches may observe the new parent before the replacement is + * published. This is safe because the old and new parent chains contain + * the same frames and both remain RCU-live during the transition. + */ static void trie_reparent_children(struct stack_depot_trie_node *parent) { - const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_children *children; unsigned int i; - children = trie_load_children_slot(&parent->children); + lockdep_assert_held(&stack_depot_trie_writer_lock); + + children = trie_load_children(&parent->children); if (!children) return; - /* - * COW updates reuse unchanged descendant subtrees. Repoint their parent - * links before retiring the old parent so fetch never follows a freed node. - * Lockless fetches may see the new parent before structural publication, but - * the old and new parent chains contain the same frames and remain RCU-live. - */ + for (i = 0; i < children->nr_children; i++) { struct stack_depot_trie_node *child; - child = (struct stack_depot_trie_node *)trie_child_array_load_child(children, i); + child = (struct stack_depot_trie_node *)trie_children_load_child(children, i); rcu_assign_pointer(child->parent, parent); } } -static void -trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_node * const *nodes, - struct stack_depot_trie_child_array * const *chain_arrays, - const struct stack_depot_trie_node **head, - const struct stack_depot_trie_node **tail) -{ +/** + * trie_path_alloc() - Allocate and initialize a linear trie path + * @parent: Parent of the first path node, or %NULL at the trie root + * @stack_id: Stack ID to assign to the final path node + * @entries: Frames to store in the path + * @nr_entries: Number of frames in @entries + * @pool_prealloc: Preallocated pool storage available to the allocator + * @node_out: Returns the final node on success + * + * Split @entries into frame runs, allocate one node per run, and link + * adjacent nodes through path children. Initialize the final node with + * @stack_id. + * The caller must hold pool_lock. A partially + * allocated path is released before returning an error. + * + * Return: The path root on success, or %NULL on allocation failure. + */ +static const struct stack_depot_trie_node * +trie_path_alloc(const struct stack_depot_trie_node *parent, u32 stack_id, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + const struct stack_depot_trie_node **node_out) +{ + struct stack_depot_trie_children *path_children = NULL; + const struct stack_depot_trie_node *path_root = NULL; + const struct stack_depot_trie_node *last_node = NULL; const struct stack_depot_trie_node *prev = parent; - unsigned int pos = 0; - unsigned int used = 0; - unsigned int i; + unsigned int entry = 0; + + lockdep_assert_held(&pool_lock); + *node_out = NULL; - while (pos < nr_entries) { + while (entry < nr_entries) { struct stack_depot_frame_run run; struct stack_depot_trie_node *node; - u32 id; - - node = nodes[used]; - frame_run_init(&entries[pos], nr_entries - pos, &run); - - id = pos + run.nr_entries == nr_entries ? leaf_id : 0; - trie_node_init(node, prev, id, &entries[pos], &run); - + unsigned int start = entry; + u32 node_stack_id = 0; + + frame_run_init(&entries[start], nr_entries - start, &run); + node = trie_pool_alloc(trie_node_bytes(&run), pool_prealloc); + if (!node) + goto err_release; + + entry += run.nr_entries; + if (entry == nr_entries) + node_stack_id = stack_id; + + trie_node_init(node, prev, node_stack_id, &entries[start], &run); + last_node = node; + if (!path_root) + path_root = node; + + if (path_children) { + trie_children_insert(path_children, last_node, 0); + path_children = NULL; + } + if (entry < nr_entries) { + path_children = trie_pool_alloc_children(1, pool_prealloc); + if (!path_children) + goto err_release; + RCU_INIT_POINTER(node->children, path_children); + } prev = node; - pos += run.nr_entries; - used++; } - for (i = 0; i + 1 < used; i++) { - const struct stack_depot_trie_node *next = nodes[i + 1]; - struct stack_depot_trie_node *node = nodes[i]; - struct stack_depot_trie_child_array *array = chain_arrays[i]; + *node_out = last_node; + return path_root; - trie_child_array_init(array, 1, &next, 1); - RCU_INIT_POINTER(node->children, array); - } +err_release: + while (last_node && last_node != parent) { + const struct stack_depot_trie_children *node_children; + const struct stack_depot_trie_node *node = last_node; - *head = nodes[0]; - *tail = nodes[used - 1]; -} - -static void trie_publish_tail_append(struct stack_depot_trie_child_array *array, - unsigned int pos, - const struct stack_depot_trie_node *head) -{ - /* - * Writers hold stack_depot_trie_writer_lock. Existing children are - * immutable, so tail append publishes the new child before increasing the - * visible count. Lockless readers that see the old count miss; readers that - * see the new count either load the initialized child or treat a transient - * NULL as a miss. The writer-lock recheck prevents permanent duplicates. - */ - rcu_assign_pointer(array->children[pos], head); - WRITE_ONCE(array->nr_children, pos + 1); + last_node = trie_load_parent(node); + node_children = trie_load_children(&node->children); + if (node_children) + trie_pool_release_children(node_children); + trie_pool_release(node, trie_node_bytes(&node->run)); + } + return NULL; } static const struct stack_depot_trie_node * stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) { - const struct stack_depot_trie_child_array *children; - unsigned int pos = 0; + const struct stack_depot_trie_children *children; + unsigned int entry = 0; - children = trie_load_children_slot(&stack_depot_trie_root); + children = trie_load_children(&stack_depot_trie_root); - while (pos < nr_entries) { + while (entry < nr_entries) { const struct stack_depot_trie_node *node; - unsigned int remaining = nr_entries - pos; + unsigned int remaining = nr_entries - entry; unsigned int matched; - unsigned int slot; + unsigned int pos; - if (!children) - return NULL; - if (!trie_child_array_find_slot(children, entries[pos], &slot)) + if (!trie_children_find_slot(children, entries[entry], &pos)) return NULL; - node = trie_child_array_load_child(children, slot); - matched = __stack_depot_trie_node_match(node, &entries[pos], remaining); + node = trie_children_load_child(children, pos); + matched = trie_node_match(node, &entries[entry], remaining); if (matched < node->run.nr_entries) return NULL; - pos += matched; - if (pos == nr_entries) - return node->leaf_id ? node : NULL; + entry += matched; + if (entry == nr_entries) + return node->stack_id ? node : NULL; - children = trie_load_children_slot(&node->children); + children = trie_load_children(&node->children); } return NULL; } -static void trie_build_split(const struct stack_depot_trie_node *child, - unsigned int matched, u32 leaf_id, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_node *prefix, - struct stack_depot_trie_node *old_tail, - struct stack_depot_trie_node * const *new_nodes, - struct stack_depot_trie_child_array * const *chain_arrays, - struct stack_depot_trie_child_array *prefix_children, - const struct stack_depot_trie_node **new_leaf) -{ - const struct stack_depot_trie_child_array *child_children; - const struct stack_depot_trie_node *child_parent; - const unsigned long *tail_entries; - const struct stack_depot_trie_node *split_children[2]; - const struct stack_depot_trie_node *new_head = NULL; - const struct stack_depot_trie_node *new_tail = NULL; - unsigned long new_frame; - unsigned long old_frame; - u32 prefix_leaf_id; - unsigned int tail_len; - bool has_new_tail; - - child_children = trie_load_children_slot(&child->children); - child_parent = trie_load_parent(child); - - has_new_tail = matched < nr_entries; - prefix_leaf_id = has_new_tail ? 0 : leaf_id; - trie_node_init_slice(prefix, child_parent, prefix_leaf_id, child, 0, - matched); - tail_len = child->run.nr_entries - matched; - trie_node_init_slice(old_tail, prefix, child->leaf_id, child, matched, - tail_len); - - if (has_new_tail) { - tail_entries = &entries[matched]; - tail_len = nr_entries - matched; - trie_build_append_chain(prefix, leaf_id, tail_entries, tail_len, - new_nodes, chain_arrays, - &new_head, &new_tail); - stack_depot_trie_node_frame(old_tail, 0, &old_frame); - stack_depot_trie_node_frame(new_head, 0, &new_frame); - if (old_frame < new_frame) { - split_children[0] = old_tail; - split_children[1] = new_head; +static u32 trie_insert_path(const struct stack_depot_trie_children __rcu **slot, + struct stack_depot_trie_node *parent, + const struct stack_depot_trie_children *children, + unsigned int pos, const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) +{ + struct stack_depot_trie_children *new_children; + const struct stack_depot_trie_node *path_root; + const struct stack_depot_trie_node *node; + unsigned int cap = 1; + unsigned long flags; + u32 new_stack_id; + + new_stack_id = trie_side_table_prepare_stack_slot(side_prealloc); + if (!new_stack_id) + return 0; + + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + trie_drain_pending_children(); + + if (children) + cap = roundup_pow_of_two(children->nr_children + 1); + + /* + * Published children are immutable, so adding a path requires a new + * children container. trie_children_init() copies existing child + * pointers, if any, and trie_children_insert() attaches the path root. + */ + new_children = trie_pool_alloc_children(cap, pool_prealloc); + if (!new_children) + goto err_release; + + path_root = trie_path_alloc(parent, new_stack_id, entries, nr_entries, + pool_prealloc, &node); + if (!path_root) + goto err_release; + + trie_side_table_publish(node, new_stack_id); + + trie_children_init(children, new_children); + trie_children_insert(new_children, path_root, pos); + + /* + * Publish the completed children container. Readers can now reach the + * new path, whose final node is already visible in the side table. + */ + rcu_assign_pointer(*slot, new_children); + + if (children) + trie_retire_children(children); + + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return new_stack_id; + +err_release: + if (new_children) + trie_pool_release_children(new_children); + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return 0; +} + +static u32 trie_split_child(const struct stack_depot_trie_children __rcu **slot, + const struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *child, + unsigned int pos, unsigned int matched, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) +{ + struct stack_depot_trie_children *prefix_children = NULL; + struct stack_depot_trie_children *new_children = NULL; + const struct stack_depot_trie_node *new_node; + const struct stack_depot_trie_node *suffix_roots[2]; + struct stack_depot_trie_node *split_prefix = NULL; + struct stack_depot_trie_node *old_suffix = NULL; + unsigned int i, nr_suffix_roots; + unsigned int old_suffix_len; + unsigned long flags; + size_t split_prefix_size; + size_t old_suffix_size; + u32 new_stack_id, prefix_stack_id; + + new_stack_id = trie_side_table_prepare_stack_slot(side_prealloc); + if (!new_stack_id) + return 0; + + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + trie_drain_pending_children(); + + split_prefix_size = trie_node_bytes_for(child->run.mode, matched); + split_prefix = trie_pool_alloc(split_prefix_size, pool_prealloc); + if (!split_prefix) + goto err_release; + + old_suffix_len = child->run.nr_entries - matched; + old_suffix_size = trie_node_bytes_for(child->run.mode, old_suffix_len); + old_suffix = trie_pool_alloc(old_suffix_size, pool_prealloc); + if (!old_suffix) + goto err_release; + + new_children = trie_pool_alloc_children(children->capacity, pool_prealloc); + if (!new_children) + goto err_release; + + if (matched < nr_entries) { + /* + * The new and existing stacks diverge inside this child. Keep the + * existing suffix as old_suffix and build a new suffix for the new + * stack. + */ + const struct stack_depot_trie_node *new_suffix; + unsigned int new_suffix_len = nr_entries - matched; + unsigned long frame; + + prefix_stack_id = 0; + nr_suffix_roots = 2; + prefix_children = trie_pool_alloc_children(nr_suffix_roots, + pool_prealloc); + if (!prefix_children) + goto err_release; + + new_suffix = trie_path_alloc(split_prefix, new_stack_id, + &entries[matched], new_suffix_len, + pool_prealloc, &new_node); + if (!new_suffix) + goto err_release; + + stack_depot_trie_node_frame(child, matched, &frame); + if (frame < entries[matched]) { + suffix_roots[0] = old_suffix; + suffix_roots[1] = new_suffix; } else { - split_children[0] = new_head; - split_children[1] = old_tail; + suffix_roots[0] = new_suffix; + suffix_roots[1] = old_suffix; } - trie_child_array_init(prefix_children, ARRAY_SIZE(split_children), - split_children, ARRAY_SIZE(split_children)); } else { - split_children[0] = old_tail; - trie_child_array_init(prefix_children, 1, split_children, 1); + /* The new stack ends at the shared prefix. */ + prefix_stack_id = new_stack_id; + nr_suffix_roots = 1; + new_node = split_prefix; + suffix_roots[0] = old_suffix; + prefix_children = trie_pool_alloc_children(nr_suffix_roots, + pool_prealloc); + if (!prefix_children) + goto err_release; } - RCU_INIT_POINTER(old_tail->children, child_children); - RCU_INIT_POINTER(prefix->children, prefix_children); - *new_leaf = has_new_tail ? new_tail : prefix; + + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + + /* Keep the shared prefix under the old parent. */ + trie_node_init_slice(split_prefix, trie_load_parent(child), prefix_stack_id, + child, 0, matched); + trie_node_init_slice(old_suffix, split_prefix, child->stack_id, child, matched, + old_suffix_len); + for (i = 0; i < nr_suffix_roots; i++) + trie_children_insert(prefix_children, suffix_roots[i], i); + + /* Keep the old children under the old suffix. */ + RCU_INIT_POINTER(old_suffix->children, trie_load_children(&child->children)); + RCU_INIT_POINTER(split_prefix->children, prefix_children); + + if (child->stack_id) + trie_side_table_publish(old_suffix, child->stack_id); + trie_side_table_publish(new_node, new_stack_id); + + /* + * Copy the current children into their replacement container, then + * store split_prefix at the position previously occupied by child. + */ + trie_children_init(children, new_children); + RCU_INIT_POINTER(new_children->nodes[pos], split_prefix); + + trie_reparent_children(old_suffix); + rcu_assign_pointer(*slot, new_children); + trie_retire_children_with_node(children, child); + + return new_stack_id; + +err_release: + if (split_prefix) + trie_pool_release(split_prefix, split_prefix_size); + if (old_suffix) + trie_pool_release(old_suffix, old_suffix_size); + if (prefix_children) + trie_pool_release_children(prefix_children); + if (new_children) + trie_pool_release_children(new_children); + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return 0; } -static unsigned int trie_size_append_chain(const unsigned long *entries, - unsigned int nr_entries, - size_t *node_sizes) +static u32 trie_promote_child(const struct stack_depot_trie_children __rcu **slot, + const struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *child, + unsigned int pos, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { - unsigned int pos = 0; - unsigned int nr_nodes = 0; + struct stack_depot_trie_children *new_children; + struct stack_depot_trie_node *promoted_node; + unsigned long flags; + size_t node_size; + u32 new_stack_id; - while (pos < nr_entries) { - struct stack_depot_frame_run run; + new_stack_id = trie_side_table_prepare_stack_slot(side_prealloc); + if (!new_stack_id) + return 0; - frame_run_init(&entries[pos], nr_entries - pos, &run); - node_sizes[nr_nodes] = trie_node_bytes(&run); - pos += run.nr_entries; - nr_nodes++; - } + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + trie_drain_pending_children(); + + node_size = trie_node_bytes(&child->run); + promoted_node = trie_pool_alloc(node_size, pool_prealloc); + if (!promoted_node) + goto out_unlock; + + new_children = trie_pool_alloc_children(children->capacity, pool_prealloc); + if (!new_children) + goto out_release_node; + + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); - return nr_nodes; + memcpy(promoted_node, child, node_size); + promoted_node->stack_id = new_stack_id; + trie_side_table_publish(promoted_node, new_stack_id); + + trie_children_init(children, new_children); + RCU_INIT_POINTER(new_children->nodes[pos], promoted_node); + + trie_reparent_children(promoted_node); + rcu_assign_pointer(*slot, new_children); + trie_retire_children_with_node(children, child); + + return new_stack_id; + +out_release_node: + trie_pool_release(promoted_node, node_size); +out_unlock: + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return 0; } -static u32 -stack_depot_trie_insert_locked(const unsigned long *entries, - unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) +static u32 stack_depot_trie_insert(const unsigned long *entries, + unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { - struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_child_array *slot_array; - struct stack_depot_trie_child_array *prefix_children; - const struct stack_depot_trie_child_array *children; - const struct stack_depot_trie_child_array __rcu **slot = + const struct stack_depot_trie_children *children; + const struct stack_depot_trie_children __rcu **slot = &stack_depot_trie_root; const struct stack_depot_trie_node *child; - const struct stack_depot_trie_node *chain_head; - const struct stack_depot_trie_node *chain_leaf; - const struct stack_depot_trie_node *split_leaf; - struct stack_depot_trie_node *split_prefix; - struct stack_depot_trie_node *old_tail; - struct stack_depot_trie_node *promoted_node; struct stack_depot_trie_node *parent = NULL; unsigned int matched; unsigned int pos; - unsigned long flags; - u32 new_leaf_id; - size_t slot_array_size; + u32 stack_id = 0; for (;;) { - children = trie_load_children_slot(slot); - if (!children) { - unsigned int nr_nodes; - - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - slot_array_size = trie_child_array_bytes(1); - nr_nodes = trie_size_append_chain(entries, nr_entries, - alloc->node_sizes); - if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, - nr_nodes - 1, 0, - slot_array_size)) - return 0; - slot_array = alloc->slot_array; - trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - alloc->nodes, alloc->chain_arrays, - &chain_head, &chain_leaf); - trie_side_table_publish_new_leaf(new_leaf_id, chain_leaf); - slot_array->nr_children = 1; - slot_array->capacity = 1; - RCU_INIT_POINTER(slot_array->children[0], chain_head); - rcu_assign_pointer(*slot, slot_array); - goto out_success; - } - if (!trie_child_array_find_slot(children, entries[0], &pos)) { - struct stack_depot_trie_child_array *tail_array; - unsigned int capacity; - unsigned int nr_nodes; - bool tail_append; - - capacity = roundup_pow_of_two(children->nr_children + 1); - tail_append = pos == children->nr_children && - children->nr_children < children->capacity; - slot_array_size = tail_append ? 0 : - trie_child_array_bytes(capacity); - if (slot_array_size > DEPOT_POOL_SIZE) - return 0; - - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - nr_nodes = trie_size_append_chain(entries, nr_entries, - alloc->node_sizes); - if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, - nr_nodes - 1, 0, - slot_array_size)) - return 0; - trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - alloc->nodes, alloc->chain_arrays, - &chain_head, &chain_leaf); - trie_side_table_publish_new_leaf(new_leaf_id, chain_leaf); - if (tail_append) { - tail_array = (struct stack_depot_trie_child_array *)children; - trie_publish_tail_append(tail_array, pos, chain_head); - } else { - slot_array = alloc->slot_array; - trie_child_array_insert_at(children, pos, chain_head, - slot_array, capacity); - rcu_assign_pointer(*slot, slot_array); - raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_child_array_locked(children); - raw_spin_unlock_irqrestore(&pool_lock, flags); - } - goto out_success; + children = trie_load_children(slot); + /* Case 1: no matching child, so add the remaining path. */ + if (!trie_children_find_slot(children, entries[0], &pos)) { + stack_id = trie_insert_path(slot, parent, children, pos, + entries, nr_entries, + pool_prealloc, + side_prealloc); + break; } - child = trie_child_array_load_child(children, pos); - matched = __stack_depot_trie_node_match(child, entries, nr_entries); + child = trie_children_load_child(children, pos); + matched = trie_node_match(child, entries, nr_entries); + + /* Case 2: the match ends inside the child run, so split it. */ if (matched < child->run.nr_entries) { - struct stack_depot_frame_run run; - unsigned int new_nodes = 0; - bool has_new_tail; - - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - - run = child->run; - run.nr_entries = matched; - alloc->node_sizes[0] = trie_node_bytes(&run); - run.nr_entries = child->run.nr_entries - matched; - alloc->node_sizes[1] = trie_node_bytes(&run); - has_new_tail = matched < nr_entries; - if (has_new_tail) - new_nodes = trie_size_append_chain(&entries[matched], - nr_entries - matched, - &alloc->node_sizes[2]); - if (trie_pool_alloc_insert(alloc, pool_prealloc, - 2 + new_nodes, - new_nodes ? new_nodes - 1 : 0, - trie_child_array_bytes(has_new_tail ? 2 : 1), - trie_child_array_bytes(children->capacity))) - return 0; - slot_array = alloc->slot_array; - prefix_children = alloc->prefix_children; - split_prefix = alloc->nodes[0]; - old_tail = alloc->nodes[1]; - trie_build_split(child, matched, new_leaf_id, entries, nr_entries, - split_prefix, old_tail, &alloc->nodes[2], - alloc->chain_arrays, prefix_children, &split_leaf); - trie_side_table_publish_split_leaves(child->leaf_id, old_tail, - new_leaf_id, split_leaf); - trie_child_array_replace_at(children, split_prefix, slot_array, pos); - trie_reparent_children(old_tail); - rcu_assign_pointer(*slot, slot_array); - trie_retire_child_array_with_node(children, child); - goto out_success; + stack_id = trie_split_child(slot, children, child, pos, + matched, entries, nr_entries, + pool_prealloc, side_prealloc); + break; } + + /* Case 3: the input ends here, so reuse or promote this child. */ if (matched == nr_entries) { - if (child->leaf_id) - return child->leaf_id; - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - alloc->node_sizes[0] = trie_node_bytes(&child->run); - slot_array_size = trie_child_array_bytes(children->capacity); - if (trie_pool_alloc_insert(alloc, pool_prealloc, 1, 0, 0, - slot_array_size)) - return 0; - promoted_node = alloc->nodes[0]; - slot_array = alloc->slot_array; - memcpy(promoted_node, child, alloc->node_sizes[0]); - promoted_node->leaf_id = new_leaf_id; - trie_side_table_publish_new_leaf(new_leaf_id, promoted_node); - trie_child_array_replace_at(children, promoted_node, slot_array, pos); - trie_reparent_children(promoted_node); - rcu_assign_pointer(*slot, slot_array); - trie_retire_child_array_with_node(children, child); - goto out_success; + if (child->stack_id) + return child->stack_id; + stack_id = trie_promote_child(slot, children, child, pos, + pool_prealloc, + side_prealloc); + break; } + /* Case 4: the child matched fully; descend with remaining frames. */ parent = (struct stack_depot_trie_node *)child; slot = &parent->children; entries += matched; nr_entries -= matched; } -out_success: - trie_side_table_last_leaf_id = new_leaf_id; - return new_leaf_id; + if (stack_id) + trie_side_table_last_stack_id = stack_id; + + return stack_id; } -static unsigned int -__stack_depot_trie_fetch_into(const struct stack_depot_trie_node *leaf, - unsigned long *entries, - unsigned int max_entries) +static unsigned int trie_fetch_into(const struct stack_depot_trie_node *node, + unsigned long *entries, + unsigned int max_entries) { - const struct stack_depot_trie_node *node; + const struct stack_depot_trie_node *cur; unsigned int total; unsigned int pos; unsigned int i; total = 0; - for (node = leaf; node; node = trie_load_parent(node)) - total += node->run.nr_entries; + for (cur = node; cur; cur = trie_load_parent(cur)) + total += cur->run.nr_entries; if (WARN_ON_ONCE(!total)) return 0; if (max_entries < total) return 0; pos = total; - for (node = leaf; node; node = trie_load_parent(node)) { - pos -= node->run.nr_entries; - for (i = 0; i < node->run.nr_entries; i++) - stack_depot_trie_node_frame(node, i, &entries[pos + i]); + for (cur = node; cur; cur = trie_load_parent(cur)) { + pos -= cur->run.nr_entries; + for (i = 0; i < cur->run.nr_entries; i++) + stack_depot_trie_node_frame(cur, i, &entries[pos + i]); } return total; } -static unsigned int -__stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, - unsigned long *entries, - unsigned int max_entries) +static unsigned int trie_fetch_handle_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries) { - const struct stack_depot_trie_node *leaf; - u32 leaf_id; + const struct stack_depot_trie_node *node; + u32 stack_id; unsigned int nr_entries; - leaf_id = __stack_depot_trie_leaf_id(handle); + stack_id = trie_stack_id(handle); rcu_read_lock_sched_notrace(); - leaf = __stack_depot_trie_side_table_lookup(leaf_id); - if (WARN_ONCE(!leaf, "corrupt trie handle %08x\n", handle)) { + node = trie_side_table_lookup(stack_id); + if (WARN_ONCE(!node, "corrupt trie handle %08x\n", handle)) { rcu_read_unlock_sched_notrace(); return 0; } - nr_entries = __stack_depot_trie_fetch_into(leaf, entries, max_entries); + nr_entries = trie_fetch_into(node, entries, max_entries); rcu_read_unlock_sched_notrace(); if (nr_entries) kmsan_unpoison_memory(entries, nr_entries * sizeof(*entries)); @@ -2363,7 +2355,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, return 0; WARN_ON_ONCE(!entries || !max_entries); if (stack_depot_handle_is_trie(handle)) - return __stack_depot_trie_fetch_handle_into(handle, entries, + return trie_fetch_handle_into(handle, entries, max_entries); stack = depot_fetch_stack(handle); @@ -2413,9 +2405,9 @@ void stack_depot_print(depot_stack_handle_t stack) if (stack_depot_handle_is_trie(stack)) { unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; - nr_entries = __stack_depot_trie_fetch_handle_into(stack, - trie_entries, - ARRAY_SIZE(trie_entries)); + nr_entries = trie_fetch_handle_into(stack, + trie_entries, + ARRAY_SIZE(trie_entries)); stack_trace_print(trie_entries, nr_entries, 0); return; } @@ -2435,9 +2427,9 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, if (stack_depot_handle_is_trie(handle)) { unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; - nr_entries = __stack_depot_trie_fetch_handle_into(handle, - trie_entries, - ARRAY_SIZE(trie_entries)); + nr_entries = trie_fetch_handle_into(handle, + trie_entries, + ARRAY_SIZE(trie_entries)); return stack_trace_snprint(buf, size, trie_entries, nr_entries, spaces); } diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 1623a644f6e62..4fba72e283312 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -336,7 +336,7 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) KUNIT_EXPECT_FALSE(test, compressed); } -#ifdef CONFIG_X86_64 +#if defined(CONFIG_X86_64) && !defined(CONFIG_UML) static void stackdepot_frame_x86_64(struct kunit *test) { unsigned long direct_map = 0xffff888000001000UL; @@ -354,7 +354,7 @@ static void stackdepot_frame_x86_64(struct kunit *test) compressed = arch_stack_depot_frame_try_compress(direct_map, &low); KUNIT_EXPECT_FALSE(test, compressed); } -#endif /* CONFIG_X86_64 */ +#endif /* CONFIG_X86_64 && !CONFIG_UML */ #ifdef CONFIG_ARM64 static void stackdepot_frame_arm64(struct kunit *test) @@ -398,7 +398,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_topology_roundtrip), KUNIT_CASE(stackdepot_frame_storage_roundtrip), KUNIT_CASE(stackdepot_frame_raw_fallback), -#ifdef CONFIG_X86_64 +#if defined(CONFIG_X86_64) && !defined(CONFIG_UML) KUNIT_CASE(stackdepot_frame_x86_64), #endif #ifdef CONFIG_ARM64