From 2c0b0b5c5ed87a040e618f09c3939639fe08402e Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 8 Jul 2026 12:27:06 +0100 Subject: [PATCH 01/38] KRN-1117: Add stackdepot trie storage Add an opt-in path-compressed trie backend for persistent stackdepot saves so related stack traces can share frame runs instead of storing every saved stack as a flat hash record. Keep refcounted and countable users on the existing hash backend, add backend-neutral fetch helpers for persistent callers, and wire the trie through stackdepot pool-backed storage with RCU-safe publication. Signed-off-by: Caleb Kan --- arch/arm64/include/asm/stackdepot.h | 42 + arch/um/include/asm/Kbuild | 1 + arch/x86/include/asm/stackdepot.h | 37 + drivers/gpu/drm/drm_modeset_lock.c | 10 +- include/asm-generic/Kbuild | 1 + include/asm-generic/stackdepot.h | 19 + include/linux/stackdepot.h | 79 +- lib/Kconfig.debug | 16 + lib/stackdepot.c | 2241 ++++++++++++++++++++++++++- lib/tests/Makefile | 1 + lib/tests/stackdepot_kunit.c | 385 +++++ mm/kmemleak.c | 4 +- mm/kmsan/kmsan_test.c | 7 +- mm/kmsan/report.c | 21 +- mm/page_owner.c | 6 +- mm/slub.c | 12 +- 16 files changed, 2805 insertions(+), 77 deletions(-) create mode 100644 arch/arm64/include/asm/stackdepot.h create mode 100644 arch/x86/include/asm/stackdepot.h create mode 100644 include/asm-generic/stackdepot.h create mode 100644 lib/tests/stackdepot_kunit.c diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h new file mode 100644 index 0000000000000..df8959d593366 --- /dev/null +++ b/arch/arm64/include/asm/stackdepot.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_STACKDEPOT_H +#define __ASM_STACKDEPOT_H + +#include +#include + +/* + * Modules are allocated inside a 2 GB relocation window containing the + * kernel image. Store a signed 32-bit offset from _text so compression is + * independent of 4 GB high-bit boundaries crossed by that window. + */ +static inline unsigned long arch_stack_depot_frame_from_payload(u32 payload) +{ + long offset; + + offset = (s32)payload; + if (offset < 0) + return (unsigned long)_text - (unsigned long)(-offset); + return (unsigned long)_text + (unsigned long)offset; +} + +static inline bool +arch_stack_depot_frame_try_compress(unsigned long frame, u32 *payload) +{ + u32 candidate; + + candidate = (u32)(frame - (unsigned long)_text); + if (arch_stack_depot_frame_from_payload(candidate) != frame) + return false; + + *payload = candidate; + return true; +} + +static inline void +arch_stack_depot_frame_decompress(u32 payload, unsigned long *frame) +{ + *frame = arch_stack_depot_frame_from_payload(payload); +} + +#endif /* __ASM_STACKDEPOT_H */ diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index 9be3ee2e37013..731e6832a3829 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -21,6 +21,7 @@ generic-y += preempt.h generic-y += ring_buffer.h generic-y += runtime-const.h generic-y += softirq_stack.h +generic-y += stackdepot.h generic-y += switch_to.h generic-y += topology.h generic-y += trace_clock.h diff --git a/arch/x86/include/asm/stackdepot.h b/arch/x86/include/asm/stackdepot.h new file mode 100644 index 0000000000000..9a8d04fa8c1c8 --- /dev/null +++ b/arch/x86/include/asm/stackdepot.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_STACKDEPOT_H +#define _ASM_X86_STACKDEPOT_H + +#include + +#ifdef CONFIG_X86_64 +/* + * Compress canonical kernel text/module addresses whose upper 32 bits are all + * ones. Other kernel virtual addresses stay raw, so decompression reconstructs + * the original frame by restoring this prefix. + */ +#define STACK_DEPOT_X86_64_FRAME_PREFIX 0xffffffff00000000UL +#define STACK_DEPOT_X86_64_FRAME_LOW_MASK 0x00000000ffffffffUL + +static inline bool +arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) +{ + if ((frame & ~STACK_DEPOT_X86_64_FRAME_LOW_MASK) != + STACK_DEPOT_X86_64_FRAME_PREFIX) + return false; + + *low = (u32)frame; + return true; +} + +static inline void +arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) +{ + *frame = STACK_DEPOT_X86_64_FRAME_PREFIX | low; +} + +#else +#include +#endif /* CONFIG_X86_64 */ + +#endif /* _ASM_X86_STACKDEPOT_H */ diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c index beb91a13a3124..e43202df77e30 100644 --- a/drivers/gpu/drm/drm_modeset_lock.c +++ b/drivers/gpu/drm/drm_modeset_lock.c @@ -81,9 +81,12 @@ static DEFINE_WW_CLASS(crtc_ww_class); #if IS_ENABLED(CONFIG_DRM_DEBUG_MODESET_LOCK) +/* Modeset-lock diagnostics only need a short caller chain. */ +#define DRM_STACK_DEPOT_MAX_FRAMES 8 + static noinline depot_stack_handle_t __drm_stack_depot_save(void) { - unsigned long entries[8]; + unsigned long entries[DRM_STACK_DEPOT_MAX_FRAMES]; unsigned int n; n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); @@ -94,16 +97,13 @@ static noinline depot_stack_handle_t __drm_stack_depot_save(void) static void __drm_stack_depot_print(depot_stack_handle_t stack_depot) { struct drm_printer p = drm_dbg_printer(NULL, DRM_UT_KMS, "drm_modeset_lock"); - unsigned long *entries; - unsigned int nr_entries; char *buf; buf = kmalloc(PAGE_SIZE, GFP_NOWAIT | __GFP_NOWARN); if (!buf) return; - nr_entries = stack_depot_fetch(stack_depot, &entries); - stack_trace_snprint(buf, PAGE_SIZE, entries, nr_entries, 2); + stack_depot_snprint(stack_depot, buf, PAGE_SIZE, 2); drm_printf(&p, "attempting to lock a contended lock without backoff:\n%s", buf); diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 295c94a3ccc1c..a126f8e237bab 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -53,6 +53,7 @@ mandatory-y += serial.h mandatory-y += shmparam.h mandatory-y += simd.h mandatory-y += softirq_stack.h +mandatory-y += stackdepot.h mandatory-y += switch_to.h mandatory-y += timex.h mandatory-y += tlbflush.h diff --git a/include/asm-generic/stackdepot.h b/include/asm-generic/stackdepot.h new file mode 100644 index 0000000000000..846975767bdd4 --- /dev/null +++ b/include/asm-generic/stackdepot.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_STACKDEPOT_H +#define __ASM_GENERIC_STACKDEPOT_H + +#include + +static inline bool +arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) +{ + return false; +} + +static inline void +arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) +{ + /* Generic code never compresses frames, so this hook is unreachable. */ +} + +#endif /* __ASM_GENERIC_STACKDEPOT_H */ diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 2cc21ffcdaf9e..700e0e7faa44c 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -53,7 +53,8 @@ union handle_parts { struct stack_record { struct list_head hash_list; /* Links in the hash table */ u32 hash; /* Hash in hash table */ - u32 size; /* Number of stored frames */ + u16 size; /* Number of stored frames */ + u16 flags; union handle_parts handle; /* Constant after initialization */ refcount_t count; union { @@ -84,8 +85,9 @@ typedef u32 depot_flags_t; */ #define STACK_DEPOT_FLAG_CAN_ALLOC ((depot_flags_t)0x0001) #define STACK_DEPOT_FLAG_GET ((depot_flags_t)0x0002) +#define STACK_DEPOT_FLAG_COUNTABLE ((depot_flags_t)0x0004) -#define STACK_DEPOT_FLAGS_NUM 2 +#define STACK_DEPOT_FLAGS_NUM 3 #define STACK_DEPOT_FLAGS_MASK ((depot_flags_t)((1 << STACK_DEPOT_FLAGS_NUM) - 1)) /* @@ -144,6 +146,16 @@ static inline int stack_depot_early_init(void) { return 0; } * Users of this flag must also call stack_depot_put() when keeping the stack * trace is no longer required to avoid overflowing the refcount. * + * If STACK_DEPOT_FLAG_COUNTABLE is set in @depot_flags, stack depot stores the + * stack in hash-backed storage for callers that need direct stack_record count + * access. This flag does not imply %STACK_DEPOT_FLAG_CAN_ALLOC and is mutually + * exclusive with %STACK_DEPOT_FLAG_GET. + * + * When trie storage is enabled, persistent non-refcounted saves use trie + * storage. Constrained contexts remain best effort and can return 0 if a + * required trylock or preallocated backing storage is unavailable; trie + * failures do not fall back to hash storage. + * * If the provided stack trace comes from the interrupt context, only the part * up to the interrupt entry is saved. * @@ -169,6 +181,10 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, * Does not increment the refcount on the saved stack trace; see * stack_depot_save_flags() for more details. * + * When trie storage is enabled, this can return trie-backed handles. Use + * stack_depot_fetch_into(), stack_depot_print(), or stack_depot_snprint() for + * backend-independent access to the stack contents. + * * Context: Contexts where allocations via alloc_pages() are allowed; * see stack_depot_save_flags() for more details. * @@ -178,27 +194,66 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags); /** - * __stack_depot_get_stack_record - Get a pointer to a stack_record struct + * __stack_depot_get_stack_record - Get a hash-backed stack record * * @handle: Stack depot handle * - * This function is only for internal purposes. + * This function is only for internal purposes. @handle must have been saved + * with %STACK_DEPOT_FLAG_COUNTABLE. * - * Return: Returns a pointer to a stack_record struct + * Return: Returns a pointer to a stack_record struct. */ struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle); /** * stack_depot_fetch - Fetch a stack trace from stack depot * - * @handle: Stack depot handle returned from stack_depot_save() + * @handle: Hash-backed stack depot handle * @entries: Pointer to store the address of the stack trace * + * This helper returns a pointer to stackdepot-owned contiguous storage for + * legacy hash-backed handles. Callers that need backend-independent access to + * stack contents should use stack_depot_fetch_into(), stack_depot_print(), or + * stack_depot_snprint(). Passing a trie-backed handle is invalid and may WARN. + * * Return: Number of frames for the fetched stack */ unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries); +/** + * stack_depot_fetch_into - Fetch a stack trace into caller-owned storage + * + * @handle: Stack depot handle returned from stack_depot_save() + * @entries: Caller-owned buffer to copy the stack trace into + * @max_entries: Number of frames that fit in @entries + * + * Copies the stored frames into caller-owned @entries. If fewer frames are + * stored than @max_entries, only the stored frames are written and their count + * is returned. If more frames are stored than @max_entries, the copy is skipped + * entirely and 0 is returned. + * + * Callers should size @entries to match the save-side stack depth cap (for + * example, %CONFIG_STACKDEPOT_MAX_FRAMES or the local stack_trace_save() limit) + * when losing diagnostics on an undersized buffer would be surprising. + * + * A non-zero invalid or post-put @handle is treated like stack_depot_fetch(): it + * returns 0 and may WARN because such handles indicate a corrupt caller state. + * + * Callers must ensure @handle remains valid for the duration of this call. + * Persistent handles saved without %STACK_DEPOT_FLAG_GET require no extra + * reference; handles saved with %STACK_DEPOT_FLAG_GET require a held reference. + * Callers must not call stack_depot_put() on persistent handles. + * Racing this helper with stack_depot_put() on the same handle is invalid. + * + * Return: Number of frames copied, 0 if @entries is NULL, @max_entries is 0, + * @handle is 0 or invalid, stack depot is disabled, or @max_entries is less + * than the number of stored frames. + */ +unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries); + /** * stack_depot_print - Print a stack trace from stack depot * @@ -224,10 +279,14 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, * * @handle: Stack depot handle returned from stack_depot_save() * - * The stack trace is evicted from stack depot once all references to it have - * been dropped (once the number of stack_depot_evict() calls matches the - * number of stack_depot_save_flags() calls with STACK_DEPOT_FLAG_GET set for - * this stack trace). + * Drop a reference acquired by stack_depot_save_flags() with + * %STACK_DEPOT_FLAG_GET. Calling this for a handle saved without + * %STACK_DEPOT_FLAG_GET is invalid; persistent handles, including trie-backed + * handles, are owned by stack depot for the lifetime of the system. + * + * The stack trace is evicted once the number of stack_depot_put() calls matches + * the number of successful stack_depot_save_flags() calls with + * %STACK_DEPOT_FLAG_GET for this stack trace. */ void stack_depot_put(depot_stack_handle_t handle); diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 21cd68084e468..3a1e7c9e6c1bd 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2706,6 +2706,22 @@ config RESOURCE_KUNIT_TEST If unsure, say N. +config STACKDEPOT_KUNIT_TEST + bool "KUnit test for stack depot" if !KUNIT_ALL_TESTS + depends on KUNIT=y && STACKDEPOT + default KUNIT_ALL_TESTS + help + Enable this option to test stack depot API behavior at boot. + This test is built in because it exercises internal, non-exported + stack depot helpers, so KUNIT must also be built in. + + KUnit tests run during boot and output the results to the debug log + in TAP format (https://testanything.org/). Only useful for kernel + developers running the KUnit test harness, and not intended for + inclusion into a production build. + + If unsure, say N. + config SYSCTL_KUNIT_TEST tristate "KUnit test for sysctl" if !KUNIT_ALL_TESTS depends on KUNIT diff --git a/lib/stackdepot.c b/lib/stackdepot.c index de0b0025af2b9..4eb141383c73e 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2,9 +2,11 @@ /* * Stack depot - a stack trace storage that avoids duplication. * - * Internally, stack depot maintains a hash table of unique stacktraces. The - * stack traces themselves are stored contiguously one after another in a set - * of separate page allocations. + * Internally, stack depot has two storage backends. Refcounted entries and + * callers that request STACK_DEPOT_FLAG_COUNTABLE use the legacy hash table with + * contiguous stack records in stack pools. Persistent non-refcounted entries + * can use trie storage when enabled; trie nodes share common frame prefixes and + * are published through RCU/COW child arrays. * * Author: Alexander Potapenko * Copyright (C) 2016 Google, Inc. @@ -14,14 +16,21 @@ #define pr_fmt(fmt) "stackdepot: " fmt +#include +#include #include +#include #include #include +#include #include +#include #include #include #include +#include #include +#include #include #include #include @@ -36,9 +45,12 @@ #include #include +#include + /* * The pool_index is offset by 1 so the first record does not have a 0 handle. */ +/* Parsed before mm_core_init(); trie handle decoding assumes this is then fixed. */ static unsigned int stack_max_pools __read_mostly = MIN((1LL << DEPOT_POOL_INDEX_BITS) - 1, 8192); @@ -61,39 +73,1263 @@ static unsigned int stack_bucket_number_order; /* Hash mask for indexing the table. */ static unsigned int stack_hash_mask; -/* Array of memory regions that store stack records. */ -static void **stack_pools; -/* Newly allocated pool that is not yet added to stack_pools. */ -static void *new_pool; -/* Number of pools in stack_pools. */ -static int pools_num; -/* Offset to the unused space in the currently used pool. */ -static size_t pool_offset = DEPOT_POOL_SIZE; -/* Freelist of stack records within stack_pools. */ -static LIST_HEAD(free_stacks); -/* The lock must be held when performing pool or freelist modifications. */ -static DEFINE_RAW_SPINLOCK(pool_lock); +/* Array of memory regions that store stack records. */ +static void **stack_pools; +/* Newly allocated pool that is not yet added to stack_pools. */ +static void *new_pool; +/* Number of pools in stack_pools. */ +static int pools_num; +/* Offset to the unused space in the currently used pool. */ +static size_t pool_offset = DEPOT_POOL_SIZE; +/* Freelist of stack records within stack_pools. */ +static LIST_HEAD(free_stacks); +/* The lock must be held when performing pool or freelist modifications. */ +static DEFINE_RAW_SPINLOCK(pool_lock); + +/* Statistics counters for debugfs. */ +enum depot_counter_id { + DEPOT_COUNTER_REFD_ALLOCS, + DEPOT_COUNTER_REFD_FREES, + DEPOT_COUNTER_REFD_INUSE, + DEPOT_COUNTER_FREELIST_SIZE, + DEPOT_COUNTER_PERSIST_COUNT, + DEPOT_COUNTER_PERSIST_BYTES, + DEPOT_COUNTER_COUNT, +}; +static long counters[DEPOT_COUNTER_COUNT]; +static const char *const counter_names[] = { + [DEPOT_COUNTER_REFD_ALLOCS] = "refcounted_allocations", + [DEPOT_COUNTER_REFD_FREES] = "refcounted_frees", + [DEPOT_COUNTER_REFD_INUSE] = "refcounted_in_use", + [DEPOT_COUNTER_FREELIST_SIZE] = "freelist_size", + [DEPOT_COUNTER_PERSIST_COUNT] = "persistent_count", + [DEPOT_COUNTER_PERSIST_BYTES] = "persistent_bytes", +}; +static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); + +enum stack_depot_frame_mode { + STACK_DEPOT_FRAME_RAW, + STACK_DEPOT_FRAME_COMPRESSED, +}; + +/* + * A trie node stores one run of frames that all use the same payload format. + * Architectures may compress some frames to 32-bit payloads; mixed raw and + * compressed input is split across multiple trie nodes so each node has one + * decoding mode. + */ +struct stack_depot_frame_run { + u16 nr_entries; + u8 mode; +}; + +static_assert(CONFIG_STACKDEPOT_MAX_FRAMES <= U16_MAX); + +struct stack_depot_trie_child_array; + +struct stack_depot_trie_node { + /* Parent links let fetch rebuild a full stack from a leaf to the root. */ + const struct stack_depot_trie_node *parent; + /* Child arrays are separate RCU/COW generations. */ + const struct stack_depot_trie_child_array *children; + u32 leaf_id; + struct stack_depot_frame_run run; + unsigned char data[]; +}; + +/* + * Children are sorted by first frame and searched by insertion slot. + * Writers may append to spare capacity at the sorted tail, but never change + * existing child pointers. Other updates build and publish a replacement array. + */ +struct stack_depot_trie_child_array { + unsigned int nr_children; + unsigned int capacity; + const struct stack_depot_trie_node *children[]; +}; + +/* Headerless reusable storage for trie nodes. */ +struct stack_depot_trie_free_node { + struct list_head list; +}; + +/* + * Reusable object storage for child arrays and other payloads that need an + * object header. A retired child array can carry the old child node that was + * replaced with it; both become reusable after the array's RCU grace period. + */ +struct stack_depot_trie_free_object { + struct list_head list; + unsigned long rcu_state; + size_t size; + struct stack_depot_trie_node *pending_node; +}; + +static_assert(sizeof(struct stack_depot_trie_node) >= + sizeof(struct stack_depot_trie_free_node)); + +#define STACK_DEPOT_TRIE_MAX_NODES (CONFIG_STACKDEPOT_MAX_FRAMES + 1) +#define STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS CONFIG_STACKDEPOT_MAX_FRAMES + +struct stack_depot_trie_alloc_workspace { + struct stack_depot_trie_node *nodes[STACK_DEPOT_TRIE_MAX_NODES]; + size_t node_sizes[STACK_DEPOT_TRIE_MAX_NODES]; + struct stack_depot_trie_child_array *child_arrays[STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS]; + u32 scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; + struct stack_depot_trie_child_array *split_child_array; + struct stack_depot_trie_child_array *child_array_storage; +}; + +static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); +static const struct stack_depot_trie_child_array *stack_depot_trie_root; +static struct stack_depot_trie_alloc_workspace __rcu *stack_depot_trie_workspace; +static DEFINE_RAW_SPINLOCK(stack_depot_trie_writer_lock); +static bool stack_depot_trie_requested; + +module_param_named(trie_enabled, stack_depot_trie_requested, bool, 0); +MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); + +#define DEPOT_POOL_INDEX_MASK ((1U << DEPOT_POOL_INDEX_BITS) - 1) +#define DEPOT_OFFSET_MASK ((1U << DEPOT_OFFSET_BITS) - 1) + +/* Size classes bucket reusable trie storage by aligned allocation size. */ +#define STACK_DEPOT_TRIE_FREE_CLASSES \ + ((DEPOT_POOL_SIZE >> DEPOT_STACK_ALIGN) + 1) + +/* + * Trie storage is suballocated from stackdepot pools, not slab caches, so pool + * pressure stays visible through stack_depot_max_pools and no-spin callers can + * fail without allocator recursion. Trie COW insertion retires child arrays + * and sometimes the node they replaced. Objects carry the RCU cookie for + * child-array payloads; headerless node fragments either live directly on + * free_trie_nodes or are attached to a pending object until that object's grace + * period has elapsed. + */ +static struct list_head free_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; +static struct list_head pending_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; +static struct list_head free_trie_nodes[STACK_DEPOT_TRIE_FREE_CLASSES]; +static DECLARE_BITMAP(free_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); +static DECLARE_BITMAP(pending_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); +static DECLARE_BITMAP(free_trie_node_map, STACK_DEPOT_TRIE_FREE_CLASSES); + +static u32 trie_side_table_max_id; + +/* + * Hash handles use pool_index_plus_1 <= stack_max_pools. Trie handles use + * pool_index_plus_1 > stack_max_pools and reinterpret the remaining handle + * bits as a dense leaf_id, which the side table maps to a trie leaf. + */ +static u32 __stack_depot_trie_max_leaf_id(void) +{ + u64 max_id; + + if (stack_max_pools >= DEPOT_POOL_INDEX_MASK - 1) + return 0; + + max_id = (u64)(DEPOT_POOL_INDEX_MASK - stack_max_pools - 1) << + DEPOT_OFFSET_BITS; + return min_t(u64, max_id, U32_MAX); +} + +static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) +{ + union handle_parts parts = {}; + u64 pool_index_plus_1; + u32 pool_delta; + u32 index; + + if (!leaf_id || leaf_id > trie_side_table_max_id) + return 0; + + index = leaf_id - 1; + pool_delta = index >> DEPOT_OFFSET_BITS; + pool_index_plus_1 = (u64)stack_max_pools + 1 + pool_delta; + if (pool_index_plus_1 >= DEPOT_POOL_INDEX_MASK) + return 0; + + parts.pool_index_plus_1 = pool_index_plus_1; + parts.offset = index & DEPOT_OFFSET_MASK; + return parts.handle; +} + +static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) +{ + union handle_parts parts = { .handle = handle }; + u64 leaf_id; + u32 pool_delta; + + parts.extra = 0; + if (parts.pool_index_plus_1 <= stack_max_pools) + return 0; + + pool_delta = parts.pool_index_plus_1 - stack_max_pools - 1; + if ((u64)pool_delta + stack_max_pools + 1 >= DEPOT_POOL_INDEX_MASK) + return 0; + + leaf_id = ((u64)pool_delta << DEPOT_OFFSET_BITS) + parts.offset + 1; + if (leaf_id > trie_side_table_max_id) + return 0; + + return leaf_id; +} + +/* + * Trie handles encode a dense leaf ID. The side table maps that ID to a leaf + * pointer for lockless fetch/print paths, which can run from diagnostic + * contexts where taking trie_side_table_lock would be unsafe. Init installs the + * root and first chunk only; additional directories/chunks are preallocated and + * published lazily as leaf IDs grow. RCU pointer publication makes fully + * initialized dirs, chunks, and leaves visible to those lockless readers. + */ +#define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 +#define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ + (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) +#define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS 9 +#define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE \ + (1U << STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS) + +struct stack_depot_trie_side_dir { + const struct stack_depot_trie_node __rcu * __rcu * + chunks[STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE]; +}; + +struct stack_depot_trie_side_root { + unsigned int dir_capacity; + struct stack_depot_trie_side_dir __rcu *dirs[]; +}; + +struct stack_depot_trie_side_prealloc { + /* Preallocated side-table directory page for sparse growth. */ + struct stack_depot_trie_side_dir *dir; + /* Preallocated side-table leaf chunk for sparse growth. */ + const struct stack_depot_trie_node __rcu **chunk; +}; + +static struct stack_depot_trie_side_root __rcu *trie_side_table_root; +static DEFINE_RAW_SPINLOCK(trie_side_table_lock); +static u32 trie_side_table_last_leaf_id; + +/* Lock order: writer_lock -> pool_lock -> trie_side_table_lock. */ + +static inline bool __stack_depot_trie_enabled(void) +{ + return static_branch_unlikely(&stack_depot_trie_enabled); +} + +static void stack_depot_trie_enable(void) +{ + if (__stack_depot_trie_enabled()) + return; + + static_branch_enable(&stack_depot_trie_enabled); +} + +static inline size_t stack_depot_frame_run_entry_bytes(enum stack_depot_frame_mode mode) +{ + if (mode == STACK_DEPOT_FRAME_COMPRESSED) + return sizeof(u32); + return sizeof(unsigned long); +} + +static inline size_t stack_depot_frame_run_bytes(const struct stack_depot_frame_run *run) +{ + return run->nr_entries * stack_depot_frame_run_entry_bytes(run->mode); +} + +static inline size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) +{ + return ALIGN(offsetof(struct stack_depot_trie_node, data) + + stack_depot_frame_run_bytes(run), sizeof(unsigned long)); +} + +static inline unsigned int trie_child_array_capacity(unsigned int nr_children) +{ + return nr_children ? roundup_pow_of_two(nr_children) : 0; +} + +static size_t trie_child_array_size_for_capacity(unsigned int capacity) +{ + size_t size; + + size = struct_size_t(struct stack_depot_trie_child_array, children, + capacity); + if (size == SIZE_MAX) + return 0; + + return ALIGN(size, sizeof(unsigned long)); +} + +static inline size_t __stack_depot_trie_child_array_size(unsigned int nr_children) +{ + return trie_child_array_size_for_capacity(trie_child_array_capacity(nr_children)); +} + +static void trie_child_array_init(void *storage, unsigned int capacity, + const struct stack_depot_trie_node * const *nodes, + unsigned int nr_children) +{ + struct stack_depot_trie_child_array *array = storage; + unsigned int i; + + array->nr_children = nr_children; + array->capacity = capacity; + for (i = 0; i < nr_children; i++) + array->children[i] = nodes[i]; + for (i = nr_children; i < capacity; i++) + array->children[i] = NULL; +} + +static inline struct stack_depot_trie_alloc_workspace *stack_depot_trie_load_workspace(void) +{ + /* Installed once and never freed; acquire the init-time publication. */ + return rcu_dereference_check(stack_depot_trie_workspace, true); +} + +static inline struct stack_depot_trie_side_root *trie_side_table_load_root(void) +{ + /* Installed once and never freed; acquire the init-time publication. */ + return rcu_dereference_check(trie_side_table_root, true); +} + +static inline bool trie_side_table_is_initialized(void) +{ + return !!trie_side_table_load_root(); +} + +static inline bool __stack_depot_trie_ready(void) +{ + return __stack_depot_trie_enabled() && + stack_depot_trie_load_workspace() && + trie_side_table_is_initialized(); +} + +static inline unsigned int trie_side_table_top_index(u32 id) +{ + return (id - 1) >> STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS; +} + +static inline unsigned int trie_side_table_root_index(u32 id) +{ + return trie_side_table_top_index(id) >> STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS; +} + +static inline unsigned int trie_side_table_dir_index(u32 id) +{ + return trie_side_table_top_index(id) & + (STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE - 1); +} + +static inline unsigned int trie_side_table_slot_index(u32 id) +{ + return (id - 1) & (STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE - 1); +} + +static struct stack_depot_trie_side_dir *trie_side_table_load_dir(unsigned int root) +{ + struct stack_depot_trie_side_root *root_vec; + + root_vec = trie_side_table_load_root(); + if (!root_vec || root >= root_vec->dir_capacity) + return NULL; + /* Pairs with side-table directory rcu_assign_pointer(). */ + return rcu_dereference_check(root_vec->dirs[root], + lockdep_is_held(&trie_side_table_lock) || + rcu_read_lock_sched_held()); +} + +static inline const struct stack_depot_trie_node __rcu ** +trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, + unsigned int idx) +{ + /* Pairs with the chunk rcu_assign_pointer() in leaf ID preparation. */ + return rcu_dereference_check(dir->chunks[idx], + lockdep_is_held(&trie_side_table_lock) || + rcu_read_lock_sched_held()); +} + +static u32 trie_side_table_next_leaf_id(void) +{ + unsigned long flags; + u32 id = 0; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + id = trie_side_table_last_leaf_id + 1; + /* ID zero wraps the 32-bit counter; max_id is trie handle capacity. */ + if (!id || id > trie_side_table_max_id) + id = 0; + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return id; +} + +static bool +trie_side_table_ensure_leaf_slot(u32 id, + struct stack_depot_trie_side_prealloc *prealloc) +{ + const struct stack_depot_trie_node __rcu **chunk; + struct stack_depot_trie_side_dir *dir; + struct stack_depot_trie_side_root *root_vec; + unsigned long flags; + unsigned int root; + unsigned int idx; + bool ret = false; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + root_vec = trie_side_table_load_root(); + root = trie_side_table_root_index(id); + dir = trie_side_table_load_dir(root); + if (!dir) { + /* Sparse growth preallocation can lose a race to another writer. */ + if (!prealloc->dir) + goto out; + dir = prealloc->dir; + prealloc->dir = NULL; + /* Publish the zeroed directory before readers can load it locklessly. */ + rcu_assign_pointer(root_vec->dirs[root], dir); + } + + idx = trie_side_table_dir_index(id); + chunk = trie_side_table_dir_load_chunk(dir, idx); + if (!chunk) { + /* Sparse growth preallocation can lose a race to another writer. */ + if (!prealloc->chunk) + goto out; + chunk = prealloc->chunk; + prealloc->chunk = NULL; + rcu_assign_pointer(dir->chunks[idx], chunk); + } + + ret = true; +out: + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return ret; +} + +static size_t trie_side_table_root_bytes(unsigned int root_size) +{ + size_t bytes; + + bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); + if (bytes == SIZE_MAX) + return 0; + return PAGE_ALIGN(bytes); +} + +static inline size_t trie_side_table_dir_bytes(void) +{ + return PAGE_ALIGN(sizeof(struct stack_depot_trie_side_dir)); +} + +static inline unsigned int trie_side_table_dir_order(void) +{ + return get_order(trie_side_table_dir_bytes()); +} + +static inline size_t trie_side_table_chunk_bytes(void) +{ + return PAGE_ALIGN(STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE * + sizeof(struct stack_depot_trie_node *)); +} + +static inline unsigned int trie_side_table_chunk_order(void) +{ + return get_order(trie_side_table_chunk_bytes()); +} + +static void trie_side_table_free_dir(struct stack_depot_trie_side_dir *dir) +{ + if (dir) + free_pages((unsigned long)dir, trie_side_table_dir_order()); +} + +static void trie_side_table_root_init(struct stack_depot_trie_side_root *root_vec, + unsigned int root_size, u32 max_id) +{ + root_vec->dir_capacity = root_size; + trie_side_table_max_id = max_id; + trie_side_table_last_leaf_id = 0; +} + +static inline unsigned int trie_side_table_root_size_for_max_id(u32 max_leaf_id) +{ + unsigned int top_size; + + top_size = DIV_ROUND_UP(max_leaf_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); + return DIV_ROUND_UP(top_size, STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE); +} + +static int __init __stack_depot_trie_side_table_init_memblock(void) +{ + struct stack_depot_trie_side_root *root_vec; + struct stack_depot_trie_side_dir *first_dir; + const struct stack_depot_trie_node __rcu **first_chunk; + size_t dir_bytes; + size_t chunk_bytes; + size_t root_bytes; + u32 max_leaf_id; + unsigned int root_size; + + if (trie_side_table_is_initialized()) + return 0; + + max_leaf_id = __stack_depot_trie_max_leaf_id(); + if (!max_leaf_id) + return -EINVAL; + root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_bytes = trie_side_table_root_bytes(root_size); + dir_bytes = trie_side_table_dir_bytes(); + chunk_bytes = trie_side_table_chunk_bytes(); + if (!root_bytes || !dir_bytes || !chunk_bytes) + return -ENOMEM; + + root_vec = memblock_alloc(root_bytes, PAGE_SIZE); + if (!root_vec) + return -ENOMEM; + memset(root_vec, 0, root_bytes); + first_dir = memblock_alloc(dir_bytes, PAGE_SIZE); + if (!first_dir) { + memblock_free(root_vec, root_bytes); + return -ENOMEM; + } + memset(first_dir, 0, dir_bytes); + first_chunk = memblock_alloc(chunk_bytes, PAGE_SIZE); + if (!first_chunk) { + memblock_free(first_dir, dir_bytes); + memblock_free(root_vec, root_bytes); + return -ENOMEM; + } + memset(first_chunk, 0, chunk_bytes); + + trie_side_table_root_init(root_vec, root_size, max_leaf_id); + RCU_INIT_POINTER(root_vec->dirs[0], first_dir); + RCU_INIT_POINTER(first_dir->chunks[0], first_chunk); + rcu_assign_pointer(trie_side_table_root, root_vec); + return 0; +} + +static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) +{ + struct stack_depot_trie_side_root *root_vec; + unsigned int root_size; + size_t root_bytes; + u32 max_leaf_id; + + if (trie_side_table_is_initialized()) + return 0; + + max_leaf_id = __stack_depot_trie_max_leaf_id(); + if (!max_leaf_id) + return -EINVAL; + + root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_bytes = trie_side_table_root_bytes(root_size); + if (!root_bytes) + return -ENOMEM; + root_vec = kvzalloc(root_bytes, gfp_flags); + if (!root_vec) + return -ENOMEM; + + trie_side_table_root_init(root_vec, root_size, max_leaf_id); + rcu_assign_pointer(trie_side_table_root, root_vec); + return 0; +} + +static void trie_free_object_buckets_init(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) { + INIT_LIST_HEAD(&free_trie_objects[i]); + INIT_LIST_HEAD(&pending_trie_objects[i]); + INIT_LIST_HEAD(&free_trie_nodes[i]); + } +} + +static int __init stack_depot_trie_init_memblock(void) +{ + struct stack_depot_trie_alloc_workspace *workspace = NULL; + size_t size; + int ret; + + if (__stack_depot_trie_ready()) + return 0; + + if (!stack_depot_trie_load_workspace()) { + size = sizeof(*stack_depot_trie_workspace); + workspace = memblock_alloc(size, __alignof__(*workspace)); + if (!workspace) + return -ENOMEM; + memset(workspace, 0, size); + } + ret = __stack_depot_trie_side_table_init_memblock(); + if (ret) { + if (workspace) + memblock_free(workspace, size); + return ret; + } + if (workspace) + rcu_assign_pointer(stack_depot_trie_workspace, workspace); + + trie_free_object_buckets_init(); + stack_depot_trie_enable(); + return 0; +} + +static int stack_depot_trie_init(gfp_t gfp_flags) +{ + struct stack_depot_trie_alloc_workspace *workspace = NULL; + int ret; + + if (__stack_depot_trie_ready()) + return 0; + + if (!stack_depot_trie_load_workspace()) { + workspace = kvzalloc(sizeof(*stack_depot_trie_workspace), gfp_flags); + if (!workspace) + return -ENOMEM; + } + ret = __stack_depot_trie_side_table_init(gfp_flags); + if (ret) { + kvfree(workspace); + return ret; + } + if (workspace) + rcu_assign_pointer(stack_depot_trie_workspace, workspace); + + trie_free_object_buckets_init(); + stack_depot_trie_enable(); + return 0; +} + +static inline const struct stack_depot_trie_node * +trie_side_table_load_leaf(const struct stack_depot_trie_node __rcu **slot) +{ + /* Pairs with side-table leaf rcu_assign_pointer(). */ + return rcu_dereference_check(*slot, + lockdep_is_held(&trie_side_table_lock) || + rcu_read_lock_sched_held()); +} + +static void *trie_side_table_alloc_page(gfp_t gfp_flags, unsigned int order) +{ + struct page *page; + + page = alloc_pages(gfp_nested_mask(gfp_flags) | __GFP_ZERO, + order); + return page ? page_address(page) : NULL; +} + +static int +__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, + struct stack_depot_trie_side_prealloc *prealloc) +{ + struct stack_depot_trie_side_dir *dir; + unsigned long flags; + bool need_chunk; + bool need_dir; + u32 id; + unsigned int root; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + id = trie_side_table_last_leaf_id + 1; + if (!id || id > trie_side_table_max_id) { + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return 0; + } + root = trie_side_table_root_index(id); + dir = trie_side_table_load_dir(root); + need_dir = !dir; + need_chunk = need_dir || !trie_side_table_dir_load_chunk(dir, + trie_side_table_dir_index(id)); + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + + if (need_dir) { + unsigned int order = trie_side_table_dir_order(); + + prealloc->dir = trie_side_table_alloc_page(gfp_flags, order); + if (!prealloc->dir) + return -ENOMEM; + } + if (need_chunk) { + unsigned int order = trie_side_table_chunk_order(); + + prealloc->chunk = trie_side_table_alloc_page(gfp_flags, order); + if (!prealloc->chunk) { + trie_side_table_free_dir(prealloc->dir); + prealloc->dir = NULL; + return -ENOMEM; + } + } + + return 0; +} + +static void +__stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_prealloc *prealloc) +{ + trie_side_table_free_dir(prealloc->dir); + if (prealloc->chunk) + free_pages((unsigned long)prealloc->chunk, + trie_side_table_chunk_order()); + prealloc->dir = NULL; + prealloc->chunk = NULL; +} + +static const struct stack_depot_trie_node __rcu **trie_side_table_leaf_slot(u32 id) +{ + const struct stack_depot_trie_node __rcu **chunk; + struct stack_depot_trie_side_dir *dir; + unsigned int root; + + root = trie_side_table_root_index(id); + dir = trie_side_table_load_dir(root); + chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); + + return &chunk[trie_side_table_slot_index(id)]; +} + +static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup(u32 id) +{ + const struct stack_depot_trie_node __rcu **chunk; + struct stack_depot_trie_side_dir *dir; + unsigned int root; + + if (!id) + return NULL; + if (!trie_side_table_is_initialized()) + return NULL; + + root = trie_side_table_root_index(id); + dir = trie_side_table_load_dir(root); + if (!dir) + return NULL; + chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); + if (!chunk) + return NULL; + + return trie_side_table_load_leaf(&chunk[trie_side_table_slot_index(id)]); +} + +static size_t __stack_depot_trie_pool_alloc_size(size_t size) +{ + size_t align = 1UL << DEPOT_STACK_ALIGN; + size_t aligned; + + if (!size || size > DEPOT_POOL_SIZE) + return 0; + if (check_add_overflow(size, align - 1, &aligned)) + return 0; + aligned = ALIGN(size, align); + return aligned <= DEPOT_POOL_SIZE ? aligned : 0; +} + +static inline size_t trie_object_header_size(void) +{ + return ALIGN(sizeof(struct stack_depot_trie_free_object), + 1UL << DEPOT_STACK_ALIGN); +} + +static size_t trie_object_alloc_size(size_t size) +{ + size_t alloc_size; + + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return 0; + if (check_add_overflow(trie_object_header_size(), size, + &alloc_size)) + return 0; + return alloc_size <= DEPOT_POOL_SIZE ? alloc_size : 0; +} + +static inline struct stack_depot_trie_free_object *trie_object_header(const void *ptr) +{ + return (struct stack_depot_trie_free_object *)((const char *)ptr - + trie_object_header_size()); +} + +static inline void *trie_object_payload(struct stack_depot_trie_free_object *free) +{ + return (char *)free + trie_object_header_size(); +} + +static inline unsigned int trie_free_class(size_t size) +{ + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return 0; + + return size >> DEPOT_STACK_ALIGN; +} + +static void trie_free_list_add(struct list_head *entry, struct list_head *heads, + unsigned long *map, unsigned int class, bool tail) +{ + lockdep_assert_held(&pool_lock); + + if (tail) + list_add_tail(entry, &heads[class]); + else + list_add(entry, &heads[class]); + __set_bit(class, map); +} + +static void trie_free_list_del(struct list_head *entry, struct list_head *heads, + unsigned long *map, unsigned int class) +{ + lockdep_assert_held(&pool_lock); + + list_del_init(entry); + if (list_empty(&heads[class])) + __clear_bit(class, map); +} + +static bool depot_init_pool(void **prealloc); + +static unsigned int trie_find_next_set_class(unsigned long *map, unsigned int class) +{ + for (; class < STACK_DEPOT_TRIE_FREE_CLASSES; class++) { + if (map[class / BITS_PER_LONG] & BIT(class % BITS_PER_LONG)) + return class; + } + + return STACK_DEPOT_TRIE_FREE_CLASSES; +} + +static void *trie_object_init_fresh(void *ptr, size_t size) +{ + struct stack_depot_trie_free_object *free = ptr; + + free->size = __stack_depot_trie_pool_alloc_size(size); + free->rcu_state = 0; + free->pending_node = NULL; + INIT_LIST_HEAD(&free->list); + return trie_object_payload(free); +} + +static void trie_free_object_locked(const void *ptr, unsigned long rcu_state) +{ + struct stack_depot_trie_free_object *free; + unsigned int class; + + lockdep_assert_held(&pool_lock); + + free = trie_object_header(ptr); + free->rcu_state = rcu_state; + class = trie_free_class(free->size); + INIT_LIST_HEAD(&free->list); + if (poll_state_synchronize_rcu(rcu_state)) + trie_free_list_add(&free->list, free_trie_objects, + free_trie_object_map, class, false); + else + trie_free_list_add(&free->list, pending_trie_objects, + pending_trie_object_map, class, true); +} + +static void trie_add_free_node_locked(void *ptr, size_t size) +{ + struct stack_depot_trie_free_node *free = ptr; + unsigned int class; + + lockdep_assert_held(&pool_lock); + + size = __stack_depot_trie_pool_alloc_size(size); + if (size < sizeof(*free)) + return; + + INIT_LIST_HEAD(&free->list); + class = trie_free_class(size); + trie_free_list_add(&free->list, free_trie_nodes, free_trie_node_map, + class, false); +} + +static void trie_drain_free_object_node_locked(struct stack_depot_trie_free_object *free) +{ + size_t size; + + lockdep_assert_held(&pool_lock); + + if (!free->pending_node) + return; + size = __stack_depot_trie_node_size(&free->pending_node->run); + trie_add_free_node_locked(free->pending_node, size); + free->pending_node = NULL; +} + +static void trie_drain_pending_objects_locked(void) +{ + struct stack_depot_trie_free_object *free; + struct stack_depot_trie_free_object *tmp; + unsigned int class; + + lockdep_assert_held(&pool_lock); + + for (class = trie_find_next_set_class(pending_trie_object_map, 0); + class < STACK_DEPOT_TRIE_FREE_CLASSES; + class = trie_find_next_set_class(pending_trie_object_map, class + 1)) { + list_for_each_entry_safe(free, tmp, &pending_trie_objects[class], list) { + /* Pending lists are FIFO; later entries cannot be ready yet. */ + if (!poll_state_synchronize_rcu(free->rcu_state)) + break; + trie_drain_free_object_node_locked(free); + trie_free_list_del(&free->list, pending_trie_objects, + pending_trie_object_map, class); + free->rcu_state = 0; + trie_free_list_add(&free->list, free_trie_objects, + free_trie_object_map, class, false); + } + } +} + +static void trie_free_object_tail_locked(void *ptr, size_t size) +{ + struct stack_depot_trie_free_object *free = ptr; + size_t header_size = trie_object_header_size(); + size_t tail_size; + + lockdep_assert_held(&pool_lock); + + if (size >= header_size + (1UL << DEPOT_STACK_ALIGN)) { + tail_size = size - header_size; + free->size = tail_size; + free->rcu_state = get_completed_synchronize_rcu(); + free->pending_node = NULL; + INIT_LIST_HEAD(&free->list); + trie_free_list_add(&free->list, free_trie_objects, + free_trie_object_map, trie_free_class(tail_size), + false); + return; + } + + trie_add_free_node_locked(ptr, size); +} + +static void *trie_pop_free_node(size_t size) +{ + struct stack_depot_trie_free_node *free; + unsigned int class; + size_t old_size; + + lockdep_assert_held(&pool_lock); + + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return NULL; + class = trie_free_class(size); + class = trie_find_next_set_class(free_trie_node_map, class); + if (class >= STACK_DEPOT_TRIE_FREE_CLASSES) + return NULL; + free = list_first_entry(&free_trie_nodes[class], typeof(*free), list); + old_size = (size_t)class << DEPOT_STACK_ALIGN; + trie_free_list_del(&free->list, free_trie_nodes, free_trie_node_map, + class); + if (old_size > size) + trie_free_object_tail_locked((char *)free + size, old_size - size); + return free; +} + +static void trie_retire_child_array_locked(const void *ptr) +{ + struct stack_depot_trie_free_object *free; + + lockdep_assert_held(&pool_lock); + free = trie_object_header(ptr); + free->pending_node = NULL; + free->rcu_state = get_state_synchronize_rcu(); + trie_free_list_add(&free->list, pending_trie_objects, + pending_trie_object_map, + trie_free_class(free->size), true); +} + +static void +trie_retire_child_array_with_node(const void *ptr, + const struct stack_depot_trie_node *node) +{ + struct stack_depot_trie_free_object *free; + unsigned long flags; + + raw_spin_lock_irqsave(&pool_lock, flags); + trie_retire_child_array_locked(ptr); + free = trie_object_header(ptr); + free->pending_node = (struct stack_depot_trie_node *)node; + raw_spin_unlock_irqrestore(&pool_lock, flags); +} + +static void *trie_pop_free_object(size_t size) +{ + struct stack_depot_trie_free_object *free; + void *tail; + size_t old_size; + size_t tail_size; + unsigned int class; + + lockdep_assert_held(&pool_lock); + + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return NULL; + class = trie_free_class(size); + class = trie_find_next_set_class(free_trie_object_map, class); + if (class >= STACK_DEPOT_TRIE_FREE_CLASSES) + return NULL; + + free = list_first_entry(&free_trie_objects[class], typeof(*free), list); + trie_drain_free_object_node_locked(free); + trie_free_list_del(&free->list, free_trie_objects, + free_trie_object_map, class); + free->rcu_state = 0; + old_size = free->size; + if (old_size > size) { + free->size = size; + tail = trie_object_payload(free) + size; + tail_size = old_size - size; + trie_free_object_tail_locked(tail, tail_size); + } + return trie_object_payload(free); +} + +/* + * Preallocate resources that cannot be allocated while trie writers hold raw + * spinlocks. Side-table growth is mandatory before a new leaf ID can be + * used, so side-table preallocation failure disables insertion for this + * save. Pool preallocation is opportunistic: reusable trie storage or active + * pool space may still satisfy the insertion, and pool_carve() reports + * -ENOSPC if they do not. Callers without spinning allocation context skip + * insertion and perform only best-effort lookup. + */ +static int +__stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) +{ + unsigned long flags; + bool can_alloc; + bool need_pool; + int ret = 0; + + can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && + gfpflags_allow_spinning(alloc_flags); + if (can_alloc) { + raw_spin_lock_irqsave(&pool_lock, flags); + need_pool = !new_pool; + raw_spin_unlock_irqrestore(&pool_lock, flags); + if (need_pool) { + struct page *page; + + page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); + if (page) + *pool_prealloc = page_address(page); + } + ret = __stack_depot_trie_side_table_prealloc(alloc_flags, side_prealloc); + } + + if (ret) { + if (*pool_prealloc) { + free_pages((unsigned long)*pool_prealloc, DEPOT_POOL_ORDER); + *pool_prealloc = NULL; + } + return -ENOSPC; + } + return 0; +} + +static int trie_pool_add_object_size(size_t size, size_t *total) +{ + size_t alloc_size; + + alloc_size = trie_object_alloc_size(size); + if (!alloc_size) + return -ENOSPC; + if (check_add_overflow(*total, alloc_size, total)) + return -ENOSPC; + return *total <= DEPOT_POOL_SIZE ? 0 : -ENOSPC; +} + +/* + * Allocate pool-backed storage for one trie insertion. Reusable retired + * fragments are preferred; any missing storage is carved as one contiguous range + * from the current stackdepot pool, possibly after installing @prealloc as a new + * pool. If allocation fails, any free-list pops are returned locally and + * pool_offset is not advanced. + * The caller must not publish any returned storage before side-table and trie + * publication succeeds. + */ +static int trie_pool_carve(struct stack_depot_trie_alloc_workspace *workspace, + void **pool_prealloc, unsigned int nr_nodes, + unsigned int nr_child_arrays, + size_t split_child_array_size, + size_t child_array_size) +{ + unsigned long flags; + unsigned long completed; + size_t one_child_size; + unsigned int i; + size_t alloc_size; + size_t offset; + size_t total = 0; + void *pool; + int ret = -ENOSPC; + + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + trie_drain_pending_objects_locked(); + one_child_size = __stack_depot_trie_child_array_size(1); + for (i = 0; i < nr_nodes; i++) { + workspace->nodes[i] = trie_pop_free_node(workspace->node_sizes[i]); + if (!workspace->nodes[i]) { + alloc_size = __stack_depot_trie_pool_alloc_size(workspace->node_sizes[i]); + if (!alloc_size || check_add_overflow(total, alloc_size, &total) || + total > DEPOT_POOL_SIZE) + goto out_discard; + } + } + for (i = 0; i < nr_child_arrays; i++) { + workspace->child_arrays[i] = trie_pop_free_object(one_child_size); + if (!workspace->child_arrays[i] && + trie_pool_add_object_size(one_child_size, &total)) + goto out_discard; + } + if (split_child_array_size) { + workspace->split_child_array = + trie_pop_free_object(split_child_array_size); + if (!workspace->split_child_array && + trie_pool_add_object_size(split_child_array_size, &total)) + goto out_discard; + } + if (child_array_size) { + workspace->child_array_storage = trie_pop_free_object(child_array_size); + if (!workspace->child_array_storage && + trie_pool_add_object_size(child_array_size, &total)) + goto out_discard; + } + + if (pools_num < 1) { + if (!depot_init_pool(pool_prealloc)) { + ret = -ENOSPC; + goto out_discard; + } + } + if (total > DEPOT_POOL_SIZE - pool_offset) { + if (!depot_init_pool(pool_prealloc)) { + ret = -ENOSPC; + goto out_discard; + } + } + + pool = stack_pools[pools_num - 1]; + + offset = pool_offset; + for (i = 0; i < nr_nodes; i++) { + if (workspace->nodes[i]) + continue; + workspace->nodes[i] = pool + offset; + offset += __stack_depot_trie_pool_alloc_size(workspace->node_sizes[i]); + } + for (i = 0; i < nr_child_arrays; i++) { + if (workspace->child_arrays[i]) + continue; + workspace->child_arrays[i] = + trie_object_init_fresh(pool + offset, one_child_size); + offset += trie_object_alloc_size(one_child_size); + } + if (split_child_array_size && !workspace->split_child_array) { + workspace->split_child_array = + trie_object_init_fresh(pool + offset, split_child_array_size); + offset += trie_object_alloc_size(split_child_array_size); + } + if (child_array_size && !workspace->child_array_storage) + workspace->child_array_storage = + trie_object_init_fresh(pool + offset, child_array_size); + pool_offset += total; + ret = 0; + goto out; +out_discard: + completed = get_completed_synchronize_rcu(); + for (i = 0; i < nr_nodes; i++) { + struct stack_depot_trie_node *node = workspace->nodes[i]; + + if (node) { + trie_add_free_node_locked(node, workspace->node_sizes[i]); + workspace->nodes[i] = NULL; + } + } + for (i = 0; i < nr_child_arrays; i++) { + struct stack_depot_trie_child_array *array = workspace->child_arrays[i]; + + if (!array) + continue; + trie_free_object_locked(array, completed); + workspace->child_arrays[i] = NULL; + } + if (split_child_array_size && workspace->split_child_array) { + trie_free_object_locked(workspace->split_child_array, completed); + workspace->split_child_array = NULL; + } + if (child_array_size && workspace->child_array_storage) { + trie_free_object_locked(workspace->child_array_storage, completed); + workspace->child_array_storage = NULL; + } +out: + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return ret; +} + +static const struct stack_depot_trie_node * +stack_depot_trie_lookup(const struct stack_depot_trie_child_array * const *root_slot, + const unsigned long *entries, unsigned int nr_entries); + +static depot_stack_handle_t +trie_find_handle(const struct stack_depot_trie_child_array * const *root_slot, + const unsigned long *entries, unsigned int nr_entries) +{ + depot_stack_handle_t handle = 0; + const struct stack_depot_trie_node *leaf; + + rcu_read_lock_sched_notrace(); + leaf = stack_depot_trie_lookup(root_slot, entries, nr_entries); + if (leaf) + handle = __stack_depot_trie_handle(leaf->leaf_id); + rcu_read_unlock_sched_notrace(); + + return handle; +} + +static void trie_side_table_publish_new_leaf(u32 leaf_id, + const struct stack_depot_trie_node *leaf) +{ + const struct stack_depot_trie_node __rcu **slot; + unsigned long flags; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + slot = trie_side_table_leaf_slot(leaf_id); + /* Pairs with trie_side_table_load_leaf(). */ + rcu_assign_pointer(*slot, leaf); + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); +} + +static void trie_side_table_publish_split_leaves(u32 old_leaf_id, + const struct stack_depot_trie_node *old_leaf, + u32 new_leaf_id, + const struct stack_depot_trie_node *new_leaf) +{ + const struct stack_depot_trie_node __rcu **new_slot; + const struct stack_depot_trie_node __rcu **old_slot = NULL; + unsigned long flags; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + if (old_leaf_id) + old_slot = trie_side_table_leaf_slot(old_leaf_id); -/* Statistics counters for debugfs. */ -enum depot_counter_id { - DEPOT_COUNTER_REFD_ALLOCS, - DEPOT_COUNTER_REFD_FREES, - DEPOT_COUNTER_REFD_INUSE, - DEPOT_COUNTER_FREELIST_SIZE, - DEPOT_COUNTER_PERSIST_COUNT, - DEPOT_COUNTER_PERSIST_BYTES, - DEPOT_COUNTER_COUNT, -}; -static long counters[DEPOT_COUNTER_COUNT]; -static const char *const counter_names[] = { - [DEPOT_COUNTER_REFD_ALLOCS] = "refcounted_allocations", - [DEPOT_COUNTER_REFD_FREES] = "refcounted_frees", - [DEPOT_COUNTER_REFD_INUSE] = "refcounted_in_use", - [DEPOT_COUNTER_FREELIST_SIZE] = "freelist_size", - [DEPOT_COUNTER_PERSIST_COUNT] = "persistent_count", - [DEPOT_COUNTER_PERSIST_BYTES] = "persistent_bytes", -}; -static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); + new_slot = trie_side_table_leaf_slot(new_leaf_id); + if (old_slot) { + /* Pairs with trie_side_table_load_leaf(). */ + rcu_assign_pointer(*old_slot, old_leaf); + } + + /* Pairs with trie_side_table_load_leaf(). */ + rcu_assign_pointer(*new_slot, new_leaf); + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); +} static int __init disable_stack_depot(char *str) { @@ -220,6 +1456,10 @@ int __init stack_depot_early_init(void) stack_depot_disabled = true; return -ENOMEM; } + if (stack_depot_trie_requested && stack_depot_trie_init_memblock()) { + pr_warn("trie storage initialization failed, disabling trie storage\n"); + stack_depot_trie_requested = false; + } return 0; } @@ -233,8 +1473,10 @@ int stack_depot_init(void) mutex_lock(&stack_depot_init_mutex); - if (stack_depot_disabled || stack_table) + if (stack_depot_disabled) goto out_unlock; + if (stack_table) + goto init_trie; /* * Similarly to stack_depot_early_init, use stack_bucket_number_order @@ -278,6 +1520,16 @@ int stack_depot_init(void) kvfree(stack_table); stack_depot_disabled = true; ret = -ENOMEM; + goto out_unlock; + } +init_trie: + if (!ret && stack_depot_trie_requested) { + ret = stack_depot_trie_init(GFP_KERNEL); + if (ret) { + pr_warn("trie storage initialization failed, disabling trie storage\n"); + stack_depot_trie_requested = false; + ret = 0; + } } out_unlock: @@ -462,6 +1714,7 @@ depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, dep /* Save the stack trace. */ stack->hash = hash; stack->size = nr_entries; + stack->flags = flags & STACK_DEPOT_FLAGS_MASK; /* stack->handle is already filled in by depot_pop_free_pool(). */ memcpy(stack->entries, entries, flex_array_size(stack, entries, nr_entries)); @@ -585,6 +1838,7 @@ static inline struct stack_record *find_stack(struct list_head *bucket, unsigned long *entries, int size, u32 hash, depot_flags_t flags) { + depot_flags_t mode = STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE; struct stack_record *stack, *ret = NULL; /* @@ -601,6 +1855,9 @@ static inline struct stack_record *find_stack(struct list_head *bucket, list_for_each_entry_rcu(stack, bucket, hash_list) { if (stack->hash != hash || stack->size != size) continue; + /* Plain, refcounted, and countable records have distinct lifetimes. */ + if ((stack->flags & mode) != (flags & mode)) + continue; /* * This may race with depot_free_stack() accessing the freelist @@ -630,6 +1887,94 @@ static inline struct stack_record *find_stack(struct list_head *bucket, return ret; } +static int +stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_slot, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace, + u32 *leaf_id, + u32 *scratch); + +static depot_stack_handle_t +stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags, depot_flags_t depot_flags) +{ + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_side_prealloc side_prealloc = {}; + void *pool_prealloc = NULL; + depot_stack_handle_t handle; + unsigned long flags; + bool can_alloc; + bool retried = false; + u32 leaf_id; + int ret; + + workspace = stack_depot_trie_load_workspace(); + can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && + gfpflags_allow_spinning(alloc_flags); + +retry: + handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); + if (handle) + return handle; + /* + * No-spin callers cannot wait for the workspace lock or allocate side-table + * or pool storage. After the lockless lookup misses, trylock and recheck: a + * concurrent writer may have inserted the stack. Otherwise fail instead of + * spinning or publishing a new leaf. + */ + if (in_nmi() || !gfpflags_allow_spinning(alloc_flags)) { + if (!raw_spin_trylock_irqsave(&stack_depot_trie_writer_lock, flags)) + return 0; + handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); + raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); + return handle; + } + + ret = __stack_depot_trie_alloc_prealloc(alloc_flags, depot_flags, + &pool_prealloc, + &side_prealloc); + if (ret) + goto out_free; + + raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); + memset(workspace, 0, sizeof(*workspace)); + ret = stack_depot_trie_insert_locked(&stack_depot_trie_root, + entries, nr_entries, + &pool_prealloc, &side_prealloc, + workspace, &leaf_id, + workspace->scratch); + if (!ret) + handle = __stack_depot_trie_handle(leaf_id); + raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); + if (!handle && ret == -ENOSPC && can_alloc && !retried) { + retried = true; + if (pool_prealloc) { + raw_spin_lock_irqsave(&pool_lock, flags); + depot_keep_new_pool(&pool_prealloc); + raw_spin_unlock_irqrestore(&pool_lock, flags); + } + if (pool_prealloc) { + free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); + pool_prealloc = NULL; + } + __stack_depot_trie_side_table_free_prealloc(&side_prealloc); + goto retry; + } + +out_free: + if (pool_prealloc) { + raw_spin_lock_irqsave(&pool_lock, flags); + depot_keep_new_pool(&pool_prealloc); + raw_spin_unlock_irqrestore(&pool_lock, flags); + } + if (pool_prealloc) + free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); + __stack_depot_trie_side_table_free_prealloc(&side_prealloc); + return handle; +} + depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags, @@ -642,11 +1987,15 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, void *prealloc = NULL; bool allow_spin = gfpflags_allow_spinning(alloc_flags); bool can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && allow_spin; + bool trie_candidate; unsigned long flags; u32 hash; if (WARN_ON(depot_flags & ~STACK_DEPOT_FLAGS_MASK)) return 0; + if (WARN_ON_ONCE((depot_flags & STACK_DEPOT_FLAG_GET) && + (depot_flags & STACK_DEPOT_FLAG_COUNTABLE))) + return 0; /* * If this stack trace is from an interrupt, including anything before @@ -661,6 +2010,19 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, if (unlikely(nr_entries == 0) || stack_depot_disabled) return 0; + trie_candidate = !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE)) && + __stack_depot_trie_ready(); + if (trie_candidate) { + if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) + nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; + handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, + depot_flags); + if (handle) + return handle; + /* Keep trie failures visible; hash fallback hides trie pool pressure. */ + return 0; + } + hash = hash_stack(entries, nr_entries); bucket = &stack_table[hash & stack_hash_mask]; @@ -743,10 +2105,760 @@ EXPORT_SYMBOL_GPL(stack_depot_save); struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle) { + struct stack_record *stack; + if (!handle) return NULL; + if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) + return NULL; + + stack = depot_fetch_stack(handle); + if (!stack) + return NULL; + if (WARN_ON_ONCE(!(stack->flags & STACK_DEPOT_FLAG_COUNTABLE))) + return NULL; + + return stack; +} + +static void frame_run_init(const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_frame_run *run) +{ + u32 payload; + unsigned int i; + bool compressed; + + compressed = arch_stack_depot_frame_try_compress(entries[0], &payload); + for (i = 1; i < nr_entries; i++) { + bool next; + + next = arch_stack_depot_frame_try_compress(entries[i], &payload); + if (next != compressed) + break; + } + + /* @i is the first non-matching frame, or @nr_entries if all matched. */ + run->mode = compressed ? STACK_DEPOT_FRAME_COMPRESSED : STACK_DEPOT_FRAME_RAW; + run->nr_entries = i; +} + +static void +stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, + unsigned int index, unsigned long *frame) +{ + u32 payload; + + if (node->run.mode == STACK_DEPOT_FRAME_RAW) { + memcpy(frame, node->data + index * sizeof(*frame), + sizeof(*frame)); + return; + } + + memcpy(&payload, node->data + index * sizeof(payload), sizeof(payload)); + arch_stack_depot_frame_decompress(payload, frame); +} + +static void trie_node_init(void *storage, + const struct stack_depot_trie_node *parent, u32 leaf_id, + const unsigned long *entries, unsigned int nr_entries, + u32 *scratch) +{ + struct stack_depot_trie_node *node = storage; + struct stack_depot_frame_run run; + + frame_run_init(entries, nr_entries, &run); + + /* Caller-owned storage is not publishable unless the payload write succeeds. */ + if (run.mode == STACK_DEPOT_FRAME_COMPRESSED) { + unsigned int i; + + for (i = 0; i < run.nr_entries; i++) + arch_stack_depot_frame_try_compress(entries[i], &scratch[i]); + memcpy(node->data, scratch, stack_depot_frame_run_bytes(&run)); + } else { + memcpy(node->data, entries, stack_depot_frame_run_bytes(&run)); + } + + node->parent = parent; + node->children = NULL; + node->leaf_id = leaf_id; + node->run = run; +} + +static void trie_node_init_slice(void *storage, + const struct stack_depot_trie_node *parent, u32 leaf_id, + const struct stack_depot_trie_node *src_node, + unsigned int start, unsigned int nr_entries) +{ + struct stack_depot_trie_node *node = storage; + struct stack_depot_frame_run run; + size_t entry_bytes; + + run = src_node->run; + run.nr_entries = nr_entries; + + entry_bytes = stack_depot_frame_run_entry_bytes(src_node->run.mode); + memcpy(node->data, src_node->data + start * entry_bytes, + stack_depot_frame_run_bytes(&run)); + node->parent = parent; + node->children = NULL; + node->leaf_id = leaf_id; + node->run = run; +} + +static unsigned int +__stack_depot_trie_node_match(const struct stack_depot_trie_node *node, + const unsigned long *entries, + unsigned int nr_entries) +{ + unsigned int limit; + unsigned int i; + + limit = min(node->run.nr_entries, nr_entries); + if (node->run.mode == STACK_DEPOT_FRAME_RAW) { + for (i = 0; i < limit; i++) { + unsigned long frame; + + memcpy(&frame, node->data + i * sizeof(frame), sizeof(frame)); + if (frame != entries[i]) + break; + } + + return i; + } + + for (i = 0; i < limit; i++) { + unsigned long frame; + + stack_depot_trie_node_frame(node, i, &frame); + if (frame != entries[i]) + break; + } + + return i; +} + +static inline const struct stack_depot_trie_node * +trie_load_parent(const struct stack_depot_trie_node *node) +{ + const struct stack_depot_trie_node __rcu * const *slot; + + slot = (const struct stack_depot_trie_node __rcu * const *)&node->parent; + return rcu_dereference_check(*slot, + lockdep_is_held(&stack_depot_trie_writer_lock) || + rcu_read_lock_sched_held()); +} + +static inline const struct stack_depot_trie_child_array * +trie_load_children_slot(const struct stack_depot_trie_child_array * const *slot) +{ + const struct stack_depot_trie_child_array __rcu * const *rcu_slot; + + rcu_slot = (const struct stack_depot_trie_child_array __rcu * const *)slot; + return rcu_dereference_check(*rcu_slot, + lockdep_is_held(&stack_depot_trie_writer_lock) || + rcu_read_lock_sched_held()); +} + +static inline const struct stack_depot_trie_node * +trie_child_array_load_child(const struct stack_depot_trie_child_array *array, + unsigned int pos) +{ + const struct stack_depot_trie_node __rcu * const *slot; + + slot = (const struct stack_depot_trie_node __rcu * const *)&array->children[pos]; + return rcu_dereference_check(*slot, + lockdep_is_held(&stack_depot_trie_writer_lock) || + rcu_read_lock_sched_held()); +} + +static void +trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, + unsigned long frame, unsigned int *pos, bool *found) +{ + unsigned int left = 0; + unsigned int right; + + *pos = 0; + *found = false; + + right = READ_ONCE(array->nr_children); + while (left < right) { + unsigned int mid = left + (right - left) / 2; + const struct stack_depot_trie_node *node; + unsigned long mid_frame; + + node = trie_child_array_load_child(array, mid); + if (!node) { + /* Tail append may produce a transient lockless lookup miss. */ + right = mid; + continue; + } + stack_depot_trie_node_frame(node, 0, &mid_frame); + if (mid_frame < frame) { + left = mid + 1; + } else if (mid_frame > frame) { + right = mid; + } else { + *pos = mid; + *found = true; + return; + } + } + + *pos = left; +} - return depot_fetch_stack(handle); +static void +trie_child_array_insert_at(const struct stack_depot_trie_child_array *old, + unsigned int pos, + const struct stack_depot_trie_node *node, + struct stack_depot_trie_child_array *new_array, + unsigned int new_capacity) +{ + unsigned int nr_old; + unsigned int i; + + nr_old = old->nr_children; + + new_array->nr_children = nr_old + 1; + new_array->capacity = new_capacity; + for (i = 0; i < pos; i++) + new_array->children[i] = trie_child_array_load_child(old, i); + new_array->children[pos] = node; + for (i = pos; i < nr_old; i++) + new_array->children[i + 1] = trie_child_array_load_child(old, i); + for (i = nr_old + 1; i < new_array->capacity; i++) + new_array->children[i] = NULL; +} + +static void +trie_publish_children_slot(const struct stack_depot_trie_child_array **slot, + const struct stack_depot_trie_child_array *children) +{ + const struct stack_depot_trie_child_array __rcu **rcu_slot; + + rcu_slot = (const struct stack_depot_trie_child_array __rcu **)slot; + rcu_assign_pointer(*rcu_slot, children); +} + +static void +trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array, + const struct stack_depot_trie_node *new_child, + struct stack_depot_trie_child_array *new_array, + unsigned int pos) +{ + unsigned int i; + + new_array->nr_children = old_array->nr_children; + new_array->capacity = old_array->capacity; + for (i = 0; i < old_array->nr_children; i++) + new_array->children[i] = trie_child_array_load_child(old_array, i); + new_array->children[pos] = new_child; + for (i = old_array->nr_children; i < new_array->capacity; i++) + new_array->children[i] = NULL; +} + +static void trie_reparent_children(struct stack_depot_trie_node *parent) +{ + const struct stack_depot_trie_child_array *children = parent->children; + unsigned int i; + + if (!children) + return; + /* + * COW updates reuse unchanged descendant subtrees. Repoint their parent + * links before retiring the old parent so fetch never follows a freed node. + * Lockless lookups that still observe the old child array may see the new + * parent early and miss. A writer-lock recheck observes the new array before + * inserting, so the miss cannot create a permanent duplicate. + */ + for (i = 0; i < children->nr_children; i++) { + struct stack_depot_trie_node *child; + const struct stack_depot_trie_node __rcu **slot; + + child = (struct stack_depot_trie_node *)trie_child_array_load_child(children, i); + slot = (const struct stack_depot_trie_node __rcu **)&child->parent; + rcu_assign_pointer(*slot, parent); + } +} + +static void +trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, + const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_trie_node * const *nodes, + struct stack_depot_trie_child_array * const *child_arrays, + u32 *scratch, + const struct stack_depot_trie_node **head, + const struct stack_depot_trie_node **tail) +{ + const struct stack_depot_trie_node *prev = parent; + unsigned int pos = 0; + unsigned int used = 0; + unsigned int i; + + while (pos < nr_entries) { + struct stack_depot_frame_run run; + struct stack_depot_trie_node *node; + u32 id; + + node = nodes[used]; + frame_run_init(&entries[pos], nr_entries - pos, &run); + + id = pos + run.nr_entries == nr_entries ? leaf_id : 0; + trie_node_init(node, prev, id, &entries[pos], run.nr_entries, + scratch); + + prev = node; + pos += run.nr_entries; + used++; + } + + for (i = 0; i + 1 < used; i++) { + const struct stack_depot_trie_node *next = nodes[i + 1]; + struct stack_depot_trie_node *node = nodes[i]; + struct stack_depot_trie_child_array *array = child_arrays[i]; + + trie_child_array_init(array, 1, &next, 1); + node->children = array; + } + + *head = nodes[0]; + *tail = nodes[used - 1]; +} + +static void trie_publish_tail_append(struct stack_depot_trie_child_array *array, + unsigned int pos, + const struct stack_depot_trie_node *head) +{ + const struct stack_depot_trie_node __rcu **slot; + + /* + * Writers hold stack_depot_trie_writer_lock. Existing children are + * immutable, so tail append publishes the new child before increasing the + * visible count. Lockless readers that see the old count miss; readers that + * see the new count either load the initialized child or treat a transient + * NULL as a miss. The writer-lock recheck prevents permanent duplicates. + */ + slot = (const struct stack_depot_trie_node __rcu **)&array->children[pos]; + rcu_assign_pointer(*slot, head); + WRITE_ONCE(array->nr_children, pos + 1); +} + +static const struct stack_depot_trie_node * +stack_depot_trie_lookup(const struct stack_depot_trie_child_array * const *root_slot, + const unsigned long *entries, unsigned int nr_entries) +{ + const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_node *parent = NULL; + unsigned int pos = 0; + + children = trie_load_children_slot(root_slot); + + while (pos < nr_entries) { + const struct stack_depot_trie_node *node; + unsigned int remaining = nr_entries - pos; + unsigned int matched; + unsigned int slot; + bool found; + + if (!children) + return NULL; + trie_child_array_find_slot(children, entries[pos], &slot, &found); + if (!found) + return NULL; + + node = trie_child_array_load_child(children, slot); + if (!node) + return NULL; + if (trie_load_parent(node) != parent) + return NULL; + + matched = __stack_depot_trie_node_match(node, &entries[pos], remaining); + if (!matched || matched < node->run.nr_entries) + return NULL; + pos += matched; + if (pos == nr_entries) + return node->leaf_id ? node : NULL; + + parent = node; + children = trie_load_children_slot(&node->children); + } + + return NULL; +} + +static void trie_build_split(const struct stack_depot_trie_node *child, + unsigned int matched, u32 leaf_id, + const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_trie_node * const *nodes, + struct stack_depot_trie_child_array * const *child_arrays, + struct stack_depot_trie_child_array *split_child_array, + u32 *scratch, + const struct stack_depot_trie_node **prefix, + const struct stack_depot_trie_node **old_tail, + const struct stack_depot_trie_node **new_leaf) +{ + const struct stack_depot_trie_child_array *child_children; + const struct stack_depot_trie_node *child_parent; + const unsigned long *tail_entries; + const struct stack_depot_trie_node *split_children[2]; + const struct stack_depot_trie_node *new_head = NULL; + const struct stack_depot_trie_node *new_tail = NULL; + struct stack_depot_trie_node *old_tail_node; + struct stack_depot_trie_node *pref; + unsigned long new_frame; + unsigned long old_frame; + u32 prefix_leaf_id; + unsigned int tail_len; + bool has_new_tail; + + child_children = trie_load_children_slot(&child->children); + child_parent = trie_load_parent(child); + + has_new_tail = matched < nr_entries; + pref = nodes[0]; + old_tail_node = nodes[1]; + prefix_leaf_id = has_new_tail ? 0 : leaf_id; + trie_node_init_slice(pref, child_parent, prefix_leaf_id, child, 0, + matched); + tail_len = child->run.nr_entries - matched; + trie_node_init_slice(old_tail_node, pref, child->leaf_id, child, matched, + tail_len); + + if (has_new_tail) { + tail_entries = &entries[matched]; + tail_len = nr_entries - matched; + trie_build_append_chain(pref, leaf_id, tail_entries, tail_len, + &nodes[2], child_arrays, scratch, + &new_head, &new_tail); + stack_depot_trie_node_frame(old_tail_node, 0, &old_frame); + stack_depot_trie_node_frame(new_head, 0, &new_frame); + if (old_frame < new_frame) { + split_children[0] = old_tail_node; + split_children[1] = new_head; + } else { + split_children[0] = new_head; + split_children[1] = old_tail_node; + } + trie_child_array_init(split_child_array, ARRAY_SIZE(split_children), + split_children, ARRAY_SIZE(split_children)); + } else { + split_children[0] = old_tail_node; + trie_child_array_init(split_child_array, 1, split_children, 1); + } + old_tail_node->children = child_children; + pref->children = split_child_array; + *prefix = pref; + *old_tail = old_tail_node; + *new_leaf = has_new_tail ? new_tail : pref; +} + +static void trie_size_append_chain(const unsigned long *entries, + unsigned int nr_entries, + size_t *node_sizes, + unsigned int *nr_nodes, + unsigned int *nr_child_arrays) +{ + unsigned int pos = 0; + unsigned int used = 0; + + while (pos < nr_entries) { + struct stack_depot_frame_run run; + + frame_run_init(&entries[pos], nr_entries - pos, &run); + node_sizes[used] = __stack_depot_trie_node_size(&run); + pos += run.nr_entries; + used++; + } + + *nr_nodes = used; + *nr_child_arrays = used > 1 ? used - 1 : 0; +} + +static int +trie_pool_alloc_append_chain(struct stack_depot_trie_alloc_workspace *workspace, + void **pool_prealloc, + const unsigned long *entries, + unsigned int nr_entries, + size_t child_array_size) +{ + unsigned int nr_child_arrays; + unsigned int nr_nodes; + + trie_size_append_chain(entries, nr_entries, workspace->node_sizes, + &nr_nodes, &nr_child_arrays); + return trie_pool_carve(workspace, pool_prealloc, nr_nodes, nr_child_arrays, + 0, child_array_size); +} + +static int +trie_pool_alloc_split(struct stack_depot_trie_alloc_workspace *workspace, + void **pool_prealloc, + const struct stack_depot_trie_child_array *children, + const struct stack_depot_trie_node *child, + unsigned int matched, const unsigned long *entries, + unsigned int nr_entries) +{ + struct stack_depot_frame_run old_tail_run; + struct stack_depot_frame_run prefix_run; + unsigned int new_child_arrays = 0; + unsigned int new_used = 0; + unsigned int nr_child_arrays; + unsigned int nr_nodes; + bool has_new_tail; + size_t child_array_size; + size_t split_child_array_size; + + prefix_run = child->run; + prefix_run.nr_entries = matched; + old_tail_run = child->run; + old_tail_run.nr_entries = child->run.nr_entries - matched; + has_new_tail = matched < nr_entries; + + workspace->node_sizes[0] = __stack_depot_trie_node_size(&prefix_run); + workspace->node_sizes[1] = __stack_depot_trie_node_size(&old_tail_run); + if (has_new_tail) + trie_size_append_chain(&entries[matched], nr_entries - matched, + &workspace->node_sizes[2], &new_used, + &new_child_arrays); + + child_array_size = trie_child_array_size_for_capacity(children->capacity); + split_child_array_size = + __stack_depot_trie_child_array_size(has_new_tail ? 2 : 1); + nr_nodes = 2 + new_used; + nr_child_arrays = new_child_arrays; + return trie_pool_carve(workspace, pool_prealloc, nr_nodes, nr_child_arrays, + split_child_array_size, child_array_size); +} + +static int +stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_slot, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace, + u32 *leaf_id, + u32 *scratch) +{ + struct stack_depot_trie_child_array *new_array; + struct stack_depot_trie_child_array *split_child_array; + const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_child_array **slot = root_slot; + const struct stack_depot_trie_node *child; + const struct stack_depot_trie_node *head; + const struct stack_depot_trie_node *last; + const struct stack_depot_trie_node *old_tail; + const struct stack_depot_trie_node *new_leaf; + struct stack_depot_trie_node *new_node; + struct stack_depot_trie_node *parent = NULL; + unsigned int matched; + unsigned int pos; + unsigned long flags; + u32 new_leaf_id; + size_t child_array_size; + bool found; + int ret; + + for (;;) { + children = trie_load_children_slot(slot); + if (!children) { + new_leaf_id = trie_side_table_next_leaf_id(); + if (!new_leaf_id) + return -ENOSPC; + if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) + return -ENOSPC; + child_array_size = __stack_depot_trie_child_array_size(1); + ret = trie_pool_alloc_append_chain(workspace, pool_prealloc, entries, + nr_entries, child_array_size); + if (ret) + return ret; + new_array = workspace->child_array_storage; + trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, + workspace->nodes, workspace->child_arrays, + scratch, &head, &last); + trie_side_table_publish_new_leaf(new_leaf_id, last); + new_array->nr_children = 1; + new_array->capacity = 1; + new_array->children[0] = head; + /* Publish the fully initialized replacement array last. */ + trie_publish_children_slot(slot, new_array); + goto out_success; + } + trie_child_array_find_slot(children, entries[0], &pos, &found); + if (!found) { + struct stack_depot_trie_child_array *tail_array; + bool tail_append; + + tail_append = pos == children->nr_children && + children->nr_children < children->capacity; + + new_leaf_id = trie_side_table_next_leaf_id(); + if (!new_leaf_id) + return -ENOSPC; + if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) + return -ENOSPC; + child_array_size = tail_append ? 0 : + __stack_depot_trie_child_array_size(children->nr_children + 1); + ret = trie_pool_alloc_append_chain(workspace, pool_prealloc, entries, + nr_entries, child_array_size); + if (ret) + return ret; + new_array = workspace->child_array_storage; + trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, + workspace->nodes, workspace->child_arrays, + scratch, &head, &last); + trie_side_table_publish_new_leaf(new_leaf_id, last); + if (tail_append) { + tail_array = (struct stack_depot_trie_child_array *)children; + trie_publish_tail_append(tail_array, pos, head); + } else { + unsigned int capacity; + + capacity = trie_child_array_capacity(children->nr_children + 1); + trie_child_array_insert_at(children, pos, head, + new_array, capacity); + /* Publish the fully initialized replacement array last. */ + trie_publish_children_slot(slot, new_array); + raw_spin_lock_irqsave(&pool_lock, flags); + trie_retire_child_array_locked(children); + raw_spin_unlock_irqrestore(&pool_lock, flags); + } + goto out_success; + } + + child = trie_child_array_load_child(children, pos); + matched = __stack_depot_trie_node_match(child, entries, nr_entries); + if (matched < child->run.nr_entries) { + new_leaf_id = trie_side_table_next_leaf_id(); + if (!new_leaf_id) + return -ENOSPC; + if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) + return -ENOSPC; + ret = trie_pool_alloc_split(workspace, pool_prealloc, children, child, + matched, entries, nr_entries); + if (ret) + return ret; + new_array = workspace->child_array_storage; + split_child_array = workspace->split_child_array; + trie_build_split(child, matched, new_leaf_id, entries, nr_entries, + workspace->nodes, workspace->child_arrays, + split_child_array, scratch, &head, &old_tail, + &new_leaf); + trie_side_table_publish_split_leaves(child->leaf_id, old_tail, + new_leaf_id, new_leaf); + trie_child_array_replace_at(children, head, new_array, pos); + trie_reparent_children((struct stack_depot_trie_node *)old_tail); + /* Publish the fully initialized replacement array last. */ + trie_publish_children_slot(slot, new_array); + trie_retire_child_array_with_node(children, child); + goto out_success; + } + if (matched == nr_entries) { + if (child->leaf_id) { + *leaf_id = child->leaf_id; + return 0; + } + new_leaf_id = trie_side_table_next_leaf_id(); + if (!new_leaf_id) + return -ENOSPC; + if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) + return -ENOSPC; + workspace->node_sizes[0] = __stack_depot_trie_node_size(&child->run); + child_array_size = trie_child_array_size_for_capacity(children->capacity); + ret = trie_pool_carve(workspace, pool_prealloc, 1, 0, 0, + child_array_size); + if (ret) + return ret; + new_node = workspace->nodes[0]; + new_array = workspace->child_array_storage; + memcpy(new_node, child, __stack_depot_trie_node_size(&child->run)); + new_node->leaf_id = new_leaf_id; + trie_side_table_publish_new_leaf(new_leaf_id, new_node); + trie_child_array_replace_at(children, new_node, new_array, pos); + trie_reparent_children(new_node); + /* Publish the fully initialized replacement array last. */ + trie_publish_children_slot(slot, new_array); + trie_retire_child_array_with_node(children, child); + goto out_success; + } + + parent = (struct stack_depot_trie_node *)child; + slot = &parent->children; + entries += matched; + nr_entries -= matched; + } + +out_success: + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + trie_side_table_last_leaf_id = new_leaf_id; + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + *leaf_id = new_leaf_id; + return 0; +} + +static unsigned int +__stack_depot_trie_fetch_into(const struct stack_depot_trie_node *leaf, + unsigned long *entries, + unsigned int max_entries) +{ + const struct stack_depot_trie_node *node; + unsigned int total; + unsigned int seen = 0; + unsigned int pos; + unsigned int i; + + total = 0; + for (node = leaf; node; node = trie_load_parent(node)) + total += node->run.nr_entries; + if (max_entries < total) + return 0; + + pos = total; + for (node = leaf; node; node = trie_load_parent(node)) { + if (node->run.nr_entries > pos) + return 0; + pos -= node->run.nr_entries; + for (i = 0; i < node->run.nr_entries; i++) { + stack_depot_trie_node_frame(node, i, &entries[pos + i]); + seen++; + } + } + if (seen != total || pos) + return 0; + + return total; +} + +static unsigned int +__stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries) +{ + const struct stack_depot_trie_node *leaf; + u32 leaf_id; + unsigned int nr_entries; + + if (!handle || !entries || !max_entries) + return 0; + + leaf_id = __stack_depot_trie_leaf_id(handle); + if (!leaf_id) + return 0; + + rcu_read_lock_sched_notrace(); + leaf = __stack_depot_trie_side_table_lookup(leaf_id); + if (WARN_ONCE(!leaf, "corrupt trie handle %08x\n", handle)) { + rcu_read_unlock_sched_notrace(); + return 0; + } + nr_entries = __stack_depot_trie_fetch_into(leaf, entries, max_entries); + rcu_read_unlock_sched_notrace(); + if (nr_entries) + kmsan_unpoison_memory(entries, nr_entries * sizeof(*entries)); + + return nr_entries; } unsigned int stack_depot_fetch(depot_stack_handle_t handle, @@ -763,6 +2875,8 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, if (!handle || stack_depot_disabled) return 0; + if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) + return 0; stack = depot_fetch_stack(handle); /* @@ -777,12 +2891,42 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, } EXPORT_SYMBOL_GPL(stack_depot_fetch); +unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries) +{ + struct stack_record *stack; + unsigned int nr_entries; + + if (!handle || !entries || !max_entries) + return 0; + if (stack_depot_disabled) + return 0; + if (__stack_depot_trie_leaf_id(handle)) + return __stack_depot_trie_fetch_handle_into(handle, entries, + max_entries); + + stack = depot_fetch_stack(handle); + if (!stack) + return 0; + nr_entries = stack->size; + if (!nr_entries || nr_entries > max_entries) + return 0; + + memcpy(entries, stack->entries, nr_entries * sizeof(*entries)); + kmsan_unpoison_memory(entries, nr_entries * sizeof(*entries)); + return nr_entries; +} +EXPORT_SYMBOL_GPL(stack_depot_fetch_into); + void stack_depot_put(depot_stack_handle_t handle) { struct stack_record *stack; if (!handle || stack_depot_disabled) return; + if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) + return; stack = depot_fetch_stack(handle); /* @@ -791,7 +2935,8 @@ void stack_depot_put(depot_stack_handle_t handle) */ if (WARN(!stack, "corrupt handle or unbalanced stack_depot_put()")) return; - + if (WARN_ON_ONCE(!(stack->flags & STACK_DEPOT_FLAG_GET))) + return; if (refcount_dec_and_test(&stack->count)) depot_free_stack(stack); } @@ -802,6 +2947,16 @@ void stack_depot_print(depot_stack_handle_t stack) unsigned long *entries; unsigned int nr_entries; + if (__stack_depot_trie_leaf_id(stack)) { + unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; + const unsigned int max_entries = ARRAY_SIZE(trie_entries); + + nr_entries = __stack_depot_trie_fetch_handle_into(stack, trie_entries, max_entries); + if (nr_entries) + stack_trace_print(trie_entries, nr_entries, 0); + return; + } + nr_entries = stack_depot_fetch(stack, &entries); if (nr_entries > 0) stack_trace_print(entries, nr_entries, 0); @@ -814,6 +2969,16 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, unsigned long *entries; unsigned int nr_entries; + if (__stack_depot_trie_leaf_id(handle)) { + unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; + const unsigned int max_entries = ARRAY_SIZE(trie_entries); + + nr_entries = __stack_depot_trie_fetch_handle_into(handle, + trie_entries, max_entries); + return nr_entries ? stack_trace_snprint(buf, size, trie_entries, + nr_entries, spaces) : 0; + } + nr_entries = stack_depot_fetch(handle, &entries); return nr_entries ? stack_trace_snprint(buf, size, entries, nr_entries, spaces) : 0; diff --git a/lib/tests/Makefile b/lib/tests/Makefile index f7460831cfdd4..1d0954575ad5d 100644 --- a/lib/tests/Makefile +++ b/lib/tests/Makefile @@ -40,6 +40,7 @@ obj-$(CONFIG_SCANF_KUNIT_TEST) += scanf_kunit.o obj-$(CONFIG_SEQ_BUF_KUNIT_TEST) += seq_buf_kunit.o obj-$(CONFIG_SIPHASH_KUNIT_TEST) += siphash_kunit.o obj-$(CONFIG_SLUB_KUNIT_TEST) += slub_kunit.o +obj-$(CONFIG_STACKDEPOT_KUNIT_TEST) += stackdepot_kunit.o obj-$(CONFIG_TEST_SORT) += test_sort.o CFLAGS_stackinit_kunit.o += $(call cc-disable-warning, switch-unreachable) obj-$(CONFIG_STACKINIT_KUNIT_TEST) += stackinit_kunit.o diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c new file mode 100644 index 0000000000000..1e9e61277b218 --- /dev/null +++ b/lib/tests/stackdepot_kunit.c @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef CONFIG_ARM64 +#include + +static inline unsigned long stackdepot_arm64_frame(long offset) +{ + return (unsigned long)((long)_text + offset); +} +#endif + +static void stackdepot_fetch_into_roundtrip(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800010000UL, + 0x1234567800020000UL, + 0x1234567800030000UL, + }; + unsigned long exact[ARRAY_SIZE(entries)] = {}; + unsigned long fetched[ARRAY_SIZE(entries) + 1] = { + [ARRAY_SIZE(entries)] = 0xa5a5a5a5a5a5a5a5UL, + }; + unsigned long expected_tail = fetched[ARRAY_SIZE(entries)]; + depot_stack_handle_t handle; + unsigned int nr_entries; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + nr_entries = stack_depot_fetch_into(handle, exact, ARRAY_SIZE(exact)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, exact, entries, sizeof(entries)); + + nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); + KUNIT_EXPECT_EQ(test, fetched[ARRAY_SIZE(entries)], expected_tail); +} + +static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800110000UL, + 0x1234567800120000UL, + 0x1234567800130000UL, + }; + unsigned long fetched[ARRAY_SIZE(entries)] = { + 0xa1a1a1a1a1a1a1a1UL, + 0xb2b2b2b2b2b2b2b2UL, + 0xc3c3c3c3c3c3c3c3UL, + }; + unsigned long expected[ARRAY_SIZE(fetched)]; + depot_stack_handle_t handle; + unsigned int nr_entries; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + memcpy(expected, fetched, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(0, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(0, NULL, 0); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + + nr_entries = stack_depot_fetch_into(handle, NULL, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(handle, fetched, 0); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(handle, fetched, + ARRAY_SIZE(fetched) - 1); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); +} + +static depot_stack_handle_t save_countable(unsigned long *entries, unsigned int nr) +{ + depot_flags_t flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE; + + return stack_depot_save_flags(entries, nr, GFP_KERNEL, flags); +} + +static void stackdepot_countable_flag_roundtrip(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800210000UL, + 0x1234567800220000UL, + 0x1234567800230000UL, + }; + unsigned long fetched[ARRAY_SIZE(entries)] = {}; + depot_stack_handle_t handle; + unsigned int nr_entries; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = save_countable(entries, ARRAY_SIZE(entries)); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); +} + +static depot_stack_handle_t save_noalloc(unsigned long *entries, unsigned int nr) +{ + gfp_t no_spin = GFP_NOWAIT & ~__GFP_RECLAIM; + + return stack_depot_save_flags(entries, nr, no_spin, 0); +} + +static void stackdepot_save_flags_public(struct kunit *test) +{ + unsigned long entries[] = { 0x501000UL, 0x502000UL, 0x503000UL }; + unsigned long get_entries[] = { 0x601000UL, 0x602000UL }; + unsigned long count_entries[] = { 0x603000UL, 0x604000UL }; + unsigned long fetched[ARRAY_SIZE(entries)] = {}; + depot_stack_handle_t noalloc_handle; + depot_stack_handle_t overlong_handle; + depot_stack_handle_t count_handle; + depot_stack_handle_t plain_handle; + depot_stack_handle_t get_handle; + depot_stack_handle_t again; + depot_stack_handle_t extra; + depot_flags_t flags; + unsigned long *overlong_fetched; + unsigned long *overlong_entries; + unsigned int overlong_nr = CONFIG_STACKDEPOT_MAX_FRAMES + 1; + unsigned int nr_entries; + size_t overlong_size; + unsigned int i; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + overlong_entries = kunit_kcalloc(test, overlong_nr, + sizeof(*overlong_entries), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, overlong_entries); + overlong_fetched = kunit_kcalloc(test, CONFIG_STACKDEPOT_MAX_FRAMES, + sizeof(*overlong_fetched), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, overlong_fetched); + for (i = 0; i < overlong_nr; i++) + overlong_entries[i] = 0x800000UL + i * 0x1000UL; + + plain_handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, plain_handle, (depot_stack_handle_t)0); + again = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_EXPECT_EQ(test, again, plain_handle); + + nr_entries = stack_depot_fetch_into(plain_handle, fetched, + ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); + + noalloc_handle = save_noalloc(entries, ARRAY_SIZE(entries)); + KUNIT_EXPECT_EQ(test, noalloc_handle, plain_handle); + + flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; + get_handle = stack_depot_save_flags(get_entries, ARRAY_SIZE(get_entries), + GFP_KERNEL, flags); + KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); + stack_depot_put(get_handle); + + flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE; + count_handle = stack_depot_save_flags(count_entries, + ARRAY_SIZE(count_entries), + GFP_KERNEL, flags); + KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); + + overlong_handle = stack_depot_save(overlong_entries, overlong_nr, + GFP_KERNEL); + KUNIT_ASSERT_NE(test, overlong_handle, (depot_stack_handle_t)0); + nr_entries = stack_depot_fetch_into(overlong_handle, overlong_fetched, + CONFIG_STACKDEPOT_MAX_FRAMES); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)CONFIG_STACKDEPOT_MAX_FRAMES); + overlong_size = CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(*overlong_entries); + KUNIT_EXPECT_MEMEQ(test, overlong_fetched, overlong_entries, overlong_size); + + extra = stack_depot_set_extra_bits(plain_handle, 7); + KUNIT_ASSERT_NE(test, extra, (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, stack_depot_get_extra_bits(extra), 7U); + memset(fetched, 0, sizeof(fetched)); + nr_entries = stack_depot_fetch_into(extra, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); +} + +static void stackdepot_snprint_public(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + char expected[256]; + char actual[256]; + depot_stack_handle_t handle; + unsigned int expected_len; + int actual_len; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + expected_len = stack_trace_snprint(expected, sizeof(expected), entries, + ARRAY_SIZE(entries), 2); + actual_len = stack_depot_snprint(handle, actual, sizeof(actual), 2); + KUNIT_EXPECT_EQ(test, actual_len, (int)expected_len); + KUNIT_EXPECT_STREQ(test, actual, expected); +} + +static void stackdepot_get_stack_record(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800310000UL, + 0x1234567800320000UL, + 0x1234567800330000UL, + }; + struct stack_record *record; + depot_stack_handle_t handle; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = save_countable(entries, ARRAY_SIZE(entries)); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + record = __stack_depot_get_stack_record(handle); + KUNIT_ASSERT_NOT_NULL(test, record); + KUNIT_EXPECT_EQ(test, record->size, (u16)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, record->entries, entries, sizeof(entries)); +} + +static void stackdepot_countable_does_not_alias_other_modes(struct kunit *test) +{ + unsigned long plain_entries[] = { + 0x1234567800410000UL, + 0x1234567800420000UL, + 0x1234567800430000UL, + }; + unsigned long get_entries[] = { + 0x1234567800510000UL, + 0x1234567800520000UL, + 0x1234567800530000UL, + }; + depot_flags_t get = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; + struct stack_record *record; + depot_stack_handle_t count_handle; + depot_stack_handle_t plain_handle; + depot_stack_handle_t get_handle; + unsigned int get_nr = ARRAY_SIZE(get_entries); + unsigned int plain_nr = ARRAY_SIZE(plain_entries); + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + plain_handle = stack_depot_save(plain_entries, plain_nr, GFP_KERNEL); + KUNIT_ASSERT_NE(test, plain_handle, (depot_stack_handle_t)0); + count_handle = save_countable(plain_entries, plain_nr); + KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); + record = __stack_depot_get_stack_record(count_handle); + KUNIT_ASSERT_NOT_NULL(test, record); + KUNIT_EXPECT_MEMEQ(test, record->entries, plain_entries, + sizeof(plain_entries)); + + get_handle = stack_depot_save_flags(get_entries, get_nr, GFP_KERNEL, get); + KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); + count_handle = save_countable(get_entries, get_nr); + KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); + record = __stack_depot_get_stack_record(count_handle); + KUNIT_ASSERT_NOT_NULL(test, record); + KUNIT_EXPECT_MEMEQ(test, record->entries, get_entries, sizeof(get_entries)); + + stack_depot_put(get_handle); +} + +static void stackdepot_frame_raw_fallback(struct kunit *test) +{ + unsigned long frame = 0xffff888000001000UL; + bool compressed; + u32 payload = 0xfeedbeef; + +#ifdef CONFIG_ARM64 + if ((unsigned long)_text <= ULONG_MAX - ((unsigned long)S32_MAX + 1UL)) + frame = (unsigned long)_text + (unsigned long)S32_MAX + 1UL; + else + frame = stackdepot_arm64_frame((long)S32_MIN - 1L); +#endif + + compressed = arch_stack_depot_frame_try_compress(frame, &payload); + KUNIT_EXPECT_FALSE(test, compressed); + KUNIT_EXPECT_EQ(test, payload, (u32)0xfeedbeef); +} + +#ifdef CONFIG_X86_64 +static void stackdepot_frame_x86_64(struct kunit *test) +{ + unsigned long direct_map = 0xffff888000001000UL; + unsigned long frame = 0xffffffff81234567UL; + unsigned long out; + bool compressed; + u32 low; + + compressed = arch_stack_depot_frame_try_compress(frame, &low); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, low, (u32)0x81234567); + arch_stack_depot_frame_decompress(low, &out); + KUNIT_EXPECT_EQ(test, out, frame); + + compressed = arch_stack_depot_frame_try_compress(direct_map, &low); + KUNIT_EXPECT_FALSE(test, compressed); +} +#endif /* CONFIG_X86_64 */ + +#ifdef CONFIG_ARM64 +static void stackdepot_frame_arm64(struct kunit *test) +{ + long negative_offset = S32_MIN; + long positive_offset = S32_MAX; + long offset = 0x123456; + unsigned long frame = stackdepot_arm64_frame(offset); + unsigned long out; + bool compressed; + u32 payload; + + compressed = arch_stack_depot_frame_try_compress(frame, &payload); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, payload, (u32)(s32)offset); + arch_stack_depot_frame_decompress(payload, &out); + KUNIT_EXPECT_EQ(test, out, frame); + + frame = stackdepot_arm64_frame(negative_offset); + compressed = arch_stack_depot_frame_try_compress(frame, &payload); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, payload, (u32)(s32)negative_offset); + arch_stack_depot_frame_decompress(payload, &out); + KUNIT_EXPECT_EQ(test, out, frame); + + frame = stackdepot_arm64_frame(positive_offset); + compressed = arch_stack_depot_frame_try_compress(frame, &payload); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, payload, (u32)(s32)positive_offset); + arch_stack_depot_frame_decompress(payload, &out); + KUNIT_EXPECT_EQ(test, out, frame); +} +#endif /* CONFIG_ARM64 */ + +static struct kunit_case stackdepot_test_cases[] = { + KUNIT_CASE(stackdepot_fetch_into_roundtrip), + KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), + KUNIT_CASE(stackdepot_countable_flag_roundtrip), + KUNIT_CASE(stackdepot_save_flags_public), + KUNIT_CASE(stackdepot_snprint_public), + KUNIT_CASE(stackdepot_get_stack_record), + KUNIT_CASE(stackdepot_countable_does_not_alias_other_modes), + KUNIT_CASE(stackdepot_frame_raw_fallback), +#ifdef CONFIG_X86_64 + KUNIT_CASE(stackdepot_frame_x86_64), +#endif +#ifdef CONFIG_ARM64 + KUNIT_CASE(stackdepot_frame_arm64), +#endif + {} +}; + +static struct kunit_suite stackdepot_test_suite = { + .name = "stackdepot", + .test_cases = stackdepot_test_cases, +}; + +kunit_test_suite(stackdepot_test_suite); + +MODULE_DESCRIPTION("KUnit tests for stack depot"); +MODULE_AUTHOR("Caleb Kan "); +MODULE_LICENSE("GPL"); diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 1ac56ceb29b6b..d7fbcc9f121c9 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -364,10 +364,10 @@ static void print_unreferenced(struct seq_file *seq, struct kmemleak_object *object) { int i; - unsigned long *entries; + unsigned long entries[MAX_TRACE]; unsigned int nr_entries; - nr_entries = stack_depot_fetch(object->trace_handle, &entries); + nr_entries = stack_depot_fetch_into(object->trace_handle, entries, ARRAY_SIZE(entries)); warn_or_seq_printf(seq, "unreferenced object%s 0x%08lx (size %zu):\n", __object_type_str(object), object->pointer, object->size); diff --git a/mm/kmsan/kmsan_test.c b/mm/kmsan/kmsan_test.c index 902ec48b1e3e6..123bcc9565721 100644 --- a/mm/kmsan/kmsan_test.c +++ b/mm/kmsan/kmsan_test.c @@ -610,7 +610,7 @@ static void test_long_origin_chain(struct kunit *test) */ static void test_stackdepot_roundtrip(struct kunit *test) { - unsigned long src_entries[16], *dst_entries; + unsigned long src_entries[16], dst_entries[16]; unsigned int src_nentries, dst_nentries; EXPECTATION_NO_REPORT(expect); depot_stack_handle_t handle; @@ -621,11 +621,10 @@ static void test_stackdepot_roundtrip(struct kunit *test) stack_trace_save(src_entries, ARRAY_SIZE(src_entries), 1); handle = stack_depot_save(src_entries, src_nentries, GFP_KERNEL); stack_depot_print(handle); - dst_nentries = stack_depot_fetch(handle, &dst_entries); + dst_nentries = stack_depot_fetch_into(handle, dst_entries, ARRAY_SIZE(dst_entries)); KUNIT_EXPECT_TRUE(test, src_nentries == dst_nentries); - kmsan_check_memory((void *)dst_entries, - sizeof(*dst_entries) * dst_nentries); + kmsan_check_memory(dst_entries, sizeof(*dst_entries) * dst_nentries); KUNIT_EXPECT_TRUE(test, report_matches(&expect)); } diff --git a/mm/kmsan/report.c b/mm/kmsan/report.c index d6853ce089541..8d7b86b38d84c 100644 --- a/mm/kmsan/report.c +++ b/mm/kmsan/report.c @@ -85,7 +85,8 @@ static char *pretty_descr(char *descr) void kmsan_print_origin(depot_stack_handle_t origin) { - unsigned long *entries = NULL, *chained_entries = NULL; + unsigned long entries[KMSAN_STACK_DEPTH]; + const unsigned int max_entries = ARRAY_SIZE(entries); unsigned int nr_entries, chained_nr_entries, skipnr; void *pc1 = NULL, *pc2 = NULL; depot_stack_handle_t head; @@ -97,7 +98,7 @@ void kmsan_print_origin(depot_stack_handle_t origin) return; while (true) { - nr_entries = stack_depot_fetch(origin, &entries); + nr_entries = stack_depot_fetch_into(origin, entries, max_entries); depth = kmsan_depth_from_eb(stack_depot_get_extra_bits(origin)); magic = nr_entries ? entries[0] : 0; if ((nr_entries == 4) && (magic == KMSAN_ALLOCA_MAGIC_ORIGIN)) { @@ -122,16 +123,14 @@ void kmsan_print_origin(depot_stack_handle_t origin) head = entries[1]; origin = entries[2]; pr_err("Uninit was stored to memory at:\n"); + /* Reuse entries after saving head and origin above. */ chained_nr_entries = - stack_depot_fetch(head, &chained_entries); - kmsan_internal_unpoison_memory( - chained_entries, - chained_nr_entries * sizeof(*chained_entries), - /*checked*/ false); - skipnr = get_stack_skipnr(chained_entries, - chained_nr_entries); - stack_trace_print(chained_entries + skipnr, - chained_nr_entries - skipnr, 0); + stack_depot_fetch_into(head, entries, max_entries); + if (chained_nr_entries) { + skipnr = get_stack_skipnr(entries, chained_nr_entries); + stack_trace_print(entries + skipnr, + chained_nr_entries - skipnr, 0); + } pr_err("\n"); continue; } diff --git a/mm/page_owner.c b/mm/page_owner.c index bc26764142ba5..9f78413fee60a 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -92,7 +92,8 @@ static __always_inline depot_stack_handle_t create_dummy_stack(void) unsigned int nr_entries; nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); - return stack_depot_save(entries, nr_entries, GFP_KERNEL); + return stack_depot_save_flags(entries, nr_entries, GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE); } static noinline void register_dummy_stack(void) @@ -154,7 +155,8 @@ static noinline depot_stack_handle_t save_stack(gfp_t flags) set_current_in_page_owner(); nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); - handle = stack_depot_save(entries, nr_entries, flags); + handle = stack_depot_save_flags(entries, nr_entries, flags, + STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE); if (!handle) handle = failure_handle; unset_current_in_page_owner(); diff --git a/mm/slub.c b/mm/slub.c index 71e09c675047d..a49aca4ee6d1a 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -8150,12 +8150,12 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) #ifdef CONFIG_STACKDEPOT { depot_stack_handle_t handle; - unsigned long *entries; + unsigned long entries[TRACK_ADDRS_COUNT]; unsigned int nr_entries; handle = READ_ONCE(trackp->handle); if (handle) { - nr_entries = stack_depot_fetch(handle, &entries); + nr_entries = stack_depot_fetch_into(handle, entries, ARRAY_SIZE(entries)); for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) kpp->kp_stack[i] = (void *)entries[i]; } @@ -8163,7 +8163,7 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) trackp = get_track(s, objp, TRACK_FREE); handle = READ_ONCE(trackp->handle); if (handle) { - nr_entries = stack_depot_fetch(handle, &entries); + nr_entries = stack_depot_fetch_into(handle, entries, ARRAY_SIZE(entries)); for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) kpp->kp_free_stack[i] = (void *)entries[i]; } @@ -9933,12 +9933,14 @@ static int slab_debugfs_show(struct seq_file *seq, void *v) #ifdef CONFIG_STACKDEPOT { depot_stack_handle_t handle; - unsigned long *entries; + unsigned long entries[TRACK_ADDRS_COUNT]; unsigned int nr_entries, j; handle = READ_ONCE(l->handle); if (handle) { - nr_entries = stack_depot_fetch(handle, &entries); + nr_entries = + stack_depot_fetch_into(handle, entries, + ARRAY_SIZE(entries)); seq_puts(seq, "\n"); for (j = 0; j < nr_entries; j++) seq_printf(seq, " %pS\n", (void *)entries[j]); From 94f15d74d549d94b4815061bdd0ac1ca01a4349b Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 8 Jul 2026 12:50:10 +0100 Subject: [PATCH 02/38] KRN-1117: Separate stackdepot put validation checks Keep the GET-only stack_depot_put() misuse warning visually separate from the corrupt-handle check so the final port diff is easier to review. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 4eb141383c73e..b1733e322365e 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2935,6 +2935,7 @@ void stack_depot_put(depot_stack_handle_t handle) */ if (WARN(!stack, "corrupt handle or unbalanced stack_depot_put()")) return; + if (WARN_ON_ONCE(!(stack->flags & STACK_DEPOT_FLAG_GET))) return; if (refcount_dec_and_test(&stack->count)) From 799719be2a365fd6ba9d7f949da00d3f05b63086 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 9 Jul 2026 15:46:40 +0100 Subject: [PATCH 03/38] KRN-1117: Tighten stackdepot fetch_into misuse handling Do not let stack_depot_fetch_into() silently accept caller-owned output buffer misuse for a valid handle. A NULL destination should fault naturally, and a zero-sized destination is now reported as an invalid caller contract instead of being treated as a normal missing stack. Keep the existing no-stack and undersized-buffer semantics intact, but warn when new trie code is asked to encode an impossible leaf ID or materializes an impossible zero-length stack. Update the KUnit coverage and API comment to match the tightened contract. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 8 +++++--- lib/stackdepot.c | 16 +++++++++++----- lib/tests/stackdepot_kunit.c | 12 ++---------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 700e0e7faa44c..e71be7ef4063a 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -233,6 +233,9 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * is returned. If more frames are stored than @max_entries, the copy is skipped * entirely and 0 is returned. * + * Passing a NULL @entries buffer or zero @max_entries for a valid @handle is + * invalid. Callers must provide storage for @max_entries frames. + * * Callers should size @entries to match the save-side stack depth cap (for * example, %CONFIG_STACKDEPOT_MAX_FRAMES or the local stack_trace_save() limit) * when losing diagnostics on an undersized buffer would be surprising. @@ -246,9 +249,8 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * Callers must not call stack_depot_put() on persistent handles. * Racing this helper with stack_depot_put() on the same handle is invalid. * - * Return: Number of frames copied, 0 if @entries is NULL, @max_entries is 0, - * @handle is 0 or invalid, stack depot is disabled, or @max_entries is less - * than the number of stored frames. + * Return: Number of frames copied, 0 if @handle is 0 or invalid, stack depot is + * disabled, or @max_entries is less than the number of stored frames. */ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, unsigned long *entries, diff --git a/lib/stackdepot.c b/lib/stackdepot.c index b1733e322365e..f0c51ce27c560 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -238,13 +238,13 @@ static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) u32 pool_delta; u32 index; - if (!leaf_id || leaf_id > trie_side_table_max_id) + if (WARN_ON_ONCE(!leaf_id || leaf_id > trie_side_table_max_id)) return 0; index = leaf_id - 1; pool_delta = index >> DEPOT_OFFSET_BITS; pool_index_plus_1 = (u64)stack_max_pools + 1 + pool_delta; - if (pool_index_plus_1 >= DEPOT_POOL_INDEX_MASK) + if (WARN_ON_ONCE(pool_index_plus_1 >= DEPOT_POOL_INDEX_MASK)) return 0; parts.pool_index_plus_1 = pool_index_plus_1; @@ -2812,6 +2812,8 @@ __stack_depot_trie_fetch_into(const struct stack_depot_trie_node *leaf, total = 0; for (node = leaf; node; node = trie_load_parent(node)) total += node->run.nr_entries; + if (WARN_ON_ONCE(!total)) + return 0; if (max_entries < total) return 0; @@ -2840,7 +2842,7 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, u32 leaf_id; unsigned int nr_entries; - if (!handle || !entries || !max_entries) + if (!handle) return 0; leaf_id = __stack_depot_trie_leaf_id(handle); @@ -2898,10 +2900,12 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, struct stack_record *stack; unsigned int nr_entries; - if (!handle || !entries || !max_entries) + if (!handle) return 0; if (stack_depot_disabled) return 0; + if (WARN_ON_ONCE(!max_entries)) + return 0; if (__stack_depot_trie_leaf_id(handle)) return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); @@ -2910,7 +2914,9 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, if (!stack) return 0; nr_entries = stack->size; - if (!nr_entries || nr_entries > max_entries) + if (WARN_ON_ONCE(!nr_entries)) + return 0; + if (nr_entries > max_entries) return 0; memcpy(entries, stack->entries, nr_entries * sizeof(*entries)); diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 1e9e61277b218..2f35ae836615c 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -49,7 +49,7 @@ static void stackdepot_fetch_into_roundtrip(struct kunit *test) KUNIT_EXPECT_EQ(test, fetched[ARRAY_SIZE(entries)], expected_tail); } -static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) +static void stackdepot_fetch_into_rejects_missing_or_short_stack(struct kunit *test) { unsigned long entries[] = { 0x1234567800110000UL, @@ -78,14 +78,6 @@ static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) nr_entries = stack_depot_fetch_into(0, NULL, 0); KUNIT_EXPECT_EQ(test, nr_entries, 0U); - nr_entries = stack_depot_fetch_into(handle, NULL, ARRAY_SIZE(fetched)); - KUNIT_EXPECT_EQ(test, nr_entries, 0U); - KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); - - nr_entries = stack_depot_fetch_into(handle, fetched, 0); - KUNIT_EXPECT_EQ(test, nr_entries, 0U); - KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); - nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched) - 1); KUNIT_EXPECT_EQ(test, nr_entries, 0U); @@ -357,7 +349,7 @@ static void stackdepot_frame_arm64(struct kunit *test) static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), - KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), + KUNIT_CASE(stackdepot_fetch_into_rejects_missing_or_short_stack), KUNIT_CASE(stackdepot_countable_flag_roundtrip), KUNIT_CASE(stackdepot_save_flags_public), KUNIT_CASE(stackdepot_snprint_public), From ddc92c82c77692bd7e48a0f18fc74ec8724e51a0 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 10 Jul 2026 14:37:50 +0100 Subject: [PATCH 04/38] KRN-1117: Simplify stackdepot trie state management Reduce the trie port's state and failure surface while retaining the pool-backed reuse and live tail append required for its memory savings. Type topology pointers as RCU data, combine leaf ID and slot preparation, remove duplicate frame-run work and scratch storage, and return leaf IDs directly from insertion. Simplify retired-object quarantine to one FIFO and overlay mutually exclusive header state. Preserve base hash behavior outside the COUNTABLE distinction, reserve trie handle space on large-page kernels, and derive side-table geometry from PAGE_SIZE. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 6 +- lib/stackdepot.c | 561 ++++++++++++++----------------------- 2 files changed, 219 insertions(+), 348 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index e71be7ef4063a..9cf773de84560 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -152,9 +152,9 @@ static inline int stack_depot_early_init(void) { return 0; } * exclusive with %STACK_DEPOT_FLAG_GET. * * When trie storage is enabled, persistent non-refcounted saves use trie - * storage. Constrained contexts remain best effort and can return 0 if a - * required trylock or preallocated backing storage is unavailable; trie - * failures do not fall back to hash storage. + * storage. Constrained callers only look up existing stacks and perform a + * trylocked recheck; they do not insert a missing stack. Trie failures do not + * fall back to hash storage. * * If the provided stack trace comes from the interrupt context, only the part * up to the interrupt entry is saved. diff --git a/lib/stackdepot.c b/lib/stackdepot.c index f0c51ce27c560..3af80f3a60838 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -86,7 +86,7 @@ static LIST_HEAD(free_stacks); /* The lock must be held when performing pool or freelist modifications. */ static DEFINE_RAW_SPINLOCK(pool_lock); -/* Statistics counters for debugfs. */ +/* Hash-backend statistics counters for debugfs. */ enum depot_counter_id { DEPOT_COUNTER_REFD_ALLOCS, DEPOT_COUNTER_REFD_FREES, @@ -102,8 +102,8 @@ static const char *const counter_names[] = { [DEPOT_COUNTER_REFD_FREES] = "refcounted_frees", [DEPOT_COUNTER_REFD_INUSE] = "refcounted_in_use", [DEPOT_COUNTER_FREELIST_SIZE] = "freelist_size", - [DEPOT_COUNTER_PERSIST_COUNT] = "persistent_count", - [DEPOT_COUNTER_PERSIST_BYTES] = "persistent_bytes", + [DEPOT_COUNTER_PERSIST_COUNT] = "hash_persistent_count", + [DEPOT_COUNTER_PERSIST_BYTES] = "hash_persistent_bytes", }; static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); @@ -129,9 +129,9 @@ struct stack_depot_trie_child_array; struct stack_depot_trie_node { /* Parent links let fetch rebuild a full stack from a leaf to the root. */ - const struct stack_depot_trie_node *parent; + const struct stack_depot_trie_node __rcu *parent; /* Child arrays are separate RCU/COW generations. */ - const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_child_array __rcu *children; u32 leaf_id; struct stack_depot_frame_run run; unsigned char data[]; @@ -145,7 +145,7 @@ struct stack_depot_trie_node { struct stack_depot_trie_child_array { unsigned int nr_children; unsigned int capacity; - const struct stack_depot_trie_node *children[]; + const struct stack_depot_trie_node __rcu *children[]; }; /* Headerless reusable storage for trie nodes. */ @@ -160,9 +160,13 @@ struct stack_depot_trie_free_node { */ struct stack_depot_trie_free_object { struct list_head list; - unsigned long rcu_state; - size_t size; - struct stack_depot_trie_node *pending_node; + union { + size_t size; + struct { + unsigned long rcu_state; + struct stack_depot_trie_node *pending_node; + }; + }; }; static_assert(sizeof(struct stack_depot_trie_node) >= @@ -171,18 +175,21 @@ static_assert(sizeof(struct stack_depot_trie_node) >= #define STACK_DEPOT_TRIE_MAX_NODES (CONFIG_STACKDEPOT_MAX_FRAMES + 1) #define STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS CONFIG_STACKDEPOT_MAX_FRAMES +/* Size classes bucket reusable trie storage by aligned allocation size. */ +#define STACK_DEPOT_TRIE_FREE_CLASSES \ + ((DEPOT_POOL_SIZE >> DEPOT_STACK_ALIGN) + 1) + struct stack_depot_trie_alloc_workspace { struct stack_depot_trie_node *nodes[STACK_DEPOT_TRIE_MAX_NODES]; size_t node_sizes[STACK_DEPOT_TRIE_MAX_NODES]; struct stack_depot_trie_child_array *child_arrays[STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS]; - u32 scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; struct stack_depot_trie_child_array *split_child_array; struct stack_depot_trie_child_array *child_array_storage; }; static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); -static const struct stack_depot_trie_child_array *stack_depot_trie_root; -static struct stack_depot_trie_alloc_workspace __rcu *stack_depot_trie_workspace; +static const struct stack_depot_trie_child_array __rcu *stack_depot_trie_root; +static struct stack_depot_trie_alloc_workspace *stack_depot_trie_workspace; static DEFINE_RAW_SPINLOCK(stack_depot_trie_writer_lock); static bool stack_depot_trie_requested; @@ -192,10 +199,6 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); #define DEPOT_POOL_INDEX_MASK ((1U << DEPOT_POOL_INDEX_BITS) - 1) #define DEPOT_OFFSET_MASK ((1U << DEPOT_OFFSET_BITS) - 1) -/* Size classes bucket reusable trie storage by aligned allocation size. */ -#define STACK_DEPOT_TRIE_FREE_CLASSES \ - ((DEPOT_POOL_SIZE >> DEPOT_STACK_ALIGN) + 1) - /* * Trie storage is suballocated from stackdepot pools, not slab caches, so pool * pressure stays visible through stack_depot_max_pools and no-spin callers can @@ -206,10 +209,9 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); * period has elapsed. */ static struct list_head free_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; -static struct list_head pending_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; static struct list_head free_trie_nodes[STACK_DEPOT_TRIE_FREE_CLASSES]; +static LIST_HEAD(pending_trie_objects); static DECLARE_BITMAP(free_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); -static DECLARE_BITMAP(pending_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); static DECLARE_BITMAP(free_trie_node_map, STACK_DEPOT_TRIE_FREE_CLASSES); static u32 trie_side_table_max_id; @@ -231,6 +233,14 @@ static u32 __stack_depot_trie_max_leaf_id(void) return min_t(u64, max_id, U32_MAX); } +static void stack_depot_trie_reserve_handle_space(void) +{ + /* Reserve one pool-index value for trie leaf IDs when none is available. */ + if (stack_depot_trie_requested && + stack_max_pools >= DEPOT_POOL_INDEX_MASK - 1) + stack_max_pools = DEPOT_POOL_INDEX_MASK - 2; +} + static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) { union handle_parts parts = {}; @@ -238,14 +248,9 @@ static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) u32 pool_delta; u32 index; - if (WARN_ON_ONCE(!leaf_id || leaf_id > trie_side_table_max_id)) - return 0; - index = leaf_id - 1; pool_delta = index >> DEPOT_OFFSET_BITS; pool_index_plus_1 = (u64)stack_max_pools + 1 + pool_delta; - if (WARN_ON_ONCE(pool_index_plus_1 >= DEPOT_POOL_INDEX_MASK)) - return 0; parts.pool_index_plus_1 = pool_index_plus_1; parts.offset = index & DEPOT_OFFSET_MASK; @@ -276,17 +281,16 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) /* * Trie handles encode a dense leaf ID. The side table maps that ID to a leaf * pointer for lockless fetch/print paths, which can run from diagnostic - * contexts where taking trie_side_table_lock would be unsafe. Init installs the - * root and first chunk only; additional directories/chunks are preallocated and - * published lazily as leaf IDs grow. RCU pointer publication makes fully - * initialized dirs, chunks, and leaves visible to those lockless readers. + * contexts where taking trie_side_table_lock would be unsafe. Initialization + * installs the root; early initialization also installs the first directory and + * chunk. Additional directories and chunks are preallocated and published + * lazily as leaf IDs grow. RCU pointer publication makes fully initialized + * directories, chunks, and leaves visible to those lockless readers. */ -#define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ - (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) -#define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS 9 + (PAGE_SIZE / sizeof(struct stack_depot_trie_node *)) #define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE \ - (1U << STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS) + (PAGE_SIZE / sizeof(struct stack_depot_trie_node **)) struct stack_depot_trie_side_dir { const struct stack_depot_trie_node __rcu * __rcu * @@ -305,7 +309,7 @@ struct stack_depot_trie_side_prealloc { const struct stack_depot_trie_node __rcu **chunk; }; -static struct stack_depot_trie_side_root __rcu *trie_side_table_root; +static struct stack_depot_trie_side_root *trie_side_table_root; static DEFINE_RAW_SPINLOCK(trie_side_table_lock); static u32 trie_side_table_last_leaf_id; @@ -318,9 +322,6 @@ static inline bool __stack_depot_trie_enabled(void) static void stack_depot_trie_enable(void) { - if (__stack_depot_trie_enabled()) - return; - static_branch_enable(&stack_depot_trie_enabled); } @@ -374,61 +375,38 @@ static void trie_child_array_init(void *storage, unsigned int capacity, array->nr_children = nr_children; array->capacity = capacity; for (i = 0; i < nr_children; i++) - array->children[i] = nodes[i]; + RCU_INIT_POINTER(array->children[i], nodes[i]); for (i = nr_children; i < capacity; i++) - array->children[i] = NULL; -} - -static inline struct stack_depot_trie_alloc_workspace *stack_depot_trie_load_workspace(void) -{ - /* Installed once and never freed; acquire the init-time publication. */ - return rcu_dereference_check(stack_depot_trie_workspace, true); -} - -static inline struct stack_depot_trie_side_root *trie_side_table_load_root(void) -{ - /* Installed once and never freed; acquire the init-time publication. */ - return rcu_dereference_check(trie_side_table_root, true); -} - -static inline bool trie_side_table_is_initialized(void) -{ - return !!trie_side_table_load_root(); -} - -static inline bool __stack_depot_trie_ready(void) -{ - return __stack_depot_trie_enabled() && - stack_depot_trie_load_workspace() && - trie_side_table_is_initialized(); + RCU_INIT_POINTER(array->children[i], NULL); } static inline unsigned int trie_side_table_top_index(u32 id) { - return (id - 1) >> STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS; + return (id - 1) / STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE; } static inline unsigned int trie_side_table_root_index(u32 id) { - return trie_side_table_top_index(id) >> STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS; + return trie_side_table_top_index(id) / + STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE; } static inline unsigned int trie_side_table_dir_index(u32 id) { - return trie_side_table_top_index(id) & - (STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE - 1); + return trie_side_table_top_index(id) % + STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE; } static inline unsigned int trie_side_table_slot_index(u32 id) { - return (id - 1) & (STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE - 1); + return (id - 1) % STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE; } static struct stack_depot_trie_side_dir *trie_side_table_load_dir(unsigned int root) { struct stack_depot_trie_side_root *root_vec; - root_vec = trie_side_table_load_root(); + root_vec = trie_side_table_root; if (!root_vec || root >= root_vec->dir_capacity) return NULL; /* Pairs with side-table directory rcu_assign_pointer(). */ @@ -447,23 +425,8 @@ trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, rcu_read_lock_sched_held()); } -static u32 trie_side_table_next_leaf_id(void) -{ - unsigned long flags; - u32 id = 0; - - raw_spin_lock_irqsave(&trie_side_table_lock, flags); - id = trie_side_table_last_leaf_id + 1; - /* ID zero wraps the 32-bit counter; max_id is trie handle capacity. */ - if (!id || id > trie_side_table_max_id) - id = 0; - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return id; -} - -static bool -trie_side_table_ensure_leaf_slot(u32 id, - struct stack_depot_trie_side_prealloc *prealloc) +static u32 +trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *prealloc) { const struct stack_depot_trie_node __rcu **chunk; struct stack_depot_trie_side_dir *dir; @@ -471,16 +434,21 @@ trie_side_table_ensure_leaf_slot(u32 id, unsigned long flags; unsigned int root; unsigned int idx; - bool ret = false; + u32 id; raw_spin_lock_irqsave(&trie_side_table_lock, flags); - root_vec = trie_side_table_load_root(); + id = trie_side_table_last_leaf_id + 1; + /* ID zero wraps the 32-bit counter; max_id is trie handle capacity. */ + if (!id || id > trie_side_table_max_id) + goto out_fail; + + root_vec = trie_side_table_root; root = trie_side_table_root_index(id); dir = trie_side_table_load_dir(root); if (!dir) { /* Sparse growth preallocation can lose a race to another writer. */ if (!prealloc->dir) - goto out; + goto out_fail; dir = prealloc->dir; prealloc->dir = NULL; /* Publish the zeroed directory before readers can load it locklessly. */ @@ -492,19 +460,21 @@ trie_side_table_ensure_leaf_slot(u32 id, if (!chunk) { /* Sparse growth preallocation can lose a race to another writer. */ if (!prealloc->chunk) - goto out; + goto out_fail; chunk = prealloc->chunk; prealloc->chunk = NULL; rcu_assign_pointer(dir->chunks[idx], chunk); } - ret = true; + goto out; +out_fail: + id = 0; out: raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return ret; + return id; } -static size_t trie_side_table_root_bytes(unsigned int root_size) +static inline size_t trie_side_table_root_bytes(unsigned int root_size) { size_t bytes; @@ -516,7 +486,7 @@ static size_t trie_side_table_root_bytes(unsigned int root_size) static inline size_t trie_side_table_dir_bytes(void) { - return PAGE_ALIGN(sizeof(struct stack_depot_trie_side_dir)); + return PAGE_SIZE; } static inline unsigned int trie_side_table_dir_order(void) @@ -526,8 +496,7 @@ static inline unsigned int trie_side_table_dir_order(void) static inline size_t trie_side_table_chunk_bytes(void) { - return PAGE_ALIGN(STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE * - sizeof(struct stack_depot_trie_node *)); + return PAGE_SIZE; } static inline unsigned int trie_side_table_chunk_order(void) @@ -568,9 +537,6 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) u32 max_leaf_id; unsigned int root_size; - if (trie_side_table_is_initialized()) - return 0; - max_leaf_id = __stack_depot_trie_max_leaf_id(); if (!max_leaf_id) return -EINVAL; @@ -602,7 +568,7 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) trie_side_table_root_init(root_vec, root_size, max_leaf_id); RCU_INIT_POINTER(root_vec->dirs[0], first_dir); RCU_INIT_POINTER(first_dir->chunks[0], first_chunk); - rcu_assign_pointer(trie_side_table_root, root_vec); + trie_side_table_root = root_vec; return 0; } @@ -613,9 +579,6 @@ static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) size_t root_bytes; u32 max_leaf_id; - if (trie_side_table_is_initialized()) - return 0; - max_leaf_id = __stack_depot_trie_max_leaf_id(); if (!max_leaf_id) return -EINVAL; @@ -629,7 +592,7 @@ static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) return -ENOMEM; trie_side_table_root_init(root_vec, root_size, max_leaf_id); - rcu_assign_pointer(trie_side_table_root, root_vec); + trie_side_table_root = root_vec; return 0; } @@ -639,35 +602,32 @@ static void trie_free_object_buckets_init(void) for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) { INIT_LIST_HEAD(&free_trie_objects[i]); - INIT_LIST_HEAD(&pending_trie_objects[i]); INIT_LIST_HEAD(&free_trie_nodes[i]); } + INIT_LIST_HEAD(&pending_trie_objects); } static int __init stack_depot_trie_init_memblock(void) { - struct stack_depot_trie_alloc_workspace *workspace = NULL; + struct stack_depot_trie_alloc_workspace *workspace; size_t size; int ret; - if (__stack_depot_trie_ready()) + if (__stack_depot_trie_enabled()) return 0; - if (!stack_depot_trie_load_workspace()) { - size = sizeof(*stack_depot_trie_workspace); - workspace = memblock_alloc(size, __alignof__(*workspace)); - if (!workspace) - return -ENOMEM; - memset(workspace, 0, size); - } + size = sizeof(*stack_depot_trie_workspace); + workspace = memblock_alloc(size, __alignof__(*workspace)); + if (!workspace) + return -ENOMEM; + memset(workspace, 0, size); + ret = __stack_depot_trie_side_table_init_memblock(); if (ret) { - if (workspace) - memblock_free(workspace, size); + memblock_free(workspace, size); return ret; } - if (workspace) - rcu_assign_pointer(stack_depot_trie_workspace, workspace); + stack_depot_trie_workspace = workspace; trie_free_object_buckets_init(); stack_depot_trie_enable(); @@ -676,39 +636,28 @@ static int __init stack_depot_trie_init_memblock(void) static int stack_depot_trie_init(gfp_t gfp_flags) { - struct stack_depot_trie_alloc_workspace *workspace = NULL; + struct stack_depot_trie_alloc_workspace *workspace; int ret; - if (__stack_depot_trie_ready()) + if (__stack_depot_trie_enabled()) return 0; - if (!stack_depot_trie_load_workspace()) { - workspace = kvzalloc(sizeof(*stack_depot_trie_workspace), gfp_flags); - if (!workspace) - return -ENOMEM; - } + workspace = kvzalloc(sizeof(*stack_depot_trie_workspace), gfp_flags); + if (!workspace) + return -ENOMEM; + ret = __stack_depot_trie_side_table_init(gfp_flags); if (ret) { kvfree(workspace); return ret; } - if (workspace) - rcu_assign_pointer(stack_depot_trie_workspace, workspace); + stack_depot_trie_workspace = workspace; trie_free_object_buckets_init(); stack_depot_trie_enable(); return 0; } -static inline const struct stack_depot_trie_node * -trie_side_table_load_leaf(const struct stack_depot_trie_node __rcu **slot) -{ - /* Pairs with side-table leaf rcu_assign_pointer(). */ - return rcu_dereference_check(*slot, - lockdep_is_held(&trie_side_table_lock) || - rcu_read_lock_sched_held()); -} - static void *trie_side_table_alloc_page(gfp_t gfp_flags, unsigned int order) { struct page *page; @@ -793,11 +742,6 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( struct stack_depot_trie_side_dir *dir; unsigned int root; - if (!id) - return NULL; - if (!trie_side_table_is_initialized()) - return NULL; - root = trie_side_table_root_index(id); dir = trie_side_table_load_dir(root); if (!dir) @@ -806,7 +750,10 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( if (!chunk) return NULL; - return trie_side_table_load_leaf(&chunk[trie_side_table_slot_index(id)]); + /* Pairs with side-table leaf rcu_assign_pointer(). */ + return rcu_dereference_check(chunk[trie_side_table_slot_index(id)], + lockdep_is_held(&trie_side_table_lock) || + rcu_read_lock_sched_held()); } static size_t __stack_depot_trie_pool_alloc_size(size_t size) @@ -862,14 +809,11 @@ static inline unsigned int trie_free_class(size_t size) } static void trie_free_list_add(struct list_head *entry, struct list_head *heads, - unsigned long *map, unsigned int class, bool tail) + unsigned long *map, unsigned int class) { lockdep_assert_held(&pool_lock); - if (tail) - list_add_tail(entry, &heads[class]); - else - list_add(entry, &heads[class]); + list_add(entry, &heads[class]); __set_bit(class, map); } @@ -900,13 +844,11 @@ static void *trie_object_init_fresh(void *ptr, size_t size) struct stack_depot_trie_free_object *free = ptr; free->size = __stack_depot_trie_pool_alloc_size(size); - free->rcu_state = 0; - free->pending_node = NULL; INIT_LIST_HEAD(&free->list); return trie_object_payload(free); } -static void trie_free_object_locked(const void *ptr, unsigned long rcu_state) +static void trie_free_object_locked(const void *ptr) { struct stack_depot_trie_free_object *free; unsigned int class; @@ -914,15 +856,10 @@ static void trie_free_object_locked(const void *ptr, unsigned long rcu_state) lockdep_assert_held(&pool_lock); free = trie_object_header(ptr); - free->rcu_state = rcu_state; class = trie_free_class(free->size); INIT_LIST_HEAD(&free->list); - if (poll_state_synchronize_rcu(rcu_state)) - trie_free_list_add(&free->list, free_trie_objects, - free_trie_object_map, class, false); - else - trie_free_list_add(&free->list, pending_trie_objects, - pending_trie_object_map, class, true); + trie_free_list_add(&free->list, free_trie_objects, + free_trie_object_map, class); } static void trie_add_free_node_locked(void *ptr, size_t size) @@ -939,7 +876,7 @@ static void trie_add_free_node_locked(void *ptr, size_t size) INIT_LIST_HEAD(&free->list); class = trie_free_class(size); trie_free_list_add(&free->list, free_trie_nodes, free_trie_node_map, - class, false); + class); } static void trie_drain_free_object_node_locked(struct stack_depot_trie_free_object *free) @@ -957,26 +894,24 @@ static void trie_drain_free_object_node_locked(struct stack_depot_trie_free_obje static void trie_drain_pending_objects_locked(void) { + struct stack_depot_trie_child_array *array; struct stack_depot_trie_free_object *free; struct stack_depot_trie_free_object *tmp; unsigned int class; lockdep_assert_held(&pool_lock); - for (class = trie_find_next_set_class(pending_trie_object_map, 0); - class < STACK_DEPOT_TRIE_FREE_CLASSES; - class = trie_find_next_set_class(pending_trie_object_map, class + 1)) { - list_for_each_entry_safe(free, tmp, &pending_trie_objects[class], list) { - /* Pending lists are FIFO; later entries cannot be ready yet. */ - if (!poll_state_synchronize_rcu(free->rcu_state)) - break; - trie_drain_free_object_node_locked(free); - trie_free_list_del(&free->list, pending_trie_objects, - pending_trie_object_map, class); - free->rcu_state = 0; - trie_free_list_add(&free->list, free_trie_objects, - free_trie_object_map, class, false); - } + list_for_each_entry_safe(free, tmp, &pending_trie_objects, list) { + /* Pending objects are FIFO; later entries cannot be ready yet. */ + if (!poll_state_synchronize_rcu(free->rcu_state)) + break; + trie_drain_free_object_node_locked(free); + array = trie_object_payload(free); + free->size = trie_child_array_size_for_capacity(array->capacity); + class = trie_free_class(free->size); + list_del_init(&free->list); + trie_free_list_add(&free->list, free_trie_objects, + free_trie_object_map, class); } } @@ -991,12 +926,10 @@ static void trie_free_object_tail_locked(void *ptr, size_t size) if (size >= header_size + (1UL << DEPOT_STACK_ALIGN)) { tail_size = size - header_size; free->size = tail_size; - free->rcu_state = get_completed_synchronize_rcu(); - free->pending_node = NULL; INIT_LIST_HEAD(&free->list); trie_free_list_add(&free->list, free_trie_objects, - free_trie_object_map, trie_free_class(tail_size), - false); + free_trie_object_map, + trie_free_class(tail_size)); return; } @@ -1035,9 +968,7 @@ static void trie_retire_child_array_locked(const void *ptr) free = trie_object_header(ptr); free->pending_node = NULL; free->rcu_state = get_state_synchronize_rcu(); - trie_free_list_add(&free->list, pending_trie_objects, - pending_trie_object_map, - trie_free_class(free->size), true); + list_add_tail(&free->list, &pending_trie_objects); } static void @@ -1073,10 +1004,8 @@ static void *trie_pop_free_object(size_t size) return NULL; free = list_first_entry(&free_trie_objects[class], typeof(*free), list); - trie_drain_free_object_node_locked(free); trie_free_list_del(&free->list, free_trie_objects, free_trie_object_map, class); - free->rcu_state = 0; old_size = free->size; if (old_size > size) { free->size = size; @@ -1160,7 +1089,6 @@ static int trie_pool_carve(struct stack_depot_trie_alloc_workspace *workspace, size_t child_array_size) { unsigned long flags; - unsigned long completed; size_t one_child_size; unsigned int i; size_t alloc_size; @@ -1243,7 +1171,6 @@ static int trie_pool_carve(struct stack_depot_trie_alloc_workspace *workspace, ret = 0; goto out; out_discard: - completed = get_completed_synchronize_rcu(); for (i = 0; i < nr_nodes; i++) { struct stack_depot_trie_node *node = workspace->nodes[i]; @@ -1257,15 +1184,15 @@ static int trie_pool_carve(struct stack_depot_trie_alloc_workspace *workspace, if (!array) continue; - trie_free_object_locked(array, completed); + trie_free_object_locked(array); workspace->child_arrays[i] = NULL; } if (split_child_array_size && workspace->split_child_array) { - trie_free_object_locked(workspace->split_child_array, completed); + trie_free_object_locked(workspace->split_child_array); workspace->split_child_array = NULL; } if (child_array_size && workspace->child_array_storage) { - trie_free_object_locked(workspace->child_array_storage, completed); + trie_free_object_locked(workspace->child_array_storage); workspace->child_array_storage = NULL; } out: @@ -1275,11 +1202,11 @@ static int trie_pool_carve(struct stack_depot_trie_alloc_workspace *workspace, } static const struct stack_depot_trie_node * -stack_depot_trie_lookup(const struct stack_depot_trie_child_array * const *root_slot, +stack_depot_trie_lookup(const struct stack_depot_trie_child_array __rcu * const *root_slot, const unsigned long *entries, unsigned int nr_entries); static depot_stack_handle_t -trie_find_handle(const struct stack_depot_trie_child_array * const *root_slot, +trie_find_handle(const struct stack_depot_trie_child_array __rcu * const *root_slot, const unsigned long *entries, unsigned int nr_entries) { depot_stack_handle_t handle = 0; @@ -1302,7 +1229,7 @@ static void trie_side_table_publish_new_leaf(u32 leaf_id, raw_spin_lock_irqsave(&trie_side_table_lock, flags); slot = trie_side_table_leaf_slot(leaf_id); - /* Pairs with trie_side_table_load_leaf(). */ + /* Pairs with __stack_depot_trie_side_table_lookup(). */ rcu_assign_pointer(*slot, leaf); raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); } @@ -1322,11 +1249,11 @@ static void trie_side_table_publish_split_leaves(u32 old_leaf_id, new_slot = trie_side_table_leaf_slot(new_leaf_id); if (old_slot) { - /* Pairs with trie_side_table_load_leaf(). */ + /* Pairs with __stack_depot_trie_side_table_lookup(). */ rcu_assign_pointer(*old_slot, old_leaf); } - /* Pairs with trie_side_table_load_leaf(). */ + /* Pairs with __stack_depot_trie_side_table_lookup(). */ rcu_assign_pointer(*new_slot, new_leaf); raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); } @@ -1382,7 +1309,7 @@ static void init_stack_table(unsigned long entries) INIT_LIST_HEAD(&stack_table[i]); } -/* Allocates a hash table via memblock. Can only be used during early boot. */ +/* Initializes hash and optional trie storage during early boot. */ int __init stack_depot_early_init(void) { unsigned long entries = 0; @@ -1446,6 +1373,7 @@ int __init stack_depot_early_init(void) } init_stack_table(entries); + stack_depot_trie_reserve_handle_space(); pr_info("allocating space for %u stack pools via memblock\n", stack_max_pools); stack_pools = @@ -1464,7 +1392,7 @@ int __init stack_depot_early_init(void) return 0; } -/* Allocates a hash table via kvcalloc. Can be used after boot. */ +/* Initializes hash and optional trie storage after boot. */ int stack_depot_init(void) { static DEFINE_MUTEX(stack_depot_init_mutex); @@ -1512,6 +1440,7 @@ int stack_depot_init(void) stack_hash_mask = entries - 1; init_stack_table(entries); + stack_depot_trie_reserve_handle_space(); pr_info("allocating space for %u stack pools via kvcalloc\n", stack_max_pools); stack_pools = kvcalloc(stack_max_pools, sizeof(void *), GFP_KERNEL); @@ -1523,7 +1452,7 @@ int stack_depot_init(void) goto out_unlock; } init_trie: - if (!ret && stack_depot_trie_requested) { + if (stack_depot_trie_requested) { ret = stack_depot_trie_init(GFP_KERNEL); if (ret) { pr_warn("trie storage initialization failed, disabling trie storage\n"); @@ -1714,7 +1643,7 @@ depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, dep /* Save the stack trace. */ stack->hash = hash; stack->size = nr_entries; - stack->flags = flags & STACK_DEPOT_FLAGS_MASK; + stack->flags = flags & STACK_DEPOT_FLAG_COUNTABLE; /* stack->handle is already filled in by depot_pop_free_pool(). */ memcpy(stack->entries, entries, flex_array_size(stack, entries, nr_entries)); @@ -1838,7 +1767,6 @@ static inline struct stack_record *find_stack(struct list_head *bucket, unsigned long *entries, int size, u32 hash, depot_flags_t flags) { - depot_flags_t mode = STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE; struct stack_record *stack, *ret = NULL; /* @@ -1855,8 +1783,8 @@ static inline struct stack_record *find_stack(struct list_head *bucket, list_for_each_entry_rcu(stack, bucket, hash_list) { if (stack->hash != hash || stack->size != size) continue; - /* Plain, refcounted, and countable records have distinct lifetimes. */ - if ((stack->flags & mode) != (flags & mode)) + /* Page owner countable records have a distinct count lifetime. */ + if ((stack->flags ^ flags) & STACK_DEPOT_FLAG_COUNTABLE) continue; /* @@ -1887,14 +1815,12 @@ static inline struct stack_record *find_stack(struct list_head *bucket, return ret; } -static int -stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_slot, +static u32 +stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu **root_slot, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace, - u32 *leaf_id, - u32 *scratch); + struct stack_depot_trie_alloc_workspace *workspace); static depot_stack_handle_t stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, @@ -1910,7 +1836,7 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, u32 leaf_id; int ret; - workspace = stack_depot_trie_load_workspace(); + workspace = stack_depot_trie_workspace; can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && gfpflags_allow_spinning(alloc_flags); @@ -1925,6 +1851,7 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, * spinning or publishing a new leaf. */ if (in_nmi() || !gfpflags_allow_spinning(alloc_flags)) { + WARN_ON_ONCE(can_alloc); if (!raw_spin_trylock_irqsave(&stack_depot_trie_writer_lock, flags)) return 0; handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); @@ -1940,15 +1867,14 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); memset(workspace, 0, sizeof(*workspace)); - ret = stack_depot_trie_insert_locked(&stack_depot_trie_root, - entries, nr_entries, - &pool_prealloc, &side_prealloc, - workspace, &leaf_id, - workspace->scratch); - if (!ret) + leaf_id = stack_depot_trie_insert_locked(&stack_depot_trie_root, + entries, nr_entries, + &pool_prealloc, &side_prealloc, + workspace); + if (leaf_id) handle = __stack_depot_trie_handle(leaf_id); raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); - if (!handle && ret == -ENOSPC && can_alloc && !retried) { + if (!handle && can_alloc && !retried) { retried = true; if (pool_prealloc) { raw_spin_lock_irqsave(&pool_lock, flags); @@ -2011,7 +1937,7 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, return 0; trie_candidate = !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE)) && - __stack_depot_trie_ready(); + __stack_depot_trie_enabled(); if (trie_candidate) { if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; @@ -2161,29 +2087,29 @@ stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, static void trie_node_init(void *storage, const struct stack_depot_trie_node *parent, u32 leaf_id, - const unsigned long *entries, unsigned int nr_entries, - u32 *scratch) + const unsigned long *entries, + const struct stack_depot_frame_run *run) { struct stack_depot_trie_node *node = storage; - struct stack_depot_frame_run run; - - frame_run_init(entries, nr_entries, &run); - /* Caller-owned storage is not publishable unless the payload write succeeds. */ - if (run.mode == STACK_DEPOT_FRAME_COMPRESSED) { + if (run->mode == STACK_DEPOT_FRAME_COMPRESSED) { unsigned int i; - for (i = 0; i < run.nr_entries; i++) - arch_stack_depot_frame_try_compress(entries[i], &scratch[i]); - memcpy(node->data, scratch, stack_depot_frame_run_bytes(&run)); + for (i = 0; i < run->nr_entries; i++) { + u32 payload; + + arch_stack_depot_frame_try_compress(entries[i], &payload); + memcpy(node->data + i * sizeof(payload), &payload, + sizeof(payload)); + } } else { - memcpy(node->data, entries, stack_depot_frame_run_bytes(&run)); + memcpy(node->data, entries, stack_depot_frame_run_bytes(run)); } - node->parent = parent; - node->children = NULL; + RCU_INIT_POINTER(node->parent, parent); + RCU_INIT_POINTER(node->children, NULL); node->leaf_id = leaf_id; - node->run = run; + node->run = *run; } static void trie_node_init_slice(void *storage, @@ -2201,8 +2127,8 @@ static void trie_node_init_slice(void *storage, entry_bytes = stack_depot_frame_run_entry_bytes(src_node->run.mode); memcpy(node->data, src_node->data + start * entry_bytes, stack_depot_frame_run_bytes(&run)); - node->parent = parent; - node->children = NULL; + RCU_INIT_POINTER(node->parent, parent); + RCU_INIT_POINTER(node->children, NULL); node->leaf_id = leaf_id; node->run = run; } @@ -2242,21 +2168,15 @@ __stack_depot_trie_node_match(const struct stack_depot_trie_node *node, static inline const struct stack_depot_trie_node * trie_load_parent(const struct stack_depot_trie_node *node) { - const struct stack_depot_trie_node __rcu * const *slot; - - slot = (const struct stack_depot_trie_node __rcu * const *)&node->parent; - return rcu_dereference_check(*slot, + return rcu_dereference_check(node->parent, lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } static inline const struct stack_depot_trie_child_array * -trie_load_children_slot(const struct stack_depot_trie_child_array * const *slot) +trie_load_children_slot(const struct stack_depot_trie_child_array __rcu * const *slot) { - const struct stack_depot_trie_child_array __rcu * const *rcu_slot; - - rcu_slot = (const struct stack_depot_trie_child_array __rcu * const *)slot; - return rcu_dereference_check(*rcu_slot, + return rcu_dereference_check(*slot, lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } @@ -2265,10 +2185,7 @@ static inline const struct stack_depot_trie_node * trie_child_array_load_child(const struct stack_depot_trie_child_array *array, unsigned int pos) { - const struct stack_depot_trie_node __rcu * const *slot; - - slot = (const struct stack_depot_trie_node __rcu * const *)&array->children[pos]; - return rcu_dereference_check(*slot, + return rcu_dereference_check(array->children[pos], lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } @@ -2325,22 +2242,22 @@ trie_child_array_insert_at(const struct stack_depot_trie_child_array *old, new_array->nr_children = nr_old + 1; new_array->capacity = new_capacity; for (i = 0; i < pos; i++) - new_array->children[i] = trie_child_array_load_child(old, i); - new_array->children[pos] = node; + RCU_INIT_POINTER(new_array->children[i], + trie_child_array_load_child(old, i)); + RCU_INIT_POINTER(new_array->children[pos], node); for (i = pos; i < nr_old; i++) - new_array->children[i + 1] = trie_child_array_load_child(old, i); + RCU_INIT_POINTER(new_array->children[i + 1], + trie_child_array_load_child(old, i)); for (i = nr_old + 1; i < new_array->capacity; i++) - new_array->children[i] = NULL; + RCU_INIT_POINTER(new_array->children[i], NULL); } static void -trie_publish_children_slot(const struct stack_depot_trie_child_array **slot, +trie_publish_children_slot(const struct stack_depot_trie_child_array __rcu **slot, const struct stack_depot_trie_child_array *children) { - const struct stack_depot_trie_child_array __rcu **rcu_slot; - - rcu_slot = (const struct stack_depot_trie_child_array __rcu **)slot; - rcu_assign_pointer(*rcu_slot, children); + /* Publish the fully initialized replacement array last. */ + rcu_assign_pointer(*slot, children); } static void @@ -2354,33 +2271,32 @@ trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array new_array->nr_children = old_array->nr_children; new_array->capacity = old_array->capacity; for (i = 0; i < old_array->nr_children; i++) - new_array->children[i] = trie_child_array_load_child(old_array, i); - new_array->children[pos] = new_child; + RCU_INIT_POINTER(new_array->children[i], + trie_child_array_load_child(old_array, i)); + RCU_INIT_POINTER(new_array->children[pos], new_child); for (i = old_array->nr_children; i < new_array->capacity; i++) - new_array->children[i] = NULL; + RCU_INIT_POINTER(new_array->children[i], NULL); } static void trie_reparent_children(struct stack_depot_trie_node *parent) { - const struct stack_depot_trie_child_array *children = parent->children; + const struct stack_depot_trie_child_array *children; unsigned int i; + children = trie_load_children_slot(&parent->children); if (!children) return; /* * COW updates reuse unchanged descendant subtrees. Repoint their parent * links before retiring the old parent so fetch never follows a freed node. - * Lockless lookups that still observe the old child array may see the new - * parent early and miss. A writer-lock recheck observes the new array before - * inserting, so the miss cannot create a permanent duplicate. + * Lockless fetches may see the new parent before structural publication, but + * the old and new parent chains contain the same frames and remain RCU-live. */ for (i = 0; i < children->nr_children; i++) { struct stack_depot_trie_node *child; - const struct stack_depot_trie_node __rcu **slot; child = (struct stack_depot_trie_node *)trie_child_array_load_child(children, i); - slot = (const struct stack_depot_trie_node __rcu **)&child->parent; - rcu_assign_pointer(*slot, parent); + rcu_assign_pointer(child->parent, parent); } } @@ -2389,7 +2305,6 @@ trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_node * const *nodes, struct stack_depot_trie_child_array * const *child_arrays, - u32 *scratch, const struct stack_depot_trie_node **head, const struct stack_depot_trie_node **tail) { @@ -2407,8 +2322,7 @@ trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, frame_run_init(&entries[pos], nr_entries - pos, &run); id = pos + run.nr_entries == nr_entries ? leaf_id : 0; - trie_node_init(node, prev, id, &entries[pos], run.nr_entries, - scratch); + trie_node_init(node, prev, id, &entries[pos], &run); prev = node; pos += run.nr_entries; @@ -2421,7 +2335,7 @@ trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, struct stack_depot_trie_child_array *array = child_arrays[i]; trie_child_array_init(array, 1, &next, 1); - node->children = array; + RCU_INIT_POINTER(node->children, array); } *head = nodes[0]; @@ -2432,8 +2346,6 @@ static void trie_publish_tail_append(struct stack_depot_trie_child_array *array, unsigned int pos, const struct stack_depot_trie_node *head) { - const struct stack_depot_trie_node __rcu **slot; - /* * Writers hold stack_depot_trie_writer_lock. Existing children are * immutable, so tail append publishes the new child before increasing the @@ -2441,17 +2353,15 @@ static void trie_publish_tail_append(struct stack_depot_trie_child_array *array, * see the new count either load the initialized child or treat a transient * NULL as a miss. The writer-lock recheck prevents permanent duplicates. */ - slot = (const struct stack_depot_trie_node __rcu **)&array->children[pos]; - rcu_assign_pointer(*slot, head); + rcu_assign_pointer(array->children[pos], head); WRITE_ONCE(array->nr_children, pos + 1); } static const struct stack_depot_trie_node * -stack_depot_trie_lookup(const struct stack_depot_trie_child_array * const *root_slot, +stack_depot_trie_lookup(const struct stack_depot_trie_child_array __rcu * const *root_slot, const unsigned long *entries, unsigned int nr_entries) { const struct stack_depot_trie_child_array *children; - const struct stack_depot_trie_node *parent = NULL; unsigned int pos = 0; children = trie_load_children_slot(root_slot); @@ -2472,8 +2382,6 @@ stack_depot_trie_lookup(const struct stack_depot_trie_child_array * const *root_ node = trie_child_array_load_child(children, slot); if (!node) return NULL; - if (trie_load_parent(node) != parent) - return NULL; matched = __stack_depot_trie_node_match(node, &entries[pos], remaining); if (!matched || matched < node->run.nr_entries) @@ -2482,7 +2390,6 @@ stack_depot_trie_lookup(const struct stack_depot_trie_child_array * const *root_ if (pos == nr_entries) return node->leaf_id ? node : NULL; - parent = node; children = trie_load_children_slot(&node->children); } @@ -2495,7 +2402,6 @@ static void trie_build_split(const struct stack_depot_trie_node *child, struct stack_depot_trie_node * const *nodes, struct stack_depot_trie_child_array * const *child_arrays, struct stack_depot_trie_child_array *split_child_array, - u32 *scratch, const struct stack_depot_trie_node **prefix, const struct stack_depot_trie_node **old_tail, const struct stack_depot_trie_node **new_leaf) @@ -2531,7 +2437,7 @@ static void trie_build_split(const struct stack_depot_trie_node *child, tail_entries = &entries[matched]; tail_len = nr_entries - matched; trie_build_append_chain(pref, leaf_id, tail_entries, tail_len, - &nodes[2], child_arrays, scratch, + &nodes[2], child_arrays, &new_head, &new_tail); stack_depot_trie_node_frame(old_tail_node, 0, &old_frame); stack_depot_trie_node_frame(new_head, 0, &new_frame); @@ -2548,8 +2454,8 @@ static void trie_build_split(const struct stack_depot_trie_node *child, split_children[0] = old_tail_node; trie_child_array_init(split_child_array, 1, split_children, 1); } - old_tail_node->children = child_children; - pref->children = split_child_array; + RCU_INIT_POINTER(old_tail_node->children, child_children); + RCU_INIT_POINTER(pref->children, split_child_array); *prefix = pref; *old_tail = old_tail_node; *new_leaf = has_new_tail ? new_tail : pref; @@ -2633,19 +2539,17 @@ trie_pool_alloc_split(struct stack_depot_trie_alloc_workspace *workspace, split_child_array_size, child_array_size); } -static int -stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_slot, +static u32 +stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu **root_slot, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace, - u32 *leaf_id, - u32 *scratch) + struct stack_depot_trie_alloc_workspace *workspace) { struct stack_depot_trie_child_array *new_array; struct stack_depot_trie_child_array *split_child_array; const struct stack_depot_trie_child_array *children; - const struct stack_depot_trie_child_array **slot = root_slot; + const struct stack_depot_trie_child_array __rcu **slot = root_slot; const struct stack_depot_trie_node *child; const struct stack_depot_trie_node *head; const struct stack_depot_trie_node *last; @@ -2659,30 +2563,26 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_ u32 new_leaf_id; size_t child_array_size; bool found; - int ret; for (;;) { children = trie_load_children_slot(slot); if (!children) { - new_leaf_id = trie_side_table_next_leaf_id(); + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) - return -ENOSPC; - if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) - return -ENOSPC; + return 0; child_array_size = __stack_depot_trie_child_array_size(1); - ret = trie_pool_alloc_append_chain(workspace, pool_prealloc, entries, - nr_entries, child_array_size); - if (ret) - return ret; + if (trie_pool_alloc_append_chain(workspace, pool_prealloc, + entries, nr_entries, + child_array_size)) + return 0; new_array = workspace->child_array_storage; trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, workspace->nodes, workspace->child_arrays, - scratch, &head, &last); + &head, &last); trie_side_table_publish_new_leaf(new_leaf_id, last); new_array->nr_children = 1; new_array->capacity = 1; - new_array->children[0] = head; - /* Publish the fully initialized replacement array last. */ + RCU_INIT_POINTER(new_array->children[0], head); trie_publish_children_slot(slot, new_array); goto out_success; } @@ -2694,21 +2594,19 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_ tail_append = pos == children->nr_children && children->nr_children < children->capacity; - new_leaf_id = trie_side_table_next_leaf_id(); + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) - return -ENOSPC; - if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) - return -ENOSPC; + return 0; child_array_size = tail_append ? 0 : __stack_depot_trie_child_array_size(children->nr_children + 1); - ret = trie_pool_alloc_append_chain(workspace, pool_prealloc, entries, - nr_entries, child_array_size); - if (ret) - return ret; + if (trie_pool_alloc_append_chain(workspace, pool_prealloc, + entries, nr_entries, + child_array_size)) + return 0; new_array = workspace->child_array_storage; trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, workspace->nodes, workspace->child_arrays, - scratch, &head, &last); + &head, &last); trie_side_table_publish_new_leaf(new_leaf_id, last); if (tail_append) { tail_array = (struct stack_depot_trie_child_array *)children; @@ -2719,7 +2617,6 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_ capacity = trie_child_array_capacity(children->nr_children + 1); trie_child_array_insert_at(children, pos, head, new_array, capacity); - /* Publish the fully initialized replacement array last. */ trie_publish_children_slot(slot, new_array); raw_spin_lock_irqsave(&pool_lock, flags); trie_retire_child_array_locked(children); @@ -2731,46 +2628,37 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_ child = trie_child_array_load_child(children, pos); matched = __stack_depot_trie_node_match(child, entries, nr_entries); if (matched < child->run.nr_entries) { - new_leaf_id = trie_side_table_next_leaf_id(); + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) - return -ENOSPC; - if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) - return -ENOSPC; - ret = trie_pool_alloc_split(workspace, pool_prealloc, children, child, - matched, entries, nr_entries); - if (ret) - return ret; + return 0; + if (trie_pool_alloc_split(workspace, pool_prealloc, children, + child, matched, entries, nr_entries)) + return 0; new_array = workspace->child_array_storage; split_child_array = workspace->split_child_array; trie_build_split(child, matched, new_leaf_id, entries, nr_entries, workspace->nodes, workspace->child_arrays, - split_child_array, scratch, &head, &old_tail, + split_child_array, &head, &old_tail, &new_leaf); trie_side_table_publish_split_leaves(child->leaf_id, old_tail, new_leaf_id, new_leaf); trie_child_array_replace_at(children, head, new_array, pos); trie_reparent_children((struct stack_depot_trie_node *)old_tail); - /* Publish the fully initialized replacement array last. */ trie_publish_children_slot(slot, new_array); trie_retire_child_array_with_node(children, child); goto out_success; } if (matched == nr_entries) { - if (child->leaf_id) { - *leaf_id = child->leaf_id; - return 0; - } - new_leaf_id = trie_side_table_next_leaf_id(); + if (child->leaf_id) + return child->leaf_id; + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) - return -ENOSPC; - if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) - return -ENOSPC; + return 0; workspace->node_sizes[0] = __stack_depot_trie_node_size(&child->run); child_array_size = trie_child_array_size_for_capacity(children->capacity); - ret = trie_pool_carve(workspace, pool_prealloc, 1, 0, 0, - child_array_size); - if (ret) - return ret; + if (trie_pool_carve(workspace, pool_prealloc, 1, 0, 0, + child_array_size)) + return 0; new_node = workspace->nodes[0]; new_array = workspace->child_array_storage; memcpy(new_node, child, __stack_depot_trie_node_size(&child->run)); @@ -2778,7 +2666,6 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_ trie_side_table_publish_new_leaf(new_leaf_id, new_node); trie_child_array_replace_at(children, new_node, new_array, pos); trie_reparent_children(new_node); - /* Publish the fully initialized replacement array last. */ trie_publish_children_slot(slot, new_array); trie_retire_child_array_with_node(children, child); goto out_success; @@ -2794,8 +2681,7 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_ raw_spin_lock_irqsave(&trie_side_table_lock, flags); trie_side_table_last_leaf_id = new_leaf_id; raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - *leaf_id = new_leaf_id; - return 0; + return new_leaf_id; } static unsigned int @@ -2805,7 +2691,6 @@ __stack_depot_trie_fetch_into(const struct stack_depot_trie_node *leaf, { const struct stack_depot_trie_node *node; unsigned int total; - unsigned int seen = 0; unsigned int pos; unsigned int i; @@ -2819,16 +2704,10 @@ __stack_depot_trie_fetch_into(const struct stack_depot_trie_node *leaf, pos = total; for (node = leaf; node; node = trie_load_parent(node)) { - if (node->run.nr_entries > pos) - return 0; pos -= node->run.nr_entries; - for (i = 0; i < node->run.nr_entries; i++) { + for (i = 0; i < node->run.nr_entries; i++) stack_depot_trie_node_frame(node, i, &entries[pos + i]); - seen++; - } } - if (seen != total || pos) - return 0; return total; } @@ -2842,13 +2721,7 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, u32 leaf_id; unsigned int nr_entries; - if (!handle) - return 0; - leaf_id = __stack_depot_trie_leaf_id(handle); - if (!leaf_id) - return 0; - rcu_read_lock_sched_notrace(); leaf = __stack_depot_trie_side_table_lookup(leaf_id); if (WARN_ONCE(!leaf, "corrupt trie handle %08x\n", handle)) { @@ -2904,8 +2777,6 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, return 0; if (stack_depot_disabled) return 0; - if (WARN_ON_ONCE(!max_entries)) - return 0; if (__stack_depot_trie_leaf_id(handle)) return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); @@ -2942,7 +2813,7 @@ void stack_depot_put(depot_stack_handle_t handle) if (WARN(!stack, "corrupt handle or unbalanced stack_depot_put()")) return; - if (WARN_ON_ONCE(!(stack->flags & STACK_DEPOT_FLAG_GET))) + if (WARN_ON_ONCE(stack->flags & STACK_DEPOT_FLAG_COUNTABLE)) return; if (refcount_dec_and_test(&stack->count)) depot_free_stack(stack); From 6fe0cb5154e14e4d90bbd654ee222bad2af768a2 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 10 Jul 2026 14:38:44 +0100 Subject: [PATCH 05/38] KRN-1117: Make stackdepot KUnit fixtures portable Use synthetic addresses that fit in unsigned long on 32-bit builds so the new public API tests compile on every supported architecture. Declare the suite's three-frame configuration requirement instead of allowing valid one-frame or two-frame configurations to run tests whose fixtures would be truncated. Signed-off-by: Caleb Kan --- lib/Kconfig.debug | 1 + lib/tests/stackdepot_kunit.c | 46 ++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 3a1e7c9e6c1bd..ee281e7197251 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2709,6 +2709,7 @@ config RESOURCE_KUNIT_TEST config STACKDEPOT_KUNIT_TEST bool "KUnit test for stack depot" if !KUNIT_ALL_TESTS depends on KUNIT=y && STACKDEPOT + depends on STACKDEPOT_MAX_FRAMES >= 3 default KUNIT_ALL_TESTS help Enable this option to test stack depot API behavior at boot. diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 2f35ae836615c..a7e1209861984 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -22,13 +22,13 @@ static inline unsigned long stackdepot_arm64_frame(long offset) static void stackdepot_fetch_into_roundtrip(struct kunit *test) { unsigned long entries[] = { - 0x1234567800010000UL, - 0x1234567800020000UL, - 0x1234567800030000UL, + 0x101000UL, + 0x102000UL, + 0x103000UL, }; unsigned long exact[ARRAY_SIZE(entries)] = {}; unsigned long fetched[ARRAY_SIZE(entries) + 1] = { - [ARRAY_SIZE(entries)] = 0xa5a5a5a5a5a5a5a5UL, + [ARRAY_SIZE(entries)] = 0xa5a5a5a5UL, }; unsigned long expected_tail = fetched[ARRAY_SIZE(entries)]; depot_stack_handle_t handle; @@ -52,14 +52,14 @@ static void stackdepot_fetch_into_roundtrip(struct kunit *test) static void stackdepot_fetch_into_rejects_missing_or_short_stack(struct kunit *test) { unsigned long entries[] = { - 0x1234567800110000UL, - 0x1234567800120000UL, - 0x1234567800130000UL, + 0x111000UL, + 0x112000UL, + 0x113000UL, }; unsigned long fetched[ARRAY_SIZE(entries)] = { - 0xa1a1a1a1a1a1a1a1UL, - 0xb2b2b2b2b2b2b2b2UL, - 0xc3c3c3c3c3c3c3c3UL, + 0xa1a1a1a1UL, + 0xb2b2b2b2UL, + 0xc3c3c3c3UL, }; unsigned long expected[ARRAY_SIZE(fetched)]; depot_stack_handle_t handle; @@ -94,9 +94,9 @@ static depot_stack_handle_t save_countable(unsigned long *entries, unsigned int static void stackdepot_countable_flag_roundtrip(struct kunit *test) { unsigned long entries[] = { - 0x1234567800210000UL, - 0x1234567800220000UL, - 0x1234567800230000UL, + 0x121000UL, + 0x122000UL, + 0x123000UL, }; unsigned long fetched[ARRAY_SIZE(entries)] = {}; depot_stack_handle_t handle; @@ -216,9 +216,9 @@ static void stackdepot_snprint_public(struct kunit *test) static void stackdepot_get_stack_record(struct kunit *test) { unsigned long entries[] = { - 0x1234567800310000UL, - 0x1234567800320000UL, - 0x1234567800330000UL, + 0x131000UL, + 0x132000UL, + 0x133000UL, }; struct stack_record *record; depot_stack_handle_t handle; @@ -237,14 +237,14 @@ static void stackdepot_get_stack_record(struct kunit *test) static void stackdepot_countable_does_not_alias_other_modes(struct kunit *test) { unsigned long plain_entries[] = { - 0x1234567800410000UL, - 0x1234567800420000UL, - 0x1234567800430000UL, + 0x141000UL, + 0x142000UL, + 0x143000UL, }; unsigned long get_entries[] = { - 0x1234567800510000UL, - 0x1234567800520000UL, - 0x1234567800530000UL, + 0x151000UL, + 0x152000UL, + 0x153000UL, }; depot_flags_t get = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; struct stack_record *record; @@ -278,7 +278,7 @@ static void stackdepot_countable_does_not_alias_other_modes(struct kunit *test) static void stackdepot_frame_raw_fallback(struct kunit *test) { - unsigned long frame = 0xffff888000001000UL; + unsigned long frame = 0x1000UL; bool compressed; u32 payload = 0xfeedbeef; From 6fbddb5664558d6cd414b8ab52e6e797cb2f3296 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 10 Jul 2026 14:39:22 +0100 Subject: [PATCH 06/38] KRN-1117: Minimize the KMSAN fetch_into conversion Keep the single KMSAN stack buffer required for bounded report-path stack usage, but remove the redundant capacity alias and zero-result branch from the caller conversion. Restore the existing explicit void-pointer cast so the port does not carry unrelated cleanup in the KMSAN test. Signed-off-by: Caleb Kan --- mm/kmsan/kmsan_test.c | 3 ++- mm/kmsan/report.c | 16 +++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/mm/kmsan/kmsan_test.c b/mm/kmsan/kmsan_test.c index 123bcc9565721..da4d3a56b8d97 100644 --- a/mm/kmsan/kmsan_test.c +++ b/mm/kmsan/kmsan_test.c @@ -624,7 +624,8 @@ static void test_stackdepot_roundtrip(struct kunit *test) dst_nentries = stack_depot_fetch_into(handle, dst_entries, ARRAY_SIZE(dst_entries)); KUNIT_EXPECT_TRUE(test, src_nentries == dst_nentries); - kmsan_check_memory(dst_entries, sizeof(*dst_entries) * dst_nentries); + kmsan_check_memory((void *)dst_entries, + sizeof(*dst_entries) * dst_nentries); KUNIT_EXPECT_TRUE(test, report_matches(&expect)); } diff --git a/mm/kmsan/report.c b/mm/kmsan/report.c index 8d7b86b38d84c..c20c24cffde55 100644 --- a/mm/kmsan/report.c +++ b/mm/kmsan/report.c @@ -86,7 +86,6 @@ static char *pretty_descr(char *descr) void kmsan_print_origin(depot_stack_handle_t origin) { unsigned long entries[KMSAN_STACK_DEPTH]; - const unsigned int max_entries = ARRAY_SIZE(entries); unsigned int nr_entries, chained_nr_entries, skipnr; void *pc1 = NULL, *pc2 = NULL; depot_stack_handle_t head; @@ -98,7 +97,8 @@ void kmsan_print_origin(depot_stack_handle_t origin) return; while (true) { - nr_entries = stack_depot_fetch_into(origin, entries, max_entries); + nr_entries = + stack_depot_fetch_into(origin, entries, ARRAY_SIZE(entries)); depth = kmsan_depth_from_eb(stack_depot_get_extra_bits(origin)); magic = nr_entries ? entries[0] : 0; if ((nr_entries == 4) && (magic == KMSAN_ALLOCA_MAGIC_ORIGIN)) { @@ -123,14 +123,12 @@ void kmsan_print_origin(depot_stack_handle_t origin) head = entries[1]; origin = entries[2]; pr_err("Uninit was stored to memory at:\n"); - /* Reuse entries after saving head and origin above. */ chained_nr_entries = - stack_depot_fetch_into(head, entries, max_entries); - if (chained_nr_entries) { - skipnr = get_stack_skipnr(entries, chained_nr_entries); - stack_trace_print(entries + skipnr, - chained_nr_entries - skipnr, 0); - } + stack_depot_fetch_into(head, entries, + ARRAY_SIZE(entries)); + skipnr = get_stack_skipnr(entries, chained_nr_entries); + stack_trace_print(entries + skipnr, + chained_nr_entries - skipnr, 0); pr_err("\n"); continue; } From 1bba11f1111ad9d9f27e76d3bfbeab7c60915bf4 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 10 Jul 2026 14:40:06 +0100 Subject: [PATCH 07/38] KRN-1117: Reject trie handles in the stackdepot GDB helper The GDB helper can only materialize contiguous hash-backed stack records. Detect the trie handle range explicitly instead of reporting a valid trie handle as an out-of-bounds hash pool index. Signed-off-by: Caleb Kan --- scripts/gdb/linux/stackdepot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/gdb/linux/stackdepot.py b/scripts/gdb/linux/stackdepot.py index 37313a5a51a0d..82aeb9f532c3d 100644 --- a/scripts/gdb/linux/stackdepot.py +++ b/scripts/gdb/linux/stackdepot.py @@ -37,6 +37,10 @@ def stack_depot_fetch(handle): if handle == 0: raise gdb.GdbError("handle is 0\n") + stack_max_pools = gdb.parse_and_eval('stack_max_pools') + if parts['pool_index_plus_1'] > stack_max_pools: + raise gdb.GdbError("trie-backed stack depot handles are not supported\n") + pool_index = parts['pool_index_plus_1'] - 1 if pool_index >= pools_num: gdb.write("pool index %d out of bounds (%d) for stack id 0x%08x\n" % (parts['pool_index'], pools_num, handle)) From 4da813bf7684d59ecc9bcf307e3e6152a61571e4 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 13 Jul 2026 11:55:26 +0100 Subject: [PATCH 08/38] KRN-1117: Preserve hash capacity when trie init fails Calculate the hash pool limit required for trie handles without changing the active limit, and size the side table from that prospective value. Commit the reduced limit only after all fallible trie initialization is complete. This leaves the hash backend at its configured capacity when optional trie initialization fails. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 3af80f3a60838..ff919a3a335d1 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -221,24 +221,25 @@ static u32 trie_side_table_max_id; * pool_index_plus_1 > stack_max_pools and reinterpret the remaining handle * bits as a dense leaf_id, which the side table maps to a trie leaf. */ -static u32 __stack_depot_trie_max_leaf_id(void) +static u32 __stack_depot_trie_max_leaf_id(unsigned int max_pools) { u64 max_id; - if (stack_max_pools >= DEPOT_POOL_INDEX_MASK - 1) + if (max_pools >= DEPOT_POOL_INDEX_MASK - 1) return 0; - max_id = (u64)(DEPOT_POOL_INDEX_MASK - stack_max_pools - 1) << + max_id = (u64)(DEPOT_POOL_INDEX_MASK - max_pools - 1) << DEPOT_OFFSET_BITS; return min_t(u64, max_id, U32_MAX); } -static void stack_depot_trie_reserve_handle_space(void) +static unsigned int stack_depot_trie_hash_max_pools(void) { /* Reserve one pool-index value for trie leaf IDs when none is available. */ - if (stack_depot_trie_requested && - stack_max_pools >= DEPOT_POOL_INDEX_MASK - 1) - stack_max_pools = DEPOT_POOL_INDEX_MASK - 2; + if (stack_max_pools >= DEPOT_POOL_INDEX_MASK - 1) + return DEPOT_POOL_INDEX_MASK - 2; + + return stack_max_pools; } static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) @@ -320,8 +321,10 @@ static inline bool __stack_depot_trie_enabled(void) return static_branch_unlikely(&stack_depot_trie_enabled); } -static void stack_depot_trie_enable(void) +static void stack_depot_trie_enable(unsigned int max_pools) { + /* Commit the handle split only after all trie state is ready. */ + stack_max_pools = max_pools; static_branch_enable(&stack_depot_trie_enabled); } @@ -526,7 +529,8 @@ static inline unsigned int trie_side_table_root_size_for_max_id(u32 max_leaf_id) return DIV_ROUND_UP(top_size, STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE); } -static int __init __stack_depot_trie_side_table_init_memblock(void) +static int __init +__stack_depot_trie_side_table_init_memblock(unsigned int max_pools) { struct stack_depot_trie_side_root *root_vec; struct stack_depot_trie_side_dir *first_dir; @@ -537,7 +541,7 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) u32 max_leaf_id; unsigned int root_size; - max_leaf_id = __stack_depot_trie_max_leaf_id(); + max_leaf_id = __stack_depot_trie_max_leaf_id(max_pools); if (!max_leaf_id) return -EINVAL; root_size = trie_side_table_root_size_for_max_id(max_leaf_id); @@ -572,14 +576,15 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) return 0; } -static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) +static int __stack_depot_trie_side_table_init(gfp_t gfp_flags, + unsigned int max_pools) { struct stack_depot_trie_side_root *root_vec; unsigned int root_size; size_t root_bytes; u32 max_leaf_id; - max_leaf_id = __stack_depot_trie_max_leaf_id(); + max_leaf_id = __stack_depot_trie_max_leaf_id(max_pools); if (!max_leaf_id) return -EINVAL; @@ -610,11 +615,13 @@ static void trie_free_object_buckets_init(void) static int __init stack_depot_trie_init_memblock(void) { struct stack_depot_trie_alloc_workspace *workspace; + unsigned int max_pools; size_t size; int ret; if (__stack_depot_trie_enabled()) return 0; + max_pools = stack_depot_trie_hash_max_pools(); size = sizeof(*stack_depot_trie_workspace); workspace = memblock_alloc(size, __alignof__(*workspace)); @@ -622,7 +629,7 @@ static int __init stack_depot_trie_init_memblock(void) return -ENOMEM; memset(workspace, 0, size); - ret = __stack_depot_trie_side_table_init_memblock(); + ret = __stack_depot_trie_side_table_init_memblock(max_pools); if (ret) { memblock_free(workspace, size); return ret; @@ -630,23 +637,25 @@ static int __init stack_depot_trie_init_memblock(void) stack_depot_trie_workspace = workspace; trie_free_object_buckets_init(); - stack_depot_trie_enable(); + stack_depot_trie_enable(max_pools); return 0; } static int stack_depot_trie_init(gfp_t gfp_flags) { struct stack_depot_trie_alloc_workspace *workspace; + unsigned int max_pools; int ret; if (__stack_depot_trie_enabled()) return 0; + max_pools = stack_depot_trie_hash_max_pools(); workspace = kvzalloc(sizeof(*stack_depot_trie_workspace), gfp_flags); if (!workspace) return -ENOMEM; - ret = __stack_depot_trie_side_table_init(gfp_flags); + ret = __stack_depot_trie_side_table_init(gfp_flags, max_pools); if (ret) { kvfree(workspace); return ret; @@ -654,7 +663,7 @@ static int stack_depot_trie_init(gfp_t gfp_flags) stack_depot_trie_workspace = workspace; trie_free_object_buckets_init(); - stack_depot_trie_enable(); + stack_depot_trie_enable(max_pools); return 0; } @@ -1373,7 +1382,6 @@ int __init stack_depot_early_init(void) } init_stack_table(entries); - stack_depot_trie_reserve_handle_space(); pr_info("allocating space for %u stack pools via memblock\n", stack_max_pools); stack_pools = @@ -1440,7 +1448,6 @@ int stack_depot_init(void) stack_hash_mask = entries - 1; init_stack_table(entries); - stack_depot_trie_reserve_handle_space(); pr_info("allocating space for %u stack pools via kvcalloc\n", stack_max_pools); stack_pools = kvcalloc(stack_max_pools, sizeof(void *), GFP_KERNEL); From 0fcb66090cfe45482ea7520e519b5eaf8fd176f3 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 13 Jul 2026 16:15:32 +0100 Subject: [PATCH 09/38] KRN-1117: Surface invalid stackdepot fetch buffers Warn when a nonzero stackdepot handle is fetched with a NULL or zero-capacity output buffer so caller misuse is not silently reported as a missing stack. Keep zero handles and disabled stackdepot as no-ops, preserve the existing undersized-buffer contract, and allow NULL destinations to fault naturally when a copy is attempted. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index ff919a3a335d1..1ed5be026a23d 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2784,6 +2784,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, return 0; if (stack_depot_disabled) return 0; + WARN_ON_ONCE(!entries || !max_entries); if (__stack_depot_trie_leaf_id(handle)) return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); From 32db89c179758995418658a603d1177b21ae9d09 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 14 Jul 2026 14:55:19 +0100 Subject: [PATCH 10/38] KRN-1117: Preserve configured stackdepot pool capacity Use the configured stack_depot_max_pools value as the immutable hash/trie handle split. If no encoded values remain for trie IDs, let optional trie initialization fail instead of silently reducing hash pool capacity. Use the full pool-index field for trie IDs. Allocate the side-table root at its exact size and natural alignment because only directories and chunks require page-sized storage. Remove the redundant fetch capacity aliases. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 70 ++++++++++++------------------------------------ 1 file changed, 17 insertions(+), 53 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 1ed5be026a23d..0b67f545eb856 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -221,27 +221,15 @@ static u32 trie_side_table_max_id; * pool_index_plus_1 > stack_max_pools and reinterpret the remaining handle * bits as a dense leaf_id, which the side table maps to a trie leaf. */ -static u32 __stack_depot_trie_max_leaf_id(unsigned int max_pools) +static u32 __stack_depot_trie_max_leaf_id(void) { u64 max_id; - if (max_pools >= DEPOT_POOL_INDEX_MASK - 1) - return 0; - - max_id = (u64)(DEPOT_POOL_INDEX_MASK - max_pools - 1) << + max_id = (u64)(DEPOT_POOL_INDEX_MASK - stack_max_pools) << DEPOT_OFFSET_BITS; return min_t(u64, max_id, U32_MAX); } -static unsigned int stack_depot_trie_hash_max_pools(void) -{ - /* Reserve one pool-index value for trie leaf IDs when none is available. */ - if (stack_max_pools >= DEPOT_POOL_INDEX_MASK - 1) - return DEPOT_POOL_INDEX_MASK - 2; - - return stack_max_pools; -} - static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) { union handle_parts parts = {}; @@ -269,9 +257,6 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) return 0; pool_delta = parts.pool_index_plus_1 - stack_max_pools - 1; - if ((u64)pool_delta + stack_max_pools + 1 >= DEPOT_POOL_INDEX_MASK) - return 0; - leaf_id = ((u64)pool_delta << DEPOT_OFFSET_BITS) + parts.offset + 1; if (leaf_id > trie_side_table_max_id) return 0; @@ -321,13 +306,6 @@ static inline bool __stack_depot_trie_enabled(void) return static_branch_unlikely(&stack_depot_trie_enabled); } -static void stack_depot_trie_enable(unsigned int max_pools) -{ - /* Commit the handle split only after all trie state is ready. */ - stack_max_pools = max_pools; - static_branch_enable(&stack_depot_trie_enabled); -} - static inline size_t stack_depot_frame_run_entry_bytes(enum stack_depot_frame_mode mode) { if (mode == STACK_DEPOT_FRAME_COMPRESSED) @@ -479,12 +457,7 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo static inline size_t trie_side_table_root_bytes(unsigned int root_size) { - size_t bytes; - - bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); - if (bytes == SIZE_MAX) - return 0; - return PAGE_ALIGN(bytes); + return struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); } static inline size_t trie_side_table_dir_bytes(void) @@ -529,8 +502,7 @@ static inline unsigned int trie_side_table_root_size_for_max_id(u32 max_leaf_id) return DIV_ROUND_UP(top_size, STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE); } -static int __init -__stack_depot_trie_side_table_init_memblock(unsigned int max_pools) +static int __init __stack_depot_trie_side_table_init_memblock(void) { struct stack_depot_trie_side_root *root_vec; struct stack_depot_trie_side_dir *first_dir; @@ -541,17 +513,15 @@ __stack_depot_trie_side_table_init_memblock(unsigned int max_pools) u32 max_leaf_id; unsigned int root_size; - max_leaf_id = __stack_depot_trie_max_leaf_id(max_pools); + max_leaf_id = __stack_depot_trie_max_leaf_id(); if (!max_leaf_id) return -EINVAL; root_size = trie_side_table_root_size_for_max_id(max_leaf_id); root_bytes = trie_side_table_root_bytes(root_size); dir_bytes = trie_side_table_dir_bytes(); chunk_bytes = trie_side_table_chunk_bytes(); - if (!root_bytes || !dir_bytes || !chunk_bytes) - return -ENOMEM; - root_vec = memblock_alloc(root_bytes, PAGE_SIZE); + root_vec = memblock_alloc(root_bytes, __alignof__(*root_vec)); if (!root_vec) return -ENOMEM; memset(root_vec, 0, root_bytes); @@ -576,22 +546,19 @@ __stack_depot_trie_side_table_init_memblock(unsigned int max_pools) return 0; } -static int __stack_depot_trie_side_table_init(gfp_t gfp_flags, - unsigned int max_pools) +static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) { struct stack_depot_trie_side_root *root_vec; unsigned int root_size; size_t root_bytes; u32 max_leaf_id; - max_leaf_id = __stack_depot_trie_max_leaf_id(max_pools); + max_leaf_id = __stack_depot_trie_max_leaf_id(); if (!max_leaf_id) return -EINVAL; root_size = trie_side_table_root_size_for_max_id(max_leaf_id); root_bytes = trie_side_table_root_bytes(root_size); - if (!root_bytes) - return -ENOMEM; root_vec = kvzalloc(root_bytes, gfp_flags); if (!root_vec) return -ENOMEM; @@ -615,13 +582,11 @@ static void trie_free_object_buckets_init(void) static int __init stack_depot_trie_init_memblock(void) { struct stack_depot_trie_alloc_workspace *workspace; - unsigned int max_pools; size_t size; int ret; if (__stack_depot_trie_enabled()) return 0; - max_pools = stack_depot_trie_hash_max_pools(); size = sizeof(*stack_depot_trie_workspace); workspace = memblock_alloc(size, __alignof__(*workspace)); @@ -629,7 +594,7 @@ static int __init stack_depot_trie_init_memblock(void) return -ENOMEM; memset(workspace, 0, size); - ret = __stack_depot_trie_side_table_init_memblock(max_pools); + ret = __stack_depot_trie_side_table_init_memblock(); if (ret) { memblock_free(workspace, size); return ret; @@ -637,25 +602,23 @@ static int __init stack_depot_trie_init_memblock(void) stack_depot_trie_workspace = workspace; trie_free_object_buckets_init(); - stack_depot_trie_enable(max_pools); + static_branch_enable(&stack_depot_trie_enabled); return 0; } static int stack_depot_trie_init(gfp_t gfp_flags) { struct stack_depot_trie_alloc_workspace *workspace; - unsigned int max_pools; int ret; if (__stack_depot_trie_enabled()) return 0; - max_pools = stack_depot_trie_hash_max_pools(); workspace = kvzalloc(sizeof(*stack_depot_trie_workspace), gfp_flags); if (!workspace) return -ENOMEM; - ret = __stack_depot_trie_side_table_init(gfp_flags, max_pools); + ret = __stack_depot_trie_side_table_init(gfp_flags); if (ret) { kvfree(workspace); return ret; @@ -663,7 +626,7 @@ static int stack_depot_trie_init(gfp_t gfp_flags) stack_depot_trie_workspace = workspace; trie_free_object_buckets_init(); - stack_depot_trie_enable(max_pools); + static_branch_enable(&stack_depot_trie_enabled); return 0; } @@ -2835,9 +2798,10 @@ void stack_depot_print(depot_stack_handle_t stack) if (__stack_depot_trie_leaf_id(stack)) { unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; - const unsigned int max_entries = ARRAY_SIZE(trie_entries); - nr_entries = __stack_depot_trie_fetch_handle_into(stack, trie_entries, max_entries); + nr_entries = __stack_depot_trie_fetch_handle_into(stack, + trie_entries, + ARRAY_SIZE(trie_entries)); if (nr_entries) stack_trace_print(trie_entries, nr_entries, 0); return; @@ -2857,10 +2821,10 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, if (__stack_depot_trie_leaf_id(handle)) { unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; - const unsigned int max_entries = ARRAY_SIZE(trie_entries); nr_entries = __stack_depot_trie_fetch_handle_into(handle, - trie_entries, max_entries); + trie_entries, + ARRAY_SIZE(trie_entries)); return nr_entries ? stack_trace_snprint(buf, size, trie_entries, nr_entries, spaces) : 0; } From 0ffd8509e429ba2be5af7b2eb3d1a98143e96454 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 14 Jul 2026 15:22:08 +0100 Subject: [PATCH 11/38] KRN-1117: Inline the stackdepot leaf limit helper Mark the pure leaf limit computation inline to match the neighboring stackdepot size and index helpers and make its role explicit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 0b67f545eb856..e184fc257f806 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -221,7 +221,7 @@ static u32 trie_side_table_max_id; * pool_index_plus_1 > stack_max_pools and reinterpret the remaining handle * bits as a dense leaf_id, which the side table maps to a trie leaf. */ -static u32 __stack_depot_trie_max_leaf_id(void) +static inline u32 __stack_depot_trie_max_leaf_id(void) { u64 max_id; From 61af459f78ba6a51425d8d36482411abda87ea43 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 15 Jul 2026 13:34:56 +0100 Subject: [PATCH 12/38] KRN-1117: Separate stackdepot trie handle classification Separate backend classification from leaf ID decoding so malformed trie handles remain in the trie namespace and warn instead of being interpreted as hash handles. Clarify the recycler, handle encoding, and RCU publication comments. Rename size helpers by their units and remove redundant child-array and side-table index helpers to reduce the number of trie helper functions. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 88 ++++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index e184fc257f806..9815cf9d640c2 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -160,6 +160,11 @@ struct stack_depot_trie_free_node { */ struct stack_depot_trie_free_object { struct list_head list; + /* + * @size is valid for freshly allocated and reusable objects. Retirement + * overwrites it with the fields below, which remain valid while the + * object is on pending_trie_objects. + */ union { size_t size; struct { @@ -217,9 +222,11 @@ static DECLARE_BITMAP(free_trie_node_map, STACK_DEPOT_TRIE_FREE_CLASSES); static u32 trie_side_table_max_id; /* - * Hash handles use pool_index_plus_1 <= stack_max_pools. Trie handles use - * pool_index_plus_1 > stack_max_pools and reinterpret the remaining handle - * bits as a dense leaf_id, which the side table maps to a trie leaf. + * stack_max_pools is the split point between hash and trie handle encodings. + * A handle with pool_index_plus_1 in 1..stack_max_pools names a hash-backed + * stack pool. Larger pool-index values cannot refer to hash pools, so trie + * storage uses that handle space to encode a dense leaf_id. The side table + * maps each leaf_id to its trie leaf node. */ static inline u32 __stack_depot_trie_max_leaf_id(void) { @@ -246,6 +253,13 @@ static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) return parts.handle; } +static inline bool stack_depot_handle_is_trie(depot_stack_handle_t handle) +{ + union handle_parts parts = { .handle = handle }; + + return parts.pool_index_plus_1 > stack_max_pools; +} + static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) { union handle_parts parts = { .handle = handle }; @@ -253,13 +267,9 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) u32 pool_delta; parts.extra = 0; - if (parts.pool_index_plus_1 <= stack_max_pools) - return 0; - pool_delta = parts.pool_index_plus_1 - stack_max_pools - 1; leaf_id = ((u64)pool_delta << DEPOT_OFFSET_BITS) + parts.offset + 1; - if (leaf_id > trie_side_table_max_id) - return 0; + WARN_ON_ONCE(leaf_id > trie_side_table_max_id); return leaf_id; } @@ -279,6 +289,7 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) (PAGE_SIZE / sizeof(struct stack_depot_trie_node **)) struct stack_depot_trie_side_dir { + /* Both the chunk pointer and each leaf pointer in it are RCU-published. */ const struct stack_depot_trie_node __rcu * __rcu * chunks[STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE]; }; @@ -318,7 +329,7 @@ static inline size_t stack_depot_frame_run_bytes(const struct stack_depot_frame_ return run->nr_entries * stack_depot_frame_run_entry_bytes(run->mode); } -static inline size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) +static inline size_t trie_node_bytes(const struct stack_depot_frame_run *run) { return ALIGN(offsetof(struct stack_depot_trie_node, data) + stack_depot_frame_run_bytes(run), sizeof(unsigned long)); @@ -329,7 +340,7 @@ static inline unsigned int trie_child_array_capacity(unsigned int nr_children) return nr_children ? roundup_pow_of_two(nr_children) : 0; } -static size_t trie_child_array_size_for_capacity(unsigned int capacity) +static size_t trie_child_array_bytes(unsigned int capacity) { size_t size; @@ -341,11 +352,6 @@ static size_t trie_child_array_size_for_capacity(unsigned int capacity) return ALIGN(size, sizeof(unsigned long)); } -static inline size_t __stack_depot_trie_child_array_size(unsigned int nr_children) -{ - return trie_child_array_size_for_capacity(trie_child_array_capacity(nr_children)); -} - static void trie_child_array_init(void *storage, unsigned int capacity, const struct stack_depot_trie_node * const *nodes, unsigned int nr_children) @@ -361,20 +367,15 @@ static void trie_child_array_init(void *storage, unsigned int capacity, RCU_INIT_POINTER(array->children[i], NULL); } -static inline unsigned int trie_side_table_top_index(u32 id) -{ - return (id - 1) / STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE; -} - static inline unsigned int trie_side_table_root_index(u32 id) { - return trie_side_table_top_index(id) / + return ((id - 1) / STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE) / STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE; } static inline unsigned int trie_side_table_dir_index(u32 id) { - return trie_side_table_top_index(id) % + return ((id - 1) / STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE) % STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE; } @@ -859,7 +860,7 @@ static void trie_drain_free_object_node_locked(struct stack_depot_trie_free_obje if (!free->pending_node) return; - size = __stack_depot_trie_node_size(&free->pending_node->run); + size = trie_node_bytes(&free->pending_node->run); trie_add_free_node_locked(free->pending_node, size); free->pending_node = NULL; } @@ -879,7 +880,7 @@ static void trie_drain_pending_objects_locked(void) break; trie_drain_free_object_node_locked(free); array = trie_object_payload(free); - free->size = trie_child_array_size_for_capacity(array->capacity); + free->size = trie_child_array_bytes(array->capacity); class = trie_free_class(free->size); list_del_init(&free->list); trie_free_list_add(&free->list, free_trie_objects, @@ -1072,7 +1073,7 @@ static int trie_pool_carve(struct stack_depot_trie_alloc_workspace *workspace, raw_spin_lock_irqsave(&pool_lock, flags); printk_deferred_enter(); trie_drain_pending_objects_locked(); - one_child_size = __stack_depot_trie_child_array_size(1); + one_child_size = trie_child_array_bytes(1); for (i = 0; i < nr_nodes; i++) { workspace->nodes[i] = trie_pop_free_node(workspace->node_sizes[i]); if (!workspace->nodes[i]) { @@ -2005,7 +2006,7 @@ struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle) if (!handle) return NULL; - if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) + if (WARN_ON_ONCE(stack_depot_handle_is_trie(handle))) return NULL; stack = depot_fetch_stack(handle); @@ -2444,7 +2445,7 @@ static void trie_size_append_chain(const unsigned long *entries, struct stack_depot_frame_run run; frame_run_init(&entries[pos], nr_entries - pos, &run); - node_sizes[used] = __stack_depot_trie_node_size(&run); + node_sizes[used] = trie_node_bytes(&run); pos += run.nr_entries; used++; } @@ -2493,16 +2494,16 @@ trie_pool_alloc_split(struct stack_depot_trie_alloc_workspace *workspace, old_tail_run.nr_entries = child->run.nr_entries - matched; has_new_tail = matched < nr_entries; - workspace->node_sizes[0] = __stack_depot_trie_node_size(&prefix_run); - workspace->node_sizes[1] = __stack_depot_trie_node_size(&old_tail_run); + workspace->node_sizes[0] = trie_node_bytes(&prefix_run); + workspace->node_sizes[1] = trie_node_bytes(&old_tail_run); if (has_new_tail) trie_size_append_chain(&entries[matched], nr_entries - matched, &workspace->node_sizes[2], &new_used, &new_child_arrays); - child_array_size = trie_child_array_size_for_capacity(children->capacity); + child_array_size = trie_child_array_bytes(children->capacity); split_child_array_size = - __stack_depot_trie_child_array_size(has_new_tail ? 2 : 1); + trie_child_array_bytes(has_new_tail ? 2 : 1); nr_nodes = 2 + new_used; nr_child_arrays = new_child_arrays; return trie_pool_carve(workspace, pool_prealloc, nr_nodes, nr_child_arrays, @@ -2540,7 +2541,7 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - child_array_size = __stack_depot_trie_child_array_size(1); + child_array_size = trie_child_array_bytes(1); if (trie_pool_alloc_append_chain(workspace, pool_prealloc, entries, nr_entries, child_array_size)) @@ -2559,8 +2560,10 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * trie_child_array_find_slot(children, entries[0], &pos, &found); if (!found) { struct stack_depot_trie_child_array *tail_array; + unsigned int capacity; bool tail_append; + capacity = trie_child_array_capacity(children->nr_children + 1); tail_append = pos == children->nr_children && children->nr_children < children->capacity; @@ -2568,7 +2571,7 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * if (!new_leaf_id) return 0; child_array_size = tail_append ? 0 : - __stack_depot_trie_child_array_size(children->nr_children + 1); + trie_child_array_bytes(capacity); if (trie_pool_alloc_append_chain(workspace, pool_prealloc, entries, nr_entries, child_array_size)) @@ -2582,9 +2585,6 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * tail_array = (struct stack_depot_trie_child_array *)children; trie_publish_tail_append(tail_array, pos, head); } else { - unsigned int capacity; - - capacity = trie_child_array_capacity(children->nr_children + 1); trie_child_array_insert_at(children, pos, head, new_array, capacity); trie_publish_children_slot(slot, new_array); @@ -2624,14 +2624,14 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - workspace->node_sizes[0] = __stack_depot_trie_node_size(&child->run); - child_array_size = trie_child_array_size_for_capacity(children->capacity); + workspace->node_sizes[0] = trie_node_bytes(&child->run); + child_array_size = trie_child_array_bytes(children->capacity); if (trie_pool_carve(workspace, pool_prealloc, 1, 0, 0, child_array_size)) return 0; new_node = workspace->nodes[0]; new_array = workspace->child_array_storage; - memcpy(new_node, child, __stack_depot_trie_node_size(&child->run)); + memcpy(new_node, child, trie_node_bytes(&child->run)); new_node->leaf_id = new_leaf_id; trie_side_table_publish_new_leaf(new_leaf_id, new_node); trie_child_array_replace_at(children, new_node, new_array, pos); @@ -2720,7 +2720,7 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, if (!handle || stack_depot_disabled) return 0; - if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) + if (WARN_ON_ONCE(stack_depot_handle_is_trie(handle))) return 0; stack = depot_fetch_stack(handle); @@ -2748,7 +2748,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, if (stack_depot_disabled) return 0; WARN_ON_ONCE(!entries || !max_entries); - if (__stack_depot_trie_leaf_id(handle)) + if (stack_depot_handle_is_trie(handle)) return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); @@ -2773,7 +2773,7 @@ void stack_depot_put(depot_stack_handle_t handle) if (!handle || stack_depot_disabled) return; - if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) + if (WARN_ON_ONCE(stack_depot_handle_is_trie(handle))) return; stack = depot_fetch_stack(handle); @@ -2796,7 +2796,7 @@ void stack_depot_print(depot_stack_handle_t stack) unsigned long *entries; unsigned int nr_entries; - if (__stack_depot_trie_leaf_id(stack)) { + if (stack_depot_handle_is_trie(stack)) { unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; nr_entries = __stack_depot_trie_fetch_handle_into(stack, @@ -2819,7 +2819,7 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, unsigned long *entries; unsigned int nr_entries; - if (__stack_depot_trie_leaf_id(handle)) { + if (stack_depot_handle_is_trie(handle)) { unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; nr_entries = __stack_depot_trie_fetch_handle_into(handle, From e2eff7775e3d75fe9a0374aefa660c155ca36a92 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 16 Jul 2026 12:44:55 +0100 Subject: [PATCH 13/38] KRN-1117: Simplify stackdepot trie preallocation Route constrained trie saves through a single lookup before insertion so the allocation path can assume complete preallocated resources. Replace speculative side-table topology checks with a cached directory and chunk pair, and defer leaf selection until insertion. Use explicit recycler payload storage and loud allocator invariant checks while keeping expected fanout exhaustion on the normal failure path. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 5 +- lib/stackdepot.c | 171 +++++++++++++++---------------------- 2 files changed, 72 insertions(+), 104 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 9cf773de84560..143b26cb5bb19 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -152,9 +152,8 @@ static inline int stack_depot_early_init(void) { return 0; } * exclusive with %STACK_DEPOT_FLAG_GET. * * When trie storage is enabled, persistent non-refcounted saves use trie - * storage. Constrained callers only look up existing stacks and perform a - * trylocked recheck; they do not insert a missing stack. Trie failures do not - * fall back to hash storage. + * storage. Constrained callers only look up existing stacks; they do not insert + * a missing stack. Trie failures do not fall back to hash storage. * * If the provided stack trace comes from the interrupt context, only the part * up to the interrupt entry is saved. diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 9815cf9d640c2..d2388686a7c0f 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -172,10 +172,13 @@ struct stack_depot_trie_free_object { struct stack_depot_trie_node *pending_node; }; }; + unsigned char data[]; }; static_assert(sizeof(struct stack_depot_trie_node) >= sizeof(struct stack_depot_trie_free_node)); +static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_free_object, data), + 1UL << DEPOT_STACK_ALIGN)); #define STACK_DEPOT_TRIE_MAX_NODES (CONFIG_STACKDEPOT_MAX_FRAMES + 1) #define STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS CONFIG_STACKDEPOT_MAX_FRAMES @@ -206,12 +209,11 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); /* * Trie storage is suballocated from stackdepot pools, not slab caches, so pool - * pressure stays visible through stack_depot_max_pools and no-spin callers can - * fail without allocator recursion. Trie COW insertion retires child arrays - * and sometimes the node they replaced. Objects carry the RCU cookie for - * child-array payloads; headerless node fragments either live directly on - * free_trie_nodes or are attached to a pending object until that object's grace - * period has elapsed. + * pressure stays visible through stack_depot_max_pools. Trie COW insertion + * retires child arrays and sometimes the node they replaced. Objects carry the + * RCU cookie for child-array payloads; headerless node fragments either live + * directly on free_trie_nodes or are attached to a pending object until that + * object's grace period has elapsed. */ static struct list_head free_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; static struct list_head free_trie_nodes[STACK_DEPOT_TRIE_FREE_CLASSES]; @@ -308,6 +310,8 @@ struct stack_depot_trie_side_prealloc { static struct stack_depot_trie_side_root *trie_side_table_root; static DEFINE_RAW_SPINLOCK(trie_side_table_lock); +/* Zeroed unpublished pages; get/put transfer ownership under the lock. */ +static struct stack_depot_trie_side_prealloc trie_side_table_cache; static u32 trie_side_table_last_leaf_id; /* Lock order: writer_lock -> pool_lock -> trie_side_table_lock. */ @@ -428,8 +432,7 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo root = trie_side_table_root_index(id); dir = trie_side_table_load_dir(root); if (!dir) { - /* Sparse growth preallocation can lose a race to another writer. */ - if (!prealloc->dir) + if (WARN_ON_ONCE(!prealloc->dir)) goto out_fail; dir = prealloc->dir; prealloc->dir = NULL; @@ -440,8 +443,7 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo idx = trie_side_table_dir_index(id); chunk = trie_side_table_dir_load_chunk(dir, idx); if (!chunk) { - /* Sparse growth preallocation can lose a race to another writer. */ - if (!prealloc->chunk) + if (WARN_ON_ONCE(!prealloc->chunk)) goto out_fail; chunk = prealloc->chunk; prealloc->chunk = NULL; @@ -640,54 +642,51 @@ static void *trie_side_table_alloc_page(gfp_t gfp_flags, unsigned int order) return page ? page_address(page) : NULL; } -static int -__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, - struct stack_depot_trie_side_prealloc *prealloc) +static int trie_side_table_get_prealloc(gfp_t gfp_flags, + struct stack_depot_trie_side_prealloc *prealloc) { - struct stack_depot_trie_side_dir *dir; unsigned long flags; - bool need_chunk; - bool need_dir; - u32 id; - unsigned int root; raw_spin_lock_irqsave(&trie_side_table_lock, flags); - id = trie_side_table_last_leaf_id + 1; - if (!id || id > trie_side_table_max_id) { - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return 0; - } - root = trie_side_table_root_index(id); - dir = trie_side_table_load_dir(root); - need_dir = !dir; - need_chunk = need_dir || !trie_side_table_dir_load_chunk(dir, - trie_side_table_dir_index(id)); + prealloc->dir = trie_side_table_cache.dir; + prealloc->chunk = trie_side_table_cache.chunk; + trie_side_table_cache.dir = NULL; + trie_side_table_cache.chunk = NULL; raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - if (need_dir) { + if (!prealloc->dir) { unsigned int order = trie_side_table_dir_order(); prealloc->dir = trie_side_table_alloc_page(gfp_flags, order); if (!prealloc->dir) return -ENOMEM; } - if (need_chunk) { + if (!prealloc->chunk) { unsigned int order = trie_side_table_chunk_order(); prealloc->chunk = trie_side_table_alloc_page(gfp_flags, order); - if (!prealloc->chunk) { - trie_side_table_free_dir(prealloc->dir); - prealloc->dir = NULL; + if (!prealloc->chunk) return -ENOMEM; - } } return 0; } -static void -__stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_prealloc *prealloc) +static void trie_side_table_put_prealloc(struct stack_depot_trie_side_prealloc *prealloc) { + unsigned long flags; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + if (!trie_side_table_cache.dir) { + trie_side_table_cache.dir = prealloc->dir; + prealloc->dir = NULL; + } + if (!trie_side_table_cache.chunk) { + trie_side_table_cache.chunk = prealloc->chunk; + prealloc->chunk = NULL; + } + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + trie_side_table_free_dir(prealloc->dir); if (prealloc->chunk) free_pages((unsigned long)prealloc->chunk, @@ -734,7 +733,7 @@ static size_t __stack_depot_trie_pool_alloc_size(size_t size) size_t align = 1UL << DEPOT_STACK_ALIGN; size_t aligned; - if (!size || size > DEPOT_POOL_SIZE) + if (WARN_ON_ONCE(!size || size > DEPOT_POOL_SIZE)) return 0; if (check_add_overflow(size, align - 1, &aligned)) return 0; @@ -744,8 +743,7 @@ static size_t __stack_depot_trie_pool_alloc_size(size_t size) static inline size_t trie_object_header_size(void) { - return ALIGN(sizeof(struct stack_depot_trie_free_object), - 1UL << DEPOT_STACK_ALIGN); + return offsetof(struct stack_depot_trie_free_object, data); } static size_t trie_object_alloc_size(size_t size) @@ -763,13 +761,7 @@ static size_t trie_object_alloc_size(size_t size) static inline struct stack_depot_trie_free_object *trie_object_header(const void *ptr) { - return (struct stack_depot_trie_free_object *)((const char *)ptr - - trie_object_header_size()); -} - -static inline void *trie_object_payload(struct stack_depot_trie_free_object *free) -{ - return (char *)free + trie_object_header_size(); + return container_of(ptr, struct stack_depot_trie_free_object, data); } static inline unsigned int trie_free_class(size_t size) @@ -818,7 +810,7 @@ static void *trie_object_init_fresh(void *ptr, size_t size) free->size = __stack_depot_trie_pool_alloc_size(size); INIT_LIST_HEAD(&free->list); - return trie_object_payload(free); + return free->data; } static void trie_free_object_locked(const void *ptr) @@ -843,7 +835,7 @@ static void trie_add_free_node_locked(void *ptr, size_t size) lockdep_assert_held(&pool_lock); size = __stack_depot_trie_pool_alloc_size(size); - if (size < sizeof(*free)) + if (WARN_ON_ONCE(size < sizeof(*free))) return; INIT_LIST_HEAD(&free->list); @@ -879,7 +871,7 @@ static void trie_drain_pending_objects_locked(void) if (!poll_state_synchronize_rcu(free->rcu_state)) break; trie_drain_free_object_node_locked(free); - array = trie_object_payload(free); + array = (struct stack_depot_trie_child_array *)free->data; free->size = trie_child_array_bytes(array->capacity); class = trie_free_class(free->size); list_del_init(&free->list); @@ -982,11 +974,11 @@ static void *trie_pop_free_object(size_t size) old_size = free->size; if (old_size > size) { free->size = size; - tail = trie_object_payload(free) + size; + tail = free->data + size; tail_size = old_size - size; trie_free_object_tail_locked(tail, tail_size); } - return trie_object_payload(free); + return free->data; } /* @@ -995,34 +987,27 @@ static void *trie_pop_free_object(size_t size) * used, so side-table preallocation failure disables insertion for this * save. Pool preallocation is opportunistic: reusable trie storage or active * pool space may still satisfy the insertion, and pool_carve() reports - * -ENOSPC if they do not. Callers without spinning allocation context skip - * insertion and perform only best-effort lookup. + * -ENOSPC if they do not. */ static int -__stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, - void **pool_prealloc, +__stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { unsigned long flags; - bool can_alloc; bool need_pool; - int ret = 0; + int ret; - can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && - gfpflags_allow_spinning(alloc_flags); - if (can_alloc) { - raw_spin_lock_irqsave(&pool_lock, flags); - need_pool = !new_pool; - raw_spin_unlock_irqrestore(&pool_lock, flags); - if (need_pool) { - struct page *page; + raw_spin_lock_irqsave(&pool_lock, flags); + need_pool = !new_pool; + raw_spin_unlock_irqrestore(&pool_lock, flags); + if (need_pool) { + struct page *page; - page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); - if (page) - *pool_prealloc = page_address(page); - } - ret = __stack_depot_trie_side_table_prealloc(alloc_flags, side_prealloc); + page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); + if (page) + *pool_prealloc = page_address(page); } + ret = trie_side_table_get_prealloc(alloc_flags, side_prealloc); if (ret) { if (*pool_prealloc) { @@ -1795,43 +1780,25 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * static depot_stack_handle_t stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, - gfp_t alloc_flags, depot_flags_t depot_flags) + gfp_t alloc_flags) { struct stack_depot_trie_alloc_workspace *workspace; struct stack_depot_trie_side_prealloc side_prealloc = {}; void *pool_prealloc = NULL; depot_stack_handle_t handle; unsigned long flags; - bool can_alloc; bool retried = false; u32 leaf_id; int ret; workspace = stack_depot_trie_workspace; - can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && - gfpflags_allow_spinning(alloc_flags); retry: handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); if (handle) return handle; - /* - * No-spin callers cannot wait for the workspace lock or allocate side-table - * or pool storage. After the lockless lookup misses, trylock and recheck: a - * concurrent writer may have inserted the stack. Otherwise fail instead of - * spinning or publishing a new leaf. - */ - if (in_nmi() || !gfpflags_allow_spinning(alloc_flags)) { - WARN_ON_ONCE(can_alloc); - if (!raw_spin_trylock_irqsave(&stack_depot_trie_writer_lock, flags)) - return 0; - handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); - raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); - return handle; - } - ret = __stack_depot_trie_alloc_prealloc(alloc_flags, depot_flags, - &pool_prealloc, + ret = __stack_depot_trie_alloc_prealloc(alloc_flags, &pool_prealloc, &side_prealloc); if (ret) goto out_free; @@ -1845,7 +1812,7 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, if (leaf_id) handle = __stack_depot_trie_handle(leaf_id); raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); - if (!handle && can_alloc && !retried) { + if (!handle && !retried) { retried = true; if (pool_prealloc) { raw_spin_lock_irqsave(&pool_lock, flags); @@ -1856,7 +1823,7 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); pool_prealloc = NULL; } - __stack_depot_trie_side_table_free_prealloc(&side_prealloc); + trie_side_table_put_prealloc(&side_prealloc); goto retry; } @@ -1868,7 +1835,7 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, } if (pool_prealloc) free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); - __stack_depot_trie_side_table_free_prealloc(&side_prealloc); + trie_side_table_put_prealloc(&side_prealloc); return handle; } @@ -1912,12 +1879,12 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, if (trie_candidate) { if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; - handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, - depot_flags); - if (handle) - return handle; - /* Keep trie failures visible; hash fallback hides trie pool pressure. */ - return 0; + if (in_nmi() || !can_alloc) { + WARN_ON_ONCE(can_alloc); + return trie_find_handle(&stack_depot_trie_root, entries, + nr_entries); + } + return stack_depot_trie_save(entries, nr_entries, alloc_flags); } hash = hash_stack(entries, nr_entries); @@ -2566,12 +2533,14 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * capacity = trie_child_array_capacity(children->nr_children + 1); tail_append = pos == children->nr_children && children->nr_children < children->capacity; + child_array_size = tail_append ? 0 : + trie_child_array_bytes(capacity); + if (child_array_size > DEPOT_POOL_SIZE) + return 0; new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - child_array_size = tail_append ? 0 : - trie_child_array_bytes(capacity); if (trie_pool_alloc_append_chain(workspace, pool_prealloc, entries, nr_entries, child_array_size)) From b0b581d9ea0e55ab04798f63985b1762bff60298 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 16 Jul 2026 13:16:31 +0100 Subject: [PATCH 14/38] KRN-1117: Simplify stackdepot side-table page handling Side-table directories and chunks are always exactly one page, so carrying byte and order helpers obscures their fixed allocation contract. Use PAGE_SIZE directly for early memblock storage and pair runtime get_zeroed_page() allocations with free_page(). Normalize nested GFP flags once and remove the redundant size, order, allocation, and free wrappers. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 64 +++++++++--------------------------------------- 1 file changed, 11 insertions(+), 53 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index d2388686a7c0f..277bf0d00729f 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -463,32 +463,6 @@ static inline size_t trie_side_table_root_bytes(unsigned int root_size) return struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); } -static inline size_t trie_side_table_dir_bytes(void) -{ - return PAGE_SIZE; -} - -static inline unsigned int trie_side_table_dir_order(void) -{ - return get_order(trie_side_table_dir_bytes()); -} - -static inline size_t trie_side_table_chunk_bytes(void) -{ - return PAGE_SIZE; -} - -static inline unsigned int trie_side_table_chunk_order(void) -{ - return get_order(trie_side_table_chunk_bytes()); -} - -static void trie_side_table_free_dir(struct stack_depot_trie_side_dir *dir) -{ - if (dir) - free_pages((unsigned long)dir, trie_side_table_dir_order()); -} - static void trie_side_table_root_init(struct stack_depot_trie_side_root *root_vec, unsigned int root_size, u32 max_id) { @@ -510,8 +484,6 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) struct stack_depot_trie_side_root *root_vec; struct stack_depot_trie_side_dir *first_dir; const struct stack_depot_trie_node __rcu **first_chunk; - size_t dir_bytes; - size_t chunk_bytes; size_t root_bytes; u32 max_leaf_id; unsigned int root_size; @@ -521,26 +493,24 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) return -EINVAL; root_size = trie_side_table_root_size_for_max_id(max_leaf_id); root_bytes = trie_side_table_root_bytes(root_size); - dir_bytes = trie_side_table_dir_bytes(); - chunk_bytes = trie_side_table_chunk_bytes(); root_vec = memblock_alloc(root_bytes, __alignof__(*root_vec)); if (!root_vec) return -ENOMEM; memset(root_vec, 0, root_bytes); - first_dir = memblock_alloc(dir_bytes, PAGE_SIZE); + first_dir = memblock_alloc(PAGE_SIZE, PAGE_SIZE); if (!first_dir) { memblock_free(root_vec, root_bytes); return -ENOMEM; } - memset(first_dir, 0, dir_bytes); - first_chunk = memblock_alloc(chunk_bytes, PAGE_SIZE); + memset(first_dir, 0, PAGE_SIZE); + first_chunk = memblock_alloc(PAGE_SIZE, PAGE_SIZE); if (!first_chunk) { - memblock_free(first_dir, dir_bytes); + memblock_free(first_dir, PAGE_SIZE); memblock_free(root_vec, root_bytes); return -ENOMEM; } - memset(first_chunk, 0, chunk_bytes); + memset(first_chunk, 0, PAGE_SIZE); trie_side_table_root_init(root_vec, root_size, max_leaf_id); RCU_INIT_POINTER(root_vec->dirs[0], first_dir); @@ -633,20 +603,12 @@ static int stack_depot_trie_init(gfp_t gfp_flags) return 0; } -static void *trie_side_table_alloc_page(gfp_t gfp_flags, unsigned int order) -{ - struct page *page; - - page = alloc_pages(gfp_nested_mask(gfp_flags) | __GFP_ZERO, - order); - return page ? page_address(page) : NULL; -} - static int trie_side_table_get_prealloc(gfp_t gfp_flags, struct stack_depot_trie_side_prealloc *prealloc) { unsigned long flags; + gfp_flags = gfp_nested_mask(gfp_flags); raw_spin_lock_irqsave(&trie_side_table_lock, flags); prealloc->dir = trie_side_table_cache.dir; prealloc->chunk = trie_side_table_cache.chunk; @@ -655,16 +617,12 @@ static int trie_side_table_get_prealloc(gfp_t gfp_flags, raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); if (!prealloc->dir) { - unsigned int order = trie_side_table_dir_order(); - - prealloc->dir = trie_side_table_alloc_page(gfp_flags, order); + prealloc->dir = (void *)get_zeroed_page(gfp_flags); if (!prealloc->dir) return -ENOMEM; } if (!prealloc->chunk) { - unsigned int order = trie_side_table_chunk_order(); - - prealloc->chunk = trie_side_table_alloc_page(gfp_flags, order); + prealloc->chunk = (void *)get_zeroed_page(gfp_flags); if (!prealloc->chunk) return -ENOMEM; } @@ -687,10 +645,10 @@ static void trie_side_table_put_prealloc(struct stack_depot_trie_side_prealloc * } raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - trie_side_table_free_dir(prealloc->dir); + if (prealloc->dir) + free_page((unsigned long)prealloc->dir); if (prealloc->chunk) - free_pages((unsigned long)prealloc->chunk, - trie_side_table_chunk_order()); + free_page((unsigned long)prealloc->chunk); prealloc->dir = NULL; prealloc->chunk = NULL; } From c09b693071d353b142086a49f8fc4ff34e96eaba Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 16 Jul 2026 17:18:33 +0100 Subject: [PATCH 15/38] KRN-1117: Drop unrelated stackdepot kernel-doc change The trie port does not need to change punctuation in the existing __stack_depot_get_stack_record() kernel-doc. Restore the original wording. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 143b26cb5bb19..2f4bc8873e003 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -200,7 +200,7 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, * This function is only for internal purposes. @handle must have been saved * with %STACK_DEPOT_FLAG_COUNTABLE. * - * Return: Returns a pointer to a stack_record struct. + * Return: Returns a pointer to a stack_record struct */ struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle); From 81ad006c9729f41f72f568c7dc453a15067f663c Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 20 Jul 2026 10:52:42 +0100 Subject: [PATCH 16/38] KRN-1117: Simplify stackdepot trie insertion allocation The generic workspace obscures how append, split, and promotion use its storage and initializes allocation results before insertion selects an operation. Replace it with writer-owned insertion allocation state, use role-specific names, and initialize only requested outputs at the carve boundary. Derive chain-array counts from node counts and fold split sizing into the selected path so allocation roles remain explicit without changing recycler, pool, or RCU publication behavior. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 384 +++++++++++++++++++++-------------------------- 1 file changed, 172 insertions(+), 212 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 277bf0d00729f..45b08cdbd8c5f 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -187,17 +187,18 @@ static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_free_object, data), #define STACK_DEPOT_TRIE_FREE_CLASSES \ ((DEPOT_POOL_SIZE >> DEPOT_STACK_ALIGN) + 1) -struct stack_depot_trie_alloc_workspace { +/* Writer-owned storage for one unpublished trie insertion. */ +struct stack_depot_trie_insert_alloc { struct stack_depot_trie_node *nodes[STACK_DEPOT_TRIE_MAX_NODES]; size_t node_sizes[STACK_DEPOT_TRIE_MAX_NODES]; - struct stack_depot_trie_child_array *child_arrays[STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS]; - struct stack_depot_trie_child_array *split_child_array; - struct stack_depot_trie_child_array *child_array_storage; + struct stack_depot_trie_child_array *chain_arrays[STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS]; + struct stack_depot_trie_child_array *prefix_children; + struct stack_depot_trie_child_array *slot_array; }; static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); static const struct stack_depot_trie_child_array __rcu *stack_depot_trie_root; -static struct stack_depot_trie_alloc_workspace *stack_depot_trie_workspace; +static struct stack_depot_trie_insert_alloc *stack_depot_trie_alloc; static DEFINE_RAW_SPINLOCK(stack_depot_trie_writer_lock); static bool stack_depot_trie_requested; @@ -554,25 +555,24 @@ static void trie_free_object_buckets_init(void) static int __init stack_depot_trie_init_memblock(void) { - struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_insert_alloc *alloc; size_t size; int ret; if (__stack_depot_trie_enabled()) return 0; - size = sizeof(*stack_depot_trie_workspace); - workspace = memblock_alloc(size, __alignof__(*workspace)); - if (!workspace) + size = sizeof(*stack_depot_trie_alloc); + alloc = memblock_alloc(size, __alignof__(*alloc)); + if (!alloc) return -ENOMEM; - memset(workspace, 0, size); ret = __stack_depot_trie_side_table_init_memblock(); if (ret) { - memblock_free(workspace, size); + memblock_free(alloc, size); return ret; } - stack_depot_trie_workspace = workspace; + stack_depot_trie_alloc = alloc; trie_free_object_buckets_init(); static_branch_enable(&stack_depot_trie_enabled); @@ -581,22 +581,22 @@ static int __init stack_depot_trie_init_memblock(void) static int stack_depot_trie_init(gfp_t gfp_flags) { - struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_insert_alloc *alloc; int ret; if (__stack_depot_trie_enabled()) return 0; - workspace = kvzalloc(sizeof(*stack_depot_trie_workspace), gfp_flags); - if (!workspace) + alloc = kvzalloc(sizeof(*stack_depot_trie_alloc), gfp_flags); + if (!alloc) return -ENOMEM; ret = __stack_depot_trie_side_table_init(gfp_flags); if (ret) { - kvfree(workspace); + kvfree(alloc); return ret; } - stack_depot_trie_workspace = workspace; + stack_depot_trie_alloc = alloc; trie_free_object_buckets_init(); static_branch_enable(&stack_depot_trie_enabled); @@ -998,11 +998,11 @@ static int trie_pool_add_object_size(size_t size, size_t *total) * The caller must not publish any returned storage before side-table and trie * publication succeeds. */ -static int trie_pool_carve(struct stack_depot_trie_alloc_workspace *workspace, +static int trie_pool_carve(struct stack_depot_trie_insert_alloc *alloc, void **pool_prealloc, unsigned int nr_nodes, - unsigned int nr_child_arrays, - size_t split_child_array_size, - size_t child_array_size) + unsigned int nr_chain_arrays, + size_t prefix_children_size, + size_t slot_array_size) { unsigned long flags; size_t one_child_size; @@ -1013,36 +1013,41 @@ static int trie_pool_carve(struct stack_depot_trie_alloc_workspace *workspace, void *pool; int ret = -ENOSPC; + memset(alloc->nodes, 0, nr_nodes * sizeof(*alloc->nodes)); + memset(alloc->chain_arrays, 0, + nr_chain_arrays * sizeof(*alloc->chain_arrays)); + alloc->prefix_children = NULL; + alloc->slot_array = NULL; + raw_spin_lock_irqsave(&pool_lock, flags); printk_deferred_enter(); trie_drain_pending_objects_locked(); one_child_size = trie_child_array_bytes(1); for (i = 0; i < nr_nodes; i++) { - workspace->nodes[i] = trie_pop_free_node(workspace->node_sizes[i]); - if (!workspace->nodes[i]) { - alloc_size = __stack_depot_trie_pool_alloc_size(workspace->node_sizes[i]); + alloc->nodes[i] = trie_pop_free_node(alloc->node_sizes[i]); + if (!alloc->nodes[i]) { + alloc_size = __stack_depot_trie_pool_alloc_size(alloc->node_sizes[i]); if (!alloc_size || check_add_overflow(total, alloc_size, &total) || total > DEPOT_POOL_SIZE) goto out_discard; } } - for (i = 0; i < nr_child_arrays; i++) { - workspace->child_arrays[i] = trie_pop_free_object(one_child_size); - if (!workspace->child_arrays[i] && + for (i = 0; i < nr_chain_arrays; i++) { + alloc->chain_arrays[i] = trie_pop_free_object(one_child_size); + if (!alloc->chain_arrays[i] && trie_pool_add_object_size(one_child_size, &total)) goto out_discard; } - if (split_child_array_size) { - workspace->split_child_array = - trie_pop_free_object(split_child_array_size); - if (!workspace->split_child_array && - trie_pool_add_object_size(split_child_array_size, &total)) + if (prefix_children_size) { + alloc->prefix_children = trie_pop_free_object(prefix_children_size); + if (!alloc->prefix_children && + trie_pool_add_object_size(prefix_children_size, &total)) goto out_discard; } - if (child_array_size) { - workspace->child_array_storage = trie_pop_free_object(child_array_size); - if (!workspace->child_array_storage && - trie_pool_add_object_size(child_array_size, &total)) + if (slot_array_size) { + alloc->slot_array = trie_pop_free_object(slot_array_size); + if (!alloc->slot_array && + trie_pool_add_object_size(slot_array_size, &total)) goto out_discard; } @@ -1063,54 +1068,47 @@ static int trie_pool_carve(struct stack_depot_trie_alloc_workspace *workspace, offset = pool_offset; for (i = 0; i < nr_nodes; i++) { - if (workspace->nodes[i]) + if (alloc->nodes[i]) continue; - workspace->nodes[i] = pool + offset; - offset += __stack_depot_trie_pool_alloc_size(workspace->node_sizes[i]); + alloc->nodes[i] = pool + offset; + offset += __stack_depot_trie_pool_alloc_size(alloc->node_sizes[i]); } - for (i = 0; i < nr_child_arrays; i++) { - if (workspace->child_arrays[i]) + for (i = 0; i < nr_chain_arrays; i++) { + if (alloc->chain_arrays[i]) continue; - workspace->child_arrays[i] = + alloc->chain_arrays[i] = trie_object_init_fresh(pool + offset, one_child_size); offset += trie_object_alloc_size(one_child_size); } - if (split_child_array_size && !workspace->split_child_array) { - workspace->split_child_array = - trie_object_init_fresh(pool + offset, split_child_array_size); - offset += trie_object_alloc_size(split_child_array_size); + if (prefix_children_size && !alloc->prefix_children) { + alloc->prefix_children = + trie_object_init_fresh(pool + offset, prefix_children_size); + offset += trie_object_alloc_size(prefix_children_size); } - if (child_array_size && !workspace->child_array_storage) - workspace->child_array_storage = - trie_object_init_fresh(pool + offset, child_array_size); + if (slot_array_size && !alloc->slot_array) + alloc->slot_array = + trie_object_init_fresh(pool + offset, slot_array_size); pool_offset += total; ret = 0; goto out; out_discard: for (i = 0; i < nr_nodes; i++) { - struct stack_depot_trie_node *node = workspace->nodes[i]; + struct stack_depot_trie_node *node = alloc->nodes[i]; - if (node) { - trie_add_free_node_locked(node, workspace->node_sizes[i]); - workspace->nodes[i] = NULL; - } + if (node) + trie_add_free_node_locked(node, alloc->node_sizes[i]); } - for (i = 0; i < nr_child_arrays; i++) { - struct stack_depot_trie_child_array *array = workspace->child_arrays[i]; + for (i = 0; i < nr_chain_arrays; i++) { + struct stack_depot_trie_child_array *array = alloc->chain_arrays[i]; if (!array) continue; trie_free_object_locked(array); - workspace->child_arrays[i] = NULL; - } - if (split_child_array_size && workspace->split_child_array) { - trie_free_object_locked(workspace->split_child_array); - workspace->split_child_array = NULL; - } - if (child_array_size && workspace->child_array_storage) { - trie_free_object_locked(workspace->child_array_storage); - workspace->child_array_storage = NULL; } + if (prefix_children_size && alloc->prefix_children) + trie_free_object_locked(alloc->prefix_children); + if (slot_array_size && alloc->slot_array) + trie_free_object_locked(alloc->slot_array); out: printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -1733,14 +1731,12 @@ static u32 stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu **root_slot, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace); + struct stack_depot_trie_side_prealloc *side_prealloc); static depot_stack_handle_t stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags) { - struct stack_depot_trie_alloc_workspace *workspace; struct stack_depot_trie_side_prealloc side_prealloc = {}; void *pool_prealloc = NULL; depot_stack_handle_t handle; @@ -1749,8 +1745,6 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, u32 leaf_id; int ret; - workspace = stack_depot_trie_workspace; - retry: handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); if (handle) @@ -1762,11 +1756,9 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, goto out_free; raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); - memset(workspace, 0, sizeof(*workspace)); leaf_id = stack_depot_trie_insert_locked(&stack_depot_trie_root, entries, nr_entries, - &pool_prealloc, &side_prealloc, - workspace); + &pool_prealloc, &side_prealloc); if (leaf_id) handle = __stack_depot_trie_handle(leaf_id); raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); @@ -2200,7 +2192,7 @@ static void trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_node * const *nodes, - struct stack_depot_trie_child_array * const *child_arrays, + struct stack_depot_trie_child_array * const *chain_arrays, const struct stack_depot_trie_node **head, const struct stack_depot_trie_node **tail) { @@ -2228,7 +2220,7 @@ trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, for (i = 0; i + 1 < used; i++) { const struct stack_depot_trie_node *next = nodes[i + 1]; struct stack_depot_trie_node *node = nodes[i]; - struct stack_depot_trie_child_array *array = child_arrays[i]; + struct stack_depot_trie_child_array *array = chain_arrays[i]; trie_child_array_init(array, 1, &next, 1); RCU_INIT_POINTER(node->children, array); @@ -2295,11 +2287,11 @@ stack_depot_trie_lookup(const struct stack_depot_trie_child_array __rcu * const static void trie_build_split(const struct stack_depot_trie_node *child, unsigned int matched, u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_node * const *nodes, - struct stack_depot_trie_child_array * const *child_arrays, - struct stack_depot_trie_child_array *split_child_array, - const struct stack_depot_trie_node **prefix, - const struct stack_depot_trie_node **old_tail, + struct stack_depot_trie_node *prefix, + struct stack_depot_trie_node *old_tail, + struct stack_depot_trie_node * const *new_nodes, + struct stack_depot_trie_child_array * const *chain_arrays, + struct stack_depot_trie_child_array *prefix_children, const struct stack_depot_trie_node **new_leaf) { const struct stack_depot_trie_child_array *child_children; @@ -2308,8 +2300,6 @@ static void trie_build_split(const struct stack_depot_trie_node *child, const struct stack_depot_trie_node *split_children[2]; const struct stack_depot_trie_node *new_head = NULL; const struct stack_depot_trie_node *new_tail = NULL; - struct stack_depot_trie_node *old_tail_node; - struct stack_depot_trie_node *pref; unsigned long new_frame; unsigned long old_frame; u32 prefix_leaf_id; @@ -2320,144 +2310,96 @@ static void trie_build_split(const struct stack_depot_trie_node *child, child_parent = trie_load_parent(child); has_new_tail = matched < nr_entries; - pref = nodes[0]; - old_tail_node = nodes[1]; prefix_leaf_id = has_new_tail ? 0 : leaf_id; - trie_node_init_slice(pref, child_parent, prefix_leaf_id, child, 0, + trie_node_init_slice(prefix, child_parent, prefix_leaf_id, child, 0, matched); tail_len = child->run.nr_entries - matched; - trie_node_init_slice(old_tail_node, pref, child->leaf_id, child, matched, + trie_node_init_slice(old_tail, prefix, child->leaf_id, child, matched, tail_len); if (has_new_tail) { tail_entries = &entries[matched]; tail_len = nr_entries - matched; - trie_build_append_chain(pref, leaf_id, tail_entries, tail_len, - &nodes[2], child_arrays, + trie_build_append_chain(prefix, leaf_id, tail_entries, tail_len, + new_nodes, chain_arrays, &new_head, &new_tail); - stack_depot_trie_node_frame(old_tail_node, 0, &old_frame); + stack_depot_trie_node_frame(old_tail, 0, &old_frame); stack_depot_trie_node_frame(new_head, 0, &new_frame); if (old_frame < new_frame) { - split_children[0] = old_tail_node; + split_children[0] = old_tail; split_children[1] = new_head; } else { split_children[0] = new_head; - split_children[1] = old_tail_node; + split_children[1] = old_tail; } - trie_child_array_init(split_child_array, ARRAY_SIZE(split_children), + trie_child_array_init(prefix_children, ARRAY_SIZE(split_children), split_children, ARRAY_SIZE(split_children)); } else { - split_children[0] = old_tail_node; - trie_child_array_init(split_child_array, 1, split_children, 1); + split_children[0] = old_tail; + trie_child_array_init(prefix_children, 1, split_children, 1); } - RCU_INIT_POINTER(old_tail_node->children, child_children); - RCU_INIT_POINTER(pref->children, split_child_array); - *prefix = pref; - *old_tail = old_tail_node; - *new_leaf = has_new_tail ? new_tail : pref; + RCU_INIT_POINTER(old_tail->children, child_children); + RCU_INIT_POINTER(prefix->children, prefix_children); + *new_leaf = has_new_tail ? new_tail : prefix; } -static void trie_size_append_chain(const unsigned long *entries, - unsigned int nr_entries, - size_t *node_sizes, - unsigned int *nr_nodes, - unsigned int *nr_child_arrays) +static unsigned int trie_size_append_chain(const unsigned long *entries, + unsigned int nr_entries, + size_t *node_sizes) { unsigned int pos = 0; - unsigned int used = 0; + unsigned int nr_nodes = 0; while (pos < nr_entries) { struct stack_depot_frame_run run; frame_run_init(&entries[pos], nr_entries - pos, &run); - node_sizes[used] = trie_node_bytes(&run); + node_sizes[nr_nodes] = trie_node_bytes(&run); pos += run.nr_entries; - used++; + nr_nodes++; } - *nr_nodes = used; - *nr_child_arrays = used > 1 ? used - 1 : 0; + return nr_nodes; } static int -trie_pool_alloc_append_chain(struct stack_depot_trie_alloc_workspace *workspace, +trie_pool_alloc_append_chain(struct stack_depot_trie_insert_alloc *alloc, void **pool_prealloc, const unsigned long *entries, unsigned int nr_entries, - size_t child_array_size) + size_t slot_array_size) { - unsigned int nr_child_arrays; - unsigned int nr_nodes; - - trie_size_append_chain(entries, nr_entries, workspace->node_sizes, - &nr_nodes, &nr_child_arrays); - return trie_pool_carve(workspace, pool_prealloc, nr_nodes, nr_child_arrays, - 0, child_array_size); -} - -static int -trie_pool_alloc_split(struct stack_depot_trie_alloc_workspace *workspace, - void **pool_prealloc, - const struct stack_depot_trie_child_array *children, - const struct stack_depot_trie_node *child, - unsigned int matched, const unsigned long *entries, - unsigned int nr_entries) -{ - struct stack_depot_frame_run old_tail_run; - struct stack_depot_frame_run prefix_run; - unsigned int new_child_arrays = 0; - unsigned int new_used = 0; - unsigned int nr_child_arrays; unsigned int nr_nodes; - bool has_new_tail; - size_t child_array_size; - size_t split_child_array_size; - prefix_run = child->run; - prefix_run.nr_entries = matched; - old_tail_run = child->run; - old_tail_run.nr_entries = child->run.nr_entries - matched; - has_new_tail = matched < nr_entries; - - workspace->node_sizes[0] = trie_node_bytes(&prefix_run); - workspace->node_sizes[1] = trie_node_bytes(&old_tail_run); - if (has_new_tail) - trie_size_append_chain(&entries[matched], nr_entries - matched, - &workspace->node_sizes[2], &new_used, - &new_child_arrays); - - child_array_size = trie_child_array_bytes(children->capacity); - split_child_array_size = - trie_child_array_bytes(has_new_tail ? 2 : 1); - nr_nodes = 2 + new_used; - nr_child_arrays = new_child_arrays; - return trie_pool_carve(workspace, pool_prealloc, nr_nodes, nr_child_arrays, - split_child_array_size, child_array_size); + nr_nodes = trie_size_append_chain(entries, nr_entries, alloc->node_sizes); + return trie_pool_carve(alloc, pool_prealloc, nr_nodes, nr_nodes - 1, 0, + slot_array_size); } static u32 stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu **root_slot, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace) + struct stack_depot_trie_side_prealloc *side_prealloc) { - struct stack_depot_trie_child_array *new_array; - struct stack_depot_trie_child_array *split_child_array; + struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; + struct stack_depot_trie_child_array *slot_array; + struct stack_depot_trie_child_array *prefix_children; const struct stack_depot_trie_child_array *children; const struct stack_depot_trie_child_array __rcu **slot = root_slot; const struct stack_depot_trie_node *child; - const struct stack_depot_trie_node *head; - const struct stack_depot_trie_node *last; - const struct stack_depot_trie_node *old_tail; - const struct stack_depot_trie_node *new_leaf; - struct stack_depot_trie_node *new_node; + const struct stack_depot_trie_node *chain_head; + const struct stack_depot_trie_node *chain_leaf; + const struct stack_depot_trie_node *split_leaf; + struct stack_depot_trie_node *split_prefix; + struct stack_depot_trie_node *old_tail; + struct stack_depot_trie_node *promoted_node; struct stack_depot_trie_node *parent = NULL; unsigned int matched; unsigned int pos; unsigned long flags; u32 new_leaf_id; - size_t child_array_size; + size_t slot_array_size; bool found; for (;;) { @@ -2466,20 +2408,20 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - child_array_size = trie_child_array_bytes(1); - if (trie_pool_alloc_append_chain(workspace, pool_prealloc, + slot_array_size = trie_child_array_bytes(1); + if (trie_pool_alloc_append_chain(alloc, pool_prealloc, entries, nr_entries, - child_array_size)) + slot_array_size)) return 0; - new_array = workspace->child_array_storage; + slot_array = alloc->slot_array; trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - workspace->nodes, workspace->child_arrays, - &head, &last); - trie_side_table_publish_new_leaf(new_leaf_id, last); - new_array->nr_children = 1; - new_array->capacity = 1; - RCU_INIT_POINTER(new_array->children[0], head); - trie_publish_children_slot(slot, new_array); + alloc->nodes, alloc->chain_arrays, + &chain_head, &chain_leaf); + trie_side_table_publish_new_leaf(new_leaf_id, chain_leaf); + slot_array->nr_children = 1; + slot_array->capacity = 1; + RCU_INIT_POINTER(slot_array->children[0], chain_head); + trie_publish_children_slot(slot, slot_array); goto out_success; } trie_child_array_find_slot(children, entries[0], &pos, &found); @@ -2491,30 +2433,30 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * capacity = trie_child_array_capacity(children->nr_children + 1); tail_append = pos == children->nr_children && children->nr_children < children->capacity; - child_array_size = tail_append ? 0 : + slot_array_size = tail_append ? 0 : trie_child_array_bytes(capacity); - if (child_array_size > DEPOT_POOL_SIZE) + if (slot_array_size > DEPOT_POOL_SIZE) return 0; new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - if (trie_pool_alloc_append_chain(workspace, pool_prealloc, + if (trie_pool_alloc_append_chain(alloc, pool_prealloc, entries, nr_entries, - child_array_size)) + slot_array_size)) return 0; - new_array = workspace->child_array_storage; trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - workspace->nodes, workspace->child_arrays, - &head, &last); - trie_side_table_publish_new_leaf(new_leaf_id, last); + alloc->nodes, alloc->chain_arrays, + &chain_head, &chain_leaf); + trie_side_table_publish_new_leaf(new_leaf_id, chain_leaf); if (tail_append) { tail_array = (struct stack_depot_trie_child_array *)children; - trie_publish_tail_append(tail_array, pos, head); + trie_publish_tail_append(tail_array, pos, chain_head); } else { - trie_child_array_insert_at(children, pos, head, - new_array, capacity); - trie_publish_children_slot(slot, new_array); + slot_array = alloc->slot_array; + trie_child_array_insert_at(children, pos, chain_head, + slot_array, capacity); + trie_publish_children_slot(slot, slot_array); raw_spin_lock_irqsave(&pool_lock, flags); trie_retire_child_array_locked(children); raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -2525,23 +2467,41 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * child = trie_child_array_load_child(children, pos); matched = __stack_depot_trie_node_match(child, entries, nr_entries); if (matched < child->run.nr_entries) { + struct stack_depot_frame_run run; + unsigned int new_nodes = 0; + bool has_new_tail; + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - if (trie_pool_alloc_split(workspace, pool_prealloc, children, - child, matched, entries, nr_entries)) + + run = child->run; + run.nr_entries = matched; + alloc->node_sizes[0] = trie_node_bytes(&run); + run.nr_entries = child->run.nr_entries - matched; + alloc->node_sizes[1] = trie_node_bytes(&run); + has_new_tail = matched < nr_entries; + if (has_new_tail) + new_nodes = trie_size_append_chain(&entries[matched], + nr_entries - matched, + &alloc->node_sizes[2]); + if (trie_pool_carve(alloc, pool_prealloc, 2 + new_nodes, + new_nodes ? new_nodes - 1 : 0, + trie_child_array_bytes(has_new_tail ? 2 : 1), + trie_child_array_bytes(children->capacity))) return 0; - new_array = workspace->child_array_storage; - split_child_array = workspace->split_child_array; + slot_array = alloc->slot_array; + prefix_children = alloc->prefix_children; + split_prefix = alloc->nodes[0]; + old_tail = alloc->nodes[1]; trie_build_split(child, matched, new_leaf_id, entries, nr_entries, - workspace->nodes, workspace->child_arrays, - split_child_array, &head, &old_tail, - &new_leaf); + split_prefix, old_tail, &alloc->nodes[2], + alloc->chain_arrays, prefix_children, &split_leaf); trie_side_table_publish_split_leaves(child->leaf_id, old_tail, - new_leaf_id, new_leaf); - trie_child_array_replace_at(children, head, new_array, pos); - trie_reparent_children((struct stack_depot_trie_node *)old_tail); - trie_publish_children_slot(slot, new_array); + new_leaf_id, split_leaf); + trie_child_array_replace_at(children, split_prefix, slot_array, pos); + trie_reparent_children(old_tail); + trie_publish_children_slot(slot, slot_array); trie_retire_child_array_with_node(children, child); goto out_success; } @@ -2551,19 +2511,19 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - workspace->node_sizes[0] = trie_node_bytes(&child->run); - child_array_size = trie_child_array_bytes(children->capacity); - if (trie_pool_carve(workspace, pool_prealloc, 1, 0, 0, - child_array_size)) + alloc->node_sizes[0] = trie_node_bytes(&child->run); + slot_array_size = trie_child_array_bytes(children->capacity); + if (trie_pool_carve(alloc, pool_prealloc, 1, 0, 0, + slot_array_size)) return 0; - new_node = workspace->nodes[0]; - new_array = workspace->child_array_storage; - memcpy(new_node, child, trie_node_bytes(&child->run)); - new_node->leaf_id = new_leaf_id; - trie_side_table_publish_new_leaf(new_leaf_id, new_node); - trie_child_array_replace_at(children, new_node, new_array, pos); - trie_reparent_children(new_node); - trie_publish_children_slot(slot, new_array); + promoted_node = alloc->nodes[0]; + slot_array = alloc->slot_array; + memcpy(promoted_node, child, alloc->node_sizes[0]); + promoted_node->leaf_id = new_leaf_id; + trie_side_table_publish_new_leaf(new_leaf_id, promoted_node); + trie_child_array_replace_at(children, promoted_node, slot_array, pos); + trie_reparent_children(promoted_node); + trie_publish_children_slot(slot, slot_array); trie_retire_child_array_with_node(children, child); goto out_success; } From 8abd0ab155467251f5bd0bcd9f3fd716c0fe1986 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 20 Jul 2026 11:16:45 +0100 Subject: [PATCH 17/38] KRN-1117: Inline the stackdepot trie append allocation The append allocation helper only sizes a new chain and forwards the derived node and child-array counts to trie_pool_carve(), while both callers are adjacent branches of the insertion path. Inline the allocation recipe where each append operation is selected so its size and carve requirements remain visible without another helper layer. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 45b08cdbd8c5f..f30c56e17ff8a 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2362,20 +2362,6 @@ static unsigned int trie_size_append_chain(const unsigned long *entries, return nr_nodes; } -static int -trie_pool_alloc_append_chain(struct stack_depot_trie_insert_alloc *alloc, - void **pool_prealloc, - const unsigned long *entries, - unsigned int nr_entries, - size_t slot_array_size) -{ - unsigned int nr_nodes; - - nr_nodes = trie_size_append_chain(entries, nr_entries, alloc->node_sizes); - return trie_pool_carve(alloc, pool_prealloc, nr_nodes, nr_nodes - 1, 0, - slot_array_size); -} - static u32 stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu **root_slot, const unsigned long *entries, unsigned int nr_entries, @@ -2405,13 +2391,16 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * for (;;) { children = trie_load_children_slot(slot); if (!children) { + unsigned int nr_nodes; + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; slot_array_size = trie_child_array_bytes(1); - if (trie_pool_alloc_append_chain(alloc, pool_prealloc, - entries, nr_entries, - slot_array_size)) + nr_nodes = trie_size_append_chain(entries, nr_entries, + alloc->node_sizes); + if (trie_pool_carve(alloc, pool_prealloc, nr_nodes, + nr_nodes - 1, 0, slot_array_size)) return 0; slot_array = alloc->slot_array; trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, @@ -2428,6 +2417,7 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * if (!found) { struct stack_depot_trie_child_array *tail_array; unsigned int capacity; + unsigned int nr_nodes; bool tail_append; capacity = trie_child_array_capacity(children->nr_children + 1); @@ -2441,9 +2431,10 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - if (trie_pool_alloc_append_chain(alloc, pool_prealloc, - entries, nr_entries, - slot_array_size)) + nr_nodes = trie_size_append_chain(entries, nr_entries, + alloc->node_sizes); + if (trie_pool_carve(alloc, pool_prealloc, nr_nodes, + nr_nodes - 1, 0, slot_array_size)) return 0; trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, alloc->nodes, alloc->chain_arrays, From 79059e3fbfc413f4167c70321c4a4819c9a052a6 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 20 Jul 2026 17:04:11 +0100 Subject: [PATCH 18/38] KRN-1117: Remove redundant stackdepot trie checks Trie handle, pool, and child-array sizes are bounded by their encoding and allocation invariants. Remove overflow and state checks that cannot fire so the implementation does not imply unsupported recovery paths. Inline the single-use child-array capacity calculation and keep expected resource failures and corrupt-handle diagnostics unchanged. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 67 ++++++++++++------------------------------------ 1 file changed, 17 insertions(+), 50 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index f30c56e17ff8a..ebd0039e6897b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include @@ -233,11 +232,8 @@ static u32 trie_side_table_max_id; */ static inline u32 __stack_depot_trie_max_leaf_id(void) { - u64 max_id; - - max_id = (u64)(DEPOT_POOL_INDEX_MASK - stack_max_pools) << - DEPOT_OFFSET_BITS; - return min_t(u64, max_id, U32_MAX); + return (DEPOT_POOL_INDEX_MASK - stack_max_pools) << + DEPOT_OFFSET_BITS; } static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) @@ -266,15 +262,11 @@ static inline bool stack_depot_handle_is_trie(depot_stack_handle_t handle) static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) { union handle_parts parts = { .handle = handle }; - u64 leaf_id; u32 pool_delta; parts.extra = 0; pool_delta = parts.pool_index_plus_1 - stack_max_pools - 1; - leaf_id = ((u64)pool_delta << DEPOT_OFFSET_BITS) + parts.offset + 1; - WARN_ON_ONCE(leaf_id > trie_side_table_max_id); - - return leaf_id; + return (pool_delta << DEPOT_OFFSET_BITS) + parts.offset + 1; } /* @@ -340,20 +332,12 @@ static inline size_t trie_node_bytes(const struct stack_depot_frame_run *run) stack_depot_frame_run_bytes(run), sizeof(unsigned long)); } -static inline unsigned int trie_child_array_capacity(unsigned int nr_children) -{ - return nr_children ? roundup_pow_of_two(nr_children) : 0; -} - static size_t trie_child_array_bytes(unsigned int capacity) { size_t size; size = struct_size_t(struct stack_depot_trie_child_array, children, capacity); - if (size == SIZE_MAX) - return 0; - return ALIGN(size, sizeof(unsigned long)); } @@ -425,8 +409,7 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo raw_spin_lock_irqsave(&trie_side_table_lock, flags); id = trie_side_table_last_leaf_id + 1; - /* ID zero wraps the 32-bit counter; max_id is trie handle capacity. */ - if (!id || id > trie_side_table_max_id) + if (id > trie_side_table_max_id) goto out_fail; root_vec = trie_side_table_root; @@ -559,9 +542,6 @@ static int __init stack_depot_trie_init_memblock(void) size_t size; int ret; - if (__stack_depot_trie_enabled()) - return 0; - size = sizeof(*stack_depot_trie_alloc); alloc = memblock_alloc(size, __alignof__(*alloc)); if (!alloc) @@ -688,15 +668,10 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( static size_t __stack_depot_trie_pool_alloc_size(size_t size) { - size_t align = 1UL << DEPOT_STACK_ALIGN; - size_t aligned; - if (WARN_ON_ONCE(!size || size > DEPOT_POOL_SIZE)) return 0; - if (check_add_overflow(size, align - 1, &aligned)) - return 0; - aligned = ALIGN(size, align); - return aligned <= DEPOT_POOL_SIZE ? aligned : 0; + + return ALIGN(size, 1UL << DEPOT_STACK_ALIGN); } static inline size_t trie_object_header_size(void) @@ -711,9 +686,7 @@ static size_t trie_object_alloc_size(size_t size) size = __stack_depot_trie_pool_alloc_size(size); if (!size) return 0; - if (check_add_overflow(trie_object_header_size(), size, - &alloc_size)) - return 0; + alloc_size = trie_object_header_size() + size; return alloc_size <= DEPOT_POOL_SIZE ? alloc_size : 0; } @@ -725,9 +698,6 @@ static inline struct stack_depot_trie_free_object *trie_object_header(const void static inline unsigned int trie_free_class(size_t size) { size = __stack_depot_trie_pool_alloc_size(size); - if (!size) - return 0; - return size >> DEPOT_STACK_ALIGN; } @@ -984,8 +954,7 @@ static int trie_pool_add_object_size(size_t size, size_t *total) alloc_size = trie_object_alloc_size(size); if (!alloc_size) return -ENOSPC; - if (check_add_overflow(*total, alloc_size, total)) - return -ENOSPC; + *total += alloc_size; return *total <= DEPOT_POOL_SIZE ? 0 : -ENOSPC; } @@ -1027,8 +996,10 @@ static int trie_pool_carve(struct stack_depot_trie_insert_alloc *alloc, alloc->nodes[i] = trie_pop_free_node(alloc->node_sizes[i]); if (!alloc->nodes[i]) { alloc_size = __stack_depot_trie_pool_alloc_size(alloc->node_sizes[i]); - if (!alloc_size || check_add_overflow(total, alloc_size, &total) || - total > DEPOT_POOL_SIZE) + if (!alloc_size) + goto out_discard; + total += alloc_size; + if (total > DEPOT_POOL_SIZE) goto out_discard; } } @@ -2268,11 +2239,8 @@ stack_depot_trie_lookup(const struct stack_depot_trie_child_array __rcu * const return NULL; node = trie_child_array_load_child(children, slot); - if (!node) - return NULL; - matched = __stack_depot_trie_node_match(node, &entries[pos], remaining); - if (!matched || matched < node->run.nr_entries) + if (matched < node->run.nr_entries) return NULL; pos += matched; if (pos == nr_entries) @@ -2420,7 +2388,7 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * unsigned int nr_nodes; bool tail_append; - capacity = trie_child_array_capacity(children->nr_children + 1); + capacity = roundup_pow_of_two(children->nr_children + 1); tail_append = pos == children->nr_children && children->nr_children < children->capacity; slot_array_size = tail_append ? 0 : @@ -2680,8 +2648,7 @@ void stack_depot_print(depot_stack_handle_t stack) nr_entries = __stack_depot_trie_fetch_handle_into(stack, trie_entries, ARRAY_SIZE(trie_entries)); - if (nr_entries) - stack_trace_print(trie_entries, nr_entries, 0); + stack_trace_print(trie_entries, nr_entries, 0); return; } @@ -2703,8 +2670,8 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, nr_entries = __stack_depot_trie_fetch_handle_into(handle, trie_entries, ARRAY_SIZE(trie_entries)); - return nr_entries ? stack_trace_snprint(buf, size, trie_entries, - nr_entries, spaces) : 0; + return stack_trace_snprint(buf, size, trie_entries, nr_entries, + spaces); } nr_entries = stack_depot_fetch(handle, &entries); From 48f7d5fcbcbbff5ba40f125ec1656454d96725eb Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 20 Jul 2026 17:04:45 +0100 Subject: [PATCH 19/38] KRN-1117: Expand stackdepot trie KUnit coverage Exercise divergent splits, internal-node promotion, descendant appends, child-array growth, tail append, and old-handle materialization through the public stackdepot APIs. Add mixed compressed and raw frame roundtrip coverage, while keeping every test vector within the configured three-frame minimum. Signed-off-by: Caleb Kan --- lib/tests/stackdepot_kunit.c | 70 +++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index a7e1209861984..9e1df756d766f 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -276,6 +276,69 @@ static void stackdepot_countable_does_not_alias_other_modes(struct kunit *test) stack_depot_put(get_handle); } +static void stackdepot_trie_topology_roundtrip(struct kunit *test) +{ + unsigned long stacks[][3] = { + { 0x201000UL, 0x202000UL }, + { 0x201000UL, 0x203000UL }, + { 0x201000UL }, + { 0x201000UL, 0x203000UL, 0x204000UL }, + { 0x201000UL, 0x205000UL }, + { 0x201000UL, 0x206000UL }, + }; + unsigned int nr_entries[] = { 2, 2, 1, 3, 2, 2 }; + depot_stack_handle_t handles[ARRAY_SIZE(stacks)]; + unsigned long fetched[ARRAY_SIZE(stacks[0])]; + unsigned int i; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + for (i = 0; i < ARRAY_SIZE(stacks); i++) { + handles[i] = stack_depot_save(stacks[i], nr_entries[i], GFP_KERNEL); + KUNIT_ASSERT_NE(test, handles[i], (depot_stack_handle_t)0); + } + + for (i = 0; i < ARRAY_SIZE(stacks); i++) { + memset(fetched, 0, sizeof(fetched)); + KUNIT_EXPECT_EQ(test, + stack_depot_fetch_into(handles[i], fetched, + ARRAY_SIZE(fetched)), + nr_entries[i]); + KUNIT_EXPECT_MEMEQ(test, fetched, stacks[i], + nr_entries[i] * sizeof(fetched[0])); + } +} + +static void stackdepot_frame_storage_roundtrip(struct kunit *test) +{ + unsigned long fetched[3] = {}; + depot_stack_handle_t handle; + unsigned int nr_entries; +#if defined(CONFIG_ARM64) + unsigned long entries[] = { + stackdepot_arm64_frame(S32_MIN), + 0x1000UL, + stackdepot_arm64_frame(S32_MAX), + }; +#elif defined(CONFIG_X86_64) + unsigned long entries[] = { + 0xffffffff81234567UL, + 0xffff888000001000UL, + 0xffffffff89abcdefUL, + }; +#else + unsigned long entries[] = { 0x301000UL, 0x302000UL, 0x303000UL }; +#endif + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); +} + static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0x1000UL; @@ -283,10 +346,7 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) u32 payload = 0xfeedbeef; #ifdef CONFIG_ARM64 - if ((unsigned long)_text <= ULONG_MAX - ((unsigned long)S32_MAX + 1UL)) - frame = (unsigned long)_text + (unsigned long)S32_MAX + 1UL; - else - frame = stackdepot_arm64_frame((long)S32_MIN - 1L); + frame = (unsigned long)_text + (unsigned long)S32_MAX + 1UL; #endif compressed = arch_stack_depot_frame_try_compress(frame, &payload); @@ -355,6 +415,8 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_snprint_public), KUNIT_CASE(stackdepot_get_stack_record), KUNIT_CASE(stackdepot_countable_does_not_alias_other_modes), + KUNIT_CASE(stackdepot_trie_topology_roundtrip), + KUNIT_CASE(stackdepot_frame_storage_roundtrip), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), From 63fa41f2a8f90fd108d530fd6dfe26085b8b28b8 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 20 Jul 2026 18:45:16 +0100 Subject: [PATCH 20/38] KRN-1117: Simplify singleton stackdepot trie paths Stackdepot owns one process-lifetime trie, but lookup and insertion helpers still accept a root argument that is always stack_depot_trie_root. Remove the unused generality so the singleton ownership is explicit at each operation. Document why the recycler scans raw bitmap words instead of using the instrumented multi-word bitmap helper. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index ebd0039e6897b..b4b6bce4baebc 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -724,6 +724,7 @@ static bool depot_init_pool(void **prealloc); static unsigned int trie_find_next_set_class(unsigned long *map, unsigned int class) { + /* Avoid instrumented multi-word find_next_bit() from this file. */ for (; class < STACK_DEPOT_TRIE_FREE_CLASSES; class++) { if (map[class / BITS_PER_LONG] & BIT(class % BITS_PER_LONG)) return class; @@ -1087,18 +1088,16 @@ static int trie_pool_carve(struct stack_depot_trie_insert_alloc *alloc, } static const struct stack_depot_trie_node * -stack_depot_trie_lookup(const struct stack_depot_trie_child_array __rcu * const *root_slot, - const unsigned long *entries, unsigned int nr_entries); +stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries); static depot_stack_handle_t -trie_find_handle(const struct stack_depot_trie_child_array __rcu * const *root_slot, - const unsigned long *entries, unsigned int nr_entries) +trie_find_handle(const unsigned long *entries, unsigned int nr_entries) { depot_stack_handle_t handle = 0; const struct stack_depot_trie_node *leaf; rcu_read_lock_sched_notrace(); - leaf = stack_depot_trie_lookup(root_slot, entries, nr_entries); + leaf = stack_depot_trie_lookup(entries, nr_entries); if (leaf) handle = __stack_depot_trie_handle(leaf->leaf_id); rcu_read_unlock_sched_notrace(); @@ -1699,8 +1698,8 @@ static inline struct stack_record *find_stack(struct list_head *bucket, } static u32 -stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu **root_slot, - const unsigned long *entries, unsigned int nr_entries, +stack_depot_trie_insert_locked(const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc); @@ -1717,7 +1716,7 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, int ret; retry: - handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); + handle = trie_find_handle(entries, nr_entries); if (handle) return handle; @@ -1727,8 +1726,7 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, goto out_free; raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); - leaf_id = stack_depot_trie_insert_locked(&stack_depot_trie_root, - entries, nr_entries, + leaf_id = stack_depot_trie_insert_locked(entries, nr_entries, &pool_prealloc, &side_prealloc); if (leaf_id) handle = __stack_depot_trie_handle(leaf_id); @@ -1802,8 +1800,7 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; if (in_nmi() || !can_alloc) { WARN_ON_ONCE(can_alloc); - return trie_find_handle(&stack_depot_trie_root, entries, - nr_entries); + return trie_find_handle(entries, nr_entries); } return stack_depot_trie_save(entries, nr_entries, alloc_flags); } @@ -2217,13 +2214,12 @@ static void trie_publish_tail_append(struct stack_depot_trie_child_array *array, } static const struct stack_depot_trie_node * -stack_depot_trie_lookup(const struct stack_depot_trie_child_array __rcu * const *root_slot, - const unsigned long *entries, unsigned int nr_entries) +stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) { const struct stack_depot_trie_child_array *children; unsigned int pos = 0; - children = trie_load_children_slot(root_slot); + children = trie_load_children_slot(&stack_depot_trie_root); while (pos < nr_entries) { const struct stack_depot_trie_node *node; @@ -2331,8 +2327,8 @@ static unsigned int trie_size_append_chain(const unsigned long *entries, } static u32 -stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu **root_slot, - const unsigned long *entries, unsigned int nr_entries, +stack_depot_trie_insert_locked(const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { @@ -2340,7 +2336,8 @@ stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array __rcu * struct stack_depot_trie_child_array *slot_array; struct stack_depot_trie_child_array *prefix_children; const struct stack_depot_trie_child_array *children; - const struct stack_depot_trie_child_array __rcu **slot = root_slot; + const struct stack_depot_trie_child_array __rcu **slot = + &stack_depot_trie_root; const struct stack_depot_trie_node *child; const struct stack_depot_trie_node *chain_head; const struct stack_depot_trie_node *chain_leaf; From 7b14626ca9d78fa500913dce2f514c85ad68f2c7 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 20 Jul 2026 18:46:02 +0100 Subject: [PATCH 21/38] KRN-1117: Correct stackdepot post-put documentation Refcounted stack records can be recycled after their RCU grace period while retaining the same encoded handle. A post-put fetch can therefore resolve to an unrelated replacement stack instead of reliably returning zero. Describe post-put handles as invalid with undefined results and keep the defined zero-return cases limited to valid API outcomes. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 2f4bc8873e003..fa9fa66433d24 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -239,8 +239,9 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * example, %CONFIG_STACKDEPOT_MAX_FRAMES or the local stack_trace_save() limit) * when losing diagnostics on an undersized buffer would be surprising. * - * A non-zero invalid or post-put @handle is treated like stack_depot_fetch(): it - * returns 0 and may WARN because such handles indicate a corrupt caller state. + * A non-zero invalid @handle, including a post-put handle, may WARN. Its return + * value and copied contents are undefined because the record may have been + * reused for another stack. * * Callers must ensure @handle remains valid for the duration of this call. * Persistent handles saved without %STACK_DEPOT_FLAG_GET require no extra @@ -248,8 +249,8 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * Callers must not call stack_depot_put() on persistent handles. * Racing this helper with stack_depot_put() on the same handle is invalid. * - * Return: Number of frames copied, 0 if @handle is 0 or invalid, stack depot is - * disabled, or @max_entries is less than the number of stored frames. + * Return: Number of frames copied, 0 if @handle is 0, stack depot is disabled, + * or @max_entries is less than the number of stored frames. */ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, unsigned long *entries, From 628dd4c7fa656d06df807b4bd392da310929affb Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 20 Jul 2026 18:46:24 +0100 Subject: [PATCH 22/38] KRN-1117: Verify stackdepot trie KUnit backend Trie-specific public API tests can otherwise pass after optional trie initialization falls back to hash storage. Add a test-only expected pool split and verify saved handles use the trie namespace. Skip trie-specific cases when no split is supplied so ordinary hash runs do not claim trie topology or compressed-storage coverage. Signed-off-by: Caleb Kan --- lib/tests/stackdepot_kunit.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 9e1df756d766f..c14845f53250a 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -4,12 +4,17 @@ #include #include #include +#include #include #include #include #include +static int expected_trie_pool_limit = -1; +module_param_named(trie_pool_limit, expected_trie_pool_limit, int, 0); +MODULE_PARM_DESC(trie_pool_limit, "Expected stackdepot hash/trie pool split"); + #ifdef CONFIG_ARM64 #include @@ -278,6 +283,7 @@ static void stackdepot_countable_does_not_alias_other_modes(struct kunit *test) static void stackdepot_trie_topology_roundtrip(struct kunit *test) { + union handle_parts parts; unsigned long stacks[][3] = { { 0x201000UL, 0x202000UL }, { 0x201000UL, 0x203000UL }, @@ -289,14 +295,21 @@ static void stackdepot_trie_topology_roundtrip(struct kunit *test) unsigned int nr_entries[] = { 2, 2, 1, 3, 2, 2 }; depot_stack_handle_t handles[ARRAY_SIZE(stacks)]; unsigned long fetched[ARRAY_SIZE(stacks[0])]; + u32 pool_index_plus_1; unsigned int i; + if (expected_trie_pool_limit < 0) + kunit_skip(test, "trie pool limit was not provided"); KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); for (i = 0; i < ARRAY_SIZE(stacks); i++) { handles[i] = stack_depot_save(stacks[i], nr_entries[i], GFP_KERNEL); KUNIT_ASSERT_NE(test, handles[i], (depot_stack_handle_t)0); } + parts.handle = handles[0]; + pool_index_plus_1 = parts.pool_index_plus_1; + KUNIT_ASSERT_GT(test, pool_index_plus_1, + (u32)expected_trie_pool_limit); for (i = 0; i < ARRAY_SIZE(stacks); i++) { memset(fetched, 0, sizeof(fetched)); @@ -311,8 +324,10 @@ static void stackdepot_trie_topology_roundtrip(struct kunit *test) static void stackdepot_frame_storage_roundtrip(struct kunit *test) { + union handle_parts parts; unsigned long fetched[3] = {}; depot_stack_handle_t handle; + u32 pool_index_plus_1; unsigned int nr_entries; #if defined(CONFIG_ARM64) unsigned long entries[] = { @@ -330,9 +345,15 @@ static void stackdepot_frame_storage_roundtrip(struct kunit *test) unsigned long entries[] = { 0x301000UL, 0x302000UL, 0x303000UL }; #endif + if (expected_trie_pool_limit < 0) + kunit_skip(test, "trie pool limit was not provided"); KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + parts.handle = handle; + pool_index_plus_1 = parts.pool_index_plus_1; + KUNIT_ASSERT_GT(test, pool_index_plus_1, + (u32)expected_trie_pool_limit); nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); From fa5c309a723b4bdf65ecf4520794db30c2f2a44a Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 21 Jul 2026 09:26:54 +0100 Subject: [PATCH 23/38] KRN-1117: Minimize the stackdepot DRM conversion The DRM modeset-lock caller only needs to switch from direct fetch to the backend-neutral stack_depot_snprint() API. Restore its existing literal frame-array size and remove the one-use macro and comment so the port carries no unrelated caller abstraction. Signed-off-by: Caleb Kan --- drivers/gpu/drm/drm_modeset_lock.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c index e43202df77e30..c614ca6fb7f6b 100644 --- a/drivers/gpu/drm/drm_modeset_lock.c +++ b/drivers/gpu/drm/drm_modeset_lock.c @@ -81,12 +81,9 @@ static DEFINE_WW_CLASS(crtc_ww_class); #if IS_ENABLED(CONFIG_DRM_DEBUG_MODESET_LOCK) -/* Modeset-lock diagnostics only need a short caller chain. */ -#define DRM_STACK_DEPOT_MAX_FRAMES 8 - static noinline depot_stack_handle_t __drm_stack_depot_save(void) { - unsigned long entries[DRM_STACK_DEPOT_MAX_FRAMES]; + unsigned long entries[8]; unsigned int n; n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); From 57d5cb4cc484a3b0057c7e59b60b8dd35ab832f7 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 21 Jul 2026 09:27:23 +0100 Subject: [PATCH 24/38] KRN-1117: Avoid over-specifying stackdepot compression failure Compression output is valid only when the architecture hook returns true. Requiring a failed compression attempt to preserve the caller-provided payload adds a contract that no stackdepot caller needs. Keep the raw-fallback result check while allowing architecture hooks to clobber the output payload on failure. Signed-off-by: Caleb Kan --- lib/tests/stackdepot_kunit.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index c14845f53250a..62ca87d241e1e 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -364,7 +364,7 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0x1000UL; bool compressed; - u32 payload = 0xfeedbeef; + u32 payload; #ifdef CONFIG_ARM64 frame = (unsigned long)_text + (unsigned long)S32_MAX + 1UL; @@ -372,7 +372,6 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) compressed = arch_stack_depot_frame_try_compress(frame, &payload); KUNIT_EXPECT_FALSE(test, compressed); - KUNIT_EXPECT_EQ(test, payload, (u32)0xfeedbeef); } #ifdef CONFIG_X86_64 From ee43f244688e8002b5d0ae801303ade555715e2e Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 21 Jul 2026 15:40:18 +0100 Subject: [PATCH 25/38] KRN-1117: Simplify stackdepot trie reuse with bitmap slots Replace the variable-size trie recycler and its static size-class tables with fixed 16-byte slots tracked by one bitmap per trie pool. Keep retired slots reserved until their RCU grace period completes, then make them available for first-fit reuse. This removes best-fit lists, tail splitting, fragment bookkeeping, and ten allocator helpers while retaining pool accounting and local unwind for unpublished insertions. Production boots used 69-72 pools compared with 69-70 for the recycler and 90-92 for hash storage. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 508 +++++++++++++++-------------------------------- 1 file changed, 165 insertions(+), 343 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index b4b6bce4baebc..6373538e10bd0 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -72,13 +72,13 @@ static unsigned int stack_bucket_number_order; /* Hash mask for indexing the table. */ static unsigned int stack_hash_mask; -/* Array of memory regions that store stack records. */ +/* Array of memory regions used by both stack depot backends. */ static void **stack_pools; /* Newly allocated pool that is not yet added to stack_pools. */ static void *new_pool; /* Number of pools in stack_pools. */ static int pools_num; -/* Offset to the unused space in the currently used pool. */ +/* Offset to unused hash storage in the current pool. */ static size_t pool_offset = DEPOT_POOL_SIZE; /* Freelist of stack records within stack_pools. */ static LIST_HEAD(free_stacks); @@ -147,44 +147,34 @@ struct stack_depot_trie_child_array { const struct stack_depot_trie_node __rcu *children[]; }; -/* Headerless reusable storage for trie nodes. */ -struct stack_depot_trie_free_node { +/* A retired child array carries an optional node through its RCU grace period. */ +struct stack_depot_trie_retired_array { struct list_head list; -}; - -/* - * Reusable object storage for child arrays and other payloads that need an - * object header. A retired child array can carry the old child node that was - * replaced with it; both become reusable after the array's RCU grace period. - */ -struct stack_depot_trie_free_object { - struct list_head list; - /* - * @size is valid for freshly allocated and reusable objects. Retirement - * overwrites it with the fields below, which remain valid while the - * object is on pending_trie_objects. - */ - union { - size_t size; - struct { - unsigned long rcu_state; - struct stack_depot_trie_node *pending_node; - }; - }; + unsigned long rcu_state; + struct stack_depot_trie_node *pending_node; unsigned char data[]; }; -static_assert(sizeof(struct stack_depot_trie_node) >= - sizeof(struct stack_depot_trie_free_node)); -static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_free_object, data), +static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_retired_array, data), 1UL << DEPOT_STACK_ALIGN)); #define STACK_DEPOT_TRIE_MAX_NODES (CONFIG_STACKDEPOT_MAX_FRAMES + 1) #define STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS CONFIG_STACKDEPOT_MAX_FRAMES +#define STACK_DEPOT_TRIE_SLOT_SIZE BIT(DEPOT_STACK_ALIGN) +#define STACK_DEPOT_TRIE_POOL_SLOTS \ + (DEPOT_POOL_SIZE / STACK_DEPOT_TRIE_SLOT_SIZE) -/* Size classes bucket reusable trie storage by aligned allocation size. */ -#define STACK_DEPOT_TRIE_FREE_CLASSES \ - ((DEPOT_POOL_SIZE >> DEPOT_STACK_ALIGN) + 1) +struct stack_depot_trie_pool { + struct list_head list; + unsigned int free_slots; + DECLARE_BITMAP(used, STACK_DEPOT_TRIE_POOL_SLOTS); +}; + +#define STACK_DEPOT_TRIE_POOL_FIRST_SLOT \ + DIV_ROUND_UP(sizeof(struct stack_depot_trie_pool), \ + STACK_DEPOT_TRIE_SLOT_SIZE) + +static_assert(STACK_DEPOT_TRIE_POOL_FIRST_SLOT < STACK_DEPOT_TRIE_POOL_SLOTS); /* Writer-owned storage for one unpublished trie insertion. */ struct stack_depot_trie_insert_alloc { @@ -207,19 +197,9 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); #define DEPOT_POOL_INDEX_MASK ((1U << DEPOT_POOL_INDEX_BITS) - 1) #define DEPOT_OFFSET_MASK ((1U << DEPOT_OFFSET_BITS) - 1) -/* - * Trie storage is suballocated from stackdepot pools, not slab caches, so pool - * pressure stays visible through stack_depot_max_pools. Trie COW insertion - * retires child arrays and sometimes the node they replaced. Objects carry the - * RCU cookie for child-array payloads; headerless node fragments either live - * directly on free_trie_nodes or are attached to a pending object until that - * object's grace period has elapsed. - */ -static struct list_head free_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; -static struct list_head free_trie_nodes[STACK_DEPOT_TRIE_FREE_CLASSES]; -static LIST_HEAD(pending_trie_objects); -static DECLARE_BITMAP(free_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); -static DECLARE_BITMAP(free_trie_node_map, STACK_DEPOT_TRIE_FREE_CLASSES); +/* Retired fixed-size slots remain reserved until their RCU grace period ends. */ +static LIST_HEAD(stack_depot_trie_pools); +static LIST_HEAD(pending_trie_arrays); static u32 trie_side_table_max_id; @@ -525,17 +505,6 @@ static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) return 0; } -static void trie_free_object_buckets_init(void) -{ - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) { - INIT_LIST_HEAD(&free_trie_objects[i]); - INIT_LIST_HEAD(&free_trie_nodes[i]); - } - INIT_LIST_HEAD(&pending_trie_objects); -} - static int __init stack_depot_trie_init_memblock(void) { struct stack_depot_trie_insert_alloc *alloc; @@ -554,7 +523,6 @@ static int __init stack_depot_trie_init_memblock(void) } stack_depot_trie_alloc = alloc; - trie_free_object_buckets_init(); static_branch_enable(&stack_depot_trie_enabled); return 0; } @@ -578,7 +546,6 @@ static int stack_depot_trie_init(gfp_t gfp_flags) } stack_depot_trie_alloc = alloc; - trie_free_object_buckets_init(); static_branch_enable(&stack_depot_trie_enabled); return 0; } @@ -666,257 +633,169 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( rcu_read_lock_sched_held()); } -static size_t __stack_depot_trie_pool_alloc_size(size_t size) +static inline size_t trie_array_alloc_size(size_t size) { - if (WARN_ON_ONCE(!size || size > DEPOT_POOL_SIZE)) - return 0; - - return ALIGN(size, 1UL << DEPOT_STACK_ALIGN); + return offsetof(struct stack_depot_trie_retired_array, data) + size; } -static inline size_t trie_object_header_size(void) +static inline struct stack_depot_trie_retired_array * +trie_retired_array(const void *ptr) { - return offsetof(struct stack_depot_trie_free_object, data); + return container_of(ptr, struct stack_depot_trie_retired_array, data); } -static size_t trie_object_alloc_size(size_t size) -{ - size_t alloc_size; - - size = __stack_depot_trie_pool_alloc_size(size); - if (!size) - return 0; - alloc_size = trie_object_header_size() + size; - return alloc_size <= DEPOT_POOL_SIZE ? alloc_size : 0; -} +static bool depot_init_pool(void **prealloc); -static inline struct stack_depot_trie_free_object *trie_object_header(const void *ptr) +static unsigned int trie_pool_reserve_slots(struct stack_depot_trie_pool *pool, + unsigned int nr_slots) { - return container_of(ptr, struct stack_depot_trie_free_object, data); -} + unsigned int run_start = STACK_DEPOT_TRIE_POOL_FIRST_SLOT; + unsigned int run = 0; + unsigned int i; + unsigned int slot; -static inline unsigned int trie_free_class(size_t size) -{ - size = __stack_depot_trie_pool_alloc_size(size); - return size >> DEPOT_STACK_ALIGN; -} + if (pool->free_slots < nr_slots) + return STACK_DEPOT_TRIE_POOL_SLOTS; -static void trie_free_list_add(struct list_head *entry, struct list_head *heads, - unsigned long *map, unsigned int class) -{ - lockdep_assert_held(&pool_lock); + for (slot = STACK_DEPOT_TRIE_POOL_FIRST_SLOT; + slot < STACK_DEPOT_TRIE_POOL_SLOTS; slot++) { + if (pool->used[slot / BITS_PER_LONG] & + BIT(slot % BITS_PER_LONG)) { + run = 0; + continue; + } + if (!run) + run_start = slot; + if (++run != nr_slots) + continue; + + for (i = run_start; i < run_start + nr_slots; i++) + pool->used[i / BITS_PER_LONG] |= BIT(i % BITS_PER_LONG); + pool->free_slots -= nr_slots; + return run_start; + } - list_add(entry, &heads[class]); - __set_bit(class, map); + return STACK_DEPOT_TRIE_POOL_SLOTS; } -static void trie_free_list_del(struct list_head *entry, struct list_head *heads, - unsigned long *map, unsigned int class) +static void *trie_pool_alloc(size_t size, void **prealloc) { - lockdep_assert_held(&pool_lock); + struct stack_depot_trie_pool *pool; + unsigned int nr_slots; + unsigned int slot; - list_del_init(entry); - if (list_empty(&heads[class])) - __clear_bit(class, map); -} - -static bool depot_init_pool(void **prealloc); + lockdep_assert_held(&pool_lock); -static unsigned int trie_find_next_set_class(unsigned long *map, unsigned int class) -{ - /* Avoid instrumented multi-word find_next_bit() from this file. */ - for (; class < STACK_DEPOT_TRIE_FREE_CLASSES; class++) { - if (map[class / BITS_PER_LONG] & BIT(class % BITS_PER_LONG)) - return class; + nr_slots = DIV_ROUND_UP(size, STACK_DEPOT_TRIE_SLOT_SIZE); + list_for_each_entry_reverse(pool, &stack_depot_trie_pools, list) { + slot = trie_pool_reserve_slots(pool, nr_slots); + if (slot != STACK_DEPOT_TRIE_POOL_SLOTS) + return (char *)pool + slot * STACK_DEPOT_TRIE_SLOT_SIZE; } - return STACK_DEPOT_TRIE_FREE_CLASSES; -} - -static void *trie_object_init_fresh(void *ptr, size_t size) -{ - struct stack_depot_trie_free_object *free = ptr; + if (!depot_init_pool(prealloc)) + return NULL; + pool = stack_pools[pools_num - 1]; + /* Keep hash records out of this bitmap-owned pool. */ + pool_offset = DEPOT_POOL_SIZE; + memset(pool, 0, sizeof(*pool)); + pool->free_slots = STACK_DEPOT_TRIE_POOL_SLOTS - + STACK_DEPOT_TRIE_POOL_FIRST_SLOT; + list_add_tail(&pool->list, &stack_depot_trie_pools); - free->size = __stack_depot_trie_pool_alloc_size(size); - INIT_LIST_HEAD(&free->list); - return free->data; + slot = trie_pool_reserve_slots(pool, nr_slots); + return (char *)pool + slot * STACK_DEPOT_TRIE_SLOT_SIZE; } -static void trie_free_object_locked(const void *ptr) +static void trie_pool_release(const void *ptr, size_t size) { - struct stack_depot_trie_free_object *free; - unsigned int class; + struct stack_depot_trie_pool *pool; + unsigned long pfn; + unsigned int nr_slots; + unsigned int slot; + unsigned int i; lockdep_assert_held(&pool_lock); - free = trie_object_header(ptr); - class = trie_free_class(free->size); - INIT_LIST_HEAD(&free->list); - trie_free_list_add(&free->list, free_trie_objects, - free_trie_object_map, class); + pfn = page_to_pfn(virt_to_page(ptr)); + pfn &= ~(BIT(DEPOT_POOL_ORDER) - 1); + pool = page_address(pfn_to_page(pfn)); + slot = ((unsigned long)ptr - (unsigned long)pool) >> DEPOT_STACK_ALIGN; + nr_slots = DIV_ROUND_UP(size, STACK_DEPOT_TRIE_SLOT_SIZE); + for (i = slot; i < slot + nr_slots; i++) + pool->used[i / BITS_PER_LONG] &= ~BIT(i % BITS_PER_LONG); + pool->free_slots += nr_slots; } -static void trie_add_free_node_locked(void *ptr, size_t size) +static struct stack_depot_trie_child_array * +trie_pool_alloc_array(size_t size, void **prealloc) { - struct stack_depot_trie_free_node *free = ptr; - unsigned int class; - - lockdep_assert_held(&pool_lock); - - size = __stack_depot_trie_pool_alloc_size(size); - if (WARN_ON_ONCE(size < sizeof(*free))) - return; + struct stack_depot_trie_retired_array *retired; - INIT_LIST_HEAD(&free->list); - class = trie_free_class(size); - trie_free_list_add(&free->list, free_trie_nodes, free_trie_node_map, - class); + retired = trie_pool_alloc(trie_array_alloc_size(size), prealloc); + return retired ? (void *)retired->data : NULL; } -static void trie_drain_free_object_node_locked(struct stack_depot_trie_free_object *free) +static void trie_pool_release_array(const void *ptr, size_t size) { - size_t size; - - lockdep_assert_held(&pool_lock); - - if (!free->pending_node) - return; - size = trie_node_bytes(&free->pending_node->run); - trie_add_free_node_locked(free->pending_node, size); - free->pending_node = NULL; + trie_pool_release(trie_retired_array(ptr), trie_array_alloc_size(size)); } -static void trie_drain_pending_objects_locked(void) +static void trie_drain_pending_arrays(void) { + struct stack_depot_trie_retired_array *retired; + struct stack_depot_trie_retired_array *tmp; struct stack_depot_trie_child_array *array; - struct stack_depot_trie_free_object *free; - struct stack_depot_trie_free_object *tmp; - unsigned int class; lockdep_assert_held(&pool_lock); - list_for_each_entry_safe(free, tmp, &pending_trie_objects, list) { - /* Pending objects are FIFO; later entries cannot be ready yet. */ - if (!poll_state_synchronize_rcu(free->rcu_state)) + list_for_each_entry_safe(retired, tmp, &pending_trie_arrays, list) { + /* Pending arrays are FIFO; later entries cannot be ready yet. */ + if (!poll_state_synchronize_rcu(retired->rcu_state)) break; - trie_drain_free_object_node_locked(free); - array = (struct stack_depot_trie_child_array *)free->data; - free->size = trie_child_array_bytes(array->capacity); - class = trie_free_class(free->size); - list_del_init(&free->list); - trie_free_list_add(&free->list, free_trie_objects, - free_trie_object_map, class); + array = (void *)retired->data; + list_del(&retired->list); + if (retired->pending_node) + trie_pool_release(retired->pending_node, + trie_node_bytes(&retired->pending_node->run)); + trie_pool_release_array(array, + trie_child_array_bytes(array->capacity)); } } -static void trie_free_object_tail_locked(void *ptr, size_t size) -{ - struct stack_depot_trie_free_object *free = ptr; - size_t header_size = trie_object_header_size(); - size_t tail_size; - - lockdep_assert_held(&pool_lock); - - if (size >= header_size + (1UL << DEPOT_STACK_ALIGN)) { - tail_size = size - header_size; - free->size = tail_size; - INIT_LIST_HEAD(&free->list); - trie_free_list_add(&free->list, free_trie_objects, - free_trie_object_map, - trie_free_class(tail_size)); - return; - } - - trie_add_free_node_locked(ptr, size); -} - -static void *trie_pop_free_node(size_t size) -{ - struct stack_depot_trie_free_node *free; - unsigned int class; - size_t old_size; - - lockdep_assert_held(&pool_lock); - - size = __stack_depot_trie_pool_alloc_size(size); - if (!size) - return NULL; - class = trie_free_class(size); - class = trie_find_next_set_class(free_trie_node_map, class); - if (class >= STACK_DEPOT_TRIE_FREE_CLASSES) - return NULL; - free = list_first_entry(&free_trie_nodes[class], typeof(*free), list); - old_size = (size_t)class << DEPOT_STACK_ALIGN; - trie_free_list_del(&free->list, free_trie_nodes, free_trie_node_map, - class); - if (old_size > size) - trie_free_object_tail_locked((char *)free + size, old_size - size); - return free; -} - static void trie_retire_child_array_locked(const void *ptr) { - struct stack_depot_trie_free_object *free; + struct stack_depot_trie_retired_array *retired; lockdep_assert_held(&pool_lock); - free = trie_object_header(ptr); - free->pending_node = NULL; - free->rcu_state = get_state_synchronize_rcu(); - list_add_tail(&free->list, &pending_trie_objects); + + retired = trie_retired_array(ptr); + retired->pending_node = NULL; + retired->rcu_state = get_state_synchronize_rcu(); + list_add_tail(&retired->list, &pending_trie_arrays); } static void trie_retire_child_array_with_node(const void *ptr, const struct stack_depot_trie_node *node) { - struct stack_depot_trie_free_object *free; + struct stack_depot_trie_retired_array *retired; unsigned long flags; raw_spin_lock_irqsave(&pool_lock, flags); trie_retire_child_array_locked(ptr); - free = trie_object_header(ptr); - free->pending_node = (struct stack_depot_trie_node *)node; + retired = trie_retired_array(ptr); + retired->pending_node = (struct stack_depot_trie_node *)node; raw_spin_unlock_irqrestore(&pool_lock, flags); } -static void *trie_pop_free_object(size_t size) -{ - struct stack_depot_trie_free_object *free; - void *tail; - size_t old_size; - size_t tail_size; - unsigned int class; - - lockdep_assert_held(&pool_lock); - - size = __stack_depot_trie_pool_alloc_size(size); - if (!size) - return NULL; - class = trie_free_class(size); - class = trie_find_next_set_class(free_trie_object_map, class); - if (class >= STACK_DEPOT_TRIE_FREE_CLASSES) - return NULL; - - free = list_first_entry(&free_trie_objects[class], typeof(*free), list); - trie_free_list_del(&free->list, free_trie_objects, - free_trie_object_map, class); - old_size = free->size; - if (old_size > size) { - free->size = size; - tail = free->data + size; - tail_size = old_size - size; - trie_free_object_tail_locked(tail, tail_size); - } - return free->data; -} - /* * Preallocate resources that cannot be allocated while trie writers hold raw * spinlocks. Side-table growth is mandatory before a new leaf ID can be * used, so side-table preallocation failure disables insertion for this - * save. Pool preallocation is opportunistic: reusable trie storage or active - * pool space may still satisfy the insertion, and pool_carve() reports - * -ENOSPC if they do not. + * save. Pool preallocation is opportunistic: reusable trie slots may still + * satisfy the insertion, and trie_pool_alloc_insert() reports -ENOSPC if they + * do not. */ static int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, void **pool_prealloc, @@ -948,39 +827,22 @@ __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, void **pool_prealloc, return 0; } -static int trie_pool_add_object_size(size_t size, size_t *total) -{ - size_t alloc_size; - - alloc_size = trie_object_alloc_size(size); - if (!alloc_size) - return -ENOSPC; - *total += alloc_size; - return *total <= DEPOT_POOL_SIZE ? 0 : -ENOSPC; -} - /* - * Allocate pool-backed storage for one trie insertion. Reusable retired - * fragments are preferred; any missing storage is carved as one contiguous range - * from the current stackdepot pool, possibly after installing @prealloc as a new - * pool. If allocation fails, any free-list pops are returned locally and - * pool_offset is not advanced. + * Reserve fixed-size pool slots for one trie insertion. If any reservation + * fails, all slots reserved by this attempt are released locally. * The caller must not publish any returned storage before side-table and trie * publication succeeds. */ -static int trie_pool_carve(struct stack_depot_trie_insert_alloc *alloc, - void **pool_prealloc, unsigned int nr_nodes, - unsigned int nr_chain_arrays, - size_t prefix_children_size, - size_t slot_array_size) +static int trie_pool_alloc_insert(struct stack_depot_trie_insert_alloc *alloc, + void **pool_prealloc, + unsigned int nr_nodes, + unsigned int nr_chain_arrays, + size_t prefix_children_size, + size_t slot_array_size) { unsigned long flags; size_t one_child_size; unsigned int i; - size_t alloc_size; - size_t offset; - size_t total = 0; - void *pool; int ret = -ENOSPC; memset(alloc->nodes, 0, nr_nodes * sizeof(*alloc->nodes)); @@ -991,76 +853,32 @@ static int trie_pool_carve(struct stack_depot_trie_insert_alloc *alloc, raw_spin_lock_irqsave(&pool_lock, flags); printk_deferred_enter(); - trie_drain_pending_objects_locked(); + trie_drain_pending_arrays(); one_child_size = trie_child_array_bytes(1); for (i = 0; i < nr_nodes; i++) { - alloc->nodes[i] = trie_pop_free_node(alloc->node_sizes[i]); - if (!alloc->nodes[i]) { - alloc_size = __stack_depot_trie_pool_alloc_size(alloc->node_sizes[i]); - if (!alloc_size) - goto out_discard; - total += alloc_size; - if (total > DEPOT_POOL_SIZE) - goto out_discard; - } + alloc->nodes[i] = trie_pool_alloc(alloc->node_sizes[i], + pool_prealloc); + if (!alloc->nodes[i]) + goto out_discard; } for (i = 0; i < nr_chain_arrays; i++) { - alloc->chain_arrays[i] = trie_pop_free_object(one_child_size); - if (!alloc->chain_arrays[i] && - trie_pool_add_object_size(one_child_size, &total)) + alloc->chain_arrays[i] = + trie_pool_alloc_array(one_child_size, pool_prealloc); + if (!alloc->chain_arrays[i]) goto out_discard; } if (prefix_children_size) { - alloc->prefix_children = trie_pop_free_object(prefix_children_size); - if (!alloc->prefix_children && - trie_pool_add_object_size(prefix_children_size, &total)) + alloc->prefix_children = + trie_pool_alloc_array(prefix_children_size, pool_prealloc); + if (!alloc->prefix_children) goto out_discard; } if (slot_array_size) { - alloc->slot_array = trie_pop_free_object(slot_array_size); - if (!alloc->slot_array && - trie_pool_add_object_size(slot_array_size, &total)) + alloc->slot_array = + trie_pool_alloc_array(slot_array_size, pool_prealloc); + if (!alloc->slot_array) goto out_discard; } - - if (pools_num < 1) { - if (!depot_init_pool(pool_prealloc)) { - ret = -ENOSPC; - goto out_discard; - } - } - if (total > DEPOT_POOL_SIZE - pool_offset) { - if (!depot_init_pool(pool_prealloc)) { - ret = -ENOSPC; - goto out_discard; - } - } - - pool = stack_pools[pools_num - 1]; - - offset = pool_offset; - for (i = 0; i < nr_nodes; i++) { - if (alloc->nodes[i]) - continue; - alloc->nodes[i] = pool + offset; - offset += __stack_depot_trie_pool_alloc_size(alloc->node_sizes[i]); - } - for (i = 0; i < nr_chain_arrays; i++) { - if (alloc->chain_arrays[i]) - continue; - alloc->chain_arrays[i] = - trie_object_init_fresh(pool + offset, one_child_size); - offset += trie_object_alloc_size(one_child_size); - } - if (prefix_children_size && !alloc->prefix_children) { - alloc->prefix_children = - trie_object_init_fresh(pool + offset, prefix_children_size); - offset += trie_object_alloc_size(prefix_children_size); - } - if (slot_array_size && !alloc->slot_array) - alloc->slot_array = - trie_object_init_fresh(pool + offset, slot_array_size); - pool_offset += total; ret = 0; goto out; out_discard: @@ -1068,19 +886,20 @@ static int trie_pool_carve(struct stack_depot_trie_insert_alloc *alloc, struct stack_depot_trie_node *node = alloc->nodes[i]; if (node) - trie_add_free_node_locked(node, alloc->node_sizes[i]); + trie_pool_release(node, alloc->node_sizes[i]); } for (i = 0; i < nr_chain_arrays; i++) { struct stack_depot_trie_child_array *array = alloc->chain_arrays[i]; if (!array) continue; - trie_free_object_locked(array); + trie_pool_release_array(array, one_child_size); } - if (prefix_children_size && alloc->prefix_children) - trie_free_object_locked(alloc->prefix_children); - if (slot_array_size && alloc->slot_array) - trie_free_object_locked(alloc->slot_array); + if (alloc->prefix_children) + trie_pool_release_array(alloc->prefix_children, + prefix_children_size); + if (alloc->slot_array) + trie_pool_release_array(alloc->slot_array, slot_array_size); out: printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -2364,8 +2183,9 @@ stack_depot_trie_insert_locked(const unsigned long *entries, slot_array_size = trie_child_array_bytes(1); nr_nodes = trie_size_append_chain(entries, nr_entries, alloc->node_sizes); - if (trie_pool_carve(alloc, pool_prealloc, nr_nodes, - nr_nodes - 1, 0, slot_array_size)) + if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, + nr_nodes - 1, 0, + slot_array_size)) return 0; slot_array = alloc->slot_array; trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, @@ -2398,8 +2218,9 @@ stack_depot_trie_insert_locked(const unsigned long *entries, return 0; nr_nodes = trie_size_append_chain(entries, nr_entries, alloc->node_sizes); - if (trie_pool_carve(alloc, pool_prealloc, nr_nodes, - nr_nodes - 1, 0, slot_array_size)) + if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, + nr_nodes - 1, 0, + slot_array_size)) return 0; trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, alloc->nodes, alloc->chain_arrays, @@ -2441,10 +2262,11 @@ stack_depot_trie_insert_locked(const unsigned long *entries, new_nodes = trie_size_append_chain(&entries[matched], nr_entries - matched, &alloc->node_sizes[2]); - if (trie_pool_carve(alloc, pool_prealloc, 2 + new_nodes, - new_nodes ? new_nodes - 1 : 0, - trie_child_array_bytes(has_new_tail ? 2 : 1), - trie_child_array_bytes(children->capacity))) + if (trie_pool_alloc_insert(alloc, pool_prealloc, + 2 + new_nodes, + new_nodes ? new_nodes - 1 : 0, + trie_child_array_bytes(has_new_tail ? 2 : 1), + trie_child_array_bytes(children->capacity))) return 0; slot_array = alloc->slot_array; prefix_children = alloc->prefix_children; @@ -2469,8 +2291,8 @@ stack_depot_trie_insert_locked(const unsigned long *entries, return 0; alloc->node_sizes[0] = trie_node_bytes(&child->run); slot_array_size = trie_child_array_bytes(children->capacity); - if (trie_pool_carve(alloc, pool_prealloc, 1, 0, 0, - slot_array_size)) + if (trie_pool_alloc_insert(alloc, pool_prealloc, 1, 0, 0, + slot_array_size)) return 0; promoted_node = alloc->nodes[0]; slot_array = alloc->slot_array; From 58bdc01c6d56fc9b0cf0b8fc6ee7df67ffc537ab Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 21 Jul 2026 17:30:55 +0100 Subject: [PATCH 26/38] KRN-1117: Remove remaining stackdepot helper indirection Inline the static-key check and side-table root sizing because each helper only hides one expression used at two call sites. Keep the public API comments backend-neutral now that handles may refer to either hash records or trie leaves. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 4 ++-- lib/stackdepot.c | 18 ++++-------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index fa9fa66433d24..788737eb0c4a2 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -163,7 +163,7 @@ static inline int stack_depot_early_init(void) { return 0; } * this is the case for contexts where neither %GFP_ATOMIC nor * %GFP_NOWAIT can be used (NMI, raw_spin_lock). * - * Return: Handle of the stack struct stored in depot, 0 on failure + * Return: Handle of the stack trace stored in depot, 0 on failure */ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, unsigned int nr_entries, @@ -223,7 +223,7 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, /** * stack_depot_fetch_into - Fetch a stack trace into caller-owned storage * - * @handle: Stack depot handle returned from stack_depot_save() + * @handle: Stack depot handle * @entries: Caller-owned buffer to copy the stack trace into * @max_entries: Number of frames that fit in @entries * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 6373538e10bd0..6a7f2678d040e 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -289,11 +289,6 @@ static u32 trie_side_table_last_leaf_id; /* Lock order: writer_lock -> pool_lock -> trie_side_table_lock. */ -static inline bool __stack_depot_trie_enabled(void) -{ - return static_branch_unlikely(&stack_depot_trie_enabled); -} - static inline size_t stack_depot_frame_run_entry_bytes(enum stack_depot_frame_mode mode) { if (mode == STACK_DEPOT_FRAME_COMPRESSED) @@ -422,11 +417,6 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo return id; } -static inline size_t trie_side_table_root_bytes(unsigned int root_size) -{ - return struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); -} - static void trie_side_table_root_init(struct stack_depot_trie_side_root *root_vec, unsigned int root_size, u32 max_id) { @@ -456,7 +446,7 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) if (!max_leaf_id) return -EINVAL; root_size = trie_side_table_root_size_for_max_id(max_leaf_id); - root_bytes = trie_side_table_root_bytes(root_size); + root_bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); root_vec = memblock_alloc(root_bytes, __alignof__(*root_vec)); if (!root_vec) @@ -495,7 +485,7 @@ static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) return -EINVAL; root_size = trie_side_table_root_size_for_max_id(max_leaf_id); - root_bytes = trie_side_table_root_bytes(root_size); + root_bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); root_vec = kvzalloc(root_bytes, gfp_flags); if (!root_vec) return -ENOMEM; @@ -532,7 +522,7 @@ static int stack_depot_trie_init(gfp_t gfp_flags) struct stack_depot_trie_insert_alloc *alloc; int ret; - if (__stack_depot_trie_enabled()) + if (static_branch_unlikely(&stack_depot_trie_enabled)) return 0; alloc = kvzalloc(sizeof(*stack_depot_trie_alloc), gfp_flags); @@ -1613,7 +1603,7 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, return 0; trie_candidate = !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE)) && - __stack_depot_trie_enabled(); + static_branch_unlikely(&stack_depot_trie_enabled); if (trie_candidate) { if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; From fbaf11709d0d0c09741fe3f414c070f98feafbf3 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 21 Jul 2026 17:31:48 +0100 Subject: [PATCH 27/38] KRN-1117: Consolidate stackdepot KUnit coverage Fold countable fetch, direct-record, and namespace-isolation assertions into one public behavior test. Inline one-use no-allocation and flag wrappers so the suite exposes the API calls directly. Preserve every distinct assertion while reducing the suite from eleven cases to nine. Signed-off-by: Caleb Kan --- lib/tests/stackdepot_kunit.c | 99 +++++++++--------------------------- 1 file changed, 23 insertions(+), 76 deletions(-) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 62ca87d241e1e..bffa34198a30c 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -89,55 +89,18 @@ static void stackdepot_fetch_into_rejects_missing_or_short_stack(struct kunit *t KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); } -static depot_stack_handle_t save_countable(unsigned long *entries, unsigned int nr) -{ - depot_flags_t flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE; - - return stack_depot_save_flags(entries, nr, GFP_KERNEL, flags); -} - -static void stackdepot_countable_flag_roundtrip(struct kunit *test) -{ - unsigned long entries[] = { - 0x121000UL, - 0x122000UL, - 0x123000UL, - }; - unsigned long fetched[ARRAY_SIZE(entries)] = {}; - depot_stack_handle_t handle; - unsigned int nr_entries; - - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - - handle = save_countable(entries, ARRAY_SIZE(entries)); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - - nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); - KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); -} - -static depot_stack_handle_t save_noalloc(unsigned long *entries, unsigned int nr) -{ - gfp_t no_spin = GFP_NOWAIT & ~__GFP_RECLAIM; - - return stack_depot_save_flags(entries, nr, no_spin, 0); -} - static void stackdepot_save_flags_public(struct kunit *test) { unsigned long entries[] = { 0x501000UL, 0x502000UL, 0x503000UL }; unsigned long get_entries[] = { 0x601000UL, 0x602000UL }; - unsigned long count_entries[] = { 0x603000UL, 0x604000UL }; unsigned long fetched[ARRAY_SIZE(entries)] = {}; depot_stack_handle_t noalloc_handle; depot_stack_handle_t overlong_handle; - depot_stack_handle_t count_handle; depot_stack_handle_t plain_handle; depot_stack_handle_t get_handle; depot_stack_handle_t again; depot_stack_handle_t extra; - depot_flags_t flags; + gfp_t no_spin = GFP_NOWAIT & ~__GFP_RECLAIM; unsigned long *overlong_fetched; unsigned long *overlong_entries; unsigned int overlong_nr = CONFIG_STACKDEPOT_MAX_FRAMES + 1; @@ -165,21 +128,16 @@ static void stackdepot_save_flags_public(struct kunit *test) KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); - noalloc_handle = save_noalloc(entries, ARRAY_SIZE(entries)); + noalloc_handle = stack_depot_save_flags(entries, ARRAY_SIZE(entries), no_spin, 0); KUNIT_EXPECT_EQ(test, noalloc_handle, plain_handle); - flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; get_handle = stack_depot_save_flags(get_entries, ARRAY_SIZE(get_entries), - GFP_KERNEL, flags); + GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC | + STACK_DEPOT_FLAG_GET); KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); stack_depot_put(get_handle); - flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE; - count_handle = stack_depot_save_flags(count_entries, - ARRAY_SIZE(count_entries), - GFP_KERNEL, flags); - KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); - overlong_handle = stack_depot_save(overlong_entries, overlong_nr, GFP_KERNEL); KUNIT_ASSERT_NE(test, overlong_handle, (depot_stack_handle_t)0); @@ -218,28 +176,7 @@ static void stackdepot_snprint_public(struct kunit *test) KUNIT_EXPECT_STREQ(test, actual, expected); } -static void stackdepot_get_stack_record(struct kunit *test) -{ - unsigned long entries[] = { - 0x131000UL, - 0x132000UL, - 0x133000UL, - }; - struct stack_record *record; - depot_stack_handle_t handle; - - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - - handle = save_countable(entries, ARRAY_SIZE(entries)); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - - record = __stack_depot_get_stack_record(handle); - KUNIT_ASSERT_NOT_NULL(test, record); - KUNIT_EXPECT_EQ(test, record->size, (u16)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, record->entries, entries, sizeof(entries)); -} - -static void stackdepot_countable_does_not_alias_other_modes(struct kunit *test) +static void stackdepot_countable_public(struct kunit *test) { unsigned long plain_entries[] = { 0x141000UL, @@ -251,28 +188,40 @@ static void stackdepot_countable_does_not_alias_other_modes(struct kunit *test) 0x152000UL, 0x153000UL, }; - depot_flags_t get = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; + unsigned long fetched[ARRAY_SIZE(plain_entries)] = {}; + depot_flags_t countable = STACK_DEPOT_FLAG_CAN_ALLOC | + STACK_DEPOT_FLAG_COUNTABLE; struct stack_record *record; depot_stack_handle_t count_handle; depot_stack_handle_t plain_handle; depot_stack_handle_t get_handle; unsigned int get_nr = ARRAY_SIZE(get_entries); unsigned int plain_nr = ARRAY_SIZE(plain_entries); + unsigned int nr_entries; KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); plain_handle = stack_depot_save(plain_entries, plain_nr, GFP_KERNEL); KUNIT_ASSERT_NE(test, plain_handle, (depot_stack_handle_t)0); - count_handle = save_countable(plain_entries, plain_nr); + count_handle = stack_depot_save_flags(plain_entries, plain_nr, GFP_KERNEL, + countable); KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); record = __stack_depot_get_stack_record(count_handle); KUNIT_ASSERT_NOT_NULL(test, record); + KUNIT_EXPECT_EQ(test, record->size, (u16)plain_nr); KUNIT_EXPECT_MEMEQ(test, record->entries, plain_entries, sizeof(plain_entries)); + nr_entries = stack_depot_fetch_into(count_handle, fetched, + ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, plain_nr); + KUNIT_EXPECT_MEMEQ(test, fetched, plain_entries, sizeof(plain_entries)); - get_handle = stack_depot_save_flags(get_entries, get_nr, GFP_KERNEL, get); + get_handle = stack_depot_save_flags(get_entries, get_nr, GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC | + STACK_DEPOT_FLAG_GET); KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); - count_handle = save_countable(get_entries, get_nr); + count_handle = stack_depot_save_flags(get_entries, get_nr, GFP_KERNEL, + countable); KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); record = __stack_depot_get_stack_record(count_handle); KUNIT_ASSERT_NOT_NULL(test, record); @@ -430,11 +379,9 @@ static void stackdepot_frame_arm64(struct kunit *test) static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), KUNIT_CASE(stackdepot_fetch_into_rejects_missing_or_short_stack), - KUNIT_CASE(stackdepot_countable_flag_roundtrip), KUNIT_CASE(stackdepot_save_flags_public), KUNIT_CASE(stackdepot_snprint_public), - KUNIT_CASE(stackdepot_get_stack_record), - KUNIT_CASE(stackdepot_countable_does_not_alias_other_modes), + KUNIT_CASE(stackdepot_countable_public), KUNIT_CASE(stackdepot_trie_topology_roundtrip), KUNIT_CASE(stackdepot_frame_storage_roundtrip), KUNIT_CASE(stackdepot_frame_raw_fallback), From aac13cdcc47d1d053bd6de555d296ac33b5a552a Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 22 Jul 2026 10:16:00 +0100 Subject: [PATCH 28/38] KRN-1117: Simplify stackdepot trie helpers Remove derived side-table limit state, redundant assignments, and over-general storage parameters. Use the authoritative handle limit and RCU publication primitive directly so the implementation carries fewer states and wrappers without changing trie behavior. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 48 ++++++++++++++---------------------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 6a7f2678d040e..9467daa55a79f 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -201,8 +201,6 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); static LIST_HEAD(stack_depot_trie_pools); static LIST_HEAD(pending_trie_arrays); -static u32 trie_side_table_max_id; - /* * stack_max_pools is the split point between hash and trie handle encodings. * A handle with pool_index_plus_1 in 1..stack_max_pools names a hash-backed @@ -244,7 +242,6 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) union handle_parts parts = { .handle = handle }; u32 pool_delta; - parts.extra = 0; pool_delta = parts.pool_index_plus_1 - stack_max_pools - 1; return (pool_delta << DEPOT_OFFSET_BITS) + parts.offset + 1; } @@ -316,11 +313,11 @@ static size_t trie_child_array_bytes(unsigned int capacity) return ALIGN(size, sizeof(unsigned long)); } -static void trie_child_array_init(void *storage, unsigned int capacity, +static void trie_child_array_init(struct stack_depot_trie_child_array *array, + unsigned int capacity, const struct stack_depot_trie_node * const *nodes, unsigned int nr_children) { - struct stack_depot_trie_child_array *array = storage; unsigned int i; array->nr_children = nr_children; @@ -384,7 +381,7 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo raw_spin_lock_irqsave(&trie_side_table_lock, flags); id = trie_side_table_last_leaf_id + 1; - if (id > trie_side_table_max_id) + if (id > __stack_depot_trie_max_leaf_id()) goto out_fail; root_vec = trie_side_table_root; @@ -418,10 +415,9 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo } static void trie_side_table_root_init(struct stack_depot_trie_side_root *root_vec, - unsigned int root_size, u32 max_id) + unsigned int root_size) { root_vec->dir_capacity = root_size; - trie_side_table_max_id = max_id; trie_side_table_last_leaf_id = 0; } @@ -466,7 +462,7 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) } memset(first_chunk, 0, PAGE_SIZE); - trie_side_table_root_init(root_vec, root_size, max_leaf_id); + trie_side_table_root_init(root_vec, root_size); RCU_INIT_POINTER(root_vec->dirs[0], first_dir); RCU_INIT_POINTER(first_dir->chunks[0], first_chunk); trie_side_table_root = root_vec; @@ -490,7 +486,7 @@ static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) if (!root_vec) return -ENOMEM; - trie_side_table_root_init(root_vec, root_size, max_leaf_id); + trie_side_table_root_init(root_vec, root_size); trie_side_table_root = root_vec; return 0; } @@ -586,8 +582,6 @@ static void trie_side_table_put_prealloc(struct stack_depot_trie_side_prealloc * free_page((unsigned long)prealloc->dir); if (prealloc->chunk) free_page((unsigned long)prealloc->chunk); - prealloc->dir = NULL; - prealloc->chunk = NULL; } static const struct stack_depot_trie_node __rcu **trie_side_table_leaf_slot(u32 id) @@ -1579,7 +1573,6 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, void *prealloc = NULL; bool allow_spin = gfpflags_allow_spinning(alloc_flags); bool can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && allow_spin; - bool trie_candidate; unsigned long flags; u32 hash; @@ -1602,9 +1595,8 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, if (unlikely(nr_entries == 0) || stack_depot_disabled) return 0; - trie_candidate = !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE)) && - static_branch_unlikely(&stack_depot_trie_enabled); - if (trie_candidate) { + if (!(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE)) && + static_branch_unlikely(&stack_depot_trie_enabled)) { if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; if (in_nmi() || !can_alloc) { @@ -1750,13 +1742,11 @@ stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, arch_stack_depot_frame_decompress(payload, frame); } -static void trie_node_init(void *storage, +static void trie_node_init(struct stack_depot_trie_node *node, const struct stack_depot_trie_node *parent, u32 leaf_id, const unsigned long *entries, const struct stack_depot_frame_run *run) { - struct stack_depot_trie_node *node = storage; - if (run->mode == STACK_DEPOT_FRAME_COMPRESSED) { unsigned int i; @@ -1777,12 +1767,11 @@ static void trie_node_init(void *storage, node->run = *run; } -static void trie_node_init_slice(void *storage, +static void trie_node_init_slice(struct stack_depot_trie_node *node, const struct stack_depot_trie_node *parent, u32 leaf_id, const struct stack_depot_trie_node *src_node, unsigned int start, unsigned int nr_entries) { - struct stack_depot_trie_node *node = storage; struct stack_depot_frame_run run; size_t entry_bytes; @@ -1862,7 +1851,6 @@ trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, unsigned int left = 0; unsigned int right; - *pos = 0; *found = false; right = READ_ONCE(array->nr_children); @@ -1917,14 +1905,6 @@ trie_child_array_insert_at(const struct stack_depot_trie_child_array *old, RCU_INIT_POINTER(new_array->children[i], NULL); } -static void -trie_publish_children_slot(const struct stack_depot_trie_child_array __rcu **slot, - const struct stack_depot_trie_child_array *children) -{ - /* Publish the fully initialized replacement array last. */ - rcu_assign_pointer(*slot, children); -} - static void trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array, const struct stack_depot_trie_node *new_child, @@ -2185,7 +2165,7 @@ stack_depot_trie_insert_locked(const unsigned long *entries, slot_array->nr_children = 1; slot_array->capacity = 1; RCU_INIT_POINTER(slot_array->children[0], chain_head); - trie_publish_children_slot(slot, slot_array); + rcu_assign_pointer(*slot, slot_array); goto out_success; } trie_child_array_find_slot(children, entries[0], &pos, &found); @@ -2223,7 +2203,7 @@ stack_depot_trie_insert_locked(const unsigned long *entries, slot_array = alloc->slot_array; trie_child_array_insert_at(children, pos, chain_head, slot_array, capacity); - trie_publish_children_slot(slot, slot_array); + rcu_assign_pointer(*slot, slot_array); raw_spin_lock_irqsave(&pool_lock, flags); trie_retire_child_array_locked(children); raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -2269,7 +2249,7 @@ stack_depot_trie_insert_locked(const unsigned long *entries, new_leaf_id, split_leaf); trie_child_array_replace_at(children, split_prefix, slot_array, pos); trie_reparent_children(old_tail); - trie_publish_children_slot(slot, slot_array); + rcu_assign_pointer(*slot, slot_array); trie_retire_child_array_with_node(children, child); goto out_success; } @@ -2291,7 +2271,7 @@ stack_depot_trie_insert_locked(const unsigned long *entries, trie_side_table_publish_new_leaf(new_leaf_id, promoted_node); trie_child_array_replace_at(children, promoted_node, slot_array, pos); trie_reparent_children(promoted_node); - trie_publish_children_slot(slot, slot_array); + rcu_assign_pointer(*slot, slot_array); trie_retire_child_array_with_node(children, child); goto out_success; } From 5ad12153bab0aecd8452f641866714828a148be3 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 22 Jul 2026 10:16:25 +0100 Subject: [PATCH 29/38] KRN-1117: Cover remaining stackdepot trie topology paths Exercise same-capacity interior insertion and splitting a node with existing descendants, then verify that every earlier handle still materializes. Keep the fixtures valid at the KUnit suite's three-frame minimum and move x86 synthetic frames away from real IRQ text ranges. Signed-off-by: Caleb Kan --- lib/tests/stackdepot_kunit.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index bffa34198a30c..221b7b8db3313 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -239,9 +239,14 @@ static void stackdepot_trie_topology_roundtrip(struct kunit *test) { 0x201000UL }, { 0x201000UL, 0x203000UL, 0x204000UL }, { 0x201000UL, 0x205000UL }, + { 0x201000UL, 0x204000UL }, { 0x201000UL, 0x206000UL }, + { 0x201000UL, 0x207000UL }, + { 0x301000UL, 0x302000UL }, + { 0x301000UL, 0x302000UL, 0x303000UL }, + { 0x301000UL, 0x304000UL }, }; - unsigned int nr_entries[] = { 2, 2, 1, 3, 2, 2 }; + unsigned int nr_entries[] = { 2, 2, 1, 3, 2, 2, 2, 2, 2, 3, 2 }; depot_stack_handle_t handles[ARRAY_SIZE(stacks)]; unsigned long fetched[ARRAY_SIZE(stacks[0])]; u32 pool_index_plus_1; @@ -286,9 +291,9 @@ static void stackdepot_frame_storage_roundtrip(struct kunit *test) }; #elif defined(CONFIG_X86_64) unsigned long entries[] = { - 0xffffffff81234567UL, + 0xffffffff10001000UL, 0xffff888000001000UL, - 0xffffffff89abcdefUL, + 0xffffffff20002000UL, }; #else unsigned long entries[] = { 0x301000UL, 0x302000UL, 0x303000UL }; From 6075957edf28954a5a43c0d1e223050e9f3b1ded Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 23 Jul 2026 11:27:29 +0100 Subject: [PATCH 30/38] KRN-1117: Simplify stackdepot trie state ownership Use the trie writer lock as the sole owner of side-table topology and leaf IDs, while retaining a separate lock only for the preallocation cache. This removes redundant nested locking without adding lockless state access. Restore the base one-time initialization guard and return child-slot match state directly instead of carrying a second output parameter. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 75 +++++++++++++++++------------------------------- 1 file changed, 27 insertions(+), 48 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 9467daa55a79f..6bb63890d0bdc 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -249,7 +249,7 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) /* * Trie handles encode a dense leaf ID. The side table maps that ID to a leaf * pointer for lockless fetch/print paths, which can run from diagnostic - * contexts where taking trie_side_table_lock would be unsafe. Initialization + * contexts where taking a lock would be unsafe. Initialization * installs the root; early initialization also installs the first directory and * chunk. Additional directories and chunks are preallocated and published * lazily as leaf IDs grow. RCU pointer publication makes fully initialized @@ -279,12 +279,12 @@ struct stack_depot_trie_side_prealloc { }; static struct stack_depot_trie_side_root *trie_side_table_root; -static DEFINE_RAW_SPINLOCK(trie_side_table_lock); -/* Zeroed unpublished pages; get/put transfer ownership under the lock. */ +static DEFINE_RAW_SPINLOCK(trie_side_table_cache_lock); +/* Zeroed unpublished pages; get/put transfer ownership under the cache lock. */ static struct stack_depot_trie_side_prealloc trie_side_table_cache; static u32 trie_side_table_last_leaf_id; -/* Lock order: writer_lock -> pool_lock -> trie_side_table_lock. */ +/* Lock order: writer_lock -> pool_lock. The cache lock is never nested. */ static inline size_t stack_depot_frame_run_entry_bytes(enum stack_depot_frame_mode mode) { @@ -354,7 +354,7 @@ static struct stack_depot_trie_side_dir *trie_side_table_load_dir(unsigned int r return NULL; /* Pairs with side-table directory rcu_assign_pointer(). */ return rcu_dereference_check(root_vec->dirs[root], - lockdep_is_held(&trie_side_table_lock) || + lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } @@ -364,7 +364,7 @@ trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, { /* Pairs with the chunk rcu_assign_pointer() in leaf ID preparation. */ return rcu_dereference_check(dir->chunks[idx], - lockdep_is_held(&trie_side_table_lock) || + lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } @@ -374,22 +374,22 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo const struct stack_depot_trie_node __rcu **chunk; struct stack_depot_trie_side_dir *dir; struct stack_depot_trie_side_root *root_vec; - unsigned long flags; unsigned int root; unsigned int idx; u32 id; - raw_spin_lock_irqsave(&trie_side_table_lock, flags); + lockdep_assert_held(&stack_depot_trie_writer_lock); + id = trie_side_table_last_leaf_id + 1; if (id > __stack_depot_trie_max_leaf_id()) - goto out_fail; + return 0; root_vec = trie_side_table_root; root = trie_side_table_root_index(id); dir = trie_side_table_load_dir(root); if (!dir) { if (WARN_ON_ONCE(!prealloc->dir)) - goto out_fail; + return 0; dir = prealloc->dir; prealloc->dir = NULL; /* Publish the zeroed directory before readers can load it locklessly. */ @@ -400,17 +400,12 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo chunk = trie_side_table_dir_load_chunk(dir, idx); if (!chunk) { if (WARN_ON_ONCE(!prealloc->chunk)) - goto out_fail; + return 0; chunk = prealloc->chunk; prealloc->chunk = NULL; rcu_assign_pointer(dir->chunks[idx], chunk); } - goto out; -out_fail: - id = 0; -out: - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); return id; } @@ -518,9 +513,6 @@ static int stack_depot_trie_init(gfp_t gfp_flags) struct stack_depot_trie_insert_alloc *alloc; int ret; - if (static_branch_unlikely(&stack_depot_trie_enabled)) - return 0; - alloc = kvzalloc(sizeof(*stack_depot_trie_alloc), gfp_flags); if (!alloc) return -ENOMEM; @@ -542,12 +534,12 @@ static int trie_side_table_get_prealloc(gfp_t gfp_flags, unsigned long flags; gfp_flags = gfp_nested_mask(gfp_flags); - raw_spin_lock_irqsave(&trie_side_table_lock, flags); + raw_spin_lock_irqsave(&trie_side_table_cache_lock, flags); prealloc->dir = trie_side_table_cache.dir; prealloc->chunk = trie_side_table_cache.chunk; trie_side_table_cache.dir = NULL; trie_side_table_cache.chunk = NULL; - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + raw_spin_unlock_irqrestore(&trie_side_table_cache_lock, flags); if (!prealloc->dir) { prealloc->dir = (void *)get_zeroed_page(gfp_flags); @@ -567,7 +559,7 @@ static void trie_side_table_put_prealloc(struct stack_depot_trie_side_prealloc * { unsigned long flags; - raw_spin_lock_irqsave(&trie_side_table_lock, flags); + raw_spin_lock_irqsave(&trie_side_table_cache_lock, flags); if (!trie_side_table_cache.dir) { trie_side_table_cache.dir = prealloc->dir; prealloc->dir = NULL; @@ -576,7 +568,7 @@ static void trie_side_table_put_prealloc(struct stack_depot_trie_side_prealloc * trie_side_table_cache.chunk = prealloc->chunk; prealloc->chunk = NULL; } - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + raw_spin_unlock_irqrestore(&trie_side_table_cache_lock, flags); if (prealloc->dir) free_page((unsigned long)prealloc->dir); @@ -613,7 +605,7 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( /* Pairs with side-table leaf rcu_assign_pointer(). */ return rcu_dereference_check(chunk[trie_side_table_slot_index(id)], - lockdep_is_held(&trie_side_table_lock) || + lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } @@ -912,13 +904,12 @@ static void trie_side_table_publish_new_leaf(u32 leaf_id, const struct stack_depot_trie_node *leaf) { const struct stack_depot_trie_node __rcu **slot; - unsigned long flags; - raw_spin_lock_irqsave(&trie_side_table_lock, flags); + lockdep_assert_held(&stack_depot_trie_writer_lock); + slot = trie_side_table_leaf_slot(leaf_id); /* Pairs with __stack_depot_trie_side_table_lookup(). */ rcu_assign_pointer(*slot, leaf); - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); } static void trie_side_table_publish_split_leaves(u32 old_leaf_id, @@ -928,9 +919,9 @@ static void trie_side_table_publish_split_leaves(u32 old_leaf_id, { const struct stack_depot_trie_node __rcu **new_slot; const struct stack_depot_trie_node __rcu **old_slot = NULL; - unsigned long flags; - raw_spin_lock_irqsave(&trie_side_table_lock, flags); + lockdep_assert_held(&stack_depot_trie_writer_lock); + if (old_leaf_id) old_slot = trie_side_table_leaf_slot(old_leaf_id); @@ -942,7 +933,6 @@ static void trie_side_table_publish_split_leaves(u32 old_leaf_id, /* Pairs with __stack_depot_trie_side_table_lookup(). */ rcu_assign_pointer(*new_slot, new_leaf); - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); } static int __init disable_stack_depot(char *str) @@ -1087,10 +1077,8 @@ int stack_depot_init(void) mutex_lock(&stack_depot_init_mutex); - if (stack_depot_disabled) + if (stack_depot_disabled || stack_table) goto out_unlock; - if (stack_table) - goto init_trie; /* * Similarly to stack_depot_early_init, use stack_bucket_number_order @@ -1136,7 +1124,6 @@ int stack_depot_init(void) ret = -ENOMEM; goto out_unlock; } -init_trie: if (stack_depot_trie_requested) { ret = stack_depot_trie_init(GFP_KERNEL); if (ret) { @@ -1844,15 +1831,13 @@ trie_child_array_load_child(const struct stack_depot_trie_child_array *array, rcu_read_lock_sched_held()); } -static void +static bool trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, - unsigned long frame, unsigned int *pos, bool *found) + unsigned long frame, unsigned int *pos) { unsigned int left = 0; unsigned int right; - *found = false; - right = READ_ONCE(array->nr_children); while (left < right) { unsigned int mid = left + (right - left) / 2; @@ -1872,12 +1857,12 @@ trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, right = mid; } else { *pos = mid; - *found = true; - return; + return true; } } *pos = left; + return false; } static void @@ -2015,12 +2000,10 @@ stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) unsigned int remaining = nr_entries - pos; unsigned int matched; unsigned int slot; - bool found; if (!children) return NULL; - trie_child_array_find_slot(children, entries[pos], &slot, &found); - if (!found) + if (!trie_child_array_find_slot(children, entries[pos], &slot)) return NULL; node = trie_child_array_load_child(children, slot); @@ -2140,7 +2123,6 @@ stack_depot_trie_insert_locked(const unsigned long *entries, unsigned long flags; u32 new_leaf_id; size_t slot_array_size; - bool found; for (;;) { children = trie_load_children_slot(slot); @@ -2168,8 +2150,7 @@ stack_depot_trie_insert_locked(const unsigned long *entries, rcu_assign_pointer(*slot, slot_array); goto out_success; } - trie_child_array_find_slot(children, entries[0], &pos, &found); - if (!found) { + if (!trie_child_array_find_slot(children, entries[0], &pos)) { struct stack_depot_trie_child_array *tail_array; unsigned int capacity; unsigned int nr_nodes; @@ -2283,9 +2264,7 @@ stack_depot_trie_insert_locked(const unsigned long *entries, } out_success: - raw_spin_lock_irqsave(&trie_side_table_lock, flags); trie_side_table_last_leaf_id = new_leaf_id; - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); return new_leaf_id; } From d356157fecf962e20f5c701d8f2b7b0da4f572cd Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 23 Jul 2026 11:28:18 +0100 Subject: [PATCH 31/38] KRN-1117: Verify constrained stackdepot trie misses Extend public save-flags coverage to verify that an allocation-constrained trie save returns zero for a missing stack without inserting it. Keep hash mode unchanged and use the existing backend parameter to scope the assertion. Signed-off-by: Caleb Kan --- lib/tests/stackdepot_kunit.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 221b7b8db3313..1623a644f6e62 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -93,6 +93,7 @@ static void stackdepot_save_flags_public(struct kunit *test) { unsigned long entries[] = { 0x501000UL, 0x502000UL, 0x503000UL }; unsigned long get_entries[] = { 0x601000UL, 0x602000UL }; + unsigned long missing_entries[] = { 0x701000UL, 0x702000UL }; unsigned long fetched[ARRAY_SIZE(entries)] = {}; depot_stack_handle_t noalloc_handle; depot_stack_handle_t overlong_handle; @@ -130,6 +131,13 @@ static void stackdepot_save_flags_public(struct kunit *test) noalloc_handle = stack_depot_save_flags(entries, ARRAY_SIZE(entries), no_spin, 0); KUNIT_EXPECT_EQ(test, noalloc_handle, plain_handle); + if (expected_trie_pool_limit >= 0) { + noalloc_handle = + stack_depot_save_flags(missing_entries, + ARRAY_SIZE(missing_entries), + no_spin, 0); + KUNIT_EXPECT_EQ(test, noalloc_handle, (depot_stack_handle_t)0); + } get_handle = stack_depot_save_flags(get_entries, ARRAY_SIZE(get_entries), GFP_KERNEL, From d41c11d60a9009b32f4e72d2ea4a1149a9092895 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 29 Jul 2026 13:07:10 +0100 Subject: [PATCH 32/38] KRN-1117: Simplify stackdepot trie insertion Split trie insertion into path, split, and promotion helpers so the writer loop only selects an operation or descends through a matching node. Derive topology under the writer lock, consolidate leaf publication, and remove one-use split and append output. Inline preallocation into the save path and make its bounded two-attempt policy explicit. A failed insertion can register the first of the two pools needed by the largest supported stack, so the second attempt is required. Use the existing lockless pool hint while preserving cleanup for partial preallocation and failed reservations. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 554 ++++++++++++++++++++++------------------------- 1 file changed, 255 insertions(+), 299 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 6bb63890d0bdc..4e1e52f3d7e8b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -739,7 +739,7 @@ static void trie_drain_pending_arrays(void) } } -static void trie_retire_child_array_locked(const void *ptr) +static void trie_retire_child_array(const void *ptr) { struct stack_depot_trie_retired_array *retired; @@ -759,50 +759,12 @@ trie_retire_child_array_with_node(const void *ptr, unsigned long flags; raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_child_array_locked(ptr); + trie_retire_child_array(ptr); retired = trie_retired_array(ptr); retired->pending_node = (struct stack_depot_trie_node *)node; raw_spin_unlock_irqrestore(&pool_lock, flags); } -/* - * Preallocate resources that cannot be allocated while trie writers hold raw - * spinlocks. Side-table growth is mandatory before a new leaf ID can be - * used, so side-table preallocation failure disables insertion for this - * save. Pool preallocation is opportunistic: reusable trie slots may still - * satisfy the insertion, and trie_pool_alloc_insert() reports -ENOSPC if they - * do not. - */ -static int -__stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) -{ - unsigned long flags; - bool need_pool; - int ret; - - raw_spin_lock_irqsave(&pool_lock, flags); - need_pool = !new_pool; - raw_spin_unlock_irqrestore(&pool_lock, flags); - if (need_pool) { - struct page *page; - - page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); - if (page) - *pool_prealloc = page_address(page); - } - ret = trie_side_table_get_prealloc(alloc_flags, side_prealloc); - - if (ret) { - if (*pool_prealloc) { - free_pages((unsigned long)*pool_prealloc, DEPOT_POOL_ORDER); - *pool_prealloc = NULL; - } - return -ENOSPC; - } - return 0; -} - /* * Reserve fixed-size pool slots for one trie insertion. If any reservation * fails, all slots reserved by this attempt are released locally. @@ -900,8 +862,8 @@ trie_find_handle(const unsigned long *entries, unsigned int nr_entries) return handle; } -static void trie_side_table_publish_new_leaf(u32 leaf_id, - const struct stack_depot_trie_node *leaf) +static void trie_side_table_publish_leaf(u32 leaf_id, + const struct stack_depot_trie_node *leaf) { const struct stack_depot_trie_node __rcu **slot; @@ -912,29 +874,6 @@ static void trie_side_table_publish_new_leaf(u32 leaf_id, rcu_assign_pointer(*slot, leaf); } -static void trie_side_table_publish_split_leaves(u32 old_leaf_id, - const struct stack_depot_trie_node *old_leaf, - u32 new_leaf_id, - const struct stack_depot_trie_node *new_leaf) -{ - const struct stack_depot_trie_node __rcu **new_slot; - const struct stack_depot_trie_node __rcu **old_slot = NULL; - - lockdep_assert_held(&stack_depot_trie_writer_lock); - - if (old_leaf_id) - old_slot = trie_side_table_leaf_slot(old_leaf_id); - - new_slot = trie_side_table_leaf_slot(new_leaf_id); - if (old_slot) { - /* Pairs with __stack_depot_trie_side_table_lookup(). */ - rcu_assign_pointer(*old_slot, old_leaf); - } - - /* Pairs with __stack_depot_trie_side_table_lookup(). */ - rcu_assign_pointer(*new_slot, new_leaf); -} - static int __init disable_stack_depot(char *str) { return kstrtobool(str, &stack_depot_disabled); @@ -1488,64 +1427,62 @@ static inline struct stack_record *find_stack(struct list_head *bucket, } static u32 -stack_depot_trie_insert_locked(const unsigned long *entries, - unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc); +stack_depot_trie_insert(const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc); static depot_stack_handle_t stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags) { - struct stack_depot_trie_side_prealloc side_prealloc = {}; - void *pool_prealloc = NULL; - depot_stack_handle_t handle; - unsigned long flags; - bool retried = false; - u32 leaf_id; - int ret; + unsigned int attempt; -retry: - handle = trie_find_handle(entries, nr_entries); - if (handle) - return handle; - - ret = __stack_depot_trie_alloc_prealloc(alloc_flags, &pool_prealloc, - &side_prealloc); - if (ret) - goto out_free; + /* The largest insertion fits in two pools; retry can use the first one. */ + for (attempt = 0; attempt < 2; attempt++) { + struct stack_depot_trie_side_prealloc side_prealloc = {}; + void *pool_prealloc = NULL; + depot_stack_handle_t handle; + unsigned long flags; + struct page *page; + u32 leaf_id = 0; + int ret; + + handle = trie_find_handle(entries, nr_entries); + if (handle) + return handle; + + /* The hint may race; a missing page is recovered by the retry. */ + if (!READ_ONCE(new_pool)) { + page = alloc_pages(gfp_nested_mask(alloc_flags), + DEPOT_POOL_ORDER); + if (page) + pool_prealloc = page_address(page); + } + ret = trie_side_table_get_prealloc(alloc_flags, &side_prealloc); + if (!ret) { + raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); + leaf_id = stack_depot_trie_insert(entries, nr_entries, + &pool_prealloc, + &side_prealloc); + raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, + flags); + } - raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); - leaf_id = stack_depot_trie_insert_locked(entries, nr_entries, - &pool_prealloc, &side_prealloc); - if (leaf_id) - handle = __stack_depot_trie_handle(leaf_id); - raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); - if (!handle && !retried) { - retried = true; - if (pool_prealloc) { + if (pool_prealloc && !ret) { raw_spin_lock_irqsave(&pool_lock, flags); depot_keep_new_pool(&pool_prealloc); raw_spin_unlock_irqrestore(&pool_lock, flags); } - if (pool_prealloc) { + if (pool_prealloc) free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); - pool_prealloc = NULL; - } trie_side_table_put_prealloc(&side_prealloc); - goto retry; + if (ret) + return 0; + if (leaf_id) + return __stack_depot_trie_handle(leaf_id); } -out_free: - if (pool_prealloc) { - raw_spin_lock_irqsave(&pool_lock, flags); - depot_keep_new_pool(&pool_prealloc); - raw_spin_unlock_irqrestore(&pool_lock, flags); - } - if (pool_prealloc) - free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); - trie_side_table_put_prealloc(&side_prealloc); - return handle; + return 0; } depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, @@ -1934,9 +1871,7 @@ static void trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_node * const *nodes, - struct stack_depot_trie_child_array * const *chain_arrays, - const struct stack_depot_trie_node **head, - const struct stack_depot_trie_node **tail) + struct stack_depot_trie_child_array * const *chain_arrays) { const struct stack_depot_trie_node *prev = parent; unsigned int pos = 0; @@ -1967,9 +1902,6 @@ trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, trie_child_array_init(array, 1, &next, 1); RCU_INIT_POINTER(node->children, array); } - - *head = nodes[0]; - *tail = nodes[used - 1]; } static void trie_publish_tail_append(struct stack_depot_trie_child_array *array, @@ -2020,52 +1952,158 @@ stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) return NULL; } -static void trie_build_split(const struct stack_depot_trie_node *child, - unsigned int matched, u32 leaf_id, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_node *prefix, - struct stack_depot_trie_node *old_tail, - struct stack_depot_trie_node * const *new_nodes, - struct stack_depot_trie_child_array * const *chain_arrays, - struct stack_depot_trie_child_array *prefix_children, - const struct stack_depot_trie_node **new_leaf) +static unsigned int trie_size_append_chain(const unsigned long *entries, + unsigned int nr_entries, + size_t *node_sizes) +{ + unsigned int pos = 0; + unsigned int nr_nodes = 0; + + while (pos < nr_entries) { + struct stack_depot_frame_run run; + + frame_run_init(&entries[pos], nr_entries - pos, &run); + node_sizes[nr_nodes] = trie_node_bytes(&run); + pos += run.nr_entries; + nr_nodes++; + } + + return nr_nodes; +} + +static u32 +trie_insert_path(const struct stack_depot_trie_child_array __rcu **slot, + struct stack_depot_trie_node *parent, + unsigned int pos, const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { - const struct stack_depot_trie_child_array *child_children; - const struct stack_depot_trie_node *child_parent; - const unsigned long *tail_entries; + struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; + struct stack_depot_trie_child_array *slot_array; + const struct stack_depot_trie_child_array *children; + unsigned int capacity = 1; + unsigned int nr_nodes; + unsigned long flags; + u32 new_leaf_id; + size_t slot_array_size; + bool tail_append = false; + + lockdep_assert_held(&stack_depot_trie_writer_lock); + + children = trie_load_children_slot(slot); + if (children) { + capacity = roundup_pow_of_two(children->nr_children + 1); + tail_append = pos == children->nr_children && + children->nr_children < children->capacity; + } + slot_array_size = tail_append ? 0 : trie_child_array_bytes(capacity); + if (slot_array_size > DEPOT_POOL_SIZE) + return 0; + + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); + if (!new_leaf_id) + return 0; + nr_nodes = trie_size_append_chain(entries, nr_entries, alloc->node_sizes); + if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, + nr_nodes - 1, 0, slot_array_size)) + return 0; + trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, + alloc->nodes, alloc->chain_arrays); + trie_side_table_publish_leaf(new_leaf_id, alloc->nodes[nr_nodes - 1]); + + if (!children) { + slot_array = alloc->slot_array; + slot_array->nr_children = 1; + slot_array->capacity = 1; + RCU_INIT_POINTER(slot_array->children[0], alloc->nodes[0]); + rcu_assign_pointer(*slot, slot_array); + return new_leaf_id; + } + + if (tail_append) { + trie_publish_tail_append((struct stack_depot_trie_child_array *)children, + pos, alloc->nodes[0]); + } else { + slot_array = alloc->slot_array; + trie_child_array_insert_at(children, pos, alloc->nodes[0], + slot_array, capacity); + rcu_assign_pointer(*slot, slot_array); + raw_spin_lock_irqsave(&pool_lock, flags); + trie_retire_child_array(children); + raw_spin_unlock_irqrestore(&pool_lock, flags); + } + + return new_leaf_id; +} + +static u32 +trie_split_child(const struct stack_depot_trie_child_array __rcu **slot, + unsigned int pos, unsigned int matched, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) +{ + struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; + struct stack_depot_trie_child_array *prefix_children; + struct stack_depot_trie_child_array *slot_array; + const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_node *child; const struct stack_depot_trie_node *split_children[2]; - const struct stack_depot_trie_node *new_head = NULL; - const struct stack_depot_trie_node *new_tail = NULL; + struct stack_depot_frame_run run; + struct stack_depot_trie_node *split_prefix; + struct stack_depot_trie_node *old_tail; unsigned long new_frame; unsigned long old_frame; - u32 prefix_leaf_id; + unsigned int new_nodes = 0; unsigned int tail_len; + u32 new_leaf_id; bool has_new_tail; - child_children = trie_load_children_slot(&child->children); - child_parent = trie_load_parent(child); + lockdep_assert_held(&stack_depot_trie_writer_lock); + + children = trie_load_children_slot(slot); + child = trie_child_array_load_child(children, pos); + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); + if (!new_leaf_id) + return 0; + run = child->run; + run.nr_entries = matched; + alloc->node_sizes[0] = trie_node_bytes(&run); + run.nr_entries = child->run.nr_entries - matched; + alloc->node_sizes[1] = trie_node_bytes(&run); has_new_tail = matched < nr_entries; - prefix_leaf_id = has_new_tail ? 0 : leaf_id; - trie_node_init_slice(prefix, child_parent, prefix_leaf_id, child, 0, - matched); + if (has_new_tail) + new_nodes = trie_size_append_chain(&entries[matched], + nr_entries - matched, + &alloc->node_sizes[2]); + if (trie_pool_alloc_insert(alloc, pool_prealloc, 2 + new_nodes, + new_nodes ? new_nodes - 1 : 0, + trie_child_array_bytes(has_new_tail ? 2 : 1), + trie_child_array_bytes(children->capacity))) + return 0; + + slot_array = alloc->slot_array; + prefix_children = alloc->prefix_children; + split_prefix = alloc->nodes[0]; + old_tail = alloc->nodes[1]; + trie_node_init_slice(split_prefix, trie_load_parent(child), + has_new_tail ? 0 : new_leaf_id, child, 0, matched); tail_len = child->run.nr_entries - matched; - trie_node_init_slice(old_tail, prefix, child->leaf_id, child, matched, - tail_len); + trie_node_init_slice(old_tail, split_prefix, child->leaf_id, child, + matched, tail_len); if (has_new_tail) { - tail_entries = &entries[matched]; - tail_len = nr_entries - matched; - trie_build_append_chain(prefix, leaf_id, tail_entries, tail_len, - new_nodes, chain_arrays, - &new_head, &new_tail); + trie_build_append_chain(split_prefix, new_leaf_id, &entries[matched], + nr_entries - matched, &alloc->nodes[2], + alloc->chain_arrays); stack_depot_trie_node_frame(old_tail, 0, &old_frame); - stack_depot_trie_node_frame(new_head, 0, &new_frame); + stack_depot_trie_node_frame(alloc->nodes[2], 0, &new_frame); if (old_frame < new_frame) { split_children[0] = old_tail; - split_children[1] = new_head; + split_children[1] = alloc->nodes[2]; } else { - split_children[0] = new_head; + split_children[0] = alloc->nodes[2]; split_children[1] = old_tail; } trie_child_array_init(prefix_children, ARRAY_SIZE(split_children), @@ -2074,187 +2112,105 @@ static void trie_build_split(const struct stack_depot_trie_node *child, split_children[0] = old_tail; trie_child_array_init(prefix_children, 1, split_children, 1); } - RCU_INIT_POINTER(old_tail->children, child_children); - RCU_INIT_POINTER(prefix->children, prefix_children); - *new_leaf = has_new_tail ? new_tail : prefix; + RCU_INIT_POINTER(old_tail->children, + trie_load_children_slot(&child->children)); + RCU_INIT_POINTER(split_prefix->children, prefix_children); + + if (child->leaf_id) + trie_side_table_publish_leaf(child->leaf_id, old_tail); + trie_side_table_publish_leaf(new_leaf_id, + has_new_tail ? alloc->nodes[new_nodes + 1] : + split_prefix); + trie_child_array_replace_at(children, split_prefix, slot_array, pos); + trie_reparent_children(old_tail); + rcu_assign_pointer(*slot, slot_array); + trie_retire_child_array_with_node(children, child); + + return new_leaf_id; } -static unsigned int trie_size_append_chain(const unsigned long *entries, - unsigned int nr_entries, - size_t *node_sizes) +static u32 +trie_promote_child(const struct stack_depot_trie_child_array __rcu **slot, + unsigned int pos, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { - unsigned int pos = 0; - unsigned int nr_nodes = 0; + struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; + struct stack_depot_trie_child_array *slot_array; + const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_node *child; + struct stack_depot_trie_node *promoted_node; + u32 new_leaf_id; + size_t slot_array_size; - while (pos < nr_entries) { - struct stack_depot_frame_run run; + lockdep_assert_held(&stack_depot_trie_writer_lock); - frame_run_init(&entries[pos], nr_entries - pos, &run); - node_sizes[nr_nodes] = trie_node_bytes(&run); - pos += run.nr_entries; - nr_nodes++; - } + children = trie_load_children_slot(slot); + child = trie_child_array_load_child(children, pos); + new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); + if (!new_leaf_id) + return 0; + alloc->node_sizes[0] = trie_node_bytes(&child->run); + slot_array_size = trie_child_array_bytes(children->capacity); + if (trie_pool_alloc_insert(alloc, pool_prealloc, 1, 0, 0, + slot_array_size)) + return 0; - return nr_nodes; + promoted_node = alloc->nodes[0]; + slot_array = alloc->slot_array; + memcpy(promoted_node, child, alloc->node_sizes[0]); + promoted_node->leaf_id = new_leaf_id; + trie_side_table_publish_leaf(new_leaf_id, promoted_node); + trie_child_array_replace_at(children, promoted_node, slot_array, pos); + trie_reparent_children(promoted_node); + rcu_assign_pointer(*slot, slot_array); + trie_retire_child_array_with_node(children, child); + + return new_leaf_id; } static u32 -stack_depot_trie_insert_locked(const unsigned long *entries, - unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) +stack_depot_trie_insert(const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { - struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_child_array *slot_array; - struct stack_depot_trie_child_array *prefix_children; const struct stack_depot_trie_child_array *children; const struct stack_depot_trie_child_array __rcu **slot = &stack_depot_trie_root; const struct stack_depot_trie_node *child; - const struct stack_depot_trie_node *chain_head; - const struct stack_depot_trie_node *chain_leaf; - const struct stack_depot_trie_node *split_leaf; - struct stack_depot_trie_node *split_prefix; - struct stack_depot_trie_node *old_tail; - struct stack_depot_trie_node *promoted_node; struct stack_depot_trie_node *parent = NULL; unsigned int matched; unsigned int pos; - unsigned long flags; - u32 new_leaf_id; - size_t slot_array_size; + u32 leaf_id; + + lockdep_assert_held(&stack_depot_trie_writer_lock); for (;;) { + pos = 0; children = trie_load_children_slot(slot); - if (!children) { - unsigned int nr_nodes; - - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - slot_array_size = trie_child_array_bytes(1); - nr_nodes = trie_size_append_chain(entries, nr_entries, - alloc->node_sizes); - if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, - nr_nodes - 1, 0, - slot_array_size)) - return 0; - slot_array = alloc->slot_array; - trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - alloc->nodes, alloc->chain_arrays, - &chain_head, &chain_leaf); - trie_side_table_publish_new_leaf(new_leaf_id, chain_leaf); - slot_array->nr_children = 1; - slot_array->capacity = 1; - RCU_INIT_POINTER(slot_array->children[0], chain_head); - rcu_assign_pointer(*slot, slot_array); - goto out_success; - } - if (!trie_child_array_find_slot(children, entries[0], &pos)) { - struct stack_depot_trie_child_array *tail_array; - unsigned int capacity; - unsigned int nr_nodes; - bool tail_append; - - capacity = roundup_pow_of_two(children->nr_children + 1); - tail_append = pos == children->nr_children && - children->nr_children < children->capacity; - slot_array_size = tail_append ? 0 : - trie_child_array_bytes(capacity); - if (slot_array_size > DEPOT_POOL_SIZE) - return 0; - - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - nr_nodes = trie_size_append_chain(entries, nr_entries, - alloc->node_sizes); - if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, - nr_nodes - 1, 0, - slot_array_size)) - return 0; - trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - alloc->nodes, alloc->chain_arrays, - &chain_head, &chain_leaf); - trie_side_table_publish_new_leaf(new_leaf_id, chain_leaf); - if (tail_append) { - tail_array = (struct stack_depot_trie_child_array *)children; - trie_publish_tail_append(tail_array, pos, chain_head); - } else { - slot_array = alloc->slot_array; - trie_child_array_insert_at(children, pos, chain_head, - slot_array, capacity); - rcu_assign_pointer(*slot, slot_array); - raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_child_array_locked(children); - raw_spin_unlock_irqrestore(&pool_lock, flags); - } - goto out_success; + if (!children || + !trie_child_array_find_slot(children, entries[0], &pos)) { + leaf_id = trie_insert_path(slot, parent, pos, entries, + nr_entries, pool_prealloc, + side_prealloc); + break; } child = trie_child_array_load_child(children, pos); matched = __stack_depot_trie_node_match(child, entries, nr_entries); if (matched < child->run.nr_entries) { - struct stack_depot_frame_run run; - unsigned int new_nodes = 0; - bool has_new_tail; - - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - - run = child->run; - run.nr_entries = matched; - alloc->node_sizes[0] = trie_node_bytes(&run); - run.nr_entries = child->run.nr_entries - matched; - alloc->node_sizes[1] = trie_node_bytes(&run); - has_new_tail = matched < nr_entries; - if (has_new_tail) - new_nodes = trie_size_append_chain(&entries[matched], - nr_entries - matched, - &alloc->node_sizes[2]); - if (trie_pool_alloc_insert(alloc, pool_prealloc, - 2 + new_nodes, - new_nodes ? new_nodes - 1 : 0, - trie_child_array_bytes(has_new_tail ? 2 : 1), - trie_child_array_bytes(children->capacity))) - return 0; - slot_array = alloc->slot_array; - prefix_children = alloc->prefix_children; - split_prefix = alloc->nodes[0]; - old_tail = alloc->nodes[1]; - trie_build_split(child, matched, new_leaf_id, entries, nr_entries, - split_prefix, old_tail, &alloc->nodes[2], - alloc->chain_arrays, prefix_children, &split_leaf); - trie_side_table_publish_split_leaves(child->leaf_id, old_tail, - new_leaf_id, split_leaf); - trie_child_array_replace_at(children, split_prefix, slot_array, pos); - trie_reparent_children(old_tail); - rcu_assign_pointer(*slot, slot_array); - trie_retire_child_array_with_node(children, child); - goto out_success; + leaf_id = trie_split_child(slot, pos, matched, entries, + nr_entries, + pool_prealloc, + side_prealloc); + break; } + if (matched == nr_entries) { if (child->leaf_id) return child->leaf_id; - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) - return 0; - alloc->node_sizes[0] = trie_node_bytes(&child->run); - slot_array_size = trie_child_array_bytes(children->capacity); - if (trie_pool_alloc_insert(alloc, pool_prealloc, 1, 0, 0, - slot_array_size)) - return 0; - promoted_node = alloc->nodes[0]; - slot_array = alloc->slot_array; - memcpy(promoted_node, child, alloc->node_sizes[0]); - promoted_node->leaf_id = new_leaf_id; - trie_side_table_publish_new_leaf(new_leaf_id, promoted_node); - trie_child_array_replace_at(children, promoted_node, slot_array, pos); - trie_reparent_children(promoted_node); - rcu_assign_pointer(*slot, slot_array); - trie_retire_child_array_with_node(children, child); - goto out_success; + leaf_id = trie_promote_child(slot, pos, pool_prealloc, + side_prealloc); + break; } parent = (struct stack_depot_trie_node *)child; @@ -2263,9 +2219,9 @@ stack_depot_trie_insert_locked(const unsigned long *entries, nr_entries -= matched; } -out_success: - trie_side_table_last_leaf_id = new_leaf_id; - return new_leaf_id; + if (leaf_id) + trie_side_table_last_leaf_id = leaf_id; + return leaf_id; } static unsigned int From 65e196c221116243b12f84396b80ebe24d9ec44a Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 29 Jul 2026 13:08:42 +0100 Subject: [PATCH 33/38] KRN-1117: Skip x86 stackdepot codec test on UML UML can define CONFIG_X86_64 while using the generic stackdepot frame codec. The generic compressor always returns false, so the x86-specific compression assertions do not describe the code under test. Exclude the x86 codec case under CONFIG_UML while retaining generic raw fallback coverage. Signed-off-by: Caleb Kan --- lib/tests/stackdepot_kunit.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 1623a644f6e62..4fba72e283312 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -336,7 +336,7 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) KUNIT_EXPECT_FALSE(test, compressed); } -#ifdef CONFIG_X86_64 +#if defined(CONFIG_X86_64) && !defined(CONFIG_UML) static void stackdepot_frame_x86_64(struct kunit *test) { unsigned long direct_map = 0xffff888000001000UL; @@ -354,7 +354,7 @@ static void stackdepot_frame_x86_64(struct kunit *test) compressed = arch_stack_depot_frame_try_compress(direct_map, &low); KUNIT_EXPECT_FALSE(test, compressed); } -#endif /* CONFIG_X86_64 */ +#endif /* CONFIG_X86_64 && !CONFIG_UML */ #ifdef CONFIG_ARM64 static void stackdepot_frame_arm64(struct kunit *test) @@ -398,7 +398,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_topology_roundtrip), KUNIT_CASE(stackdepot_frame_storage_roundtrip), KUNIT_CASE(stackdepot_frame_raw_fallback), -#ifdef CONFIG_X86_64 +#if defined(CONFIG_X86_64) && !defined(CONFIG_UML) KUNIT_CASE(stackdepot_frame_x86_64), #endif #ifdef CONFIG_ARM64 From b0139f52b9d61287630b676567498c6f38f25938 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 30 Jul 2026 11:59:15 +0100 Subject: [PATCH 34/38] KRN-1117: Simplify stackdepot trie insertion allocation Use children containers that own nodes[] and consistently distinguish entries, positions, and bitmap slots. Keep capacity immutable so release size remains a property of the allocation. Remove the generic insertion allocator. Let path insertion, splitting, and promotion reserve and unwind exactly the resources they own, without clearing or scanning the maximum workspace. Reject impossible fanout before reservation and keep split sizing and construction outside pool_lock. Inline the one-use tail publisher and keep three bounded save attempts. One attempt can lose the racy pool hint, while a maximum insertion can need two newly registered pools. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 642 +++++++++++++++++++++++------------------------ 1 file changed, 316 insertions(+), 326 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 4e1e52f3d7e8b..3875c79b5af53 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -6,7 +6,7 @@ * callers that request STACK_DEPOT_FLAG_COUNTABLE use the legacy hash table with * contiguous stack records in stack pools. Persistent non-refcounted entries * can use trie storage when enabled; trie nodes share common frame prefixes and - * are published through RCU/COW child arrays. + * are published through RCU/COW children containers. * * Author: Alexander Potapenko * Copyright (C) 2016 Google, Inc. @@ -124,42 +124,42 @@ struct stack_depot_frame_run { static_assert(CONFIG_STACKDEPOT_MAX_FRAMES <= U16_MAX); -struct stack_depot_trie_child_array; +struct stack_depot_trie_children; struct stack_depot_trie_node { /* Parent links let fetch rebuild a full stack from a leaf to the root. */ const struct stack_depot_trie_node __rcu *parent; - /* Child arrays are separate RCU/COW generations. */ - const struct stack_depot_trie_child_array __rcu *children; + /* Children are separate RCU/COW generations. */ + const struct stack_depot_trie_children __rcu *children; u32 leaf_id; struct stack_depot_frame_run run; unsigned char data[]; }; /* - * Children are sorted by first frame and searched by insertion slot. + * Child nodes are sorted by first frame and searched by insertion position. * Writers may append to spare capacity at the sorted tail, but never change - * existing child pointers. Other updates build and publish a replacement array. + * existing node pointers. Other updates publish a replacement container. */ -struct stack_depot_trie_child_array { +struct stack_depot_trie_children { unsigned int nr_children; unsigned int capacity; - const struct stack_depot_trie_node __rcu *children[]; + const struct stack_depot_trie_node __rcu *nodes[]; }; -/* A retired child array carries an optional node through its RCU grace period. */ -struct stack_depot_trie_retired_array { +/* Retired children carry an optional node through their RCU grace period. */ +struct stack_depot_trie_retired_children { struct list_head list; unsigned long rcu_state; struct stack_depot_trie_node *pending_node; unsigned char data[]; }; -static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_retired_array, data), +static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_retired_children, data), 1UL << DEPOT_STACK_ALIGN)); #define STACK_DEPOT_TRIE_MAX_NODES (CONFIG_STACKDEPOT_MAX_FRAMES + 1) -#define STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS CONFIG_STACKDEPOT_MAX_FRAMES +#define STACK_DEPOT_TRIE_MAX_PATH_CHILDREN CONFIG_STACKDEPOT_MAX_FRAMES #define STACK_DEPOT_TRIE_SLOT_SIZE BIT(DEPOT_STACK_ALIGN) #define STACK_DEPOT_TRIE_POOL_SLOTS \ (DEPOT_POOL_SIZE / STACK_DEPOT_TRIE_SLOT_SIZE) @@ -173,6 +173,9 @@ struct stack_depot_trie_pool { #define STACK_DEPOT_TRIE_POOL_FIRST_SLOT \ DIV_ROUND_UP(sizeof(struct stack_depot_trie_pool), \ STACK_DEPOT_TRIE_SLOT_SIZE) +#define STACK_DEPOT_TRIE_POOL_USABLE_SIZE \ + ((STACK_DEPOT_TRIE_POOL_SLOTS - STACK_DEPOT_TRIE_POOL_FIRST_SLOT) * \ + STACK_DEPOT_TRIE_SLOT_SIZE) static_assert(STACK_DEPOT_TRIE_POOL_FIRST_SLOT < STACK_DEPOT_TRIE_POOL_SLOTS); @@ -180,13 +183,12 @@ static_assert(STACK_DEPOT_TRIE_POOL_FIRST_SLOT < STACK_DEPOT_TRIE_POOL_SLOTS); struct stack_depot_trie_insert_alloc { struct stack_depot_trie_node *nodes[STACK_DEPOT_TRIE_MAX_NODES]; size_t node_sizes[STACK_DEPOT_TRIE_MAX_NODES]; - struct stack_depot_trie_child_array *chain_arrays[STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS]; - struct stack_depot_trie_child_array *prefix_children; - struct stack_depot_trie_child_array *slot_array; + struct stack_depot_trie_children + *path_children[STACK_DEPOT_TRIE_MAX_PATH_CHILDREN]; }; static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); -static const struct stack_depot_trie_child_array __rcu *stack_depot_trie_root; +static const struct stack_depot_trie_children __rcu *stack_depot_trie_root; static struct stack_depot_trie_insert_alloc *stack_depot_trie_alloc; static DEFINE_RAW_SPINLOCK(stack_depot_trie_writer_lock); static bool stack_depot_trie_requested; @@ -199,7 +201,7 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); /* Retired fixed-size slots remain reserved until their RCU grace period ends. */ static LIST_HEAD(stack_depot_trie_pools); -static LIST_HEAD(pending_trie_arrays); +static LIST_HEAD(pending_trie_children); /* * stack_max_pools is the split point between hash and trie handle encodings. @@ -304,28 +306,27 @@ static inline size_t trie_node_bytes(const struct stack_depot_frame_run *run) stack_depot_frame_run_bytes(run), sizeof(unsigned long)); } -static size_t trie_child_array_bytes(unsigned int capacity) +static size_t trie_children_alloc_size(unsigned int capacity) { size_t size; - size = struct_size_t(struct stack_depot_trie_child_array, children, + size = struct_size_t(struct stack_depot_trie_children, nodes, capacity); - return ALIGN(size, sizeof(unsigned long)); + return offsetof(struct stack_depot_trie_retired_children, data) + + ALIGN(size, sizeof(unsigned long)); } -static void trie_child_array_init(struct stack_depot_trie_child_array *array, - unsigned int capacity, - const struct stack_depot_trie_node * const *nodes, - unsigned int nr_children) +static void trie_children_init(struct stack_depot_trie_children *children, + const struct stack_depot_trie_node * const *nodes, + unsigned int nr_children) { unsigned int i; - array->nr_children = nr_children; - array->capacity = capacity; + children->nr_children = nr_children; for (i = 0; i < nr_children; i++) - RCU_INIT_POINTER(array->children[i], nodes[i]); - for (i = nr_children; i < capacity; i++) - RCU_INIT_POINTER(array->children[i], NULL); + RCU_INIT_POINTER(children->nodes[i], nodes[i]); + for (i = nr_children; i < children->capacity; i++) + RCU_INIT_POINTER(children->nodes[i], NULL); } static inline unsigned int trie_side_table_root_index(u32 id) @@ -609,15 +610,10 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( rcu_read_lock_sched_held()); } -static inline size_t trie_array_alloc_size(size_t size) +static inline struct stack_depot_trie_retired_children * +trie_retired_children(const void *ptr) { - return offsetof(struct stack_depot_trie_retired_array, data) + size; -} - -static inline struct stack_depot_trie_retired_array * -trie_retired_array(const void *ptr) -{ - return container_of(ptr, struct stack_depot_trie_retired_array, data); + return container_of(ptr, struct stack_depot_trie_retired_children, data); } static bool depot_init_pool(void **prealloc); @@ -662,6 +658,8 @@ static void *trie_pool_alloc(size_t size, void **prealloc) lockdep_assert_held(&pool_lock); + if (size > STACK_DEPOT_TRIE_POOL_USABLE_SIZE) + return NULL; nr_slots = DIV_ROUND_UP(size, STACK_DEPOT_TRIE_SLOT_SIZE); list_for_each_entry_reverse(pool, &stack_depot_trie_pools, list) { slot = trie_pool_reserve_slots(pool, nr_slots); @@ -703,147 +701,75 @@ static void trie_pool_release(const void *ptr, size_t size) pool->free_slots += nr_slots; } -static struct stack_depot_trie_child_array * -trie_pool_alloc_array(size_t size, void **prealloc) +static struct stack_depot_trie_children * +trie_pool_alloc_children(unsigned int capacity, void **prealloc) { - struct stack_depot_trie_retired_array *retired; + struct stack_depot_trie_retired_children *retired; + struct stack_depot_trie_children *children; - retired = trie_pool_alloc(trie_array_alloc_size(size), prealloc); - return retired ? (void *)retired->data : NULL; + retired = trie_pool_alloc(trie_children_alloc_size(capacity), prealloc); + if (!retired) + return NULL; + + children = (void *)retired->data; + children->capacity = capacity; + return children; } -static void trie_pool_release_array(const void *ptr, size_t size) +static void +trie_pool_release_children(const struct stack_depot_trie_children *children) { - trie_pool_release(trie_retired_array(ptr), trie_array_alloc_size(size)); + trie_pool_release(trie_retired_children(children), + trie_children_alloc_size(children->capacity)); } -static void trie_drain_pending_arrays(void) +static void trie_drain_pending_children(void) { - struct stack_depot_trie_retired_array *retired; - struct stack_depot_trie_retired_array *tmp; - struct stack_depot_trie_child_array *array; + struct stack_depot_trie_retired_children *retired; + struct stack_depot_trie_retired_children *tmp; + struct stack_depot_trie_children *children; lockdep_assert_held(&pool_lock); - list_for_each_entry_safe(retired, tmp, &pending_trie_arrays, list) { - /* Pending arrays are FIFO; later entries cannot be ready yet. */ + list_for_each_entry_safe(retired, tmp, &pending_trie_children, list) { + /* Pending children are FIFO; later entries cannot be ready yet. */ if (!poll_state_synchronize_rcu(retired->rcu_state)) break; - array = (void *)retired->data; + children = (void *)retired->data; list_del(&retired->list); if (retired->pending_node) trie_pool_release(retired->pending_node, trie_node_bytes(&retired->pending_node->run)); - trie_pool_release_array(array, - trie_child_array_bytes(array->capacity)); + trie_pool_release_children(children); } } -static void trie_retire_child_array(const void *ptr) +static void trie_retire_children(const void *ptr) { - struct stack_depot_trie_retired_array *retired; + struct stack_depot_trie_retired_children *retired; lockdep_assert_held(&pool_lock); - retired = trie_retired_array(ptr); + retired = trie_retired_children(ptr); retired->pending_node = NULL; retired->rcu_state = get_state_synchronize_rcu(); - list_add_tail(&retired->list, &pending_trie_arrays); + list_add_tail(&retired->list, &pending_trie_children); } static void -trie_retire_child_array_with_node(const void *ptr, - const struct stack_depot_trie_node *node) +trie_retire_children_with_node(const void *ptr, + const struct stack_depot_trie_node *node) { - struct stack_depot_trie_retired_array *retired; + struct stack_depot_trie_retired_children *retired; unsigned long flags; raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_child_array(ptr); - retired = trie_retired_array(ptr); + trie_retire_children(ptr); + retired = trie_retired_children(ptr); retired->pending_node = (struct stack_depot_trie_node *)node; raw_spin_unlock_irqrestore(&pool_lock, flags); } -/* - * Reserve fixed-size pool slots for one trie insertion. If any reservation - * fails, all slots reserved by this attempt are released locally. - * The caller must not publish any returned storage before side-table and trie - * publication succeeds. - */ -static int trie_pool_alloc_insert(struct stack_depot_trie_insert_alloc *alloc, - void **pool_prealloc, - unsigned int nr_nodes, - unsigned int nr_chain_arrays, - size_t prefix_children_size, - size_t slot_array_size) -{ - unsigned long flags; - size_t one_child_size; - unsigned int i; - int ret = -ENOSPC; - - memset(alloc->nodes, 0, nr_nodes * sizeof(*alloc->nodes)); - memset(alloc->chain_arrays, 0, - nr_chain_arrays * sizeof(*alloc->chain_arrays)); - alloc->prefix_children = NULL; - alloc->slot_array = NULL; - - raw_spin_lock_irqsave(&pool_lock, flags); - printk_deferred_enter(); - trie_drain_pending_arrays(); - one_child_size = trie_child_array_bytes(1); - for (i = 0; i < nr_nodes; i++) { - alloc->nodes[i] = trie_pool_alloc(alloc->node_sizes[i], - pool_prealloc); - if (!alloc->nodes[i]) - goto out_discard; - } - for (i = 0; i < nr_chain_arrays; i++) { - alloc->chain_arrays[i] = - trie_pool_alloc_array(one_child_size, pool_prealloc); - if (!alloc->chain_arrays[i]) - goto out_discard; - } - if (prefix_children_size) { - alloc->prefix_children = - trie_pool_alloc_array(prefix_children_size, pool_prealloc); - if (!alloc->prefix_children) - goto out_discard; - } - if (slot_array_size) { - alloc->slot_array = - trie_pool_alloc_array(slot_array_size, pool_prealloc); - if (!alloc->slot_array) - goto out_discard; - } - ret = 0; - goto out; -out_discard: - for (i = 0; i < nr_nodes; i++) { - struct stack_depot_trie_node *node = alloc->nodes[i]; - - if (node) - trie_pool_release(node, alloc->node_sizes[i]); - } - for (i = 0; i < nr_chain_arrays; i++) { - struct stack_depot_trie_child_array *array = alloc->chain_arrays[i]; - - if (!array) - continue; - trie_pool_release_array(array, one_child_size); - } - if (alloc->prefix_children) - trie_pool_release_array(alloc->prefix_children, - prefix_children_size); - if (alloc->slot_array) - trie_pool_release_array(alloc->slot_array, slot_array_size); -out: - printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); - return ret; -} - static const struct stack_depot_trie_node * stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries); @@ -1437,8 +1363,8 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, { unsigned int attempt; - /* The largest insertion fits in two pools; retry can use the first one. */ - for (attempt = 0; attempt < 2; attempt++) { + /* Allow one stale pool hint before the two pools a largest insert needs. */ + for (attempt = 0; attempt < 3; attempt++) { struct stack_depot_trie_side_prealloc side_prealloc = {}; void *pool_prealloc = NULL; depot_stack_handle_t handle; @@ -1751,8 +1677,8 @@ trie_load_parent(const struct stack_depot_trie_node *node) rcu_read_lock_sched_held()); } -static inline const struct stack_depot_trie_child_array * -trie_load_children_slot(const struct stack_depot_trie_child_array __rcu * const *slot) +static inline const struct stack_depot_trie_children * +trie_load_children(const struct stack_depot_trie_children __rcu * const *slot) { return rcu_dereference_check(*slot, lockdep_is_held(&stack_depot_trie_writer_lock) || @@ -1760,28 +1686,28 @@ trie_load_children_slot(const struct stack_depot_trie_child_array __rcu * const } static inline const struct stack_depot_trie_node * -trie_child_array_load_child(const struct stack_depot_trie_child_array *array, - unsigned int pos) +trie_children_load_node(const struct stack_depot_trie_children *children, + unsigned int pos) { - return rcu_dereference_check(array->children[pos], + return rcu_dereference_check(children->nodes[pos], lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } static bool -trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, - unsigned long frame, unsigned int *pos) +trie_children_find_position(const struct stack_depot_trie_children *children, + unsigned long frame, unsigned int *pos) { unsigned int left = 0; unsigned int right; - right = READ_ONCE(array->nr_children); + right = READ_ONCE(children->nr_children); while (left < right) { unsigned int mid = left + (right - left) / 2; const struct stack_depot_trie_node *node; unsigned long mid_frame; - node = trie_child_array_load_child(array, mid); + node = trie_children_load_node(children, mid); if (!node) { /* Tail append may produce a transient lockless lookup miss. */ right = mid; @@ -1803,54 +1729,51 @@ trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, } static void -trie_child_array_insert_at(const struct stack_depot_trie_child_array *old, - unsigned int pos, - const struct stack_depot_trie_node *node, - struct stack_depot_trie_child_array *new_array, - unsigned int new_capacity) +trie_children_insert_at(const struct stack_depot_trie_children *old_children, + unsigned int pos, + const struct stack_depot_trie_node *node, + struct stack_depot_trie_children *new_children) { unsigned int nr_old; unsigned int i; - nr_old = old->nr_children; + nr_old = old_children->nr_children; - new_array->nr_children = nr_old + 1; - new_array->capacity = new_capacity; + new_children->nr_children = nr_old + 1; for (i = 0; i < pos; i++) - RCU_INIT_POINTER(new_array->children[i], - trie_child_array_load_child(old, i)); - RCU_INIT_POINTER(new_array->children[pos], node); + RCU_INIT_POINTER(new_children->nodes[i], + trie_children_load_node(old_children, i)); + RCU_INIT_POINTER(new_children->nodes[pos], node); for (i = pos; i < nr_old; i++) - RCU_INIT_POINTER(new_array->children[i + 1], - trie_child_array_load_child(old, i)); - for (i = nr_old + 1; i < new_array->capacity; i++) - RCU_INIT_POINTER(new_array->children[i], NULL); + RCU_INIT_POINTER(new_children->nodes[i + 1], + trie_children_load_node(old_children, i)); + for (i = nr_old + 1; i < new_children->capacity; i++) + RCU_INIT_POINTER(new_children->nodes[i], NULL); } static void -trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array, - const struct stack_depot_trie_node *new_child, - struct stack_depot_trie_child_array *new_array, - unsigned int pos) +trie_children_replace_at(const struct stack_depot_trie_children *old_children, + const struct stack_depot_trie_node *new_node, + struct stack_depot_trie_children *new_children, + unsigned int pos) { unsigned int i; - new_array->nr_children = old_array->nr_children; - new_array->capacity = old_array->capacity; - for (i = 0; i < old_array->nr_children; i++) - RCU_INIT_POINTER(new_array->children[i], - trie_child_array_load_child(old_array, i)); - RCU_INIT_POINTER(new_array->children[pos], new_child); - for (i = old_array->nr_children; i < new_array->capacity; i++) - RCU_INIT_POINTER(new_array->children[i], NULL); + new_children->nr_children = old_children->nr_children; + for (i = 0; i < old_children->nr_children; i++) + RCU_INIT_POINTER(new_children->nodes[i], + trie_children_load_node(old_children, i)); + RCU_INIT_POINTER(new_children->nodes[pos], new_node); + for (i = old_children->nr_children; i < new_children->capacity; i++) + RCU_INIT_POINTER(new_children->nodes[i], NULL); } static void trie_reparent_children(struct stack_depot_trie_node *parent) { - const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_children *children; unsigned int i; - children = trie_load_children_slot(&parent->children); + children = trie_load_children(&parent->children); if (!children) return; /* @@ -1862,7 +1785,7 @@ static void trie_reparent_children(struct stack_depot_trie_node *parent) for (i = 0; i < children->nr_children; i++) { struct stack_depot_trie_node *child; - child = (struct stack_depot_trie_node *)trie_child_array_load_child(children, i); + child = (struct stack_depot_trie_node *)trie_children_load_node(children, i); rcu_assign_pointer(child->parent, parent); } } @@ -1871,82 +1794,67 @@ static void trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_node * const *nodes, - struct stack_depot_trie_child_array * const *chain_arrays) + struct stack_depot_trie_children * const *path_children) { const struct stack_depot_trie_node *prev = parent; - unsigned int pos = 0; + unsigned int entry = 0; unsigned int used = 0; unsigned int i; - while (pos < nr_entries) { + while (entry < nr_entries) { struct stack_depot_frame_run run; struct stack_depot_trie_node *node; u32 id; node = nodes[used]; - frame_run_init(&entries[pos], nr_entries - pos, &run); + frame_run_init(&entries[entry], nr_entries - entry, &run); - id = pos + run.nr_entries == nr_entries ? leaf_id : 0; - trie_node_init(node, prev, id, &entries[pos], &run); + id = entry + run.nr_entries == nr_entries ? leaf_id : 0; + trie_node_init(node, prev, id, &entries[entry], &run); prev = node; - pos += run.nr_entries; + entry += run.nr_entries; used++; } for (i = 0; i + 1 < used; i++) { const struct stack_depot_trie_node *next = nodes[i + 1]; struct stack_depot_trie_node *node = nodes[i]; - struct stack_depot_trie_child_array *array = chain_arrays[i]; + struct stack_depot_trie_children *children = path_children[i]; - trie_child_array_init(array, 1, &next, 1); - RCU_INIT_POINTER(node->children, array); + trie_children_init(children, &next, 1); + RCU_INIT_POINTER(node->children, children); } } -static void trie_publish_tail_append(struct stack_depot_trie_child_array *array, - unsigned int pos, - const struct stack_depot_trie_node *head) -{ - /* - * Writers hold stack_depot_trie_writer_lock. Existing children are - * immutable, so tail append publishes the new child before increasing the - * visible count. Lockless readers that see the old count miss; readers that - * see the new count either load the initialized child or treat a transient - * NULL as a miss. The writer-lock recheck prevents permanent duplicates. - */ - rcu_assign_pointer(array->children[pos], head); - WRITE_ONCE(array->nr_children, pos + 1); -} - static const struct stack_depot_trie_node * stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) { - const struct stack_depot_trie_child_array *children; - unsigned int pos = 0; + const struct stack_depot_trie_children *children; + unsigned int entry = 0; - children = trie_load_children_slot(&stack_depot_trie_root); + children = trie_load_children(&stack_depot_trie_root); - while (pos < nr_entries) { + while (entry < nr_entries) { const struct stack_depot_trie_node *node; - unsigned int remaining = nr_entries - pos; + unsigned int remaining = nr_entries - entry; unsigned int matched; - unsigned int slot; + unsigned int pos; if (!children) return NULL; - if (!trie_child_array_find_slot(children, entries[pos], &slot)) + if (!trie_children_find_position(children, entries[entry], &pos)) return NULL; - node = trie_child_array_load_child(children, slot); - matched = __stack_depot_trie_node_match(node, &entries[pos], remaining); + node = trie_children_load_node(children, pos); + matched = __stack_depot_trie_node_match(node, &entries[entry], remaining); if (matched < node->run.nr_entries) return NULL; - pos += matched; - if (pos == nr_entries) + entry += matched; + if (entry == nr_entries) return node->leaf_id ? node : NULL; - children = trie_load_children_slot(&node->children); + children = trie_load_children(&node->children); } return NULL; @@ -1956,15 +1864,15 @@ static unsigned int trie_size_append_chain(const unsigned long *entries, unsigned int nr_entries, size_t *node_sizes) { - unsigned int pos = 0; + unsigned int entry = 0; unsigned int nr_nodes = 0; - while (pos < nr_entries) { + while (entry < nr_entries) { struct stack_depot_frame_run run; - frame_run_init(&entries[pos], nr_entries - pos, &run); + frame_run_init(&entries[entry], nr_entries - entry, &run); node_sizes[nr_nodes] = trie_node_bytes(&run); - pos += run.nr_entries; + entry += run.nr_entries; nr_nodes++; } @@ -1972,97 +1880,136 @@ static unsigned int trie_size_append_chain(const unsigned long *entries, } static u32 -trie_insert_path(const struct stack_depot_trie_child_array __rcu **slot, +trie_insert_path(const struct stack_depot_trie_children __rcu **slot, struct stack_depot_trie_node *parent, + const struct stack_depot_trie_children *children, unsigned int pos, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_child_array *slot_array; - const struct stack_depot_trie_child_array *children; + struct stack_depot_trie_children *new_children; unsigned int capacity = 1; + unsigned int nr_nodes_allocated = 0; + unsigned int nr_path_children_allocated = 0; unsigned int nr_nodes; + unsigned int i; unsigned long flags; u32 new_leaf_id; - size_t slot_array_size; bool tail_append = false; - lockdep_assert_held(&stack_depot_trie_writer_lock); - - children = trie_load_children_slot(slot); if (children) { capacity = roundup_pow_of_two(children->nr_children + 1); tail_append = pos == children->nr_children && children->nr_children < children->capacity; } - slot_array_size = tail_append ? 0 : trie_child_array_bytes(capacity); - if (slot_array_size > DEPOT_POOL_SIZE) + if (!tail_append && + trie_children_alloc_size(capacity) > STACK_DEPOT_TRIE_POOL_USABLE_SIZE) return 0; new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; nr_nodes = trie_size_append_chain(entries, nr_entries, alloc->node_sizes); - if (trie_pool_alloc_insert(alloc, pool_prealloc, nr_nodes, - nr_nodes - 1, 0, slot_array_size)) - return 0; + + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + trie_drain_pending_children(); + + for (i = 0; i < nr_nodes; i++) { + alloc->nodes[i] = trie_pool_alloc(alloc->node_sizes[i], + pool_prealloc); + if (!alloc->nodes[i]) + goto err_release; + nr_nodes_allocated++; + } + for (i = 0; i + 1 < nr_nodes; i++) { + alloc->path_children[i] = + trie_pool_alloc_children(1, pool_prealloc); + if (!alloc->path_children[i]) + goto err_release; + nr_path_children_allocated++; + } + if (!tail_append) { + new_children = trie_pool_alloc_children(capacity, pool_prealloc); + if (!new_children) + goto err_release; + } + + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - alloc->nodes, alloc->chain_arrays); + alloc->nodes, alloc->path_children); trie_side_table_publish_leaf(new_leaf_id, alloc->nodes[nr_nodes - 1]); if (!children) { - slot_array = alloc->slot_array; - slot_array->nr_children = 1; - slot_array->capacity = 1; - RCU_INIT_POINTER(slot_array->children[0], alloc->nodes[0]); - rcu_assign_pointer(*slot, slot_array); + new_children->nr_children = 1; + RCU_INIT_POINTER(new_children->nodes[0], alloc->nodes[0]); + rcu_assign_pointer(*slot, new_children); return new_leaf_id; } if (tail_append) { - trie_publish_tail_append((struct stack_depot_trie_child_array *)children, - pos, alloc->nodes[0]); + struct stack_depot_trie_children *tail_children = + (struct stack_depot_trie_children *)children; + + /* + * Publish the node before the count read by READ_ONCE(). Readers + * either miss on the old count or on a transient NULL; the writer + * recheck prevents a permanent duplicate. + */ + rcu_assign_pointer(tail_children->nodes[pos], alloc->nodes[0]); + WRITE_ONCE(tail_children->nr_children, pos + 1); } else { - slot_array = alloc->slot_array; - trie_child_array_insert_at(children, pos, alloc->nodes[0], - slot_array, capacity); - rcu_assign_pointer(*slot, slot_array); + trie_children_insert_at(children, pos, alloc->nodes[0], new_children); + rcu_assign_pointer(*slot, new_children); raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_child_array(children); + trie_retire_children(children); raw_spin_unlock_irqrestore(&pool_lock, flags); } return new_leaf_id; + +err_release: + for (i = 0; i < nr_nodes_allocated; i++) + trie_pool_release(alloc->nodes[i], alloc->node_sizes[i]); + for (i = 0; i < nr_path_children_allocated; i++) + trie_pool_release_children(alloc->path_children[i]); + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return 0; } static u32 -trie_split_child(const struct stack_depot_trie_child_array __rcu **slot, +trie_split_child(const struct stack_depot_trie_children __rcu **slot, + const struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *child, unsigned int pos, unsigned int matched, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_child_array *prefix_children; - struct stack_depot_trie_child_array *slot_array; - const struct stack_depot_trie_child_array *children; - const struct stack_depot_trie_node *child; - const struct stack_depot_trie_node *split_children[2]; + struct stack_depot_trie_children *prefix_children = NULL; + struct stack_depot_trie_children *new_children; + const struct stack_depot_trie_node *new_tail; + const struct stack_depot_trie_node *split_leaf; + const struct stack_depot_trie_node *split_nodes[2]; struct stack_depot_frame_run run; struct stack_depot_trie_node *split_prefix; struct stack_depot_trie_node *old_tail; - unsigned long new_frame; - unsigned long old_frame; - unsigned int new_nodes = 0; - unsigned int tail_len; + unsigned long old_tail_frame; + unsigned int nr_nodes_allocated = 0; + unsigned int nr_path_children_allocated = 0; + unsigned int nr_tail_nodes = 0; + unsigned int nr_split_nodes; + unsigned int old_tail_len; + unsigned int i; + unsigned long flags; u32 new_leaf_id; bool has_new_tail; - lockdep_assert_held(&stack_depot_trie_writer_lock); - - children = trie_load_children_slot(slot); - child = trie_child_array_load_child(children, pos); new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; @@ -2070,102 +2017,143 @@ trie_split_child(const struct stack_depot_trie_child_array __rcu **slot, run = child->run; run.nr_entries = matched; alloc->node_sizes[0] = trie_node_bytes(&run); - run.nr_entries = child->run.nr_entries - matched; + old_tail_len = child->run.nr_entries - matched; + run.nr_entries = old_tail_len; alloc->node_sizes[1] = trie_node_bytes(&run); has_new_tail = matched < nr_entries; + nr_split_nodes = has_new_tail ? 2 : 1; if (has_new_tail) - new_nodes = trie_size_append_chain(&entries[matched], - nr_entries - matched, - &alloc->node_sizes[2]); - if (trie_pool_alloc_insert(alloc, pool_prealloc, 2 + new_nodes, - new_nodes ? new_nodes - 1 : 0, - trie_child_array_bytes(has_new_tail ? 2 : 1), - trie_child_array_bytes(children->capacity))) - return 0; + nr_tail_nodes = trie_size_append_chain(&entries[matched], + nr_entries - matched, + &alloc->node_sizes[2]); + + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + trie_drain_pending_children(); + + for (i = 0; i < 2 + nr_tail_nodes; i++) { + alloc->nodes[i] = trie_pool_alloc(alloc->node_sizes[i], + pool_prealloc); + if (!alloc->nodes[i]) + goto err_release; + nr_nodes_allocated++; + } + for (i = 0; i + 1 < nr_tail_nodes; i++) { + alloc->path_children[i] = + trie_pool_alloc_children(1, pool_prealloc); + if (!alloc->path_children[i]) + goto err_release; + nr_path_children_allocated++; + } + prefix_children = trie_pool_alloc_children(nr_split_nodes, pool_prealloc); + if (!prefix_children) + goto err_release; + new_children = trie_pool_alloc_children(children->capacity, pool_prealloc); + if (!new_children) + goto err_release; + + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); - slot_array = alloc->slot_array; - prefix_children = alloc->prefix_children; split_prefix = alloc->nodes[0]; old_tail = alloc->nodes[1]; - trie_node_init_slice(split_prefix, trie_load_parent(child), - has_new_tail ? 0 : new_leaf_id, child, 0, matched); - tail_len = child->run.nr_entries - matched; - trie_node_init_slice(old_tail, split_prefix, child->leaf_id, child, - matched, tail_len); if (has_new_tail) { trie_build_append_chain(split_prefix, new_leaf_id, &entries[matched], nr_entries - matched, &alloc->nodes[2], - alloc->chain_arrays); - stack_depot_trie_node_frame(old_tail, 0, &old_frame); - stack_depot_trie_node_frame(alloc->nodes[2], 0, &new_frame); - if (old_frame < new_frame) { - split_children[0] = old_tail; - split_children[1] = alloc->nodes[2]; + alloc->path_children); + new_tail = alloc->nodes[2]; + split_leaf = alloc->nodes[nr_tail_nodes + 1]; + stack_depot_trie_node_frame(child, matched, &old_tail_frame); + if (old_tail_frame < entries[matched]) { + split_nodes[0] = old_tail; + split_nodes[1] = new_tail; } else { - split_children[0] = alloc->nodes[2]; - split_children[1] = old_tail; + split_nodes[0] = new_tail; + split_nodes[1] = old_tail; } - trie_child_array_init(prefix_children, ARRAY_SIZE(split_children), - split_children, ARRAY_SIZE(split_children)); } else { - split_children[0] = old_tail; - trie_child_array_init(prefix_children, 1, split_children, 1); + split_leaf = split_prefix; + split_nodes[0] = old_tail; } + + trie_node_init_slice(split_prefix, trie_load_parent(child), + has_new_tail ? 0 : new_leaf_id, child, 0, matched); + trie_node_init_slice(old_tail, split_prefix, child->leaf_id, child, + matched, old_tail_len); + trie_children_init(prefix_children, split_nodes, nr_split_nodes); RCU_INIT_POINTER(old_tail->children, - trie_load_children_slot(&child->children)); + trie_load_children(&child->children)); RCU_INIT_POINTER(split_prefix->children, prefix_children); if (child->leaf_id) trie_side_table_publish_leaf(child->leaf_id, old_tail); - trie_side_table_publish_leaf(new_leaf_id, - has_new_tail ? alloc->nodes[new_nodes + 1] : - split_prefix); - trie_child_array_replace_at(children, split_prefix, slot_array, pos); + trie_side_table_publish_leaf(new_leaf_id, split_leaf); + trie_children_replace_at(children, split_prefix, new_children, pos); trie_reparent_children(old_tail); - rcu_assign_pointer(*slot, slot_array); - trie_retire_child_array_with_node(children, child); + rcu_assign_pointer(*slot, new_children); + trie_retire_children_with_node(children, child); return new_leaf_id; + +err_release: + for (i = 0; i < nr_nodes_allocated; i++) + trie_pool_release(alloc->nodes[i], alloc->node_sizes[i]); + for (i = 0; i < nr_path_children_allocated; i++) + trie_pool_release_children(alloc->path_children[i]); + if (prefix_children) + trie_pool_release_children(prefix_children); + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return 0; } static u32 -trie_promote_child(const struct stack_depot_trie_child_array __rcu **slot, +trie_promote_child(const struct stack_depot_trie_children __rcu **slot, + const struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *child, unsigned int pos, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { - struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_child_array *slot_array; - const struct stack_depot_trie_child_array *children; - const struct stack_depot_trie_node *child; + struct stack_depot_trie_children *new_children; struct stack_depot_trie_node *promoted_node; + unsigned long flags; + size_t node_size; u32 new_leaf_id; - size_t slot_array_size; - - lockdep_assert_held(&stack_depot_trie_writer_lock); - children = trie_load_children_slot(slot); - child = trie_child_array_load_child(children, pos); new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); if (!new_leaf_id) return 0; - alloc->node_sizes[0] = trie_node_bytes(&child->run); - slot_array_size = trie_child_array_bytes(children->capacity); - if (trie_pool_alloc_insert(alloc, pool_prealloc, 1, 0, 0, - slot_array_size)) - return 0; + node_size = trie_node_bytes(&child->run); - promoted_node = alloc->nodes[0]; - slot_array = alloc->slot_array; - memcpy(promoted_node, child, alloc->node_sizes[0]); + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + trie_drain_pending_children(); + promoted_node = trie_pool_alloc(node_size, pool_prealloc); + if (!promoted_node) + goto out_unlock; + new_children = trie_pool_alloc_children(children->capacity, pool_prealloc); + if (!new_children) + goto out_release_node; + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + + memcpy(promoted_node, child, node_size); promoted_node->leaf_id = new_leaf_id; trie_side_table_publish_leaf(new_leaf_id, promoted_node); - trie_child_array_replace_at(children, promoted_node, slot_array, pos); + trie_children_replace_at(children, promoted_node, new_children, pos); trie_reparent_children(promoted_node); - rcu_assign_pointer(*slot, slot_array); - trie_retire_child_array_with_node(children, child); + rcu_assign_pointer(*slot, new_children); + trie_retire_children_with_node(children, child); return new_leaf_id; + +out_release_node: + trie_pool_release(promoted_node, node_size); +out_unlock: + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return 0; } static u32 @@ -2173,8 +2161,8 @@ stack_depot_trie_insert(const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { - const struct stack_depot_trie_child_array *children; - const struct stack_depot_trie_child_array __rcu **slot = + const struct stack_depot_trie_children *children; + const struct stack_depot_trie_children __rcu **slot = &stack_depot_trie_root; const struct stack_depot_trie_node *child; struct stack_depot_trie_node *parent = NULL; @@ -2186,19 +2174,20 @@ stack_depot_trie_insert(const unsigned long *entries, for (;;) { pos = 0; - children = trie_load_children_slot(slot); + children = trie_load_children(slot); if (!children || - !trie_child_array_find_slot(children, entries[0], &pos)) { - leaf_id = trie_insert_path(slot, parent, pos, entries, + !trie_children_find_position(children, entries[0], &pos)) { + leaf_id = trie_insert_path(slot, parent, children, pos, entries, nr_entries, pool_prealloc, side_prealloc); break; } - child = trie_child_array_load_child(children, pos); + child = trie_children_load_node(children, pos); matched = __stack_depot_trie_node_match(child, entries, nr_entries); if (matched < child->run.nr_entries) { - leaf_id = trie_split_child(slot, pos, matched, entries, + leaf_id = trie_split_child(slot, children, child, pos, + matched, entries, nr_entries, pool_prealloc, side_prealloc); @@ -2208,7 +2197,8 @@ stack_depot_trie_insert(const unsigned long *entries, if (matched == nr_entries) { if (child->leaf_id) return child->leaf_id; - leaf_id = trie_promote_child(slot, pos, pool_prealloc, + leaf_id = trie_promote_child(slot, children, child, pos, + pool_prealloc, side_prealloc); break; } From f9d58e1fc1557839fe37ae4df37dea4b6aec0fdc Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 30 Jul 2026 12:00:32 +0100 Subject: [PATCH 35/38] KRN-1117: Strengthen stackdepot trie topology coverage Add a shorter-stack split where the input ends inside an existing run. Re-save every topology fixture after all mutations and require its original handle, so the test covers lookup and deduplication as well as materialization. Signed-off-by: Caleb Kan --- lib/tests/stackdepot_kunit.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 4fba72e283312..e74a04a2ac171 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -253,8 +253,10 @@ static void stackdepot_trie_topology_roundtrip(struct kunit *test) { 0x301000UL, 0x302000UL }, { 0x301000UL, 0x302000UL, 0x303000UL }, { 0x301000UL, 0x304000UL }, + { 0x401000UL, 0x402000UL, 0x403000UL }, + { 0x401000UL, 0x402000UL }, }; - unsigned int nr_entries[] = { 2, 2, 1, 3, 2, 2, 2, 2, 2, 3, 2 }; + unsigned int nr_entries[] = { 2, 2, 1, 3, 2, 2, 2, 2, 2, 3, 2, 3, 2 }; depot_stack_handle_t handles[ARRAY_SIZE(stacks)]; unsigned long fetched[ARRAY_SIZE(stacks[0])]; u32 pool_index_plus_1; @@ -281,6 +283,9 @@ static void stackdepot_trie_topology_roundtrip(struct kunit *test) nr_entries[i]); KUNIT_EXPECT_MEMEQ(test, fetched, stacks[i], nr_entries[i] * sizeof(fetched[0])); + KUNIT_EXPECT_EQ(test, + stack_depot_save(stacks[i], nr_entries[i], GFP_KERNEL), + handles[i]); } } From 7e200829185d8a885d4c97642b104b1a4f1fdfb3 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 30 Jul 2026 16:49:42 +0100 Subject: [PATCH 36/38] KRN-1117: Simplify stackdepot trie path construction Replace the maximum-sized writer workspace and separate sizing, reservation, construction, and rollback passes with trie_path_alloc(). Build each frame run once and unwind an unpublished path through parent links, removing init-time workspace allocation and bulk cleanup state. Use stack ID, path, prefix, and suffix terminology consistently. Inline one-use publication and replacement helpers, derive publication IDs from nodes, and fold pass-through initialization wrappers while preserving allocator-specific memblock and runtime paths. Retain bounded retry, complete pool-size guards, and tail append. A stalled-RCU fanout probe used 622 pools with always-COW children versus four with tail append, so the simpler topology still preserves bounded pool behavior. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 816 ++++++++++++++++++++--------------------------- 1 file changed, 350 insertions(+), 466 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 3875c79b5af53..e2179b631af1c 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -6,7 +6,7 @@ * callers that request STACK_DEPOT_FLAG_COUNTABLE use the legacy hash table with * contiguous stack records in stack pools. Persistent non-refcounted entries * can use trie storage when enabled; trie nodes share common frame prefixes and - * are published through RCU/COW children containers. + * are published through RCU children containers. * * Author: Alexander Potapenko * Copyright (C) 2016 Google, Inc. @@ -127,19 +127,20 @@ static_assert(CONFIG_STACKDEPOT_MAX_FRAMES <= U16_MAX); struct stack_depot_trie_children; struct stack_depot_trie_node { - /* Parent links let fetch rebuild a full stack from a leaf to the root. */ + /* Parent links let fetch rebuild a full stack from a node to the root. */ const struct stack_depot_trie_node __rcu *parent; - /* Children are separate RCU/COW generations. */ + /* Children are RCU-published containers. */ const struct stack_depot_trie_children __rcu *children; - u32 leaf_id; + /* Non-zero when a stored stack ends at this node. */ + u32 stack_id; struct stack_depot_frame_run run; unsigned char data[]; }; /* * Child nodes are sorted by first frame and searched by insertion position. - * Writers may append to spare capacity at the sorted tail, but never change - * existing node pointers. Other updates publish a replacement container. + * Existing child pointers are immutable. Writers may publish into unused tail + * capacity; other updates publish a replacement container. */ struct stack_depot_trie_children { unsigned int nr_children; @@ -151,15 +152,13 @@ struct stack_depot_trie_children { struct stack_depot_trie_retired_children { struct list_head list; unsigned long rcu_state; - struct stack_depot_trie_node *pending_node; + const struct stack_depot_trie_node *pending_node; unsigned char data[]; }; static_assert(IS_ALIGNED(offsetof(struct stack_depot_trie_retired_children, data), 1UL << DEPOT_STACK_ALIGN)); -#define STACK_DEPOT_TRIE_MAX_NODES (CONFIG_STACKDEPOT_MAX_FRAMES + 1) -#define STACK_DEPOT_TRIE_MAX_PATH_CHILDREN CONFIG_STACKDEPOT_MAX_FRAMES #define STACK_DEPOT_TRIE_SLOT_SIZE BIT(DEPOT_STACK_ALIGN) #define STACK_DEPOT_TRIE_POOL_SLOTS \ (DEPOT_POOL_SIZE / STACK_DEPOT_TRIE_SLOT_SIZE) @@ -179,17 +178,8 @@ struct stack_depot_trie_pool { static_assert(STACK_DEPOT_TRIE_POOL_FIRST_SLOT < STACK_DEPOT_TRIE_POOL_SLOTS); -/* Writer-owned storage for one unpublished trie insertion. */ -struct stack_depot_trie_insert_alloc { - struct stack_depot_trie_node *nodes[STACK_DEPOT_TRIE_MAX_NODES]; - size_t node_sizes[STACK_DEPOT_TRIE_MAX_NODES]; - struct stack_depot_trie_children - *path_children[STACK_DEPOT_TRIE_MAX_PATH_CHILDREN]; -}; - static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); static const struct stack_depot_trie_children __rcu *stack_depot_trie_root; -static struct stack_depot_trie_insert_alloc *stack_depot_trie_alloc; static DEFINE_RAW_SPINLOCK(stack_depot_trie_writer_lock); static bool stack_depot_trie_requested; @@ -207,23 +197,23 @@ static LIST_HEAD(pending_trie_children); * stack_max_pools is the split point between hash and trie handle encodings. * A handle with pool_index_plus_1 in 1..stack_max_pools names a hash-backed * stack pool. Larger pool-index values cannot refer to hash pools, so trie - * storage uses that handle space to encode a dense leaf_id. The side table - * maps each leaf_id to its trie leaf node. + * storage uses that handle space to encode a dense stack ID. The side table + * maps each stack ID to its trie node. */ -static inline u32 __stack_depot_trie_max_leaf_id(void) +static inline u32 trie_max_stack_id(void) { return (DEPOT_POOL_INDEX_MASK - stack_max_pools) << DEPOT_OFFSET_BITS; } -static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) +static depot_stack_handle_t trie_handle(u32 stack_id) { union handle_parts parts = {}; u64 pool_index_plus_1; u32 pool_delta; u32 index; - index = leaf_id - 1; + index = stack_id - 1; pool_delta = index >> DEPOT_OFFSET_BITS; pool_index_plus_1 = (u64)stack_max_pools + 1 + pool_delta; @@ -239,7 +229,7 @@ static inline bool stack_depot_handle_is_trie(depot_stack_handle_t handle) return parts.pool_index_plus_1 > stack_max_pools; } -static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) +static u32 trie_stack_id(depot_stack_handle_t handle) { union handle_parts parts = { .handle = handle }; u32 pool_delta; @@ -249,13 +239,11 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) } /* - * Trie handles encode a dense leaf ID. The side table maps that ID to a leaf - * pointer for lockless fetch/print paths, which can run from diagnostic - * contexts where taking a lock would be unsafe. Initialization - * installs the root; early initialization also installs the first directory and - * chunk. Additional directories and chunks are preallocated and published - * lazily as leaf IDs grow. RCU pointer publication makes fully initialized - * directories, chunks, and leaves visible to those lockless readers. + * Trie handles encode a dense stack ID. The side table maps that ID to a node + * pointer for lockless fetch and print paths, which can run from diagnostic + * contexts where taking a lock would be unsafe. Initialization installs the + * root; early initialization also installs the first directory and chunk. + * Additional directories and chunks are published lazily as stack IDs grow. */ #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ (PAGE_SIZE / sizeof(struct stack_depot_trie_node *)) @@ -263,7 +251,7 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) (PAGE_SIZE / sizeof(struct stack_depot_trie_node **)) struct stack_depot_trie_side_dir { - /* Both the chunk pointer and each leaf pointer in it are RCU-published. */ + /* Both the chunk pointer and each node pointer in it are RCU-published. */ const struct stack_depot_trie_node __rcu * __rcu * chunks[STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE]; }; @@ -276,7 +264,7 @@ struct stack_depot_trie_side_root { struct stack_depot_trie_side_prealloc { /* Preallocated side-table directory page for sparse growth. */ struct stack_depot_trie_side_dir *dir; - /* Preallocated side-table leaf chunk for sparse growth. */ + /* Preallocated side-table pointer chunk for sparse growth. */ const struct stack_depot_trie_node __rcu **chunk; }; @@ -284,7 +272,7 @@ static struct stack_depot_trie_side_root *trie_side_table_root; static DEFINE_RAW_SPINLOCK(trie_side_table_cache_lock); /* Zeroed unpublished pages; get/put transfer ownership under the cache lock. */ static struct stack_depot_trie_side_prealloc trie_side_table_cache; -static u32 trie_side_table_last_leaf_id; +static u32 trie_side_table_last_stack_id; /* Lock order: writer_lock -> pool_lock. The cache lock is never nested. */ @@ -316,19 +304,6 @@ static size_t trie_children_alloc_size(unsigned int capacity) ALIGN(size, sizeof(unsigned long)); } -static void trie_children_init(struct stack_depot_trie_children *children, - const struct stack_depot_trie_node * const *nodes, - unsigned int nr_children) -{ - unsigned int i; - - children->nr_children = nr_children; - for (i = 0; i < nr_children; i++) - RCU_INIT_POINTER(children->nodes[i], nodes[i]); - for (i = nr_children; i < children->capacity; i++) - RCU_INIT_POINTER(children->nodes[i], NULL); -} - static inline unsigned int trie_side_table_root_index(u32 id) { return ((id - 1) / STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE) / @@ -363,14 +338,14 @@ static inline const struct stack_depot_trie_node __rcu ** trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, unsigned int idx) { - /* Pairs with the chunk rcu_assign_pointer() in leaf ID preparation. */ + /* Pairs with the chunk rcu_assign_pointer() in stack ID preparation. */ return rcu_dereference_check(dir->chunks[idx], lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); } static u32 -trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *prealloc) +trie_side_table_prepare_stack_slot(struct stack_depot_trie_side_prealloc *prealloc) { const struct stack_depot_trie_node __rcu **chunk; struct stack_depot_trie_side_dir *dir; @@ -381,16 +356,14 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo lockdep_assert_held(&stack_depot_trie_writer_lock); - id = trie_side_table_last_leaf_id + 1; - if (id > __stack_depot_trie_max_leaf_id()) + id = trie_side_table_last_stack_id + 1; + if (id > trie_max_stack_id()) return 0; root_vec = trie_side_table_root; root = trie_side_table_root_index(id); dir = trie_side_table_load_dir(root); if (!dir) { - if (WARN_ON_ONCE(!prealloc->dir)) - return 0; dir = prealloc->dir; prealloc->dir = NULL; /* Publish the zeroed directory before readers can load it locklessly. */ @@ -400,8 +373,6 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo idx = trie_side_table_dir_index(id); chunk = trie_side_table_dir_load_chunk(dir, idx); if (!chunk) { - if (WARN_ON_ONCE(!prealloc->chunk)) - return 0; chunk = prealloc->chunk; prealloc->chunk = NULL; rcu_assign_pointer(dir->chunks[idx], chunk); @@ -410,121 +381,71 @@ trie_side_table_prepare_leaf_slot(struct stack_depot_trie_side_prealloc *preallo return id; } -static void trie_side_table_root_init(struct stack_depot_trie_side_root *root_vec, - unsigned int root_size) -{ - root_vec->dir_capacity = root_size; - trie_side_table_last_leaf_id = 0; -} - -static inline unsigned int trie_side_table_root_size_for_max_id(u32 max_leaf_id) +static inline unsigned int trie_side_table_root_size_for_max_id(u32 max_stack_id) { unsigned int top_size; - top_size = DIV_ROUND_UP(max_leaf_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); + top_size = DIV_ROUND_UP(max_stack_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); return DIV_ROUND_UP(top_size, STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE); } -static int __init __stack_depot_trie_side_table_init_memblock(void) +static int __init stack_depot_trie_init_memblock(void) { struct stack_depot_trie_side_root *root_vec; struct stack_depot_trie_side_dir *first_dir; const struct stack_depot_trie_node __rcu **first_chunk; size_t root_bytes; - u32 max_leaf_id; + u32 max_stack_id; unsigned int root_size; - max_leaf_id = __stack_depot_trie_max_leaf_id(); - if (!max_leaf_id) + max_stack_id = trie_max_stack_id(); + if (!max_stack_id) return -EINVAL; - root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_size = trie_side_table_root_size_for_max_id(max_stack_id); root_bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); root_vec = memblock_alloc(root_bytes, __alignof__(*root_vec)); if (!root_vec) return -ENOMEM; - memset(root_vec, 0, root_bytes); first_dir = memblock_alloc(PAGE_SIZE, PAGE_SIZE); if (!first_dir) { memblock_free(root_vec, root_bytes); return -ENOMEM; } - memset(first_dir, 0, PAGE_SIZE); first_chunk = memblock_alloc(PAGE_SIZE, PAGE_SIZE); if (!first_chunk) { memblock_free(first_dir, PAGE_SIZE); memblock_free(root_vec, root_bytes); return -ENOMEM; } - memset(first_chunk, 0, PAGE_SIZE); - trie_side_table_root_init(root_vec, root_size); + root_vec->dir_capacity = root_size; RCU_INIT_POINTER(root_vec->dirs[0], first_dir); RCU_INIT_POINTER(first_dir->chunks[0], first_chunk); trie_side_table_root = root_vec; + static_branch_enable(&stack_depot_trie_enabled); return 0; } -static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) +static int stack_depot_trie_init(void) { struct stack_depot_trie_side_root *root_vec; unsigned int root_size; size_t root_bytes; - u32 max_leaf_id; + u32 max_stack_id; - max_leaf_id = __stack_depot_trie_max_leaf_id(); - if (!max_leaf_id) + max_stack_id = trie_max_stack_id(); + if (!max_stack_id) return -EINVAL; - root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_size = trie_side_table_root_size_for_max_id(max_stack_id); root_bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); - root_vec = kvzalloc(root_bytes, gfp_flags); + root_vec = kvzalloc(root_bytes, GFP_KERNEL); if (!root_vec) return -ENOMEM; - trie_side_table_root_init(root_vec, root_size); + root_vec->dir_capacity = root_size; trie_side_table_root = root_vec; - return 0; -} - -static int __init stack_depot_trie_init_memblock(void) -{ - struct stack_depot_trie_insert_alloc *alloc; - size_t size; - int ret; - - size = sizeof(*stack_depot_trie_alloc); - alloc = memblock_alloc(size, __alignof__(*alloc)); - if (!alloc) - return -ENOMEM; - - ret = __stack_depot_trie_side_table_init_memblock(); - if (ret) { - memblock_free(alloc, size); - return ret; - } - stack_depot_trie_alloc = alloc; - - static_branch_enable(&stack_depot_trie_enabled); - return 0; -} - -static int stack_depot_trie_init(gfp_t gfp_flags) -{ - struct stack_depot_trie_insert_alloc *alloc; - int ret; - - alloc = kvzalloc(sizeof(*stack_depot_trie_alloc), gfp_flags); - if (!alloc) - return -ENOMEM; - - ret = __stack_depot_trie_side_table_init(gfp_flags); - if (ret) { - kvfree(alloc); - return ret; - } - stack_depot_trie_alloc = alloc; - static_branch_enable(&stack_depot_trie_enabled); return 0; } @@ -577,20 +498,7 @@ static void trie_side_table_put_prealloc(struct stack_depot_trie_side_prealloc * free_page((unsigned long)prealloc->chunk); } -static const struct stack_depot_trie_node __rcu **trie_side_table_leaf_slot(u32 id) -{ - const struct stack_depot_trie_node __rcu **chunk; - struct stack_depot_trie_side_dir *dir; - unsigned int root; - - root = trie_side_table_root_index(id); - dir = trie_side_table_load_dir(root); - chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); - - return &chunk[trie_side_table_slot_index(id)]; -} - -static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup(u32 id) +static const struct stack_depot_trie_node *trie_side_table_lookup(u32 id) { const struct stack_depot_trie_node __rcu **chunk; struct stack_depot_trie_side_dir *dir; @@ -604,7 +512,7 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( if (!chunk) return NULL; - /* Pairs with side-table leaf rcu_assign_pointer(). */ + /* Pairs with side-table node publication. */ return rcu_dereference_check(chunk[trie_side_table_slot_index(id)], lockdep_is_held(&stack_depot_trie_writer_lock) || rcu_read_lock_sched_held()); @@ -621,7 +529,6 @@ static bool depot_init_pool(void **prealloc); static unsigned int trie_pool_reserve_slots(struct stack_depot_trie_pool *pool, unsigned int nr_slots) { - unsigned int run_start = STACK_DEPOT_TRIE_POOL_FIRST_SLOT; unsigned int run = 0; unsigned int i; unsigned int slot; @@ -636,20 +543,19 @@ static unsigned int trie_pool_reserve_slots(struct stack_depot_trie_pool *pool, run = 0; continue; } - if (!run) - run_start = slot; if (++run != nr_slots) continue; - for (i = run_start; i < run_start + nr_slots; i++) + for (i = slot + 1 - nr_slots; i <= slot; i++) pool->used[i / BITS_PER_LONG] |= BIT(i % BITS_PER_LONG); pool->free_slots -= nr_slots; - return run_start; + return slot + 1 - nr_slots; } return STACK_DEPOT_TRIE_POOL_SLOTS; } +/* Allocate at least @size bytes from one contiguous trie-pool slot run. */ static void *trie_pool_alloc(size_t size, void **prealloc) { struct stack_depot_trie_pool *pool; @@ -681,6 +587,7 @@ static void *trie_pool_alloc(size_t size, void **prealloc) return (char *)pool + slot * STACK_DEPOT_TRIE_SLOT_SIZE; } +/* Release the slots for the byte count originally passed to allocation. */ static void trie_pool_release(const void *ptr, size_t size) { struct stack_depot_trie_pool *pool; @@ -707,11 +614,13 @@ trie_pool_alloc_children(unsigned int capacity, void **prealloc) struct stack_depot_trie_retired_children *retired; struct stack_depot_trie_children *children; + /* Capacity counts child-pointer entries; allocation includes RCU metadata. */ retired = trie_pool_alloc(trie_children_alloc_size(capacity), prealloc); if (!retired) return NULL; children = (void *)retired->data; + children->nr_children = 0; children->capacity = capacity; return children; } @@ -719,10 +628,16 @@ trie_pool_alloc_children(unsigned int capacity, void **prealloc) static void trie_pool_release_children(const struct stack_depot_trie_children *children) { + /* Capacity is immutable and therefore recovers the allocation byte size. */ trie_pool_release(trie_retired_children(children), trie_children_alloc_size(children->capacity)); } +/* + * Return RCU-ready objects before allocating. Pending children are FIFO, so + * stop at the first incomplete grace period. A replaced node shares the same + * retirement cookie and is released with its former children container. + */ static void trie_drain_pending_children(void) { struct stack_depot_trie_retired_children *retired; @@ -732,7 +647,6 @@ static void trie_drain_pending_children(void) lockdep_assert_held(&pool_lock); list_for_each_entry_safe(retired, tmp, &pending_trie_children, list) { - /* Pending children are FIFO; later entries cannot be ready yet. */ if (!poll_state_synchronize_rcu(retired->rcu_state)) break; children = (void *)retired->data; @@ -744,30 +658,30 @@ static void trie_drain_pending_children(void) } } -static void trie_retire_children(const void *ptr) +static void trie_retire_children(const struct stack_depot_trie_children *children) { struct stack_depot_trie_retired_children *retired; lockdep_assert_held(&pool_lock); - retired = trie_retired_children(ptr); + retired = trie_retired_children(children); retired->pending_node = NULL; retired->rcu_state = get_state_synchronize_rcu(); list_add_tail(&retired->list, &pending_trie_children); } static void -trie_retire_children_with_node(const void *ptr, +trie_retire_children_with_node(const struct stack_depot_trie_children *children, const struct stack_depot_trie_node *node) { struct stack_depot_trie_retired_children *retired; - unsigned long flags; - raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_children(ptr); - retired = trie_retired_children(ptr); - retired->pending_node = (struct stack_depot_trie_node *)node; - raw_spin_unlock_irqrestore(&pool_lock, flags); + lockdep_assert_held(&stack_depot_trie_writer_lock); + raw_spin_lock(&pool_lock); + trie_retire_children(children); + retired = trie_retired_children(children); + retired->pending_node = node; + raw_spin_unlock(&pool_lock); } static const struct stack_depot_trie_node * @@ -777,27 +691,37 @@ static depot_stack_handle_t trie_find_handle(const unsigned long *entries, unsigned int nr_entries) { depot_stack_handle_t handle = 0; - const struct stack_depot_trie_node *leaf; + const struct stack_depot_trie_node *node; rcu_read_lock_sched_notrace(); - leaf = stack_depot_trie_lookup(entries, nr_entries); - if (leaf) - handle = __stack_depot_trie_handle(leaf->leaf_id); + node = stack_depot_trie_lookup(entries, nr_entries); + if (node) + handle = trie_handle(node->stack_id); rcu_read_unlock_sched_notrace(); return handle; } -static void trie_side_table_publish_leaf(u32 leaf_id, - const struct stack_depot_trie_node *leaf) +/* + * Publish only after the node and its path are fully initialized and all + * fallible allocation is complete. Publication commits the path, so it cannot + * then be rolled back. The side-table mapping must precede the trie slot that + * makes a new stack ID reachable from lookup. Published storage remains valid + * until RCU retirement; only descendant parent links may change meanwhile. + */ +static void trie_side_table_publish(const struct stack_depot_trie_node *node) { - const struct stack_depot_trie_node __rcu **slot; + const struct stack_depot_trie_node __rcu **chunk; + struct stack_depot_trie_side_dir *dir; + u32 stack_id = node->stack_id; lockdep_assert_held(&stack_depot_trie_writer_lock); - slot = trie_side_table_leaf_slot(leaf_id); - /* Pairs with __stack_depot_trie_side_table_lookup(). */ - rcu_assign_pointer(*slot, leaf); + dir = trie_side_table_load_dir(trie_side_table_root_index(stack_id)); + chunk = trie_side_table_dir_load_chunk(dir, + trie_side_table_dir_index(stack_id)); + /* Pairs with trie_side_table_lookup(). */ + rcu_assign_pointer(chunk[trie_side_table_slot_index(stack_id)], node); } static int __init disable_stack_depot(char *str) @@ -990,7 +914,7 @@ int stack_depot_init(void) goto out_unlock; } if (stack_depot_trie_requested) { - ret = stack_depot_trie_init(GFP_KERNEL); + ret = stack_depot_trie_init(); if (ret) { pr_warn("trie storage initialization failed, disabling trie storage\n"); stack_depot_trie_requested = false; @@ -1370,13 +1294,17 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, depot_stack_handle_t handle; unsigned long flags; struct page *page; - u32 leaf_id = 0; - int ret; + u32 stack_id = 0; handle = trie_find_handle(entries, nr_entries); if (handle) return handle; + if (trie_side_table_get_prealloc(alloc_flags, &side_prealloc)) { + trie_side_table_put_prealloc(&side_prealloc); + return 0; + } + /* The hint may race; a missing page is recovered by the retry. */ if (!READ_ONCE(new_pool)) { page = alloc_pages(gfp_nested_mask(alloc_flags), @@ -1384,17 +1312,13 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, if (page) pool_prealloc = page_address(page); } - ret = trie_side_table_get_prealloc(alloc_flags, &side_prealloc); - if (!ret) { - raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); - leaf_id = stack_depot_trie_insert(entries, nr_entries, - &pool_prealloc, - &side_prealloc); - raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, - flags); - } - if (pool_prealloc && !ret) { + raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); + stack_id = stack_depot_trie_insert(entries, nr_entries, + &pool_prealloc, &side_prealloc); + raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); + + if (pool_prealloc) { raw_spin_lock_irqsave(&pool_lock, flags); depot_keep_new_pool(&pool_prealloc); raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -1402,10 +1326,8 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, if (pool_prealloc) free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); trie_side_table_put_prealloc(&side_prealloc); - if (ret) - return 0; - if (leaf_id) - return __stack_depot_trie_handle(leaf_id); + if (stack_id) + return trie_handle(stack_id); } return 0; @@ -1593,7 +1515,7 @@ stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, } static void trie_node_init(struct stack_depot_trie_node *node, - const struct stack_depot_trie_node *parent, u32 leaf_id, + const struct stack_depot_trie_node *parent, u32 stack_id, const unsigned long *entries, const struct stack_depot_frame_run *run) { @@ -1613,12 +1535,12 @@ static void trie_node_init(struct stack_depot_trie_node *node, RCU_INIT_POINTER(node->parent, parent); RCU_INIT_POINTER(node->children, NULL); - node->leaf_id = leaf_id; + node->stack_id = stack_id; node->run = *run; } static void trie_node_init_slice(struct stack_depot_trie_node *node, - const struct stack_depot_trie_node *parent, u32 leaf_id, + const struct stack_depot_trie_node *parent, u32 stack_id, const struct stack_depot_trie_node *src_node, unsigned int start, unsigned int nr_entries) { @@ -1633,14 +1555,13 @@ static void trie_node_init_slice(struct stack_depot_trie_node *node, stack_depot_frame_run_bytes(&run)); RCU_INIT_POINTER(node->parent, parent); RCU_INIT_POINTER(node->children, NULL); - node->leaf_id = leaf_id; + node->stack_id = stack_id; node->run = run; } -static unsigned int -__stack_depot_trie_node_match(const struct stack_depot_trie_node *node, - const unsigned long *entries, - unsigned int nr_entries) +static unsigned int trie_node_match(const struct stack_depot_trie_node *node, + const unsigned long *entries, + unsigned int nr_entries) { unsigned int limit; unsigned int i; @@ -1686,8 +1607,8 @@ trie_load_children(const struct stack_depot_trie_children __rcu * const *slot) } static inline const struct stack_depot_trie_node * -trie_children_load_node(const struct stack_depot_trie_children *children, - unsigned int pos) +trie_children_load_child(const struct stack_depot_trie_children *children, + unsigned int pos) { return rcu_dereference_check(children->nodes[pos], lockdep_is_held(&stack_depot_trie_writer_lock) || @@ -1707,7 +1628,7 @@ trie_children_find_position(const struct stack_depot_trie_children *children, const struct stack_depot_trie_node *node; unsigned long mid_frame; - node = trie_children_load_node(children, mid); + node = trie_children_load_child(children, mid); if (!node) { /* Tail append may produce a transient lockless lookup miss. */ right = mid; @@ -1728,44 +1649,31 @@ trie_children_find_position(const struct stack_depot_trie_children *children, return false; } -static void -trie_children_insert_at(const struct stack_depot_trie_children *old_children, - unsigned int pos, - const struct stack_depot_trie_node *node, - struct stack_depot_trie_children *new_children) +/* Initialize an unpublished container from a stable published prefix. */ +static void trie_children_init(const struct stack_depot_trie_children *old, + struct stack_depot_trie_children *new) { - unsigned int nr_old; + unsigned int nr_old = old->nr_children; unsigned int i; - nr_old = old_children->nr_children; - - new_children->nr_children = nr_old + 1; - for (i = 0; i < pos; i++) - RCU_INIT_POINTER(new_children->nodes[i], - trie_children_load_node(old_children, i)); - RCU_INIT_POINTER(new_children->nodes[pos], node); - for (i = pos; i < nr_old; i++) - RCU_INIT_POINTER(new_children->nodes[i + 1], - trie_children_load_node(old_children, i)); - for (i = nr_old + 1; i < new_children->capacity; i++) - RCU_INIT_POINTER(new_children->nodes[i], NULL); + new->nr_children = nr_old; + for (i = 0; i < nr_old; i++) + RCU_INIT_POINTER(new->nodes[i], trie_children_load_child(old, i)); + for (i = nr_old; i < new->capacity; i++) + RCU_INIT_POINTER(new->nodes[i], NULL); } -static void -trie_children_replace_at(const struct stack_depot_trie_children *old_children, - const struct stack_depot_trie_node *new_node, - struct stack_depot_trie_children *new_children, - unsigned int pos) +static void trie_children_insert(struct stack_depot_trie_children *children, + const struct stack_depot_trie_node *node, + unsigned int pos) { unsigned int i; - new_children->nr_children = old_children->nr_children; - for (i = 0; i < old_children->nr_children; i++) - RCU_INIT_POINTER(new_children->nodes[i], - trie_children_load_node(old_children, i)); - RCU_INIT_POINTER(new_children->nodes[pos], new_node); - for (i = old_children->nr_children; i < new_children->capacity; i++) - RCU_INIT_POINTER(new_children->nodes[i], NULL); + for (i = children->nr_children; i > pos; i--) + RCU_INIT_POINTER(children->nodes[i], + trie_children_load_child(children, i - 1)); + RCU_INIT_POINTER(children->nodes[pos], node); + children->nr_children++; } static void trie_reparent_children(struct stack_depot_trie_node *parent) @@ -1773,58 +1681,86 @@ static void trie_reparent_children(struct stack_depot_trie_node *parent) const struct stack_depot_trie_children *children; unsigned int i; + lockdep_assert_held(&stack_depot_trie_writer_lock); + children = trie_load_children(&parent->children); if (!children) return; /* - * COW updates reuse unchanged descendant subtrees. Repoint their parent - * links before retiring the old parent so fetch never follows a freed node. - * Lockless fetches may see the new parent before structural publication, but - * the old and new parent chains contain the same frames and remain RCU-live. + * Replacement nodes reuse unchanged descendant subtrees. Repoint their + * parent links before retiring the old parent so fetch never follows a freed + * node. Lockless fetches may see the new parent before publication, but the + * old and new parent chains contain the same frames and remain RCU-live. */ for (i = 0; i < children->nr_children; i++) { struct stack_depot_trie_node *child; - child = (struct stack_depot_trie_node *)trie_children_load_node(children, i); + child = (struct stack_depot_trie_node *)trie_children_load_child(children, i); rcu_assign_pointer(child->parent, parent); } } -static void -trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_node * const *nodes, - struct stack_depot_trie_children * const *path_children) -{ - const struct stack_depot_trie_node *prev = parent; +/* + * Split entries into runs, allocate and initialize each node once, and link + * adjacent nodes through singleton children. Both trie locks must be held. + * Failure walks the unpublished parent chain and releases local ownership. + */ +static const struct stack_depot_trie_node * +trie_path_alloc(const struct stack_depot_trie_node *parent, u32 stack_id, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + const struct stack_depot_trie_node **node_out) +{ + struct stack_depot_trie_children *path_children = NULL; + const struct stack_depot_trie_node *path_root = NULL; + const struct stack_depot_trie_node *last_node = parent; unsigned int entry = 0; - unsigned int used = 0; - unsigned int i; + + lockdep_assert_held(&pool_lock); + lockdep_assert_held(&stack_depot_trie_writer_lock); while (entry < nr_entries) { struct stack_depot_frame_run run; struct stack_depot_trie_node *node; - u32 id; - node = nodes[used]; frame_run_init(&entries[entry], nr_entries - entry, &run); + node = trie_pool_alloc(trie_node_bytes(&run), pool_prealloc); + if (!node) + goto err_release; - id = entry + run.nr_entries == nr_entries ? leaf_id : 0; - trie_node_init(node, prev, id, &entries[entry], &run); - - prev = node; + trie_node_init(node, last_node, + entry + run.nr_entries == nr_entries ? stack_id : 0, + &entries[entry], &run); entry += run.nr_entries; - used++; + last_node = node; + if (!path_root) + path_root = node; + + if (path_children) + trie_children_insert(path_children, last_node, 0); + if (entry < nr_entries) { + path_children = trie_pool_alloc_children(1, pool_prealloc); + if (!path_children) + goto err_release; + RCU_INIT_POINTER(node->children, path_children); + } } - for (i = 0; i + 1 < used; i++) { - const struct stack_depot_trie_node *next = nodes[i + 1]; - struct stack_depot_trie_node *node = nodes[i]; - struct stack_depot_trie_children *children = path_children[i]; + *node_out = last_node; + return path_root; - trie_children_init(children, &next, 1); - RCU_INIT_POINTER(node->children, children); +err_release: + while (last_node != parent) { + const struct stack_depot_trie_children *node_children; + const struct stack_depot_trie_node *node = last_node; + + last_node = trie_load_parent(node); + node_children = trie_load_children(&node->children); + if (node_children) + trie_pool_release_children(node_children); + trie_pool_release(node, trie_node_bytes(&node->run)); } + return NULL; } static const struct stack_depot_trie_node * @@ -1846,13 +1782,13 @@ stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) if (!trie_children_find_position(children, entries[entry], &pos)) return NULL; - node = trie_children_load_node(children, pos); - matched = __stack_depot_trie_node_match(node, &entries[entry], remaining); + node = trie_children_load_child(children, pos); + matched = trie_node_match(node, &entries[entry], remaining); if (matched < node->run.nr_entries) return NULL; entry += matched; if (entry == nr_entries) - return node->leaf_id ? node : NULL; + return node->stack_id ? node : NULL; children = trie_load_children(&node->children); } @@ -1860,25 +1796,6 @@ stack_depot_trie_lookup(const unsigned long *entries, unsigned int nr_entries) return NULL; } -static unsigned int trie_size_append_chain(const unsigned long *entries, - unsigned int nr_entries, - size_t *node_sizes) -{ - unsigned int entry = 0; - unsigned int nr_nodes = 0; - - while (entry < nr_entries) { - struct stack_depot_frame_run run; - - frame_run_init(&entries[entry], nr_entries - entry, &run); - node_sizes[nr_nodes] = trie_node_bytes(&run); - entry += run.nr_entries; - nr_nodes++; - } - - return nr_nodes; -} - static u32 trie_insert_path(const struct stack_depot_trie_children __rcu **slot, struct stack_depot_trie_node *parent, @@ -1887,97 +1804,75 @@ trie_insert_path(const struct stack_depot_trie_children __rcu **slot, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { - struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; - struct stack_depot_trie_children *new_children; + struct stack_depot_trie_children *new_children = NULL; + const struct stack_depot_trie_node *path_root; + const struct stack_depot_trie_node *node; unsigned int capacity = 1; - unsigned int nr_nodes_allocated = 0; - unsigned int nr_path_children_allocated = 0; - unsigned int nr_nodes; - unsigned int i; - unsigned long flags; - u32 new_leaf_id; + u32 new_stack_id; bool tail_append = false; + /* + * Reuse spare capacity only for a sorted tail append. Other insertions + * replace the children container without modifying visible pointers. + */ if (children) { capacity = roundup_pow_of_two(children->nr_children + 1); tail_append = pos == children->nr_children && children->nr_children < children->capacity; } - if (!tail_append && - trie_children_alloc_size(capacity) > STACK_DEPOT_TRIE_POOL_USABLE_SIZE) + if (!tail_append && trie_children_alloc_size(capacity) > + STACK_DEPOT_TRIE_POOL_USABLE_SIZE) return 0; - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) + new_stack_id = trie_side_table_prepare_stack_slot(side_prealloc); + if (!new_stack_id) return 0; - nr_nodes = trie_size_append_chain(entries, nr_entries, alloc->node_sizes); - raw_spin_lock_irqsave(&pool_lock, flags); + raw_spin_lock(&pool_lock); printk_deferred_enter(); trie_drain_pending_children(); - for (i = 0; i < nr_nodes; i++) { - alloc->nodes[i] = trie_pool_alloc(alloc->node_sizes[i], - pool_prealloc); - if (!alloc->nodes[i]) - goto err_release; - nr_nodes_allocated++; - } - for (i = 0; i + 1 < nr_nodes; i++) { - alloc->path_children[i] = - trie_pool_alloc_children(1, pool_prealloc); - if (!alloc->path_children[i]) - goto err_release; - nr_path_children_allocated++; - } + /* Reserve replacement topology before the path, the final fallible step. */ if (!tail_append) { new_children = trie_pool_alloc_children(capacity, pool_prealloc); if (!new_children) goto err_release; } + path_root = trie_path_alloc(parent, new_stack_id, entries, nr_entries, + pool_prealloc, &node); + if (!path_root) + goto err_release; - printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); - - trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, - alloc->nodes, alloc->path_children); - trie_side_table_publish_leaf(new_leaf_id, alloc->nodes[nr_nodes - 1]); - - if (!children) { - new_children->nr_children = 1; - RCU_INIT_POINTER(new_children->nodes[0], alloc->nodes[0]); - rcu_assign_pointer(*slot, new_children); - return new_leaf_id; - } - + /* Commit the stack ID before making the path reachable from the trie. */ + trie_side_table_publish(node); if (tail_append) { struct stack_depot_trie_children *tail_children = (struct stack_depot_trie_children *)children; /* - * Publish the node before the count read by READ_ONCE(). Readers - * either miss on the old count or on a transient NULL; the writer - * recheck prevents a permanent duplicate. + * Publish the node before the visible count. Readers may transiently + * see NULL and miss; the writer-lock recheck prevents duplicates. */ - rcu_assign_pointer(tail_children->nodes[pos], alloc->nodes[0]); + rcu_assign_pointer(tail_children->nodes[pos], path_root); WRITE_ONCE(tail_children->nr_children, pos + 1); } else { - trie_children_insert_at(children, pos, alloc->nodes[0], new_children); + if (children) + trie_children_init(children, new_children); + trie_children_insert(new_children, path_root, pos); rcu_assign_pointer(*slot, new_children); - raw_spin_lock_irqsave(&pool_lock, flags); - trie_retire_children(children); - raw_spin_unlock_irqrestore(&pool_lock, flags); + if (children) + trie_retire_children(children); } - return new_leaf_id; + printk_deferred_exit(); + raw_spin_unlock(&pool_lock); + return new_stack_id; err_release: - for (i = 0; i < nr_nodes_allocated; i++) - trie_pool_release(alloc->nodes[i], alloc->node_sizes[i]); - for (i = 0; i < nr_path_children_allocated; i++) - trie_pool_release_children(alloc->path_children[i]); + if (new_children) + trie_pool_release_children(new_children); printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); + raw_spin_unlock(&pool_lock); return 0; } @@ -1990,121 +1885,114 @@ trie_split_child(const struct stack_depot_trie_children __rcu **slot, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { - struct stack_depot_trie_insert_alloc *alloc = stack_depot_trie_alloc; struct stack_depot_trie_children *prefix_children = NULL; - struct stack_depot_trie_children *new_children; - const struct stack_depot_trie_node *new_tail; - const struct stack_depot_trie_node *split_leaf; - const struct stack_depot_trie_node *split_nodes[2]; + struct stack_depot_trie_children *new_children = NULL; + const struct stack_depot_trie_node *new_node; + const struct stack_depot_trie_node *suffix_roots[2]; struct stack_depot_frame_run run; - struct stack_depot_trie_node *split_prefix; - struct stack_depot_trie_node *old_tail; - unsigned long old_tail_frame; - unsigned int nr_nodes_allocated = 0; - unsigned int nr_path_children_allocated = 0; - unsigned int nr_tail_nodes = 0; - unsigned int nr_split_nodes; - unsigned int old_tail_len; + struct stack_depot_trie_node *split_prefix = NULL; + struct stack_depot_trie_node *old_suffix = NULL; + unsigned int nr_suffix_roots; + unsigned int old_suffix_len; unsigned int i; - unsigned long flags; - u32 new_leaf_id; - bool has_new_tail; + size_t split_prefix_size; + size_t old_suffix_size; + u32 new_stack_id; + bool has_new_suffix; - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) + new_stack_id = trie_side_table_prepare_stack_slot(side_prealloc); + if (!new_stack_id) return 0; + /* Rebuild the child's run as newly allocated prefix and old suffix nodes. */ run = child->run; run.nr_entries = matched; - alloc->node_sizes[0] = trie_node_bytes(&run); - old_tail_len = child->run.nr_entries - matched; - run.nr_entries = old_tail_len; - alloc->node_sizes[1] = trie_node_bytes(&run); - has_new_tail = matched < nr_entries; - nr_split_nodes = has_new_tail ? 2 : 1; - if (has_new_tail) - nr_tail_nodes = trie_size_append_chain(&entries[matched], - nr_entries - matched, - &alloc->node_sizes[2]); - - raw_spin_lock_irqsave(&pool_lock, flags); + split_prefix_size = trie_node_bytes(&run); + old_suffix_len = child->run.nr_entries - matched; + run.nr_entries = old_suffix_len; + old_suffix_size = trie_node_bytes(&run); + has_new_suffix = matched < nr_entries; + nr_suffix_roots = has_new_suffix ? 2 : 1; + + raw_spin_lock(&pool_lock); printk_deferred_enter(); trie_drain_pending_children(); - for (i = 0; i < 2 + nr_tail_nodes; i++) { - alloc->nodes[i] = trie_pool_alloc(alloc->node_sizes[i], - pool_prealloc); - if (!alloc->nodes[i]) - goto err_release; - nr_nodes_allocated++; - } - for (i = 0; i + 1 < nr_tail_nodes; i++) { - alloc->path_children[i] = - trie_pool_alloc_children(1, pool_prealloc); - if (!alloc->path_children[i]) - goto err_release; - nr_path_children_allocated++; - } - prefix_children = trie_pool_alloc_children(nr_split_nodes, pool_prealloc); - if (!prefix_children) + /* Reserve fixed split topology before the optional new suffix path. */ + split_prefix = trie_pool_alloc(split_prefix_size, pool_prealloc); + if (!split_prefix) + goto err_release; + old_suffix = trie_pool_alloc(old_suffix_size, pool_prealloc); + if (!old_suffix) goto err_release; new_children = trie_pool_alloc_children(children->capacity, pool_prealloc); if (!new_children) goto err_release; + prefix_children = trie_pool_alloc_children(nr_suffix_roots, pool_prealloc); + if (!prefix_children) + goto err_release; - printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); + if (has_new_suffix) { + const struct stack_depot_trie_node *new_suffix; + unsigned long old_suffix_frame; - split_prefix = alloc->nodes[0]; - old_tail = alloc->nodes[1]; - - if (has_new_tail) { - trie_build_append_chain(split_prefix, new_leaf_id, &entries[matched], - nr_entries - matched, &alloc->nodes[2], - alloc->path_children); - new_tail = alloc->nodes[2]; - split_leaf = alloc->nodes[nr_tail_nodes + 1]; - stack_depot_trie_node_frame(child, matched, &old_tail_frame); - if (old_tail_frame < entries[matched]) { - split_nodes[0] = old_tail; - split_nodes[1] = new_tail; + new_suffix = trie_path_alloc(split_prefix, new_stack_id, + &entries[matched], nr_entries - matched, + pool_prealloc, &new_node); + if (!new_suffix) + goto err_release; + stack_depot_trie_node_frame(child, matched, &old_suffix_frame); + /* Children remain sorted by the first frame of each suffix. */ + if (old_suffix_frame < entries[matched]) { + suffix_roots[0] = old_suffix; + suffix_roots[1] = new_suffix; } else { - split_nodes[0] = new_tail; - split_nodes[1] = old_tail; + suffix_roots[0] = new_suffix; + suffix_roots[1] = old_suffix; } } else { - split_leaf = split_prefix; - split_nodes[0] = old_tail; + new_node = split_prefix; + suffix_roots[0] = old_suffix; } + printk_deferred_exit(); + raw_spin_unlock(&pool_lock); + + /* Rebuild the old path as prefix -> old suffix and attach suffix roots. */ trie_node_init_slice(split_prefix, trie_load_parent(child), - has_new_tail ? 0 : new_leaf_id, child, 0, matched); - trie_node_init_slice(old_tail, split_prefix, child->leaf_id, child, - matched, old_tail_len); - trie_children_init(prefix_children, split_nodes, nr_split_nodes); - RCU_INIT_POINTER(old_tail->children, + has_new_suffix ? 0 : new_stack_id, child, 0, matched); + trie_node_init_slice(old_suffix, split_prefix, child->stack_id, child, + matched, old_suffix_len); + for (i = 0; i < nr_suffix_roots; i++) + trie_children_insert(prefix_children, suffix_roots[i], i); + RCU_INIT_POINTER(old_suffix->children, trie_load_children(&child->children)); RCU_INIT_POINTER(split_prefix->children, prefix_children); - if (child->leaf_id) - trie_side_table_publish_leaf(child->leaf_id, old_tail); - trie_side_table_publish_leaf(new_leaf_id, split_leaf); - trie_children_replace_at(children, split_prefix, new_children, pos); - trie_reparent_children(old_tail); + /* Publish IDs, reparent descendants, then replace and retire topology. */ + if (child->stack_id) + trie_side_table_publish(old_suffix); + trie_side_table_publish(new_node); + /* Old and replacement chains contain identical frames during transition. */ + trie_children_init(children, new_children); + RCU_INIT_POINTER(new_children->nodes[pos], split_prefix); + trie_reparent_children(old_suffix); rcu_assign_pointer(*slot, new_children); trie_retire_children_with_node(children, child); - return new_leaf_id; + return new_stack_id; err_release: - for (i = 0; i < nr_nodes_allocated; i++) - trie_pool_release(alloc->nodes[i], alloc->node_sizes[i]); - for (i = 0; i < nr_path_children_allocated; i++) - trie_pool_release_children(alloc->path_children[i]); + if (split_prefix) + trie_pool_release(split_prefix, split_prefix_size); + if (old_suffix) + trie_pool_release(old_suffix, old_suffix_size); if (prefix_children) trie_pool_release_children(prefix_children); + if (new_children) + trie_pool_release_children(new_children); printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); + raw_spin_unlock(&pool_lock); return 0; } @@ -2117,16 +2005,16 @@ trie_promote_child(const struct stack_depot_trie_children __rcu **slot, { struct stack_depot_trie_children *new_children; struct stack_depot_trie_node *promoted_node; - unsigned long flags; size_t node_size; - u32 new_leaf_id; + u32 new_stack_id; - new_leaf_id = trie_side_table_prepare_leaf_slot(side_prealloc); - if (!new_leaf_id) + new_stack_id = trie_side_table_prepare_stack_slot(side_prealloc); + if (!new_stack_id) return 0; node_size = trie_node_bytes(&child->run); - raw_spin_lock_irqsave(&pool_lock, flags); + /* Reserve a clone and replacement children container before publication. */ + raw_spin_lock(&pool_lock); printk_deferred_enter(); trie_drain_pending_children(); promoted_node = trie_pool_alloc(node_size, pool_prealloc); @@ -2136,23 +2024,25 @@ trie_promote_child(const struct stack_depot_trie_children __rcu **slot, if (!new_children) goto out_release_node; printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); + raw_spin_unlock(&pool_lock); + /* Add the stack ID through a clone, then reparent before retirement. */ memcpy(promoted_node, child, node_size); - promoted_node->leaf_id = new_leaf_id; - trie_side_table_publish_leaf(new_leaf_id, promoted_node); - trie_children_replace_at(children, promoted_node, new_children, pos); + promoted_node->stack_id = new_stack_id; + trie_side_table_publish(promoted_node); + trie_children_init(children, new_children); + RCU_INIT_POINTER(new_children->nodes[pos], promoted_node); trie_reparent_children(promoted_node); rcu_assign_pointer(*slot, new_children); trie_retire_children_with_node(children, child); - return new_leaf_id; + return new_stack_id; out_release_node: trie_pool_release(promoted_node, node_size); out_unlock: printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); + raw_spin_unlock(&pool_lock); return 0; } @@ -2168,97 +2058,94 @@ stack_depot_trie_insert(const unsigned long *entries, struct stack_depot_trie_node *parent = NULL; unsigned int matched; unsigned int pos; - u32 leaf_id; + u32 stack_id; lockdep_assert_held(&stack_depot_trie_writer_lock); for (;;) { pos = 0; children = trie_load_children(slot); + /* No matching child: attach the remaining path. */ if (!children || !trie_children_find_position(children, entries[0], &pos)) { - leaf_id = trie_insert_path(slot, parent, children, pos, entries, - nr_entries, pool_prealloc, - side_prealloc); + stack_id = trie_insert_path(slot, parent, children, pos, + entries, nr_entries, pool_prealloc, + side_prealloc); break; } - child = trie_children_load_node(children, pos); - matched = __stack_depot_trie_node_match(child, entries, nr_entries); + child = trie_children_load_child(children, pos); + matched = trie_node_match(child, entries, nr_entries); + /* A partial child match requires a prefix/suffix split. */ if (matched < child->run.nr_entries) { - leaf_id = trie_split_child(slot, children, child, pos, - matched, entries, - nr_entries, - pool_prealloc, - side_prealloc); + stack_id = trie_split_child(slot, children, child, pos, + matched, entries, nr_entries, + pool_prealloc, side_prealloc); break; } + /* The input ends here: reuse a stack node or promote an internal one. */ if (matched == nr_entries) { - if (child->leaf_id) - return child->leaf_id; - leaf_id = trie_promote_child(slot, children, child, pos, - pool_prealloc, - side_prealloc); + if (child->stack_id) + return child->stack_id; + stack_id = trie_promote_child(slot, children, child, pos, + pool_prealloc, side_prealloc); break; } + /* The child matched completely; continue with the remaining frames. */ parent = (struct stack_depot_trie_node *)child; slot = &parent->children; entries += matched; nr_entries -= matched; } - if (leaf_id) - trie_side_table_last_leaf_id = leaf_id; - return leaf_id; + if (stack_id) + trie_side_table_last_stack_id = stack_id; + return stack_id; } -static unsigned int -__stack_depot_trie_fetch_into(const struct stack_depot_trie_node *leaf, - unsigned long *entries, - unsigned int max_entries) +static unsigned int trie_fetch_into(const struct stack_depot_trie_node *node, + unsigned long *entries, + unsigned int max_entries) { - const struct stack_depot_trie_node *node; + const struct stack_depot_trie_node *cur; unsigned int total; unsigned int pos; unsigned int i; total = 0; - for (node = leaf; node; node = trie_load_parent(node)) - total += node->run.nr_entries; - if (WARN_ON_ONCE(!total)) - return 0; + for (cur = node; cur; cur = trie_load_parent(cur)) + total += cur->run.nr_entries; if (max_entries < total) return 0; pos = total; - for (node = leaf; node; node = trie_load_parent(node)) { - pos -= node->run.nr_entries; - for (i = 0; i < node->run.nr_entries; i++) - stack_depot_trie_node_frame(node, i, &entries[pos + i]); + for (cur = node; cur; cur = trie_load_parent(cur)) { + pos -= cur->run.nr_entries; + for (i = 0; i < cur->run.nr_entries; i++) + stack_depot_trie_node_frame(cur, i, &entries[pos + i]); } return total; } -static unsigned int -__stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, - unsigned long *entries, - unsigned int max_entries) +static unsigned int trie_fetch_handle_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries) { - const struct stack_depot_trie_node *leaf; - u32 leaf_id; + const struct stack_depot_trie_node *node; + u32 stack_id; unsigned int nr_entries; - leaf_id = __stack_depot_trie_leaf_id(handle); + stack_id = trie_stack_id(handle); rcu_read_lock_sched_notrace(); - leaf = __stack_depot_trie_side_table_lookup(leaf_id); - if (WARN_ONCE(!leaf, "corrupt trie handle %08x\n", handle)) { + node = trie_side_table_lookup(stack_id); + if (WARN_ONCE(!node, "corrupt trie handle %08x\n", handle)) { rcu_read_unlock_sched_notrace(); return 0; } - nr_entries = __stack_depot_trie_fetch_into(leaf, entries, max_entries); + nr_entries = trie_fetch_into(node, entries, max_entries); rcu_read_unlock_sched_notrace(); if (nr_entries) kmsan_unpoison_memory(entries, nr_entries * sizeof(*entries)); @@ -2309,8 +2196,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, return 0; WARN_ON_ONCE(!entries || !max_entries); if (stack_depot_handle_is_trie(handle)) - return __stack_depot_trie_fetch_handle_into(handle, entries, - max_entries); + return trie_fetch_handle_into(handle, entries, max_entries); stack = depot_fetch_stack(handle); if (!stack) @@ -2359,9 +2245,8 @@ void stack_depot_print(depot_stack_handle_t stack) if (stack_depot_handle_is_trie(stack)) { unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; - nr_entries = __stack_depot_trie_fetch_handle_into(stack, - trie_entries, - ARRAY_SIZE(trie_entries)); + nr_entries = trie_fetch_handle_into(stack, trie_entries, + ARRAY_SIZE(trie_entries)); stack_trace_print(trie_entries, nr_entries, 0); return; } @@ -2381,9 +2266,8 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, if (stack_depot_handle_is_trie(handle)) { unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; - nr_entries = __stack_depot_trie_fetch_handle_into(handle, - trie_entries, - ARRAY_SIZE(trie_entries)); + nr_entries = trie_fetch_handle_into(handle, trie_entries, + ARRAY_SIZE(trie_entries)); return stack_trace_snprint(buf, size, trie_entries, nr_entries, spaces); } From 6f7a250b686dbdc8f7bb4ade84bfdebc9f394c20 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 30 Jul 2026 16:50:57 +0100 Subject: [PATCH 37/38] KRN-1117: Cover maximum stackdepot trie paths Place a maximum-depth roundtrip first and alternate compressed and raw frames on arm64 and x86_64. Require a trie handle, exact fetch, and stable deduplication; with 256 frames this covers one node per frame and the two-pool insertion and retry path. Order the remaining functions and cases by public save, print, countable, fetch, topology, storage, fallback, and architecture codec behavior so definitions match execution order. Signed-off-by: Caleb Kan --- lib/tests/stackdepot_kunit.c | 167 ++++++++++++++++++++++------------- 1 file changed, 108 insertions(+), 59 deletions(-) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index e74a04a2ac171..be14cae98fcfb 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -24,69 +24,52 @@ static inline unsigned long stackdepot_arm64_frame(long offset) } #endif -static void stackdepot_fetch_into_roundtrip(struct kunit *test) +static void stackdepot_trie_max_path_roundtrip(struct kunit *test) { - unsigned long entries[] = { - 0x101000UL, - 0x102000UL, - 0x103000UL, - }; - unsigned long exact[ARRAY_SIZE(entries)] = {}; - unsigned long fetched[ARRAY_SIZE(entries) + 1] = { - [ARRAY_SIZE(entries)] = 0xa5a5a5a5UL, - }; - unsigned long expected_tail = fetched[ARRAY_SIZE(entries)]; - depot_stack_handle_t handle; - unsigned int nr_entries; - - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - - handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - - nr_entries = stack_depot_fetch_into(handle, exact, ARRAY_SIZE(exact)); - KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, exact, entries, sizeof(entries)); - - nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); - KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); - KUNIT_EXPECT_EQ(test, fetched[ARRAY_SIZE(entries)], expected_tail); -} - -static void stackdepot_fetch_into_rejects_missing_or_short_stack(struct kunit *test) -{ - unsigned long entries[] = { - 0x111000UL, - 0x112000UL, - 0x113000UL, - }; - unsigned long fetched[ARRAY_SIZE(entries)] = { - 0xa1a1a1a1UL, - 0xb2b2b2b2UL, - 0xc3c3c3c3UL, - }; - unsigned long expected[ARRAY_SIZE(fetched)]; + union handle_parts parts; + unsigned long *entries; + unsigned long *fetched; depot_stack_handle_t handle; - unsigned int nr_entries; + size_t size = CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(*entries); + u32 pool_index_plus_1; + unsigned int i; + if (expected_trie_pool_limit < 0) + kunit_skip(test, "trie pool limit was not provided"); KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + entries = kunit_kcalloc(test, CONFIG_STACKDEPOT_MAX_FRAMES, + sizeof(*entries), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, entries); + fetched = kunit_kcalloc(test, CONFIG_STACKDEPOT_MAX_FRAMES, + sizeof(*fetched), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fetched); + for (i = 0; i < CONFIG_STACKDEPOT_MAX_FRAMES; i++) { +#ifdef CONFIG_ARM64 + entries[i] = i & 1 ? 0x1000UL + i * 0x1000UL : + stackdepot_arm64_frame(i * 4); +#elif defined(CONFIG_X86_64) && !defined(CONFIG_UML) + entries[i] = i & 1 ? 0xffff888000000000UL + i * 0x1000UL : + 0xffffffff10000000UL + i * 0x10UL; +#else + entries[i] = 0x1000UL + i * 0x1000UL; +#endif + } - handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + handle = stack_depot_save(entries, CONFIG_STACKDEPOT_MAX_FRAMES, + GFP_KERNEL); KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - memcpy(expected, fetched, sizeof(expected)); - - nr_entries = stack_depot_fetch_into(0, fetched, ARRAY_SIZE(fetched)); - KUNIT_EXPECT_EQ(test, nr_entries, 0U); - KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); - - nr_entries = stack_depot_fetch_into(0, NULL, 0); - KUNIT_EXPECT_EQ(test, nr_entries, 0U); - - nr_entries = stack_depot_fetch_into(handle, fetched, - ARRAY_SIZE(fetched) - 1); - KUNIT_EXPECT_EQ(test, nr_entries, 0U); - KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + parts.handle = handle; + pool_index_plus_1 = parts.pool_index_plus_1; + KUNIT_EXPECT_GT(test, pool_index_plus_1, (u32)expected_trie_pool_limit); + KUNIT_EXPECT_EQ(test, + stack_depot_fetch_into(handle, fetched, + CONFIG_STACKDEPOT_MAX_FRAMES), + (unsigned int)CONFIG_STACKDEPOT_MAX_FRAMES); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, size); + KUNIT_EXPECT_EQ(test, + stack_depot_save(entries, CONFIG_STACKDEPOT_MAX_FRAMES, + GFP_KERNEL), + handle); } static void stackdepot_save_flags_public(struct kunit *test) @@ -238,6 +221,71 @@ static void stackdepot_countable_public(struct kunit *test) stack_depot_put(get_handle); } +static void stackdepot_fetch_into_roundtrip(struct kunit *test) +{ + unsigned long entries[] = { + 0x101000UL, + 0x102000UL, + 0x103000UL, + }; + unsigned long exact[ARRAY_SIZE(entries)] = {}; + unsigned long fetched[ARRAY_SIZE(entries) + 1] = { + [ARRAY_SIZE(entries)] = 0xa5a5a5a5UL, + }; + unsigned long expected_tail = fetched[ARRAY_SIZE(entries)]; + depot_stack_handle_t handle; + unsigned int nr_entries; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + nr_entries = stack_depot_fetch_into(handle, exact, ARRAY_SIZE(exact)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, exact, entries, sizeof(entries)); + + nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); + KUNIT_EXPECT_EQ(test, fetched[ARRAY_SIZE(entries)], expected_tail); +} + +static void stackdepot_fetch_into_rejects_missing_or_short_stack(struct kunit *test) +{ + unsigned long entries[] = { + 0x111000UL, + 0x112000UL, + 0x113000UL, + }; + unsigned long fetched[ARRAY_SIZE(entries)] = { + 0xa1a1a1a1UL, + 0xb2b2b2b2UL, + 0xc3c3c3c3UL, + }; + unsigned long expected[ARRAY_SIZE(fetched)]; + depot_stack_handle_t handle; + unsigned int nr_entries; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + memcpy(expected, fetched, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(0, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(0, NULL, 0); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + + nr_entries = stack_depot_fetch_into(handle, fetched, + ARRAY_SIZE(fetched) - 1); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); +} + static void stackdepot_trie_topology_roundtrip(struct kunit *test) { union handle_parts parts; @@ -395,11 +443,12 @@ static void stackdepot_frame_arm64(struct kunit *test) #endif /* CONFIG_ARM64 */ static struct kunit_case stackdepot_test_cases[] = { - KUNIT_CASE(stackdepot_fetch_into_roundtrip), - KUNIT_CASE(stackdepot_fetch_into_rejects_missing_or_short_stack), + KUNIT_CASE(stackdepot_trie_max_path_roundtrip), KUNIT_CASE(stackdepot_save_flags_public), KUNIT_CASE(stackdepot_snprint_public), KUNIT_CASE(stackdepot_countable_public), + KUNIT_CASE(stackdepot_fetch_into_roundtrip), + KUNIT_CASE(stackdepot_fetch_into_rejects_missing_or_short_stack), KUNIT_CASE(stackdepot_trie_topology_roundtrip), KUNIT_CASE(stackdepot_frame_storage_roundtrip), KUNIT_CASE(stackdepot_frame_raw_fallback), From 8b068e379583430a763d5c76d6d495026bdb6289 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 31 Jul 2026 11:42:03 +0100 Subject: [PATCH 38/38] KRN-1117: Tighten stackdepot pool and trie invariants Mark the spare pool pointer as poisoned when registering the final allowed pool so later saves do not allocate and retain an unusable order-2 page after capacity is reached. Document why bitmap allocation scans every physical slot and clarify that side-table mappings precede topology publication for new and remapped nodes. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index e2179b631af1c..f33e6f87cbdcf 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -536,6 +536,7 @@ static unsigned int trie_pool_reserve_slots(struct stack_depot_trie_pool *pool, if (pool->free_slots < nr_slots) return STACK_DEPOT_TRIE_POOL_SLOTS; + /* A free run can cross any previous allocation position. */ for (slot = STACK_DEPOT_TRIE_POOL_FIRST_SLOT; slot < STACK_DEPOT_TRIE_POOL_SLOTS; slot++) { if (pool->used[slot / BITS_PER_LONG] & @@ -705,9 +706,10 @@ trie_find_handle(const unsigned long *entries, unsigned int nr_entries) /* * Publish only after the node and its path are fully initialized and all * fallible allocation is complete. Publication commits the path, so it cannot - * then be rolled back. The side-table mapping must precede the trie slot that - * makes a new stack ID reachable from lookup. Published storage remains valid - * until RCU retirement; only descendant parent links may change meanwhile. + * then be rolled back. Side-table mappings must precede trie topology + * publication that makes new or remapped nodes reachable from lookup. + * Published storage remains valid until RCU retirement; only descendant parent + * links may change meanwhile. */ static void trie_side_table_publish(const struct stack_depot_trie_node *node) { @@ -964,7 +966,7 @@ static bool depot_init_pool(void **prealloc) * NULL; do not reset to NULL if we have reached the maximum number of * pools. */ - if (pools_num < stack_max_pools) + if (pools_num + 1 < stack_max_pools) WRITE_ONCE(new_pool, NULL); else WRITE_ONCE(new_pool, STACK_DEPOT_POISON);