From b96d171e13890dadc1ddb71d916a63952399fd5d Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 26 May 2026 16:54:04 +0100 Subject: [PATCH 001/129] KRN-1117: Add caller-owned stack trace fetch helper Add stack_depot_fetch_into() so callers can copy a saved stack trace into caller-owned storage instead of receiving a pointer to stackdepot-owned memory. This is a preparatory API for trie-backed stack storage, where a saved stack may be spread across multiple trie nodes and must be materialized before use. Add KUnit coverage for successful copy-out and invalid or too-small buffer handling. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 13 ++++++ lib/Kconfig.debug | 14 +++++++ lib/stackdepot.c | 19 +++++++++ lib/tests/Makefile | 1 + lib/tests/stackdepot_kunit.c | 81 ++++++++++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+) create mode 100644 lib/tests/stackdepot_kunit.c diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 2cc21ffcdaf9e..991dd160a6830 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -199,6 +199,19 @@ struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle) unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries); +/** + * stack_depot_fetch_into - Fetch a stack trace into caller-owned storage + * + * @handle: Stack depot handle returned from stack_depot_save() + * @entries: Caller-owned buffer to copy the stack trace into + * @max_entries: Number of frames that fit in @entries + * + * Return: Number of frames copied, 0 on invalid handle or insufficient space + */ +unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries); + /** * stack_depot_print - Print a stack trace from stack depot * diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 21cd68084e468..8fd4f350cae7e 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2693,6 +2693,20 @@ config HASH_KUNIT_TEST This is intended to help people writing architecture-specific optimized versions. If unsure, say N. +config STACKDEPOT_KUNIT_TEST + tristate "KUnit test for stack depot" if !KUNIT_ALL_TESTS + depends on KUNIT && STACKDEPOT + default KUNIT_ALL_TESTS + help + Enable this option to test stack depot API behavior at boot. + + KUnit tests run during boot and output the results to the debug log + in TAP format (https://testanything.org/). Only useful for kernel + developers running the KUnit test harness, and not intended for + inclusion into a production build. + + If unsure, say N. + config RESOURCE_KUNIT_TEST tristate "KUnit test for resource API" if !KUNIT_ALL_TESTS depends on KUNIT diff --git a/lib/stackdepot.c b/lib/stackdepot.c index de0b0025af2b9..90b1e3ae60b0b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -777,6 +777,25 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, } EXPORT_SYMBOL_GPL(stack_depot_fetch); +unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries) +{ + unsigned long *stack_entries; + unsigned int nr_entries; + + if (!entries) + return 0; + + nr_entries = stack_depot_fetch(handle, &stack_entries); + if (!nr_entries || nr_entries > max_entries) + return 0; + + memcpy(entries, stack_entries, nr_entries * sizeof(*entries)); + return nr_entries; +} +EXPORT_SYMBOL_GPL(stack_depot_fetch_into); + void stack_depot_put(depot_stack_handle_t handle) { struct stack_record *stack; diff --git a/lib/tests/Makefile b/lib/tests/Makefile index f7460831cfdd4..1d0954575ad5d 100644 --- a/lib/tests/Makefile +++ b/lib/tests/Makefile @@ -40,6 +40,7 @@ obj-$(CONFIG_SCANF_KUNIT_TEST) += scanf_kunit.o obj-$(CONFIG_SEQ_BUF_KUNIT_TEST) += seq_buf_kunit.o obj-$(CONFIG_SIPHASH_KUNIT_TEST) += siphash_kunit.o obj-$(CONFIG_SLUB_KUNIT_TEST) += slub_kunit.o +obj-$(CONFIG_STACKDEPOT_KUNIT_TEST) += stackdepot_kunit.o obj-$(CONFIG_TEST_SORT) += test_sort.o CFLAGS_stackinit_kunit.o += $(call cc-disable-warning, switch-unreachable) obj-$(CONFIG_STACKINIT_KUNIT_TEST) += stackinit_kunit.o diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c new file mode 100644 index 0000000000000..4244c26bc01ee --- /dev/null +++ b/lib/tests/stackdepot_kunit.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include + +static void stackdepot_fetch_into_roundtrip(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800010000UL, + 0x1234567800020000UL, + 0x1234567800030000UL, + }; + unsigned long fetched[ARRAY_SIZE(entries)] = {}; + depot_stack_handle_t handle; + unsigned int nr_entries; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + nr_entries = + stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); +} + +static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800110000UL, + 0x1234567800120000UL, + 0x1234567800130000UL, + }; + unsigned long fetched[ARRAY_SIZE(entries)] = { + 0xa1a1a1a1a1a1a1a1UL, + 0xb2b2b2b2b2b2b2b2UL, + 0xc3c3c3c3c3c3c3c3UL, + }; + unsigned long expected[ARRAY_SIZE(fetched)]; + depot_stack_handle_t handle; + unsigned int nr_entries; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + memcpy(expected, fetched, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(0, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, 0); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(handle, NULL, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, 0); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(handle, fetched, + ARRAY_SIZE(fetched) - 1); + KUNIT_EXPECT_EQ(test, nr_entries, 0); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); +} + +static struct kunit_case stackdepot_test_cases[] = { + KUNIT_CASE(stackdepot_fetch_into_roundtrip), + KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), + {} +}; + +static struct kunit_suite stackdepot_test_suite = { + .name = "stackdepot", + .test_cases = stackdepot_test_cases, +}; + +kunit_test_suite(stackdepot_test_suite); + +MODULE_DESCRIPTION("KUnit tests for stack depot"); +MODULE_LICENSE("GPL"); From 0ffe3447f042b2d5f7bf48093fda2c304759938f Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 27 May 2026 10:41:50 +0100 Subject: [PATCH 002/129] KRN-1117: Add stackdepot files to MAINTAINERS Cover the stackdepot public header and KUnit test under LIBRARY CODE so get_maintainer.pl reports the library maintainer for the new test file and checkpatch no longer warns that MAINTAINERS may need updating. Signed-off-by: Caleb Kan --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 554e881b05bea..2752f5ba5fcc0 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14281,7 +14281,9 @@ M: Andrew Morton L: linux-kernel@vger.kernel.org S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-nonmm-unstable +F: include/linux/stackdepot.h F: lib/* +F: lib/tests/stackdepot_kunit.c LICENSES and SPDX stuff M: Thomas Gleixner From 90214d6c9ad1313e12d4b8605bcde86281de7cb9 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 27 May 2026 12:24:18 +0100 Subject: [PATCH 003/129] KRN-1117: Clarify stack_depot_fetch_into contract Document the caller-owned copy and handle lifetime contract for stack_depot_fetch_into() so the helper is explicit about how trie-backed materialization will be consumed. Cover the exact-fit, oversized-buffer, and zero-sized-buffer cases in KUnit to lock the all-or-nothing copy semantics. Keep the stackdepot KUnit entry ordered with nearby tests and broaden MAINTAINERS coverage so future stackdepot tests are routed to the library maintainer. Signed-off-by: Caleb Kan --- MAINTAINERS | 2 +- include/linux/stackdepot.h | 13 ++++++++++++- lib/Kconfig.debug | 26 +++++++++++++------------- lib/stackdepot.c | 6 +++++- lib/tests/stackdepot_kunit.c | 19 ++++++++++++++++++- 5 files changed, 49 insertions(+), 17 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 2752f5ba5fcc0..2b367e6caffed 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14283,7 +14283,7 @@ S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-nonmm-unstable F: include/linux/stackdepot.h F: lib/* -F: lib/tests/stackdepot_kunit.c +F: lib/tests/stackdepot* LICENSES and SPDX stuff M: Thomas Gleixner diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 991dd160a6830..ea7768d34cb0a 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -206,7 +206,18 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * @entries: Caller-owned buffer to copy the stack trace into * @max_entries: Number of frames that fit in @entries * - * Return: Number of frames copied, 0 on invalid handle or insufficient space + * Copies the stack trace into caller-owned @entries if the whole trace fits. + * No partial copy is performed on failure. + * + * Callers must ensure @handle remains valid for the duration of this call. + * Handles saved with %STACK_DEPOT_FLAG_GET require a held reference; handles + * saved without %STACK_DEPOT_FLAG_GET are persistent, and callers must not call + * stack_depot_put() on them. + * Racing this helper with stack_depot_put() on the same handle is invalid. + * + * Return: Number of frames copied, 0 if @entries is NULL or @max_entries is 0, + * if the underlying fetch fails, or if @max_entries is less than the number of + * stored frames. */ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, unsigned long *entries, diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 8fd4f350cae7e..af021cbe83ddc 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2693,6 +2693,19 @@ config HASH_KUNIT_TEST This is intended to help people writing architecture-specific optimized versions. If unsure, say N. +config RESOURCE_KUNIT_TEST + tristate "KUnit test for resource API" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + select GET_FREE_REGION + help + This builds the resource API unit test. + Tests the logic of API provided by resource.c and ioport.h. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + config STACKDEPOT_KUNIT_TEST tristate "KUnit test for stack depot" if !KUNIT_ALL_TESTS depends on KUNIT && STACKDEPOT @@ -2707,19 +2720,6 @@ config STACKDEPOT_KUNIT_TEST If unsure, say N. -config RESOURCE_KUNIT_TEST - tristate "KUnit test for resource API" if !KUNIT_ALL_TESTS - depends on KUNIT - default KUNIT_ALL_TESTS - select GET_FREE_REGION - help - This builds the resource API unit test. - Tests the logic of API provided by resource.c and ioport.h. - For more information on KUnit and unit tests in general please refer - to the KUnit documentation in Documentation/dev-tools/kunit/. - - If unsure, say N. - config SYSCTL_KUNIT_TEST tristate "KUnit test for sysctl" if !KUNIT_ALL_TESTS depends on KUNIT diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 90b1e3ae60b0b..6528b45b15445 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -784,13 +784,17 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, unsigned long *stack_entries; unsigned int nr_entries; - if (!entries) + if (!entries || !max_entries) return 0; nr_entries = stack_depot_fetch(handle, &stack_entries); if (!nr_entries || nr_entries > max_entries) return 0; + /* + * stack_depot_fetch() returns stackdepot-owned storage; the caller must + * keep the handle valid while this helper copies from it. + */ memcpy(entries, stack_entries, nr_entries * sizeof(*entries)); return nr_entries; } diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 4244c26bc01ee..fe844e67cfb2a 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -13,7 +13,11 @@ static void stackdepot_fetch_into_roundtrip(struct kunit *test) 0x1234567800020000UL, 0x1234567800030000UL, }; - unsigned long fetched[ARRAY_SIZE(entries)] = {}; + unsigned long exact[ARRAY_SIZE(entries)] = {}; + unsigned long fetched[ARRAY_SIZE(entries) + 1] = { + [ARRAY_SIZE(entries)] = 0xa5a5a5a5a5a5a5a5UL, + }; + unsigned long expected_tail = fetched[ARRAY_SIZE(entries)]; depot_stack_handle_t handle; unsigned int nr_entries; @@ -22,10 +26,16 @@ static void stackdepot_fetch_into_roundtrip(struct kunit *test) handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + nr_entries = + stack_depot_fetch_into(handle, exact, ARRAY_SIZE(exact)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, exact, entries, sizeof(entries)); + nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); + KUNIT_EXPECT_EQ(test, fetched[ARRAY_SIZE(entries)], expected_tail); } static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) @@ -54,10 +64,17 @@ static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) KUNIT_EXPECT_EQ(test, nr_entries, 0); KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + nr_entries = stack_depot_fetch_into(0, NULL, 0); + KUNIT_EXPECT_EQ(test, nr_entries, 0); + nr_entries = stack_depot_fetch_into(handle, NULL, ARRAY_SIZE(fetched)); KUNIT_EXPECT_EQ(test, nr_entries, 0); KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + nr_entries = stack_depot_fetch_into(handle, fetched, 0); + KUNIT_EXPECT_EQ(test, nr_entries, 0); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched) - 1); KUNIT_EXPECT_EQ(test, nr_entries, 0); From 0becc55daf91471c44dfe00cd882bdc07aa98312 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 27 May 2026 14:22:35 +0100 Subject: [PATCH 004/129] KRN-1117: Stop page_owner reading stack entries directly Use stack_depot_fetch_into() when rendering page_owner_stacks so the diagnostic path no longer depends on stackdepot's flat entries[] storage. Store the materialized stack in seq_file private data to avoid a large stack buffer and keep output formatting unchanged. Clarify the fetch_into() oversized-buffer documentation while adding the first page_owner caller that relies on caller-owned stack materialization. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 8 +++++--- mm/page_owner.c | 35 +++++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index ea7768d34cb0a..b3b83f4a4761a 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -208,6 +208,8 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * * Copies the stack trace into caller-owned @entries if the whole trace fits. * No partial copy is performed on failure. + * If @max_entries is larger than the stored stack trace, only the stored frames + * are copied and their count is returned. * * Callers must ensure @handle remains valid for the duration of this call. * Handles saved with %STACK_DEPOT_FLAG_GET require a held reference; handles @@ -215,9 +217,9 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * stack_depot_put() on them. * Racing this helper with stack_depot_put() on the same handle is invalid. * - * Return: Number of frames copied, 0 if @entries is NULL or @max_entries is 0, - * if the underlying fetch fails, or if @max_entries is less than the number of - * stored frames. + * Return: Number of frames copied, 0 if @entries is NULL, @max_entries is 0, + * @handle is 0 or invalid, stack depot is disabled, or @max_entries is less + * than the number of stored frames. */ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, unsigned long *entries, diff --git a/mm/page_owner.c b/mm/page_owner.c index bc26764142ba5..aa9638c67dece 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -40,6 +40,12 @@ struct stack { struct stack_record *stack_record; struct stack *next; }; + +struct page_owner_stack_seq { + struct stack *stack; + unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; +}; + static struct stack dummy_stack; static struct stack failure_stack; static struct stack *stack_list; @@ -858,6 +864,7 @@ static const struct file_operations proc_page_owner_operations = { static void *stack_start(struct seq_file *m, loff_t *ppos) { + struct page_owner_stack_seq *priv = m->private; struct stack *stack; if (*ppos == -1UL) @@ -870,21 +877,22 @@ static void *stack_start(struct seq_file *m, loff_t *ppos) * value of stack_list. */ stack = smp_load_acquire(&stack_list); - m->private = stack; } else { - stack = m->private; + stack = priv->stack; } + priv->stack = stack; return stack; } static void *stack_next(struct seq_file *m, void *v, loff_t *ppos) { + struct page_owner_stack_seq *priv = m->private; struct stack *stack = v; stack = stack->next; *ppos = stack ? *ppos + 1 : -1UL; - m->private = stack; + priv->stack = stack; return stack; } @@ -893,24 +901,30 @@ static unsigned long page_owner_pages_threshold; static int stack_print(struct seq_file *m, void *v) { - int i, nr_base_pages; + struct page_owner_stack_seq *priv = m->private; struct stack *stack = v; - unsigned long *entries; - unsigned long nr_entries; struct stack_record *stack_record = stack->stack_record; + unsigned int i, nr_entries; + int nr_base_pages; if (!stack->stack_record) return 0; - nr_entries = stack_record->size; - entries = stack_record->entries; nr_base_pages = refcount_read(&stack_record->count) - 1; if (nr_base_pages < 1 || nr_base_pages < page_owner_pages_threshold) return 0; + /* Keep show_stacks independent of stackdepot's internal storage layout. */ + nr_entries = stack_depot_fetch_into(stack_record->handle.handle, + priv->entries, + ARRAY_SIZE(priv->entries)); + /* Buffer matches the stored-depth cap; failure means an unresolved handle. */ + if (!nr_entries) + return 0; + for (i = 0; i < nr_entries; i++) - seq_printf(m, " %pS\n", (void *)entries[i]); + seq_printf(m, " %pS\n", (void *)priv->entries[i]); seq_printf(m, "nr_base_pages: %d\n\n", nr_base_pages); return 0; @@ -929,7 +943,8 @@ static const struct seq_operations page_owner_stack_op = { static int page_owner_stack_open(struct inode *inode, struct file *file) { - return seq_open_private(file, &page_owner_stack_op, 0); + return seq_open_private(file, &page_owner_stack_op, + sizeof(struct page_owner_stack_seq)); } static const struct file_operations page_owner_stack_operations = { From d328e216667e798ac1a54e7129c79c01df5ef91f Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 27 May 2026 16:42:27 +0100 Subject: [PATCH 005/129] KRN-1117: Store page_owner stack handles in stack list Store page_owner stack handles in the show_stacks list instead of stackdepot internal stack_record pointers. This keeps the debugfs listing path from depending on stackdepot's flat record storage while preserving the existing page count accounting for a later helper conversion. Materialize stack frames into seq_file private storage with stack_depot_fetch_into() when printing page_owner_stacks so the output format stays unchanged and no large stack-local buffer is needed. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 8 +++---- lib/tests/stackdepot_kunit.c | 1 + mm/page_owner.c | 44 ++++++++++++++++++++++-------------- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index b3b83f4a4761a..7d10054b35477 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -206,10 +206,10 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * @entries: Caller-owned buffer to copy the stack trace into * @max_entries: Number of frames that fit in @entries * - * Copies the stack trace into caller-owned @entries if the whole trace fits. - * No partial copy is performed on failure. - * If @max_entries is larger than the stored stack trace, only the stored frames - * are copied and their count is returned. + * Copies the stored frames into caller-owned @entries. If fewer frames are + * stored than @max_entries, only the stored frames are written and their count + * is returned. If more frames are stored than @max_entries, the copy is skipped + * entirely and 0 is returned. * * Callers must ensure @handle remains valid for the duration of this call. * Handles saved with %STACK_DEPOT_FLAG_GET require a held reference; handles diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index fe844e67cfb2a..4892deaaf5924 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -66,6 +66,7 @@ static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) nr_entries = stack_depot_fetch_into(0, NULL, 0); KUNIT_EXPECT_EQ(test, nr_entries, 0); + /* No buffer is supplied for this invalid-input combination. */ nr_entries = stack_depot_fetch_into(handle, NULL, ARRAY_SIZE(fetched)); KUNIT_EXPECT_EQ(test, nr_entries, 0); diff --git a/mm/page_owner.c b/mm/page_owner.c index aa9638c67dece..2626d8adc2abe 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -37,7 +37,7 @@ struct page_owner { }; struct stack { - struct stack_record *stack_record; + depot_stack_handle_t handle; struct stack *next; }; @@ -118,6 +118,8 @@ static noinline void register_early_stack(void) static __init void init_page_owner(void) { + struct stack_record *stack_record; + if (!page_owner_enabled) return; @@ -126,12 +128,14 @@ static __init void init_page_owner(void) register_early_stack(); init_early_allocated_pages(); /* Initialize dummy and failure stacks and link them to stack_list */ - dummy_stack.stack_record = __stack_depot_get_stack_record(dummy_handle); - failure_stack.stack_record = __stack_depot_get_stack_record(failure_handle); - if (dummy_stack.stack_record) - refcount_set(&dummy_stack.stack_record->count, 1); - if (failure_stack.stack_record) - refcount_set(&failure_stack.stack_record->count, 1); + dummy_stack.handle = dummy_handle; + failure_stack.handle = failure_handle; + stack_record = __stack_depot_get_stack_record(dummy_handle); + if (stack_record) + refcount_set(&stack_record->count, 1); + stack_record = __stack_depot_get_stack_record(failure_handle); + if (stack_record) + refcount_set(&stack_record->count, 1); dummy_stack.next = &failure_stack; stack_list = &dummy_stack; static_branch_enable(&page_owner_inited); @@ -168,12 +172,14 @@ static noinline depot_stack_handle_t save_stack(gfp_t flags) return handle; } -static void add_stack_record_to_list(struct stack_record *stack_record, - gfp_t gfp_mask) +static void add_stack_record_to_list(depot_stack_handle_t handle, gfp_t gfp_mask) { unsigned long flags; struct stack *stack; + if (!handle) + return; + if (!gfpflags_allow_spinning(gfp_mask)) return; @@ -185,7 +191,7 @@ static void add_stack_record_to_list(struct stack_record *stack_record, } unset_current_in_page_owner(); - stack->stack_record = stack_record; + stack->handle = handle; stack->next = NULL; spin_lock_irqsave(&stack_list_lock, flags); @@ -219,8 +225,8 @@ static void inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, int old = REFCOUNT_SATURATED; if (atomic_try_cmpxchg_relaxed(&stack_record->count.refs, &old, 1)) - /* Add the new stack_record to our list */ - add_stack_record_to_list(stack_record, gfp_mask); + /* Add the new stack to our list */ + add_stack_record_to_list(handle, gfp_mask); } refcount_add(nr_base_pages, &stack_record->count); } @@ -903,11 +909,16 @@ static int stack_print(struct seq_file *m, void *v) { struct page_owner_stack_seq *priv = m->private; struct stack *stack = v; - struct stack_record *stack_record = stack->stack_record; + depot_stack_handle_t handle = stack->handle; + struct stack_record *stack_record; unsigned int i, nr_entries; int nr_base_pages; - if (!stack->stack_record) + if (!handle) + return 0; + + stack_record = __stack_depot_get_stack_record(handle); + if (!stack_record) return 0; nr_base_pages = refcount_read(&stack_record->count) - 1; @@ -916,10 +927,9 @@ static int stack_print(struct seq_file *m, void *v) return 0; /* Keep show_stacks independent of stackdepot's internal storage layout. */ - nr_entries = stack_depot_fetch_into(stack_record->handle.handle, - priv->entries, + nr_entries = stack_depot_fetch_into(handle, priv->entries, ARRAY_SIZE(priv->entries)); - /* Buffer matches the stored-depth cap; failure means an unresolved handle. */ + /* Buffer matches the stored-depth cap; failure means no stack is available. */ if (!nr_entries) return 0; From 3a5759efb69d6befdb8a1aa8b9ee7171dec99800 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 28 May 2026 08:57:36 +0100 Subject: [PATCH 006/129] KRN-1117: Hide page_owner stack count internals Add internal stackdepot count helpers and use them from page_owner so page_owner no longer reads or mutates struct stack_record directly. This keeps page_owner behind handle-based stackdepot access before the persistent stack storage layout changes for trie-backed records. Keep the stackdepot KUnit test built-in-only because it exercises internal helpers, and add coverage for the saturated-to-counted transition and count boundary cases. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 63 +++++++++++++++++++++++++++-- lib/Kconfig.debug | 4 +- lib/stackdepot.c | 77 +++++++++++++++++++++++++++++++++++- lib/tests/stackdepot_kunit.c | 67 +++++++++++++++++++++++++++++++ mm/page_owner.c | 61 ++++++++-------------------- 5 files changed, 222 insertions(+), 50 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 7d10054b35477..c75c86f2cc087 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -50,6 +50,7 @@ union handle_parts { }; }; +/* Internal only. New users should not inspect stack records directly. */ struct stack_record { struct list_head hash_list; /* Links in the hash table */ u32 hash; /* Hash in hash table */ @@ -188,6 +189,62 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, */ struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle); +/** + * __stack_depot_get_count - Get a counted stack record count + * + * @handle: Stack depot handle + * @count: Pointer to store the count + * + * This function is only for internal purposes. + * + * Return: true on success, false if @handle is invalid, @count is NULL, or the + * stack record is not in counted mode. + */ +bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count); + +/** + * __stack_depot_set_count - Set a stack record count + * + * @handle: Stack depot handle + * @count: Count to set + * + * This function is only for internal purposes. + * @count must be greater than 0 and less than %INT_MAX. + */ +void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); + +/** + * __stack_depot_inc_count - Increment a stack record count + * + * @handle: Stack depot handle + * @count: Count to add + * + * This function is only for internal purposes. + * @count must be greater than 0 and less than %INT_MAX. + * + * Persistent stack records start with refcount set to %REFCOUNT_SATURATED. If + * this helper switches a saturated record to counted mode, it stores @count + 1. + * + * Return: true if this call switched the record from saturated to counted, + * false otherwise. + */ +bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count); + +/** + * __stack_depot_dec_count_and_test - Decrement a stack record count + * + * @handle: Stack depot handle + * @count: Count to subtract + * + * This function is only for internal purposes. + * @count must be greater than 0 and less than %INT_MAX. + * + * Return: true if the resulting count is 0, false if the resulting count is + * non-zero or @handle is invalid. + */ +bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, + unsigned int count); + /** * stack_depot_fetch - Fetch a stack trace from stack depot * @@ -212,9 +269,9 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * entirely and 0 is returned. * * Callers must ensure @handle remains valid for the duration of this call. - * Handles saved with %STACK_DEPOT_FLAG_GET require a held reference; handles - * saved without %STACK_DEPOT_FLAG_GET are persistent, and callers must not call - * stack_depot_put() on them. + * Persistent handles saved without %STACK_DEPOT_FLAG_GET require no extra + * reference; handles saved with %STACK_DEPOT_FLAG_GET require a held reference. + * Callers must not call stack_depot_put() on persistent handles. * Racing this helper with stack_depot_put() on the same handle is invalid. * * Return: Number of frames copied, 0 if @entries is NULL, @max_entries is 0, diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index af021cbe83ddc..9160324736572 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2707,8 +2707,8 @@ config RESOURCE_KUNIT_TEST If unsure, say N. config STACKDEPOT_KUNIT_TEST - tristate "KUnit test for stack depot" if !KUNIT_ALL_TESTS - depends on KUNIT && STACKDEPOT + bool "KUnit test for stack depot" if !KUNIT_ALL_TESTS + depends on KUNIT=y && STACKDEPOT default KUNIT_ALL_TESTS help Enable this option to test stack depot API behavior at boot. diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 6528b45b15445..93f1f43a1059e 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -749,6 +749,81 @@ struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle) return depot_fetch_stack(handle); } +bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count) +{ + struct stack_record *stack; + unsigned int raw; + + if (!handle || !count) + return false; + + stack = depot_fetch_stack(handle); + if (!stack) + return false; + + raw = refcount_read(&stack->count); + /* Saturated records are persistent but not in counted mode. */ + if (raw > INT_MAX) + return false; + + *count = raw; + return true; +} + +void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count) +{ + struct stack_record *stack; + + /* Reject values outside positive refcount space. */ + if (!handle || !count || count >= INT_MAX) + return; + + stack = depot_fetch_stack(handle); + if (!stack) + return; + + refcount_set(&stack->count, (int)count); +} + +bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) +{ + struct stack_record *stack; + int new; + int old = REFCOUNT_SATURATED; + bool was_saturated = false; + + if (!handle || !count || count >= INT_MAX) + return false; + + stack = depot_fetch_stack(handle); + if (!stack) + return false; + + new = 1 + (int)count; + /* Stack records are already published; only the counter value changes. */ + if (atomic_try_cmpxchg_relaxed(&stack->count.refs, &old, new)) + was_saturated = true; + else + refcount_add((int)count, &stack->count); + + return was_saturated; +} + +bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, + unsigned int count) +{ + struct stack_record *stack; + + if (!handle || !count || count >= INT_MAX) + return false; + + stack = depot_fetch_stack(handle); + if (!stack) + return false; + + return refcount_sub_and_test((int)count, &stack->count); +} + unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries) { @@ -784,7 +859,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, unsigned long *stack_entries; unsigned int nr_entries; - if (!entries || !max_entries) + if (!handle || !entries || !max_entries) return 0; nr_entries = stack_depot_fetch(handle, &stack_entries); diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 4892deaaf5924..844100473dbbf 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -82,9 +82,76 @@ static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); } +static void stackdepot_count_helpers(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800210000UL, + 0x1234567800220000UL, + 0x1234567800230000UL, + }; + unsigned long zero_entries[] = { + 0x1234567800310000UL, + 0x1234567800320000UL, + 0x1234567800330000UL, + }; + depot_stack_handle_t handle; + depot_stack_handle_t zero_handle; + unsigned int count; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(0, &count)); + KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(0, NULL)); + __stack_depot_set_count(0, 1); + __stack_depot_set_count(0, 0); + __stack_depot_set_count(0, INT_MAX); + KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(0, 1)); + KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(0, INT_MAX - 1)); + KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(0, 1)); + + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(handle, INT_MAX)); + KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); + + KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 2)); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 3); + + KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(handle, 4)); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 7); + + KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 5)); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 2); + __stack_depot_set_count(handle, 0); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 2); + __stack_depot_set_count(handle, INT_MAX); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 2); + __stack_depot_set_count(handle, 6); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 6); + KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(handle, INT_MAX)); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 6); + + zero_handle = stack_depot_save(zero_entries, ARRAY_SIZE(zero_entries), + GFP_KERNEL); + KUNIT_ASSERT_NE(test, zero_handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(zero_handle, 1)); + KUNIT_EXPECT_TRUE(test, + __stack_depot_dec_count_and_test(zero_handle, 2)); +} + static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), + KUNIT_CASE(stackdepot_count_helpers), {} }; diff --git a/mm/page_owner.c b/mm/page_owner.c index 2626d8adc2abe..6d50e1976ddbf 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -118,8 +118,6 @@ static noinline void register_early_stack(void) static __init void init_page_owner(void) { - struct stack_record *stack_record; - if (!page_owner_enabled) return; @@ -130,12 +128,10 @@ static __init void init_page_owner(void) /* Initialize dummy and failure stacks and link them to stack_list */ dummy_stack.handle = dummy_handle; failure_stack.handle = failure_handle; - stack_record = __stack_depot_get_stack_record(dummy_handle); - if (stack_record) - refcount_set(&stack_record->count, 1); - stack_record = __stack_depot_get_stack_record(failure_handle); - if (stack_record) - refcount_set(&stack_record->count, 1); + if (dummy_handle) + __stack_depot_set_count(dummy_handle, 1); + if (failure_handle) + __stack_depot_set_count(failure_handle, 1); dummy_stack.next = &failure_stack; stack_list = &dummy_stack; static_branch_enable(&page_owner_inited); @@ -207,39 +203,17 @@ static void add_stack_record_to_list(depot_stack_handle_t handle, gfp_t gfp_mask } static void inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, - int nr_base_pages) + unsigned int nr_base_pages) { - struct stack_record *stack_record = __stack_depot_get_stack_record(handle); - - if (!stack_record) - return; - - /* - * New stack_record's that do not use STACK_DEPOT_FLAG_GET start - * with REFCOUNT_SATURATED to catch spurious increments of their - * refcount. - * Since we do not use STACK_DEPOT_FLAG_GET API, let us - * set a refcount of 1 ourselves. - */ - if (refcount_read(&stack_record->count) == REFCOUNT_SATURATED) { - int old = REFCOUNT_SATURATED; - - if (atomic_try_cmpxchg_relaxed(&stack_record->count.refs, &old, 1)) - /* Add the new stack to our list */ - add_stack_record_to_list(handle, gfp_mask); - } - refcount_add(nr_base_pages, &stack_record->count); + /* The first count is the marker for stack_list membership. */ + if (__stack_depot_inc_count(handle, nr_base_pages)) + add_stack_record_to_list(handle, gfp_mask); } static void dec_stack_record_count(depot_stack_handle_t handle, - int nr_base_pages) + unsigned int nr_base_pages) { - struct stack_record *stack_record = __stack_depot_get_stack_record(handle); - - if (!stack_record) - return; - - if (refcount_sub_and_test(nr_base_pages, &stack_record->count)) + if (__stack_depot_dec_count_and_test(handle, nr_base_pages)) pr_warn("%s: refcount went to 0 for %u handle\n", __func__, handle); } @@ -910,20 +884,19 @@ static int stack_print(struct seq_file *m, void *v) struct page_owner_stack_seq *priv = m->private; struct stack *stack = v; depot_stack_handle_t handle = stack->handle; - struct stack_record *stack_record; + unsigned int nr_base_pages; unsigned int i, nr_entries; - int nr_base_pages; if (!handle) return 0; - stack_record = __stack_depot_get_stack_record(handle); - if (!stack_record) + if (!__stack_depot_get_count(handle, &nr_base_pages) || !nr_base_pages) return 0; + nr_base_pages--; - nr_base_pages = refcount_read(&stack_record->count) - 1; - - if (nr_base_pages < 1 || nr_base_pages < page_owner_pages_threshold) + /* Drop the list marker before applying the page-count threshold. */ + if (!nr_base_pages || + (unsigned long)nr_base_pages < page_owner_pages_threshold) return 0; /* Keep show_stacks independent of stackdepot's internal storage layout. */ @@ -935,7 +908,7 @@ static int stack_print(struct seq_file *m, void *v) for (i = 0; i < nr_entries; i++) seq_printf(m, " %pS\n", (void *)priv->entries[i]); - seq_printf(m, "nr_base_pages: %d\n\n", nr_base_pages); + seq_printf(m, "nr_base_pages: %u\n\n", nr_base_pages); return 0; } From 42a90b13d4d60e5a08c697d80352ad0c01ddd4ce Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 28 May 2026 12:40:44 +0100 Subject: [PATCH 007/129] KRN-1117: Keep stackdepot record layout private Move the flat stackdepot record and handle bitfield layout out of the public stackdepot header and into lib/stackdepot.c. Runtime callers now use handle-based helpers, so the flat storage details no longer need to be exposed before the persistent storage backend changes. Keep the page_owner count helpers internal to stackdepot, document their counted-mode semantics, and extend KUnit coverage for saturated, counted, and invalid-count cases. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 66 +++++++----------------------------- lib/Kconfig.debug | 1 + lib/stackdepot.c | 66 +++++++++++++++++++++++++++++------- lib/tests/stackdepot_kunit.c | 21 +++++++----- mm/page_owner.c | 8 ++--- 5 files changed, 85 insertions(+), 77 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index c75c86f2cc087..eb6ee9a49f9c8 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -39,44 +39,6 @@ typedef u32 depot_stack_handle_t; #define DEPOT_POOL_INDEX_BITS (DEPOT_HANDLE_BITS - DEPOT_OFFSET_BITS - \ STACK_DEPOT_EXTRA_BITS) -#ifdef CONFIG_STACKDEPOT -/* Compact structure that stores a reference to a stack. */ -union handle_parts { - depot_stack_handle_t handle; - struct { - u32 pool_index_plus_1 : DEPOT_POOL_INDEX_BITS; - u32 offset : DEPOT_OFFSET_BITS; - u32 extra : STACK_DEPOT_EXTRA_BITS; - }; -}; - -/* Internal only. New users should not inspect stack records directly. */ -struct stack_record { - struct list_head hash_list; /* Links in the hash table */ - u32 hash; /* Hash in hash table */ - u32 size; /* Number of stored frames */ - union handle_parts handle; /* Constant after initialization */ - refcount_t count; - union { - unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; /* Frames */ - struct { - /* - * An important invariant of the implementation is to - * only place a stack record onto the freelist iff its - * refcount is zero. Because stack records with a zero - * refcount are never considered as valid, it is safe to - * union @entries and freelist management state below. - * Conversely, as soon as an entry is off the freelist - * and its refcount becomes non-zero, the below must not - * be accessed until being placed back on the freelist. - */ - struct list_head free_list; /* Links in the freelist */ - unsigned long rcu_state; /* RCU cookie */ - }; - }; -}; -#endif - typedef u32 depot_flags_t; /* @@ -178,17 +140,6 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, depot_stack_handle_t stack_depot_save(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags); -/** - * __stack_depot_get_stack_record - Get a pointer to a stack_record struct - * - * @handle: Stack depot handle - * - * This function is only for internal purposes. - * - * Return: Returns a pointer to a stack_record struct - */ -struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle); - /** * __stack_depot_get_count - Get a counted stack record count * @@ -209,7 +160,11 @@ bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count); * @count: Count to set * * This function is only for internal purposes. - * @count must be greater than 0 and less than %INT_MAX. + * If @count is 0 or greater than or equal to %INT_MAX, this function is a + * no-op. + * Callers that use this to switch a saturated record to counted mode must + * separately make the record discoverable by their own tracking structure. + * Callers must have exclusive access to the stack record count. */ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); @@ -220,10 +175,12 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); * @count: Count to add * * This function is only for internal purposes. - * @count must be greater than 0 and less than %INT_MAX. + * @count must be greater than 0 and less than %INT_MAX - 1. * * Persistent stack records start with refcount set to %REFCOUNT_SATURATED. If * this helper switches a saturated record to counted mode, it stores @count + 1. + * For records already in counted mode, cumulative overflow is handled by the + * underlying refcount_add() warning and saturation semantics. * * Return: true if this call switched the record from saturated to counted, * false otherwise. @@ -237,10 +194,11 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count); * @count: Count to subtract * * This function is only for internal purposes. - * @count must be greater than 0 and less than %INT_MAX. + * @count must be greater than 0 and less than %INT_MAX - 1. * * Return: true if the resulting count is 0, false if the resulting count is - * non-zero or @handle is invalid. + * non-zero, @handle is invalid, the stack record is not in counted mode, or + * @count is greater than the current count. */ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, unsigned int count); @@ -277,6 +235,8 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * Return: Number of frames copied, 0 if @entries is NULL, @max_entries is 0, * @handle is 0 or invalid, stack depot is disabled, or @max_entries is less * than the number of stored frames. + * An invalid or post-put @handle may also trigger a warning from the underlying + * stack_depot_fetch() call. */ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, unsigned long *entries, diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 9160324736572..5f77493c669c7 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2712,6 +2712,7 @@ config STACKDEPOT_KUNIT_TEST default KUNIT_ALL_TESTS help Enable this option to test stack depot API behavior at boot. + This test is built in because it exercises internal stack depot helpers. KUnit tests run during boot and output the results to the debug log in TAP format (https://testanything.org/). Only useful for kernel diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 93f1f43a1059e..da740533460a1 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -54,6 +54,41 @@ static bool __stack_depot_early_init_passed __initdata; /* Initial seed for jhash2. */ #define STACK_HASH_SEED 0x9747b28c +/* Compact structure that stores a reference to a stack. */ +union handle_parts { + depot_stack_handle_t handle; + struct { + u32 pool_index_plus_1 : DEPOT_POOL_INDEX_BITS; + u32 offset : DEPOT_OFFSET_BITS; + u32 extra : STACK_DEPOT_EXTRA_BITS; + }; +}; + +struct stack_record { + struct list_head hash_list; /* Links in the hash table */ + u32 hash; /* Hash in hash table */ + u32 size; /* Number of stored frames */ + union handle_parts handle; /* Constant after initialization */ + refcount_t count; + union { + unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; /* Frames */ + struct { + /* + * An important invariant of the implementation is to + * only place a stack record onto the freelist iff its + * refcount is zero. Because stack records with a zero + * refcount are never considered as valid, it is safe to + * union @entries and freelist management state below. + * Conversely, as soon as an entry is off the freelist + * and its refcount becomes non-zero, the below must not + * be accessed until being placed back on the freelist. + */ + struct list_head free_list; /* Links in the freelist */ + unsigned long rcu_state; /* RCU cookie */ + }; + }; +}; + /* Hash table of stored stack records. */ static struct list_head *stack_table; /* Fixed order of the number of table buckets. Used when KASAN is enabled. */ @@ -741,14 +776,6 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, } EXPORT_SYMBOL_GPL(stack_depot_save); -struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle) -{ - if (!handle) - return NULL; - - return depot_fetch_stack(handle); -} - bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count) { struct stack_record *stack; @@ -761,7 +788,8 @@ bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count) if (!stack) return false; - raw = refcount_read(&stack->count); + /* Negative saturated counts wrap above INT_MAX when converted to unsigned. */ + raw = (unsigned int)refcount_read(&stack->count); /* Saturated records are persistent but not in counted mode. */ if (raw > INT_MAX) return false; @@ -792,7 +820,7 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) int old = REFCOUNT_SATURATED; bool was_saturated = false; - if (!handle || !count || count >= INT_MAX) + if (!handle || !count || count >= (unsigned int)INT_MAX - 1) return false; stack = depot_fetch_stack(handle); @@ -804,6 +832,7 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) if (atomic_try_cmpxchg_relaxed(&stack->count.refs, &old, new)) was_saturated = true; else + /* Preserve refcount_add() overflow warning and saturation semantics. */ refcount_add((int)count, &stack->count); return was_saturated; @@ -813,15 +842,28 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, unsigned int count) { struct stack_record *stack; + int new; + int old; - if (!handle || !count || count >= INT_MAX) + if (!handle || !count || count >= (unsigned int)INT_MAX - 1) return false; stack = depot_fetch_stack(handle); if (!stack) return false; - return refcount_sub_and_test((int)count, &stack->count); + old = refcount_read(&stack->count); + do { + if (old <= 0 || count > old) + return false; + + new = old - (int)count; + } while (!atomic_try_cmpxchg_release(&stack->count.refs, &old, new)); + + if (!new) + smp_acquire__after_ctrl_dep(); + + return !new; } unsigned int stack_depot_fetch(depot_stack_handle_t handle, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 844100473dbbf..128b402dedeb5 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -95,7 +95,7 @@ static void stackdepot_count_helpers(struct kunit *test) 0x1234567800330000UL, }; depot_stack_handle_t handle; - depot_stack_handle_t zero_handle; + depot_stack_handle_t second_handle; unsigned int count; KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); @@ -106,7 +106,7 @@ static void stackdepot_count_helpers(struct kunit *test) __stack_depot_set_count(0, 0); __stack_depot_set_count(0, INT_MAX); KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(0, 1)); - KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(0, INT_MAX - 1)); + KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(0, INT_MAX - 2)); KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(0, 1)); handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); @@ -114,12 +114,15 @@ static void stackdepot_count_helpers(struct kunit *test) KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(handle, INT_MAX)); + KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(handle, INT_MAX - 1)); + KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 1)); KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 2)); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 3); + /* Already-counted records take the refcount_add() path. */ KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(handle, 4)); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 7); @@ -140,12 +143,14 @@ static void stackdepot_count_helpers(struct kunit *test) KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 6); - zero_handle = stack_depot_save(zero_entries, ARRAY_SIZE(zero_entries), - GFP_KERNEL); - KUNIT_ASSERT_NE(test, zero_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(zero_handle, 1)); - KUNIT_EXPECT_TRUE(test, - __stack_depot_dec_count_and_test(zero_handle, 2)); + second_handle = stack_depot_save(zero_entries, ARRAY_SIZE(zero_entries), + GFP_KERNEL); + KUNIT_ASSERT_NE(test, second_handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(second_handle, 1)); + KUNIT_EXPECT_FALSE(test, + __stack_depot_dec_count_and_test(second_handle, 1)); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(second_handle, &count)); + KUNIT_EXPECT_EQ(test, count, 1); } static struct kunit_case stackdepot_test_cases[] = { diff --git a/mm/page_owner.c b/mm/page_owner.c index 6d50e1976ddbf..2b6a1398f4b29 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -125,9 +125,10 @@ static __init void init_page_owner(void) register_failure_stack(); register_early_stack(); init_early_allocated_pages(); - /* Initialize dummy and failure stacks and link them to stack_list */ + /* Initialize dummy and failure stacks and link them to stack_list. */ dummy_stack.handle = dummy_handle; failure_stack.handle = failure_handle; + /* These counts are the stack_list membership markers. */ if (dummy_handle) __stack_depot_set_count(dummy_handle, 1); if (failure_handle) @@ -205,7 +206,7 @@ static void add_stack_record_to_list(depot_stack_handle_t handle, gfp_t gfp_mask static void inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, unsigned int nr_base_pages) { - /* The first count is the marker for stack_list membership. */ + /* The saturated-to-counted transition reserves the stack_list marker. */ if (__stack_depot_inc_count(handle, nr_base_pages)) add_stack_record_to_list(handle, gfp_mask); } @@ -895,8 +896,7 @@ static int stack_print(struct seq_file *m, void *v) nr_base_pages--; /* Drop the list marker before applying the page-count threshold. */ - if (!nr_base_pages || - (unsigned long)nr_base_pages < page_owner_pages_threshold) + if (!nr_base_pages || nr_base_pages < page_owner_pages_threshold) return 0; /* Keep show_stacks independent of stackdepot's internal storage layout. */ From 75f1a4d80eb33c4bddc5f6be20c90492725ea024 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 28 May 2026 14:59:30 +0100 Subject: [PATCH 008/129] KRN-1117: Add stackdepot frame compression hooks Add generic architecture hooks for stackdepot frame compression so future x86-64 and arm64 implementations can opt in without changing common stackdepot callers. The generic asm fallback keeps today's behavior by declining compression and storing raw frames. Keep stackdepot_fetch_into() safe across the copy by holding the same notrace RCU read-side section used by stackdepot lookups, and cover the raw frame fallback plus count-helper edge cases in KUnit. Signed-off-by: Caleb Kan --- arch/um/include/asm/Kbuild | 1 + include/asm-generic/Kbuild | 1 + include/asm-generic/stackdepot.h | 19 +++++++++++++++++ include/linux/stackdepot.h | 32 ++++++++++++++++++++++++++++ lib/stackdepot.c | 33 ++++++++++++++++++++++++++--- lib/tests/stackdepot_kunit.c | 36 ++++++++++++++++++++++++++++++++ 6 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 include/asm-generic/stackdepot.h diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index 9be3ee2e37013..731e6832a3829 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -21,6 +21,7 @@ generic-y += preempt.h generic-y += ring_buffer.h generic-y += runtime-const.h generic-y += softirq_stack.h +generic-y += stackdepot.h generic-y += switch_to.h generic-y += topology.h generic-y += trace_clock.h diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 295c94a3ccc1c..a126f8e237bab 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -53,6 +53,7 @@ mandatory-y += serial.h mandatory-y += shmparam.h mandatory-y += simd.h mandatory-y += softirq_stack.h +mandatory-y += stackdepot.h mandatory-y += switch_to.h mandatory-y += timex.h mandatory-y += tlbflush.h diff --git a/include/asm-generic/stackdepot.h b/include/asm-generic/stackdepot.h new file mode 100644 index 0000000000000..f9bacfaf72ad8 --- /dev/null +++ b/include/asm-generic/stackdepot.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_STACKDEPOT_H +#define __ASM_GENERIC_STACKDEPOT_H + +#include + +static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, + u8 *prefix_id, u32 *low) +{ + return false; +} + +static inline bool arch_stack_depot_frame_decompress(u8 prefix_id, u32 low, + unsigned long *frame) +{ + return false; +} + +#endif /* __ASM_GENERIC_STACKDEPOT_H */ diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index eb6ee9a49f9c8..6158985ddb165 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -203,6 +203,38 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count); bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, unsigned int count); +/** + * __stack_depot_frame_try_compress - Try to compress a stack frame + * + * @frame: Stack frame address + * @prefix_id: Storage for the architecture prefix id + * @low: Storage for the compressed low bits + * + * This function is only for internal purposes. The generic implementation is a + * raw fallback and never compresses. + * @prefix_id and @low must be non-NULL. + * + * Return: true if @frame was compressed, false otherwise. + */ +bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, + u32 *low); + +/** + * __stack_depot_frame_decompress - Decompress a stack frame + * + * @prefix_id: Architecture prefix id returned by compression + * @low: Compressed low bits returned by compression + * @frame: Storage for the decompressed frame + * + * This function is only for internal purposes. The generic raw fallback has no + * compressed representation to decode. + * @frame must be non-NULL. + * + * Return: true if @frame was decompressed, false otherwise. + */ +bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, + unsigned long *frame); + /** * stack_depot_fetch - Fetch a stack trace from stack depot * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index da740533460a1..bddef705c6e85 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -36,6 +36,8 @@ #include #include +#include + /* * The pool_index is offset by 1 so the first record does not have a 0 handle. */ @@ -854,7 +856,7 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, old = refcount_read(&stack->count); do { - if (old <= 0 || count > old) + if (old <= 0 || count > (unsigned int)old) return false; new = old - (int)count; @@ -866,6 +868,24 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, return !new; } +bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, + u32 *low) +{ + if (!prefix_id || !low) + return false; + + return arch_stack_depot_frame_try_compress(frame, prefix_id, low); +} + +bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, + unsigned long *frame) +{ + if (!frame) + return false; + + return arch_stack_depot_frame_decompress(prefix_id, low, frame); +} + unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries) { @@ -900,20 +920,27 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, { unsigned long *stack_entries; unsigned int nr_entries; + unsigned int copied = 0; if (!handle || !entries || !max_entries) return 0; + /* Protect against reuse if stack_depot_put() retires the record mid-copy. */ + rcu_read_lock_sched_notrace(); nr_entries = stack_depot_fetch(handle, &stack_entries); if (!nr_entries || nr_entries > max_entries) - return 0; + goto out; /* * stack_depot_fetch() returns stackdepot-owned storage; the caller must * keep the handle valid while this helper copies from it. */ memcpy(entries, stack_entries, nr_entries * sizeof(*entries)); - return nr_entries; + copied = nr_entries; + +out: + rcu_read_unlock_sched_notrace(); + return copied; } EXPORT_SYMBOL_GPL(stack_depot_fetch_into); diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 128b402dedeb5..6fd5a6a98779a 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -94,8 +94,14 @@ static void stackdepot_count_helpers(struct kunit *test) 0x1234567800320000UL, 0x1234567800330000UL, }; + unsigned long seeded_entries[] = { + 0x1234567800410000UL, + 0x1234567800420000UL, + 0x1234567800430000UL, + }; depot_stack_handle_t handle; depot_stack_handle_t second_handle; + depot_stack_handle_t seeded_handle; unsigned int count; KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); @@ -151,12 +157,42 @@ static void stackdepot_count_helpers(struct kunit *test) __stack_depot_dec_count_and_test(second_handle, 1)); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(second_handle, &count)); KUNIT_EXPECT_EQ(test, count, 1); + + seeded_handle = stack_depot_save(seeded_entries, ARRAY_SIZE(seeded_entries), + GFP_KERNEL); + KUNIT_ASSERT_NE(test, seeded_handle, (depot_stack_handle_t)0); + __stack_depot_set_count(seeded_handle, 3); + KUNIT_EXPECT_FALSE(test, + __stack_depot_dec_count_and_test(seeded_handle, 1)); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(seeded_handle, &count)); + KUNIT_EXPECT_EQ(test, count, 2); +} + +static void stackdepot_frame_raw_fallback(struct kunit *test) +{ + unsigned long frame = 0xffffffff81234567UL; + unsigned long out = 0x12345678UL; + u32 low = 0xfeedbeef; + u8 prefix_id = 0xaa; + + KUNIT_EXPECT_FALSE(test, + __stack_depot_frame_try_compress(frame, &prefix_id, &low)); + KUNIT_EXPECT_EQ(test, prefix_id, (u8)0xaa); + KUNIT_EXPECT_EQ(test, low, (u32)0xfeedbeef); + + KUNIT_EXPECT_FALSE(test, + __stack_depot_frame_decompress(0, 0x81234567, &out)); + KUNIT_EXPECT_EQ(test, out, 0x12345678UL); + + KUNIT_EXPECT_FALSE(test, + __stack_depot_frame_decompress(0, 0x81234567, NULL)); } static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), KUNIT_CASE(stackdepot_count_helpers), + KUNIT_CASE(stackdepot_frame_raw_fallback), {} }; From 250c3b78a174a613b206ebd59cb8f6683e122b77 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 28 May 2026 15:19:26 +0100 Subject: [PATCH 009/129] KRN-1117: Warn on stackdepot count underflow Keep the stackdepot count decrement helper from silently discarding attempts to subtract more than the current counted value. Preserve the old refcount diagnostic behavior by warning before returning false on an underflow attempt. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index bddef705c6e85..89d44ed0587ee 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -856,7 +856,10 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, old = refcount_read(&stack->count); do { - if (old <= 0 || count > (unsigned int)old) + if (old <= 0) + return false; + if (WARN_ONCE(count > (unsigned int)old, + "stack depot count underflow\n")) return false; new = old - (int)count; From 03f8f0a2a4b3417b4f382b5e570ba9a5a450eb55 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 28 May 2026 16:15:55 +0100 Subject: [PATCH 010/129] KRN-1117: Compress x86-64 stackdepot frames Add the x86-64 stackdepot frame compression hook for frames whose high 32 bits match the kernel-text prefix. This keeps non-x86-64 builds on the generic raw fallback while allowing the common code to use the architecture hook added earlier. Extend stackdepot KUnit coverage for the x86-64 round trip and for non-compressible direct-map frames. Signed-off-by: Caleb Kan --- arch/x86/include/asm/stackdepot.h | 37 +++++++++++++++++++++++++++++++ lib/tests/stackdepot_kunit.c | 32 ++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 arch/x86/include/asm/stackdepot.h diff --git a/arch/x86/include/asm/stackdepot.h b/arch/x86/include/asm/stackdepot.h new file mode 100644 index 0000000000000..52835ea95b7f3 --- /dev/null +++ b/arch/x86/include/asm/stackdepot.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_STACKDEPOT_H +#define _ASM_X86_STACKDEPOT_H + +#include + +#ifdef CONFIG_X86_64 +#define STACK_DEPOT_X86_64_FRAME_PREFIX 0xffffffff00000000UL +#define STACK_DEPOT_X86_64_FRAME_LOW_MASK 0x00000000ffffffffUL + +static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, + u8 *prefix_id, u32 *low) +{ + if ((frame & ~STACK_DEPOT_X86_64_FRAME_LOW_MASK) != + STACK_DEPOT_X86_64_FRAME_PREFIX) + return false; + + *prefix_id = 0; + *low = (u32)frame; + return true; +} + +static inline bool arch_stack_depot_frame_decompress(u8 prefix_id, u32 low, + unsigned long *frame) +{ + if (prefix_id) + return false; + + *frame = STACK_DEPOT_X86_64_FRAME_PREFIX | low; + return true; +} + +#else +#include +#endif /* CONFIG_X86_64 */ + +#endif /* _ASM_X86_STACKDEPOT_H */ diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 6fd5a6a98779a..c579393412f76 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -170,7 +170,7 @@ static void stackdepot_count_helpers(struct kunit *test) static void stackdepot_frame_raw_fallback(struct kunit *test) { - unsigned long frame = 0xffffffff81234567UL; + unsigned long frame = 0xffff888000001000UL; unsigned long out = 0x12345678UL; u32 low = 0xfeedbeef; u8 prefix_id = 0xaa; @@ -181,18 +181,46 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) KUNIT_EXPECT_EQ(test, low, (u32)0xfeedbeef); KUNIT_EXPECT_FALSE(test, - __stack_depot_frame_decompress(0, 0x81234567, &out)); + __stack_depot_frame_decompress(1, 0x81234567, &out)); KUNIT_EXPECT_EQ(test, out, 0x12345678UL); KUNIT_EXPECT_FALSE(test, __stack_depot_frame_decompress(0, 0x81234567, NULL)); } +#ifdef CONFIG_X86_64 +static void stackdepot_frame_x86_64(struct kunit *test) +{ + unsigned long direct_map = 0xffff888000001000UL; + unsigned long frame = 0xffffffff81234567UL; + unsigned long out; + bool compressed; + u32 low; + u8 prefix_id; + + KUNIT_EXPECT_TRUE(test, + __stack_depot_frame_try_compress(frame, &prefix_id, &low)); + KUNIT_EXPECT_EQ(test, prefix_id, (u8)0); + KUNIT_EXPECT_EQ(test, low, (u32)0x81234567); + KUNIT_EXPECT_TRUE(test, + __stack_depot_frame_decompress(prefix_id, low, &out)); + KUNIT_EXPECT_EQ(test, out, frame); + + compressed = __stack_depot_frame_try_compress(direct_map, &prefix_id, &low); + KUNIT_EXPECT_FALSE(test, compressed); + KUNIT_EXPECT_FALSE(test, + __stack_depot_frame_decompress(1, low, &out)); +} +#endif /* CONFIG_X86_64 */ + static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), KUNIT_CASE(stackdepot_count_helpers), KUNIT_CASE(stackdepot_frame_raw_fallback), +#ifdef CONFIG_X86_64 + KUNIT_CASE(stackdepot_frame_x86_64), +#endif {} }; From 8f7a08f3abecc4e35cd1bd0f5d090dd98fa51013 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 29 May 2026 09:20:21 +0100 Subject: [PATCH 011/129] KRN-1117: Add stackdepot frame-run encoding helpers Add the internal frame-run encoding layer used by the trie-backed stackdepot prototype. Runs are described as either raw frames or compressed low 32-bit frame values that share one architecture prefix, with caller-provided scratch storage so the kernel port does not hide CONFIG_STACKDEPOT_MAX_FRAMES-sized arrays on the stack. Enable the corresponding arm64 compression hook using the runtime _text prefix plus adjacent 4 GiB prefixes so KASLR and module windows round-trip exactly, while non-matching frames keep the raw fallback. Harden the existing x86 and generic hooks and extend KUnit coverage for raw, compressed, invalid, and max-frame cases. Signed-off-by: Caleb Kan --- MAINTAINERS | 2 + arch/arm64/include/asm/stackdepot.h | 91 ++++++++++ arch/x86/include/asm/stackdepot.h | 6 + include/linux/stackdepot.h | 79 ++++++++- lib/stackdepot.c | 193 +++++++++++++++++++++- lib/tests/stackdepot_kunit.c | 247 +++++++++++++++++++++++++++- mm/page_owner.c | 2 +- 7 files changed, 609 insertions(+), 11 deletions(-) create mode 100644 arch/arm64/include/asm/stackdepot.h diff --git a/MAINTAINERS b/MAINTAINERS index 2b367e6caffed..3d1871e43d7ee 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14281,6 +14281,8 @@ M: Andrew Morton L: linux-kernel@vger.kernel.org S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-nonmm-unstable +F: arch/*/include/asm/stackdepot.h +F: include/asm-generic/stackdepot.h F: include/linux/stackdepot.h F: lib/* F: lib/tests/stackdepot* diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h new file mode 100644 index 0000000000000..9491413c4a1d5 --- /dev/null +++ b/arch/arm64/include/asm/stackdepot.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_STACKDEPOT_H +#define __ASM_STACKDEPOT_H + +#include +#include +#include + +#define STACK_DEPOT_ARM64_FRAME_LOW_MASK 0x00000000ffffffffUL +#define STACK_DEPOT_ARM64_FRAME_PREFIX_MASK (~STACK_DEPOT_ARM64_FRAME_LOW_MASK) + +/* + * The kernel image is KASLR-relocated on arm64, and modules are allocated + * inside a 2 GB relocation window that contains the image. Store the runtime + * text prefix and the two adjacent 4 GB prefixes so both sides of any window + * boundary can round-trip. + */ +#define STACK_DEPOT_ARM64_PREV_PREFIX_ID 0 +#define STACK_DEPOT_ARM64_TEXT_PREFIX_ID 1 +#define STACK_DEPOT_ARM64_NEXT_PREFIX_ID 2 + +static inline unsigned long arch_stack_depot_frame_text_prefix(void) +{ + return (unsigned long)_text & STACK_DEPOT_ARM64_FRAME_PREFIX_MASK; +} + +static inline bool arch_stack_depot_frame_prefix(u8 prefix_id, + unsigned long *prefix) +{ + unsigned long text_prefix = arch_stack_depot_frame_text_prefix(); + + switch (prefix_id) { + case STACK_DEPOT_ARM64_PREV_PREFIX_ID: + if (text_prefix < SZ_4G) + return false; + *prefix = text_prefix - SZ_4G; + return true; + case STACK_DEPOT_ARM64_TEXT_PREFIX_ID: + *prefix = text_prefix; + return true; + case STACK_DEPOT_ARM64_NEXT_PREFIX_ID: + if (text_prefix > ~0UL - SZ_4G) + return false; + *prefix = text_prefix + SZ_4G; + return true; + default: + return false; + } +} + +static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, + u8 *prefix_id, u32 *low) +{ + unsigned long prefix = frame & STACK_DEPOT_ARM64_FRAME_PREFIX_MASK; + unsigned long candidate; + u8 i; + + if (!prefix_id || !low) + return false; + + for (i = STACK_DEPOT_ARM64_PREV_PREFIX_ID; + i <= STACK_DEPOT_ARM64_NEXT_PREFIX_ID; i++) { + if (!arch_stack_depot_frame_prefix(i, &candidate)) + continue; + if (prefix != candidate) + continue; + + *prefix_id = i; + *low = (u32)frame; + return true; + } + + return false; +} + +static inline bool arch_stack_depot_frame_decompress(u8 prefix_id, u32 low, + unsigned long *frame) +{ + unsigned long prefix; + + if (!frame) + return false; + + if (!arch_stack_depot_frame_prefix(prefix_id, &prefix)) + return false; + + *frame = prefix | low; + return true; +} + +#endif /* __ASM_STACKDEPOT_H */ diff --git a/arch/x86/include/asm/stackdepot.h b/arch/x86/include/asm/stackdepot.h index 52835ea95b7f3..d61eef4dc1f3f 100644 --- a/arch/x86/include/asm/stackdepot.h +++ b/arch/x86/include/asm/stackdepot.h @@ -11,6 +11,9 @@ static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, u32 *low) { + if (!prefix_id || !low) + return false; + if ((frame & ~STACK_DEPOT_X86_64_FRAME_LOW_MASK) != STACK_DEPOT_X86_64_FRAME_PREFIX) return false; @@ -23,6 +26,9 @@ static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, static inline bool arch_stack_depot_frame_decompress(u8 prefix_id, u32 low, unsigned long *frame) { + if (!frame) + return false; + if (prefix_id) return false; diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 6158985ddb165..5fe31ff9508c8 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -51,6 +51,18 @@ typedef u32 depot_flags_t; #define STACK_DEPOT_FLAGS_NUM 2 #define STACK_DEPOT_FLAGS_MASK ((depot_flags_t)((1 << STACK_DEPOT_FLAGS_NUM) - 1)) +enum stack_depot_frame_mode { + STACK_DEPOT_FRAME_RAW, + STACK_DEPOT_FRAME_COMPRESSED, +}; + +struct stack_depot_frame_run { + enum stack_depot_frame_mode mode; + u8 prefix_id; + unsigned int nr_entries; + size_t bytes; +}; + /* * Using stack depot requires its initialization, which can be done in 3 ways: * @@ -180,7 +192,9 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); * Persistent stack records start with refcount set to %REFCOUNT_SATURATED. If * this helper switches a saturated record to counted mode, it stores @count + 1. * For records already in counted mode, cumulative overflow is handled by the - * underlying refcount_add() warning and saturation semantics. + * underlying refcount_add() warning and saturation semantics. If such an + * overflow happens, later count get/decrement attempts treat the record as no + * longer counted and fail closed. * * Return: true if this call switched the record from saturated to counted, * false otherwise. @@ -235,6 +249,69 @@ bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, unsigned long *frame); +/** + * __stack_depot_frame_run_init - Describe a homogeneous stack frame run + * + * @entries: Stack frames that start the run + * @nr_entries: Number of frames available in @entries + * @run: Storage for the resulting run description + * + * This function is only for internal purposes. It describes the longest prefix + * of @entries that can be stored with one payload format: raw frames, or low + * bits for frames that all share one architecture prefix id. + * + * Return: 0 on success, -EINVAL on invalid input. + */ +int __stack_depot_frame_run_init(const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_frame_run *run); + +/** + * __stack_depot_frame_run_write - Write a stack frame run payload + * + * @run: Run description returned by __stack_depot_frame_run_init() + * @entries: Stack frames to encode + * @dst: Payload buffer to write + * @dst_size: Size of @dst in bytes + * @scratch: Scratch buffer for compressed frame payloads + * @nr_scratch: Number of 32-bit entries that fit in @scratch + * + * This function is only for internal purposes. It does not write partial + * compressed payloads: if any frame does not match @run, @dst is unchanged. + * Compressed runs require @scratch to hold at least @run->nr_entries entries; + * raw runs do not use @scratch. + * + * Return: 0 on success, -EINVAL on invalid input. + */ +int __stack_depot_frame_run_write(const struct stack_depot_frame_run *run, + const unsigned long *entries, void *dst, + size_t dst_size, u32 *scratch, + unsigned int nr_scratch); + +/** + * __stack_depot_frame_run_read - Read a stack frame run payload + * + * @run: Run description for the payload + * @src: Payload buffer to read + * @src_size: Size of @src in bytes + * @entries: Storage for decoded stack frames + * @max_entries: Number of frames that fit in @entries + * @scratch: Scratch buffer for decoded compressed frames + * @nr_scratch: Number of frames that fit in @scratch + * + * This function is only for internal purposes. It does not write partial + * compressed output: if any frame cannot be decoded, @entries is unchanged. + * Compressed runs require @scratch to hold at least @run->nr_entries entries; + * raw runs do not use @scratch. + * + * Return: 0 on success, -EINVAL on invalid input. + */ +int __stack_depot_frame_run_read(const struct stack_depot_frame_run *run, + const void *src, size_t src_size, + unsigned long *entries, unsigned int max_entries, + unsigned long *scratch, + unsigned int nr_scratch); + /** * stack_depot_fetch - Fetch a stack trace from stack depot * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 89d44ed0587ee..014a4be381201 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -15,6 +15,7 @@ #define pr_fmt(fmt) "stackdepot: " fmt #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -830,7 +832,7 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) return false; new = 1 + (int)count; - /* Stack records are already published; only the counter value changes. */ + /* No refcount_t helper conditionally converts saturation to a count. */ if (atomic_try_cmpxchg_relaxed(&stack->count.refs, &old, new)) was_saturated = true; else @@ -854,12 +856,14 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, if (!stack) return false; + /* refcount_sub_and_test() would saturate; underflow must not change count. */ old = refcount_read(&stack->count); do { + /* Saturated counts are negative and intentionally fail closed here. */ if (old <= 0) return false; - if (WARN_ONCE(count > (unsigned int)old, - "stack depot count underflow\n")) + if (WARN_RATELIMIT(count > (unsigned int)old, + "stack depot count underflow\n")) return false; new = old - (int)count; @@ -871,8 +875,7 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, return !new; } -bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, - u32 *low) +static bool frame_try_compress(unsigned long frame, u8 *prefix_id, u32 *low) { if (!prefix_id || !low) return false; @@ -880,8 +883,13 @@ bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, return arch_stack_depot_frame_try_compress(frame, prefix_id, low); } -bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, - unsigned long *frame) +bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, + u32 *low) +{ + return frame_try_compress(frame, prefix_id, low); +} + +static bool frame_decompress(u8 prefix_id, u32 low, unsigned long *frame) { if (!frame) return false; @@ -889,6 +897,175 @@ bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, return arch_stack_depot_frame_decompress(prefix_id, low, frame); } +bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, + unsigned long *frame) +{ + return frame_decompress(prefix_id, low, frame); +} + +static size_t stack_depot_frame_run_entry_bytes(enum stack_depot_frame_mode mode) +{ + if (mode == STACK_DEPOT_FRAME_COMPRESSED) + return sizeof(u32); + return sizeof(unsigned long); +} + +static int stack_depot_frame_run_validate(const struct stack_depot_frame_run *run) +{ + size_t bytes; + + if (!run || !run->nr_entries || + run->nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) + return -EINVAL; + + switch (run->mode) { + case STACK_DEPOT_FRAME_RAW: + case STACK_DEPOT_FRAME_COMPRESSED: + break; + default: + return -EINVAL; + } + + bytes = run->nr_entries * stack_depot_frame_run_entry_bytes(run->mode); + if (run->bytes != bytes) + return -EINVAL; + + return 0; +} + +int __stack_depot_frame_run_init(const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_frame_run *run) +{ + u8 first_prefix = 0; + u32 low; + unsigned int i; + bool compressed; + + if (!entries || !nr_entries || !run || + nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) + return -EINVAL; + + /* Only prefix ids classify a run; low bits are scratch for the arch hook. */ + compressed = frame_try_compress(entries[0], &first_prefix, &low); + for (i = 1; i < nr_entries; i++) { + u8 prefix_id; + bool next; + + next = frame_try_compress(entries[i], &prefix_id, &low); + if (next != compressed) + break; + if (compressed && prefix_id != first_prefix) + break; + } + + /* @i is the first non-matching frame, or @nr_entries if all matched. */ + run->mode = compressed ? STACK_DEPOT_FRAME_COMPRESSED : STACK_DEPOT_FRAME_RAW; + run->prefix_id = compressed ? first_prefix : 0; + run->nr_entries = i; + run->bytes = i * stack_depot_frame_run_entry_bytes(run->mode); + + return 0; +} + +static int +stack_depot_frame_run_write_compressed(const struct stack_depot_frame_run *run, + const unsigned long *entries, void *dst, + u32 *scratch, unsigned int nr_scratch) +{ + unsigned int i; + + if (!scratch || nr_scratch < run->nr_entries) + return -EINVAL; + + for (i = 0; i < run->nr_entries; i++) { + u8 prefix_id; + + if (!frame_try_compress(entries[i], &prefix_id, &scratch[i])) + return -EINVAL; + if (prefix_id != run->prefix_id) + return -EINVAL; + } + + memcpy(dst, scratch, run->bytes); + return 0; +} + +int __stack_depot_frame_run_write(const struct stack_depot_frame_run *run, + const unsigned long *entries, void *dst, + size_t dst_size, u32 *scratch, + unsigned int nr_scratch) +{ + int ret; + + if (!entries || !dst) + return -EINVAL; + + ret = stack_depot_frame_run_validate(run); + if (ret) + return ret; + if (dst_size < run->bytes) + return -EINVAL; + + if (run->mode == STACK_DEPOT_FRAME_RAW) { + memcpy(dst, entries, run->bytes); + return 0; + } + + return stack_depot_frame_run_write_compressed(run, entries, dst, scratch, + nr_scratch); +} + +static int +stack_depot_frame_run_read_compressed(const struct stack_depot_frame_run *run, + const void *src, unsigned long *entries, + unsigned long *scratch, + unsigned int nr_scratch) +{ + unsigned int i; + + if (!scratch || nr_scratch < run->nr_entries) + return -EINVAL; + + /* Stage lows first so a bad compressed run cannot leave a partial write. */ + for (i = 0; i < run->nr_entries; i++) { + u32 low; + + memcpy(&low, (const char *)src + i * sizeof(low), sizeof(low)); + if (!frame_decompress(run->prefix_id, low, &scratch[i])) + return -EINVAL; + } + + memcpy(entries, scratch, run->nr_entries * sizeof(*entries)); + return 0; +} + +int __stack_depot_frame_run_read(const struct stack_depot_frame_run *run, + const void *src, size_t src_size, + unsigned long *entries, unsigned int max_entries, + unsigned long *scratch, + unsigned int nr_scratch) +{ + int ret; + + if (!src || !entries) + return -EINVAL; + + ret = stack_depot_frame_run_validate(run); + if (ret) + return ret; + if (src_size < run->bytes || max_entries < run->nr_entries) + return -EINVAL; + + if (run->mode == STACK_DEPOT_FRAME_RAW) { + memcpy(entries, src, run->bytes); + return 0; + } + + return stack_depot_frame_run_read_compressed(run, src, entries, scratch, + nr_scratch); +} + unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries) { @@ -928,7 +1105,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, if (!handle || !entries || !max_entries) return 0; - /* Protect against reuse if stack_depot_put() retires the record mid-copy. */ + /* Follow stackdepot's non-traceable RCU read-side convention while copying. */ rcu_read_lock_sched_notrace(); nr_entries = stack_depot_fetch(handle, &stack_entries); if (!nr_entries || nr_entries > max_entries) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index c579393412f76..71dddbf670c54 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -2,10 +2,40 @@ #include #include +#include #include +#include #include #include +#ifdef CONFIG_ARM64 +#include +#endif + +static int +frame_run_init(const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_frame_run *run) +{ + return __stack_depot_frame_run_init(entries, nr_entries, run); +} + +static int frame_run_write(const struct stack_depot_frame_run *run, + const unsigned long *entries, void *dst, size_t dst_size, + u32 *scratch, unsigned int nr_scratch) +{ + return __stack_depot_frame_run_write(run, entries, dst, dst_size, + scratch, nr_scratch); +} + +static int frame_run_read(const struct stack_depot_frame_run *run, + const void *src, size_t src_size, + unsigned long *entries, unsigned int max_entries, + unsigned long *scratch, unsigned int nr_scratch) +{ + return __stack_depot_frame_run_read(run, src, src_size, entries, + max_entries, scratch, nr_scratch); +} + static void stackdepot_fetch_into_roundtrip(struct kunit *test) { unsigned long entries[] = { @@ -99,9 +129,15 @@ static void stackdepot_count_helpers(struct kunit *test) 0x1234567800420000UL, 0x1234567800430000UL, }; + unsigned long max_entries[] = { + 0x1234567800510000UL, + 0x1234567800520000UL, + 0x1234567800530000UL, + }; depot_stack_handle_t handle; depot_stack_handle_t second_handle; depot_stack_handle_t seeded_handle; + depot_stack_handle_t max_handle; unsigned int count; KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); @@ -124,6 +160,12 @@ static void stackdepot_count_helpers(struct kunit *test) KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 1)); KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); + max_handle = stack_depot_save(max_entries, ARRAY_SIZE(max_entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, max_handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(max_handle, INT_MAX - 2)); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(max_handle, &count)); + KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX - 1); + KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 2)); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 3); @@ -175,13 +217,23 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) u32 low = 0xfeedbeef; u8 prefix_id = 0xaa; +#ifdef CONFIG_ARM64 + frame = arch_stack_depot_frame_text_prefix(); + if (frame <= ~0UL - 2 * SZ_4G) + frame += 2 * SZ_4G; + else + frame -= 2 * SZ_4G; + frame |= 0x1000UL; +#endif + + /* Arch hooks may exist, but this frame is chosen to stay raw. */ KUNIT_EXPECT_FALSE(test, __stack_depot_frame_try_compress(frame, &prefix_id, &low)); KUNIT_EXPECT_EQ(test, prefix_id, (u8)0xaa); KUNIT_EXPECT_EQ(test, low, (u32)0xfeedbeef); KUNIT_EXPECT_FALSE(test, - __stack_depot_frame_decompress(1, 0x81234567, &out)); + __stack_depot_frame_decompress(0xff, 0x81234567, &out)); KUNIT_EXPECT_EQ(test, out, 0x12345678UL); KUNIT_EXPECT_FALSE(test, @@ -213,6 +265,189 @@ static void stackdepot_frame_x86_64(struct kunit *test) } #endif /* CONFIG_X86_64 */ +#ifdef CONFIG_ARM64 +static void stackdepot_frame_arm64(struct kunit *test) +{ + unsigned long frame = (unsigned long)stackdepot_frame_arm64; + unsigned long text_prefix = arch_stack_depot_frame_text_prefix(); + unsigned long out; + bool decoded; + u32 low; + u8 prefix_id; + + KUNIT_EXPECT_TRUE(test, + __stack_depot_frame_try_compress(frame, &prefix_id, &low)); + KUNIT_EXPECT_EQ(test, low, (u32)frame); + KUNIT_EXPECT_TRUE(test, + __stack_depot_frame_decompress(prefix_id, low, &out)); + KUNIT_EXPECT_EQ(test, out, frame); + + if (text_prefix >= SZ_4G) { + frame = (text_prefix - SZ_4G) | 0x12345678UL; + KUNIT_EXPECT_TRUE(test, + __stack_depot_frame_try_compress(frame, &prefix_id, &low)); + KUNIT_EXPECT_EQ(test, prefix_id, (u8)STACK_DEPOT_ARM64_PREV_PREFIX_ID); + KUNIT_EXPECT_TRUE(test, + __stack_depot_frame_decompress(prefix_id, low, &out)); + KUNIT_EXPECT_EQ(test, out, frame); + } else { + prefix_id = STACK_DEPOT_ARM64_PREV_PREFIX_ID; + decoded = __stack_depot_frame_decompress(prefix_id, 0, &out); + KUNIT_EXPECT_FALSE(test, decoded); + } + + if (text_prefix <= ~0UL - SZ_4G) { + frame = (text_prefix + SZ_4G) | 0x87654321UL; + KUNIT_EXPECT_TRUE(test, + __stack_depot_frame_try_compress(frame, &prefix_id, &low)); + KUNIT_EXPECT_EQ(test, prefix_id, (u8)STACK_DEPOT_ARM64_NEXT_PREFIX_ID); + KUNIT_EXPECT_TRUE(test, + __stack_depot_frame_decompress(prefix_id, low, &out)); + KUNIT_EXPECT_EQ(test, out, frame); + } else { + prefix_id = STACK_DEPOT_ARM64_NEXT_PREFIX_ID; + decoded = __stack_depot_frame_decompress(prefix_id, 0, &out); + KUNIT_EXPECT_FALSE(test, decoded); + } + + KUNIT_EXPECT_FALSE(test, + __stack_depot_frame_decompress(3, low, &out)); +} +#endif /* CONFIG_ARM64 */ + +static void stackdepot_frame_run_raw_roundtrip(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + unsigned long out[ARRAY_SIZE(entries)] = {}; + struct stack_depot_frame_run run; + unsigned char payload[sizeof(entries)]; + int ret; + + ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, run.mode, STACK_DEPOT_FRAME_RAW); + KUNIT_EXPECT_EQ(test, run.nr_entries, + (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_EQ(test, run.bytes, sizeof(entries)); + + ret = frame_run_write(&run, entries, payload, sizeof(payload), NULL, 0); + KUNIT_EXPECT_EQ(test, ret, 0); + ret = frame_run_read(&run, payload, run.bytes, out, ARRAY_SIZE(out), NULL, 0); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); +} + +#ifdef CONFIG_X86_64 +static void stackdepot_frame_run_x86_64_roundtrip(struct kunit *test) +{ + unsigned long entries[] = { + 0xffffffff81000001UL, + 0xffffffff81000002UL, + 0xffffffff81000003UL, + }; + unsigned long out[ARRAY_SIZE(entries)] = {}; + unsigned long read_scratch[ARRAY_SIZE(entries)]; + struct stack_depot_frame_run run; + u32 payload[ARRAY_SIZE(entries)]; + u32 write_scratch[ARRAY_SIZE(entries)]; + int ret; + + ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, run.mode, STACK_DEPOT_FRAME_COMPRESSED); + KUNIT_EXPECT_EQ(test, run.prefix_id, (u8)0); + KUNIT_EXPECT_EQ(test, run.nr_entries, + (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_EQ(test, run.bytes, sizeof(payload)); + + ret = frame_run_write(&run, entries, payload, sizeof(payload), + write_scratch, ARRAY_SIZE(write_scratch)); + KUNIT_EXPECT_EQ(test, ret, 0); + ret = frame_run_read(&run, payload, run.bytes, out, ARRAY_SIZE(out), + read_scratch, ARRAY_SIZE(read_scratch)); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); +} + +static void stackdepot_frame_run_x86_64_boundary(struct kunit *test) +{ + unsigned long entries[] = { + 0xffffffff81000001UL, + 0xffffffff81000002UL, + 0xffff888000000003UL, + 0xffffffff81000004UL, + }; + struct stack_depot_frame_run run; + int ret; + + ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, run.mode, STACK_DEPOT_FRAME_COMPRESSED); + KUNIT_EXPECT_EQ(test, run.nr_entries, 2U); + + ret = frame_run_init(&entries[2], 2, &run); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, run.mode, STACK_DEPOT_FRAME_RAW); + KUNIT_EXPECT_EQ(test, run.nr_entries, 1U); +} + +static void stackdepot_frame_run_x86_64_write_rejects_mismatch(struct kunit *test) +{ + unsigned long good[] = { + 0xffffffff81000001UL, + 0xffffffff81000002UL, + }; + unsigned long bad[] = { + 0xffffffff81000001UL, + 0xffff888000000002UL, + }; + u32 payload[ARRAY_SIZE(good)] = { 0xa5a5a5a5, 0xb6b6b6b6 }; + u32 scratch[ARRAY_SIZE(good)]; + u32 old[ARRAY_SIZE(payload)]; + struct stack_depot_frame_run run; + int ret; + + memcpy(old, payload, sizeof(old)); + ret = frame_run_init(good, ARRAY_SIZE(good), &run); + KUNIT_EXPECT_EQ(test, ret, 0); + ret = frame_run_write(&run, bad, payload, sizeof(payload), scratch, + ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_MEMEQ(test, payload, old, sizeof(payload)); +} +#endif /* CONFIG_X86_64 */ + +static void stackdepot_frame_run_invalid_inputs(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + unsigned long out[ARRAY_SIZE(entries)] = { 0xa5a5a5a5UL }; + unsigned long old[ARRAY_SIZE(out)]; + struct stack_depot_frame_run run; + unsigned char payload[sizeof(entries)]; + int ret; + + memcpy(old, out, sizeof(old)); + ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); + KUNIT_EXPECT_EQ(test, ret, 0); + + ret = frame_run_init(NULL, ARRAY_SIZE(entries), &run); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = frame_run_init(entries, 0, &run); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = frame_run_init(entries, ARRAY_SIZE(entries), NULL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = frame_run_write(&run, entries, NULL, run.bytes, NULL, 0); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = frame_run_write(&run, entries, payload, run.bytes - 1, NULL, 0); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = frame_run_read(&run, payload, run.bytes - 1, out, ARRAY_SIZE(out), + NULL, 0); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = frame_run_read(&run, payload, run.bytes, out, 0, NULL, 0); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_MEMEQ(test, out, old, sizeof(out)); +} + static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), @@ -221,6 +456,16 @@ static struct kunit_case stackdepot_test_cases[] = { #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), #endif +#ifdef CONFIG_ARM64 + KUNIT_CASE(stackdepot_frame_arm64), +#endif + KUNIT_CASE(stackdepot_frame_run_raw_roundtrip), +#ifdef CONFIG_X86_64 + KUNIT_CASE(stackdepot_frame_run_x86_64_roundtrip), + KUNIT_CASE(stackdepot_frame_run_x86_64_boundary), + KUNIT_CASE(stackdepot_frame_run_x86_64_write_rejects_mismatch), +#endif + KUNIT_CASE(stackdepot_frame_run_invalid_inputs), {} }; diff --git a/mm/page_owner.c b/mm/page_owner.c index 2b6a1398f4b29..a5ae7fac5a83b 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -885,7 +885,7 @@ static int stack_print(struct seq_file *m, void *v) struct page_owner_stack_seq *priv = m->private; struct stack *stack = v; depot_stack_handle_t handle = stack->handle; - unsigned int nr_base_pages; + unsigned int nr_base_pages = 0; unsigned int i, nr_entries; if (!handle) From fe07f9299c14306f5903b67a8186fec606a3fb66 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 29 May 2026 12:28:21 +0100 Subject: [PATCH 012/129] KRN-1117: Tighten page_owner stack display accounting Keep the page_owner stack display buffer sized to the depth that page_owner actually saves, and make count_threshold use the same unsigned int range as the stackdepot page counters. This keeps the stack-list marker subtraction explicit before threshold filtering and rejects oversized debugfs thresholds instead of silently suppressing every stack. Signed-off-by: Caleb Kan --- mm/page_owner.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index a5ae7fac5a83b..233331867fcdb 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -43,7 +44,7 @@ struct stack { struct page_owner_stack_seq { struct stack *stack; - unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; + unsigned long entries[PAGE_OWNER_STACK_DEPTH]; }; static struct stack dummy_stack; @@ -878,7 +879,7 @@ static void *stack_next(struct seq_file *m, void *v, loff_t *ppos) return stack; } -static unsigned long page_owner_pages_threshold; +static unsigned int page_owner_pages_threshold; static int stack_print(struct seq_file *m, void *v) { @@ -891,12 +892,12 @@ static int stack_print(struct seq_file *m, void *v) if (!handle) return 0; - if (!__stack_depot_get_count(handle, &nr_base_pages) || !nr_base_pages) + if (!__stack_depot_get_count(handle, &nr_base_pages) || nr_base_pages <= 1) return 0; nr_base_pages--; /* Drop the list marker before applying the page-count threshold. */ - if (!nr_base_pages || nr_base_pages < page_owner_pages_threshold) + if (nr_base_pages < page_owner_pages_threshold) return 0; /* Keep show_stacks independent of stackdepot's internal storage layout. */ @@ -945,7 +946,10 @@ static int page_owner_threshold_get(void *data, u64 *val) static int page_owner_threshold_set(void *data, u64 val) { - WRITE_ONCE(page_owner_pages_threshold, val); + if (val > UINT_MAX) + return -ERANGE; + + WRITE_ONCE(page_owner_pages_threshold, (unsigned int)val); return 0; } From dd85540545f14446b3577dc587817d08f10646c2 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 29 May 2026 12:29:33 +0100 Subject: [PATCH 013/129] KRN-1117: Materialize stackdepot trie frame runs Add the private trie-node storage primitive from the userspace prototype. A node stores one homogeneous raw or compressed frame run in caller-provided storage and records immutable parent and stack-length metadata so a leaf parent chain can be materialized into caller-owned output. Keep the helper inert for now: stack_depot_save() and the public fetch path still use existing hash records. KUnit covers node sizing, raw and compressed round trips, parent-chain materialization, mixed-run rejection, short storage, and invalid fetch inputs. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 73 +++++++++++- lib/stackdepot.c | 214 ++++++++++++++++++++++++++++++++--- lib/tests/stackdepot_kunit.c | 176 ++++++++++++++++++++++++++++ 3 files changed, 442 insertions(+), 21 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 5fe31ff9508c8..0998e6d704dbf 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -192,9 +192,10 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); * Persistent stack records start with refcount set to %REFCOUNT_SATURATED. If * this helper switches a saturated record to counted mode, it stores @count + 1. * For records already in counted mode, cumulative overflow is handled by the - * underlying refcount_add() warning and saturation semantics. If such an - * overflow happens, later count get/decrement attempts treat the record as no - * longer counted and fail closed. + * underlying refcount_add() saturation semantics; whether that also emits a + * warning depends on the refcount configuration. If such an overflow happens, + * later count get/decrement attempts treat the record as no longer counted and + * fail closed. * * Return: true if this call switched the record from saturated to counted, * false otherwise. @@ -212,7 +213,8 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count); * * Return: true if the resulting count is 0, false if the resulting count is * non-zero, @handle is invalid, the stack record is not in counted mode, or - * @count is greater than the current count. + * @count is greater than the current count. Saturated persistent records are + * not in counted mode and fail closed without changing the record. */ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, unsigned int count); @@ -258,7 +260,9 @@ bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, * * This function is only for internal purposes. It describes the longest prefix * of @entries that can be stored with one payload format: raw frames, or low - * bits for frames that all share one architecture prefix id. + * bits for frames that all share one architecture prefix id. It does not write + * frame payload data; callers that need payload storage must call + * __stack_depot_frame_run_write(). * * Return: 0 on success, -EINVAL on invalid input. */ @@ -302,7 +306,8 @@ int __stack_depot_frame_run_write(const struct stack_depot_frame_run *run, * This function is only for internal purposes. It does not write partial * compressed output: if any frame cannot be decoded, @entries is unchanged. * Compressed runs require @scratch to hold at least @run->nr_entries entries; - * raw runs do not use @scratch. + * raw runs do not use @scratch. For compressed runs, @entries and @scratch + * must not overlap. * * Return: 0 on success, -EINVAL on invalid input. */ @@ -312,6 +317,62 @@ int __stack_depot_frame_run_read(const struct stack_depot_frame_run *run, unsigned long *scratch, unsigned int nr_scratch); +/** + * __stack_depot_trie_node_size - Get storage size for a trie node + * + * @run: Frame run to store in the node + * + * This function is only for internal purposes. + * + * Return: Aligned node storage size, 0 on invalid input. + */ +size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run); + +/** + * __stack_depot_trie_node_init - Initialize a trie node in caller storage + * + * @storage: Node storage to initialize + * @storage_size: Size of @storage in bytes + * @parent: Parent node or NULL for a root node + * @leaf_id: Non-zero id when this node terminates a stored stack + * @entries: Homogeneous frame run to store in this node + * @nr_entries: Number of frames in @entries + * @scratch: Scratch buffer for compressed frame payloads + * @nr_scratch: Number of 32-bit entries that fit in @scratch + * + * This function is only for internal purposes. It does not publish @storage; + * all @entries must fit in one raw or same-prefix compressed frame run. Callers + * remain responsible for lifetime and visibility. Callers must discard @storage + * unless this function returns 0. + * + * Return: 0 on success, -EINVAL on invalid input. + */ +int __stack_depot_trie_node_init(void *storage, size_t storage_size, + const void *parent, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, u32 *scratch, + unsigned int nr_scratch); + +/** + * __stack_depot_trie_fetch_into - Materialize a trie parent chain + * + * @leaf: Leaf node to materialize from + * @entries: Caller-owned output buffer + * @max_entries: Number of frames that fit in @entries + * @scratch: Caller-owned scratch buffer for staged output + * @nr_scratch: Number of frames that fit in @scratch + * + * This function is only for internal purposes. It stages the full stack into + * @scratch first, so failures do not partially write @entries. @scratch is + * caller-owned temporary storage and may be modified on failure. + * + * Return: Number of frames copied, 0 on invalid input or too-small buffers. + */ +unsigned int +__stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, + unsigned int max_entries, unsigned long *scratch, + unsigned int nr_scratch); + /** * stack_depot_fetch - Fetch a stack trace from stack depot * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 014a4be381201..0dd2dcd60892d 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -93,6 +94,14 @@ struct stack_record { }; }; +struct stack_depot_trie_node { + const struct stack_depot_trie_node *parent; + u32 leaf_id; + u32 stack_len; + struct stack_depot_frame_run run; + unsigned char data[]; +}; + /* Hash table of stored stack records. */ static struct list_head *stack_table; /* Fixed order of the number of table buckets. Used when KASAN is enabled. */ @@ -832,7 +841,12 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) return false; new = 1 + (int)count; - /* No refcount_t helper conditionally converts saturation to a count. */ + /* + * Intentional refcount_t internals use: no helper conditionally + * converts the persistent REFCOUNT_SATURATED sentinel to a positive + * page_owner count. The cmpxchg only performs that one-way transition; + * normal counted records continue through refcount_add(). + */ if (atomic_try_cmpxchg_relaxed(&stack->count.refs, &old, new)) was_saturated = true; else @@ -856,9 +870,14 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, if (!stack) return false; - /* refcount_sub_and_test() would saturate; underflow must not change count. */ + /* + * Intentional refcount_t internals use: refcount_sub_and_test() would + * saturate on underflow, but page_owner accounting must warn and leave + * the existing count unchanged. + */ old = refcount_read(&stack->count); do { + /* Retry checks use the current count as the linearization point. */ /* Saturated counts are negative and intentionally fail closed here. */ if (old <= 0) return false; @@ -933,9 +952,10 @@ static int stack_depot_frame_run_validate(const struct stack_depot_frame_run *ru return 0; } -int __stack_depot_frame_run_init(const unsigned long *entries, - unsigned int nr_entries, - struct stack_depot_frame_run *run) +static int frame_run_init_lows(const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_frame_run *run, + u32 *lows, unsigned int nr_lows) { u8 first_prefix = 0; u32 low; @@ -945,9 +965,14 @@ int __stack_depot_frame_run_init(const unsigned long *entries, if (!entries || !nr_entries || !run || nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) return -EINVAL; + /* The run length is not known yet, so scratch must cover the input. */ + if (lows && nr_entries > nr_lows) + return -EINVAL; /* Only prefix ids classify a run; low bits are scratch for the arch hook. */ compressed = frame_try_compress(entries[0], &first_prefix, &low); + if (compressed && lows) + lows[0] = low; for (i = 1; i < nr_entries; i++) { u8 prefix_id; bool next; @@ -957,6 +982,8 @@ int __stack_depot_frame_run_init(const unsigned long *entries, break; if (compressed && prefix_id != first_prefix) break; + if (compressed && lows) + lows[i] = low; } /* @i is the first non-matching frame, or @nr_entries if all matched. */ @@ -968,6 +995,13 @@ int __stack_depot_frame_run_init(const unsigned long *entries, return 0; } +int __stack_depot_frame_run_init(const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_frame_run *run) +{ + return frame_run_init_lows(entries, nr_entries, run, NULL, 0); +} + static int stack_depot_frame_run_write_compressed(const struct stack_depot_frame_run *run, const unsigned long *entries, void *dst, @@ -991,10 +1025,9 @@ stack_depot_frame_run_write_compressed(const struct stack_depot_frame_run *run, return 0; } -int __stack_depot_frame_run_write(const struct stack_depot_frame_run *run, - const unsigned long *entries, void *dst, - size_t dst_size, u32 *scratch, - unsigned int nr_scratch) +static int frame_run_write(const struct stack_depot_frame_run *run, + const unsigned long *entries, void *dst, size_t dst_size, + u32 *scratch, unsigned int nr_scratch) { int ret; @@ -1016,6 +1049,14 @@ int __stack_depot_frame_run_write(const struct stack_depot_frame_run *run, nr_scratch); } +int __stack_depot_frame_run_write(const struct stack_depot_frame_run *run, + const unsigned long *entries, void *dst, + size_t dst_size, u32 *scratch, + unsigned int nr_scratch) +{ + return frame_run_write(run, entries, dst, dst_size, scratch, nr_scratch); +} + static int stack_depot_frame_run_read_compressed(const struct stack_depot_frame_run *run, const void *src, unsigned long *entries, @@ -1036,15 +1077,39 @@ stack_depot_frame_run_read_compressed(const struct stack_depot_frame_run *run, return -EINVAL; } - memcpy(entries, scratch, run->nr_entries * sizeof(*entries)); + if (entries != scratch) + memcpy(entries, scratch, run->nr_entries * sizeof(*entries)); return 0; } -int __stack_depot_frame_run_read(const struct stack_depot_frame_run *run, - const void *src, size_t src_size, - unsigned long *entries, unsigned int max_entries, - unsigned long *scratch, - unsigned int nr_scratch) +static int frame_run_read_to_scratch(const struct stack_depot_frame_run *run, + const void *src, unsigned long *scratch, + unsigned int nr_scratch) +{ + return stack_depot_frame_run_read_compressed(run, src, scratch, scratch, + nr_scratch); +} + +static bool stack_depot_ranges_overlap(const void *a, const void *b, size_t size) +{ + unsigned long a_start = (unsigned long)a; + unsigned long b_start = (unsigned long)b; + unsigned long a_end; + unsigned long b_end; + + if (!size) + return false; + if (check_add_overflow(a_start, size, &a_end) || + check_add_overflow(b_start, size, &b_end)) + return true; + + return a_start < b_end && b_start < a_end; +} + +static int frame_run_read(const struct stack_depot_frame_run *run, + const void *src, size_t src_size, + unsigned long *entries, unsigned int max_entries, + unsigned long *scratch, unsigned int nr_scratch) { int ret; @@ -1061,11 +1126,130 @@ int __stack_depot_frame_run_read(const struct stack_depot_frame_run *run, memcpy(entries, src, run->bytes); return 0; } + if (!scratch || nr_scratch < run->nr_entries) + return -EINVAL; + if (stack_depot_ranges_overlap(entries, scratch, + run->nr_entries * sizeof(*entries))) + return -EINVAL; return stack_depot_frame_run_read_compressed(run, src, entries, scratch, nr_scratch); } +int __stack_depot_frame_run_read(const struct stack_depot_frame_run *run, + const void *src, size_t src_size, + unsigned long *entries, unsigned int max_entries, + unsigned long *scratch, + unsigned int nr_scratch) +{ + return frame_run_read(run, src, src_size, entries, max_entries, scratch, + nr_scratch); +} + +size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) +{ + size_t size; + + if (stack_depot_frame_run_validate(run)) + return 0; + size = sizeof(struct stack_depot_trie_node); + if (check_add_overflow(size, run->bytes, &size)) + return 0; + + return ALIGN(size, sizeof(unsigned long)); +} + +int __stack_depot_trie_node_init(void *storage, size_t storage_size, + const void *parent, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, u32 *scratch, + unsigned int nr_scratch) +{ + const struct stack_depot_trie_node *parent_node = parent; + struct stack_depot_trie_node *node = storage; + struct stack_depot_frame_run run; + u32 stack_len; + int ret; + + if (!node || !entries) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)node, __alignof__(*node))) + return -EINVAL; + + ret = frame_run_init_lows(entries, nr_entries, &run, scratch, nr_scratch); + if (ret) + return ret; + if (run.nr_entries != nr_entries) + return -EINVAL; + if (storage_size < __stack_depot_trie_node_size(&run)) + return -EINVAL; + /* frame_run_init_lows() permits NULL scratch for raw runs only. */ + if (run.mode == STACK_DEPOT_FRAME_COMPRESSED && + (!scratch || nr_scratch < run.nr_entries)) + return -EINVAL; + if (parent_node) { + if (!parent_node->stack_len || + parent_node->stack_len > U32_MAX - run.nr_entries) + return -EINVAL; + stack_len = parent_node->stack_len + run.nr_entries; + } else { + stack_len = run.nr_entries; + } + + /* Caller-owned storage is not publishable unless the payload write succeeds. */ + if (run.mode == STACK_DEPOT_FRAME_COMPRESSED) + /* Copy low-bit payloads staged by frame_run_init_lows(). */ + memcpy(node->data, scratch, run.bytes); + else + memcpy(node->data, entries, run.bytes); + + node->parent = parent_node; + node->leaf_id = leaf_id; + node->stack_len = stack_len; + node->run = run; + return 0; +} + +unsigned int +__stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, + unsigned int max_entries, unsigned long *scratch, + unsigned int nr_scratch) +{ + const struct stack_depot_trie_node *node = leaf; + unsigned int pos; + unsigned int total; + int ret; + + if (!node || !entries || !scratch || !node->stack_len || !node->leaf_id) + return 0; + + total = node->stack_len; + if (max_entries < total || nr_scratch < total) + return 0; + + pos = total; + for (node = leaf; node; node = node->parent) { + if (node->stack_len != pos || node->run.nr_entries > pos) + return 0; + pos -= node->run.nr_entries; + /* Decode directly into the staged output; node->data is separate. */ + if (node->run.mode == STACK_DEPOT_FRAME_COMPRESSED) + ret = frame_run_read_to_scratch(&node->run, node->data, + &scratch[pos], nr_scratch - pos); + else + ret = frame_run_read(&node->run, node->data, + node->run.bytes, &scratch[pos], + nr_scratch - pos, NULL, 0); + if (ret) + return 0; + } + if (pos) + return 0; + + memcpy(entries, scratch, total * sizeof(*entries)); + return total; +} + unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries) { diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 71dddbf670c54..e917eebf22ff9 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -36,6 +36,44 @@ static int frame_run_read(const struct stack_depot_frame_run *run, max_entries, scratch, nr_scratch); } +static int tnode_init(void *storage, size_t storage_size, const void *parent, + u32 leaf_id, const unsigned long *entries, + unsigned int nr_entries, u32 *scratch, + unsigned int nr_scratch) +{ + return __stack_depot_trie_node_init(storage, storage_size, parent, leaf_id, + entries, nr_entries, scratch, + nr_scratch); +} + +static unsigned int tfetch(const void *leaf, unsigned long *entries, + unsigned int max_entries, unsigned long *scratch, + unsigned int nr_scratch) +{ + return __stack_depot_trie_fetch_into(leaf, entries, max_entries, scratch, + nr_scratch); +} + +static void +trie_node_alloc(struct kunit *test, const unsigned long *entries, + unsigned int nr_entries, const void *parent, u32 leaf_id, + void **node) +{ + u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; + struct stack_depot_frame_run run; + size_t size; + int ret; + + KUNIT_ASSERT_EQ(test, frame_run_init(entries, nr_entries, &run), 0); + size = __stack_depot_trie_node_size(&run); + KUNIT_ASSERT_GT(test, size, (size_t)0); + *node = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, *node); + ret = tnode_init(*node, size, parent, leaf_id, entries, nr_entries, + write_scratch, ARRAY_SIZE(write_scratch)); + KUNIT_ASSERT_EQ(test, ret, 0); +} + static void stackdepot_fetch_into_roundtrip(struct kunit *test) { unsigned long entries[] = { @@ -448,6 +486,133 @@ static void stackdepot_frame_run_invalid_inputs(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, out, old, sizeof(out)); } +static void stackdepot_trie_node_raw_roundtrip(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + unsigned long scratch[ARRAY_SIZE(entries)]; + unsigned long out[ARRAY_SIZE(entries)] = {}; + unsigned int fetched; + void *node; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 7, &node); + fetched = tfetch(node, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); +} + +static void stackdepot_trie_node_parent_chain(struct kunit *test) +{ + unsigned long root_entries[] = { 0x1000UL, 0x2000UL }; + unsigned long child_entries[] = { 0x3000UL, 0x4000UL }; + unsigned long expected[] = { 0x1000UL, 0x2000UL, 0x3000UL, 0x4000UL }; + unsigned long scratch[ARRAY_SIZE(expected)]; + unsigned long out[ARRAY_SIZE(expected)] = {}; + unsigned int fetched; + void *root; + void *child; + + trie_node_alloc(test, root_entries, ARRAY_SIZE(root_entries), NULL, 0, + &root); + trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), root, 9, + &child); + fetched = tfetch(child, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); + KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); +} + +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) +static void stackdepot_trie_node_compressed_roundtrip(struct kunit *test) +{ + unsigned long entries[] = { +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x1000UL, + arch_stack_depot_frame_text_prefix() | 0x2000UL, +#else + 0xffffffff81001000UL, + 0xffffffff81002000UL, +#endif + }; + unsigned long scratch[ARRAY_SIZE(entries)]; + unsigned long out[ARRAY_SIZE(entries)] = {}; + unsigned int fetched; + void *node; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 11, &node); + fetched = tfetch(node, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); +} +#endif + +static void stackdepot_trie_node_rejects_short_storage(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_frame_run run; + unsigned char storage[sizeof(unsigned long)]; + size_t size; + int ret; + + KUNIT_ASSERT_EQ(test, frame_run_init(entries, ARRAY_SIZE(entries), &run), 0); + size = __stack_depot_trie_node_size(&run); + KUNIT_ASSERT_GT(test, size, sizeof(storage)); + ret = tnode_init(storage, sizeof(storage), NULL, 1, entries, + ARRAY_SIZE(entries), NULL, 0); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) +static void stackdepot_trie_node_rejects_mixed_run(struct kunit *test) +{ + unsigned long storage[32]; + u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; + unsigned long entries[] = { +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x1000UL, + 0x1000UL, +#else + 0xffffffff81001000UL, + 0xffff888000001000UL, +#endif + }; + int ret; + + ret = tnode_init(storage, sizeof(storage), NULL, 1, entries, + ARRAY_SIZE(entries), write_scratch, + ARRAY_SIZE(write_scratch)); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} +#endif + +static void stackdepot_trie_fetch_rejects_bad_inputs(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + unsigned long root_entries[] = { 0x3000UL }; + unsigned long out[ARRAY_SIZE(entries)] = { 0xa5a5UL, 0xb6b6UL }; + unsigned long expected[ARRAY_SIZE(out)]; + unsigned long scratch[ARRAY_SIZE(entries)]; + unsigned int fetched; + void *node; + void *root; + + memcpy(expected, out, sizeof(expected)); + trie_node_alloc(test, root_entries, ARRAY_SIZE(root_entries), NULL, 0, + &root); + fetched = tfetch(root, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 0); + KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 5, &node); + fetched = tfetch(node, out, ARRAY_SIZE(out) - 1, scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 0); + KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); + fetched = tfetch(node, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch) - 1); + KUNIT_EXPECT_EQ(test, fetched, 0); + KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); + fetched = tfetch(NULL, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 0); + KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); +} + static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), @@ -466,6 +631,16 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_frame_run_x86_64_write_rejects_mismatch), #endif KUNIT_CASE(stackdepot_frame_run_invalid_inputs), + KUNIT_CASE(stackdepot_trie_node_raw_roundtrip), + KUNIT_CASE(stackdepot_trie_node_parent_chain), +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) + KUNIT_CASE(stackdepot_trie_node_compressed_roundtrip), +#endif + KUNIT_CASE(stackdepot_trie_node_rejects_short_storage), +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) + KUNIT_CASE(stackdepot_trie_node_rejects_mixed_run), +#endif + KUNIT_CASE(stackdepot_trie_fetch_rejects_bad_inputs), {} }; @@ -477,4 +652,5 @@ static struct kunit_suite stackdepot_test_suite = { kunit_test_suite(stackdepot_test_suite); MODULE_DESCRIPTION("KUnit tests for stack depot"); +MODULE_AUTHOR("Caleb Kan "); MODULE_LICENSE("GPL"); From 83e18286b48417cfbbea071ed941bcc51db1bfcb Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 29 May 2026 14:41:34 +0100 Subject: [PATCH 014/129] KRN-1117: Add stackdepot trie child arrays Add the sorted child-array helper layer from the userspace trie prototype. Child arrays are built in caller-owned storage, reject duplicate first-frame keys and in-place replacement, and remain unpublished so no RCU or COW visibility rules change yet. Cover initialization, lookup, empty insertion, middle insertion, unsorted input, duplicate insertion, and in-place update rejection in stackdepot KUnit. Also document the page_owner stack listing race as best-effort because counts can change while seq_file output is being generated. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 65 ++++++++++- lib/stackdepot.c | 206 +++++++++++++++++++++++++++++++++-- lib/tests/stackdepot_kunit.c | 155 +++++++++++++++++++++++++- mm/page_owner.c | 1 + 4 files changed, 412 insertions(+), 15 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 0998e6d704dbf..6eabb4f25ad08 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -187,7 +187,7 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); * @count: Count to add * * This function is only for internal purposes. - * @count must be greater than 0 and less than %INT_MAX - 1. + * @count must be greater than 0 and less than or equal to %INT_MAX - 1. * * Persistent stack records start with refcount set to %REFCOUNT_SATURATED. If * this helper switches a saturated record to counted mode, it stores @count + 1. @@ -196,6 +196,7 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); * warning depends on the refcount configuration. If such an overflow happens, * later count get/decrement attempts treat the record as no longer counted and * fail closed. + * Callers must ensure @handle remains valid for the duration of this call. * * Return: true if this call switched the record from saturated to counted, * false otherwise. @@ -209,7 +210,7 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count); * @count: Count to subtract * * This function is only for internal purposes. - * @count must be greater than 0 and less than %INT_MAX - 1. + * @count must be greater than 0 and less than or equal to %INT_MAX - 1. * * Return: true if the resulting count is 0, false if the resulting count is * non-zero, @handle is invalid, the stack record is not in counted mode, or @@ -373,6 +374,66 @@ __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries, unsigned long *scratch, unsigned int nr_scratch); +/** + * __stack_depot_trie_child_array_size - Get storage size for child pointers + * + * @nr_children: Number of child pointers stored in the array + * + * This function is only for internal purposes. + * + * Return: Aligned child-array storage size, 0 on overflow. + */ +size_t __stack_depot_trie_child_array_size(unsigned int nr_children); + +/** + * __stack_depot_trie_child_array_init - Initialize sorted child storage + * + * @storage: Child-array storage to initialize + * @storage_size: Size of @storage in bytes + * @children: Children sorted by first decoded frame + * @nr_children: Number of child pointers in @children + * + * This function is only for internal purposes. It does not publish @storage; + * callers remain responsible for lifetime and visibility. Callers must discard + * @storage unless this function returns 0. + * + * Return: 0 on success, -EINVAL on invalid input. + */ +int +__stack_depot_trie_child_array_init(void *storage, size_t storage_size, + const void * const *children, + unsigned int nr_children); + +/** + * __stack_depot_trie_child_array_find - Find a child by first frame + * + * @storage: Child-array storage initialized by child_array_init/insert + * @frame: First decoded frame to search for + * + * This function is only for internal purposes. + * + * Return: Child pointer if found, NULL otherwise. + */ +const void * +__stack_depot_trie_child_array_find(const void *storage, unsigned long frame); + +/** + * __stack_depot_trie_child_array_insert - Build replacement child storage + * + * @old_storage: Existing sorted child array, or NULL + * @child: Child node to insert + * @new_storage: Replacement child-array storage to initialize + * @new_storage_size: Size of @new_storage in bytes + * + * This function is only for internal purposes. It builds a new sorted child + * array and rejects duplicate first-frame keys and in-place updates. + * + * Return: 0 on success, -EINVAL on invalid input. + */ +int +__stack_depot_trie_child_array_insert(const void *old_storage, const void *child, + void *new_storage, size_t new_storage_size); + /** * stack_depot_fetch - Fetch a stack trace from stack depot * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 0dd2dcd60892d..1de5365386169 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -102,6 +102,11 @@ struct stack_depot_trie_node { unsigned char data[]; }; +struct stack_depot_trie_child_array { + unsigned int nr_children; + const struct stack_depot_trie_node *children[]; +}; + /* Hash table of stored stack records. */ static struct list_head *stack_table; /* Fixed order of the number of table buckets. Used when KASAN is enabled. */ @@ -803,8 +808,8 @@ bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count) /* Negative saturated counts wrap above INT_MAX when converted to unsigned. */ raw = (unsigned int)refcount_read(&stack->count); - /* Saturated records are persistent but not in counted mode. */ - if (raw > INT_MAX) + /* Saturated and zero records are not in counted mode. */ + if (!raw || raw > INT_MAX) return false; *count = raw; @@ -816,7 +821,7 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count) struct stack_record *stack; /* Reject values outside positive refcount space. */ - if (!handle || !count || count >= INT_MAX) + if (!handle || !count || count >= (unsigned int)INT_MAX) return; stack = depot_fetch_stack(handle); @@ -833,7 +838,7 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) int old = REFCOUNT_SATURATED; bool was_saturated = false; - if (!handle || !count || count >= (unsigned int)INT_MAX - 1) + if (!handle || !count || count > (unsigned int)INT_MAX - 1) return false; stack = depot_fetch_stack(handle); @@ -847,10 +852,10 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) * page_owner count. The cmpxchg only performs that one-way transition; * normal counted records continue through refcount_add(). */ - if (atomic_try_cmpxchg_relaxed(&stack->count.refs, &old, new)) + if (atomic_try_cmpxchg_release(&stack->count.refs, &old, new)) was_saturated = true; else - /* Preserve refcount_add() overflow warning and saturation semantics. */ + /* Existing counted records only need refcount_add()'s relaxed ordering. */ refcount_add((int)count, &stack->count); return was_saturated; @@ -863,7 +868,7 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, int new; int old; - if (!handle || !count || count >= (unsigned int)INT_MAX - 1) + if (!handle || !count || count > (unsigned int)INT_MAX - 1) return false; stack = depot_fetch_stack(handle); @@ -875,14 +880,22 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, * saturate on underflow, but page_owner accounting must warn and leave * the existing count unchanged. */ + /* A stale read is harmless: cmpxchg reloads @old before retry checks. */ old = refcount_read(&stack->count); do { - /* Retry checks use the current count as the linearization point. */ + bool underflow; + + /* + * Retry checks use the observed count as this operation's + * linearization point. A racing increment not observed here is + * ordered after this decrement. + */ /* Saturated counts are negative and intentionally fail closed here. */ if (old <= 0) return false; - if (WARN_RATELIMIT(count > (unsigned int)old, - "stack depot count underflow\n")) + + underflow = count > (unsigned int)old; + if (WARN_RATELIMIT(underflow, "stack depot count underflow\n")) return false; new = old - (int)count; @@ -969,6 +982,7 @@ static int frame_run_init_lows(const unsigned long *entries, if (lows && nr_entries > nr_lows) return -EINVAL; + /* On success, only lows[0..run->nr_entries - 1] are initialized. */ /* Only prefix ids classify a run; low bits are scratch for the arch hook. */ compressed = frame_try_compress(entries[0], &first_prefix, &low); if (compressed && lows) @@ -1159,6 +1173,27 @@ size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) return ALIGN(size, sizeof(unsigned long)); } +static int +stack_depot_trie_node_first_frame(const struct stack_depot_trie_node *node, + unsigned long *frame) +{ + u32 low; + + if (!node || !frame || stack_depot_frame_run_validate(&node->run)) + return -EINVAL; + + if (node->run.mode == STACK_DEPOT_FRAME_RAW) { + memcpy(frame, node->data, sizeof(*frame)); + return 0; + } + + memcpy(&low, node->data, sizeof(low)); + if (!frame_decompress(node->run.prefix_id, low, frame)) + return -EINVAL; + + return 0; +} + int __stack_depot_trie_node_init(void *storage, size_t storage_size, const void *parent, u32 leaf_id, const unsigned long *entries, @@ -1232,6 +1267,7 @@ __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, if (node->stack_len != pos || node->run.nr_entries > pos) return 0; pos -= node->run.nr_entries; + /* nr_scratch >= total, and pos tracks the remaining prefix length. */ /* Decode directly into the staged output; node->data is separate. */ if (node->run.mode == STACK_DEPOT_FRAME_COMPRESSED) ret = frame_run_read_to_scratch(&node->run, node->data, @@ -1250,6 +1286,156 @@ __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, return total; } +size_t __stack_depot_trie_child_array_size(unsigned int nr_children) +{ + size_t size; + size_t bytes; + + if (check_mul_overflow((size_t)nr_children, + sizeof(struct stack_depot_trie_node *), &bytes)) + return 0; + size = sizeof(struct stack_depot_trie_child_array); + if (check_add_overflow(size, bytes, &size)) + return 0; + + return ALIGN(size, sizeof(unsigned long)); +} + +int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, + const void * const *children, + unsigned int nr_children) +{ + struct stack_depot_trie_child_array *array = storage; + const struct stack_depot_trie_node * const *nodes = + (const struct stack_depot_trie_node * const *)children; + unsigned long last = 0; + unsigned int i; + + if (!array || storage_size < __stack_depot_trie_child_array_size(nr_children)) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)array, __alignof__(*array))) + return -EINVAL; + if (nr_children && !nodes) + return -EINVAL; + + for (i = 0; i < nr_children; i++) { + unsigned long frame; + + if (stack_depot_trie_node_first_frame(nodes[i], &frame)) + return -EINVAL; + if (!frame || (i && frame <= last)) + return -EINVAL; + last = frame; + } + + array->nr_children = nr_children; + for (i = 0; i < nr_children; i++) + array->children[i] = nodes[i]; + + return 0; +} + +static int +stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *array, + unsigned long frame, unsigned int *pos, bool *found) +{ + unsigned int left = 0; + unsigned int right; + + if (!array || !pos || !found) + return -EINVAL; + + right = array->nr_children; + while (left < right) { + unsigned int mid = left + (right - left) / 2; + unsigned long mid_frame; + + if (stack_depot_trie_node_first_frame(array->children[mid], &mid_frame)) + return -EINVAL; + if (mid_frame < frame) { + left = mid + 1; + } else if (mid_frame > frame) { + right = mid; + } else { + *pos = mid; + *found = true; + return 0; + } + } + + *pos = left; + *found = false; + return 0; +} + +const void * +__stack_depot_trie_child_array_find(const void *storage, unsigned long frame) +{ + const struct stack_depot_trie_child_array *array = storage; + unsigned int pos; + bool found; + + if (!array) + return NULL; + + if (stack_depot_trie_child_lower_bound(array, frame, &pos, &found) || + !found) + return NULL; + + return array->children[pos]; +} + +int +__stack_depot_trie_child_array_insert(const void *old_storage, const void *child, + void *new_storage, size_t new_storage_size) +{ + const struct stack_depot_trie_child_array *old_array = old_storage; + struct stack_depot_trie_child_array *new_array = new_storage; + const struct stack_depot_trie_node *node = child; + unsigned int nr_old; + unsigned int pos; + unsigned int i; + unsigned long frame; + size_t old_size; + bool found; + + if (!node || !new_array || stack_depot_trie_node_first_frame(node, &frame)) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)new_array, __alignof__(*new_array))) + return -EINVAL; + if (!frame) + return -EINVAL; + if (old_array == new_array) + return -EINVAL; + + nr_old = old_array ? old_array->nr_children : 0; + if (new_storage_size < __stack_depot_trie_child_array_size(nr_old + 1)) + return -EINVAL; + old_size = __stack_depot_trie_child_array_size(nr_old); + if (old_array && stack_depot_ranges_overlap(old_array, new_array, old_size)) + return -EINVAL; + + if (old_array) { + if (stack_depot_trie_child_lower_bound(old_array, frame, &pos, + &found)) + return -EINVAL; + if (found) + return -EINVAL; + } else { + /* NULL old array means the new child must be inserted at the start. */ + pos = 0; + } + + new_array->nr_children = nr_old + 1; + for (i = 0; i < pos; i++) + new_array->children[i] = old_array->children[i]; + new_array->children[pos] = node; + for (i = pos; i < nr_old; i++) + new_array->children[i + 1] = old_array->children[i]; + + return 0; +} + unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries) { diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index e917eebf22ff9..b2ab42f6b25c1 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -54,6 +54,25 @@ static unsigned int tfetch(const void *leaf, unsigned long *entries, nr_scratch); } +static int child_array_init(void *storage, size_t storage_size, + const void * const *children, unsigned int nr_children) +{ + return __stack_depot_trie_child_array_init(storage, storage_size, children, + nr_children); +} + +static int child_array_insert(const void *old_storage, const void *child, + void *new_storage, size_t new_storage_size) +{ + return __stack_depot_trie_child_array_insert(old_storage, child, + new_storage, new_storage_size); +} + +static const void *child_array_find(const void *storage, unsigned long frame) +{ + return __stack_depot_trie_child_array_find(storage, frame); +} + static void trie_node_alloc(struct kunit *test, const unsigned long *entries, unsigned int nr_entries, const void *parent, u32 leaf_id, @@ -194,15 +213,14 @@ static void stackdepot_count_helpers(struct kunit *test) KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(handle, INT_MAX)); - KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(handle, INT_MAX - 1)); KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 1)); KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); max_handle = stack_depot_save(max_entries, ARRAY_SIZE(max_entries), GFP_KERNEL); KUNIT_ASSERT_NE(test, max_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(max_handle, INT_MAX - 2)); + KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(max_handle, INT_MAX - 1)); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(max_handle, &count)); - KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX - 1); + KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX); KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 2)); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); @@ -216,9 +234,18 @@ static void stackdepot_count_helpers(struct kunit *test) KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 5)); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 2); + KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 3)); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 2); __stack_depot_set_count(handle, 0); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 2); + __stack_depot_set_count(handle, INT_MAX - 1); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX - 1); + __stack_depot_set_count(handle, 2); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 2); __stack_depot_set_count(handle, INT_MAX); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 2); @@ -613,6 +640,124 @@ static void stackdepot_trie_fetch_rejects_bad_inputs(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); } +static void stackdepot_trie_child_array_init_find(struct kunit *test) +{ + unsigned long first_entries[] = { 0x1000UL }; + unsigned long second_entries[] = { 0x2000UL }; + unsigned long third_entries[] = { 0x3000UL }; + const void *children[3]; + void *node; + void *array; + size_t size; + int ret; + + trie_node_alloc(test, first_entries, ARRAY_SIZE(first_entries), NULL, 1, + &node); + children[0] = node; + trie_node_alloc(test, second_entries, ARRAY_SIZE(second_entries), NULL, 2, + &node); + children[1] = node; + trie_node_alloc(test, third_entries, ARRAY_SIZE(third_entries), NULL, 3, + &node); + children[2] = node; + + size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); + KUNIT_ASSERT_GT(test, size, (size_t)0); + array = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, array); + ret = child_array_init(array, size, children, ARRAY_SIZE(children)); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, 0x1000UL), children[0]); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, 0x2000UL), children[1]); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, 0x3000UL), children[2]); + KUNIT_EXPECT_NULL(test, child_array_find(array, 0x4000UL)); + KUNIT_EXPECT_NULL(test, child_array_find(NULL, 0x1000UL)); +} + +static void stackdepot_trie_child_array_rejects_unsorted(struct kunit *test) +{ + unsigned long first_entries[] = { 0x2000UL }; + unsigned long second_entries[] = { 0x1000UL }; + const void *children[2]; + void *node; + void *array; + size_t size; + int ret; + + trie_node_alloc(test, first_entries, ARRAY_SIZE(first_entries), NULL, 1, + &node); + children[0] = node; + trie_node_alloc(test, second_entries, ARRAY_SIZE(second_entries), NULL, 2, + &node); + children[1] = node; + size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); + array = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, array); + ret = child_array_init(array, size, children, ARRAY_SIZE(children)); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void stackdepot_trie_child_array_insert(struct kunit *test) +{ + unsigned long first_entries[] = { 0x1000UL }; + unsigned long second_entries[] = { 0x3000UL }; + unsigned long middle_entries[] = { 0x2000UL }; + const void *children[2]; + void *old_array; + void *new_array; + void *middle; + void *node; + size_t old_size; + size_t new_size; + int ret; + + trie_node_alloc(test, first_entries, ARRAY_SIZE(first_entries), NULL, 1, + &node); + children[0] = node; + trie_node_alloc(test, second_entries, ARRAY_SIZE(second_entries), NULL, 2, + &node); + children[1] = node; + trie_node_alloc(test, middle_entries, ARRAY_SIZE(middle_entries), NULL, 3, + &middle); + old_size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); + new_size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children) + 1); + old_array = kunit_kzalloc(test, old_size, GFP_KERNEL); + new_array = kunit_kzalloc(test, new_size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array); + KUNIT_ASSERT_NOT_NULL(test, new_array); + KUNIT_ASSERT_EQ(test, + child_array_init(old_array, old_size, children, + ARRAY_SIZE(children)), + 0); + + ret = child_array_insert(old_array, middle, new_array, new_size); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array, 0x1000UL), children[0]); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array, 0x2000UL), middle); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array, 0x3000UL), children[1]); + ret = child_array_insert(old_array, children[0], new_array, new_size); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = child_array_insert(old_array, middle, old_array, old_size); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void stackdepot_trie_child_array_insert_empty(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + void *new_array; + void *child; + size_t size; + int ret; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 1, &child); + size = __stack_depot_trie_child_array_size(1); + new_array = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array); + ret = child_array_insert(NULL, child, new_array, size); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array, 0x1000UL), child); +} + static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), @@ -641,6 +786,10 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_node_rejects_mixed_run), #endif KUNIT_CASE(stackdepot_trie_fetch_rejects_bad_inputs), + KUNIT_CASE(stackdepot_trie_child_array_init_find), + KUNIT_CASE(stackdepot_trie_child_array_rejects_unsorted), + KUNIT_CASE(stackdepot_trie_child_array_insert), + KUNIT_CASE(stackdepot_trie_child_array_insert_empty), {} }; diff --git a/mm/page_owner.c b/mm/page_owner.c index 233331867fcdb..fe3427c3b4aa9 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -892,6 +892,7 @@ static int stack_print(struct seq_file *m, void *v) if (!handle) return 0; + /* Counts can race with page_owner updates; seq_file output is best effort. */ if (!__stack_depot_get_count(handle, &nr_base_pages) || nr_base_pages <= 1) return 0; nr_base_pages--; From 78463e7b1487824777f48de304cd4335e2ddc812 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 29 May 2026 17:13:01 +0100 Subject: [PATCH 015/129] KRN-1117: Harden stackdepot trie helper contracts Tighten the private trie helper contracts after adding child arrays. Reject overlapping frame-run buffers, validate old child-array storage alignment, and use an explicit ULONG_MAX arm64 prefix bound. Keep the page_owner count helper limits consistent across set/inc paths and document best-effort page_owner stack listing semantics. Extend stackdepot KUnit coverage for arm64 compressed frame-run round trips, compressed trie nodes without scratch storage, count-boundary handling, and child-array edge cases. The trie helpers remain inert: they only build caller-owned unpublished storage and do not change stack_depot_save() or public fetch routing. Signed-off-by: Caleb Kan --- arch/arm64/include/asm/stackdepot.h | 3 +- include/linux/stackdepot.h | 11 +++-- lib/stackdepot.c | 48 ++++++++++++++++------ lib/tests/stackdepot_kunit.c | 62 +++++++++++++++++++++++++++++ mm/page_owner.c | 1 + 5 files changed, 108 insertions(+), 17 deletions(-) diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h index 9491413c4a1d5..81a875a42e039 100644 --- a/arch/arm64/include/asm/stackdepot.h +++ b/arch/arm64/include/asm/stackdepot.h @@ -4,6 +4,7 @@ #include #include +#include #include #define STACK_DEPOT_ARM64_FRAME_LOW_MASK 0x00000000ffffffffUL @@ -39,7 +40,7 @@ static inline bool arch_stack_depot_frame_prefix(u8 prefix_id, *prefix = text_prefix; return true; case STACK_DEPOT_ARM64_NEXT_PREFIX_ID: - if (text_prefix > ~0UL - SZ_4G) + if (text_prefix > ULONG_MAX - SZ_4G) return false; *prefix = text_prefix + SZ_4G; return true; diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 6eabb4f25ad08..5ec7d726d61ee 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -159,6 +159,7 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, * @count: Pointer to store the count * * This function is only for internal purposes. + * The returned count is an unsynchronized snapshot for diagnostics. * * Return: true on success, false if @handle is invalid, @count is NULL, or the * stack record is not in counted mode. @@ -172,7 +173,7 @@ bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count); * @count: Count to set * * This function is only for internal purposes. - * If @count is 0 or greater than or equal to %INT_MAX, this function is a + * If @count is 0 or greater than %INT_MAX, this function is a * no-op. * Callers that use this to switch a saturated record to counted mode must * separately make the record discoverable by their own tracking structure. @@ -187,7 +188,8 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); * @count: Count to add * * This function is only for internal purposes. - * @count must be greater than 0 and less than or equal to %INT_MAX - 1. + * If @count is 0, this function is a no-op. Otherwise @count must be less + * than or equal to %INT_MAX - 1. * * Persistent stack records start with refcount set to %REFCOUNT_SATURATED. If * this helper switches a saturated record to counted mode, it stores @count + 1. @@ -215,7 +217,8 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count); * Return: true if the resulting count is 0, false if the resulting count is * non-zero, @handle is invalid, the stack record is not in counted mode, or * @count is greater than the current count. Saturated persistent records are - * not in counted mode and fail closed without changing the record. + * not in counted mode and fail closed without changing the record. Underflow + * attempts warn and leave the count unchanged. */ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, unsigned int count); @@ -284,7 +287,7 @@ int __stack_depot_frame_run_init(const unsigned long *entries, * This function is only for internal purposes. It does not write partial * compressed payloads: if any frame does not match @run, @dst is unchanged. * Compressed runs require @scratch to hold at least @run->nr_entries entries; - * raw runs do not use @scratch. + * raw runs do not use @scratch. @dst must not overlap @entries. * * Return: 0 on success, -EINVAL on invalid input. */ diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 1de5365386169..55405b649e1bd 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -821,7 +821,7 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count) struct stack_record *stack; /* Reject values outside positive refcount space. */ - if (!handle || !count || count >= (unsigned int)INT_MAX) + if (!handle || !count || count > (unsigned int)INT_MAX) return; stack = depot_fetch_stack(handle); @@ -852,10 +852,10 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) * page_owner count. The cmpxchg only performs that one-way transition; * normal counted records continue through refcount_add(). */ - if (atomic_try_cmpxchg_release(&stack->count.refs, &old, new)) + if (atomic_try_cmpxchg(&stack->count.refs, &old, new)) was_saturated = true; else - /* Existing counted records only need refcount_add()'s relaxed ordering. */ + /* Another caller may have won the transition; count this caller too. */ refcount_add((int)count, &stack->count); return was_saturated; @@ -935,6 +935,9 @@ bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, return frame_decompress(prefix_id, low, frame); } +static bool stack_depot_ranges_overlap(const void *a, size_t a_size, + const void *b, size_t b_size); + static size_t stack_depot_frame_run_entry_bytes(enum stack_depot_frame_mode mode) { if (mode == STACK_DEPOT_FRAME_COMPRESSED) @@ -1025,6 +1028,8 @@ stack_depot_frame_run_write_compressed(const struct stack_depot_frame_run *run, if (!scratch || nr_scratch < run->nr_entries) return -EINVAL; + if (stack_depot_ranges_overlap(dst, run->bytes, scratch, run->bytes)) + return -EINVAL; for (i = 0; i < run->nr_entries; i++) { u8 prefix_id; @@ -1053,6 +1058,9 @@ static int frame_run_write(const struct stack_depot_frame_run *run, return ret; if (dst_size < run->bytes) return -EINVAL; + if (stack_depot_ranges_overlap(dst, run->bytes, entries, + run->nr_entries * sizeof(*entries))) + return -EINVAL; if (run->mode == STACK_DEPOT_FRAME_RAW) { memcpy(dst, entries, run->bytes); @@ -1100,21 +1108,23 @@ static int frame_run_read_to_scratch(const struct stack_depot_frame_run *run, const void *src, unsigned long *scratch, unsigned int nr_scratch) { + /* Private staging helper: the public frame-run read API rejects aliasing. */ return stack_depot_frame_run_read_compressed(run, src, scratch, scratch, nr_scratch); } -static bool stack_depot_ranges_overlap(const void *a, const void *b, size_t size) +static bool stack_depot_ranges_overlap(const void *a, size_t a_size, + const void *b, size_t b_size) { unsigned long a_start = (unsigned long)a; unsigned long b_start = (unsigned long)b; unsigned long a_end; unsigned long b_end; - if (!size) + if (!a_size || !b_size) return false; - if (check_add_overflow(a_start, size, &a_end) || - check_add_overflow(b_start, size, &b_end)) + if (check_add_overflow(a_start, a_size, &a_end) || + check_add_overflow(b_start, b_size, &b_end)) return true; return a_start < b_end && b_start < a_end; @@ -1135,6 +1145,10 @@ static int frame_run_read(const struct stack_depot_frame_run *run, return ret; if (src_size < run->bytes || max_entries < run->nr_entries) return -EINVAL; + if (stack_depot_ranges_overlap(entries, + run->nr_entries * sizeof(*entries), src, + run->bytes)) + return -EINVAL; if (run->mode == STACK_DEPOT_FRAME_RAW) { memcpy(entries, src, run->bytes); @@ -1142,8 +1156,9 @@ static int frame_run_read(const struct stack_depot_frame_run *run, } if (!scratch || nr_scratch < run->nr_entries) return -EINVAL; - if (stack_depot_ranges_overlap(entries, scratch, - run->nr_entries * sizeof(*entries))) + if (stack_depot_ranges_overlap(entries, + run->nr_entries * sizeof(*entries), scratch, + run->nr_entries * sizeof(*scratch))) return -EINVAL; return stack_depot_frame_run_read_compressed(run, src, entries, scratch, @@ -1261,6 +1276,9 @@ __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, total = node->stack_len; if (max_entries < total || nr_scratch < total) return 0; + if (stack_depot_ranges_overlap(entries, total * sizeof(*entries), scratch, + total * sizeof(*scratch))) + return 0; pos = total; for (node = leaf; node; node = node->parent) { @@ -1344,6 +1362,8 @@ stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *ar if (!array || !pos || !found) return -EINVAL; + *pos = 0; + *found = false; right = array->nr_children; while (left < right) { @@ -1364,7 +1384,6 @@ stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *ar } *pos = left; - *found = false; return 0; } @@ -1397,6 +1416,7 @@ __stack_depot_trie_child_array_insert(const void *old_storage, const void *child unsigned int i; unsigned long frame; size_t old_size; + bool overlaps; bool found; if (!node || !new_array || stack_depot_trie_node_first_frame(node, &frame)) @@ -1407,12 +1427,16 @@ __stack_depot_trie_child_array_insert(const void *old_storage, const void *child return -EINVAL; if (old_array == new_array) return -EINVAL; + if (old_array && !IS_ALIGNED((unsigned long)old_array, __alignof__(*old_array))) + return -EINVAL; nr_old = old_array ? old_array->nr_children : 0; if (new_storage_size < __stack_depot_trie_child_array_size(nr_old + 1)) return -EINVAL; old_size = __stack_depot_trie_child_array_size(nr_old); - if (old_array && stack_depot_ranges_overlap(old_array, new_array, old_size)) + overlaps = old_array && stack_depot_ranges_overlap(old_array, old_size, + new_array, new_storage_size); + if (overlaps) return -EINVAL; if (old_array) { @@ -1475,7 +1499,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, if (!handle || !entries || !max_entries) return 0; - /* Follow stackdepot's non-traceable RCU read-side convention while copying. */ + /* Defer stack record reuse while copying from stackdepot-owned storage. */ rcu_read_lock_sched_notrace(); nr_entries = stack_depot_fetch(handle, &stack_entries); if (!nr_entries || nr_entries > max_entries) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index b2ab42f6b25c1..955625945523b 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -248,6 +248,9 @@ static void stackdepot_count_helpers(struct kunit *test) KUNIT_EXPECT_EQ(test, count, 2); __stack_depot_set_count(handle, INT_MAX); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX); + __stack_depot_set_count(handle, 2); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 2); __stack_depot_set_count(handle, 6); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); @@ -402,6 +405,37 @@ static void stackdepot_frame_run_raw_roundtrip(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); } +#ifdef CONFIG_ARM64 +static void stackdepot_frame_run_arm64_roundtrip(struct kunit *test) +{ + unsigned long entries[] = { + arch_stack_depot_frame_text_prefix() | 0x1000UL, + arch_stack_depot_frame_text_prefix() | 0x2000UL, + }; + unsigned long read_scratch[ARRAY_SIZE(entries)]; + unsigned long out[ARRAY_SIZE(entries)] = {}; + struct stack_depot_frame_run run; + u32 payload[ARRAY_SIZE(entries)]; + u32 write_scratch[ARRAY_SIZE(entries)]; + int ret; + + ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, run.mode, STACK_DEPOT_FRAME_COMPRESSED); + KUNIT_EXPECT_EQ(test, run.nr_entries, + (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_EQ(test, run.bytes, sizeof(payload)); + + ret = frame_run_write(&run, entries, payload, sizeof(payload), + write_scratch, ARRAY_SIZE(write_scratch)); + KUNIT_EXPECT_EQ(test, ret, 0); + ret = frame_run_read(&run, payload, run.bytes, out, ARRAY_SIZE(out), + read_scratch, ARRAY_SIZE(read_scratch)); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); +} +#endif + #ifdef CONFIG_X86_64 static void stackdepot_frame_run_x86_64_roundtrip(struct kunit *test) { @@ -569,6 +603,30 @@ static void stackdepot_trie_node_compressed_roundtrip(struct kunit *test) KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); } + +static void stackdepot_trie_node_rejects_compressed_without_scratch(struct kunit *test) +{ + unsigned long entries[] = { +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x1000UL, +#else + 0xffffffff81001000UL, +#endif + }; + struct stack_depot_frame_run run; + void *storage; + size_t size; + int ret; + + KUNIT_ASSERT_EQ(test, frame_run_init(entries, ARRAY_SIZE(entries), &run), 0); + size = __stack_depot_trie_node_size(&run); + KUNIT_ASSERT_GT(test, size, (size_t)0); + storage = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, storage); + ret = tnode_init(storage, size, NULL, 1, entries, ARRAY_SIZE(entries), + NULL, 0); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} #endif static void stackdepot_trie_node_rejects_short_storage(struct kunit *test) @@ -770,6 +828,9 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_frame_arm64), #endif KUNIT_CASE(stackdepot_frame_run_raw_roundtrip), +#ifdef CONFIG_ARM64 + KUNIT_CASE(stackdepot_frame_run_arm64_roundtrip), +#endif #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_run_x86_64_roundtrip), KUNIT_CASE(stackdepot_frame_run_x86_64_boundary), @@ -780,6 +841,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_node_parent_chain), #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) KUNIT_CASE(stackdepot_trie_node_compressed_roundtrip), + KUNIT_CASE(stackdepot_trie_node_rejects_compressed_without_scratch), #endif KUNIT_CASE(stackdepot_trie_node_rejects_short_storage), #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) diff --git a/mm/page_owner.c b/mm/page_owner.c index fe3427c3b4aa9..d9bb15f58e078 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -893,6 +893,7 @@ static int stack_print(struct seq_file *m, void *v) return 0; /* Counts can race with page_owner updates; seq_file output is best effort. */ + /* A count of 1 is only the stack_list membership marker. */ if (!__stack_depot_get_count(handle, &nr_base_pages) || nr_base_pages <= 1) return 0; nr_base_pages--; From 60a1fa05d10dcfd8900b250badac020b7e46fc85 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 1 Jun 2026 15:37:17 +0100 Subject: [PATCH 016/129] KRN-1117: Match stackdepot trie node prefixes Add the private trie node matching primitive from the userspace prototype. It decodes one raw or compressed node run and returns the matching prefix length against caller-provided frames without walking parent or child links. Extend stackdepot KUnit coverage for raw and compressed exact matches, partial matches, shorter and longer inputs, first-frame mismatches, and invalid inputs. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 17 ++++++++++ lib/stackdepot.c | 44 ++++++++++++++++++++++--- lib/tests/stackdepot_kunit.c | 62 ++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 5 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 5ec7d726d61ee..7349de3489a1b 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -357,6 +357,23 @@ int __stack_depot_trie_node_init(void *storage, size_t storage_size, unsigned int nr_entries, u32 *scratch, unsigned int nr_scratch); +/** + * __stack_depot_trie_node_match - Match entries against one trie node + * + * @node: Trie node to compare + * @entries: Stack frames to match from the start of @node + * @nr_entries: Number of frames available in @entries + * + * This function is only for internal purposes. It compares @entries against the + * decoded frame run stored in @node and does not walk parent or child links. + * + * Return: Number of matching frames, up to the smaller of the node run length + * and @nr_entries. Returns 0 on invalid input or a first-frame mismatch. + */ +unsigned int __stack_depot_trie_node_match(const void *node, + const unsigned long *entries, + unsigned int nr_entries); + /** * __stack_depot_trie_fetch_into - Materialize a trie parent chain * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 55405b649e1bd..27654f3ad651b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1189,26 +1189,35 @@ size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) } static int -stack_depot_trie_node_first_frame(const struct stack_depot_trie_node *node, - unsigned long *frame) +stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, + unsigned int index, unsigned long *frame) { u32 low; - if (!node || !frame || stack_depot_frame_run_validate(&node->run)) + if (!node || !frame || stack_depot_frame_run_validate(&node->run) || + index >= node->run.nr_entries) return -EINVAL; if (node->run.mode == STACK_DEPOT_FRAME_RAW) { - memcpy(frame, node->data, sizeof(*frame)); + memcpy(frame, node->data + index * sizeof(*frame), + sizeof(*frame)); return 0; } - memcpy(&low, node->data, sizeof(low)); + memcpy(&low, node->data + index * sizeof(low), sizeof(low)); if (!frame_decompress(node->run.prefix_id, low, frame)) return -EINVAL; return 0; } +static int +stack_depot_trie_node_first_frame(const struct stack_depot_trie_node *node, + unsigned long *frame) +{ + return stack_depot_trie_node_frame(node, 0, frame); +} + int __stack_depot_trie_node_init(void *storage, size_t storage_size, const void *parent, u32 leaf_id, const unsigned long *entries, @@ -1260,6 +1269,31 @@ int __stack_depot_trie_node_init(void *storage, size_t storage_size, return 0; } +unsigned int __stack_depot_trie_node_match(const void *node_ptr, + const unsigned long *entries, + unsigned int nr_entries) +{ + const struct stack_depot_trie_node *node = node_ptr; + unsigned int limit; + unsigned int i; + + if (!node || !entries || !nr_entries || + stack_depot_frame_run_validate(&node->run)) + return 0; + + limit = min(node->run.nr_entries, nr_entries); + for (i = 0; i < limit; i++) { + unsigned long frame; + + if (stack_depot_trie_node_frame(node, i, &frame)) + return 0; + if (frame != entries[i]) + break; + } + + return i; +} + unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries, unsigned long *scratch, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 955625945523b..f058c01b81a65 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -54,6 +54,12 @@ static unsigned int tfetch(const void *leaf, unsigned long *entries, nr_scratch); } +static unsigned int tmatch(const void *node, const unsigned long *entries, + unsigned int nr_entries) +{ + return __stack_depot_trie_node_match(node, entries, nr_entries); +} + static int child_array_init(void *storage, size_t storage_size, const void * const *children, unsigned int nr_children) { @@ -581,6 +587,33 @@ static void stackdepot_trie_node_parent_chain(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } +static void stackdepot_trie_node_match_raw(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + unsigned long mismatch[] = { 0x1000UL, 0x2222UL, 0x3000UL }; + unsigned long short_input[] = { 0x1000UL, 0x2000UL }; + unsigned long long_input[] = { + 0x1000UL, 0x2000UL, 0x3000UL, 0x4000UL, + }; + unsigned long first_mismatch[] = { 0x9000UL, 0x2000UL }; + unsigned int matched; + void *node; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 7, &node); + KUNIT_EXPECT_EQ(test, tmatch(node, entries, ARRAY_SIZE(entries)), + (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_EQ(test, tmatch(node, mismatch, ARRAY_SIZE(mismatch)), 1U); + KUNIT_EXPECT_EQ(test, tmatch(node, short_input, ARRAY_SIZE(short_input)), + (unsigned int)ARRAY_SIZE(short_input)); + KUNIT_EXPECT_EQ(test, tmatch(node, long_input, ARRAY_SIZE(long_input)), + (unsigned int)ARRAY_SIZE(entries)); + matched = tmatch(node, first_mismatch, ARRAY_SIZE(first_mismatch)); + KUNIT_EXPECT_EQ(test, matched, 0U); + KUNIT_EXPECT_EQ(test, tmatch(NULL, entries, ARRAY_SIZE(entries)), 0U); + KUNIT_EXPECT_EQ(test, tmatch(node, NULL, ARRAY_SIZE(entries)), 0U); + KUNIT_EXPECT_EQ(test, tmatch(node, entries, 0), 0U); +} + #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) static void stackdepot_trie_node_compressed_roundtrip(struct kunit *test) { @@ -604,6 +637,33 @@ static void stackdepot_trie_node_compressed_roundtrip(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); } +static void stackdepot_trie_node_match_compressed(struct kunit *test) +{ + unsigned long entries[] = { +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x1000UL, + arch_stack_depot_frame_text_prefix() | 0x2000UL, +#else + 0xffffffff81001000UL, + 0xffffffff81002000UL, +#endif + }; + unsigned long mismatch[] = { + entries[0], +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x3000UL, +#else + 0xffffffff81003000UL, +#endif + }; + void *node; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 11, &node); + KUNIT_EXPECT_EQ(test, tmatch(node, entries, ARRAY_SIZE(entries)), + (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_EQ(test, tmatch(node, mismatch, ARRAY_SIZE(mismatch)), 1U); +} + static void stackdepot_trie_node_rejects_compressed_without_scratch(struct kunit *test) { unsigned long entries[] = { @@ -839,8 +899,10 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_frame_run_invalid_inputs), KUNIT_CASE(stackdepot_trie_node_raw_roundtrip), KUNIT_CASE(stackdepot_trie_node_parent_chain), + KUNIT_CASE(stackdepot_trie_node_match_raw), #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) KUNIT_CASE(stackdepot_trie_node_compressed_roundtrip), + KUNIT_CASE(stackdepot_trie_node_match_compressed), KUNIT_CASE(stackdepot_trie_node_rejects_compressed_without_scratch), #endif KUNIT_CASE(stackdepot_trie_node_rejects_short_storage), From edfe0dd03dc0bd45922aac13828ab2bb3e86e2a7 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 1 Jun 2026 16:26:09 +0100 Subject: [PATCH 017/129] KRN-1117: Build stackdepot trie append chains Add the private append-chain primitive from the userspace trie prototype. The helper builds unpublished raw or compressed trie nodes from caller-owned storage, splitting chains at frame-run boundaries and linking newly-created nodes with one-child arrays. Keep the helper inert for now: it does not publish nodes, allocate memory, or change stack_depot_save() and fetch routing. Cover raw chains, parent chains, frame-run boundary splits, and invalid storage inputs in stackdepot KUnit. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 45 ++++++++ lib/stackdepot.c | 207 +++++++++++++++++++++++++++++++++++ lib/tests/stackdepot_kunit.c | 192 ++++++++++++++++++++++++++++++++ 3 files changed, 444 insertions(+) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 7349de3489a1b..f4de9a92bb6e7 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -63,6 +63,16 @@ struct stack_depot_frame_run { size_t bytes; }; +struct stack_depot_trie_node_slot { + void *node; + size_t size; +}; + +struct stack_depot_trie_child_array_slot { + void *array; + size_t size; +}; + /* * Using stack depot requires its initialization, which can be done in 3 ways: * @@ -374,6 +384,41 @@ unsigned int __stack_depot_trie_node_match(const void *node, const unsigned long *entries, unsigned int nr_entries); +/** + * __stack_depot_trie_append_chain - Build an unpublished node chain + * + * @parent: Parent node for the new chain, or NULL for a root chain + * @leaf_id: Non-zero id to store in the final node + * @entries: Stack frames to store in the chain + * @nr_entries: Number of frames in @entries + * @node_slots: Caller-owned storage slots for trie nodes + * @nr_node_slots: Number of entries in @node_slots + * @child_slots: Caller-owned storage slots for one-child arrays + * @nr_child_slots: Number of entries in @child_slots + * @scratch: Scratch buffer for compressed frame payloads + * @nr_scratch: Number of 32-bit entries that fit in @scratch + * @head: Storage for the first node in the chain + * @tail: Storage for the final node in the chain + * @nr_used: Storage for the number of node slots consumed + * + * This function is only for internal purposes. It builds nodes and one-child + * arrays in caller-owned unpublished storage, splitting the input at raw / + * compressed mode and compressed-prefix boundaries. Callers remain responsible + * for lifetime and visibility. + * + * Return: 0 on success, -EINVAL on invalid input. + */ +int +__stack_depot_trie_append_chain(const void *parent, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, const void **head, + const void **tail, unsigned int *nr_used); + /** * __stack_depot_trie_fetch_into - Materialize a trie parent chain * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 27654f3ad651b..3a64dc39a9326 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -96,6 +96,7 @@ struct stack_record { struct stack_depot_trie_node { const struct stack_depot_trie_node *parent; + const struct stack_depot_trie_child_array *children; u32 leaf_id; u32 stack_len; struct stack_depot_frame_run run; @@ -1263,6 +1264,7 @@ int __stack_depot_trie_node_init(void *storage, size_t storage_size, memcpy(node->data, entries, run.bytes); node->parent = parent_node; + node->children = NULL; node->leaf_id = leaf_id; node->stack_len = stack_len; node->run = run; @@ -1294,6 +1296,211 @@ unsigned int __stack_depot_trie_node_match(const void *node_ptr, return i; } +static bool trie_ancestor_overlaps(const struct stack_depot_trie_node *node, + const void *ptr, size_t size) +{ + for (; node; node = node->parent) { + size_t child_size; + size_t node_size; + + if (stack_depot_frame_run_validate(&node->run)) + return true; + + node_size = __stack_depot_trie_node_size(&node->run); + if (!node_size) + return true; + if (stack_depot_ranges_overlap(ptr, size, node, node_size)) + return true; + + if (!node->children) + continue; + child_size = __stack_depot_trie_child_array_size(node->children->nr_children); + if (!child_size) + return true; + if (stack_depot_ranges_overlap(ptr, size, node->children, + child_size)) + return true; + } + + return false; +} + +static bool +trie_node_slot_overlaps(const struct stack_depot_trie_node_slot *slots, + unsigned int used, const void *ptr, size_t size) +{ + unsigned int i; + + for (i = 0; i < used; i++) { + if (stack_depot_ranges_overlap(ptr, size, slots[i].node, + slots[i].size)) + return true; + } + + return false; +} + +static bool +trie_child_slot_overlaps(const struct stack_depot_trie_child_array_slot *slots, + unsigned int used, const void *ptr, size_t size) +{ + unsigned int i; + + for (i = 0; i < used; i++) { + if (stack_depot_ranges_overlap(ptr, size, slots[i].array, + slots[i].size)) + return true; + } + + return false; +} + +static int trie_append_chain_validate(const struct stack_depot_trie_node *parent, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, unsigned int *nr_runs) +{ + unsigned int child_slots_needed; + unsigned int pos = 0; + unsigned int used = 0; + u32 stack_len = parent ? parent->stack_len : 0; + + if (!entries || !nr_entries || !node_slots || !nr_runs) + return -EINVAL; + if (parent && !parent->stack_len) + return -EINVAL; + + while (pos < nr_entries) { + const struct stack_depot_trie_node_slot *slot; + struct stack_depot_frame_run run; + size_t size; + + if (__stack_depot_frame_run_init(&entries[pos], nr_entries - pos, + &run)) + return -EINVAL; + if (used >= nr_node_slots) + return -EINVAL; + slot = &node_slots[used]; + if (!slot->node) + return -EINVAL; + if (run.mode == STACK_DEPOT_FRAME_COMPRESSED && + (!scratch || nr_scratch < run.nr_entries)) + return -EINVAL; + if (stack_len > U32_MAX - run.nr_entries) + return -EINVAL; + + size = __stack_depot_trie_node_size(&run); + if (slot->size < size) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)slot->node, + __alignof__(struct stack_depot_trie_node))) + return -EINVAL; + if (trie_ancestor_overlaps(parent, slot->node, slot->size)) + return -EINVAL; + if (trie_node_slot_overlaps(node_slots, used, slot->node, slot->size)) + return -EINVAL; + + stack_len += run.nr_entries; + pos += run.nr_entries; + used++; + } + + child_slots_needed = used > 1 ? used - 1 : 0; + if (child_slots_needed) { + unsigned int i; + + if (!child_slots || nr_child_slots < child_slots_needed) + return -EINVAL; + for (i = 0; i < child_slots_needed; i++) { + unsigned long addr = (unsigned long)child_slots[i].array; + + if (!child_slots[i].array || child_slots[i].size < + __stack_depot_trie_child_array_size(1)) + return -EINVAL; + if (!IS_ALIGNED(addr, + __alignof__(struct stack_depot_trie_child_array))) + return -EINVAL; + if (trie_ancestor_overlaps(parent, child_slots[i].array, + child_slots[i].size)) + return -EINVAL; + if (trie_node_slot_overlaps(node_slots, used, + child_slots[i].array, + child_slots[i].size)) + return -EINVAL; + if (trie_child_slot_overlaps(child_slots, i, + child_slots[i].array, + child_slots[i].size)) + return -EINVAL; + } + } + + *nr_runs = used; + return 0; +} + +int +__stack_depot_trie_append_chain(const void *parent_ptr, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, const void **head, + const void **tail, unsigned int *nr_used) +{ + const struct stack_depot_trie_node *parent = parent_ptr; + const struct stack_depot_trie_node *prev = parent; + unsigned int pos = 0; + unsigned int used; + unsigned int i; + + if (!leaf_id || !head || !tail || !nr_used) + return -EINVAL; + if (trie_append_chain_validate(parent, entries, nr_entries, node_slots, + nr_node_slots, child_slots, nr_child_slots, + scratch, nr_scratch, &used)) + return -EINVAL; + + for (i = 0; i < used; i++) { + struct stack_depot_frame_run run; + struct stack_depot_trie_node *node = node_slots[i].node; + u32 id; + + if (__stack_depot_frame_run_init(&entries[pos], nr_entries - pos, + &run)) + return -EINVAL; + id = pos + run.nr_entries == nr_entries ? leaf_id : 0; + if (__stack_depot_trie_node_init(node, node_slots[i].size, prev, + id, &entries[pos], run.nr_entries, + scratch, nr_scratch)) + return -EINVAL; + + prev = node; + pos += run.nr_entries; + } + + for (i = 0; i + 1 < used; i++) { + const void *next = node_slots[i + 1].node; + struct stack_depot_trie_node *node = node_slots[i].node; + void *array = child_slots[i].array; + size_t size = child_slots[i].size; + + if (__stack_depot_trie_child_array_insert(NULL, next, array, size)) + return -EINVAL; + node->children = array; + } + + *head = node_slots[0].node; + *tail = node_slots[used - 1].node; + *nr_used = used; + return 0; +} + unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries, unsigned long *scratch, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index f058c01b81a65..81b65fbf5f51c 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -60,6 +60,20 @@ static unsigned int tmatch(const void *node, const unsigned long *entries, return __stack_depot_trie_node_match(node, entries, nr_entries); } +static int +append_chain(const void *parent, u32 leaf_id, const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, + const void **head, const void **tail, unsigned int *nr_used) +{ + return __stack_depot_trie_append_chain(parent, leaf_id, entries, + nr_entries, node_slots, nr_node_slots, child_slots, + nr_child_slots, scratch, nr_scratch, head, tail, nr_used); +} + static int child_array_init(void *storage, size_t storage_size, const void * const *children, unsigned int nr_children) { @@ -614,6 +628,74 @@ static void stackdepot_trie_node_match_raw(struct kunit *test) KUNIT_EXPECT_EQ(test, tmatch(node, entries, 0), 0U); } +static void stackdepot_trie_append_chain_raw(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + struct stack_depot_frame_run run; + struct stack_depot_trie_node_slot node_slots[1]; + unsigned long scratch[ARRAY_SIZE(entries)]; + unsigned long out[ARRAY_SIZE(entries)] = {}; + const void *head = NULL; + const void *tail = NULL; + unsigned int used = 0; + unsigned int fetched; + size_t size; + int ret; + + KUNIT_ASSERT_EQ(test, frame_run_init(entries, ARRAY_SIZE(entries), &run), 0); + size = __stack_depot_trie_node_size(&run); + node_slots[0].node = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, node_slots[0].node); + node_slots[0].size = size; + + ret = append_chain(NULL, 13, entries, ARRAY_SIZE(entries), node_slots, + ARRAY_SIZE(node_slots), NULL, 0, NULL, 0, &head, &tail, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, head, node_slots[0].node); + KUNIT_EXPECT_PTR_EQ(test, tail, node_slots[0].node); + KUNIT_EXPECT_EQ(test, used, 1U); + fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); +} + +static void stackdepot_trie_append_chain_parent(struct kunit *test) +{ + unsigned long parent_entries[] = { 0x1000UL }; + unsigned long entries[] = { 0x2000UL, 0x3000UL }; + unsigned long expected[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + struct stack_depot_frame_run run; + struct stack_depot_trie_node_slot node_slots[1]; + unsigned long scratch[ARRAY_SIZE(expected)]; + unsigned long out[ARRAY_SIZE(expected)] = {}; + const void *head = NULL; + const void *tail = NULL; + unsigned int used = 0; + unsigned int fetched; + void *parent; + size_t size; + int ret; + + trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, + &parent); + KUNIT_ASSERT_EQ(test, frame_run_init(entries, ARRAY_SIZE(entries), &run), 0); + size = __stack_depot_trie_node_size(&run); + node_slots[0].node = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, node_slots[0].node); + node_slots[0].size = size; + + ret = append_chain(parent, 14, entries, ARRAY_SIZE(entries), node_slots, + ARRAY_SIZE(node_slots), NULL, 0, NULL, 0, &head, &tail, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, head, tail); + KUNIT_EXPECT_EQ(test, used, 1U); + fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); + KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); +} + #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) static void stackdepot_trie_node_compressed_roundtrip(struct kunit *test) { @@ -664,6 +746,112 @@ static void stackdepot_trie_node_match_compressed(struct kunit *test) KUNIT_EXPECT_EQ(test, tmatch(node, mismatch, ARRAY_SIZE(mismatch)), 1U); } +static void stackdepot_trie_append_chain_splits_frame_runs(struct kunit *test) +{ + unsigned long entries[] = { +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x1000UL, + arch_stack_depot_frame_text_prefix() | 0x2000UL, + 0x1000UL, + arch_stack_depot_frame_text_prefix() | 0x3000UL, +#else + 0xffffffff81001000UL, + 0xffffffff81002000UL, + 0xffff888000001000UL, + 0xffffffff81003000UL, +#endif + }; + struct stack_depot_trie_node_slot node_slots[3]; + struct stack_depot_trie_child_array_slot child_slots[2]; + unsigned long read_scratch[ARRAY_SIZE(entries)]; + unsigned long out[ARRAY_SIZE(entries)] = {}; + const void *child; + u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; + const void *head = NULL; + const void *tail = NULL; + unsigned int used = 0; + unsigned int fetched; + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(node_slots); i++) { + size_t size; + + node_slots[i].size = 128; + size = node_slots[i].size; + node_slots[i].node = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, node_slots[i].node); + } + for (i = 0; i < ARRAY_SIZE(child_slots); i++) { + size_t size; + + child_slots[i].size = __stack_depot_trie_child_array_size(1); + size = child_slots[i].size; + child_slots[i].array = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_slots[i].array); + } + + ret = append_chain(NULL, 15, entries, ARRAY_SIZE(entries), node_slots, + ARRAY_SIZE(node_slots), child_slots, ARRAY_SIZE(child_slots), + write_scratch, ARRAY_SIZE(write_scratch), &head, &tail, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, head, node_slots[0].node); + KUNIT_EXPECT_PTR_EQ(test, tail, node_slots[2].node); + KUNIT_EXPECT_EQ(test, used, 3U); + child = child_array_find(child_slots[0].array, entries[2]); + KUNIT_EXPECT_PTR_EQ(test, child, node_slots[1].node); + child = child_array_find(child_slots[1].array, entries[3]); + KUNIT_EXPECT_PTR_EQ(test, child, node_slots[2].node); + fetched = tfetch(tail, out, ARRAY_SIZE(out), read_scratch, + ARRAY_SIZE(read_scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); +} + +static void stackdepot_trie_append_chain_rejects_bad_inputs(struct kunit *test) +{ + unsigned long entries[] = { +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x1000UL, + 0x1000UL, +#else + 0xffffffff81001000UL, + 0xffff888000001000UL, +#endif + }; + struct stack_depot_trie_node_slot node_slots[2]; + struct stack_depot_trie_child_array_slot child_slot; + u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; + const void *head = NULL; + const void *tail = NULL; + unsigned int used = 0; + int ret; + + node_slots[0].size = 128; + node_slots[0].node = kunit_kzalloc(test, node_slots[0].size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, node_slots[0].node); + node_slots[1].size = 128; + node_slots[1].node = kunit_kzalloc(test, node_slots[1].size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, node_slots[1].node); + child_slot.size = __stack_depot_trie_child_array_size(1); + child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_slot.array); + + ret = append_chain(NULL, 16, entries, ARRAY_SIZE(entries), node_slots, 1, + &child_slot, 1, write_scratch, ARRAY_SIZE(write_scratch), + &head, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = append_chain(NULL, 16, entries, ARRAY_SIZE(entries), node_slots, + ARRAY_SIZE(node_slots), NULL, 0, write_scratch, + ARRAY_SIZE(write_scratch), &head, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = append_chain(NULL, 16, entries, ARRAY_SIZE(entries), node_slots, + ARRAY_SIZE(node_slots), &child_slot, 1, NULL, 0, &head, + &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + static void stackdepot_trie_node_rejects_compressed_without_scratch(struct kunit *test) { unsigned long entries[] = { @@ -900,9 +1088,13 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_node_raw_roundtrip), KUNIT_CASE(stackdepot_trie_node_parent_chain), KUNIT_CASE(stackdepot_trie_node_match_raw), + KUNIT_CASE(stackdepot_trie_append_chain_raw), + KUNIT_CASE(stackdepot_trie_append_chain_parent), #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) KUNIT_CASE(stackdepot_trie_node_compressed_roundtrip), KUNIT_CASE(stackdepot_trie_node_match_compressed), + KUNIT_CASE(stackdepot_trie_append_chain_splits_frame_runs), + KUNIT_CASE(stackdepot_trie_append_chain_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_node_rejects_compressed_without_scratch), #endif KUNIT_CASE(stackdepot_trie_node_rejects_short_storage), From 2fe6b7a1a455b7c26af0034b975a98ccc0a22c8d Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 1 Jun 2026 17:09:31 +0100 Subject: [PATCH 018/129] KRN-1117: Publish stackdepot trie append chains Add the private append-publication primitive from the userspace trie prototype. The helper builds a replacement child array around an unpublished appended chain and installs it into caller-owned root or parent storage only after validation succeeds. Cover root publication, parent publication, duplicate rejection, root replacement, and bad input handling in stackdepot KUnit. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 27 +++++ lib/stackdepot.c | 91 ++++++++++++++++ lib/tests/stackdepot_kunit.c | 203 +++++++++++++++++++++++++++++++++++ 3 files changed, 321 insertions(+) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index f4de9a92bb6e7..da00d624711e1 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -73,6 +73,12 @@ struct stack_depot_trie_child_array_slot { size_t size; }; +struct stack_depot_trie_child_array; + +struct stack_depot_trie_root { + const struct stack_depot_trie_child_array *children; +}; + /* * Using stack depot requires its initialization, which can be done in 3 ways: * @@ -419,6 +425,27 @@ __stack_depot_trie_append_chain(const void *parent, u32 leaf_id, unsigned int nr_scratch, const void **head, const void **tail, unsigned int *nr_used); +/** + * __stack_depot_trie_publish_append - Publish an appended chain + * + * @root: Root storage to publish into, or NULL for a parent publish + * @parent: Parent node to publish under, or NULL for a root publish + * @head: First node of an unpublished chain + * @new_storage: Replacement child-array storage + * @new_storage_size: Size of @new_storage in bytes + * + * This function is only for internal purposes. It builds a replacement child + * array containing @head and stores it in either @root or @parent. @head must + * be the first node of an unpublished chain whose parent is @parent. Callers + * remain responsible for lifetime and visibility. + * + * Return: 0 on success, -EINVAL on invalid input. + */ +int +__stack_depot_trie_publish_append(struct stack_depot_trie_root *root, + void *parent, const void *head, + void *new_storage, size_t new_storage_size); + /** * __stack_depot_trie_fetch_into - Materialize a trie parent chain * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 3a64dc39a9326..13602486ead62 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1325,6 +1325,47 @@ static bool trie_ancestor_overlaps(const struct stack_depot_trie_node *node, return false; } +static bool trie_chain_overlaps(const struct stack_depot_trie_node *node, + const void *ptr, size_t size) +{ + unsigned int depth = 0; + + for (; node; depth++) { + const struct stack_depot_trie_child_array *children; + size_t child_size; + size_t node_size; + + if (depth >= CONFIG_STACKDEPOT_MAX_FRAMES) + return true; + if (stack_depot_frame_run_validate(&node->run)) + return true; + + node_size = __stack_depot_trie_node_size(&node->run); + if (!node_size) + return true; + if (stack_depot_ranges_overlap(ptr, size, node, node_size)) + return true; + + children = node->children; + if (!children) + break; + if (children->nr_children != 1) + return true; + child_size = __stack_depot_trie_child_array_size(1); + if (!child_size) + return true; + if (stack_depot_ranges_overlap(ptr, size, children, child_size)) + return true; + if (!children->children[0]) + return true; + if (children->children[0]->parent != node) + return true; + node = children->children[0]; + } + + return false; +} + static bool trie_node_slot_overlaps(const struct stack_depot_trie_node_slot *slots, unsigned int used, const void *ptr, size_t size) @@ -1501,6 +1542,56 @@ __stack_depot_trie_append_chain(const void *parent_ptr, u32 leaf_id, return 0; } +int +__stack_depot_trie_publish_append(struct stack_depot_trie_root *root, + void *parent_ptr, const void *head_ptr, + void *new_storage, size_t new_storage_size) +{ + const struct stack_depot_trie_child_array *old_array; + const struct stack_depot_trie_node *head = head_ptr; + const struct stack_depot_trie_child_array **slot; + struct stack_depot_trie_node *parent = parent_ptr; + struct stack_depot_trie_child_array *new_array = new_storage; + size_t storage_size = new_storage_size; + size_t new_size; + size_t old_size; + + if ((root && parent) || (!root && !parent) || !head || !new_array) + return -EINVAL; + if (head->parent != parent) + return -EINVAL; + + if (root) { + if (stack_depot_ranges_overlap(new_array, storage_size, + &root->children, sizeof(root->children))) + return -EINVAL; + slot = &root->children; + } else { + if (trie_ancestor_overlaps(parent, new_array, storage_size)) + return -EINVAL; + slot = &parent->children; + } + + old_array = *slot; + old_size = old_array ? + __stack_depot_trie_child_array_size(old_array->nr_children) : 0; + new_size = old_array ? old_array->nr_children + 1 : 1; + new_size = __stack_depot_trie_child_array_size(new_size); + if (!new_size || storage_size < new_size) + return -EINVAL; + if (old_array && + stack_depot_ranges_overlap(old_array, old_size, new_array, + storage_size)) + return -EINVAL; + if (trie_chain_overlaps(head, new_array, storage_size)) + return -EINVAL; + if (__stack_depot_trie_child_array_insert(old_array, head, new_array, storage_size)) + return -EINVAL; + + *slot = new_array; + return 0; +} + unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries, unsigned long *scratch, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 81b65fbf5f51c..6fa83e6a43ad9 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -74,6 +74,27 @@ append_chain(const void *parent, u32 leaf_id, const unsigned long *entries, nr_child_slots, scratch, nr_scratch, head, tail, nr_used); } +static int publish_append(struct stack_depot_trie_root *root, void *parent, + const void *head, void *storage, size_t storage_size) +{ + return __stack_depot_trie_publish_append(root, parent, head, storage, + storage_size); +} + +static void +trie_node_slot_alloc(struct kunit *test, + struct stack_depot_trie_node_slot *slot, + const unsigned long *entries, unsigned int nr_entries) +{ + struct stack_depot_frame_run run; + + KUNIT_ASSERT_EQ(test, frame_run_init(entries, nr_entries, &run), 0); + slot->size = __stack_depot_trie_node_size(&run); + KUNIT_ASSERT_GT(test, slot->size, (size_t)0); + slot->node = kunit_kzalloc(test, slot->size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, slot->node); +} + static int child_array_init(void *storage, size_t storage_size, const void * const *children, unsigned int nr_children) { @@ -696,6 +717,184 @@ static void stackdepot_trie_append_chain_parent(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } +static void stackdepot_trie_publish_append_root(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *head = NULL; + const void *tail = NULL; + unsigned int used = 0; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = append_chain(NULL, 17, entries, ARRAY_SIZE(entries), &node_slot, 1, + NULL, 0, NULL, 0, &head, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, head, child_array.array, + child_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, root.children, child_array.array); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, entries[0]), head); + KUNIT_EXPECT_PTR_EQ(test, head, tail); +} + +static void stackdepot_trie_publish_append_parent(struct kunit *test) +{ + unsigned long parent_entries[] = { 0x1000UL }; + unsigned long old_entries[] = { 0x2000UL }; + unsigned long new_entries[] = { 0x3000UL }; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot old_slot; + struct stack_depot_trie_node_slot new_slot; + const void *old_head = NULL; + const void *old_tail = NULL; + const void *new_head = NULL; + const void *new_tail = NULL; + unsigned int used = 0; + void *parent; + int ret; + + trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, + &parent); + trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); + trie_node_slot_alloc(test, &new_slot, new_entries, ARRAY_SIZE(new_entries)); + old_array.size = __stack_depot_trie_child_array_size(1); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + new_array.size = __stack_depot_trie_child_array_size(2); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + + ret = append_chain(parent, 18, old_entries, ARRAY_SIZE(old_entries), + &old_slot, 1, NULL, 0, NULL, 0, &old_head, &old_tail, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(NULL, parent, old_head, old_array.array, + old_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = append_chain(parent, 19, new_entries, ARRAY_SIZE(new_entries), + &new_slot, 1, NULL, 0, NULL, 0, &new_head, &new_tail, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(NULL, parent, new_head, new_array.array, + new_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array.array, old_entries[0]), + old_head); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array.array, new_entries[0]), + new_head); + KUNIT_EXPECT_PTR_EQ(test, old_head, old_tail); + KUNIT_EXPECT_PTR_EQ(test, new_head, new_tail); +} + +static void stackdepot_trie_publish_append_root_replaces_array(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL }; + unsigned long new_entries[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot dup_array; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot dup_slot; + struct stack_depot_trie_node_slot old_slot; + struct stack_depot_trie_node_slot new_slot; + struct stack_depot_trie_root root = {}; + const void *dup_head = NULL; + const void *dup_tail = NULL; + const void *old_head = NULL; + const void *old_tail = NULL; + const void *new_head = NULL; + const void *new_tail = NULL; + unsigned int used = 0; + int ret; + + trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); + trie_node_slot_alloc(test, &dup_slot, old_entries, ARRAY_SIZE(old_entries)); + trie_node_slot_alloc(test, &new_slot, new_entries, ARRAY_SIZE(new_entries)); + old_array.size = __stack_depot_trie_child_array_size(1); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + dup_array.size = __stack_depot_trie_child_array_size(2); + dup_array.array = kunit_kzalloc(test, dup_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dup_array.array); + new_array.size = __stack_depot_trie_child_array_size(2); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + + ret = append_chain(NULL, 21, old_entries, ARRAY_SIZE(old_entries), + &old_slot, 1, NULL, 0, NULL, 0, &old_head, &old_tail, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, old_head, old_array.array, + old_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = append_chain(NULL, 22, old_entries, ARRAY_SIZE(old_entries), + &dup_slot, 1, NULL, 0, NULL, 0, &dup_head, &dup_tail, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, dup_head, dup_array.array, + dup_array.size); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); + + ret = append_chain(NULL, 23, new_entries, ARRAY_SIZE(new_entries), + &new_slot, 1, NULL, 0, NULL, 0, &new_head, &new_tail, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, new_head, new_array.array, + new_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, root.children, new_array.array); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, old_entries[0]), + old_head); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, new_entries[0]), + new_head); + KUNIT_EXPECT_PTR_EQ(test, old_head, old_tail); + KUNIT_EXPECT_PTR_EQ(test, dup_head, dup_tail); + KUNIT_EXPECT_PTR_EQ(test, new_head, new_tail); +} + +static void stackdepot_trie_publish_append_rejects_bad_inputs(struct kunit *test) +{ + unsigned long parent_entries[] = { 0x1000UL }; + unsigned long entries[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *head = NULL; + const void *tail = NULL; + unsigned int used = 0; + void *parent; + int ret; + + trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, + &parent); + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = append_chain(parent, 20, entries, ARRAY_SIZE(entries), &node_slot, 1, + NULL, 0, NULL, 0, &head, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, head, child_array.array, + child_array.size); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = publish_append(NULL, parent, NULL, child_array.array, + child_array.size); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = publish_append(NULL, parent, head, node_slot.node, node_slot.size); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) static void stackdepot_trie_node_compressed_roundtrip(struct kunit *test) { @@ -1090,6 +1289,10 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_node_match_raw), KUNIT_CASE(stackdepot_trie_append_chain_raw), KUNIT_CASE(stackdepot_trie_append_chain_parent), + KUNIT_CASE(stackdepot_trie_publish_append_root), + KUNIT_CASE(stackdepot_trie_publish_append_parent), + KUNIT_CASE(stackdepot_trie_publish_append_root_replaces_array), + KUNIT_CASE(stackdepot_trie_publish_append_rejects_bad_inputs), #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) KUNIT_CASE(stackdepot_trie_node_compressed_roundtrip), KUNIT_CASE(stackdepot_trie_node_match_compressed), From 25ce56870581f9d4704efdfa422087249cbfff28 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 1 Jun 2026 19:10:20 +0100 Subject: [PATCH 019/129] KRN-1117: Order stackdepot trie append publication Make the private append-publication helper read the current child array with READ_ONCE() and publish the replacement with release ordering. This keeps the helper safe for future lockless trie readers. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 13602486ead62..c9a351040d07e 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1572,7 +1572,7 @@ __stack_depot_trie_publish_append(struct stack_depot_trie_root *root, slot = &parent->children; } - old_array = *slot; + old_array = READ_ONCE(*slot); old_size = old_array ? __stack_depot_trie_child_array_size(old_array->nr_children) : 0; new_size = old_array ? old_array->nr_children + 1 : 1; @@ -1588,7 +1588,8 @@ __stack_depot_trie_publish_append(struct stack_depot_trie_root *root, if (__stack_depot_trie_child_array_insert(old_array, head, new_array, storage_size)) return -EINVAL; - *slot = new_array; + /* Publish the fully initialized replacement array last. */ + smp_store_release(slot, new_array); return 0; } From 3bc16337e7f7306e0873a74a9d9628dd5dee7514 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 1 Jun 2026 21:05:50 +0100 Subject: [PATCH 020/129] KRN-1117: Classify stackdepot trie lookup steps Add the private trie lookup-step classifier from the userspace trie prototype. The helper reads one root or parent child array with acquire ordering, compares the matching child node against caller frames, and reports whether the caller should append, descend, split, promote, or return an existing leaf. Keep the helper inert for now: it does not allocate, publish, or change stack_depot_save() and fetch routing. Cover append, descend, found, promote, split, and invalid-input classifications in stackdepot KUnit. Also make the counted-stack increment helper fail closed if the cmpxchg loser path observes a non-positive count instead of delegating that case to refcount_add(). Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 36 +++++++++++ lib/stackdepot.c | 76 ++++++++++++++++++++++- lib/tests/stackdepot_kunit.c | 114 +++++++++++++++++++++++++++++++++++ 3 files changed, 224 insertions(+), 2 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index da00d624711e1..ebc56c72823b5 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -56,6 +56,14 @@ enum stack_depot_frame_mode { STACK_DEPOT_FRAME_COMPRESSED, }; +enum stack_depot_trie_lookup_status { + STACK_DEPOT_TRIE_LOOKUP_APPEND, + STACK_DEPOT_TRIE_LOOKUP_DESCEND, + STACK_DEPOT_TRIE_LOOKUP_FOUND, + STACK_DEPOT_TRIE_LOOKUP_PROMOTE, + STACK_DEPOT_TRIE_LOOKUP_SPLIT, +}; + struct stack_depot_frame_run { enum stack_depot_frame_mode mode; u8 prefix_id; @@ -79,6 +87,13 @@ struct stack_depot_trie_root { const struct stack_depot_trie_child_array *children; }; +struct stack_depot_trie_lookup { + enum stack_depot_trie_lookup_status status; + const void *parent; + const void *node; + unsigned int matched; +}; + /* * Using stack depot requires its initialization, which can be done in 3 ways: * @@ -446,6 +461,27 @@ __stack_depot_trie_publish_append(struct stack_depot_trie_root *root, void *parent, const void *head, void *new_storage, size_t new_storage_size); +/** + * __stack_depot_trie_lookup_step - Classify one trie lookup step + * + * @root: Root to look up in, or NULL when looking under @parent + * @parent: Parent node to look up under, or NULL when looking in @root + * @entries: Remaining stack frames to classify + * @nr_entries: Number of frames in @entries + * @lookup: Storage for the lookup result + * + * This function is only for internal purposes. It reads one root or parent + * child array and compares one child node against @entries. It does not publish, + * allocate, or walk beyond the matching node. + * + * Return: 0 on success, -EINVAL on invalid input or malformed storage. + */ +int +__stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, + const void *parent, const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_lookup *lookup); + /** * __stack_depot_trie_fetch_into - Materialize a trie parent chain * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index c9a351040d07e..741d3a16661d7 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -855,8 +855,8 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) */ if (atomic_try_cmpxchg(&stack->count.refs, &old, new)) was_saturated = true; - else - /* Another caller may have won the transition; count this caller too. */ + else if (old > 0) + /* Another caller won the transition; count this caller too. */ refcount_add((int)count, &stack->count); return was_saturated; @@ -938,6 +938,10 @@ bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, static bool stack_depot_ranges_overlap(const void *a, size_t a_size, const void *b, size_t b_size); +static int +stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *array, + unsigned long frame, unsigned int *pos, + bool *found); static size_t stack_depot_frame_run_entry_bytes(enum stack_depot_frame_mode mode) { @@ -1593,6 +1597,74 @@ __stack_depot_trie_publish_append(struct stack_depot_trie_root *root, return 0; } +int +__stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, + const void *parent_ptr, const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_lookup *lookup) +{ + const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_node *parent = parent_ptr; + const struct stack_depot_trie_node *node; + struct stack_depot_trie_lookup tmp; + unsigned int matched; + unsigned int pos; + unsigned long key; + bool found; + + if (!entries || !nr_entries || !lookup) + return -EINVAL; + if ((root && parent) || (!root && !parent)) + return -EINVAL; + + if (root) { + /* Pairs with append publication's smp_store_release(). */ + children = smp_load_acquire(&root->children); + } else { + /* Pairs with append publication's smp_store_release(). */ + children = smp_load_acquire(&parent->children); + } + + tmp.status = STACK_DEPOT_TRIE_LOOKUP_APPEND; + tmp.parent = parent; + tmp.node = NULL; + tmp.matched = 0; + if (!children) { + *lookup = tmp; + return 0; + } + + key = entries[0]; + if (stack_depot_trie_child_lower_bound(children, key, &pos, &found)) + return -EINVAL; + if (!found) { + *lookup = tmp; + return 0; + } + + node = children->children[pos]; + if (!node || node->parent != parent) + return -EINVAL; + + matched = __stack_depot_trie_node_match(node, entries, nr_entries); + if (!matched) + return -EINVAL; + + tmp.node = node; + tmp.matched = matched; + if (matched < node->run.nr_entries) + tmp.status = STACK_DEPOT_TRIE_LOOKUP_SPLIT; + else if (matched < nr_entries) + tmp.status = STACK_DEPOT_TRIE_LOOKUP_DESCEND; + else if (node->leaf_id) + tmp.status = STACK_DEPOT_TRIE_LOOKUP_FOUND; + else + tmp.status = STACK_DEPOT_TRIE_LOOKUP_PROMOTE; + + *lookup = tmp; + return 0; +} + unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries, unsigned long *scratch, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 6fa83e6a43ad9..d3950debe12fc 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -81,6 +81,15 @@ static int publish_append(struct stack_depot_trie_root *root, void *parent, storage_size); } +static int lookup_step(const struct stack_depot_trie_root *root, + const void *parent, const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_lookup *lookup) +{ + return __stack_depot_trie_lookup_step(root, parent, entries, nr_entries, + lookup); +} + static void trie_node_slot_alloc(struct kunit *test, struct stack_depot_trie_node_slot *slot, @@ -895,6 +904,109 @@ static void stackdepot_trie_publish_append_rejects_bad_inputs(struct kunit *test KUNIT_EXPECT_EQ(test, ret, -EINVAL); } +static void stackdepot_trie_lookup_step_root(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + unsigned long longer[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + unsigned long partial[] = { 0x1000UL, 0x2222UL }; + unsigned long missing[] = { 0x9000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_lookup lookup; + struct stack_depot_trie_root root = {}; + const void *head = NULL; + const void *tail = NULL; + unsigned int used = 0; + int ret; + + ret = lookup_step(&root, NULL, missing, ARRAY_SIZE(missing), &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_APPEND); + KUNIT_EXPECT_NULL(test, lookup.node); + KUNIT_EXPECT_EQ(test, lookup.matched, 0U); + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = append_chain(NULL, 24, entries, ARRAY_SIZE(entries), &node_slot, 1, + NULL, 0, NULL, 0, &head, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, head, child_array.array, + child_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = lookup_step(&root, NULL, entries, ARRAY_SIZE(entries), &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.parent, NULL); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, head); + KUNIT_EXPECT_EQ(test, lookup.matched, (unsigned int)ARRAY_SIZE(entries)); + + ret = lookup_step(&root, NULL, longer, ARRAY_SIZE(longer), &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_DESCEND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, head); + KUNIT_EXPECT_EQ(test, lookup.matched, (unsigned int)ARRAY_SIZE(entries)); + + ret = lookup_step(&root, NULL, partial, ARRAY_SIZE(partial), &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_SPLIT); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, head); + KUNIT_EXPECT_EQ(test, lookup.matched, 1U); + + ret = lookup_step(&root, NULL, missing, ARRAY_SIZE(missing), &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_APPEND); + KUNIT_EXPECT_NULL(test, lookup.node); +} + +static void stackdepot_trie_lookup_step_parent_promote(struct kunit *test) +{ + unsigned long parent_entries[] = { 0x1000UL }; + unsigned long child_entries[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_lookup lookup; + struct stack_depot_trie_root root = {}; + void *child; + void *parent; + int ret; + + trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, + &parent); + trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), parent, 0, + &child); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = publish_append(NULL, parent, child, child_array.array, + child_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = lookup_step(NULL, parent, child_entries, ARRAY_SIZE(child_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_PROMOTE); + KUNIT_EXPECT_PTR_EQ(test, lookup.parent, parent); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, child); + KUNIT_EXPECT_EQ(test, lookup.matched, + (unsigned int)ARRAY_SIZE(child_entries)); + + ret = lookup_step(&root, parent, child_entries, ARRAY_SIZE(child_entries), + &lookup); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = lookup_step(NULL, NULL, child_entries, ARRAY_SIZE(child_entries), + &lookup); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = lookup_step(NULL, parent, NULL, ARRAY_SIZE(child_entries), &lookup); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = lookup_step(NULL, parent, child_entries, 0, &lookup); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = lookup_step(NULL, parent, child_entries, ARRAY_SIZE(child_entries), + NULL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) static void stackdepot_trie_node_compressed_roundtrip(struct kunit *test) { @@ -1293,6 +1405,8 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_publish_append_parent), KUNIT_CASE(stackdepot_trie_publish_append_root_replaces_array), KUNIT_CASE(stackdepot_trie_publish_append_rejects_bad_inputs), + KUNIT_CASE(stackdepot_trie_lookup_step_root), + KUNIT_CASE(stackdepot_trie_lookup_step_parent_promote), #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) KUNIT_CASE(stackdepot_trie_node_compressed_roundtrip), KUNIT_CASE(stackdepot_trie_node_match_compressed), From 2663c87c6c6f694486577c541408e75a64c54301 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 2 Jun 2026 00:15:08 +0100 Subject: [PATCH 021/129] KRN-1117: Harden stackdepot trie lookup prep Harden the private stackdepot trie preparation helpers before the backend starts routing production saves through them. The lookup step now keeps parent-chain validation with the multi-step walker that has the full matched prefix context, and the publication and scratch-buffer contracts document the required writer serialization and aliasing rules. Tighten page_owner stack-list accounting so a failed list-node allocation cannot leave an untracked marker count behind, and keep diagnostic stack counts from being resurrected after they reach zero. The arm64 frame compression helper now rejects the synthetic zero prefix explicitly. Signed-off-by: Caleb Kan --- arch/arm64/include/asm/stackdepot.h | 3 +- include/linux/stackdepot.h | 28 +++++++++----- lib/stackdepot.c | 47 +++++++++++++++++------- lib/tests/stackdepot_kunit.c | 57 ++++++++++++++++++++++++++++- mm/page_owner.c | 51 ++++++++++++++++++++------ 5 files changed, 150 insertions(+), 36 deletions(-) diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h index 81a875a42e039..16f38a4c85872 100644 --- a/arch/arm64/include/asm/stackdepot.h +++ b/arch/arm64/include/asm/stackdepot.h @@ -32,7 +32,8 @@ static inline bool arch_stack_depot_frame_prefix(u8 prefix_id, switch (prefix_id) { case STACK_DEPOT_ARM64_PREV_PREFIX_ID: - if (text_prefix < SZ_4G) + /* Do not synthesize the zero prefix; such frames stay raw. */ + if (text_prefix <= SZ_4G) return false; *prefix = text_prefix - SZ_4G; return true; diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index ebc56c72823b5..82886edb305fa 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -225,10 +225,12 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); * Persistent stack records start with refcount set to %REFCOUNT_SATURATED. If * this helper switches a saturated record to counted mode, it stores @count + 1. * For records already in counted mode, cumulative overflow is handled by the - * underlying refcount_add() saturation semantics; whether that also emits a - * warning depends on the refcount configuration. If such an overflow happens, - * later count get/decrement attempts treat the record as no longer counted and - * fail closed. + * underlying refcount_add_not_zero() saturation semantics; whether that also + * emits a warning depends on the refcount configuration. If such an overflow + * happens, later count get/decrement attempts treat the record as no longer + * counted and fail closed. + * If a racing decrement brings an already-counted diagnostic record to zero, + * this helper does not resurrect it. * Callers must ensure @handle remains valid for the duration of this call. * * Return: true if this call switched the record from saturated to counted, @@ -318,7 +320,7 @@ int __stack_depot_frame_run_init(const unsigned long *entries, * This function is only for internal purposes. It does not write partial * compressed payloads: if any frame does not match @run, @dst is unchanged. * Compressed runs require @scratch to hold at least @run->nr_entries entries; - * raw runs do not use @scratch. @dst must not overlap @entries. + * raw runs do not use @scratch. @dst and @scratch must not overlap @entries. * * Return: 0 on success, -EINVAL on invalid input. */ @@ -452,7 +454,8 @@ __stack_depot_trie_append_chain(const void *parent, u32 leaf_id, * This function is only for internal purposes. It builds a replacement child * array containing @head and stores it in either @root or @parent. @head must * be the first node of an unpublished chain whose parent is @parent. Callers - * remain responsible for lifetime and visibility. + * must serialize publishers for the same @root or @parent. Callers remain + * responsible for lifetime and visibility. * * Return: 0 on success, -EINVAL on invalid input. */ @@ -472,7 +475,13 @@ __stack_depot_trie_publish_append(struct stack_depot_trie_root *root, * * This function is only for internal purposes. It reads one root or parent * child array and compares one child node against @entries. It does not publish, - * allocate, or walk beyond the matching node. + * allocate, walk beyond the matching node, or validate parent-chain equivalence. + * Callers must keep the trie storage alive for the duration of the lookup, for + * example by holding the stackdepot RCU read-side critical section or the writer + * lock. + * A copy-on-write split may reparent descendants before a structural publish; + * callers that walk multiple steps must validate the returned node's parent + * chain against the prefix they already matched before accepting a result. * * Return: 0 on success, -EINVAL on invalid input or malformed storage. */ @@ -522,8 +531,9 @@ size_t __stack_depot_trie_child_array_size(unsigned int nr_children); * @nr_children: Number of child pointers in @children * * This function is only for internal purposes. It does not publish @storage; - * callers remain responsible for lifetime and visibility. Callers must discard - * @storage unless this function returns 0. + * callers remain responsible for lifetime and visibility. @nr_children may be + * zero with @children set to NULL to initialize an empty array. Callers must + * discard @storage unless this function returns 0. * * Return: 0 on success, -EINVAL on invalid input. */ diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 741d3a16661d7..5aebc543473bf 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -64,15 +64,15 @@ union handle_parts { depot_stack_handle_t handle; struct { u32 pool_index_plus_1 : DEPOT_POOL_INDEX_BITS; - u32 offset : DEPOT_OFFSET_BITS; - u32 extra : STACK_DEPOT_EXTRA_BITS; + u32 offset : DEPOT_OFFSET_BITS; + u32 extra : STACK_DEPOT_EXTRA_BITS; }; }; struct stack_record { struct list_head hash_list; /* Links in the hash table */ - u32 hash; /* Hash in hash table */ - u32 size; /* Number of stored frames */ + u32 hash; /* Hash in hash table */ + u32 size; /* Number of stored frames */ union handle_parts handle; /* Constant after initialization */ refcount_t count; union { @@ -855,9 +855,16 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) */ if (atomic_try_cmpxchg(&stack->count.refs, &old, new)) was_saturated = true; - else if (old > 0) + else if (old > 0) { /* Another caller won the transition; count this caller too. */ - refcount_add((int)count, &stack->count); + if (!refcount_add_not_zero((int)count, &stack->count)) { + /* Do not resurrect a diagnostic count that already hit zero. */ + return false; + } + } else { + /* A racing decrement reached zero, or the record is not counted. */ + return false; + } return was_saturated; } @@ -896,8 +903,10 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, return false; underflow = count > (unsigned int)old; - if (WARN_RATELIMIT(underflow, "stack depot count underflow\n")) + if (underflow) { + WARN_RATELIMIT(1, "stack depot count underflow\n"); return false; + } new = old - (int)count; } while (!atomic_try_cmpxchg_release(&stack->count.refs, &old, new)); @@ -990,7 +999,7 @@ static int frame_run_init_lows(const unsigned long *entries, if (lows && nr_entries > nr_lows) return -EINVAL; - /* On success, only lows[0..run->nr_entries - 1] are initialized. */ + /* On compressed success, only lows[0..run->nr_entries - 1] are initialized. */ /* Only prefix ids classify a run; low bits are scratch for the arch hook. */ compressed = frame_try_compress(entries[0], &first_prefix, &low); if (compressed && lows) @@ -1035,6 +1044,9 @@ stack_depot_frame_run_write_compressed(const struct stack_depot_frame_run *run, return -EINVAL; if (stack_depot_ranges_overlap(dst, run->bytes, scratch, run->bytes)) return -EINVAL; + if (stack_depot_ranges_overlap(scratch, run->bytes, entries, + run->nr_entries * sizeof(*entries))) + return -EINVAL; for (i = 0; i < run->nr_entries; i++) { u8 prefix_id; @@ -1643,8 +1655,13 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, } node = children->children[pos]; - if (!node || node->parent != parent) + if (!node) return -EINVAL; + /* + * Do not validate node->parent here. COW splits may reparent descendants + * to an equivalent replacement prefix before the structural publish; the + * multi-step finder has enough prefix context to validate that equivalence. + */ matched = __stack_depot_trie_node_match(node, entries, nr_entries); if (!matched) @@ -1856,11 +1873,15 @@ __stack_depot_trie_child_array_insert(const void *old_storage, const void *child } new_array->nr_children = nr_old + 1; - for (i = 0; i < pos; i++) - new_array->children[i] = old_array->children[i]; + if (old_array) { + for (i = 0; i < pos; i++) + new_array->children[i] = old_array->children[i]; + } new_array->children[pos] = node; - for (i = pos; i < nr_old; i++) - new_array->children[i + 1] = old_array->children[i]; + if (old_array) { + for (i = pos; i < nr_old; i++) + new_array->children[i + 1] = old_array->children[i]; + } return 0; } diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index d3950debe12fc..3eb5e46997c99 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -231,6 +231,11 @@ static void stackdepot_count_helpers(struct kunit *test) 0x1234567800320000UL, 0x1234567800330000UL, }; + unsigned long zeroed_entries[] = { + 0x1234567800610000UL, + 0x1234567800620000UL, + 0x1234567800630000UL, + }; unsigned long seeded_entries[] = { 0x1234567800410000UL, 0x1234567800420000UL, @@ -245,6 +250,8 @@ static void stackdepot_count_helpers(struct kunit *test) depot_stack_handle_t second_handle; depot_stack_handle_t seeded_handle; depot_stack_handle_t max_handle; + depot_stack_handle_t zeroed_handle; + unsigned int zeroed_nr = ARRAY_SIZE(zeroed_entries); unsigned int count; KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); @@ -326,6 +333,11 @@ static void stackdepot_count_helpers(struct kunit *test) __stack_depot_dec_count_and_test(seeded_handle, 1)); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(seeded_handle, &count)); KUNIT_EXPECT_EQ(test, count, 2); + + zeroed_handle = stack_depot_save(zeroed_entries, zeroed_nr, GFP_KERNEL); + KUNIT_ASSERT_NE(test, zeroed_handle, (depot_stack_handle_t)0); + __stack_depot_set_count(zeroed_handle, 3); + KUNIT_EXPECT_TRUE(test, __stack_depot_dec_count_and_test(zeroed_handle, 3)); } static void stackdepot_frame_raw_fallback(struct kunit *test) @@ -400,7 +412,7 @@ static void stackdepot_frame_arm64(struct kunit *test) __stack_depot_frame_decompress(prefix_id, low, &out)); KUNIT_EXPECT_EQ(test, out, frame); - if (text_prefix >= SZ_4G) { + if (text_prefix > SZ_4G) { frame = (text_prefix - SZ_4G) | 0x12345678UL; KUNIT_EXPECT_TRUE(test, __stack_depot_frame_try_compress(frame, &prefix_id, &low)); @@ -1007,6 +1019,48 @@ static void stackdepot_trie_lookup_step_parent_promote(struct kunit *test) KUNIT_EXPECT_EQ(test, ret, -EINVAL); } +static void stackdepot_trie_lookup_step_accepts_reparented_child(struct kunit *test) +{ + unsigned long parent_entries[] = { 0x1000UL }; + unsigned long child_entries[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot child_slot; + struct stack_depot_trie_lookup lookup; + void *old_parent; + void *new_parent; + int ret; + + trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, + &old_parent); + trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 8, + &new_parent); + trie_node_slot_alloc(test, &child_slot, child_entries, + ARRAY_SIZE(child_entries)); + KUNIT_ASSERT_EQ(test, + tnode_init(child_slot.node, child_slot.size, old_parent, 9, + child_entries, ARRAY_SIZE(child_entries), NULL, 0), + 0); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = publish_append(NULL, old_parent, child_slot.node, child_array.array, + child_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, + tnode_init(child_slot.node, child_slot.size, new_parent, 9, + child_entries, ARRAY_SIZE(child_entries), NULL, 0), + 0); + + ret = lookup_step(NULL, old_parent, child_entries, ARRAY_SIZE(child_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.parent, old_parent); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, child_slot.node); + KUNIT_EXPECT_EQ(test, lookup.matched, + (unsigned int)ARRAY_SIZE(child_entries)); +} + #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) static void stackdepot_trie_node_compressed_roundtrip(struct kunit *test) { @@ -1407,6 +1461,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_publish_append_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_lookup_step_root), KUNIT_CASE(stackdepot_trie_lookup_step_parent_promote), + KUNIT_CASE(stackdepot_trie_lookup_step_accepts_reparented_child), #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) KUNIT_CASE(stackdepot_trie_node_compressed_roundtrip), KUNIT_CASE(stackdepot_trie_node_match_compressed), diff --git a/mm/page_owner.c b/mm/page_owner.c index d9bb15f58e078..1d462f22e4cae 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -170,25 +170,35 @@ static noinline depot_stack_handle_t save_stack(gfp_t flags) return handle; } -static void add_stack_record_to_list(depot_stack_handle_t handle, gfp_t gfp_mask) +static struct stack *alloc_stack_record(gfp_t gfp_mask) { - unsigned long flags; struct stack *stack; - if (!handle) - return; - if (!gfpflags_allow_spinning(gfp_mask)) - return; + return NULL; set_current_in_page_owner(); stack = kmalloc(sizeof(*stack), gfp_nested_mask(gfp_mask)); - if (!stack) { - unset_current_in_page_owner(); - return; - } unset_current_in_page_owner(); + return stack; +} + +static void free_stack_record(struct stack *stack) +{ + set_current_in_page_owner(); + kfree(stack); + unset_current_in_page_owner(); +} + +static void add_stack_record_to_list(depot_stack_handle_t handle, + struct stack *stack) +{ + unsigned long flags; + + if (WARN_ON_ONCE(!stack)) + return; + stack->handle = handle; stack->next = NULL; @@ -207,14 +217,30 @@ static void add_stack_record_to_list(depot_stack_handle_t handle, gfp_t gfp_mask static void inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, unsigned int nr_base_pages) { + struct stack *stack = NULL; + unsigned int count; + + if (!handle) + return; + + if (!__stack_depot_get_count(handle, &count)) { + stack = alloc_stack_record(gfp_mask); + /* Keep saturated accounting if the list marker cannot be tracked. */ + if (!stack) + return; + } + /* The saturated-to-counted transition reserves the stack_list marker. */ if (__stack_depot_inc_count(handle, nr_base_pages)) - add_stack_record_to_list(handle, gfp_mask); + add_stack_record_to_list(handle, stack); + else if (stack) + free_stack_record(stack); } static void dec_stack_record_count(depot_stack_handle_t handle, unsigned int nr_base_pages) { + /* Successful list insertion leaves a marker count after all pages free. */ if (__stack_depot_dec_count_and_test(handle, nr_base_pages)) pr_warn("%s: refcount went to 0 for %u handle\n", __func__, handle); @@ -896,6 +922,7 @@ static int stack_print(struct seq_file *m, void *v) /* A count of 1 is only the stack_list membership marker. */ if (!__stack_depot_get_count(handle, &nr_base_pages) || nr_base_pages <= 1) return 0; + /* The <= 1 guard above makes removing the list marker safe. */ nr_base_pages--; /* Drop the list marker before applying the page-count threshold. */ @@ -905,7 +932,7 @@ static int stack_print(struct seq_file *m, void *v) /* Keep show_stacks independent of stackdepot's internal storage layout. */ nr_entries = stack_depot_fetch_into(handle, priv->entries, ARRAY_SIZE(priv->entries)); - /* Buffer matches the stored-depth cap; failure means no stack is available. */ + /* Buffer matches save_stack()'s stored-depth cap. */ if (!nr_entries) return 0; From 2d85c611ae156868d2ba31a9a92fe1d5b383f29c Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 2 Jun 2026 09:30:51 +0100 Subject: [PATCH 022/129] KRN-1117: Add stackdepot trie append insertion Add the private append-only insertion helper from the userspace trie prototype. The helper handles the case where no existing child starts with the next input frame: it preflights root or parent context, storage aliasing, existing-child state, and publication storage before building the unpublished append chain and publishing the replacement child array last. Cover root and parent append insertion, multi-run append insertion, zero-frame rejection, alias rejection, existing-child rejection, and failure atomicity in stackdepot KUnit. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 38 +++ lib/stackdepot.c | 129 ++++++++++ lib/tests/stackdepot_kunit.c | 478 ++++++++++++++++++++++++++++++++++- 3 files changed, 644 insertions(+), 1 deletion(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 82886edb305fa..db0c655ab7856 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -491,6 +491,44 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, unsigned int nr_entries, struct stack_depot_trie_lookup *lookup); +/** + * __stack_depot_trie_insert_append - Insert a missing child by append + * + * @root: Root storage to insert into, or NULL when inserting under @parent + * @parent: Parent node to insert under, or NULL when inserting into @root + * @leaf_id: Non-zero id to store in the final appended node + * @entries: Stack frames to append from the insertion point + * @nr_entries: Number of frames in @entries + * @node_slots: Caller-owned storage slots for trie nodes + * @nr_node_slots: Number of entries in @node_slots + * @child_slots: Caller-owned storage slots for one-child arrays + * @nr_child_slots: Number of entries in @child_slots + * @scratch: Scratch buffer for compressed frame payloads + * @nr_scratch: Number of 32-bit entries that fit in @scratch + * @new_storage: Replacement child-array storage to publish + * @new_storage_size: Size of @new_storage in bytes + * @tail: Storage for the final appended node + * @nr_used: Storage for the number of node slots consumed + * + * This function is only for internal purposes. It handles only the append case + * where no existing child starts with @entries[0]. It builds an unpublished + * append chain in caller-owned storage, then publishes the replacement child + * array last. Callers must serialize publishers for the same @root or @parent. + * + * Return: 0 on success, -EINVAL on invalid input or existing child. + */ +int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, + void *parent, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, void *new_storage, + size_t new_storage_size, const void **tail, + unsigned int *nr_used); + /** * __stack_depot_trie_fetch_into - Materialize a trie parent chain * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 5aebc543473bf..12154ca562c46 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1412,6 +1412,93 @@ trie_child_slot_overlaps(const struct stack_depot_trie_child_array_slot *slots, return false; } +static const struct stack_depot_trie_child_array ** +trie_publish_slot(struct stack_depot_trie_root *root, + struct stack_depot_trie_node *parent) +{ + if ((root && parent) || (!root && !parent)) + return NULL; + if (root) + return &root->children; + return &parent->children; +} + +static int trie_insert_append_precheck(struct stack_depot_trie_root *root, + struct stack_depot_trie_node *parent, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, void *new_storage, + size_t new_storage_size) +{ + const struct stack_depot_trie_child_array **slot; + const struct stack_depot_trie_child_array *children; + size_t size; + unsigned int pos; + bool found; + + if (!entries || !nr_entries || !new_storage) + return -EINVAL; + if (!entries[0]) + return -EINVAL; + if ((nr_node_slots && !node_slots) || (nr_child_slots && !child_slots)) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)new_storage, + __alignof__(struct stack_depot_trie_child_array))) + return -EINVAL; + + slot = trie_publish_slot(root, parent); + if (!slot) + return -EINVAL; + if (stack_depot_ranges_overlap(new_storage, new_storage_size, slot, + sizeof(*slot))) + return -EINVAL; + if (root) { + if (trie_node_slot_overlaps(node_slots, nr_node_slots, slot, + sizeof(*slot))) + return -EINVAL; + if (trie_child_slot_overlaps(child_slots, nr_child_slots, slot, + sizeof(*slot))) + return -EINVAL; + } + if (parent && trie_ancestor_overlaps(parent, new_storage, new_storage_size)) + return -EINVAL; + if (trie_node_slot_overlaps(node_slots, nr_node_slots, new_storage, + new_storage_size) || + trie_child_slot_overlaps(child_slots, nr_child_slots, new_storage, + new_storage_size)) + return -EINVAL; + + /* Pairs with append publication's smp_store_release(). */ + children = smp_load_acquire(slot); + size = __stack_depot_trie_child_array_size(children ? + children->nr_children + 1 : 1); + if (!size || new_storage_size < size) + return -EINVAL; + if (children) { + size = __stack_depot_trie_child_array_size(children->nr_children); + if (!size) + return -EINVAL; + if (stack_depot_ranges_overlap(children, size, new_storage, + new_storage_size)) + return -EINVAL; + if (trie_node_slot_overlaps(node_slots, nr_node_slots, children, + size) || + trie_child_slot_overlaps(child_slots, nr_child_slots, children, + size)) + return -EINVAL; + if (stack_depot_trie_child_lower_bound(children, entries[0], &pos, + &found)) + return -EINVAL; + if (found) + return -EINVAL; + } + + return 0; +} + static int trie_append_chain_validate(const struct stack_depot_trie_node *parent, const unsigned long *entries, unsigned int nr_entries, @@ -1682,6 +1769,48 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, return 0; } +int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, + void *parent_ptr, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, void *new_storage, + size_t new_storage_size, const void **tail, + unsigned int *nr_used) +{ + const void *head; + const void *last; + unsigned int used; + struct stack_depot_trie_node *parent = parent_ptr; + int ret; + + if (!leaf_id || !tail || !nr_used) + return -EINVAL; + ret = trie_insert_append_precheck(root, parent, entries, nr_entries, + node_slots, nr_node_slots, child_slots, + nr_child_slots, new_storage, + new_storage_size); + if (ret) + return ret; + ret = __stack_depot_trie_append_chain(parent, leaf_id, entries, nr_entries, + node_slots, nr_node_slots, child_slots, + nr_child_slots, scratch, nr_scratch, + &head, &last, &used); + if (ret) + return ret; + ret = __stack_depot_trie_publish_append(root, parent, head, new_storage, + new_storage_size); + if (ret) + return ret; + + *tail = last; + *nr_used = used; + return 0; +} + unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries, unsigned long *scratch, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 3eb5e46997c99..af77eefd0ad71 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -87,7 +87,23 @@ static int lookup_step(const struct stack_depot_trie_root *root, struct stack_depot_trie_lookup *lookup) { return __stack_depot_trie_lookup_step(root, parent, entries, nr_entries, - lookup); + lookup); +} + +static int insert_append(struct stack_depot_trie_root *root, void *parent, + u32 leaf_id, const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, void *storage, size_t storage_size, + const void **tail, unsigned int *nr_used) +{ + return __stack_depot_trie_insert_append(root, parent, leaf_id, entries, + nr_entries, node_slots, nr_node_slots, child_slots, + nr_child_slots, scratch, nr_scratch, storage, + storage_size, tail, nr_used); } static void @@ -1061,6 +1077,452 @@ static void stackdepot_trie_lookup_step_accepts_reparented_child(struct kunit *t (unsigned int)ARRAY_SIZE(child_entries)); } +static void stackdepot_trie_insert_append_root(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *tail = NULL; + unsigned int used = 0; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append(&root, NULL, 31, entries, ARRAY_SIZE(entries), + &node_slot, 1, NULL, 0, NULL, 0, child_array.array, + child_array.size, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, tail, node_slot.node); + KUNIT_EXPECT_EQ(test, used, 1U); + KUNIT_EXPECT_PTR_EQ(test, root.children, child_array.array); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, entries[0]), + node_slot.node); +} + +static void stackdepot_trie_insert_append_parent(struct kunit *test) +{ + unsigned long parent_entries[] = { 0x1000UL }; + unsigned long old_entries[] = { 0x2000UL }; + unsigned long new_entries[] = { 0x3000UL }; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot old_slot; + struct stack_depot_trie_node_slot new_slot; + struct stack_depot_trie_lookup lookup; + const void *old_head = NULL; + const void *old_tail = NULL; + const void *new_tail = NULL; + unsigned int used = 0; + void *parent; + int ret; + + trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, + &parent); + trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); + trie_node_slot_alloc(test, &new_slot, new_entries, ARRAY_SIZE(new_entries)); + old_array.size = __stack_depot_trie_child_array_size(1); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + new_array.size = __stack_depot_trie_child_array_size(2); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + + ret = append_chain(parent, 32, old_entries, ARRAY_SIZE(old_entries), + &old_slot, 1, NULL, 0, NULL, 0, &old_head, &old_tail, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(NULL, parent, old_head, old_array.array, + old_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = insert_append(NULL, parent, 33, new_entries, ARRAY_SIZE(new_entries), + &new_slot, 1, NULL, 0, NULL, 0, new_array.array, + new_array.size, &new_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = lookup_step(NULL, parent, old_entries, ARRAY_SIZE(old_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, old_head); + ret = lookup_step(NULL, parent, new_entries, ARRAY_SIZE(new_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, new_tail); + KUNIT_EXPECT_EQ(test, used, 1U); +} + +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) +static void stackdepot_trie_insert_append_splits_frame_runs(struct kunit *test) +{ + unsigned long entries[] = { +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x1000UL, + arch_stack_depot_frame_text_prefix() | 0x2000UL, + 0x1000UL, + arch_stack_depot_frame_text_prefix() | 0x3000UL, +#else + 0xffffffff81001000UL, + 0xffffffff81002000UL, + 0xffff888000001000UL, + 0xffffffff81003000UL, +#endif + }; + struct stack_depot_trie_node_slot node_slots[3]; + struct stack_depot_trie_child_array_slot child_slots[2]; + struct stack_depot_trie_child_array_slot root_array; + struct stack_depot_trie_root root = {}; + unsigned long read_scratch[ARRAY_SIZE(entries)]; + unsigned long out[ARRAY_SIZE(entries)] = {}; + u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; + const void *tail = NULL; + unsigned int used = 0; + unsigned int fetched; + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(node_slots); i++) { + node_slots[i].size = 128; + node_slots[i].node = kunit_kzalloc(test, node_slots[i].size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, node_slots[i].node); + } + for (i = 0; i < ARRAY_SIZE(child_slots); i++) { + child_slots[i].size = __stack_depot_trie_child_array_size(1); + child_slots[i].array = kunit_kzalloc(test, child_slots[i].size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_slots[i].array); + } + root_array.size = __stack_depot_trie_child_array_size(1); + root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, root_array.array); + + ret = insert_append(&root, NULL, 42, entries, ARRAY_SIZE(entries), + node_slots, ARRAY_SIZE(node_slots), child_slots, + ARRAY_SIZE(child_slots), write_scratch, + ARRAY_SIZE(write_scratch), root_array.array, + root_array.size, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, used, 3U); + KUNIT_EXPECT_PTR_EQ(test, tail, node_slots[2].node); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, entries[0]), + node_slots[0].node); + fetched = tfetch(tail, out, ARRAY_SIZE(out), read_scratch, + ARRAY_SIZE(read_scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); +} +#endif + +static void stackdepot_trie_insert_append_rejects_existing_child(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot old_slot; + struct stack_depot_trie_node_slot dup_slot; + struct stack_depot_trie_root root = {}; + unsigned char old[128]; + const void *old_head = NULL; + const void *old_tail = NULL; + const void *tail = (const void *)1; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &old_slot, entries, ARRAY_SIZE(entries)); + trie_node_slot_alloc(test, &dup_slot, entries, ARRAY_SIZE(entries)); + KUNIT_ASSERT_LE(test, dup_slot.size, sizeof(old)); + memset(dup_slot.node, 0xaa, dup_slot.size); + memcpy(old, dup_slot.node, dup_slot.size); + old_array.size = __stack_depot_trie_child_array_size(1); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + new_array.size = __stack_depot_trie_child_array_size(2); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + + ret = append_chain(NULL, 34, entries, ARRAY_SIZE(entries), &old_slot, 1, + NULL, 0, NULL, 0, &old_head, &old_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, old_head, old_array.array, + old_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + used = 99; + + ret = insert_append(&root, NULL, 35, entries, ARRAY_SIZE(entries), + &dup_slot, 1, NULL, 0, NULL, 0, new_array.array, + new_array.size, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); + KUNIT_EXPECT_MEMEQ(test, dup_slot.node, old, dup_slot.size); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + +static void stackdepot_trie_insert_append_rejects_short_array(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *tail = (const void *)1; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + child_array.size = __stack_depot_trie_child_array_size(0); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append(&root, NULL, 36, entries, ARRAY_SIZE(entries), + &node_slot, 1, NULL, 0, NULL, 0, child_array.array, + child_array.size, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_NULL(test, root.children); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + +static void stackdepot_trie_insert_append_rejects_zero_frame(struct kunit *test) +{ + unsigned long entries[] = { 0 }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + unsigned char old[128]; + const void *tail = (const void *)1; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + KUNIT_ASSERT_LE(test, node_slot.size, sizeof(old)); + memset(node_slot.node, 0xaa, node_slot.size); + memcpy(old, node_slot.node, node_slot.size); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append(&root, NULL, 39, entries, ARRAY_SIZE(entries), + &node_slot, 1, NULL, 0, NULL, 0, child_array.array, + child_array.size, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_NULL(test, root.children); + KUNIT_EXPECT_MEMEQ(test, node_slot.node, old, node_slot.size); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + +static void stackdepot_trie_insert_append_rejects_root_with_parent(struct kunit *test) +{ + unsigned long parent_entries[] = { 0x1000UL }; + unsigned long entries[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *tail = (const void *)1; + unsigned int used = 99; + void *parent; + int ret; + + trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, + &parent); + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append(&root, parent, 37, entries, ARRAY_SIZE(entries), + &node_slot, 1, NULL, 0, NULL, 0, child_array.array, + child_array.size, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_NULL(test, root.children); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + +static void stackdepot_trie_insert_append_rejects_root_slot_alias(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *tail = (const void *)1; + unsigned int used = 99; + int ret; + + node_slot.node = &root.children; + node_slot.size = sizeof(root.children); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append(&root, NULL, 40, entries, ARRAY_SIZE(entries), + &node_slot, 1, NULL, 0, NULL, 0, child_array.array, + child_array.size, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_NULL(test, root.children); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + +static void stackdepot_trie_insert_append_rejects_parent_overlap(struct kunit *test) +{ + unsigned long parent_entries[] = { 0x1000UL }; + unsigned long child_entries[] = { 0x2000UL }; + struct stack_depot_trie_node_slot parent_slot; + struct stack_depot_trie_node_slot child_slot; + struct stack_depot_trie_lookup lookup; + const void *tail = (const void *)1; + unsigned int used = 99; + unsigned char old[128]; + int ret; + + trie_node_slot_alloc(test, &parent_slot, parent_entries, + ARRAY_SIZE(parent_entries)); + ret = tnode_init(parent_slot.node, parent_slot.size, NULL, 7, + parent_entries, ARRAY_SIZE(parent_entries), NULL, 0); + KUNIT_ASSERT_EQ(test, ret, 0); + trie_node_slot_alloc(test, &child_slot, child_entries, + ARRAY_SIZE(child_entries)); + KUNIT_ASSERT_LE(test, child_slot.size, sizeof(old)); + memset(child_slot.node, 0xaa, child_slot.size); + memcpy(old, child_slot.node, child_slot.size); + + ret = insert_append(NULL, parent_slot.node, 41, child_entries, + ARRAY_SIZE(child_entries), &child_slot, 1, NULL, 0, + NULL, 0, parent_slot.node, parent_slot.size, &tail, + &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_MEMEQ(test, child_slot.node, old, child_slot.size); + ret = lookup_step(NULL, parent_slot.node, child_entries, + ARRAY_SIZE(child_entries), &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_APPEND); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + +static void stackdepot_trie_insert_append_rejects_publish_overlap(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + unsigned char old[128]; + const void *tail = (const void *)1; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + KUNIT_ASSERT_LE(test, node_slot.size, sizeof(old)); + memset(node_slot.node, 0xaa, node_slot.size); + memcpy(old, node_slot.node, node_slot.size); + + ret = insert_append(&root, NULL, 38, entries, ARRAY_SIZE(entries), + &node_slot, 1, NULL, 0, NULL, 0, node_slot.node, + node_slot.size, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_NULL(test, root.children); + KUNIT_EXPECT_MEMEQ(test, node_slot.node, old, node_slot.size); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + +static void stackdepot_trie_insert_append_rejects_child_node_overlap(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL }; + unsigned long new_entries[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot old_slot; + struct stack_depot_trie_node_slot new_slot; + struct stack_depot_trie_root root = {}; + const void *old_head = NULL; + const void *old_tail = NULL; + const void *tail = (const void *)1; + unsigned char old[128]; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); + old_array.size = __stack_depot_trie_child_array_size(1); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = append_chain(NULL, 43, old_entries, ARRAY_SIZE(old_entries), + &old_slot, 1, NULL, 0, NULL, 0, &old_head, + &old_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, old_head, old_array.array, + old_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_LE(test, old_array.size, sizeof(old)); + memcpy(old, old_array.array, old_array.size); + new_slot.node = old_array.array; + new_slot.size = old_array.size; + new_array.size = __stack_depot_trie_child_array_size(2); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + used = 99; + + ret = insert_append(&root, NULL, 44, new_entries, ARRAY_SIZE(new_entries), + &new_slot, 1, NULL, 0, NULL, 0, new_array.array, + new_array.size, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_MEMEQ(test, old_array.array, old, old_array.size); + KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + +static void stackdepot_trie_insert_append_rejects_child_array_overlap(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL }; + unsigned long new_entries[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_child_array_slot child_slot; + struct stack_depot_trie_node_slot old_slot; + struct stack_depot_trie_node_slot new_slot; + struct stack_depot_trie_root root = {}; + const void *old_head = NULL; + const void *old_tail = NULL; + const void *tail = (const void *)1; + unsigned char old[128]; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); + trie_node_slot_alloc(test, &new_slot, new_entries, ARRAY_SIZE(new_entries)); + old_array.size = __stack_depot_trie_child_array_size(1); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = append_chain(NULL, 45, old_entries, ARRAY_SIZE(old_entries), + &old_slot, 1, NULL, 0, NULL, 0, &old_head, + &old_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, old_head, old_array.array, + old_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_LE(test, old_array.size, sizeof(old)); + memcpy(old, old_array.array, old_array.size); + child_slot.array = old_array.array; + child_slot.size = old_array.size; + new_array.size = __stack_depot_trie_child_array_size(2); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + used = 99; + + ret = insert_append(&root, NULL, 46, new_entries, ARRAY_SIZE(new_entries), + &new_slot, 1, &child_slot, 1, NULL, 0, new_array.array, + new_array.size, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_MEMEQ(test, old_array.array, old, old_array.size); + KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) static void stackdepot_trie_node_compressed_roundtrip(struct kunit *test) { @@ -1462,6 +1924,20 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_lookup_step_root), KUNIT_CASE(stackdepot_trie_lookup_step_parent_promote), KUNIT_CASE(stackdepot_trie_lookup_step_accepts_reparented_child), + KUNIT_CASE(stackdepot_trie_insert_append_root), + KUNIT_CASE(stackdepot_trie_insert_append_parent), +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) + KUNIT_CASE(stackdepot_trie_insert_append_splits_frame_runs), +#endif + KUNIT_CASE(stackdepot_trie_insert_append_rejects_existing_child), + KUNIT_CASE(stackdepot_trie_insert_append_rejects_short_array), + KUNIT_CASE(stackdepot_trie_insert_append_rejects_zero_frame), + KUNIT_CASE(stackdepot_trie_insert_append_rejects_root_with_parent), + KUNIT_CASE(stackdepot_trie_insert_append_rejects_root_slot_alias), + KUNIT_CASE(stackdepot_trie_insert_append_rejects_parent_overlap), + KUNIT_CASE(stackdepot_trie_insert_append_rejects_publish_overlap), + KUNIT_CASE(stackdepot_trie_insert_append_rejects_child_node_overlap), + KUNIT_CASE(stackdepot_trie_insert_append_rejects_child_array_overlap), #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) KUNIT_CASE(stackdepot_trie_node_compressed_roundtrip), KUNIT_CASE(stackdepot_trie_node_match_compressed), From cd0e4008197ff5e65c8fb69b78e247b507c771c9 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 2 Jun 2026 10:54:01 +0100 Subject: [PATCH 023/129] KRN-1117: Add stackdepot trie descend insertion Extend the private append-insertion helper so it can descend through fully matched trie nodes before appending the remaining stack suffix. The helper still rejects found, promote, and split cases so those operations can be implemented as separate, reviewable steps. Preflight descend storage against the published child arrays and subtrees before mutating caller-owned staging slots, preserving the copy-on-write publication contract. Cover one-level and multi-level descend insertion plus sibling-alias rejection in stackdepot KUnit. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 10 +- lib/stackdepot.c | 204 +++++++++++++++++++++++++++++++++++ lib/tests/stackdepot_kunit.c | 176 ++++++++++++++++++++++++++++++ 3 files changed, 386 insertions(+), 4 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index db0c655ab7856..5f1d6c58fb754 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -510,10 +510,12 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, * @tail: Storage for the final appended node * @nr_used: Storage for the number of node slots consumed * - * This function is only for internal purposes. It handles only the append case - * where no existing child starts with @entries[0]. It builds an unpublished - * append chain in caller-owned storage, then publishes the replacement child - * array last. Callers must serialize publishers for the same @root or @parent. + * This function is only for internal purposes. It descends through fully + * matched child nodes until no existing child starts with the next input frame, + * then builds an unpublished append chain in caller-owned storage and publishes + * the replacement child array last. Split, promote, and found cases are rejected + * for later insertion helpers. Callers must serialize publishers for the same + * @root or @parent. * * Return: 0 on success, -EINVAL on invalid input or existing child. */ diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 12154ca562c46..1b10f9a4e11c9 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1382,6 +1382,131 @@ static bool trie_chain_overlaps(const struct stack_depot_trie_node *node, return false; } +static bool +trie_child_array_subtree_overlaps(const struct stack_depot_trie_child_array *array, + const struct stack_depot_trie_node *parent, + const void *ptr, size_t size) +{ + const struct stack_depot_trie_node *node; + size_t array_size; + unsigned int depth = 0; + + if (!array) + return false; + array_size = __stack_depot_trie_child_array_size(array->nr_children); + if (!array_size) + return true; + if (stack_depot_ranges_overlap(ptr, size, array, array_size)) + return true; + if (!array->nr_children) + return false; + + node = array->children[0]; + if (!node || node->parent != parent) + return true; + + for (;;) { + const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_child_array *siblings; + const struct stack_depot_trie_node *child; + const struct stack_depot_trie_node *node_parent; + size_t child_size; + size_t node_size; + unsigned int i; + + if (depth >= CONFIG_STACKDEPOT_MAX_FRAMES) + return true; + if (stack_depot_frame_run_validate(&node->run)) + return true; + + node_size = __stack_depot_trie_node_size(&node->run); + if (!node_size) + return true; + if (stack_depot_ranges_overlap(ptr, size, node, node_size)) + return true; + + children = node->children; + if (children) { + child_size = __stack_depot_trie_child_array_size(children->nr_children); + if (!child_size) + return true; + if (stack_depot_ranges_overlap(ptr, size, children, + child_size)) + return true; + if (children->nr_children) { + child = children->children[0]; + if (!child || child->parent != node) + return true; + node = child; + depth++; + continue; + } + } + + for (;;) { + node_parent = node->parent; + if (node_parent == parent) { + siblings = array; + } else { + if (!node_parent || !node_parent->children) + return true; + siblings = node_parent->children; + } + + for (i = 0; i < siblings->nr_children; i++) { + if (siblings->children[i] == node) + break; + } + if (i == siblings->nr_children) + return true; + if (i + 1 < siblings->nr_children) { + node = siblings->children[i + 1]; + if (!node || node->parent != node_parent) + return true; + break; + } + if (node_parent == parent) + return false; + if (!depth) + return true; + node = node_parent; + depth--; + } + } +} + +static bool +trie_node_slots_subtree_overlap(const struct stack_depot_trie_child_array *array, + const struct stack_depot_trie_node *parent, + const struct stack_depot_trie_node_slot *slots, + unsigned int nr_slots) +{ + unsigned int i; + + for (i = 0; i < nr_slots; i++) { + if (trie_child_array_subtree_overlaps(array, parent, slots[i].node, slots[i].size)) + return true; + } + + return false; +} + +static bool +trie_child_slots_subtree_overlap(const struct stack_depot_trie_child_array *array, + const struct stack_depot_trie_node *parent, + const struct stack_depot_trie_child_array_slot *slots, + unsigned int nr_slots) +{ + unsigned int i; + + for (i = 0; i < nr_slots; i++) { + if (trie_child_array_subtree_overlaps(array, parent, slots[i].array, slots[i].size)) + return true; + } + + return false; +} + static bool trie_node_slot_overlaps(const struct stack_depot_trie_node_slot *slots, unsigned int used, const void *ptr, size_t size) @@ -1499,6 +1624,62 @@ static int trie_insert_append_precheck(struct stack_depot_trie_root *root, return 0; } +static int trie_insert_descend_precheck(struct stack_depot_trie_root *root, + struct stack_depot_trie_node *parent, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + void *new_storage, size_t new_storage_size) +{ + const struct stack_depot_trie_child_array **slot; + const struct stack_depot_trie_child_array *children; + size_t size; + + slot = trie_publish_slot(root, parent); + if (!slot || !new_storage) + return -EINVAL; + if ((nr_node_slots && !node_slots) || (nr_child_slots && !child_slots)) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)new_storage, + __alignof__(struct stack_depot_trie_child_array))) + return -EINVAL; + if (stack_depot_ranges_overlap(new_storage, new_storage_size, slot, + sizeof(*slot))) + return -EINVAL; + if (trie_node_slot_overlaps(node_slots, nr_node_slots, slot, + sizeof(*slot))) + return -EINVAL; + if (trie_child_slot_overlaps(child_slots, nr_child_slots, slot, + sizeof(*slot))) + return -EINVAL; + + /* Pairs with append publication's smp_store_release(). */ + children = smp_load_acquire(slot); + if (!children) + return -EINVAL; + size = __stack_depot_trie_child_array_size(children->nr_children); + if (!size) + return -EINVAL; + if (stack_depot_ranges_overlap(children, size, new_storage, + new_storage_size)) + return -EINVAL; + if (trie_node_slot_overlaps(node_slots, nr_node_slots, children, size)) + return -EINVAL; + if (trie_child_slot_overlaps(child_slots, nr_child_slots, children, + size)) + return -EINVAL; + + if (trie_node_slots_subtree_overlap(children, parent, node_slots, nr_node_slots)) + return -EINVAL; + if (trie_child_slots_subtree_overlap(children, parent, child_slots, nr_child_slots)) + return -EINVAL; + if (trie_child_array_subtree_overlaps(children, parent, new_storage, new_storage_size)) + return -EINVAL; + + return 0; +} + static int trie_append_chain_validate(const struct stack_depot_trie_node *parent, const unsigned long *entries, unsigned int nr_entries, @@ -1789,6 +1970,29 @@ int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, if (!leaf_id || !tail || !nr_used) return -EINVAL; + + for (;;) { + struct stack_depot_trie_lookup lookup; + + ret = __stack_depot_trie_lookup_step(root, parent, entries, nr_entries, &lookup); + if (ret) + return ret; + if (lookup.status != STACK_DEPOT_TRIE_LOOKUP_DESCEND) + break; + ret = trie_insert_descend_precheck(root, parent, node_slots, + nr_node_slots, child_slots, + nr_child_slots, new_storage, + new_storage_size); + if (ret) + return ret; + parent = (struct stack_depot_trie_node *)lookup.node; + root = NULL; + entries += lookup.matched; + nr_entries -= lookup.matched; + } + + if (!entries || !nr_entries) + return -EINVAL; ret = trie_insert_append_precheck(root, parent, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, new_storage, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index af77eefd0ad71..2a6463e9ba677 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1156,6 +1156,179 @@ static void stackdepot_trie_insert_append_parent(struct kunit *test) KUNIT_EXPECT_EQ(test, used, 1U); } +static void stackdepot_trie_insert_append_descends_one_level(struct kunit *test) +{ + unsigned long prefix_entries[] = { 0x1000UL }; + unsigned long child_entries[] = { 0x2000UL }; + unsigned long stack_entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_child_array_slot root_array; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot prefix_slot; + struct stack_depot_trie_node_slot child_slot; + struct stack_depot_trie_lookup lookup; + struct stack_depot_trie_root root = {}; + unsigned long scratch[ARRAY_SIZE(stack_entries)]; + unsigned long out[ARRAY_SIZE(stack_entries)] = {}; + const void *prefix = NULL; + const void *tail = NULL; + unsigned int fetched; + unsigned int used = 0; + int ret; + + trie_node_slot_alloc(test, &prefix_slot, prefix_entries, + ARRAY_SIZE(prefix_entries)); + trie_node_slot_alloc(test, &child_slot, child_entries, + ARRAY_SIZE(child_entries)); + root_array.size = __stack_depot_trie_child_array_size(1); + root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, root_array.array); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append(&root, NULL, 47, prefix_entries, + ARRAY_SIZE(prefix_entries), &prefix_slot, 1, NULL, 0, + NULL, 0, root_array.array, root_array.size, &prefix, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = insert_append(&root, NULL, 48, stack_entries, + ARRAY_SIZE(stack_entries), &child_slot, 1, NULL, 0, + NULL, 0, child_array.array, child_array.size, &tail, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, used, 1U); + ret = lookup_step(NULL, prefix, child_entries, ARRAY_SIZE(child_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, tail); + fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(stack_entries)); + KUNIT_EXPECT_MEMEQ(test, out, stack_entries, sizeof(stack_entries)); +} + +static void stackdepot_trie_insert_append_descends_multiple_levels(struct kunit *test) +{ + unsigned long first_entries[] = { 0x1000UL }; + unsigned long second_entries[] = { 0x2000UL }; + unsigned long tail_entries[] = { 0x3000UL }; + unsigned long second_stack[] = { 0x1000UL, 0x2000UL }; + unsigned long full_stack[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + struct stack_depot_trie_child_array_slot first_array; + struct stack_depot_trie_child_array_slot second_array; + struct stack_depot_trie_child_array_slot third_array; + struct stack_depot_trie_node_slot first_slot; + struct stack_depot_trie_node_slot second_slot; + struct stack_depot_trie_node_slot third_slot; + struct stack_depot_trie_lookup lookup; + struct stack_depot_trie_root root = {}; + unsigned long scratch[ARRAY_SIZE(full_stack)]; + unsigned long out[ARRAY_SIZE(full_stack)] = {}; + const void *first = NULL; + const void *second = NULL; + const void *tail = NULL; + unsigned int used = 0; + unsigned int fetched; + int ret; + + trie_node_slot_alloc(test, &first_slot, first_entries, + ARRAY_SIZE(first_entries)); + trie_node_slot_alloc(test, &second_slot, second_entries, + ARRAY_SIZE(second_entries)); + trie_node_slot_alloc(test, &third_slot, tail_entries, + ARRAY_SIZE(tail_entries)); + first_array.size = __stack_depot_trie_child_array_size(1); + first_array.array = kunit_kzalloc(test, first_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, first_array.array); + second_array.size = __stack_depot_trie_child_array_size(1); + second_array.array = kunit_kzalloc(test, second_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, second_array.array); + third_array.size = __stack_depot_trie_child_array_size(1); + third_array.array = kunit_kzalloc(test, third_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, third_array.array); + + ret = insert_append(&root, NULL, 55, first_entries, + ARRAY_SIZE(first_entries), &first_slot, 1, NULL, 0, + NULL, 0, first_array.array, first_array.size, &first, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = insert_append(&root, NULL, 56, second_stack, ARRAY_SIZE(second_stack), + &second_slot, 1, NULL, 0, NULL, 0, + second_array.array, second_array.size, &second, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = insert_append(&root, NULL, 57, full_stack, ARRAY_SIZE(full_stack), + &third_slot, 1, NULL, 0, NULL, 0, third_array.array, + third_array.size, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, used, 1U); + ret = lookup_step(NULL, second, tail_entries, ARRAY_SIZE(tail_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, tail); + fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(full_stack)); + KUNIT_EXPECT_MEMEQ(test, out, full_stack, sizeof(full_stack)); +} + +static void stackdepot_trie_insert_append_descend_rejects_sibling_overlap(struct kunit *test) +{ + unsigned long prefix_entries[] = { 0x1000UL }; + unsigned long sibling_entries[] = { 0x9000UL }; + unsigned long stack_entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_child_array_slot root_array; + struct stack_depot_trie_child_array_slot sibling_array; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot prefix_slot; + struct stack_depot_trie_node_slot sibling_slot; + struct stack_depot_trie_root root = {}; + struct stack_depot_trie_lookup lookup; + const void *prefix = NULL; + const void *sibling = NULL; + const void *tail = (const void *)1; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &prefix_slot, prefix_entries, + ARRAY_SIZE(prefix_entries)); + trie_node_slot_alloc(test, &sibling_slot, sibling_entries, + ARRAY_SIZE(sibling_entries)); + root_array.size = __stack_depot_trie_child_array_size(1); + root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, root_array.array); + sibling_array.size = __stack_depot_trie_child_array_size(2); + sibling_array.array = kunit_kzalloc(test, sibling_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sibling_array.array); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append(&root, NULL, 49, prefix_entries, + ARRAY_SIZE(prefix_entries), &prefix_slot, 1, NULL, 0, + NULL, 0, root_array.array, root_array.size, &prefix, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = insert_append(&root, NULL, 50, sibling_entries, + ARRAY_SIZE(sibling_entries), &sibling_slot, 1, NULL, 0, + NULL, 0, sibling_array.array, sibling_array.size, + &sibling, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + used = 99; + + ret = insert_append(&root, NULL, 51, stack_entries, + ARRAY_SIZE(stack_entries), &sibling_slot, 1, NULL, 0, + NULL, 0, child_array.array, child_array.size, &tail, + &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); + ret = lookup_step(&root, NULL, sibling_entries, ARRAY_SIZE(sibling_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, sibling); +} + #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) static void stackdepot_trie_insert_append_splits_frame_runs(struct kunit *test) { @@ -1926,6 +2099,9 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_lookup_step_accepts_reparented_child), KUNIT_CASE(stackdepot_trie_insert_append_root), KUNIT_CASE(stackdepot_trie_insert_append_parent), + KUNIT_CASE(stackdepot_trie_insert_append_descends_one_level), + KUNIT_CASE(stackdepot_trie_insert_append_descends_multiple_levels), + KUNIT_CASE(stackdepot_trie_insert_append_descend_rejects_sibling_overlap), #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) KUNIT_CASE(stackdepot_trie_insert_append_splits_frame_runs), #endif From a245616d89b964b127b989d654d6f5a501a59610 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 2 Jun 2026 12:24:53 +0100 Subject: [PATCH 024/129] KRN-1117: Add stackdepot maintainer routing Give stackdepot files an explicit MAINTAINERS section so stackdepot changes route to the mm tree and linux-kernel list directly instead of being matched only through the broad library-code entry. Signed-off-by: Caleb Kan --- MAINTAINERS | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 3d1871e43d7ee..997eade04f421 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14281,11 +14281,7 @@ M: Andrew Morton L: linux-kernel@vger.kernel.org S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-nonmm-unstable -F: arch/*/include/asm/stackdepot.h -F: include/asm-generic/stackdepot.h -F: include/linux/stackdepot.h F: lib/* -F: lib/tests/stackdepot* LICENSES and SPDX stuff M: Thomas Gleixner @@ -24619,6 +24615,17 @@ S: Supported F: Documentation/devicetree/bindings/interrupt-controller/starfive,jh8100-intc.yaml F: drivers/irqchip/irq-starfive-jh8100-intc.c +STACK DEPOT +M: Andrew Morton +L: linux-kernel@vger.kernel.org +S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-nonmm-unstable +F: arch/*/include/asm/stackdepot.h +F: include/asm-generic/stackdepot.h +F: include/linux/stackdepot.h +F: lib/stackdepot.c +F: lib/tests/stackdepot* + STATIC BRANCH/CALL M: Peter Zijlstra M: Josh Poimboeuf From 0040433d26c289b64701425d3f6f1a5c97bc5795 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 2 Jun 2026 12:26:02 +0100 Subject: [PATCH 025/129] KRN-1117: Clarify stackdepot prep invariants Document the stackdepot prep assumptions that came up during review: arm64 frame compression uses adjacent high-bit buckets around the 2 GB module relocation window, saturated refcounts must fail positive-count tests, and page_owner keeps saturated stacks retryable when list marker tracking cannot be allocated. Signed-off-by: Caleb Kan --- arch/arm64/include/asm/stackdepot.h | 11 ++++++----- include/linux/stackdepot.h | 5 +++-- lib/stackdepot.c | 3 +++ mm/page_owner.c | 4 ++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h index 16f38a4c85872..f91a9c0ae4ba8 100644 --- a/arch/arm64/include/asm/stackdepot.h +++ b/arch/arm64/include/asm/stackdepot.h @@ -11,10 +11,11 @@ #define STACK_DEPOT_ARM64_FRAME_PREFIX_MASK (~STACK_DEPOT_ARM64_FRAME_LOW_MASK) /* - * The kernel image is KASLR-relocated on arm64, and modules are allocated - * inside a 2 GB relocation window that contains the image. Store the runtime - * text prefix and the two adjacent 4 GB prefixes so both sides of any window - * boundary can round-trip. + * Modules are allocated inside a 2 GB relocation window containing the + * kernel image, but stackdepot compression stores only the low 32 bits of + * each frame. If that window crosses a 4 GB high-bit boundary, module text + * may have the previous or next prefix even though it is still within + * relocation range of _text. */ #define STACK_DEPOT_ARM64_PREV_PREFIX_ID 0 #define STACK_DEPOT_ARM64_TEXT_PREFIX_ID 1 @@ -32,7 +33,7 @@ static inline bool arch_stack_depot_frame_prefix(u8 prefix_id, switch (prefix_id) { case STACK_DEPOT_ARM64_PREV_PREFIX_ID: - /* Do not synthesize the zero prefix; such frames stay raw. */ + /* Avoid underflow and the zero prefix; such frames stay raw. */ if (text_prefix <= SZ_4G) return false; *prefix = text_prefix - SZ_4G; diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 5f1d6c58fb754..b98e6c64d6735 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -635,6 +635,9 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * is returned. If more frames are stored than @max_entries, the copy is skipped * entirely and 0 is returned. * + * A non-zero invalid or post-put @handle is treated like stack_depot_fetch(): it + * returns 0 and may WARN because such handles indicate a corrupt caller state. + * * Callers must ensure @handle remains valid for the duration of this call. * Persistent handles saved without %STACK_DEPOT_FLAG_GET require no extra * reference; handles saved with %STACK_DEPOT_FLAG_GET require a held reference. @@ -644,8 +647,6 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * Return: Number of frames copied, 0 if @entries is NULL, @max_entries is 0, * @handle is 0 or invalid, stack depot is disabled, or @max_entries is less * than the number of stored frames. - * An invalid or post-put @handle may also trigger a warning from the underlying - * stack_depot_fetch() call. */ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, unsigned long *entries, diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 1b10f9a4e11c9..999fe3f94bbb6 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -148,6 +148,8 @@ static const char *const counter_names[] = { [DEPOT_COUNTER_PERSIST_BYTES] = "persistent_bytes", }; static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); +/* Count helpers rely on saturated refcounts failing positive-count checks. */ +static_assert(REFCOUNT_SATURATED < 0); static int __init disable_stack_depot(char *str) { @@ -1985,6 +1987,7 @@ int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, new_storage_size); if (ret) return ret; + /* Insert callers serialize writers and may publish below this node. */ parent = (struct stack_depot_trie_node *)lookup.node; root = NULL; entries += lookup.matched; diff --git a/mm/page_owner.c b/mm/page_owner.c index 1d462f22e4cae..f4f0882159587 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -225,7 +225,7 @@ static void inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, if (!__stack_depot_get_count(handle, &count)) { stack = alloc_stack_record(gfp_mask); - /* Keep saturated accounting if the list marker cannot be tracked. */ + /* Leave saturated stacks retryable if no list marker can be tracked. */ if (!stack) return; } @@ -240,7 +240,7 @@ static void inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, static void dec_stack_record_count(depot_stack_handle_t handle, unsigned int nr_base_pages) { - /* Successful list insertion leaves a marker count after all pages free. */ + /* Successful list insertion leaves a marker count; zero means corruption. */ if (__stack_depot_dec_count_and_test(handle, nr_base_pages)) pr_warn("%s: refcount went to 0 for %u handle\n", __func__, handle); From 0569363d59db1eedd566d2c1961f6e1d6b2cd993 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 2 Jun 2026 16:14:57 +0100 Subject: [PATCH 026/129] KRN-1117: Harden stackdepot prep helper contracts Keep the trie and frame-run prep interfaces private to stackdepot so the public header only exposes APIs used outside lib/. The helpers are still inert, but their contracts now reject aliasing, overlong parent chains, count overflows, and untracked page_owner handles before later storage-routing patches can depend on them. Strengthen KUnit coverage for those edge cases and remove fixed-size test buffers so the tests scale with the configured stack depth. Signed-off-by: Caleb Kan --- MAINTAINERS | 1 + arch/arm64/include/asm/stackdepot.h | 5 +- include/linux/stackdepot.h | 425 +--------------------------- lib/Kconfig.debug | 3 +- lib/stackdepot.c | 73 +++-- lib/stackdepot_internal.h | 118 ++++++++ lib/tests/stackdepot_kunit.c | 292 ++++++++++++++++--- mm/page_owner.c | 53 +++- 8 files changed, 487 insertions(+), 483 deletions(-) create mode 100644 lib/stackdepot_internal.h diff --git a/MAINTAINERS b/MAINTAINERS index 997eade04f421..6adb143d363d2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -24624,6 +24624,7 @@ F: arch/*/include/asm/stackdepot.h F: include/asm-generic/stackdepot.h F: include/linux/stackdepot.h F: lib/stackdepot.c +F: lib/stackdepot_internal.h F: lib/tests/stackdepot* STATIC BRANCH/CALL diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h index f91a9c0ae4ba8..d718b4722de1e 100644 --- a/arch/arm64/include/asm/stackdepot.h +++ b/arch/arm64/include/asm/stackdepot.h @@ -33,12 +33,15 @@ static inline bool arch_stack_depot_frame_prefix(u8 prefix_id, switch (prefix_id) { case STACK_DEPOT_ARM64_PREV_PREFIX_ID: - /* Avoid underflow and the zero prefix; such frames stay raw. */ + /* Reject < SZ_4G for underflow and == SZ_4G for prefix value 0. */ if (text_prefix <= SZ_4G) return false; *prefix = text_prefix - SZ_4G; return true; case STACK_DEPOT_ARM64_TEXT_PREFIX_ID: + /* Prefix zero is reserved for the raw fallback. */ + if (!text_prefix) + return false; *prefix = text_prefix; return true; case STACK_DEPOT_ARM64_NEXT_PREFIX_ID: diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index b98e6c64d6735..a460cf0407abd 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -51,49 +51,6 @@ typedef u32 depot_flags_t; #define STACK_DEPOT_FLAGS_NUM 2 #define STACK_DEPOT_FLAGS_MASK ((depot_flags_t)((1 << STACK_DEPOT_FLAGS_NUM) - 1)) -enum stack_depot_frame_mode { - STACK_DEPOT_FRAME_RAW, - STACK_DEPOT_FRAME_COMPRESSED, -}; - -enum stack_depot_trie_lookup_status { - STACK_DEPOT_TRIE_LOOKUP_APPEND, - STACK_DEPOT_TRIE_LOOKUP_DESCEND, - STACK_DEPOT_TRIE_LOOKUP_FOUND, - STACK_DEPOT_TRIE_LOOKUP_PROMOTE, - STACK_DEPOT_TRIE_LOOKUP_SPLIT, -}; - -struct stack_depot_frame_run { - enum stack_depot_frame_mode mode; - u8 prefix_id; - unsigned int nr_entries; - size_t bytes; -}; - -struct stack_depot_trie_node_slot { - void *node; - size_t size; -}; - -struct stack_depot_trie_child_array_slot { - void *array; - size_t size; -}; - -struct stack_depot_trie_child_array; - -struct stack_depot_trie_root { - const struct stack_depot_trie_child_array *children; -}; - -struct stack_depot_trie_lookup { - enum stack_depot_trie_lookup_status status; - const void *parent; - const void *node; - unsigned int matched; -}; - /* * Using stack depot requires its initialization, which can be done in 3 ways: * @@ -204,8 +161,8 @@ bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count); * @count: Count to set * * This function is only for internal purposes. - * If @count is 0 or greater than %INT_MAX, this function is a - * no-op. + * If @handle is invalid, @count is 0, or @count is greater than %INT_MAX, + * this function is a no-op. * Callers that use this to switch a saturated record to counted mode must * separately make the record discoverable by their own tracking structure. * Callers must have exclusive access to the stack record count. @@ -217,26 +174,28 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); * * @handle: Stack depot handle * @count: Count to add + * @new_count: Optional storage for whether this call performed the first + * counted increment * * This function is only for internal purposes. * If @count is 0, this function is a no-op. Otherwise @count must be less - * than or equal to %INT_MAX - 1. + * than or equal to %INT_MAX - 1 so the saturated-to-counted transition can + * store the stack_list marker plus @count without overflowing. * * Persistent stack records start with refcount set to %REFCOUNT_SATURATED. If * this helper switches a saturated record to counted mode, it stores @count + 1. - * For records already in counted mode, cumulative overflow is handled by the - * underlying refcount_add_not_zero() saturation semantics; whether that also - * emits a warning depends on the refcount configuration. If such an overflow - * happens, later count get/decrement attempts treat the record as no longer - * counted and fail closed. + * For records already in counted mode, cumulative overflow is rejected and + * leaves the count unchanged. If @new_count is non-NULL, it is set to true when + * this call switches the record from saturated to counted and false otherwise. * If a racing decrement brings an already-counted diagnostic record to zero, * this helper does not resurrect it. * Callers must ensure @handle remains valid for the duration of this call. * - * Return: true if this call switched the record from saturated to counted, - * false otherwise. + * Return: true if @count was applied, false otherwise. */ -bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count); +bool __stack_depot_inc_count(depot_stack_handle_t handle, + unsigned int count, + bool *new_count); /** * __stack_depot_dec_count_and_test - Decrement a stack record count @@ -245,7 +204,7 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count); * @count: Count to subtract * * This function is only for internal purposes. - * @count must be greater than 0 and less than or equal to %INT_MAX - 1. + * @count must be greater than 0 and less than or equal to %INT_MAX. * * Return: true if the resulting count is 0, false if the resulting count is * non-zero, @handle is invalid, the stack record is not in counted mode, or @@ -256,362 +215,6 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count); bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, unsigned int count); -/** - * __stack_depot_frame_try_compress - Try to compress a stack frame - * - * @frame: Stack frame address - * @prefix_id: Storage for the architecture prefix id - * @low: Storage for the compressed low bits - * - * This function is only for internal purposes. The generic implementation is a - * raw fallback and never compresses. - * @prefix_id and @low must be non-NULL. - * - * Return: true if @frame was compressed, false otherwise. - */ -bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, - u32 *low); - -/** - * __stack_depot_frame_decompress - Decompress a stack frame - * - * @prefix_id: Architecture prefix id returned by compression - * @low: Compressed low bits returned by compression - * @frame: Storage for the decompressed frame - * - * This function is only for internal purposes. The generic raw fallback has no - * compressed representation to decode. - * @frame must be non-NULL. - * - * Return: true if @frame was decompressed, false otherwise. - */ -bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, - unsigned long *frame); - -/** - * __stack_depot_frame_run_init - Describe a homogeneous stack frame run - * - * @entries: Stack frames that start the run - * @nr_entries: Number of frames available in @entries - * @run: Storage for the resulting run description - * - * This function is only for internal purposes. It describes the longest prefix - * of @entries that can be stored with one payload format: raw frames, or low - * bits for frames that all share one architecture prefix id. It does not write - * frame payload data; callers that need payload storage must call - * __stack_depot_frame_run_write(). - * - * Return: 0 on success, -EINVAL on invalid input. - */ -int __stack_depot_frame_run_init(const unsigned long *entries, - unsigned int nr_entries, - struct stack_depot_frame_run *run); - -/** - * __stack_depot_frame_run_write - Write a stack frame run payload - * - * @run: Run description returned by __stack_depot_frame_run_init() - * @entries: Stack frames to encode - * @dst: Payload buffer to write - * @dst_size: Size of @dst in bytes - * @scratch: Scratch buffer for compressed frame payloads - * @nr_scratch: Number of 32-bit entries that fit in @scratch - * - * This function is only for internal purposes. It does not write partial - * compressed payloads: if any frame does not match @run, @dst is unchanged. - * Compressed runs require @scratch to hold at least @run->nr_entries entries; - * raw runs do not use @scratch. @dst and @scratch must not overlap @entries. - * - * Return: 0 on success, -EINVAL on invalid input. - */ -int __stack_depot_frame_run_write(const struct stack_depot_frame_run *run, - const unsigned long *entries, void *dst, - size_t dst_size, u32 *scratch, - unsigned int nr_scratch); - -/** - * __stack_depot_frame_run_read - Read a stack frame run payload - * - * @run: Run description for the payload - * @src: Payload buffer to read - * @src_size: Size of @src in bytes - * @entries: Storage for decoded stack frames - * @max_entries: Number of frames that fit in @entries - * @scratch: Scratch buffer for decoded compressed frames - * @nr_scratch: Number of frames that fit in @scratch - * - * This function is only for internal purposes. It does not write partial - * compressed output: if any frame cannot be decoded, @entries is unchanged. - * Compressed runs require @scratch to hold at least @run->nr_entries entries; - * raw runs do not use @scratch. For compressed runs, @entries and @scratch - * must not overlap. - * - * Return: 0 on success, -EINVAL on invalid input. - */ -int __stack_depot_frame_run_read(const struct stack_depot_frame_run *run, - const void *src, size_t src_size, - unsigned long *entries, unsigned int max_entries, - unsigned long *scratch, - unsigned int nr_scratch); - -/** - * __stack_depot_trie_node_size - Get storage size for a trie node - * - * @run: Frame run to store in the node - * - * This function is only for internal purposes. - * - * Return: Aligned node storage size, 0 on invalid input. - */ -size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run); - -/** - * __stack_depot_trie_node_init - Initialize a trie node in caller storage - * - * @storage: Node storage to initialize - * @storage_size: Size of @storage in bytes - * @parent: Parent node or NULL for a root node - * @leaf_id: Non-zero id when this node terminates a stored stack - * @entries: Homogeneous frame run to store in this node - * @nr_entries: Number of frames in @entries - * @scratch: Scratch buffer for compressed frame payloads - * @nr_scratch: Number of 32-bit entries that fit in @scratch - * - * This function is only for internal purposes. It does not publish @storage; - * all @entries must fit in one raw or same-prefix compressed frame run. Callers - * remain responsible for lifetime and visibility. Callers must discard @storage - * unless this function returns 0. - * - * Return: 0 on success, -EINVAL on invalid input. - */ -int __stack_depot_trie_node_init(void *storage, size_t storage_size, - const void *parent, u32 leaf_id, - const unsigned long *entries, - unsigned int nr_entries, u32 *scratch, - unsigned int nr_scratch); - -/** - * __stack_depot_trie_node_match - Match entries against one trie node - * - * @node: Trie node to compare - * @entries: Stack frames to match from the start of @node - * @nr_entries: Number of frames available in @entries - * - * This function is only for internal purposes. It compares @entries against the - * decoded frame run stored in @node and does not walk parent or child links. - * - * Return: Number of matching frames, up to the smaller of the node run length - * and @nr_entries. Returns 0 on invalid input or a first-frame mismatch. - */ -unsigned int __stack_depot_trie_node_match(const void *node, - const unsigned long *entries, - unsigned int nr_entries); - -/** - * __stack_depot_trie_append_chain - Build an unpublished node chain - * - * @parent: Parent node for the new chain, or NULL for a root chain - * @leaf_id: Non-zero id to store in the final node - * @entries: Stack frames to store in the chain - * @nr_entries: Number of frames in @entries - * @node_slots: Caller-owned storage slots for trie nodes - * @nr_node_slots: Number of entries in @node_slots - * @child_slots: Caller-owned storage slots for one-child arrays - * @nr_child_slots: Number of entries in @child_slots - * @scratch: Scratch buffer for compressed frame payloads - * @nr_scratch: Number of 32-bit entries that fit in @scratch - * @head: Storage for the first node in the chain - * @tail: Storage for the final node in the chain - * @nr_used: Storage for the number of node slots consumed - * - * This function is only for internal purposes. It builds nodes and one-child - * arrays in caller-owned unpublished storage, splitting the input at raw / - * compressed mode and compressed-prefix boundaries. Callers remain responsible - * for lifetime and visibility. - * - * Return: 0 on success, -EINVAL on invalid input. - */ -int -__stack_depot_trie_append_chain(const void *parent, u32 leaf_id, - const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, const void **head, - const void **tail, unsigned int *nr_used); - -/** - * __stack_depot_trie_publish_append - Publish an appended chain - * - * @root: Root storage to publish into, or NULL for a parent publish - * @parent: Parent node to publish under, or NULL for a root publish - * @head: First node of an unpublished chain - * @new_storage: Replacement child-array storage - * @new_storage_size: Size of @new_storage in bytes - * - * This function is only for internal purposes. It builds a replacement child - * array containing @head and stores it in either @root or @parent. @head must - * be the first node of an unpublished chain whose parent is @parent. Callers - * must serialize publishers for the same @root or @parent. Callers remain - * responsible for lifetime and visibility. - * - * Return: 0 on success, -EINVAL on invalid input. - */ -int -__stack_depot_trie_publish_append(struct stack_depot_trie_root *root, - void *parent, const void *head, - void *new_storage, size_t new_storage_size); - -/** - * __stack_depot_trie_lookup_step - Classify one trie lookup step - * - * @root: Root to look up in, or NULL when looking under @parent - * @parent: Parent node to look up under, or NULL when looking in @root - * @entries: Remaining stack frames to classify - * @nr_entries: Number of frames in @entries - * @lookup: Storage for the lookup result - * - * This function is only for internal purposes. It reads one root or parent - * child array and compares one child node against @entries. It does not publish, - * allocate, walk beyond the matching node, or validate parent-chain equivalence. - * Callers must keep the trie storage alive for the duration of the lookup, for - * example by holding the stackdepot RCU read-side critical section or the writer - * lock. - * A copy-on-write split may reparent descendants before a structural publish; - * callers that walk multiple steps must validate the returned node's parent - * chain against the prefix they already matched before accepting a result. - * - * Return: 0 on success, -EINVAL on invalid input or malformed storage. - */ -int -__stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, - const void *parent, const unsigned long *entries, - unsigned int nr_entries, - struct stack_depot_trie_lookup *lookup); - -/** - * __stack_depot_trie_insert_append - Insert a missing child by append - * - * @root: Root storage to insert into, or NULL when inserting under @parent - * @parent: Parent node to insert under, or NULL when inserting into @root - * @leaf_id: Non-zero id to store in the final appended node - * @entries: Stack frames to append from the insertion point - * @nr_entries: Number of frames in @entries - * @node_slots: Caller-owned storage slots for trie nodes - * @nr_node_slots: Number of entries in @node_slots - * @child_slots: Caller-owned storage slots for one-child arrays - * @nr_child_slots: Number of entries in @child_slots - * @scratch: Scratch buffer for compressed frame payloads - * @nr_scratch: Number of 32-bit entries that fit in @scratch - * @new_storage: Replacement child-array storage to publish - * @new_storage_size: Size of @new_storage in bytes - * @tail: Storage for the final appended node - * @nr_used: Storage for the number of node slots consumed - * - * This function is only for internal purposes. It descends through fully - * matched child nodes until no existing child starts with the next input frame, - * then builds an unpublished append chain in caller-owned storage and publishes - * the replacement child array last. Split, promote, and found cases are rejected - * for later insertion helpers. Callers must serialize publishers for the same - * @root or @parent. - * - * Return: 0 on success, -EINVAL on invalid input or existing child. - */ -int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, - void *parent, u32 leaf_id, - const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, void *new_storage, - size_t new_storage_size, const void **tail, - unsigned int *nr_used); - -/** - * __stack_depot_trie_fetch_into - Materialize a trie parent chain - * - * @leaf: Leaf node to materialize from - * @entries: Caller-owned output buffer - * @max_entries: Number of frames that fit in @entries - * @scratch: Caller-owned scratch buffer for staged output - * @nr_scratch: Number of frames that fit in @scratch - * - * This function is only for internal purposes. It stages the full stack into - * @scratch first, so failures do not partially write @entries. @scratch is - * caller-owned temporary storage and may be modified on failure. - * - * Return: Number of frames copied, 0 on invalid input or too-small buffers. - */ -unsigned int -__stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, - unsigned int max_entries, unsigned long *scratch, - unsigned int nr_scratch); - -/** - * __stack_depot_trie_child_array_size - Get storage size for child pointers - * - * @nr_children: Number of child pointers stored in the array - * - * This function is only for internal purposes. - * - * Return: Aligned child-array storage size, 0 on overflow. - */ -size_t __stack_depot_trie_child_array_size(unsigned int nr_children); - -/** - * __stack_depot_trie_child_array_init - Initialize sorted child storage - * - * @storage: Child-array storage to initialize - * @storage_size: Size of @storage in bytes - * @children: Children sorted by first decoded frame - * @nr_children: Number of child pointers in @children - * - * This function is only for internal purposes. It does not publish @storage; - * callers remain responsible for lifetime and visibility. @nr_children may be - * zero with @children set to NULL to initialize an empty array. Callers must - * discard @storage unless this function returns 0. - * - * Return: 0 on success, -EINVAL on invalid input. - */ -int -__stack_depot_trie_child_array_init(void *storage, size_t storage_size, - const void * const *children, - unsigned int nr_children); - -/** - * __stack_depot_trie_child_array_find - Find a child by first frame - * - * @storage: Child-array storage initialized by child_array_init/insert - * @frame: First decoded frame to search for - * - * This function is only for internal purposes. - * - * Return: Child pointer if found, NULL otherwise. - */ -const void * -__stack_depot_trie_child_array_find(const void *storage, unsigned long frame); - -/** - * __stack_depot_trie_child_array_insert - Build replacement child storage - * - * @old_storage: Existing sorted child array, or NULL - * @child: Child node to insert - * @new_storage: Replacement child-array storage to initialize - * @new_storage_size: Size of @new_storage in bytes - * - * This function is only for internal purposes. It builds a new sorted child - * array and rejects duplicate first-frame keys and in-place updates. - * - * Return: 0 on success, -EINVAL on invalid input. - */ -int -__stack_depot_trie_child_array_insert(const void *old_storage, const void *child, - void *new_storage, size_t new_storage_size); - /** * stack_depot_fetch - Fetch a stack trace from stack depot * diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 5f77493c669c7..5ac3d34edb84c 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2712,7 +2712,8 @@ config STACKDEPOT_KUNIT_TEST default KUNIT_ALL_TESTS help Enable this option to test stack depot API behavior at boot. - This test is built in because it exercises internal stack depot helpers. + This test is built in because it exercises internal, non-exported + stack depot helpers. KUnit tests run during boot and output the results to the debug log in TAP format (https://testanything.org/). Only useful for kernel diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 999fe3f94bbb6..8fc36bd947cba 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -41,6 +41,8 @@ #include +#include "stackdepot_internal.h" + /* * The pool_index is offset by 1 so the first record does not have a 0 handle. */ @@ -148,7 +150,7 @@ static const char *const counter_names[] = { [DEPOT_COUNTER_PERSIST_BYTES] = "persistent_bytes", }; static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); -/* Count helpers rely on saturated refcounts failing positive-count checks. */ +/* Count helpers rely on saturated refcounts looking negative. */ static_assert(REFCOUNT_SATURATED < 0); static int __init disable_stack_depot(char *str) @@ -834,13 +836,17 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count) refcount_set(&stack->count, (int)count); } -bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) +bool __stack_depot_inc_count(depot_stack_handle_t handle, + unsigned int count, + bool *new_count) { struct stack_record *stack; int new; int old = REFCOUNT_SATURATED; bool was_saturated = false; + if (new_count) + *new_count = false; if (!handle || !count || count > (unsigned int)INT_MAX - 1) return false; @@ -852,23 +858,26 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, unsigned int count) /* * Intentional refcount_t internals use: no helper conditionally * converts the persistent REFCOUNT_SATURATED sentinel to a positive - * page_owner count. The cmpxchg only performs that one-way transition; - * normal counted records continue through refcount_add(). + * page_owner count. The first cmpxchg only performs that one-way + * transition; normal counted records continue through a checked cmpxchg + * loop so overflow cannot recreate the saturated sentinel. */ - if (atomic_try_cmpxchg(&stack->count.refs, &old, new)) + if (atomic_try_cmpxchg(&stack->count.refs, &old, new)) { was_saturated = true; - else if (old > 0) { - /* Another caller won the transition; count this caller too. */ - if (!refcount_add_not_zero((int)count, &stack->count)) { - /* Do not resurrect a diagnostic count that already hit zero. */ - return false; - } } else { - /* A racing decrement reached zero, or the record is not counted. */ - return false; + /* cmpxchg reloads @old before each retry check. */ + do { + if (old <= 0) + return false; + if (count > (unsigned int)INT_MAX - (unsigned int)old) + return false; + new = old + (int)count; + } while (!atomic_try_cmpxchg(&stack->count.refs, &old, new)); } - return was_saturated; + if (new_count) + *new_count = was_saturated; + return true; } bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, @@ -878,7 +887,7 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, int new; int old; - if (!handle || !count || count > (unsigned int)INT_MAX - 1) + if (!handle || !count || count > (unsigned int)INT_MAX) return false; stack = depot_fetch_stack(handle); @@ -913,6 +922,7 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, new = old - (int)count; } while (!atomic_try_cmpxchg_release(&stack->count.refs, &old, new)); + /* Non-zero results are diagnostic counts; callers consume no ordered data. */ if (!new) smp_acquire__after_ctrl_dep(); @@ -1179,9 +1189,12 @@ static int frame_run_read(const struct stack_depot_frame_run *run, run->nr_entries * sizeof(*entries), scratch, run->nr_entries * sizeof(*scratch))) return -EINVAL; + if (stack_depot_ranges_overlap(src, run->bytes, scratch, + run->nr_entries * sizeof(*scratch))) + return -EINVAL; return stack_depot_frame_run_read_compressed(run, src, entries, scratch, - nr_scratch); + nr_scratch); } int __stack_depot_frame_run_read(const struct stack_depot_frame_run *run, @@ -1267,7 +1280,9 @@ int __stack_depot_trie_node_init(void *storage, size_t storage_size, return -EINVAL; if (parent_node) { if (!parent_node->stack_len || - parent_node->stack_len > U32_MAX - run.nr_entries) + parent_node->stack_len > U32_MAX - run.nr_entries || + parent_node->stack_len > + CONFIG_STACKDEPOT_MAX_FRAMES - run.nr_entries) return -EINVAL; stack_len = parent_node->stack_len + run.nr_entries; } else { @@ -1302,6 +1317,10 @@ unsigned int __stack_depot_trie_node_match(const void *node_ptr, return 0; limit = min(node->run.nr_entries, nr_entries); + if (node->run.mode == STACK_DEPOT_FRAME_RAW && + !memcmp(node->data, entries, limit * sizeof(*entries))) + return limit; + for (i = 0; i < limit; i++) { unsigned long frame; @@ -1717,7 +1736,8 @@ static int trie_append_chain_validate(const struct stack_depot_trie_node *parent if (run.mode == STACK_DEPOT_FRAME_COMPRESSED && (!scratch || nr_scratch < run.nr_entries)) return -EINVAL; - if (stack_len > U32_MAX - run.nr_entries) + if (stack_len > U32_MAX - run.nr_entries || + stack_len > CONFIG_STACKDEPOT_MAX_FRAMES - run.nr_entries) return -EINVAL; size = __stack_depot_trie_node_size(&run); @@ -2040,6 +2060,8 @@ __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, pos = total; for (node = leaf; node; node = node->parent) { + if (stack_depot_frame_run_validate(&node->run)) + return 0; if (node->stack_len != pos || node->run.nr_entries > pos) return 0; pos -= node->run.nr_entries; @@ -2096,9 +2118,16 @@ int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, for (i = 0; i < nr_children; i++) { unsigned long frame; + size_t size; if (stack_depot_trie_node_first_frame(nodes[i], &frame)) return -EINVAL; + size = __stack_depot_trie_node_size(&nodes[i]->run); + if (!size) + return -EINVAL; + if (stack_depot_ranges_overlap(array, storage_size, nodes[i], size)) + return -EINVAL; + /* Child key zero is reserved so NULL lookup remains unambiguous. */ if (!frame || (i && frame <= last)) return -EINVAL; last = frame; @@ -2173,12 +2202,18 @@ __stack_depot_trie_child_array_insert(const void *old_storage, const void *child unsigned int pos; unsigned int i; unsigned long frame; + size_t node_size; size_t old_size; bool overlaps; bool found; if (!node || !new_array || stack_depot_trie_node_first_frame(node, &frame)) return -EINVAL; + node_size = __stack_depot_trie_node_size(&node->run); + if (!node_size) + return -EINVAL; + if (stack_depot_ranges_overlap(new_array, new_storage_size, node, node_size)) + return -EINVAL; if (!IS_ALIGNED((unsigned long)new_array, __alignof__(*new_array))) return -EINVAL; if (!frame) @@ -2261,7 +2296,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, if (!handle || !entries || !max_entries) return 0; - /* Defer stack record reuse while copying from stackdepot-owned storage. */ + /* Extend the lookup RCU section so the fetched record cannot be reused. */ rcu_read_lock_sched_notrace(); nr_entries = stack_depot_fetch(handle, &stack_entries); if (!nr_entries || nr_entries > max_entries) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h new file mode 100644 index 0000000000000..89d54a7885c4a --- /dev/null +++ b/lib/stackdepot_internal.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _STACKDEPOT_INTERNAL_H +#define _STACKDEPOT_INTERNAL_H + +#include +#include + +enum stack_depot_frame_mode { + STACK_DEPOT_FRAME_RAW, + STACK_DEPOT_FRAME_COMPRESSED, +}; + +enum stack_depot_trie_lookup_status { + STACK_DEPOT_TRIE_LOOKUP_APPEND, + STACK_DEPOT_TRIE_LOOKUP_DESCEND, + STACK_DEPOT_TRIE_LOOKUP_FOUND, + STACK_DEPOT_TRIE_LOOKUP_PROMOTE, + STACK_DEPOT_TRIE_LOOKUP_SPLIT, +}; + +struct stack_depot_frame_run { + enum stack_depot_frame_mode mode; + u8 prefix_id; + unsigned int nr_entries; + size_t bytes; +}; + +struct stack_depot_trie_node_slot { + void *node; + size_t size; +}; + +struct stack_depot_trie_child_array_slot { + void *array; + size_t size; +}; + +struct stack_depot_trie_child_array; + +struct stack_depot_trie_root { + const struct stack_depot_trie_child_array *children; +}; + +struct stack_depot_trie_lookup { + enum stack_depot_trie_lookup_status status; + const void *parent; + const void *node; + unsigned int matched; +}; + +bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, + u32 *low); +bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, + unsigned long *frame); +int __stack_depot_frame_run_init(const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_frame_run *run); +int __stack_depot_frame_run_write(const struct stack_depot_frame_run *run, + const unsigned long *entries, void *dst, + size_t dst_size, u32 *scratch, + unsigned int nr_scratch); +int __stack_depot_frame_run_read(const struct stack_depot_frame_run *run, + const void *src, size_t src_size, + unsigned long *entries, unsigned int max_entries, + unsigned long *scratch, + unsigned int nr_scratch); +size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run); +int __stack_depot_trie_node_init(void *storage, size_t storage_size, + const void *parent, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, u32 *scratch, + unsigned int nr_scratch); +unsigned int __stack_depot_trie_node_match(const void *node, + const unsigned long *entries, + unsigned int nr_entries); +int __stack_depot_trie_append_chain(const void *parent, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, const void **head, + const void **tail, unsigned int *nr_used); +int __stack_depot_trie_publish_append(struct stack_depot_trie_root *root, + void *parent, const void *head, + void *new_storage, size_t new_storage_size); +int __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, + const void *parent, const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_lookup *lookup); +int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, + void *parent, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, void *new_storage, + size_t new_storage_size, const void **tail, + unsigned int *nr_used); +unsigned int __stack_depot_trie_fetch_into(const void *leaf, + unsigned long *entries, + unsigned int max_entries, + unsigned long *scratch, + unsigned int nr_scratch); +size_t __stack_depot_trie_child_array_size(unsigned int nr_children); +int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, + const void * const *children, + unsigned int nr_children); +const void *__stack_depot_trie_child_array_find(const void *storage, + unsigned long frame); +int __stack_depot_trie_child_array_insert(const void *old_storage, + const void *child, void *new_storage, + size_t new_storage_size); + +#endif /* _STACKDEPOT_INTERNAL_H */ diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 2a6463e9ba677..2327b83b0e533 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -8,6 +8,8 @@ #include #include +#include "../stackdepot_internal.h" + #ifdef CONFIG_ARM64 #include #endif @@ -149,6 +151,7 @@ trie_node_alloc(struct kunit *test, const unsigned long *entries, size_t size; int ret; + /* This helper allocates one trie node, so @entries must form one run. */ KUNIT_ASSERT_EQ(test, frame_run_init(entries, nr_entries, &run), 0); size = __stack_depot_trie_node_size(&run); KUNIT_ASSERT_GT(test, size, (size_t)0); @@ -159,6 +162,15 @@ trie_node_alloc(struct kunit *test, const unsigned long *entries, KUNIT_ASSERT_EQ(test, ret, 0); } +static void trie_fill_raw_entries(unsigned long *entries, unsigned int nr_entries, + unsigned long base) +{ + unsigned int i; + + for (i = 0; i < nr_entries; i++) + entries[i] = base + i * 0x10UL; +} + static void stackdepot_fetch_into_roundtrip(struct kunit *test) { unsigned long entries[] = { @@ -268,6 +280,7 @@ static void stackdepot_count_helpers(struct kunit *test) depot_stack_handle_t max_handle; depot_stack_handle_t zeroed_handle; unsigned int zeroed_nr = ARRAY_SIZE(zeroed_entries); + bool new_count; unsigned int count; KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); @@ -277,30 +290,49 @@ static void stackdepot_count_helpers(struct kunit *test) __stack_depot_set_count(0, 1); __stack_depot_set_count(0, 0); __stack_depot_set_count(0, INT_MAX); - KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(0, 1)); - KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(0, INT_MAX - 2)); + KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(0, 1, &new_count)); + KUNIT_EXPECT_FALSE(test, + __stack_depot_inc_count(0, INT_MAX - 2, &new_count)); KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(0, 1)); handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(handle, INT_MAX)); + KUNIT_EXPECT_FALSE(test, + __stack_depot_inc_count(handle, INT_MAX, &new_count)); KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 1)); KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); max_handle = stack_depot_save(max_entries, ARRAY_SIZE(max_entries), GFP_KERNEL); KUNIT_ASSERT_NE(test, max_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(max_handle, INT_MAX - 1)); + new_count = false; + KUNIT_EXPECT_TRUE(test, + __stack_depot_inc_count(max_handle, INT_MAX - 1, + &new_count)); + KUNIT_EXPECT_TRUE(test, new_count); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(max_handle, &count)); + KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX); + new_count = true; + KUNIT_EXPECT_FALSE(test, + __stack_depot_inc_count(max_handle, 1, &new_count)); + KUNIT_EXPECT_FALSE(test, new_count); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(max_handle, &count)); KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX); + KUNIT_EXPECT_TRUE(test, + __stack_depot_dec_count_and_test(max_handle, INT_MAX)); + KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(max_handle, &count)); - KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 2)); + new_count = false; + KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 2, &new_count)); + KUNIT_EXPECT_TRUE(test, new_count); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 3); - /* Already-counted records take the refcount_add() path. */ - KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(handle, 4)); + /* Already-counted records increment without needing a list marker. */ + new_count = true; + KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 4, &new_count)); + KUNIT_EXPECT_FALSE(test, new_count); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 7); @@ -328,14 +360,18 @@ static void stackdepot_count_helpers(struct kunit *test) __stack_depot_set_count(handle, 6); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 6); - KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(handle, INT_MAX)); + KUNIT_EXPECT_FALSE(test, + __stack_depot_inc_count(handle, INT_MAX, &new_count)); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); KUNIT_EXPECT_EQ(test, count, 6); second_handle = stack_depot_save(zero_entries, ARRAY_SIZE(zero_entries), GFP_KERNEL); KUNIT_ASSERT_NE(test, second_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(second_handle, 1)); + new_count = false; + KUNIT_EXPECT_TRUE(test, + __stack_depot_inc_count(second_handle, 1, &new_count)); + KUNIT_EXPECT_TRUE(test, new_count); KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(second_handle, 1)); KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(second_handle, &count)); @@ -354,6 +390,7 @@ static void stackdepot_count_helpers(struct kunit *test) KUNIT_ASSERT_NE(test, zeroed_handle, (depot_stack_handle_t)0); __stack_depot_set_count(zeroed_handle, 3); KUNIT_EXPECT_TRUE(test, __stack_depot_dec_count_and_test(zeroed_handle, 3)); + KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(zeroed_handle, 1)); } static void stackdepot_frame_raw_fallback(struct kunit *test) @@ -594,6 +631,42 @@ static void stackdepot_frame_run_x86_64_write_rejects_mismatch(struct kunit *tes } #endif /* CONFIG_X86_64 */ +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) +static void stackdepot_frame_run_compressed_rejects_src_scratch_overlap(struct kunit *test) +{ + unsigned long entries[] = { +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x1000UL, + arch_stack_depot_frame_text_prefix() | 0x2000UL, +#else + 0xffffffff81001000UL, + 0xffffffff81002000UL, +#endif + }; + unsigned long alias[ARRAY_SIZE(entries)] = {}; + unsigned long out[ARRAY_SIZE(entries)] = { 0xa5a5UL, 0xb6b6UL }; + unsigned long old[ARRAY_SIZE(out)]; + struct stack_depot_frame_run run; + u32 payload[ARRAY_SIZE(entries)]; + u32 write_scratch[ARRAY_SIZE(entries)]; + int ret; + + memcpy(old, out, sizeof(old)); + ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, run.mode, STACK_DEPOT_FRAME_COMPRESSED); + ret = frame_run_write(&run, entries, payload, sizeof(payload), + write_scratch, ARRAY_SIZE(write_scratch)); + KUNIT_ASSERT_EQ(test, ret, 0); + memcpy(alias, payload, run.bytes); + + ret = frame_run_read(&run, alias, run.bytes, out, ARRAY_SIZE(out), + alias, ARRAY_SIZE(alias)); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_MEMEQ(test, out, old, sizeof(out)); +} +#endif + static void stackdepot_frame_run_invalid_inputs(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -659,6 +732,54 @@ static void stackdepot_trie_node_parent_chain(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } +static void stackdepot_trie_node_rejects_stack_len_overflow(struct kunit *test) +{ + unsigned long exact_child[] = { 0x80000000UL }; + unsigned long overflow_child[] = { 0x80001000UL, 0x80002000UL }; + unsigned int parent_len = CONFIG_STACKDEPOT_MAX_FRAMES - 1; + struct stack_depot_frame_run run; + unsigned long *parent_entries; + void *parent; + void *child; + size_t size; + int ret; + + if (CONFIG_STACKDEPOT_MAX_FRAMES < 2) { + kunit_skip(test, "stack length overflow test needs at least two frames"); + return; + } + + parent_entries = kunit_kcalloc(test, parent_len, sizeof(*parent_entries), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, parent_entries); + trie_fill_raw_entries(parent_entries, parent_len, 0x1000UL); + KUNIT_ASSERT_EQ(test, frame_run_init(parent_entries, parent_len, &run), 0); + size = __stack_depot_trie_node_size(&run); + KUNIT_ASSERT_GT(test, size, (size_t)0); + parent = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, parent); + ret = tnode_init(parent, size, NULL, 0, parent_entries, parent_len, + NULL, 0); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = frame_run_init(exact_child, ARRAY_SIZE(exact_child), &run); + KUNIT_ASSERT_EQ(test, ret, 0); + size = __stack_depot_trie_node_size(&run); + child = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child); + ret = tnode_init(child, size, parent, 1, exact_child, + ARRAY_SIZE(exact_child), NULL, 0); + KUNIT_EXPECT_EQ(test, ret, 0); + + ret = frame_run_init(overflow_child, ARRAY_SIZE(overflow_child), &run); + KUNIT_ASSERT_EQ(test, ret, 0); + size = __stack_depot_trie_node_size(&run); + child = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child); + ret = tnode_init(child, size, parent, 2, overflow_child, + ARRAY_SIZE(overflow_child), NULL, 0); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + static void stackdepot_trie_node_match_raw(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; @@ -754,6 +875,55 @@ static void stackdepot_trie_append_chain_parent(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } +static void stackdepot_trie_append_chain_rejects_stack_len_overflow(struct kunit *test) +{ + unsigned long entries[] = { 0x80001000UL, 0x80002000UL }; + unsigned int parent_len = CONFIG_STACKDEPOT_MAX_FRAMES - 1; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_frame_run run; + unsigned long *parent_entries; + unsigned char *old; + const void *head = NULL; + const void *tail = NULL; + unsigned int used = 99; + void *parent; + size_t size; + int ret; + + if (CONFIG_STACKDEPOT_MAX_FRAMES < 2) { + kunit_skip(test, "stack length overflow test needs at least two frames"); + return; + } + + parent_entries = kunit_kcalloc(test, parent_len, sizeof(*parent_entries), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, parent_entries); + trie_fill_raw_entries(parent_entries, parent_len, 0x1000UL); + KUNIT_ASSERT_EQ(test, frame_run_init(parent_entries, parent_len, &run), 0); + size = __stack_depot_trie_node_size(&run); + parent = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, parent); + ret = tnode_init(parent, size, NULL, 0, parent_entries, parent_len, + NULL, 0); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_ASSERT_EQ(test, frame_run_init(entries, ARRAY_SIZE(entries), &run), 0); + node_slot.size = __stack_depot_trie_node_size(&run); + node_slot.node = kunit_kzalloc(test, node_slot.size, GFP_KERNEL); + old = kunit_kzalloc(test, node_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, node_slot.node); + KUNIT_ASSERT_NOT_NULL(test, old); + memset(node_slot.node, 0xaa, node_slot.size); + memcpy(old, node_slot.node, node_slot.size); + + ret = append_chain(parent, 18, entries, ARRAY_SIZE(entries), &node_slot, 1, + NULL, 0, NULL, 0, &head, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_MEMEQ(test, node_slot.node, old, node_slot.size); + KUNIT_EXPECT_NULL(test, head); + KUNIT_EXPECT_NULL(test, tail); + KUNIT_EXPECT_EQ(test, used, 99U); +} + static void stackdepot_trie_publish_append_root(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -1358,11 +1528,10 @@ static void stackdepot_trie_insert_append_splits_frame_runs(struct kunit *test) unsigned int i; int ret; - for (i = 0; i < ARRAY_SIZE(node_slots); i++) { - node_slots[i].size = 128; - node_slots[i].node = kunit_kzalloc(test, node_slots[i].size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, node_slots[i].node); - } + trie_node_slot_alloc(test, &node_slots[0], entries, 2); + trie_node_slot_alloc(test, &node_slots[1], &entries[2], 1); + trie_node_slot_alloc(test, &node_slots[2], &entries[3], 1); + for (i = 0; i < ARRAY_SIZE(child_slots); i++) { child_slots[i].size = __stack_depot_trie_child_array_size(1); child_slots[i].array = kunit_kzalloc(test, child_slots[i].size, GFP_KERNEL); @@ -1397,7 +1566,7 @@ static void stackdepot_trie_insert_append_rejects_existing_child(struct kunit *t struct stack_depot_trie_node_slot old_slot; struct stack_depot_trie_node_slot dup_slot; struct stack_depot_trie_root root = {}; - unsigned char old[128]; + unsigned char *old; const void *old_head = NULL; const void *old_tail = NULL; const void *tail = (const void *)1; @@ -1406,7 +1575,8 @@ static void stackdepot_trie_insert_append_rejects_existing_child(struct kunit *t trie_node_slot_alloc(test, &old_slot, entries, ARRAY_SIZE(entries)); trie_node_slot_alloc(test, &dup_slot, entries, ARRAY_SIZE(entries)); - KUNIT_ASSERT_LE(test, dup_slot.size, sizeof(old)); + old = kunit_kzalloc(test, dup_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old); memset(dup_slot.node, 0xaa, dup_slot.size); memcpy(old, dup_slot.node, dup_slot.size); old_array.size = __stack_depot_trie_child_array_size(1); @@ -1464,13 +1634,14 @@ static void stackdepot_trie_insert_append_rejects_zero_frame(struct kunit *test) struct stack_depot_trie_child_array_slot child_array; struct stack_depot_trie_node_slot node_slot; struct stack_depot_trie_root root = {}; - unsigned char old[128]; + unsigned char *old; const void *tail = (const void *)1; unsigned int used = 99; int ret; trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - KUNIT_ASSERT_LE(test, node_slot.size, sizeof(old)); + old = kunit_kzalloc(test, node_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old); memset(node_slot.node, 0xaa, node_slot.size); memcpy(old, node_slot.node, node_slot.size); child_array.size = __stack_depot_trie_child_array_size(1); @@ -1549,7 +1720,7 @@ static void stackdepot_trie_insert_append_rejects_parent_overlap(struct kunit *t struct stack_depot_trie_lookup lookup; const void *tail = (const void *)1; unsigned int used = 99; - unsigned char old[128]; + unsigned char *old; int ret; trie_node_slot_alloc(test, &parent_slot, parent_entries, @@ -1559,7 +1730,8 @@ static void stackdepot_trie_insert_append_rejects_parent_overlap(struct kunit *t KUNIT_ASSERT_EQ(test, ret, 0); trie_node_slot_alloc(test, &child_slot, child_entries, ARRAY_SIZE(child_entries)); - KUNIT_ASSERT_LE(test, child_slot.size, sizeof(old)); + old = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old); memset(child_slot.node, 0xaa, child_slot.size); memcpy(old, child_slot.node, child_slot.size); @@ -1582,13 +1754,14 @@ static void stackdepot_trie_insert_append_rejects_publish_overlap(struct kunit * unsigned long entries[] = { 0x1000UL }; struct stack_depot_trie_node_slot node_slot; struct stack_depot_trie_root root = {}; - unsigned char old[128]; + unsigned char *old; const void *tail = (const void *)1; unsigned int used = 99; int ret; trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - KUNIT_ASSERT_LE(test, node_slot.size, sizeof(old)); + old = kunit_kzalloc(test, node_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old); memset(node_slot.node, 0xaa, node_slot.size); memcpy(old, node_slot.node, node_slot.size); @@ -1614,7 +1787,7 @@ static void stackdepot_trie_insert_append_rejects_child_node_overlap(struct kuni const void *old_head = NULL; const void *old_tail = NULL; const void *tail = (const void *)1; - unsigned char old[128]; + unsigned char *old; unsigned int used = 99; int ret; @@ -1629,7 +1802,8 @@ static void stackdepot_trie_insert_append_rejects_child_node_overlap(struct kuni ret = publish_append(&root, NULL, old_head, old_array.array, old_array.size); KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_ASSERT_LE(test, old_array.size, sizeof(old)); + old = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old); memcpy(old, old_array.array, old_array.size); new_slot.node = old_array.array; new_slot.size = old_array.size; @@ -1661,7 +1835,7 @@ static void stackdepot_trie_insert_append_rejects_child_array_overlap(struct kun const void *old_head = NULL; const void *old_tail = NULL; const void *tail = (const void *)1; - unsigned char old[128]; + unsigned char *old; unsigned int used = 99; int ret; @@ -1677,7 +1851,8 @@ static void stackdepot_trie_insert_append_rejects_child_array_overlap(struct kun ret = publish_append(&root, NULL, old_head, old_array.array, old_array.size); KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_ASSERT_LE(test, old_array.size, sizeof(old)); + old = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old); memcpy(old, old_array.array, old_array.size); child_slot.array = old_array.array; child_slot.size = old_array.size; @@ -1774,14 +1949,10 @@ static void stackdepot_trie_append_chain_splits_frame_runs(struct kunit *test) unsigned int i; int ret; - for (i = 0; i < ARRAY_SIZE(node_slots); i++) { - size_t size; + trie_node_slot_alloc(test, &node_slots[0], entries, 2); + trie_node_slot_alloc(test, &node_slots[1], &entries[2], 1); + trie_node_slot_alloc(test, &node_slots[2], &entries[3], 1); - node_slots[i].size = 128; - size = node_slots[i].size; - node_slots[i].node = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, node_slots[i].node); - } for (i = 0; i < ARRAY_SIZE(child_slots); i++) { size_t size; @@ -1828,12 +1999,8 @@ static void stackdepot_trie_append_chain_rejects_bad_inputs(struct kunit *test) unsigned int used = 0; int ret; - node_slots[0].size = 128; - node_slots[0].node = kunit_kzalloc(test, node_slots[0].size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, node_slots[0].node); - node_slots[1].size = 128; - node_slots[1].node = kunit_kzalloc(test, node_slots[1].size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, node_slots[1].node); + trie_node_slot_alloc(test, &node_slots[0], entries, 1); + trie_node_slot_alloc(test, &node_slots[1], &entries[1], 1); child_slot.size = __stack_depot_trie_child_array_size(1); child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, child_slot.array); @@ -2003,6 +2170,27 @@ static void stackdepot_trie_child_array_rejects_unsorted(struct kunit *test) KUNIT_EXPECT_EQ(test, ret, -EINVAL); } +static void stackdepot_trie_child_array_init_rejects_child_overlap(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + const void *children[1]; + unsigned char *old; + void *node; + size_t size; + int ret; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 1, &node); + children[0] = node; + size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); + old = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old); + memcpy(old, node, size); + + ret = child_array_init(node, size, children, ARRAY_SIZE(children)); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_MEMEQ(test, node, old, size); +} + static void stackdepot_trie_child_array_insert(struct kunit *test) { unsigned long first_entries[] = { 0x1000UL }; @@ -2047,6 +2235,25 @@ static void stackdepot_trie_child_array_insert(struct kunit *test) KUNIT_EXPECT_EQ(test, ret, -EINVAL); } +static void stackdepot_trie_child_array_insert_rejects_child_overlap(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + unsigned char *old; + void *child; + size_t size; + int ret; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 1, &child); + size = __stack_depot_trie_child_array_size(1); + old = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old); + memcpy(old, child, size); + + ret = child_array_insert(NULL, child, child, size); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_MEMEQ(test, child, old, size); +} + static void stackdepot_trie_child_array_insert_empty(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -2083,13 +2290,18 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_frame_run_x86_64_roundtrip), KUNIT_CASE(stackdepot_frame_run_x86_64_boundary), KUNIT_CASE(stackdepot_frame_run_x86_64_write_rejects_mismatch), +#endif +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) + KUNIT_CASE(stackdepot_frame_run_compressed_rejects_src_scratch_overlap), #endif KUNIT_CASE(stackdepot_frame_run_invalid_inputs), KUNIT_CASE(stackdepot_trie_node_raw_roundtrip), KUNIT_CASE(stackdepot_trie_node_parent_chain), + KUNIT_CASE(stackdepot_trie_node_rejects_stack_len_overflow), KUNIT_CASE(stackdepot_trie_node_match_raw), KUNIT_CASE(stackdepot_trie_append_chain_raw), KUNIT_CASE(stackdepot_trie_append_chain_parent), + KUNIT_CASE(stackdepot_trie_append_chain_rejects_stack_len_overflow), KUNIT_CASE(stackdepot_trie_publish_append_root), KUNIT_CASE(stackdepot_trie_publish_append_parent), KUNIT_CASE(stackdepot_trie_publish_append_root_replaces_array), @@ -2128,7 +2340,9 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_fetch_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_child_array_init_find), KUNIT_CASE(stackdepot_trie_child_array_rejects_unsorted), + KUNIT_CASE(stackdepot_trie_child_array_init_rejects_child_overlap), KUNIT_CASE(stackdepot_trie_child_array_insert), + KUNIT_CASE(stackdepot_trie_child_array_insert_rejects_child_overlap), KUNIT_CASE(stackdepot_trie_child_array_insert_empty), {} }; diff --git a/mm/page_owner.c b/mm/page_owner.c index f4f0882159587..f6339a4d12577 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -130,6 +130,7 @@ static __init void init_page_owner(void) dummy_stack.handle = dummy_handle; failure_stack.handle = failure_handle; /* These counts are the stack_list membership markers. */ + /* No page_owner count updates can race before page_owner_inited flips. */ if (dummy_handle) __stack_depot_set_count(dummy_handle, 1); if (failure_handle) @@ -214,33 +215,49 @@ static void add_stack_record_to_list(depot_stack_handle_t handle, spin_unlock_irqrestore(&stack_list_lock, flags); } -static void inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, +static bool inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, unsigned int nr_base_pages) { struct stack *stack = NULL; + bool new_count = false; unsigned int count; - if (!handle) - return; + if (!handle || !nr_base_pages) + return false; + /* Snapshot only avoids allocation when the stack is already counted. */ + /* If this races a final decrement to zero, inc_count() fails safely. */ if (!__stack_depot_get_count(handle, &count)) { stack = alloc_stack_record(gfp_mask); - /* Leave saturated stacks retryable if no list marker can be tracked. */ + /* Leave saturated stacks retryable for future tracked allocations. */ if (!stack) - return; + return false; } - /* The saturated-to-counted transition reserves the stack_list marker. */ - if (__stack_depot_inc_count(handle, nr_base_pages)) + /* Racing transition losers free their unused list node below. */ + if (!__stack_depot_inc_count(handle, nr_base_pages, &new_count)) { + if (stack) + free_stack_record(stack); + return false; + } + /* new_count is only possible after allocating the list node above. */ + if (new_count) { + if (WARN_ON_ONCE(!stack)) { + __stack_depot_dec_count_and_test(handle, nr_base_pages + 1); + return false; + } add_stack_record_to_list(handle, stack); - else if (stack) + } else if (stack) { free_stack_record(stack); + } + + return true; } static void dec_stack_record_count(depot_stack_handle_t handle, unsigned int nr_base_pages) { - /* Successful list insertion leaves a marker count; zero means corruption. */ + /* Successful list insertion leaves a marker; zero means it was decremented. */ if (__stack_depot_dec_count_and_test(handle, nr_base_pages)) pr_warn("%s: refcount went to 0 for %u handle\n", __func__, handle); @@ -326,7 +343,7 @@ void __reset_page_owner(struct page *page, unsigned short order) __update_page_owner_free_handle(page, handle, order, current->pid, current->tgid, free_ts_nsec); - if (alloc_handle != early_handle) + if (alloc_handle && alloc_handle != early_handle) /* * early_handle is being set as a handle for all those * early allocated pages. See init_pages_in_zone(). @@ -342,12 +359,24 @@ noinline void __set_page_owner(struct page *page, unsigned short order, { u64 ts_nsec = local_clock(); depot_stack_handle_t handle; + bool counted; + /* Any previous allocation handle for this page was decremented at free. */ handle = save_stack(gfp_mask); + counted = inc_stack_record_count(handle, gfp_mask, 1 << order); + if (!counted && handle != failure_handle) { + /* Attribute to failure_handle only if it can be symmetrically counted. */ + handle = failure_handle; + counted = inc_stack_record_count(handle, gfp_mask, 1 << order); + } + /* Avoid storing a handle that would later decrement an unapplied count. */ + if (!counted) { + pr_warn_ratelimited("failed to count page owner stack\n"); + handle = 0; + } __update_page_owner_handle(page, handle, order, gfp_mask, -1, ts_nsec, current->pid, current->tgid, current->comm); - inc_stack_record_count(handle, gfp_mask, 1 << order); } void __folio_set_owner_migrate_reason(struct folio *folio, int reason) @@ -932,7 +961,7 @@ static int stack_print(struct seq_file *m, void *v) /* Keep show_stacks independent of stackdepot's internal storage layout. */ nr_entries = stack_depot_fetch_into(handle, priv->entries, ARRAY_SIZE(priv->entries)); - /* Buffer matches save_stack()'s stored-depth cap. */ + /* Buffer matches save_stack()'s cap, so exact-or-nothing fetch should fit. */ if (!nr_entries) return 0; From 2e18d10b8ed976fc3ba60f22ad5bfc4e8a73550a Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 3 Jun 2026 10:25:15 +0100 Subject: [PATCH 027/129] KRN-1117: Add stackdepot trie promotion Teach the private append insertion helper to promote a fully matched, childless internal trie node into a terminal leaf by publishing a copy-on-write replacement child array. Keep split and promote-with- children cases rejected until subtree COW support lands. Cover root-level promotion, descended promotion, and the childful rejection path in stackdepot KUnit. Also keep raw node matching on the direct raw-frame path so mismatch handling avoids the generic decoder. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 190 ++++++++++++++++++++++++++++++++++- lib/tests/stackdepot_kunit.c | 171 +++++++++++++++++++++++++++++++ 2 files changed, 356 insertions(+), 5 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 8fc36bd947cba..facc9e4562c91 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1317,9 +1317,17 @@ unsigned int __stack_depot_trie_node_match(const void *node_ptr, return 0; limit = min(node->run.nr_entries, nr_entries); - if (node->run.mode == STACK_DEPOT_FRAME_RAW && - !memcmp(node->data, entries, limit * sizeof(*entries))) - return limit; + if (node->run.mode == STACK_DEPOT_FRAME_RAW) { + for (i = 0; i < limit; i++) { + unsigned long frame; + + memcpy(&frame, node->data + i * sizeof(frame), sizeof(frame)); + if (frame != entries[i]) + break; + } + + return i; + } for (i = 0; i < limit; i++) { unsigned long frame; @@ -1701,6 +1709,165 @@ static int trie_insert_descend_precheck(struct stack_depot_trie_root *root, return 0; } +static int +trie_child_array_replace_precheck(const struct stack_depot_trie_child_array *old_array, + const struct stack_depot_trie_node *old_child, + void *new_storage, size_t new_storage_size, + unsigned int *pos) +{ + struct stack_depot_trie_child_array *new_array = new_storage; + unsigned long frame; + bool found; + size_t size; + + if (!old_array || !old_child || !new_array || !pos) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)new_array, + __alignof__(struct stack_depot_trie_child_array))) + return -EINVAL; + if (old_array == new_array) + return -EINVAL; + + size = __stack_depot_trie_child_array_size(old_array->nr_children); + if (!size || new_storage_size < size) + return -EINVAL; + if (stack_depot_ranges_overlap(old_array, size, new_array, new_storage_size)) + return -EINVAL; + if (stack_depot_trie_node_first_frame(old_child, &frame)) + return -EINVAL; + if (stack_depot_trie_child_lower_bound(old_array, frame, pos, &found) || + !found || old_array->children[*pos] != old_child) + return -EINVAL; + + return 0; +} + +static void +trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array, + const struct stack_depot_trie_node *new_child, + void *new_storage, unsigned int pos) +{ + struct stack_depot_trie_child_array *new_array = new_storage; + unsigned int i; + + new_array->nr_children = old_array->nr_children; + for (i = 0; i < old_array->nr_children; i++) + new_array->children[i] = old_array->children[i]; + new_array->children[pos] = new_child; +} + +static int +trie_clone_promoted_node(const struct stack_depot_trie_node *old_node, + u32 leaf_id, + const struct stack_depot_trie_node_slot *slot) +{ + struct stack_depot_trie_node *new_node; + size_t size; + + if (!old_node || !leaf_id || !slot || !slot->node) + return -EINVAL; + if (old_node->children || old_node->leaf_id || !old_node->stack_len) + return -EINVAL; + if (stack_depot_frame_run_validate(&old_node->run)) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)slot->node, + __alignof__(struct stack_depot_trie_node))) + return -EINVAL; + + size = __stack_depot_trie_node_size(&old_node->run); + if (!size || slot->size < size) + return -EINVAL; + if (stack_depot_ranges_overlap(slot->node, slot->size, old_node, size)) + return -EINVAL; + + new_node = slot->node; + memcpy(new_node, old_node, size); + new_node->leaf_id = leaf_id; + return 0; +} + +static int +trie_promote_precheck(struct stack_depot_trie_root *root, + struct stack_depot_trie_node *parent, + const struct stack_depot_trie_node *child, + const struct stack_depot_trie_node_slot *slot, + void *new_storage, size_t new_storage_size, + const struct stack_depot_trie_child_array **old_array, + unsigned int *pos) +{ + const struct stack_depot_trie_child_array **publish_slot; + const struct stack_depot_trie_child_array *array; + const struct stack_depot_trie_node *new_child; + size_t new_child_size; + + if (!child || !slot || !slot->node || !new_storage || !old_array || !pos) + return -EINVAL; + if (child->parent != parent || child->children || child->leaf_id) + return -EINVAL; + + publish_slot = trie_publish_slot(root, parent); + if (!publish_slot) + return -EINVAL; + if (stack_depot_ranges_overlap(slot->node, slot->size, publish_slot, + sizeof(*publish_slot))) + return -EINVAL; + if (stack_depot_ranges_overlap(new_storage, new_storage_size, publish_slot, + sizeof(*publish_slot))) + return -EINVAL; + if (parent && (trie_ancestor_overlaps(parent, slot->node, slot->size) || + trie_ancestor_overlaps(parent, new_storage, new_storage_size))) + return -EINVAL; + if (stack_depot_ranges_overlap(slot->node, slot->size, new_storage, + new_storage_size)) + return -EINVAL; + + /* Pairs with append and promote publication's smp_store_release(). */ + *old_array = smp_load_acquire(publish_slot); + if (!*old_array) + return -EINVAL; + array = *old_array; + new_child = slot->node; + new_child_size = __stack_depot_trie_node_size(&child->run); + if (!new_child_size || slot->size < new_child_size) + return -EINVAL; + if (trie_child_array_subtree_overlaps(array, parent, new_child, new_child_size)) + return -EINVAL; + if (trie_child_array_subtree_overlaps(array, parent, new_storage, new_storage_size)) + return -EINVAL; + + return trie_child_array_replace_precheck(array, child, new_storage, + new_storage_size, pos); +} + +static int +trie_promote_child(struct stack_depot_trie_root *root, + struct stack_depot_trie_node *parent, + const struct stack_depot_trie_node *child, u32 leaf_id, + const struct stack_depot_trie_node_slot *slot, + void *new_storage, size_t new_storage_size) +{ + const struct stack_depot_trie_child_array **publish_slot; + const struct stack_depot_trie_child_array *old_array; + unsigned int pos; + int ret; + + if (!leaf_id) + return -EINVAL; + ret = trie_promote_precheck(root, parent, child, slot, new_storage, + new_storage_size, &old_array, &pos); + if (ret) + return ret; + ret = trie_clone_promoted_node(child, leaf_id, slot); + if (ret) + return ret; + trie_child_array_replace_at(old_array, slot->node, new_storage, pos); + + publish_slot = trie_publish_slot(root, parent); + /* Publish the fully initialized replacement array last. */ + smp_store_release(publish_slot, new_storage); + return 0; +} + static int trie_append_chain_validate(const struct stack_depot_trie_node *parent, const unsigned long *entries, unsigned int nr_entries, @@ -1988,14 +2155,13 @@ int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, const void *last; unsigned int used; struct stack_depot_trie_node *parent = parent_ptr; + struct stack_depot_trie_lookup lookup; int ret; if (!leaf_id || !tail || !nr_used) return -EINVAL; for (;;) { - struct stack_depot_trie_lookup lookup; - ret = __stack_depot_trie_lookup_step(root, parent, entries, nr_entries, &lookup); if (ret) return ret; @@ -2016,6 +2182,20 @@ int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, if (!entries || !nr_entries) return -EINVAL; + if (lookup.status == STACK_DEPOT_TRIE_LOOKUP_PROMOTE) { + if (!node_slots || !nr_node_slots) + return -EINVAL; + ret = trie_promote_child(root, parent, lookup.node, leaf_id, + &node_slots[0], new_storage, new_storage_size); + if (ret) + return ret; + *tail = node_slots[0].node; + *nr_used = 1; + return 0; + } + if (lookup.status != STACK_DEPOT_TRIE_LOOKUP_APPEND) + return -EINVAL; + ret = trie_insert_append_precheck(root, parent, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, new_storage, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 2327b83b0e533..197146fed3b12 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1499,6 +1499,174 @@ static void stackdepot_trie_insert_append_descend_rejects_sibling_overlap(struct KUNIT_EXPECT_PTR_EQ(test, lookup.node, sibling); } +static void stackdepot_trie_insert_append_promotes_internal(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot promote_slot; + struct stack_depot_trie_lookup lookup; + struct stack_depot_trie_root root = {}; + unsigned long scratch[ARRAY_SIZE(entries)]; + unsigned long out[ARRAY_SIZE(entries)] = {}; + const void *children[1]; + const void *tail = (const void *)1; + unsigned int fetched; + unsigned int used = 99; + void *old_child; + int ret; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &old_child); + children[0] = old_child; + old_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = child_array_init(old_array.array, old_array.size, children, + ARRAY_SIZE(children)); + KUNIT_ASSERT_EQ(test, ret, 0); + root.children = old_array.array; + trie_node_slot_alloc(test, &promote_slot, entries, ARRAY_SIZE(entries)); + new_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + + ret = insert_append(&root, NULL, 58, entries, ARRAY_SIZE(entries), + &promote_slot, 1, NULL, 0, NULL, 0, new_array.array, + new_array.size, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, root.children, new_array.array); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, entries[0]), tail); + KUNIT_EXPECT_PTR_EQ(test, tail, promote_slot.node); + KUNIT_EXPECT_EQ(test, used, 1U); + fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); + ret = lookup_step(&root, NULL, entries, ARRAY_SIZE(entries), &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, tail); +} + +static void stackdepot_trie_insert_append_descends_to_promote(struct kunit *test) +{ + unsigned long prefix_entries[] = { 0x1000UL }; + unsigned long child_entries[] = { 0x2000UL }; + unsigned long stack_entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_child_array_slot root_array; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot promote_slot; + struct stack_depot_trie_lookup lookup; + struct stack_depot_trie_root root = {}; + unsigned long scratch[ARRAY_SIZE(stack_entries)]; + unsigned long out[ARRAY_SIZE(stack_entries)] = {}; + const void *root_children[1]; + const void *tail = (const void *)1; + unsigned int fetched; + unsigned int used = 99; + void *child; + void *prefix; + int ret; + + trie_node_alloc(test, prefix_entries, ARRAY_SIZE(prefix_entries), NULL, 0, + &prefix); + trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), prefix, 0, + &child); + root_children[0] = prefix; + root_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(root_children)); + root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, root_array.array); + ret = child_array_init(root_array.array, root_array.size, root_children, + ARRAY_SIZE(root_children)); + KUNIT_ASSERT_EQ(test, ret, 0); + root.children = root_array.array; + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = publish_append(NULL, prefix, child, child_array.array, + child_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + trie_node_slot_alloc(test, &promote_slot, child_entries, + ARRAY_SIZE(child_entries)); + new_array.size = __stack_depot_trie_child_array_size(1); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + + ret = insert_append(&root, NULL, 61, stack_entries, ARRAY_SIZE(stack_entries), + &promote_slot, 1, NULL, 0, NULL, 0, new_array.array, + new_array.size, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, root.children, root_array.array); + ret = lookup_step(NULL, prefix, child_entries, ARRAY_SIZE(child_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, tail); + KUNIT_EXPECT_EQ(test, used, 1U); + fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(stack_entries)); + KUNIT_EXPECT_MEMEQ(test, out, stack_entries, sizeof(stack_entries)); +} + +static void stackdepot_trie_insert_append_rejects_promote_with_children(struct kunit *test) +{ + unsigned long parent_entries[] = { 0x1000UL }; + unsigned long child_entries[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot root_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot promote_slot; + struct stack_depot_trie_lookup lookup; + struct stack_depot_trie_root root = {}; + unsigned char *old_slot; + const void *root_children[1]; + const void *tail = (const void *)1; + unsigned int used = 99; + void *child; + void *parent; + int ret; + + trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 0, + &parent); + trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), parent, 59, + &child); + root_children[0] = parent; + root_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(root_children)); + root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, root_array.array); + ret = child_array_init(root_array.array, root_array.size, root_children, + ARRAY_SIZE(root_children)); + KUNIT_ASSERT_EQ(test, ret, 0); + root.children = root_array.array; + new_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(root_children)); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + ret = publish_append(NULL, parent, child, new_array.array, new_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + trie_node_slot_alloc(test, &promote_slot, parent_entries, + ARRAY_SIZE(parent_entries)); + old_slot = kunit_kzalloc(test, promote_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_slot); + memcpy(old_slot, promote_slot.node, promote_slot.size); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + + ret = insert_append(&root, NULL, 60, parent_entries, + ARRAY_SIZE(parent_entries), &promote_slot, 1, NULL, 0, + NULL, 0, new_array.array, new_array.size, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, root.children, root_array.array); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, parent_entries[0]), + parent); + ret = lookup_step(NULL, parent, child_entries, ARRAY_SIZE(child_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, child); + KUNIT_EXPECT_MEMEQ(test, promote_slot.node, old_slot, promote_slot.size); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) static void stackdepot_trie_insert_append_splits_frame_runs(struct kunit *test) { @@ -2314,6 +2482,9 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_insert_append_descends_one_level), KUNIT_CASE(stackdepot_trie_insert_append_descends_multiple_levels), KUNIT_CASE(stackdepot_trie_insert_append_descend_rejects_sibling_overlap), + KUNIT_CASE(stackdepot_trie_insert_append_promotes_internal), + KUNIT_CASE(stackdepot_trie_insert_append_descends_to_promote), + KUNIT_CASE(stackdepot_trie_insert_append_rejects_promote_with_children), #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) KUNIT_CASE(stackdepot_trie_insert_append_splits_frame_runs), #endif From 5bf3754444174bb37b037b8603e1f55e00f9927e Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 3 Jun 2026 11:01:48 +0100 Subject: [PATCH 028/129] KRN-1117: Preserve children during trie promotion Allow the private trie promotion path to promote internal nodes that already have children by carrying the child array into the replacement node and reparenting descendants before publication. This keeps the copy-on-write shape intact while preparing split insertion to preserve existing subtrees. Cover the childful promotion path in stackdepot KUnit and keep the test node-slot allocator limited to single-run inputs. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 23 ++++++++++++++++++++--- lib/tests/stackdepot_kunit.c | 30 ++++++++++++++++-------------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index facc9e4562c91..1cc062365722f 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1766,7 +1766,7 @@ trie_clone_promoted_node(const struct stack_depot_trie_node *old_node, if (!old_node || !leaf_id || !slot || !slot->node) return -EINVAL; - if (old_node->children || old_node->leaf_id || !old_node->stack_len) + if (old_node->leaf_id || !old_node->stack_len) return -EINVAL; if (stack_depot_frame_run_validate(&old_node->run)) return -EINVAL; @@ -1786,6 +1786,22 @@ trie_clone_promoted_node(const struct stack_depot_trie_node *old_node, return 0; } +static void trie_reparent_children(struct stack_depot_trie_node *parent) +{ + const struct stack_depot_trie_child_array *children = parent->children; + unsigned int i; + + if (!children) + return; + for (i = 0; i < children->nr_children; i++) { + struct stack_depot_trie_node *child; + + /* Child arrays are const for readers; writers serialize reparenting. */ + child = (struct stack_depot_trie_node *)children->children[i]; + WRITE_ONCE(child->parent, parent); + } +} + static int trie_promote_precheck(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, @@ -1802,7 +1818,7 @@ trie_promote_precheck(struct stack_depot_trie_root *root, if (!child || !slot || !slot->node || !new_storage || !old_array || !pos) return -EINVAL; - if (child->parent != parent || child->children || child->leaf_id) + if (child->parent != parent || child->leaf_id) return -EINVAL; publish_slot = trie_publish_slot(root, parent); @@ -1861,6 +1877,7 @@ trie_promote_child(struct stack_depot_trie_root *root, if (ret) return ret; trie_child_array_replace_at(old_array, slot->node, new_storage, pos); + trie_reparent_children(slot->node); publish_slot = trie_publish_slot(root, parent); /* Publish the fully initialized replacement array last. */ @@ -2239,7 +2256,7 @@ __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, return 0; pos = total; - for (node = leaf; node; node = node->parent) { + for (node = leaf; node; node = READ_ONCE(node->parent)) { if (stack_depot_frame_run_validate(&node->run)) return 0; if (node->stack_len != pos || node->run.nr_entries > pos) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 197146fed3b12..8d4ae865ba79f 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -116,6 +116,7 @@ trie_node_slot_alloc(struct kunit *test, struct stack_depot_frame_run run; KUNIT_ASSERT_EQ(test, frame_run_init(entries, nr_entries, &run), 0); + KUNIT_ASSERT_EQ(test, run.nr_entries, nr_entries); slot->size = __stack_depot_trie_node_size(&run); KUNIT_ASSERT_GT(test, slot->size, (size_t)0); slot->node = kunit_kzalloc(test, slot->size, GFP_KERNEL); @@ -1608,7 +1609,7 @@ static void stackdepot_trie_insert_append_descends_to_promote(struct kunit *test KUNIT_EXPECT_MEMEQ(test, out, stack_entries, sizeof(stack_entries)); } -static void stackdepot_trie_insert_append_rejects_promote_with_children(struct kunit *test) +static void stackdepot_trie_insert_append_promotes_with_children(struct kunit *test) { unsigned long parent_entries[] = { 0x1000UL }; unsigned long child_entries[] = { 0x2000UL }; @@ -1617,9 +1618,12 @@ static void stackdepot_trie_insert_append_rejects_promote_with_children(struct k struct stack_depot_trie_node_slot promote_slot; struct stack_depot_trie_lookup lookup; struct stack_depot_trie_root root = {}; - unsigned char *old_slot; + unsigned long expected[] = { 0x1000UL, 0x2000UL }; + unsigned long scratch[ARRAY_SIZE(expected)]; + unsigned long out[ARRAY_SIZE(expected)] = {}; const void *root_children[1]; - const void *tail = (const void *)1; + const void *tail = NULL; + unsigned int fetched; unsigned int used = 99; void *child; void *parent; @@ -1644,27 +1648,25 @@ static void stackdepot_trie_insert_append_rejects_promote_with_children(struct k KUNIT_ASSERT_EQ(test, ret, 0); trie_node_slot_alloc(test, &promote_slot, parent_entries, ARRAY_SIZE(parent_entries)); - old_slot = kunit_kzalloc(test, promote_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_slot); - memcpy(old_slot, promote_slot.node, promote_slot.size); new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, new_array.array); ret = insert_append(&root, NULL, 60, parent_entries, ARRAY_SIZE(parent_entries), &promote_slot, 1, NULL, 0, NULL, 0, new_array.array, new_array.size, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_PTR_EQ(test, root.children, root_array.array); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, root.children, new_array.array); KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, parent_entries[0]), - parent); - ret = lookup_step(NULL, parent, child_entries, ARRAY_SIZE(child_entries), + tail); + ret = lookup_step(NULL, tail, child_entries, ARRAY_SIZE(child_entries), &lookup); KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); KUNIT_EXPECT_PTR_EQ(test, lookup.node, child); - KUNIT_EXPECT_MEMEQ(test, promote_slot.node, old_slot, promote_slot.size); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); + KUNIT_EXPECT_EQ(test, used, 1U); + fetched = tfetch(child, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); + KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) @@ -2484,7 +2486,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_insert_append_descend_rejects_sibling_overlap), KUNIT_CASE(stackdepot_trie_insert_append_promotes_internal), KUNIT_CASE(stackdepot_trie_insert_append_descends_to_promote), - KUNIT_CASE(stackdepot_trie_insert_append_rejects_promote_with_children), + KUNIT_CASE(stackdepot_trie_insert_append_promotes_with_children), #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) KUNIT_CASE(stackdepot_trie_insert_append_splits_frame_runs), #endif From 2f1fe8cea3904172b8216905bfb3248070646ace Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 3 Jun 2026 11:29:10 +0100 Subject: [PATCH 029/129] KRN-1117: Add stackdepot trie node slicing Add a private helper that initializes a trie node from a contiguous slice of an existing node's frame run. This gives split insertion a copy-on-write building block for prefix and old-tail nodes without decoding or recompressing stored frame payloads. Cover raw, compressed, parent-chain, and invalid slice cases in stackdepot KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 63 +++++++++++++++ lib/stackdepot_internal.h | 4 + lib/tests/stackdepot_kunit.c | 148 +++++++++++++++++++++++++++++++++++ 3 files changed, 215 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 1cc062365722f..493f909ca867b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1220,6 +1220,21 @@ size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) return ALIGN(size, sizeof(unsigned long)); } +static int stack_depot_frame_run_slice(const struct stack_depot_frame_run *src, + unsigned int start, unsigned int nr_entries, + struct stack_depot_frame_run *run) +{ + if (stack_depot_frame_run_validate(src) || !nr_entries || !run) + return -EINVAL; + if (start >= src->nr_entries || nr_entries > src->nr_entries - start) + return -EINVAL; + + *run = *src; + run->nr_entries = nr_entries; + run->bytes = nr_entries * stack_depot_frame_run_entry_bytes(run->mode); + return 0; +} + static int stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, unsigned int index, unsigned long *frame) @@ -1304,6 +1319,54 @@ int __stack_depot_trie_node_init(void *storage, size_t storage_size, return 0; } +int __stack_depot_trie_node_init_slice(void *storage, size_t storage_size, + const void *parent, u32 leaf_id, + const void *src_node, unsigned int start, + unsigned int nr_entries) +{ + const struct stack_depot_trie_node *parent_node = parent; + const struct stack_depot_trie_node *src = src_node; + struct stack_depot_trie_node *node = storage; + struct stack_depot_frame_run run; + size_t entry_bytes; + size_t src_size; + u32 stack_len; + int ret; + + if (!node || !src || !src->stack_len) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)node, __alignof__(*node))) + return -EINVAL; + + ret = stack_depot_frame_run_slice(&src->run, start, nr_entries, &run); + if (ret) + return ret; + if (storage_size < __stack_depot_trie_node_size(&run)) + return -EINVAL; + src_size = __stack_depot_trie_node_size(&src->run); + if (!src_size || stack_depot_ranges_overlap(node, storage_size, src, src_size)) + return -EINVAL; + if (parent_node) { + if (!parent_node->stack_len || + parent_node->stack_len > U32_MAX - run.nr_entries || + parent_node->stack_len > + CONFIG_STACKDEPOT_MAX_FRAMES - run.nr_entries) + return -EINVAL; + stack_len = parent_node->stack_len + run.nr_entries; + } else { + stack_len = run.nr_entries; + } + + entry_bytes = stack_depot_frame_run_entry_bytes(src->run.mode); + memcpy(node->data, src->data + start * entry_bytes, run.bytes); + node->parent = parent_node; + node->children = NULL; + node->leaf_id = leaf_id; + node->stack_len = stack_len; + node->run = run; + return 0; +} + unsigned int __stack_depot_trie_node_match(const void *node_ptr, const unsigned long *entries, unsigned int nr_entries) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 89d54a7885c4a..b1901aadb1b72 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -70,6 +70,10 @@ int __stack_depot_trie_node_init(void *storage, size_t storage_size, const unsigned long *entries, unsigned int nr_entries, u32 *scratch, unsigned int nr_scratch); +int __stack_depot_trie_node_init_slice(void *storage, size_t storage_size, + const void *parent, u32 leaf_id, + const void *src_node, unsigned int start, + unsigned int nr_entries); unsigned int __stack_depot_trie_node_match(const void *node, const unsigned long *entries, unsigned int nr_entries); diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 8d4ae865ba79f..33b94e7ca846e 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -48,6 +48,15 @@ static int tnode_init(void *storage, size_t storage_size, const void *parent, nr_scratch); } +static int +tnode_init_slice(void *storage, size_t storage_size, const void *parent, + u32 leaf_id, const void *src_node, unsigned int start, + unsigned int nr_entries) +{ + return __stack_depot_trie_node_init_slice(storage, storage_size, parent, + leaf_id, src_node, start, nr_entries); +} + static unsigned int tfetch(const void *leaf, unsigned long *entries, unsigned int max_entries, unsigned long *scratch, unsigned int nr_scratch) @@ -733,6 +742,139 @@ static void stackdepot_trie_node_parent_chain(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } +static void stackdepot_trie_node_slice_raw(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + unsigned long expected[] = { 0x2000UL, 0x3000UL }; + struct stack_depot_frame_run run; + unsigned long scratch[ARRAY_SIZE(expected)]; + unsigned long out[ARRAY_SIZE(expected)] = {}; + unsigned int fetched; + void *source; + void *slice; + size_t size; + int ret; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &source); + ret = frame_run_init(&entries[1], ARRAY_SIZE(expected), &run); + KUNIT_ASSERT_EQ(test, ret, 0); + size = __stack_depot_trie_node_size(&run); + KUNIT_ASSERT_GT(test, size, (size_t)0); + slice = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, slice); + ret = tnode_init_slice(slice, size, NULL, 10, source, 1, + ARRAY_SIZE(expected)); + KUNIT_ASSERT_EQ(test, ret, 0); + fetched = tfetch(slice, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); + KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); + KUNIT_EXPECT_EQ(test, tmatch(slice, expected, ARRAY_SIZE(expected)), + (unsigned int)ARRAY_SIZE(expected)); +} + +static void stackdepot_trie_node_slice_parent_chain(struct kunit *test) +{ + unsigned long root_entries[] = { 0x1000UL }; + unsigned long entries[] = { 0x2000UL, 0x3000UL, 0x4000UL }; + unsigned long expected[] = { 0x1000UL, 0x3000UL, 0x4000UL }; + struct stack_depot_frame_run run; + unsigned long scratch[ARRAY_SIZE(expected)]; + unsigned long out[ARRAY_SIZE(expected)] = {}; + unsigned int fetched; + void *root; + void *source; + void *slice; + size_t size; + int ret; + + trie_node_alloc(test, root_entries, ARRAY_SIZE(root_entries), NULL, 0, + &root); + trie_node_alloc(test, entries, ARRAY_SIZE(entries), root, 0, &source); + ret = frame_run_init(&entries[1], 2, &run); + KUNIT_ASSERT_EQ(test, ret, 0); + size = __stack_depot_trie_node_size(&run); + KUNIT_ASSERT_GT(test, size, (size_t)0); + slice = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, slice); + ret = tnode_init_slice(slice, size, root, 11, source, 1, 2); + KUNIT_ASSERT_EQ(test, ret, 0); + fetched = tfetch(slice, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); + KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); +} + +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) +static void stackdepot_trie_node_slice_compressed(struct kunit *test) +{ + unsigned long entries[] = { +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x1000UL, + arch_stack_depot_frame_text_prefix() | 0x2000UL, + arch_stack_depot_frame_text_prefix() | 0x3000UL, +#else + 0xffffffff81001000UL, + 0xffffffff81002000UL, + 0xffffffff81003000UL, +#endif + }; + unsigned long expected[] = { entries[1], entries[2] }; + struct stack_depot_frame_run run; + unsigned long scratch[ARRAY_SIZE(expected)]; + unsigned long out[ARRAY_SIZE(expected)] = {}; + unsigned int fetched; + void *source; + void *slice; + size_t size; + int ret; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &source); + ret = frame_run_init(&entries[1], ARRAY_SIZE(expected), &run); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, run.mode, STACK_DEPOT_FRAME_COMPRESSED); + size = __stack_depot_trie_node_size(&run); + KUNIT_ASSERT_GT(test, size, (size_t)0); + slice = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, slice); + ret = tnode_init_slice(slice, size, NULL, 12, source, 1, + ARRAY_SIZE(expected)); + KUNIT_ASSERT_EQ(test, ret, 0); + fetched = tfetch(slice, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); + KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); +} +#endif + +static void stackdepot_trie_node_slice_rejects_bad_inputs(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_frame_run run; + void *source; + void *slice; + size_t size; + int ret; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &source); + ret = frame_run_init(entries, 1, &run); + KUNIT_ASSERT_EQ(test, ret, 0); + size = __stack_depot_trie_node_size(&run); + KUNIT_ASSERT_GT(test, size, (size_t)0); + slice = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, slice); + + KUNIT_EXPECT_EQ(test, tnode_init_slice(NULL, size, NULL, 1, source, 0, 1), + -EINVAL); + KUNIT_EXPECT_EQ(test, tnode_init_slice(slice, size, NULL, 1, NULL, 0, 1), + -EINVAL); + KUNIT_EXPECT_EQ(test, tnode_init_slice(slice, size, NULL, 1, source, 0, 0), + -EINVAL); + KUNIT_EXPECT_EQ(test, tnode_init_slice(slice, size, NULL, 1, source, 2, 1), + -EINVAL); + KUNIT_EXPECT_EQ(test, tnode_init_slice(slice, size - 1, NULL, 1, source, 0, 1), + -EINVAL); + KUNIT_EXPECT_EQ(test, tnode_init_slice(source, size, NULL, 1, source, 0, 1), + -EINVAL); +} + static void stackdepot_trie_node_rejects_stack_len_overflow(struct kunit *test) { unsigned long exact_child[] = { 0x80000000UL }; @@ -2467,6 +2609,12 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_frame_run_invalid_inputs), KUNIT_CASE(stackdepot_trie_node_raw_roundtrip), KUNIT_CASE(stackdepot_trie_node_parent_chain), + KUNIT_CASE(stackdepot_trie_node_slice_raw), + KUNIT_CASE(stackdepot_trie_node_slice_parent_chain), +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) + KUNIT_CASE(stackdepot_trie_node_slice_compressed), +#endif + KUNIT_CASE(stackdepot_trie_node_slice_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_node_rejects_stack_len_overflow), KUNIT_CASE(stackdepot_trie_node_match_raw), KUNIT_CASE(stackdepot_trie_append_chain_raw), From c78948bdd2106a4c848fcda17e09a398bd6e1c64 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 3 Jun 2026 11:50:58 +0100 Subject: [PATCH 030/129] KRN-1117: Add stackdepot trie split child arrays Add a private helper that builds the child array for a future split prefix from the displaced old tail and an optional new stack tail. The helper keeps child ordering centralized and reuses existing child-array validation so split insertion can publish a fully initialized prefix subtree later. Cover one-child, sorted two-child, duplicate-key, short-storage, and storage-overlap cases in stackdepot KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 31 +++++++++++++ lib/stackdepot_internal.h | 3 ++ lib/tests/stackdepot_kunit.c | 85 ++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 493f909ca867b..e339222da5e5f 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2400,6 +2400,37 @@ int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, return 0; } +int __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, + const void *old_tail, + const void *new_head) +{ + const void *children[2]; + unsigned long new_frame; + unsigned long old_frame; + + if (!old_tail) + return -EINVAL; + if (!new_head) { + children[0] = old_tail; + return __stack_depot_trie_child_array_init(storage, storage_size, + children, 1); + } + + if (stack_depot_trie_node_first_frame(old_tail, &old_frame) || + stack_depot_trie_node_first_frame(new_head, &new_frame) || + old_frame == new_frame) + return -EINVAL; + if (old_frame < new_frame) { + children[0] = old_tail; + children[1] = new_head; + } else { + children[0] = new_head; + children[1] = old_tail; + } + + return __stack_depot_trie_child_array_init(storage, storage_size, children, 2); +} + static int stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *array, unsigned long frame, unsigned int *pos, bool *found) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index b1901aadb1b72..7f8e24dff162e 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -113,6 +113,9 @@ size_t __stack_depot_trie_child_array_size(unsigned int nr_children); int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, const void * const *children, unsigned int nr_children); +int __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, + const void *old_tail, + const void *new_head); const void *__stack_depot_trie_child_array_find(const void *storage, unsigned long frame); int __stack_depot_trie_child_array_insert(const void *old_storage, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 33b94e7ca846e..01f8a18d3cda3 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -139,6 +139,13 @@ static int child_array_init(void *storage, size_t storage_size, nr_children); } +static int split_child_array_init(void *storage, size_t storage_size, + const void *old_tail, const void *new_head) +{ + return __stack_depot_trie_split_child_array_init(storage, storage_size, + old_tail, new_head); +} + static int child_array_insert(const void *old_storage, const void *child, void *new_storage, size_t new_storage_size) { @@ -2566,6 +2573,81 @@ static void stackdepot_trie_child_array_insert_rejects_child_overlap(struct kuni KUNIT_EXPECT_MEMEQ(test, child, old, size); } +static void stackdepot_trie_split_child_array_init_one_child(struct kunit *test) +{ + unsigned long old_entries[] = { 0x2000UL }; + void *old_tail; + void *array; + size_t size; + int ret; + + trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, + &old_tail); + size = __stack_depot_trie_child_array_size(1); + array = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, array); + ret = split_child_array_init(array, size, old_tail, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, old_entries[0]), old_tail); +} + +static void stackdepot_trie_split_child_array_init_orders_children(struct kunit *test) +{ + unsigned long old_entries[] = { 0x3000UL }; + unsigned long new_entries[] = { 0x1000UL }; + void *new_head; + void *old_tail; + void *array; + size_t size; + int ret; + + trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, + &old_tail); + trie_node_alloc(test, new_entries, ARRAY_SIZE(new_entries), NULL, 2, + &new_head); + size = __stack_depot_trie_child_array_size(2); + array = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, array); + ret = split_child_array_init(array, size, old_tail, new_head); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, new_entries[0]), new_head); + KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, old_entries[0]), old_tail); +} + +static void stackdepot_trie_split_child_array_rejects_bad_inputs(struct kunit *test) +{ + unsigned long old_entries[] = { 0x2000UL }; + unsigned long dup_entries[] = { 0x2000UL }; + unsigned char *old; + void *old_tail; + void *dup_tail; + void *array; + size_t size; + + trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, + &old_tail); + trie_node_alloc(test, dup_entries, ARRAY_SIZE(dup_entries), NULL, 2, + &dup_tail); + size = __stack_depot_trie_child_array_size(2); + array = kunit_kzalloc(test, size, GFP_KERNEL); + old = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, array); + KUNIT_ASSERT_NOT_NULL(test, old); + memset(array, 0xaa, size); + memcpy(old, array, size); + + KUNIT_EXPECT_EQ(test, split_child_array_init(array, size, NULL, dup_tail), + -EINVAL); + KUNIT_EXPECT_EQ(test, split_child_array_init(array, size, old_tail, dup_tail), + -EINVAL); + size = __stack_depot_trie_child_array_size(1); + KUNIT_EXPECT_EQ(test, split_child_array_init(array, size, old_tail, dup_tail), + -EINVAL); + KUNIT_EXPECT_EQ(test, split_child_array_init(old_tail, size, old_tail, NULL), + -EINVAL); + KUNIT_EXPECT_MEMEQ(test, array, old, size); +} + static void stackdepot_trie_child_array_insert_empty(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -2664,6 +2746,9 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_child_array_init_rejects_child_overlap), KUNIT_CASE(stackdepot_trie_child_array_insert), KUNIT_CASE(stackdepot_trie_child_array_insert_rejects_child_overlap), + KUNIT_CASE(stackdepot_trie_split_child_array_init_one_child), + KUNIT_CASE(stackdepot_trie_split_child_array_init_orders_children), + KUNIT_CASE(stackdepot_trie_split_child_array_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_child_array_insert_empty), {} }; From bf4d8ffe3f84042a85ba3bc3ff3131e0a519b211 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 3 Jun 2026 12:16:05 +0100 Subject: [PATCH 031/129] KRN-1117: Add stackdepot trie split tail planning Add a private helper that counts and validates the node and child-array slots needed to build a future split's new stack tail. This lets split insertion preflight caller-owned storage before mutating any staged trie nodes or publishing a replacement child array. Cover single-run, mixed raw/compressed, short-slot, and invalid-input cases in stackdepot KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 60 +++++++++++++++++++++ lib/stackdepot_internal.h | 7 +++ lib/tests/stackdepot_kunit.c | 101 +++++++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index e339222da5e5f..ff4db13ab7e34 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2431,6 +2431,66 @@ int __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size return __stack_depot_trie_child_array_init(storage, storage_size, children, 2); } +int __stack_depot_trie_split_tail_plan(const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + unsigned int *nr_runs) +{ + unsigned int pos = 0; + unsigned int runs = 0; + unsigned int child_slots_needed; + unsigned int i; + + if (!entries || !nr_entries || nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES || + !node_slots || !nr_runs) + return -EINVAL; + + while (pos < nr_entries) { + const struct stack_depot_trie_node_slot *slot; + struct stack_depot_frame_run run; + size_t size; + + if (__stack_depot_frame_run_init(&entries[pos], nr_entries - pos, + &run)) + return -EINVAL; + if (runs >= nr_node_slots) + return -EINVAL; + slot = &node_slots[runs]; + if (!slot->node) + return -EINVAL; + size = __stack_depot_trie_node_size(&run); + if (!size || slot->size < size) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)slot->node, + __alignof__(struct stack_depot_trie_node))) + return -EINVAL; + + pos += run.nr_entries; + runs++; + } + + child_slots_needed = runs > 1 ? runs - 1 : 0; + if (child_slots_needed) { + if (!child_slots || nr_child_slots < child_slots_needed) + return -EINVAL; + for (i = 0; i < child_slots_needed; i++) { + size_t size = __stack_depot_trie_child_array_size(1); + + if (!child_slots[i].array || child_slots[i].size < size) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)child_slots[i].array, + __alignof__(struct stack_depot_trie_child_array))) + return -EINVAL; + } + } + + *nr_runs = runs; + return 0; +} + static int stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *array, unsigned long frame, unsigned int *pos, bool *found) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 7f8e24dff162e..d529202de38a8 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -116,6 +116,13 @@ int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, int __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, const void *old_tail, const void *new_head); +int __stack_depot_trie_split_tail_plan(const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + unsigned int *nr_runs); const void *__stack_depot_trie_child_array_find(const void *storage, unsigned long frame); int __stack_depot_trie_child_array_insert(const void *old_storage, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 01f8a18d3cda3..0f2ab62723fab 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -146,6 +146,17 @@ static int split_child_array_init(void *storage, size_t storage_size, old_tail, new_head); } +static int split_tail_plan(const unsigned long *entries, unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, unsigned int *nr_runs) +{ + return __stack_depot_trie_split_tail_plan(entries, nr_entries, node_slots, + nr_node_slots, child_slots, + nr_child_slots, nr_runs); +} + static int child_array_insert(const void *old_storage, const void *child, void *new_storage, size_t new_storage_size) { @@ -2648,6 +2659,91 @@ static void stackdepot_trie_split_child_array_rejects_bad_inputs(struct kunit *t KUNIT_EXPECT_MEMEQ(test, array, old, size); } +static void stackdepot_trie_split_tail_plan_raw(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_node_slot node_slot; + unsigned int nr_runs = 0; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + ret = split_tail_plan(entries, ARRAY_SIZE(entries), &node_slot, 1, NULL, 0, + &nr_runs); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, nr_runs, 1U); +} + +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) +static void stackdepot_trie_split_tail_plan_mixed_runs(struct kunit *test) +{ + unsigned long entries[] = { +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x1000UL, + 0x1000UL, + arch_stack_depot_frame_text_prefix() | 0x2000UL, +#else + 0xffffffff81001000UL, + 0xffff888000001000UL, + 0xffffffff81002000UL, +#endif + }; + struct stack_depot_trie_child_array_slot child_slots[2]; + struct stack_depot_trie_node_slot node_slots[3]; + unsigned int nr_runs = 0; + unsigned int i; + int ret; + + trie_node_slot_alloc(test, &node_slots[0], entries, 1); + trie_node_slot_alloc(test, &node_slots[1], &entries[1], 1); + trie_node_slot_alloc(test, &node_slots[2], &entries[2], 1); + for (i = 0; i < ARRAY_SIZE(child_slots); i++) { + size_t size; + + child_slots[i].size = __stack_depot_trie_child_array_size(1); + size = child_slots[i].size; + child_slots[i].array = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_slots[i].array); + } + + ret = split_tail_plan(entries, ARRAY_SIZE(entries), node_slots, + ARRAY_SIZE(node_slots), child_slots, + ARRAY_SIZE(child_slots), &nr_runs); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, nr_runs, 3U); +} +#endif + +static void stackdepot_trie_split_tail_plan_rejects_bad_inputs(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_node_slot short_slot; + struct stack_depot_frame_run run; + unsigned int nr_runs = 99; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); + KUNIT_ASSERT_EQ(test, ret, 0); + short_slot = node_slot; + short_slot.size = __stack_depot_trie_node_size(&run) - 1; + + ret = split_tail_plan(NULL, ARRAY_SIZE(entries), &node_slot, 1, NULL, 0, + &nr_runs); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = split_tail_plan(entries, 0, &node_slot, 1, NULL, 0, &nr_runs); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = split_tail_plan(entries, ARRAY_SIZE(entries), NULL, 1, NULL, 0, + &nr_runs); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = split_tail_plan(entries, ARRAY_SIZE(entries), &node_slot, 1, NULL, 0, + NULL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = split_tail_plan(entries, ARRAY_SIZE(entries), &short_slot, 1, NULL, 0, + &nr_runs); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + static void stackdepot_trie_child_array_insert_empty(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -2749,6 +2845,11 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_split_child_array_init_one_child), KUNIT_CASE(stackdepot_trie_split_child_array_init_orders_children), KUNIT_CASE(stackdepot_trie_split_child_array_rejects_bad_inputs), + KUNIT_CASE(stackdepot_trie_split_tail_plan_raw), +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) + KUNIT_CASE(stackdepot_trie_split_tail_plan_mixed_runs), +#endif + KUNIT_CASE(stackdepot_trie_split_tail_plan_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_child_array_insert_empty), {} }; From 91c3fa786cfb226d08c5788744cb03c7a26e06bc Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 3 Jun 2026 13:04:38 +0100 Subject: [PATCH 032/129] KRN-1117: Add stackdepot trie split precheck Add a private helper that verifies all caller-owned storage needed by a future split insertion before any staged trie node or replacement child array can be modified. The helper rejects overlaps with the publish slot, ancestors, sibling staging slots, and the live subtree. Cover the successful precheck path and representative alias failures in stackdepot KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 101 +++++++++++++++++++++++++++++++++++ lib/stackdepot_internal.h | 7 +++ lib/tests/stackdepot_kunit.c | 91 +++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index ff4db13ab7e34..f72b6b3cb3150 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2491,6 +2491,107 @@ int __stack_depot_trie_split_tail_plan(const unsigned long *entries, return 0; } +int __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, + const void *parent_ptr, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + void *new_storage, size_t new_storage_size) +{ + struct stack_depot_trie_node *parent = (void *)parent_ptr; + const struct stack_depot_trie_child_array **slot; + const struct stack_depot_trie_child_array *children; + unsigned int i; + size_t size; + + if (!new_storage || !new_storage_size || + (nr_node_slots && !node_slots) || (nr_child_slots && !child_slots)) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)new_storage, + __alignof__(struct stack_depot_trie_child_array))) + return -EINVAL; + + for (i = 0; i < nr_node_slots; i++) { + const struct stack_depot_trie_node_slot *node_slot = &node_slots[i]; + + if (!node_slot->node || !node_slot->size) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)node_slot->node, + __alignof__(struct stack_depot_trie_node))) + return -EINVAL; + if (parent && + trie_ancestor_overlaps(parent, node_slot->node, node_slot->size)) + return -EINVAL; + if (trie_node_slot_overlaps(node_slots, i, node_slot->node, node_slot->size)) + return -EINVAL; + } + + for (i = 0; i < nr_child_slots; i++) { + const struct stack_depot_trie_child_array_slot *child_slot = + &child_slots[i]; + void *array = child_slot->array; + size_t slot_size = child_slot->size; + + if (!array || !slot_size) + return -EINVAL; + if (!IS_ALIGNED((unsigned long)array, + __alignof__(struct stack_depot_trie_child_array))) + return -EINVAL; + if (parent && + trie_ancestor_overlaps(parent, array, slot_size)) + return -EINVAL; + if (trie_child_slot_overlaps(child_slots, i, array, slot_size)) + return -EINVAL; + } + + for (i = 0; i < nr_node_slots; i++) { + if (trie_child_slot_overlaps(child_slots, nr_child_slots, + node_slots[i].node, node_slots[i].size)) + return -EINVAL; + } + if (trie_node_slot_overlaps(node_slots, nr_node_slots, new_storage, + new_storage_size) || + trie_child_slot_overlaps(child_slots, nr_child_slots, new_storage, + new_storage_size)) + return -EINVAL; + + slot = trie_publish_slot(root, parent); + if (!slot) + return -EINVAL; + if (stack_depot_ranges_overlap(new_storage, new_storage_size, slot, + sizeof(*slot))) + return -EINVAL; + if (trie_node_slot_overlaps(node_slots, nr_node_slots, slot, sizeof(*slot)) || + trie_child_slot_overlaps(child_slots, nr_child_slots, slot, sizeof(*slot))) + return -EINVAL; + if (parent && trie_ancestor_overlaps(parent, new_storage, new_storage_size)) + return -EINVAL; + + /* Pairs with append, promote, and future split publication. */ + children = smp_load_acquire(slot); + if (!children) + return -EINVAL; + size = __stack_depot_trie_child_array_size(children->nr_children); + if (!size) + return -EINVAL; + if (stack_depot_ranges_overlap(children, size, new_storage, + new_storage_size)) + return -EINVAL; + if (trie_node_slot_overlaps(node_slots, nr_node_slots, children, size) || + trie_child_slot_overlaps(child_slots, nr_child_slots, children, size)) + return -EINVAL; + if (trie_node_slots_subtree_overlap(children, parent, node_slots, nr_node_slots)) + return -EINVAL; + if (trie_child_slots_subtree_overlap(children, parent, child_slots, nr_child_slots)) + return -EINVAL; + if (trie_child_array_subtree_overlaps(children, parent, new_storage, + new_storage_size)) + return -EINVAL; + + return 0; +} + static int stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *array, unsigned long frame, unsigned int *pos, bool *found) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index d529202de38a8..040b603630475 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -123,6 +123,13 @@ int __stack_depot_trie_split_tail_plan(const unsigned long *entries, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, unsigned int *nr_runs); +int __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, + const void *parent, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + void *new_storage, size_t new_storage_size); const void *__stack_depot_trie_child_array_find(const void *storage, unsigned long frame); int __stack_depot_trie_child_array_insert(const void *old_storage, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 0f2ab62723fab..177fc31fe4018 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -157,6 +157,18 @@ static int split_tail_plan(const unsigned long *entries, unsigned int nr_entries nr_child_slots, nr_runs); } +static int split_precheck(struct stack_depot_trie_root *root, const void *parent, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, void *new_storage, + size_t new_storage_size) +{ + return __stack_depot_trie_split_precheck(root, parent, node_slots, + nr_node_slots, child_slots, nr_child_slots, + new_storage, new_storage_size); +} + static int child_array_insert(const void *old_storage, const void *child, void *new_storage, size_t new_storage_size) { @@ -2744,6 +2756,83 @@ static void stackdepot_trie_split_tail_plan_rejects_bad_inputs(struct kunit *tes KUNIT_EXPECT_EQ(test, ret, -EINVAL); } +static void stackdepot_trie_split_precheck(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL }; + unsigned long new_entries[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot child_slot; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *children[1]; + void *old_child; + int ret; + + trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, + &old_child); + trie_node_slot_alloc(test, &node_slot, new_entries, ARRAY_SIZE(new_entries)); + children[0] = old_child; + old_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = child_array_init(old_array.array, old_array.size, children, + ARRAY_SIZE(children)); + KUNIT_ASSERT_EQ(test, ret, 0); + root.children = old_array.array; + child_slot.size = __stack_depot_trie_child_array_size(1); + child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_slot.array); + new_array.size = __stack_depot_trie_child_array_size(1); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + + ret = split_precheck(&root, NULL, &node_slot, 1, &child_slot, 1, + new_array.array, new_array.size); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void stackdepot_trie_split_precheck_rejects_aliases(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL }; + unsigned long new_entries[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot child_slot; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *children[1]; + void *old_child; + int ret; + + trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, + &old_child); + trie_node_slot_alloc(test, &node_slot, new_entries, ARRAY_SIZE(new_entries)); + children[0] = old_child; + old_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = child_array_init(old_array.array, old_array.size, children, + ARRAY_SIZE(children)); + KUNIT_ASSERT_EQ(test, ret, 0); + root.children = old_array.array; + child_slot.size = __stack_depot_trie_child_array_size(1); + child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_slot.array); + new_array.size = __stack_depot_trie_child_array_size(1); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + + ret = split_precheck(&root, NULL, &node_slot, 1, &child_slot, 1, + old_array.array, old_array.size); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + node_slot.node = old_array.array; + node_slot.size = old_array.size; + ret = split_precheck(&root, NULL, &node_slot, 1, &child_slot, 1, + new_array.array, new_array.size); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + static void stackdepot_trie_child_array_insert_empty(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -2850,6 +2939,8 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_split_tail_plan_mixed_runs), #endif KUNIT_CASE(stackdepot_trie_split_tail_plan_rejects_bad_inputs), + KUNIT_CASE(stackdepot_trie_split_precheck), + KUNIT_CASE(stackdepot_trie_split_precheck_rejects_aliases), KUNIT_CASE(stackdepot_trie_child_array_insert_empty), {} }; From 546033d06560983ff32054bdc95465417d9305f3 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 3 Jun 2026 13:59:16 +0100 Subject: [PATCH 033/129] KRN-1117: Add stackdepot trie split subtree Add a private helper that builds an unpublished copy-on-write split subtree from an existing trie node. The helper materializes the shared prefix, displaced old tail, and optional new tail chain while preserving existing descendants under the old tail. Cover divergent split construction in stackdepot KUnit so later patches can wire split publication separately. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 194 +++++++++++++++++++++++++++++++++++ lib/stackdepot_internal.h | 9 ++ lib/tests/stackdepot_kunit.c | 60 +++++++++++ 3 files changed, 263 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index f72b6b3cb3150..8fc928d8265e9 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2592,6 +2592,200 @@ int __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, return 0; } +static bool +trie_split_subtree_overlaps(const struct stack_depot_trie_node *child, + const void *ptr, size_t size) +{ + if (trie_ancestor_overlaps(child, ptr, size)) + return true; + return trie_child_array_subtree_overlaps(child->children, child, ptr, + size); +} + +static bool +trie_split_slots_overlap(const struct stack_depot_trie_node *child, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots) +{ + unsigned int i; + + for (i = 0; i < nr_node_slots; i++) { + const struct stack_depot_trie_node_slot *slot = &node_slots[i]; + + if (trie_split_subtree_overlaps(child, slot->node, slot->size)) + return true; + if (trie_node_slot_overlaps(node_slots, i, slot->node, + slot->size)) + return true; + if (trie_child_slot_overlaps(child_slots, nr_child_slots, + slot->node, slot->size)) + return true; + } + + for (i = 0; i < nr_child_slots; i++) { + const struct stack_depot_trie_child_array_slot *slot = + &child_slots[i]; + + if (trie_split_subtree_overlaps(child, slot->array, slot->size)) + return true; + if (trie_child_slot_overlaps(child_slots, i, slot->array, + slot->size)) + return true; + } + + return false; +} + +static int +trie_split_subtree_precheck(const struct stack_depot_trie_node *child, + unsigned int matched, u32 leaf_id, + const unsigned long *entries, unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, unsigned int *new_runs) +{ + struct stack_depot_frame_run old_tail_run; + struct stack_depot_frame_run prefix_run; + const unsigned long *tail_entries; + unsigned int child_slots_needed; + unsigned int slots_needed; + unsigned int tail_child_slots; + unsigned int tail_node_slots; + unsigned int tail_len; + bool has_new_tail; + size_t size; + int ret; + + if (!child || !leaf_id || !entries || !nr_entries || !node_slots || + !child_slots || !new_runs) + return -EINVAL; + if (!matched || matched >= child->run.nr_entries || + matched > nr_entries) + return -EINVAL; + if (!child->leaf_id && !child->children) + return -EINVAL; + if (stack_depot_frame_run_slice(&child->run, 0, matched, &prefix_run)) + return -EINVAL; + tail_len = child->run.nr_entries - matched; + if (stack_depot_frame_run_slice(&child->run, matched, tail_len, &old_tail_run)) + return -EINVAL; + + has_new_tail = matched < nr_entries; + *new_runs = 0; + if (has_new_tail) { + tail_entries = &entries[matched]; + tail_len = nr_entries - matched; + tail_node_slots = nr_node_slots > 2 ? nr_node_slots - 2 : 0; + tail_child_slots = nr_child_slots > 1 ? nr_child_slots - 1 : 0; + ret = __stack_depot_trie_split_tail_plan(tail_entries, tail_len, + &node_slots[2], tail_node_slots, + tail_child_slots ? &child_slots[1] : NULL, + tail_child_slots, new_runs); + if (ret) + return ret; + } + + slots_needed = 2 + *new_runs; + child_slots_needed = 1 + (*new_runs ? *new_runs - 1 : 0); + if (nr_node_slots < slots_needed || nr_child_slots < child_slots_needed) + return -EINVAL; + + size = __stack_depot_trie_node_size(&prefix_run); + if (!node_slots[0].node || node_slots[0].size < size) + return -EINVAL; + size = __stack_depot_trie_node_size(&old_tail_run); + if (!node_slots[1].node || node_slots[1].size < size) + return -EINVAL; + size = __stack_depot_trie_child_array_size(has_new_tail ? 2 : 1); + if (!child_slots[0].array || child_slots[0].size < size) + return -EINVAL; + if (trie_split_slots_overlap(child, node_slots, slots_needed, + child_slots, child_slots_needed)) + return -EINVAL; + + return 0; +} + +int __stack_depot_trie_split_subtree(const void *child_ptr, unsigned int matched, + u32 leaf_id, const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, const void **prefix, + const void **tail, unsigned int *nr_used) +{ + const struct stack_depot_trie_node *child = child_ptr; + const unsigned long *tail_entries; + const void *new_head = NULL; + const void *new_tail = NULL; + struct stack_depot_trie_node *old_tail; + struct stack_depot_trie_node *pref; + unsigned int chain_used = 0; + u32 prefix_leaf_id; + void *split_array; + size_t split_array_size; + unsigned int new_runs; + unsigned int tail_len; + bool has_new_tail; + int ret; + + if (!prefix || !tail || !nr_used) + return -EINVAL; + ret = trie_split_subtree_precheck(child, matched, leaf_id, entries, + nr_entries, node_slots, nr_node_slots, + child_slots, nr_child_slots, + &new_runs); + if (ret) + return ret; + + pref = node_slots[0].node; + old_tail = node_slots[1].node; + has_new_tail = matched < nr_entries; + prefix_leaf_id = has_new_tail ? 0 : leaf_id; + ret = __stack_depot_trie_node_init_slice(pref, node_slots[0].size, + child->parent, prefix_leaf_id, + child, 0, matched); + if (ret) + return ret; + tail_len = child->run.nr_entries - matched; + ret = __stack_depot_trie_node_init_slice(old_tail, node_slots[1].size, + pref, child->leaf_id, child, + matched, tail_len); + if (ret) + return ret; + + if (has_new_tail) { + tail_entries = &entries[matched]; + tail_len = nr_entries - matched; + ret = __stack_depot_trie_append_chain(pref, leaf_id, tail_entries, + tail_len, &node_slots[2], nr_node_slots - 2, + &child_slots[1], nr_child_slots - 1, + scratch, nr_scratch, &new_head, &new_tail, + &chain_used); + if (ret) + return ret; + } + split_array = child_slots[0].array; + split_array_size = child_slots[0].size; + ret = __stack_depot_trie_split_child_array_init(split_array, split_array_size, + old_tail, new_head); + if (ret) + return ret; + + old_tail->children = child->children; + pref->children = child_slots[0].array; + trie_reparent_children(old_tail); + *prefix = pref; + *tail = has_new_tail ? new_tail : pref; + *nr_used = 2 + chain_used; + return 0; +} + static int stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *array, unsigned long frame, unsigned int *pos, bool *found) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 040b603630475..55443df59224d 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -130,6 +130,15 @@ int __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, void *new_storage, size_t new_storage_size); +int __stack_depot_trie_split_subtree(const void *child, unsigned int matched, + u32 leaf_id, const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, const void **prefix, + const void **tail, unsigned int *nr_used); const void *__stack_depot_trie_child_array_find(const void *storage, unsigned long frame); int __stack_depot_trie_child_array_insert(const void *old_storage, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 177fc31fe4018..aa52ef47be127 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -117,6 +117,20 @@ static int insert_append(struct stack_depot_trie_root *root, void *parent, storage_size, tail, nr_used); } +static int split_subtree(const void *child, unsigned int matched, u32 leaf_id, + const unsigned long *entries, unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, const void **prefix, + const void **tail, unsigned int *nr_used) +{ + return __stack_depot_trie_split_subtree(child, matched, leaf_id, entries, + nr_entries, node_slots, nr_node_slots, child_slots, + nr_child_slots, scratch, nr_scratch, prefix, tail, nr_used); +} + static void trie_node_slot_alloc(struct kunit *test, struct stack_depot_trie_node_slot *slot, @@ -2833,6 +2847,51 @@ static void stackdepot_trie_split_precheck_rejects_aliases(struct kunit *test) KUNIT_EXPECT_EQ(test, ret, -EINVAL); } +static void stackdepot_trie_split_subtree_divergent_tail(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; + unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; + struct stack_depot_trie_child_array_slot child_slot; + struct stack_depot_trie_node_slot node_slots[3]; + struct stack_depot_trie_lookup lookup; + unsigned long scratch[ARRAY_SIZE(old_entries)]; + unsigned long out[ARRAY_SIZE(old_entries)] = {}; + const void *new_tail = NULL; + const void *old_tail; + const void *prefix = NULL; + unsigned int fetched; + unsigned int used = 99; + void *child; + int ret; + + trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, + &child); + trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); + trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); + trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); + child_slot.size = __stack_depot_trie_child_array_size(2); + child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_slot.array); + + ret = split_subtree(child, 1, 2, new_entries, ARRAY_SIZE(new_entries), + node_slots, ARRAY_SIZE(node_slots), &child_slot, 1, + NULL, 0, &prefix, &new_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, used, 3U); + ret = lookup_step(NULL, prefix, &old_entries[1], 1, &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + old_tail = lookup.node; + KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[2].node); + fetched = tfetch(old_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 2U); + KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); + memset(out, 0, sizeof(out)); + fetched = tfetch(new_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 2U); + KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); +} + static void stackdepot_trie_child_array_insert_empty(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -2941,6 +3000,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_split_tail_plan_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_split_precheck), KUNIT_CASE(stackdepot_trie_split_precheck_rejects_aliases), + KUNIT_CASE(stackdepot_trie_split_subtree_divergent_tail), KUNIT_CASE(stackdepot_trie_child_array_insert_empty), {} }; From 677886ebaeb8b946705b4581ec34ad5ee4b74d5f Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 3 Jun 2026 16:00:28 +0100 Subject: [PATCH 034/129] KRN-1117: Harden stackdepot trie split precheck A future split publication will reuse caller-provided child-array storage for the replacement published array. Reject replacement storage that is too small for the currently published array so the eventual COW replacement cannot write past the staging buffer. Add KUnit coverage for short replacement arrays, prefix-leaf splits, and preserving children when splitting internal nodes. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 2 +- lib/tests/stackdepot_kunit.c | 155 +++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 1 deletion(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 8fc928d8265e9..8081f0e98bd83 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2573,7 +2573,7 @@ int __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, if (!children) return -EINVAL; size = __stack_depot_trie_child_array_size(children->nr_children); - if (!size) + if (!size || new_storage_size < size) return -EINVAL; if (stack_depot_ranges_overlap(children, size, new_storage, new_storage_size)) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index aa52ef47be127..ef6a680e2be63 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -2847,6 +2847,47 @@ static void stackdepot_trie_split_precheck_rejects_aliases(struct kunit *test) KUNIT_EXPECT_EQ(test, ret, -EINVAL); } +static void stackdepot_trie_split_precheck_rejects_short_array(struct kunit *test) +{ + unsigned long first_entries[] = { 0x1000UL }; + unsigned long second_entries[] = { 0x2000UL }; + unsigned long new_entries[] = { 0x3000UL }; + struct stack_depot_trie_child_array_slot child_slot; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *children[2]; + void *first_child; + void *second_child; + int ret; + + trie_node_alloc(test, first_entries, ARRAY_SIZE(first_entries), NULL, 1, + &first_child); + trie_node_alloc(test, second_entries, ARRAY_SIZE(second_entries), NULL, 2, + &second_child); + trie_node_slot_alloc(test, &node_slot, new_entries, ARRAY_SIZE(new_entries)); + children[0] = first_child; + children[1] = second_child; + old_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = child_array_init(old_array.array, old_array.size, children, + ARRAY_SIZE(children)); + KUNIT_ASSERT_EQ(test, ret, 0); + root.children = old_array.array; + child_slot.size = __stack_depot_trie_child_array_size(1); + child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_slot.array); + new_array.size = __stack_depot_trie_child_array_size(1); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + + ret = split_precheck(&root, NULL, &node_slot, 1, &child_slot, 1, + new_array.array, new_array.size); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + static void stackdepot_trie_split_subtree_divergent_tail(struct kunit *test) { unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; @@ -2892,6 +2933,117 @@ static void stackdepot_trie_split_subtree_divergent_tail(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); } +static void stackdepot_trie_split_subtree_prefix_leaf(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; + unsigned long new_entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_slot; + struct stack_depot_trie_node_slot node_slots[2]; + struct stack_depot_trie_lookup lookup; + unsigned long scratch[ARRAY_SIZE(old_entries)]; + unsigned long out[ARRAY_SIZE(old_entries)] = {}; + const void *new_tail = NULL; + const void *old_tail; + const void *prefix = NULL; + unsigned int fetched; + unsigned int used = 99; + void *child; + int ret; + + trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, + &child); + trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); + trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); + child_slot.size = __stack_depot_trie_child_array_size(1); + child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_slot.array); + + ret = split_subtree(child, 1, 2, new_entries, ARRAY_SIZE(new_entries), + node_slots, ARRAY_SIZE(node_slots), &child_slot, 1, + NULL, 0, &prefix, &new_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, used, 2U); + KUNIT_EXPECT_PTR_EQ(test, new_tail, prefix); + + fetched = tfetch(prefix, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 1U); + KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); + + ret = lookup_step(NULL, prefix, &old_entries[1], 1, &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + old_tail = lookup.node; + memset(out, 0, sizeof(out)); + fetched = tfetch(old_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 2U); + KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); +} + +static void stackdepot_trie_split_subtree_preserves_children(struct kunit *test) +{ + unsigned long child_entries[] = { 0x1000UL, 0x2000UL }; + unsigned long desc_entries[] = { 0x4000UL }; + unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; + unsigned long old_tail_lookup[] = { 0x2000UL, 0x4000UL }; + unsigned long expected[] = { 0x1000UL, 0x2000UL, 0x4000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_child_array_slot split_array; + struct stack_depot_trie_node_slot desc_slot; + struct stack_depot_trie_node_slot node_slots[3]; + struct stack_depot_trie_lookup lookup; + unsigned long scratch[ARRAY_SIZE(expected)]; + unsigned long out[ARRAY_SIZE(expected)] = {}; + const void *desc_head = NULL; + const void *desc_tail = NULL; + const void *new_tail = NULL; + const void *old_tail; + const void *prefix = NULL; + unsigned int fetched; + unsigned int used = 99; + void *child; + int ret; + + trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), NULL, 0, + &child); + trie_node_slot_alloc(test, &desc_slot, desc_entries, ARRAY_SIZE(desc_entries)); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = append_chain(child, 3, desc_entries, ARRAY_SIZE(desc_entries), + &desc_slot, 1, NULL, 0, NULL, 0, &desc_head, + &desc_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(NULL, child, desc_head, child_array.array, + child_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + trie_node_slot_alloc(test, &node_slots[0], child_entries, 1); + trie_node_slot_alloc(test, &node_slots[1], &child_entries[1], 1); + trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); + split_array.size = __stack_depot_trie_child_array_size(2); + split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, split_array.array); + + ret = split_subtree(child, 1, 4, new_entries, ARRAY_SIZE(new_entries), + node_slots, ARRAY_SIZE(node_slots), &split_array, 1, + NULL, 0, &prefix, &new_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, used, 3U); + ret = lookup_step(NULL, prefix, old_tail_lookup, + ARRAY_SIZE(old_tail_lookup), &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_DESCEND); + old_tail = lookup.node; + ret = lookup_step(NULL, old_tail, desc_entries, ARRAY_SIZE(desc_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, desc_tail); + fetched = tfetch(desc_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); + KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); +} + static void stackdepot_trie_child_array_insert_empty(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -3000,7 +3152,10 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_split_tail_plan_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_split_precheck), KUNIT_CASE(stackdepot_trie_split_precheck_rejects_aliases), + KUNIT_CASE(stackdepot_trie_split_precheck_rejects_short_array), KUNIT_CASE(stackdepot_trie_split_subtree_divergent_tail), + KUNIT_CASE(stackdepot_trie_split_subtree_prefix_leaf), + KUNIT_CASE(stackdepot_trie_split_subtree_preserves_children), KUNIT_CASE(stackdepot_trie_child_array_insert_empty), {} }; From b9469d6bf51cd4c316d928509213154b07c7a000 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 3 Jun 2026 16:41:11 +0100 Subject: [PATCH 035/129] KRN-1117: Add stackdepot trie split insertion Wire split lookups into the private trie insertion helper so a partial match is handled by building a copy-on-write split subtree and publishing the replacement child array last. Keep the helper fail-closed by bounding ancestor walks, and add KUnit coverage for divergent splits, prefix-leaf splits, and parent-chain cycles. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 83 +++++++++++++++- lib/tests/stackdepot_kunit.c | 188 +++++++++++++++++++++++++++++++++++ 2 files changed, 270 insertions(+), 1 deletion(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 8081f0e98bd83..e74713a0c1dd0 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1407,10 +1407,14 @@ unsigned int __stack_depot_trie_node_match(const void *node_ptr, static bool trie_ancestor_overlaps(const struct stack_depot_trie_node *node, const void *ptr, size_t size) { - for (; node; node = node->parent) { + unsigned int depth = 0; + + for (; node; node = node->parent, depth++) { size_t child_size; size_t node_size; + if (depth >= CONFIG_STACKDEPOT_MAX_FRAMES) + return true; if (stack_depot_frame_run_validate(&node->run)) return true; @@ -2219,6 +2223,20 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, return 0; } +static int trie_split_child(struct stack_depot_trie_root *root, + struct stack_depot_trie_node *parent, + const struct stack_depot_trie_node *child, + unsigned int matched, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, void *new_storage, + size_t new_storage_size, const void **tail, + unsigned int *nr_used); + int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, void *parent_ptr, u32 leaf_id, const unsigned long *entries, @@ -2262,6 +2280,12 @@ int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, if (!entries || !nr_entries) return -EINVAL; + if (lookup.status == STACK_DEPOT_TRIE_LOOKUP_SPLIT) + return trie_split_child(root, parent, lookup.node, lookup.matched, + leaf_id, entries, nr_entries, node_slots, + nr_node_slots, child_slots, nr_child_slots, + scratch, nr_scratch, new_storage, + new_storage_size, tail, nr_used); if (lookup.status == STACK_DEPOT_TRIE_LOOKUP_PROMOTE) { if (!node_slots || !nr_node_slots) return -EINVAL; @@ -2786,6 +2810,63 @@ int __stack_depot_trie_split_subtree(const void *child_ptr, unsigned int matched return 0; } +static int trie_split_child(struct stack_depot_trie_root *root, + struct stack_depot_trie_node *parent, + const struct stack_depot_trie_node *child, + unsigned int matched, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, void *new_storage, + size_t new_storage_size, const void **tail, + unsigned int *nr_used) +{ + const struct stack_depot_trie_child_array **publish_slot; + const struct stack_depot_trie_child_array *old_array; + const void *prefix; + unsigned int pos; + unsigned int used; + int ret; + + if (!child || !tail || !nr_used) + return -EINVAL; + if (child->parent != parent) + return -EINVAL; + + ret = __stack_depot_trie_split_precheck(root, parent, node_slots, + nr_node_slots, child_slots, + nr_child_slots, new_storage, + new_storage_size); + if (ret) + return ret; + publish_slot = trie_publish_slot(root, parent); + if (!publish_slot) + return -EINVAL; + + /* Pairs with append, promote, and split publication. */ + old_array = smp_load_acquire(publish_slot); + ret = trie_child_array_replace_precheck(old_array, child, new_storage, + new_storage_size, &pos); + if (ret) + return ret; + + ret = __stack_depot_trie_split_subtree(child, matched, leaf_id, entries, + nr_entries, node_slots, nr_node_slots, + child_slots, nr_child_slots, scratch, + nr_scratch, &prefix, tail, &used); + if (ret) + return ret; + + trie_child_array_replace_at(old_array, prefix, new_storage, pos); + /* Publish the fully initialized replacement array last. */ + smp_store_release(publish_slot, new_storage); + *nr_used = used; + return 0; +} + static int stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *array, unsigned long frame, unsigned int *pos, bool *found) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index ef6a680e2be63..1c2fef9dfb45c 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1914,6 +1914,159 @@ static void stackdepot_trie_insert_append_splits_frame_runs(struct kunit *test) } #endif +static void stackdepot_trie_insert_append_splits_child(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; + unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot split_array; + struct stack_depot_trie_child_array_slot replace_array; + struct stack_depot_trie_node_slot old_slot; + struct stack_depot_trie_node_slot node_slots[3]; + struct stack_depot_trie_lookup lookup; + struct stack_depot_trie_root root = {}; + unsigned long scratch[ARRAY_SIZE(old_entries)]; + unsigned long out[ARRAY_SIZE(old_entries)] = {}; + const void *old_head = NULL; + const void *old_tail; + const void *new_tail = NULL; + const void *prefix; + unsigned int fetched; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); + old_array.size = __stack_depot_trie_child_array_size(1); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = append_chain(NULL, 1, old_entries, ARRAY_SIZE(old_entries), + &old_slot, 1, NULL, 0, NULL, 0, &old_head, + &old_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, old_head, old_array.array, + old_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); + trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); + trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); + split_array.size = __stack_depot_trie_child_array_size(2); + split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, split_array.array); + replace_array.size = __stack_depot_trie_child_array_size(1); + replace_array.array = kunit_kzalloc(test, replace_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, replace_array.array); + used = 99; + + ret = insert_append(&root, NULL, 2, new_entries, ARRAY_SIZE(new_entries), + node_slots, ARRAY_SIZE(node_slots), &split_array, 1, + NULL, 0, replace_array.array, replace_array.size, + &new_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, used, 3U); + KUNIT_EXPECT_PTR_EQ(test, root.children, replace_array.array); + KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[2].node); + + ret = lookup_step(&root, NULL, old_entries, ARRAY_SIZE(old_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_DESCEND); + prefix = lookup.node; + ret = lookup_step(NULL, prefix, &old_entries[1], 1, &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + old_tail = lookup.node; + fetched = tfetch(old_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 2U); + KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); + + memset(out, 0, sizeof(out)); + ret = lookup_step(&root, NULL, new_entries, ARRAY_SIZE(new_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_DESCEND); + ret = lookup_step(NULL, lookup.node, &new_entries[1], 1, &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, new_tail); + fetched = tfetch(new_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 2U); + KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); +} + +static void stackdepot_trie_insert_append_splits_prefix_leaf(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; + unsigned long new_entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot split_array; + struct stack_depot_trie_child_array_slot replace_array; + struct stack_depot_trie_node_slot old_slot; + struct stack_depot_trie_node_slot node_slots[2]; + struct stack_depot_trie_lookup lookup; + struct stack_depot_trie_root root = {}; + unsigned long scratch[ARRAY_SIZE(old_entries)]; + unsigned long out[ARRAY_SIZE(old_entries)] = {}; + const void *old_head = NULL; + const void *old_tail; + const void *new_tail = NULL; + unsigned int fetched; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); + old_array.size = __stack_depot_trie_child_array_size(1); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = append_chain(NULL, 1, old_entries, ARRAY_SIZE(old_entries), + &old_slot, 1, NULL, 0, NULL, 0, &old_head, + &old_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, old_head, old_array.array, + old_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); + trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); + split_array.size = __stack_depot_trie_child_array_size(1); + split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, split_array.array); + replace_array.size = __stack_depot_trie_child_array_size(1); + replace_array.array = kunit_kzalloc(test, replace_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, replace_array.array); + used = 99; + + ret = insert_append(&root, NULL, 2, new_entries, ARRAY_SIZE(new_entries), + node_slots, ARRAY_SIZE(node_slots), &split_array, 1, + NULL, 0, replace_array.array, replace_array.size, + &new_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, used, 2U); + KUNIT_EXPECT_PTR_EQ(test, root.children, replace_array.array); + KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[0].node); + ret = lookup_step(&root, NULL, new_entries, ARRAY_SIZE(new_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + KUNIT_EXPECT_PTR_EQ(test, lookup.node, new_tail); + fetched = tfetch(new_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 1U); + KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); + + ret = lookup_step(&root, NULL, old_entries, ARRAY_SIZE(old_entries), + &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_DESCEND); + ret = lookup_step(NULL, lookup.node, &old_entries[1], 1, &lookup); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); + memset(out, 0, sizeof(out)); + fetched = tfetch(lookup.node, out, ARRAY_SIZE(out), scratch, + ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 2U); + KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); +} + static void stackdepot_trie_insert_append_rejects_existing_child(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -2105,6 +2258,38 @@ static void stackdepot_trie_insert_append_rejects_parent_overlap(struct kunit *t KUNIT_EXPECT_EQ(test, used, 99U); } +static void stackdepot_trie_insert_append_rejects_parent_cycle(struct kunit *test) +{ + unsigned long parent_entries[] = { 0x1000UL }; + unsigned long entries[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot parent_slot; + struct stack_depot_trie_node_slot node_slot; + const void *tail = (const void *)1; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &parent_slot, parent_entries, + ARRAY_SIZE(parent_entries)); + ret = tnode_init(parent_slot.node, parent_slot.size, NULL, 7, + parent_entries, ARRAY_SIZE(parent_entries), NULL, 0); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = tnode_init(parent_slot.node, parent_slot.size, parent_slot.node, 7, + parent_entries, ARRAY_SIZE(parent_entries), NULL, 0); + KUNIT_ASSERT_EQ(test, ret, 0); + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append(NULL, parent_slot.node, 42, entries, ARRAY_SIZE(entries), + &node_slot, 1, NULL, 0, NULL, 0, child_array.array, + child_array.size, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + static void stackdepot_trie_insert_append_rejects_publish_overlap(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -3116,12 +3301,15 @@ static struct kunit_case stackdepot_test_cases[] = { #if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) KUNIT_CASE(stackdepot_trie_insert_append_splits_frame_runs), #endif + KUNIT_CASE(stackdepot_trie_insert_append_splits_child), + KUNIT_CASE(stackdepot_trie_insert_append_splits_prefix_leaf), KUNIT_CASE(stackdepot_trie_insert_append_rejects_existing_child), KUNIT_CASE(stackdepot_trie_insert_append_rejects_short_array), KUNIT_CASE(stackdepot_trie_insert_append_rejects_zero_frame), KUNIT_CASE(stackdepot_trie_insert_append_rejects_root_with_parent), KUNIT_CASE(stackdepot_trie_insert_append_rejects_root_slot_alias), KUNIT_CASE(stackdepot_trie_insert_append_rejects_parent_overlap), + KUNIT_CASE(stackdepot_trie_insert_append_rejects_parent_cycle), KUNIT_CASE(stackdepot_trie_insert_append_rejects_publish_overlap), KUNIT_CASE(stackdepot_trie_insert_append_rejects_child_node_overlap), KUNIT_CASE(stackdepot_trie_insert_append_rejects_child_array_overlap), From 2658c72af4d454ea222b6d4193fa1205259e985a Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 4 Jun 2026 09:18:58 +0100 Subject: [PATCH 036/129] KRN-1117: Add stackdepot trie leaf lookup Add a private trie leaf lookup helper that walks lookup-step results and returns only exact terminal stack matches. This gives the future trie-backed save path a fail-closed exists primitive before public stackdepot routing. Cover root hits, descended hits, prefix leaves with children, miss states, and parent-chain mismatches in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 46 +++++++++++ lib/stackdepot_internal.h | 3 + lib/tests/stackdepot_kunit.c | 156 +++++++++++++++++++++++++++++++++++ 3 files changed, 205 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index e74713a0c1dd0..9e85b1e980102 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2223,6 +2223,52 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, return 0; } +const void * +__stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries) +{ + const struct stack_depot_trie_root *lookup_root = root; + const struct stack_depot_trie_node *parent = NULL; + unsigned int pos = 0; + + if (!root || !entries || !nr_entries) + return NULL; + + while (pos < nr_entries) { + const struct stack_depot_trie_node *node; + struct stack_depot_trie_lookup lookup; + + if (__stack_depot_trie_lookup_step(lookup_root, parent, + &entries[pos], nr_entries - pos, + &lookup)) + return NULL; + node = lookup.node; + if (node && (node->parent != parent || + node->stack_len != pos + lookup.matched)) + return NULL; + + switch (lookup.status) { + case STACK_DEPOT_TRIE_LOOKUP_FOUND: + if (pos + lookup.matched == nr_entries) + return node; + return NULL; + case STACK_DEPOT_TRIE_LOOKUP_DESCEND: + if (!node || !lookup.matched) + return NULL; + pos += lookup.matched; + parent = node; + lookup_root = NULL; + break; + case STACK_DEPOT_TRIE_LOOKUP_APPEND: + case STACK_DEPOT_TRIE_LOOKUP_PROMOTE: + case STACK_DEPOT_TRIE_LOOKUP_SPLIT: + return NULL; + } + } + + return NULL; +} + static int trie_split_child(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, const struct stack_depot_trie_node *child, diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 55443df59224d..7dd8f685c2f63 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -93,6 +93,9 @@ int __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, const void *parent, const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_lookup *lookup); +const void * +__stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries); int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, void *parent, u32 leaf_id, const unsigned long *entries, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 1c2fef9dfb45c..5461db7f12823 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -101,6 +101,12 @@ static int lookup_step(const struct stack_depot_trie_root *root, lookup); } +static const void *find_leaf(const struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries) +{ + return __stack_depot_trie_find_leaf(root, entries, nr_entries); +} + static int insert_append(struct stack_depot_trie_root *root, void *parent, u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, @@ -1434,6 +1440,152 @@ static void stackdepot_trie_lookup_step_accepts_reparented_child(struct kunit *t (unsigned int)ARRAY_SIZE(child_entries)); } +static void stackdepot_trie_find_leaf_root(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *head = NULL; + const void *tail = NULL; + unsigned int used = 0; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = append_chain(NULL, 61, entries, ARRAY_SIZE(entries), &node_slot, 1, + NULL, 0, NULL, 0, &head, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, head, child_array.array, + child_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), + head); + KUNIT_EXPECT_NULL(test, find_leaf(NULL, entries, ARRAY_SIZE(entries))); + KUNIT_EXPECT_NULL(test, find_leaf(&root, NULL, ARRAY_SIZE(entries))); + KUNIT_EXPECT_NULL(test, find_leaf(&root, entries, 0)); +} + +static void stackdepot_trie_find_leaf_descends(struct kunit *test) +{ + unsigned long prefix_entries[] = { 0x1000UL }; + unsigned long full_entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_child_array_slot root_array; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot prefix_slot; + struct stack_depot_trie_node_slot child_slot; + struct stack_depot_trie_root root = {}; + const void *prefix = NULL; + const void *child = NULL; + unsigned int used = 0; + int ret; + + trie_node_slot_alloc(test, &prefix_slot, prefix_entries, + ARRAY_SIZE(prefix_entries)); + trie_node_slot_alloc(test, &child_slot, &full_entries[1], 1); + root_array.size = __stack_depot_trie_child_array_size(1); + root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, root_array.array); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append(&root, NULL, 62, prefix_entries, + ARRAY_SIZE(prefix_entries), &prefix_slot, 1, NULL, 0, + NULL, 0, root_array.array, root_array.size, &prefix, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = insert_append(&root, NULL, 63, full_entries, ARRAY_SIZE(full_entries), + &child_slot, 1, NULL, 0, NULL, 0, child_array.array, + child_array.size, &child, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, prefix_entries, + ARRAY_SIZE(prefix_entries)), prefix); + KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, full_entries, + ARRAY_SIZE(full_entries)), child); +} + +static void stackdepot_trie_find_leaf_misses(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + unsigned long split_miss[] = { 0x1000UL, 0x2222UL }; + unsigned long append_miss[] = { 0x3000UL }; + unsigned long prefix_miss[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *head = NULL; + const void *tail = NULL; + unsigned int used = 0; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = append_chain(NULL, 64, entries, ARRAY_SIZE(entries), &node_slot, 1, + NULL, 0, NULL, 0, &head, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, head, child_array.array, + child_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_EXPECT_NULL(test, find_leaf(&root, split_miss, + ARRAY_SIZE(split_miss))); + KUNIT_EXPECT_NULL(test, find_leaf(&root, append_miss, + ARRAY_SIZE(append_miss))); + KUNIT_EXPECT_NULL(test, find_leaf(&root, prefix_miss, + ARRAY_SIZE(prefix_miss))); +} + +static void stackdepot_trie_find_leaf_rejects_bad_parent(struct kunit *test) +{ + unsigned long parent_entries[] = { 0x1000UL }; + unsigned long full_entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_child_array_slot root_array; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot parent_slot; + struct stack_depot_trie_node_slot child_slot; + struct stack_depot_trie_root root = {}; + void *wrong_parent; + const void *parent = NULL; + const void *child = NULL; + unsigned int used = 0; + int ret; + + trie_node_slot_alloc(test, &parent_slot, parent_entries, + ARRAY_SIZE(parent_entries)); + trie_node_slot_alloc(test, &child_slot, &full_entries[1], 1); + trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 66, + &wrong_parent); + root_array.size = __stack_depot_trie_child_array_size(1); + root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, root_array.array); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append(&root, NULL, 67, parent_entries, + ARRAY_SIZE(parent_entries), &parent_slot, 1, NULL, 0, + NULL, 0, root_array.array, root_array.size, &parent, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = insert_append(&root, NULL, 68, full_entries, ARRAY_SIZE(full_entries), + &child_slot, 1, NULL, 0, NULL, 0, child_array.array, + child_array.size, &child, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = tnode_init(child_slot.node, child_slot.size, wrong_parent, 68, + &full_entries[1], 1, NULL, 0); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_EXPECT_NULL(test, find_leaf(&root, full_entries, + ARRAY_SIZE(full_entries))); +} + static void stackdepot_trie_insert_append_root(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -3290,6 +3442,10 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_lookup_step_root), KUNIT_CASE(stackdepot_trie_lookup_step_parent_promote), KUNIT_CASE(stackdepot_trie_lookup_step_accepts_reparented_child), + KUNIT_CASE(stackdepot_trie_find_leaf_root), + KUNIT_CASE(stackdepot_trie_find_leaf_descends), + KUNIT_CASE(stackdepot_trie_find_leaf_misses), + KUNIT_CASE(stackdepot_trie_find_leaf_rejects_bad_parent), KUNIT_CASE(stackdepot_trie_insert_append_root), KUNIT_CASE(stackdepot_trie_insert_append_parent), KUNIT_CASE(stackdepot_trie_insert_append_descends_one_level), From 84fe19468e53a656e6f1ff0cc7839bf353d5ae1d Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 4 Jun 2026 10:16:24 +0100 Subject: [PATCH 037/129] KRN-1117: Add stackdepot trie pre-publish hook Add a private prepare hook for trie insertion so future side-table updates can be completed immediately before structural publication. Thread the hook through append, promote, and split insertion while keeping the existing no-hook insertion helper as a wrapper. Cover prepare-before-publish ordering and callback failure paths in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 175 +++++++++++++++++++----- lib/stackdepot_internal.h | 25 ++++ lib/tests/stackdepot_kunit.c | 254 +++++++++++++++++++++++++++++++++++ 3 files changed, 419 insertions(+), 35 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 9e85b1e980102..f8386b818db9f 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1927,10 +1927,12 @@ trie_promote_child(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, const struct stack_depot_trie_node *child, u32 leaf_id, const struct stack_depot_trie_node_slot *slot, - void *new_storage, size_t new_storage_size) + void *new_storage, size_t new_storage_size, + const struct stack_depot_trie_publish_prepare *prepare) { const struct stack_depot_trie_child_array **publish_slot; const struct stack_depot_trie_child_array *old_array; + struct stack_depot_trie_leaf_update update; unsigned int pos; int ret; @@ -1944,6 +1946,15 @@ trie_promote_child(struct stack_depot_trie_root *root, if (ret) return ret; trie_child_array_replace_at(old_array, slot->node, new_storage, pos); + if (prepare) { + if (!prepare->fn) + return -EINVAL; + update.leaf_id = leaf_id; + update.leaf = slot->node; + ret = prepare->fn(&update, 1, prepare->ctx); + if (ret) + return ret; + } trie_reparent_children(slot->node); publish_slot = trie_publish_slot(root, parent); @@ -2099,10 +2110,12 @@ __stack_depot_trie_append_chain(const void *parent_ptr, u32 leaf_id, return 0; } -int -__stack_depot_trie_publish_append(struct stack_depot_trie_root *root, - void *parent_ptr, const void *head_ptr, - void *new_storage, size_t new_storage_size) +static int trie_publish_append_prepare(struct stack_depot_trie_root *root, + void *parent_ptr, const void *head_ptr, + void *new_storage, size_t new_storage_size, + const struct stack_depot_trie_publish_prepare *prepare, + u32 leaf_id, + const void *leaf) { const struct stack_depot_trie_child_array *old_array; const struct stack_depot_trie_node *head = head_ptr; @@ -2144,12 +2157,36 @@ __stack_depot_trie_publish_append(struct stack_depot_trie_root *root, return -EINVAL; if (__stack_depot_trie_child_array_insert(old_array, head, new_array, storage_size)) return -EINVAL; + if (prepare) { + struct stack_depot_trie_leaf_update update = { + .leaf_id = leaf_id, + .leaf = leaf, + }; + int ret; + + if (!prepare->fn) + return -EINVAL; + if (!leaf_id || !leaf) + return -EINVAL; + ret = prepare->fn(&update, 1, prepare->ctx); + if (ret) + return ret; + } /* Publish the fully initialized replacement array last. */ smp_store_release(slot, new_array); return 0; } +int +__stack_depot_trie_publish_append(struct stack_depot_trie_root *root, + void *parent_ptr, const void *head_ptr, + void *new_storage, size_t new_storage_size) +{ + return trie_publish_append_prepare(root, parent_ptr, head_ptr, new_storage, + new_storage_size, NULL, 0, NULL); +} + int __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, const void *parent_ptr, const unsigned long *entries, @@ -2280,20 +2317,26 @@ static int trie_split_child(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, - size_t new_storage_size, const void **tail, + size_t new_storage_size, + const struct stack_depot_trie_publish_prepare *prepare, + const void **tail, unsigned int *nr_used); -int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, - void *parent_ptr, u32 leaf_id, - const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, void *new_storage, - size_t new_storage_size, const void **tail, - unsigned int *nr_used) +int +__stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, + void *parent_ptr, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot + *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, void *new_storage, + size_t new_storage_size, + const struct stack_depot_trie_publish_prepare *prepare, + const void **tail, + unsigned int *nr_used) { const void *head; const void *last; @@ -2331,12 +2374,14 @@ int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, leaf_id, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, scratch, nr_scratch, new_storage, - new_storage_size, tail, nr_used); + new_storage_size, prepare, tail, + nr_used); if (lookup.status == STACK_DEPOT_TRIE_LOOKUP_PROMOTE) { if (!node_slots || !nr_node_slots) return -EINVAL; ret = trie_promote_child(root, parent, lookup.node, leaf_id, - &node_slots[0], new_storage, new_storage_size); + &node_slots[0], new_storage, new_storage_size, + prepare); if (ret) return ret; *tail = node_slots[0].node; @@ -2358,8 +2403,8 @@ int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, &head, &last, &used); if (ret) return ret; - ret = __stack_depot_trie_publish_append(root, parent, head, new_storage, - new_storage_size); + ret = trie_publish_append_prepare(root, parent, head, new_storage, + new_storage_size, prepare, leaf_id, last); if (ret) return ret; @@ -2368,6 +2413,27 @@ int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, return 0; } +int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, + void *parent, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, void *new_storage, + size_t new_storage_size, const void **tail, + unsigned int *nr_used) +{ + return __stack_depot_trie_insert_append_prepare(root, parent, leaf_id, + entries, nr_entries, node_slots, + nr_node_slots, child_slots, + nr_child_slots, scratch, + nr_scratch, new_storage, + new_storage_size, NULL, tail, + nr_used); +} + unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries, unsigned long *scratch, @@ -2779,23 +2845,27 @@ trie_split_subtree_precheck(const struct stack_depot_trie_node *child, return 0; } -int __stack_depot_trie_split_subtree(const void *child_ptr, unsigned int matched, - u32 leaf_id, const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, const void **prefix, - const void **tail, unsigned int *nr_used) +static int trie_split_subtree_prepare(const void *child_ptr, unsigned int matched, + u32 leaf_id, const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, + const struct stack_depot_trie_publish_prepare *prepare, + const void **prefix, + const void **tail, unsigned int *nr_used) { const struct stack_depot_trie_node *child = child_ptr; const unsigned long *tail_entries; const void *new_head = NULL; const void *new_tail = NULL; + struct stack_depot_trie_leaf_update updates[2]; struct stack_depot_trie_node *old_tail; struct stack_depot_trie_node *pref; unsigned int chain_used = 0; + unsigned int nr_updates = 0; u32 prefix_leaf_id; void *split_array; size_t split_array_size; @@ -2846,6 +2916,21 @@ int __stack_depot_trie_split_subtree(const void *child_ptr, unsigned int matched old_tail, new_head); if (ret) return ret; + if (child->leaf_id) { + updates[nr_updates].leaf_id = child->leaf_id; + updates[nr_updates].leaf = old_tail; + nr_updates++; + } + updates[nr_updates].leaf_id = leaf_id; + updates[nr_updates].leaf = has_new_tail ? new_tail : pref; + nr_updates++; + if (prepare) { + if (!prepare->fn) + return -EINVAL; + ret = prepare->fn(updates, nr_updates, prepare->ctx); + if (ret) + return ret; + } old_tail->children = child->children; pref->children = child_slots[0].array; @@ -2856,6 +2941,23 @@ int __stack_depot_trie_split_subtree(const void *child_ptr, unsigned int matched return 0; } +int __stack_depot_trie_split_subtree(const void *child, unsigned int matched, + u32 leaf_id, const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, const void **prefix, + const void **tail, unsigned int *nr_used) +{ + return trie_split_subtree_prepare(child, matched, leaf_id, entries, + nr_entries, node_slots, nr_node_slots, + child_slots, nr_child_slots, scratch, + nr_scratch, NULL, prefix, tail, + nr_used); +} + static int trie_split_child(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, const struct stack_depot_trie_node *child, @@ -2867,7 +2969,9 @@ static int trie_split_child(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, - size_t new_storage_size, const void **tail, + size_t new_storage_size, + const struct stack_depot_trie_publish_prepare *prepare, + const void **tail, unsigned int *nr_used) { const struct stack_depot_trie_child_array **publish_slot; @@ -2899,10 +3003,11 @@ static int trie_split_child(struct stack_depot_trie_root *root, if (ret) return ret; - ret = __stack_depot_trie_split_subtree(child, matched, leaf_id, entries, - nr_entries, node_slots, nr_node_slots, - child_slots, nr_child_slots, scratch, - nr_scratch, &prefix, tail, &used); + ret = trie_split_subtree_prepare(child, matched, leaf_id, entries, + nr_entries, node_slots, nr_node_slots, + child_slots, nr_child_slots, scratch, + nr_scratch, prepare, &prefix, + tail, &used); if (ret) return ret; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 7dd8f685c2f63..ebd47a645a36a 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -48,6 +48,17 @@ struct stack_depot_trie_lookup { unsigned int matched; }; +struct stack_depot_trie_leaf_update { + u32 leaf_id; + const void *leaf; +}; + +struct stack_depot_trie_publish_prepare { + int (*fn)(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, void *ctx); + void *ctx; +}; + bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, u32 *low); bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, @@ -107,6 +118,20 @@ int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, const void **tail, unsigned int *nr_used); +int +__stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, + void *parent, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot + *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, void *new_storage, + size_t new_storage_size, + const struct stack_depot_trie_publish_prepare *prepare, + const void **tail, unsigned int *nr_used); unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 5461db7f12823..7a44b70843520 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -123,6 +123,65 @@ static int insert_append(struct stack_depot_trie_root *root, void *parent, storage_size, tail, nr_used); } +static int +insert_append_prepare(struct stack_depot_trie_root *root, void *parent, + u32 leaf_id, const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, void *storage, size_t storage_size, + const struct stack_depot_trie_publish_prepare *prepare, + const void **tail, unsigned int *nr_used) +{ + return __stack_depot_trie_insert_append_prepare(root, parent, leaf_id, + entries, nr_entries, node_slots, + nr_node_slots, child_slots, + nr_child_slots, scratch, + nr_scratch, storage, + storage_size, prepare, tail, + nr_used); +} + +#define STACKDEPOT_TRIE_PREPARE_MAX_UPDATES 2 + +struct stackdepot_trie_prepare_ctx { + struct kunit *test; + const struct stack_depot_trie_child_array **visible; + const struct stack_depot_trie_child_array *expected_visible; + const void *expected_leaf[STACKDEPOT_TRIE_PREPARE_MAX_UPDATES]; + u32 expected_leaf_id[STACKDEPOT_TRIE_PREPARE_MAX_UPDATES]; + unsigned int nr_expected; + unsigned int calls; + int ret; +}; + +static int +stackdepot_trie_prepare(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, void *data) +{ + struct stackdepot_trie_prepare_ctx *ctx = data; + unsigned int i; + + ctx->calls++; + KUNIT_EXPECT_NOT_NULL(ctx->test, updates); + if (!updates) + return -EINVAL; + KUNIT_EXPECT_EQ(ctx->test, nr_updates, ctx->nr_expected); + for (i = 0; i < nr_updates && i < ctx->nr_expected; i++) { + KUNIT_EXPECT_EQ(ctx->test, updates[i].leaf_id, + ctx->expected_leaf_id[i]); + KUNIT_EXPECT_PTR_EQ(ctx->test, updates[i].leaf, + ctx->expected_leaf[i]); + } + if (ctx->visible) + KUNIT_EXPECT_PTR_EQ(ctx->test, *ctx->visible, + ctx->expected_visible); + + return ctx->ret; +} + static int split_subtree(const void *child, unsigned int matched, u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, const struct stack_depot_trie_node_slot *node_slots, @@ -1612,6 +1671,197 @@ static void stackdepot_trie_insert_append_root(struct kunit *test) node_slot.node); } +static void stackdepot_trie_insert_append_prepare_root(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + struct stackdepot_trie_prepare_ctx ctx = { + .test = test, + .visible = &root.children, + .expected_visible = NULL, + .expected_leaf_id = { 69 }, + .nr_expected = 1, + }; + struct stack_depot_trie_publish_prepare prepare = { + .fn = stackdepot_trie_prepare, + .ctx = &ctx, + }; + const void *tail = NULL; + unsigned int used = 0; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + ctx.expected_leaf[0] = node_slot.node; + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append_prepare(&root, NULL, 69, entries, ARRAY_SIZE(entries), + &node_slot, 1, NULL, 0, NULL, 0, + child_array.array, child_array.size, + &prepare, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, ctx.calls, 1U); + KUNIT_EXPECT_PTR_EQ(test, root.children, child_array.array); + KUNIT_EXPECT_PTR_EQ(test, tail, node_slot.node); + KUNIT_EXPECT_EQ(test, used, 1U); +} + +static void stackdepot_trie_insert_append_prepare_failure(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + struct stackdepot_trie_prepare_ctx ctx = { + .test = test, + .visible = &root.children, + .expected_visible = NULL, + .expected_leaf_id = { 70 }, + .nr_expected = 1, + .ret = -EAGAIN, + }; + struct stack_depot_trie_publish_prepare prepare = { + .fn = stackdepot_trie_prepare, + .ctx = &ctx, + }; + const void *tail = (const void *)1; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + ctx.expected_leaf[0] = node_slot.node; + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + + ret = insert_append_prepare(&root, NULL, 70, entries, ARRAY_SIZE(entries), + &node_slot, 1, NULL, 0, NULL, 0, + child_array.array, child_array.size, + &prepare, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EAGAIN); + KUNIT_EXPECT_EQ(test, ctx.calls, 1U); + KUNIT_EXPECT_NULL(test, root.children); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + +static void stackdepot_trie_insert_append_prepare_promote_failure(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot new_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + struct stackdepot_trie_prepare_ctx ctx = { + .test = test, + .visible = &root.children, + .expected_leaf_id = { 72 }, + .nr_expected = 1, + .ret = -EAGAIN, + }; + struct stack_depot_trie_publish_prepare prepare = { + .fn = stackdepot_trie_prepare, + .ctx = &ctx, + }; + const void *children[1]; + const void *tail = (const void *)1; + unsigned int used = 99; + void *child; + int ret; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &child); + children[0] = child; + old_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = child_array_init(old_array.array, old_array.size, children, + ARRAY_SIZE(children)); + KUNIT_ASSERT_EQ(test, ret, 0); + root.children = old_array.array; + ctx.expected_visible = old_array.array; + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + ctx.expected_leaf[0] = node_slot.node; + new_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); + new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, new_array.array); + + ret = insert_append_prepare(&root, NULL, 72, entries, ARRAY_SIZE(entries), + &node_slot, 1, NULL, 0, NULL, 0, + new_array.array, new_array.size, + &prepare, &tail, &used); + KUNIT_EXPECT_EQ(test, ret, -EAGAIN); + KUNIT_EXPECT_EQ(test, ctx.calls, 1U); + KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); + KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + +static void stackdepot_trie_insert_append_prepare_split(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; + unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot split_array; + struct stack_depot_trie_child_array_slot replace_array; + struct stack_depot_trie_node_slot old_slot; + struct stack_depot_trie_node_slot node_slots[3]; + struct stack_depot_trie_root root = {}; + struct stackdepot_trie_prepare_ctx ctx = { + .test = test, + .visible = &root.children, + .expected_leaf_id = { 73, 74 }, + .nr_expected = 2, + }; + struct stack_depot_trie_publish_prepare prepare = { + .fn = stackdepot_trie_prepare, + .ctx = &ctx, + }; + const void *old_head = NULL; + const void *old_tail = NULL; + const void *new_tail = NULL; + unsigned int used = 0; + int ret; + + trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); + old_array.size = __stack_depot_trie_child_array_size(1); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = append_chain(NULL, 73, old_entries, ARRAY_SIZE(old_entries), + &old_slot, 1, NULL, 0, NULL, 0, &old_head, + &old_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, old_head, old_array.array, + old_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + ctx.expected_visible = old_array.array; + + trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); + trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); + trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); + ctx.expected_leaf[0] = node_slots[1].node; + ctx.expected_leaf[1] = node_slots[2].node; + split_array.size = __stack_depot_trie_child_array_size(2); + split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, split_array.array); + replace_array.size = __stack_depot_trie_child_array_size(1); + replace_array.array = kunit_kzalloc(test, replace_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, replace_array.array); + + ret = insert_append_prepare(&root, NULL, 74, new_entries, + ARRAY_SIZE(new_entries), node_slots, + ARRAY_SIZE(node_slots), &split_array, 1, + NULL, 0, replace_array.array, + replace_array.size, &prepare, &new_tail, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, ctx.calls, 1U); + KUNIT_EXPECT_PTR_EQ(test, root.children, replace_array.array); + KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[2].node); +} + static void stackdepot_trie_insert_append_parent(struct kunit *test) { unsigned long parent_entries[] = { 0x1000UL }; @@ -3447,6 +3697,10 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_find_leaf_misses), KUNIT_CASE(stackdepot_trie_find_leaf_rejects_bad_parent), KUNIT_CASE(stackdepot_trie_insert_append_root), + KUNIT_CASE(stackdepot_trie_insert_append_prepare_root), + KUNIT_CASE(stackdepot_trie_insert_append_prepare_failure), + KUNIT_CASE(stackdepot_trie_insert_append_prepare_promote_failure), + KUNIT_CASE(stackdepot_trie_insert_append_prepare_split), KUNIT_CASE(stackdepot_trie_insert_append_parent), KUNIT_CASE(stackdepot_trie_insert_append_descends_one_level), KUNIT_CASE(stackdepot_trie_insert_append_descends_multiple_levels), From feb01a3e9d92e7f12bd6990819c0bc4d9ad93066 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 4 Jun 2026 11:10:07 +0100 Subject: [PATCH 038/129] KRN-1117: Add stackdepot trie insertion planning Add a private trie insertion planner that sizes caller-provided node, child-array, and replacement-array storage for append, promote, split, and descend insertion paths without allocating or publishing anything. Cover append, promote, split, descend, duplicate-leaf, malformed-child, and zero-slot promote planning cases in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 248 ++++++++++++++++++++++++++++ lib/stackdepot_internal.h | 9 ++ lib/tests/stackdepot_kunit.c | 306 +++++++++++++++++++++++++++++++++++ 3 files changed, 563 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index f8386b818db9f..874b93f6c7411 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2434,6 +2434,254 @@ int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, nr_used); } +static int trie_plan_append_chain(unsigned int base_stack_len, + const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + unsigned int *nr_used, + unsigned int *nr_child_used) +{ + unsigned int pos = 0; + unsigned int stack_len = base_stack_len; + unsigned int used = 0; + + if (!entries || !nr_entries || !node_slots || !nr_used || !nr_child_used) + return -EINVAL; + + while (pos < nr_entries) { + struct stack_depot_frame_run run; + + if (used >= nr_node_slots) + return -EINVAL; + if (__stack_depot_frame_run_init(&entries[pos], nr_entries - pos, + &run)) + return -EINVAL; + if (stack_len > U32_MAX - run.nr_entries || + stack_len > CONFIG_STACKDEPOT_MAX_FRAMES - run.nr_entries) + return -EINVAL; + + node_slots[used].node = NULL; + node_slots[used].size = __stack_depot_trie_node_size(&run); + if (!node_slots[used].size) + return -EINVAL; + stack_len += run.nr_entries; + pos += run.nr_entries; + used++; + } + + if (used > 1) { + unsigned int i; + + if (!child_slots || nr_child_slots < used - 1) + return -EINVAL; + for (i = 0; i < used - 1; i++) { + child_slots[i].array = NULL; + child_slots[i].size = __stack_depot_trie_child_array_size(1); + if (!child_slots[i].size) + return -EINVAL; + } + } + + *nr_used = used; + *nr_child_used = used > 1 ? used - 1 : 0; + return 0; +} + +static bool trie_node_depth_invalid(const struct stack_depot_trie_node *parent, + const struct stack_depot_trie_node *node) +{ + u32 base = parent ? parent->stack_len : 0; + + if (!node || node->parent != parent || !node->stack_len) + return true; + if (parent && !parent->stack_len) + return true; + if (stack_depot_frame_run_validate(&node->run)) + return true; + if (node->run.nr_entries > U32_MAX - base || + base > CONFIG_STACKDEPOT_MAX_FRAMES - node->run.nr_entries) + return true; + + return node->stack_len != base + node->run.nr_entries; +} + +static bool trie_node_chain_depth_invalid(const struct stack_depot_trie_node *node) +{ + unsigned int depth = 0; + + for (; node; node = node->parent, depth++) { + if (depth >= CONFIG_STACKDEPOT_MAX_FRAMES) + return true; + if (trie_node_depth_invalid(node->parent, node)) + return true; + } + + return false; +} + +static int trie_plan_split(const struct stack_depot_trie_child_array *children, + const struct stack_depot_trie_node *child, + unsigned int matched, const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, size_t *new_storage_size, + unsigned int *nr_used, unsigned int *nr_child_used) +{ + struct stack_depot_frame_run old_tail_run; + struct stack_depot_frame_run prefix_run; + unsigned int new_child_used = 0; + unsigned int new_used = 0; + unsigned int prefix_stack_len; + bool has_new_tail; + + if (!children || !child || !entries || !node_slots || !child_slots || + !new_storage_size || !nr_used || !nr_child_used) + return -EINVAL; + if (!matched || matched >= child->run.nr_entries || matched > nr_entries) + return -EINVAL; + if (trie_node_depth_invalid(child->parent, child)) + return -EINVAL; + if (!child->leaf_id && !child->children) + return -EINVAL; + if (nr_node_slots < 2 || nr_child_slots < 1) + return -EINVAL; + if (stack_depot_frame_run_slice(&child->run, 0, matched, &prefix_run) || + stack_depot_frame_run_slice(&child->run, matched, + child->run.nr_entries - matched, + &old_tail_run)) + return -EINVAL; + + has_new_tail = matched < nr_entries; + prefix_stack_len = child->parent ? child->parent->stack_len : 0; + if (prefix_stack_len > CONFIG_STACKDEPOT_MAX_FRAMES - matched) + return -EINVAL; + prefix_stack_len += matched; + + node_slots[0].node = NULL; + node_slots[0].size = __stack_depot_trie_node_size(&prefix_run); + node_slots[1].node = NULL; + node_slots[1].size = __stack_depot_trie_node_size(&old_tail_run); + if (!node_slots[0].size || !node_slots[1].size) + return -EINVAL; + + if (has_new_tail && + trie_plan_append_chain(prefix_stack_len, &entries[matched], + nr_entries - matched, &node_slots[2], + nr_node_slots - 2, &child_slots[1], + nr_child_slots - 1, &new_used, + &new_child_used)) + return -EINVAL; + + child_slots[0].array = NULL; + child_slots[0].size = + __stack_depot_trie_child_array_size(has_new_tail ? 2 : 1); + *new_storage_size = + __stack_depot_trie_child_array_size(children->nr_children); + if (!child_slots[0].size || !*new_storage_size) + return -EINVAL; + *nr_used = 2 + new_used; + *nr_child_used = 1 + new_child_used; + return 0; +} + +int +__stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, + const void *parent_ptr, const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, size_t *new_storage_size, + unsigned int *nr_used, unsigned int *nr_child_used) +{ + const struct stack_depot_trie_node *parent = parent_ptr; + const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_node *child; + unsigned int matched; + unsigned int pos; + bool found; + + if (!entries || !nr_entries || !node_slots || !new_storage_size || + !nr_used || !nr_child_used) + return -EINVAL; + if ((root && parent) || (!root && !parent)) + return -EINVAL; + + for (;;) { + if (parent && trie_node_chain_depth_invalid(parent)) + return -EINVAL; + if (root) { + /* Pairs with append, promote, and split publication. */ + children = smp_load_acquire(&root->children); + } else { + /* Pairs with append, promote, and split publication. */ + children = smp_load_acquire(&parent->children); + } + if (!children) { + if (trie_plan_append_chain(parent ? parent->stack_len : 0, + entries, nr_entries, node_slots, + nr_node_slots, child_slots, + nr_child_slots, nr_used, + nr_child_used)) + return -EINVAL; + *new_storage_size = __stack_depot_trie_child_array_size(1); + return *new_storage_size ? 0 : -EINVAL; + } + if (stack_depot_trie_child_lower_bound(children, entries[0], &pos, + &found)) + return -EINVAL; + if (!found) { + if (trie_plan_append_chain(parent ? parent->stack_len : 0, + entries, nr_entries, node_slots, + nr_node_slots, child_slots, + nr_child_slots, nr_used, + nr_child_used)) + return -EINVAL; + *new_storage_size = + __stack_depot_trie_child_array_size(children->nr_children + 1); + return *new_storage_size ? 0 : -EINVAL; + } + + child = children->children[pos]; + if (trie_node_depth_invalid(parent, child)) + return -EINVAL; + matched = __stack_depot_trie_node_match(child, entries, nr_entries); + if (!matched || (matched == nr_entries && + child->run.nr_entries == nr_entries && + child->leaf_id)) + return -EINVAL; + if (matched < child->run.nr_entries) + return trie_plan_split(children, child, matched, entries, + nr_entries, node_slots, nr_node_slots, + child_slots, nr_child_slots, + new_storage_size, nr_used, + nr_child_used); + if (matched == nr_entries) { + if (child->leaf_id || !nr_node_slots) + return -EINVAL; + node_slots[0].node = NULL; + node_slots[0].size = __stack_depot_trie_node_size(&child->run); + *new_storage_size = + __stack_depot_trie_child_array_size(children->nr_children); + if (!node_slots[0].size || !*new_storage_size) + return -EINVAL; + *nr_used = 1; + *nr_child_used = 0; + return 0; + } + + root = NULL; + parent = child; + entries += matched; + nr_entries -= matched; + } +} + unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries, unsigned long *scratch, diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index ebd47a645a36a..358247515a3d3 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -132,6 +132,15 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, size_t new_storage_size, const struct stack_depot_trie_publish_prepare *prepare, const void **tail, unsigned int *nr_used); +int +__stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, + const void *parent, const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, size_t *new_storage_size, + unsigned int *nr_used, unsigned int *nr_child_used); unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 7a44b70843520..a44d5f4def380 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -144,6 +144,22 @@ insert_append_prepare(struct stack_depot_trie_root *root, void *parent, nr_used); } +static int insert_plan(const struct stack_depot_trie_root *root, + const void *parent, const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, size_t *new_storage_size, + unsigned int *nr_used, unsigned int *nr_child_used) +{ + return __stack_depot_trie_insert_plan(root, parent, entries, nr_entries, + node_slots, nr_node_slots, + child_slots, nr_child_slots, + new_storage_size, nr_used, + nr_child_used); +} + #define STACKDEPOT_TRIE_PREPARE_MAX_UPDATES 2 struct stackdepot_trie_prepare_ctx { @@ -2469,6 +2485,286 @@ static void stackdepot_trie_insert_append_splits_prefix_leaf(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); } +static void stackdepot_trie_insert_plan_append(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_slot; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_frame_run run; + struct stack_depot_trie_root root = {}; + unsigned int child_used = 99; + unsigned int used = 99; + size_t publish_size = 0; + int ret; + + ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), &node_slot, + 1, &child_slot, 1, &publish_size, &used, + &child_used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_NULL(test, node_slot.node); + KUNIT_EXPECT_EQ(test, node_slot.size, __stack_depot_trie_node_size(&run)); + KUNIT_EXPECT_EQ(test, used, 1U); + KUNIT_EXPECT_EQ(test, child_used, 0U); + KUNIT_EXPECT_EQ(test, publish_size, __stack_depot_trie_child_array_size(1)); +} + +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) +static void stackdepot_trie_insert_plan_mixed_append(struct kunit *test) +{ + unsigned long entries[] = { +#ifdef CONFIG_ARM64 + arch_stack_depot_frame_text_prefix() | 0x1000UL, + 0x1000UL, +#else + 0xffffffff81001000UL, + 0xffff888000001000UL, +#endif + }; + struct stack_depot_trie_child_array_slot child_slot; + struct stack_depot_trie_node_slot node_slots[2]; + struct stack_depot_frame_run first_run; + struct stack_depot_frame_run second_run; + struct stack_depot_trie_root root = {}; + unsigned int child_used = 99; + unsigned int used = 99; + size_t publish_size = 0; + int ret; + + ret = frame_run_init(entries, ARRAY_SIZE(entries), &first_run); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = frame_run_init(&entries[first_run.nr_entries], + ARRAY_SIZE(entries) - first_run.nr_entries, + &second_run); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), node_slots, + ARRAY_SIZE(node_slots), &child_slot, 1, &publish_size, + &used, &child_used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, used, 2U); + KUNIT_EXPECT_EQ(test, child_used, 1U); + KUNIT_EXPECT_EQ(test, node_slots[0].size, + __stack_depot_trie_node_size(&first_run)); + KUNIT_EXPECT_EQ(test, node_slots[1].size, + __stack_depot_trie_node_size(&second_run)); + KUNIT_EXPECT_EQ(test, child_slot.size, + __stack_depot_trie_child_array_size(1)); + KUNIT_EXPECT_EQ(test, publish_size, __stack_depot_trie_child_array_size(1)); +} +#endif + +static void stackdepot_trie_insert_plan_promote(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_frame_run run; + struct stack_depot_trie_root root = {}; + const void *children[1]; + unsigned int child_used = 99; + unsigned int used = 99; + size_t publish_size = 0; + void *child; + int ret; + + ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); + KUNIT_ASSERT_EQ(test, ret, 0); + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &child); + children[0] = child; + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = child_array_init(child_array.array, child_array.size, children, + ARRAY_SIZE(children)); + KUNIT_ASSERT_EQ(test, ret, 0); + root.children = child_array.array; + + ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), &node_slot, + 1, NULL, 0, &publish_size, &used, &child_used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_NULL(test, node_slot.node); + KUNIT_EXPECT_EQ(test, node_slot.size, __stack_depot_trie_node_size(&run)); + KUNIT_EXPECT_EQ(test, used, 1U); + KUNIT_EXPECT_EQ(test, child_used, 0U); + KUNIT_EXPECT_EQ(test, publish_size, __stack_depot_trie_child_array_size(1)); +} + +static void stackdepot_trie_insert_plan_promote_rejects_empty_slots(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot = { + .node = (void *)1, + .size = 99, + }; + struct stack_depot_trie_root root = {}; + const void *children[1]; + unsigned int child_used = 99; + unsigned int used = 99; + size_t publish_size = 0; + void *child; + int ret; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &child); + children[0] = child; + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = child_array_init(child_array.array, child_array.size, children, + ARRAY_SIZE(children)); + KUNIT_ASSERT_EQ(test, ret, 0); + root.children = child_array.array; + + ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), &node_slot, + 0, NULL, 0, &publish_size, &used, &child_used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, node_slot.node, (void *)1); + KUNIT_EXPECT_EQ(test, node_slot.size, (size_t)99); +} + +static void stackdepot_trie_insert_plan_split(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; + unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; + struct stack_depot_trie_child_array_slot child_slot; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_node_slot node_slots[3]; + struct stack_depot_trie_node_slot old_slot; + struct stack_depot_frame_run run; + struct stack_depot_trie_root root = {}; + const void *old_head = NULL; + const void *old_tail = NULL; + unsigned int child_used = 99; + unsigned int used = 99; + size_t publish_size = 0; + int ret; + + trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); + old_array.size = __stack_depot_trie_child_array_size(1); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = append_chain(NULL, 1, old_entries, ARRAY_SIZE(old_entries), + &old_slot, 1, NULL, 0, NULL, 0, &old_head, + &old_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, old_head, old_array.array, + old_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = frame_run_init(new_entries, 1, &run); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = insert_plan(&root, NULL, new_entries, ARRAY_SIZE(new_entries), + node_slots, ARRAY_SIZE(node_slots), &child_slot, 1, + &publish_size, &used, &child_used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, used, 3U); + KUNIT_EXPECT_EQ(test, child_used, 1U); + KUNIT_EXPECT_EQ(test, node_slots[0].size, + __stack_depot_trie_node_size(&run)); + KUNIT_EXPECT_EQ(test, node_slots[1].size, + __stack_depot_trie_node_size(&run)); + KUNIT_EXPECT_EQ(test, node_slots[2].size, + __stack_depot_trie_node_size(&run)); + KUNIT_EXPECT_EQ(test, child_slot.size, + __stack_depot_trie_child_array_size(2)); + KUNIT_EXPECT_EQ(test, publish_size, __stack_depot_trie_child_array_size(1)); +} + +static void stackdepot_trie_insert_plan_descends(struct kunit *test) +{ + unsigned long prefix_entries[] = { 0x1000UL }; + unsigned long stack_entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_child_array_slot root_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_node_slot prefix_slot; + struct stack_depot_frame_run run; + struct stack_depot_trie_root root = {}; + const void *prefix = NULL; + unsigned int child_used = 99; + unsigned int used = 99; + size_t publish_size = 0; + int ret; + + trie_node_slot_alloc(test, &prefix_slot, prefix_entries, + ARRAY_SIZE(prefix_entries)); + root_array.size = __stack_depot_trie_child_array_size(1); + root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, root_array.array); + ret = insert_append(&root, NULL, 75, prefix_entries, + ARRAY_SIZE(prefix_entries), &prefix_slot, 1, NULL, 0, + NULL, 0, root_array.array, root_array.size, &prefix, + &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = frame_run_init(&stack_entries[1], 1, &run); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = insert_plan(&root, NULL, stack_entries, ARRAY_SIZE(stack_entries), + &node_slot, 1, NULL, 0, &publish_size, &used, + &child_used); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, used, 1U); + KUNIT_EXPECT_EQ(test, child_used, 0U); + KUNIT_EXPECT_EQ(test, node_slot.size, __stack_depot_trie_node_size(&run)); + KUNIT_EXPECT_EQ(test, publish_size, __stack_depot_trie_child_array_size(1)); +} + +static void stackdepot_trie_insert_plan_rejects_existing_leaf(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *tail = NULL; + unsigned int child_used = 99; + unsigned int used = 99; + size_t publish_size = 0; + int ret; + + trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = insert_append(&root, NULL, 76, entries, ARRAY_SIZE(entries), + &node_slot, 1, NULL, 0, NULL, 0, child_array.array, + child_array.size, &tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), &node_slot, + 1, NULL, 0, &publish_size, &used, &child_used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void stackdepot_trie_insert_plan_rejects_bad_child(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_root root = {}; + const void *children[1]; + unsigned int child_used = 99; + unsigned int used = 99; + size_t publish_size = 0; + void *bad_parent; + void *child; + int ret; + + trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 77, &bad_parent); + trie_node_alloc(test, entries, ARRAY_SIZE(entries), bad_parent, 78, &child); + children[0] = child; + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = child_array_init(child_array.array, child_array.size, children, + ARRAY_SIZE(children)); + KUNIT_ASSERT_EQ(test, ret, 0); + root.children = child_array.array; + + ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), &node_slot, + 1, NULL, 0, &publish_size, &used, &child_used); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + static void stackdepot_trie_insert_append_rejects_existing_child(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -3713,6 +4009,16 @@ static struct kunit_case stackdepot_test_cases[] = { #endif KUNIT_CASE(stackdepot_trie_insert_append_splits_child), KUNIT_CASE(stackdepot_trie_insert_append_splits_prefix_leaf), + KUNIT_CASE(stackdepot_trie_insert_plan_append), +#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) + KUNIT_CASE(stackdepot_trie_insert_plan_mixed_append), +#endif + KUNIT_CASE(stackdepot_trie_insert_plan_promote), + KUNIT_CASE(stackdepot_trie_insert_plan_promote_rejects_empty_slots), + KUNIT_CASE(stackdepot_trie_insert_plan_split), + KUNIT_CASE(stackdepot_trie_insert_plan_descends), + KUNIT_CASE(stackdepot_trie_insert_plan_rejects_existing_leaf), + KUNIT_CASE(stackdepot_trie_insert_plan_rejects_bad_child), KUNIT_CASE(stackdepot_trie_insert_append_rejects_existing_child), KUNIT_CASE(stackdepot_trie_insert_append_rejects_short_array), KUNIT_CASE(stackdepot_trie_insert_append_rejects_zero_frame), From 8e6de92d1ab9624e5a89d9730326924b6680e85b Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 4 Jun 2026 12:06:08 +0100 Subject: [PATCH 039/129] KRN-1117: Add stackdepot trie handle namespace Add private helpers to encode and decode trie leaf IDs in the stackdepot handle space above the configured hash-pool namespace. Keep the all-ones pool-index value reserved as an invalid sentinel and derive the maximum leaf ID from the current stack_max_pools value. Cover hash-handle rejection, extra-bit masking, offset-boundary crossing, exact namespace overflow, and namespace-unavailable configurations in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 69 ++++++++++++++++++++++++++++++++++++ lib/stackdepot_internal.h | 3 ++ lib/tests/stackdepot_kunit.c | 47 ++++++++++++++++++++++++ 3 files changed, 119 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 874b93f6c7411..a436983d47641 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -153,6 +153,75 @@ static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); /* Count helpers rely on saturated refcounts looking negative. */ static_assert(REFCOUNT_SATURATED < 0); +static u32 stack_depot_pool_index_mask(void) +{ + return (1U << DEPOT_POOL_INDEX_BITS) - 1; +} + +static u32 stack_depot_offset_mask(void) +{ + return (1U << DEPOT_OFFSET_BITS) - 1; +} + +static bool stack_depot_trie_namespace_available(void) +{ + /* Reserve the all-ones pool index as an invalid trie namespace sentinel. */ + return stack_max_pools < stack_depot_pool_index_mask() - 1; +} + +u32 __stack_depot_trie_max_leaf_id(void) +{ + if (!stack_depot_trie_namespace_available()) + return 0; + + return (stack_depot_pool_index_mask() - stack_max_pools - 1) << + DEPOT_OFFSET_BITS; +} + +depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) +{ + union handle_parts parts = {}; + u64 pool_index_plus_1; + u32 pool_delta; + u32 index; + + if (!leaf_id || !stack_depot_trie_namespace_available()) + return 0; + if (leaf_id > __stack_depot_trie_max_leaf_id()) + return 0; + + index = leaf_id - 1; + pool_delta = index >> DEPOT_OFFSET_BITS; + pool_index_plus_1 = (u64)stack_max_pools + 1 + pool_delta; + if (pool_index_plus_1 >= stack_depot_pool_index_mask()) + return 0; + + parts.pool_index_plus_1 = pool_index_plus_1; + parts.offset = index & stack_depot_offset_mask(); + return parts.handle; +} + +u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) +{ + union handle_parts parts = { .handle = handle }; + u64 leaf_id; + u32 pool_delta; + + if (!stack_depot_trie_namespace_available()) + return 0; + + parts.extra = 0; + if (parts.pool_index_plus_1 <= stack_max_pools) + return 0; + + pool_delta = parts.pool_index_plus_1 - stack_max_pools - 1; + if ((u64)pool_delta + stack_max_pools + 1 >= stack_depot_pool_index_mask()) + return 0; + + leaf_id = ((u64)pool_delta << DEPOT_OFFSET_BITS) + parts.offset + 1; + return leaf_id > U32_MAX ? 0 : leaf_id; +} + static int __init disable_stack_depot(char *str) { return kstrtobool(str, &stack_depot_disabled); diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 358247515a3d3..25c8200d02bb9 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -59,6 +59,9 @@ struct stack_depot_trie_publish_prepare { void *ctx; }; +depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id); +u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle); +u32 __stack_depot_trie_max_leaf_id(void); bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, u32 *low); bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index a44d5f4def380..1e82d4dd8794d 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -528,6 +528,52 @@ static void stackdepot_count_helpers(struct kunit *test) KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(zeroed_handle, 1)); } +static void stackdepot_trie_handle_namespace(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800710000UL, + 0x1234567800720000UL, + 0x1234567800730000UL, + }; + depot_stack_handle_t boundary_handle; + depot_stack_handle_t hash_handle; + depot_stack_handle_t tagged; + depot_stack_handle_t trie; + u32 boundary_id; + u32 max_id; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + trie = __stack_depot_trie_handle(1); + hash_handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_EXPECT_NE(test, hash_handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(hash_handle), 0U); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_handle(0), (depot_stack_handle_t)0); + + if (!trie) { + KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(0), 0U); + return; + } + + boundary_id = (1U << DEPOT_OFFSET_BITS) + 2; + max_id = __stack_depot_trie_max_leaf_id(); + boundary_handle = __stack_depot_trie_handle(boundary_id); + tagged = stack_depot_set_extra_bits(trie, 7); + + KUNIT_EXPECT_NE(test, boundary_handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(trie), 1U); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(tagged), 1U); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(boundary_handle), + boundary_id); + KUNIT_EXPECT_NE(test, max_id, 0U); + KUNIT_EXPECT_NE(test, __stack_depot_trie_handle(max_id), + (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_handle(max_id + 1), + (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_handle(U32_MAX), + (depot_stack_handle_t)0); +} + static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0xffff888000001000UL; @@ -3948,6 +3994,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), KUNIT_CASE(stackdepot_count_helpers), + KUNIT_CASE(stackdepot_trie_handle_namespace), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), From b3e2e2a7ca5548f476fc5195581be75105e2c709 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 4 Jun 2026 14:28:34 +0100 Subject: [PATCH 040/129] KRN-1117: Fix stackdepot trie COW leaf lookup COW splits and promotions can reparent descendants to an equivalent replacement prefix before the parent child-array swap becomes visible. __stack_depot_trie_lookup_step() already tolerates that state, but __stack_depot_trie_find_leaf() still rejected any exact parent pointer mismatch. A reader walking the old child array could therefore miss an existing leaf during the pre-publish window. Validate the relocated parent chain against the stack prefix instead of requiring pointer identity. Add KUnit coverage for old-root lookup after split reparenting and for split prepare failures leaving the old publication visible. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 69 +++++++++++++++++- lib/tests/stackdepot_kunit.c | 138 ++++++++++++++++++++++++++++++++++- 2 files changed, 202 insertions(+), 5 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index a436983d47641..1fea3707f0f96 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1032,6 +1032,10 @@ static int stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *array, unsigned long frame, unsigned int *pos, bool *found); +static bool +trie_parent_chain_matches_prefix(const struct stack_depot_trie_node *node, + const unsigned long *entries, + unsigned int nr_entries); static size_t stack_depot_frame_run_entry_bytes(enum stack_depot_frame_mode mode) { @@ -2349,9 +2353,17 @@ __stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, &lookup)) return NULL; node = lookup.node; - if (node && (node->parent != parent || - node->stack_len != pos + lookup.matched)) - return NULL; + if (node) { + const struct stack_depot_trie_node *node_parent; + + node_parent = READ_ONCE(node->parent); + if (node->stack_len != pos + lookup.matched) + return NULL; + if (node_parent != parent && + !trie_parent_chain_matches_prefix(node_parent, entries, + pos)) + return NULL; + } switch (lookup.status) { case STACK_DEPOT_TRIE_LOOKUP_FOUND: @@ -2591,6 +2603,57 @@ static bool trie_node_chain_depth_invalid(const struct stack_depot_trie_node *no return false; } +static bool +trie_parent_chain_matches_prefix(const struct stack_depot_trie_node *node, + const unsigned long *entries, + unsigned int nr_entries) +{ + const struct stack_depot_trie_node *cur; + unsigned int depth = 0; + unsigned int i; + + if (!node) + return nr_entries == 0; + if (!entries || node->stack_len != nr_entries) + return false; + + cur = node; + while (cur) { + const struct stack_depot_trie_node *parent; + unsigned int start; + + if (depth++ >= CONFIG_STACKDEPOT_MAX_FRAMES) + return false; + parent = READ_ONCE(cur->parent); + if (stack_depot_frame_run_validate(&cur->run)) + return false; + if (!cur->stack_len || cur->run.nr_entries > cur->stack_len) + return false; + if (parent) { + if (!parent->stack_len || + parent->stack_len > U32_MAX - cur->run.nr_entries) + return false; + if (cur->stack_len != parent->stack_len + cur->run.nr_entries) + return false; + } else if (cur->stack_len != cur->run.nr_entries) { + return false; + } + + start = cur->stack_len - cur->run.nr_entries; + for (i = 0; i < cur->run.nr_entries; i++) { + unsigned long frame; + + if (stack_depot_trie_node_frame(cur, i, &frame) || + frame != entries[start + i]) + return false; + } + + cur = parent; + } + + return true; +} + static int trie_plan_split(const struct stack_depot_trie_child_array *children, const struct stack_depot_trie_node *child, unsigned int matched, const unsigned long *entries, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 1e82d4dd8794d..9830e8f134d0d 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1630,6 +1630,65 @@ static void stackdepot_trie_find_leaf_descends(struct kunit *test) ARRAY_SIZE(full_entries)), child); } +static void stackdepot_trie_find_leaf_accepts_reparented_child(struct kunit *test) +{ + unsigned long child_entries[] = { 0x1000UL, 0x2000UL }; + unsigned long desc_entries[] = { 0x4000UL }; + unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; + unsigned long old_stack[] = { 0x1000UL, 0x2000UL, 0x4000UL }; + struct stack_depot_trie_child_array_slot root_array; + struct stack_depot_trie_child_array_slot child_array; + struct stack_depot_trie_child_array_slot split_array; + struct stack_depot_trie_node_slot desc_slot; + struct stack_depot_trie_node_slot node_slots[3]; + struct stack_depot_trie_root root = {}; + const void *desc_head = NULL; + const void *desc_tail = NULL; + const void *new_tail = NULL; + const void *prefix = NULL; + unsigned int used = 0; + void *child; + int ret; + + trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), NULL, 0, + &child); + root_array.size = __stack_depot_trie_child_array_size(1); + root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, root_array.array); + ret = publish_append(&root, NULL, child, root_array.array, + root_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + trie_node_slot_alloc(test, &desc_slot, desc_entries, + ARRAY_SIZE(desc_entries)); + child_array.size = __stack_depot_trie_child_array_size(1); + child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, child_array.array); + ret = append_chain(child, 3, desc_entries, ARRAY_SIZE(desc_entries), + &desc_slot, 1, NULL, 0, NULL, 0, &desc_head, + &desc_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(NULL, child, desc_head, child_array.array, + child_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + + trie_node_slot_alloc(test, &node_slots[0], child_entries, 1); + trie_node_slot_alloc(test, &node_slots[1], &child_entries[1], 1); + trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); + split_array.size = __stack_depot_trie_child_array_size(2); + split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, split_array.array); + ret = split_subtree(child, 1, 4, new_entries, ARRAY_SIZE(new_entries), + node_slots, ARRAY_SIZE(node_slots), &split_array, 1, + NULL, 0, &prefix, &new_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, old_stack, ARRAY_SIZE(old_stack)), + desc_tail); + KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[2].node); + KUNIT_EXPECT_PTR_EQ(test, prefix, node_slots[0].node); +} + static void stackdepot_trie_find_leaf_misses(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; @@ -1666,6 +1725,7 @@ static void stackdepot_trie_find_leaf_misses(struct kunit *test) static void stackdepot_trie_find_leaf_rejects_bad_parent(struct kunit *test) { unsigned long parent_entries[] = { 0x1000UL }; + unsigned long wrong_parent_entries[] = { 0x1111UL }; unsigned long full_entries[] = { 0x1000UL, 0x2000UL }; struct stack_depot_trie_child_array_slot root_array; struct stack_depot_trie_child_array_slot child_array; @@ -1681,8 +1741,8 @@ static void stackdepot_trie_find_leaf_rejects_bad_parent(struct kunit *test) trie_node_slot_alloc(test, &parent_slot, parent_entries, ARRAY_SIZE(parent_entries)); trie_node_slot_alloc(test, &child_slot, &full_entries[1], 1); - trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 66, - &wrong_parent); + trie_node_alloc(test, wrong_parent_entries, ARRAY_SIZE(wrong_parent_entries), + NULL, 66, &wrong_parent); root_array.size = __stack_depot_trie_child_array_size(1); root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, root_array.array); @@ -1924,6 +1984,78 @@ static void stackdepot_trie_insert_append_prepare_split(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[2].node); } +static void stackdepot_trie_insert_append_prepare_split_failure(struct kunit *test) +{ + unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; + unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; + struct stack_depot_trie_child_array_slot old_array; + struct stack_depot_trie_child_array_slot split_array; + struct stack_depot_trie_child_array_slot replace_array; + struct stack_depot_trie_node_slot old_slot; + struct stack_depot_trie_node_slot node_slots[3]; + struct stack_depot_trie_root root = {}; + const void *found; + struct stackdepot_trie_prepare_ctx ctx = { + .test = test, + .visible = &root.children, + .expected_leaf_id = { 75, 76 }, + .nr_expected = 2, + .ret = -EAGAIN, + }; + struct stack_depot_trie_publish_prepare prepare = { + .fn = stackdepot_trie_prepare, + .ctx = &ctx, + }; + const void *old_head = NULL; + const void *old_tail = NULL; + const void *new_tail = (const void *)1; + unsigned int used = 99; + int ret; + + trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); + old_array.size = __stack_depot_trie_child_array_size(1); + old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, old_array.array); + ret = append_chain(NULL, 75, old_entries, ARRAY_SIZE(old_entries), + &old_slot, 1, NULL, 0, NULL, 0, &old_head, + &old_tail, &used); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = publish_append(&root, NULL, old_head, old_array.array, + old_array.size); + KUNIT_ASSERT_EQ(test, ret, 0); + ctx.expected_visible = old_array.array; + + trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); + trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); + trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); + ctx.expected_leaf[0] = node_slots[1].node; + ctx.expected_leaf[1] = node_slots[2].node; + split_array.size = __stack_depot_trie_child_array_size(2); + split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, split_array.array); + replace_array.size = __stack_depot_trie_child_array_size(1); + replace_array.array = kunit_kzalloc(test, replace_array.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, replace_array.array); + new_tail = (const void *)1; + used = 99; + + ret = insert_append_prepare(&root, NULL, 76, new_entries, + ARRAY_SIZE(new_entries), node_slots, + ARRAY_SIZE(node_slots), &split_array, 1, + NULL, 0, replace_array.array, + replace_array.size, &prepare, &new_tail, + &used); + KUNIT_EXPECT_EQ(test, ret, -EAGAIN); + KUNIT_EXPECT_EQ(test, ctx.calls, 1U); + KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); + KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, old_entries, + ARRAY_SIZE(old_entries)), old_tail); + found = find_leaf(&root, new_entries, ARRAY_SIZE(new_entries)); + KUNIT_EXPECT_NULL(test, found); + KUNIT_EXPECT_PTR_EQ(test, new_tail, (const void *)1); + KUNIT_EXPECT_EQ(test, used, 99U); +} + static void stackdepot_trie_insert_append_parent(struct kunit *test) { unsigned long parent_entries[] = { 0x1000UL }; @@ -4037,6 +4169,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_lookup_step_accepts_reparented_child), KUNIT_CASE(stackdepot_trie_find_leaf_root), KUNIT_CASE(stackdepot_trie_find_leaf_descends), + KUNIT_CASE(stackdepot_trie_find_leaf_accepts_reparented_child), KUNIT_CASE(stackdepot_trie_find_leaf_misses), KUNIT_CASE(stackdepot_trie_find_leaf_rejects_bad_parent), KUNIT_CASE(stackdepot_trie_insert_append_root), @@ -4044,6 +4177,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_insert_append_prepare_failure), KUNIT_CASE(stackdepot_trie_insert_append_prepare_promote_failure), KUNIT_CASE(stackdepot_trie_insert_append_prepare_split), + KUNIT_CASE(stackdepot_trie_insert_append_prepare_split_failure), KUNIT_CASE(stackdepot_trie_insert_append_parent), KUNIT_CASE(stackdepot_trie_insert_append_descends_one_level), KUNIT_CASE(stackdepot_trie_insert_append_descends_multiple_levels), From b69ba25725dbca1b24b662f31d719e8cc4b18c00 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 4 Jun 2026 16:28:21 +0100 Subject: [PATCH 041/129] KRN-1117: Clarify page_owner marker reporting Clarify that page_owner stack reporting treats a count of one as the stack-list membership marker for best-effort seq_file output. The count can race with page_owner updates, so avoid describing the marker case as an absolute invariant. Signed-off-by: Caleb Kan --- mm/page_owner.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index f6339a4d12577..811dc09edd630 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -948,7 +948,7 @@ static int stack_print(struct seq_file *m, void *v) return 0; /* Counts can race with page_owner updates; seq_file output is best effort. */ - /* A count of 1 is only the stack_list membership marker. */ + /* Treat count 1 as marker-only for best-effort reporting. */ if (!__stack_depot_get_count(handle, &nr_base_pages) || nr_base_pages <= 1) return 0; /* The <= 1 guard above makes removing the list marker safe. */ From fa13111d162b48c940335a1d1066dfd411e8589d Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 4 Jun 2026 16:29:17 +0100 Subject: [PATCH 042/129] KRN-1117: Add stackdepot trie side table Add a private side table for trie leaf IDs so future trie handles can resolve stable leaf pointers without exposing trie internals through the public stackdepot API. Keep the helper inert for now: the existing hash-backed save and fetch paths are unchanged. Use a lazily allocated chunked table with caller-owned preallocation so future save paths can allocate outside the writer critical section. Serialize side-table writers internally and publish chunks and entries with release/acquire ordering for lockless lookup. Retain emptied chunks until teardown so stale lockless lookups cannot race immediate chunk reuse. Cover allocation, store, lookup, revoke, restore, byte accounting, chunk boundaries, and namespace-unavailable configurations in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 301 +++++++++++++++++++++++++++++++++++ lib/stackdepot_internal.h | 22 +++ lib/tests/stackdepot_kunit.c | 187 ++++++++++++++++++++++ 3 files changed, 510 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 1fea3707f0f96..48fa6acfd5cc9 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -222,6 +222,307 @@ u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) return leaf_id > U32_MAX ? 0 : leaf_id; } +static const void ***trie_side_table_chunks; +static DEFINE_RAW_SPINLOCK(trie_side_table_lock); +static unsigned int trie_side_table_high_water; +static unsigned int trie_side_table_nr_chunks; +static unsigned int trie_side_table_top_size; +static u32 trie_side_table_next_id; +static bool trie_side_table_initialized; + +static unsigned int trie_side_table_top_index(u32 id) +{ + return (id - 1) >> STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS; +} + +static unsigned int trie_side_table_slot_index(u32 id) +{ + return (id - 1) & (STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE - 1); +} + +static const void **trie_side_table_load_chunk(unsigned int top) +{ + /* Pairs with trie_side_table_publish_chunk(). */ + return smp_load_acquire(&trie_side_table_chunks[top]); +} + +static void trie_side_table_publish_chunk(unsigned int top, const void **chunk) +{ + /* Pairs with trie_side_table_load_chunk(). */ + smp_store_release(&trie_side_table_chunks[top], chunk); +} + +static const void * +trie_side_table_load_entry(const void **chunk, unsigned int slot) +{ + /* Pairs with trie_side_table_store_entry(). */ + return smp_load_acquire(&chunk[slot]); +} + +static void +trie_side_table_store_entry(const void **chunk, unsigned int slot, const void *entry) +{ + /* Pairs with trie_side_table_load_entry(). */ + smp_store_release(&chunk[slot], entry); +} + +static void trie_side_table_clear_entry(const void **chunk, unsigned int slot) +{ + WRITE_ONCE(chunk[slot], NULL); +} + +int __stack_depot_trie_side_table_init(gfp_t gfp_flags) +{ + u32 max_leaf_id; + + if (trie_side_table_initialized) + return 0; + + max_leaf_id = __stack_depot_trie_max_leaf_id(); + if (!max_leaf_id) + return -EINVAL; + + trie_side_table_top_size = + DIV_ROUND_UP(max_leaf_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); + trie_side_table_chunks = + kvcalloc(trie_side_table_top_size, sizeof(*trie_side_table_chunks), + gfp_flags); + if (!trie_side_table_chunks) + return -ENOMEM; + + trie_side_table_high_water = 0; + trie_side_table_nr_chunks = 0; + trie_side_table_next_id = 0; + trie_side_table_initialized = true; + return 0; +} + +void __stack_depot_trie_side_table_destroy(void) +{ + unsigned int i; + + if (!trie_side_table_initialized) + return; + + for (i = 0; i < trie_side_table_high_water; i++) + kfree(trie_side_table_chunks[i]); + kvfree(trie_side_table_chunks); + trie_side_table_chunks = NULL; + trie_side_table_high_water = 0; + trie_side_table_nr_chunks = 0; + trie_side_table_top_size = 0; + trie_side_table_next_id = 0; + trie_side_table_initialized = false; +} + +bool __stack_depot_trie_side_table_prealloc_needed(void) +{ + unsigned long flags; + bool needed; + u32 id; + unsigned int top; + + if (!trie_side_table_initialized) + return false; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + id = READ_ONCE(trie_side_table_next_id) + 1; + if (!id || id > __stack_depot_trie_max_leaf_id()) { + needed = false; + goto out; + } + + top = trie_side_table_top_index(id); + if (top >= trie_side_table_top_size) { + needed = false; + goto out; + } + + needed = !trie_side_table_load_chunk(top); +out: + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return needed; +} + +void *__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags) +{ + return kcalloc(STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE, + sizeof(*trie_side_table_chunks[0]), gfp_flags); +} + +void __stack_depot_trie_side_table_free_prealloc(void *prealloc) +{ + kfree(prealloc); +} + +u32 __stack_depot_trie_side_table_alloc_id(void **prealloc) +{ + const void **chunk; + unsigned long flags; + u32 id; + unsigned int top; + + if (!trie_side_table_initialized) + return 0; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + id = trie_side_table_next_id + 1; + if (!id || id > __stack_depot_trie_max_leaf_id()) + goto fail; + + top = trie_side_table_top_index(id); + if (top >= trie_side_table_top_size) + goto fail; + + chunk = trie_side_table_load_chunk(top); + if (!chunk) { + if (!prealloc || !*prealloc) + goto fail; + chunk = *prealloc; + *prealloc = NULL; + trie_side_table_publish_chunk(top, chunk); + trie_side_table_nr_chunks++; + if (trie_side_table_high_water < top + 1) + trie_side_table_high_water = top + 1; + } + + WRITE_ONCE(trie_side_table_next_id, id); + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return id; +fail: + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return 0; +} + +void __stack_depot_trie_side_table_revoke_latest(u32 id) +{ + const void **chunk; + unsigned long flags; + unsigned int slot; + unsigned int top; + + if (!trie_side_table_initialized || !id || + id != READ_ONCE(trie_side_table_next_id)) + return; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + if (id != trie_side_table_next_id) + goto out; + top = trie_side_table_top_index(id); + if (top >= trie_side_table_top_size) + goto out; + + chunk = trie_side_table_load_chunk(top); + if (!chunk) + goto out; + + slot = trie_side_table_slot_index(id); + trie_side_table_clear_entry(chunk, slot); + WRITE_ONCE(trie_side_table_next_id, id - 1); +out: + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); +} + +void __stack_depot_trie_side_table_restore(u32 id, const void *entry) +{ + const void **chunk; + unsigned long flags; + unsigned int top; + + if (!trie_side_table_initialized || !id) + return; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + if (id > trie_side_table_next_id) + goto out; + top = trie_side_table_top_index(id); + if (top >= trie_side_table_top_size) + goto out; + + chunk = trie_side_table_load_chunk(top); + if (!chunk) + goto out; + + trie_side_table_store_entry(chunk, trie_side_table_slot_index(id), entry); +out: + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); +} + +int __stack_depot_trie_side_table_store(u32 id, const void *entry) +{ + const void **chunk; + unsigned long flags; + unsigned int top; + int ret = -EINVAL; + + if (!trie_side_table_initialized || !id || !entry) + return -EINVAL; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + if (id > trie_side_table_next_id) + goto out; + top = trie_side_table_top_index(id); + if (top >= trie_side_table_top_size) + goto out; + + chunk = trie_side_table_load_chunk(top); + if (!chunk) + goto out; + + trie_side_table_store_entry(chunk, trie_side_table_slot_index(id), entry); + ret = 0; +out: + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return ret; +} + +const void *__stack_depot_trie_side_table_lookup(u32 id) +{ + const void **chunk; + unsigned int top; + + if (!trie_side_table_initialized || !id) + return NULL; + + top = trie_side_table_top_index(id); + if (top >= trie_side_table_top_size) + return NULL; + + chunk = trie_side_table_load_chunk(top); + if (!chunk) + return NULL; + + return trie_side_table_load_entry(chunk, trie_side_table_slot_index(id)); +} + +size_t __stack_depot_trie_side_table_entries(void) +{ + return trie_side_table_initialized ? READ_ONCE(trie_side_table_next_id) : 0; +} + +size_t __stack_depot_trie_side_table_bytes(void) +{ + unsigned int nr_chunks; + size_t bytes; + size_t top_bytes; + + if (!trie_side_table_initialized) + return 0; + if (check_mul_overflow((size_t)trie_side_table_top_size, + sizeof(*trie_side_table_chunks), &top_bytes)) + return SIZE_MAX; + + nr_chunks = READ_ONCE(trie_side_table_nr_chunks); + if (check_mul_overflow((size_t)nr_chunks, + STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE * + sizeof(*trie_side_table_chunks[0]), &bytes)) + return SIZE_MAX; + if (check_add_overflow(top_bytes, bytes, &bytes)) + return SIZE_MAX; + + return bytes; +} + static int __init disable_stack_depot(char *str) { return kstrtobool(str, &stack_depot_disabled); diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 25c8200d02bb9..1d0ba8317ca0e 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -59,9 +59,31 @@ struct stack_depot_trie_publish_prepare { void *ctx; }; +#define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 +#define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ + (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) + depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id); u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle); u32 __stack_depot_trie_max_leaf_id(void); + +/* + * Private trie leaf side table. Writers serialize internally; lookup is + * lockless. Init and destroy are controlled setup/teardown operations and must + * not race with lookup. + */ +int __stack_depot_trie_side_table_init(gfp_t gfp_flags); +void __stack_depot_trie_side_table_destroy(void); +bool __stack_depot_trie_side_table_prealloc_needed(void); +void *__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags); +void __stack_depot_trie_side_table_free_prealloc(void *prealloc); +u32 __stack_depot_trie_side_table_alloc_id(void **prealloc); +void __stack_depot_trie_side_table_revoke_latest(u32 id); +void __stack_depot_trie_side_table_restore(u32 id, const void *entry); +int __stack_depot_trie_side_table_store(u32 id, const void *entry); +const void *__stack_depot_trie_side_table_lookup(u32 id); +size_t __stack_depot_trie_side_table_entries(void); +size_t __stack_depot_trie_side_table_bytes(void); bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, u32 *low); bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 9830e8f134d0d..24edf6865a142 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -574,6 +574,185 @@ static void stackdepot_trie_handle_namespace(struct kunit *test) (depot_stack_handle_t)0); } +static void stackdepot_trie_side_table_destroy_action(void *data) +{ + __stack_depot_trie_side_table_destroy(); +} + +static void stackdepot_trie_side_table_init_or_skip(struct kunit *test) +{ + int ret; + + ret = __stack_depot_trie_side_table_init(GFP_KERNEL); + if (ret == -EINVAL && !__stack_depot_trie_max_leaf_id()) + kunit_skip(test, "trie handle namespace unavailable"); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, stackdepot_trie_side_table_destroy_action, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); +} + +static u32 stackdepot_trie_side_table_alloc(struct kunit *test) +{ + void *prealloc = NULL; + u32 id; + + if (__stack_depot_trie_side_table_prealloc_needed()) { + prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, prealloc); + } + + id = __stack_depot_trie_side_table_alloc_id(&prealloc); + __stack_depot_trie_side_table_free_prealloc(prealloc); + return id; +} + +static void stackdepot_trie_side_table_destroy_uninit(struct kunit *test) +{ + __stack_depot_trie_side_table_destroy(); + KUNIT_SUCCEED(test); +} + +static void stackdepot_trie_side_table_alloc_store_lookup(struct kunit *test) +{ + const void *entry1 = (const void *)0x1111UL; + const void *entry2 = (const void *)0x2222UL; + u32 id1; + u32 id2; + + stackdepot_trie_side_table_init_or_skip(test); + id1 = stackdepot_trie_side_table_alloc(test); + id2 = stackdepot_trie_side_table_alloc(test); + + KUNIT_ASSERT_EQ(test, id1, 1U); + KUNIT_ASSERT_EQ(test, id2, 2U); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store(id1, entry1), 0); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store(id2, entry2), 0); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), entry1); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id2), entry2); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 2UL); +} + +static void stackdepot_trie_side_table_rejects_invalid_ids(struct kunit *test) +{ + int ret; + u32 id; + + stackdepot_trie_side_table_init_or_skip(test); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(0)); + + id = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, id, 1U); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id + 1)); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store(id, NULL), -EINVAL); + ret = __stack_depot_trie_side_table_store(id + 1, (const void *)0x1UL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void stackdepot_trie_side_table_revoke_latest(struct kunit *test) +{ + const void *entry = (const void *)0xaaaaUL; + size_t bytes; + int ret; + u32 id; + + stackdepot_trie_side_table_init_or_skip(test); + id = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, id, 1U); + ret = __stack_depot_trie_side_table_store(id, entry); + KUNIT_ASSERT_EQ(test, ret, 0); + bytes = __stack_depot_trie_side_table_bytes(); + KUNIT_EXPECT_GT(test, bytes, 0UL); + + __stack_depot_trie_side_table_revoke_latest(id); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_bytes(), bytes); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id)); +} + +static void stackdepot_trie_side_table_revoke_keeps_chunk(struct kunit *test) +{ + const void *entry1 = (const void *)0x1111UL; + const void *entry2 = (const void *)0x2222UL; + size_t bytes; + u32 id1; + u32 id2; + + stackdepot_trie_side_table_init_or_skip(test); + id1 = stackdepot_trie_side_table_alloc(test); + id2 = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, id1, 1U); + KUNIT_ASSERT_EQ(test, id2, 2U); + KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id1, entry1), 0); + KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id2, entry2), 0); + bytes = __stack_depot_trie_side_table_bytes(); + + __stack_depot_trie_side_table_revoke_latest(id2); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_bytes(), bytes); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), entry1); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id2)); +} + +static void stackdepot_trie_side_table_restore(struct kunit *test) +{ + const void *entry1 = (const void *)0xaaaaUL; + const void *entry2 = (const void *)0xbbbbUL; + u32 id; + + stackdepot_trie_side_table_init_or_skip(test); + id = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, id, 1U); + KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, entry1), 0); + KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, entry2), 0); + + __stack_depot_trie_side_table_restore(id, entry1); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id), entry1); + __stack_depot_trie_side_table_restore(id, NULL); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id)); +} + +static void stackdepot_trie_side_table_chunk_boundary(struct kunit *test) +{ + void *prealloc = NULL; + u32 id = 0; + u32 i; + + stackdepot_trie_side_table_init_or_skip(test); + for (i = 0; i < STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE; i++) + id = stackdepot_trie_side_table_alloc(test); + + KUNIT_ASSERT_EQ(test, id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_side_table_prealloc_needed()); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_alloc_id(NULL), 0U); + + prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, prealloc); + id = __stack_depot_trie_side_table_alloc_id(&prealloc); + KUNIT_EXPECT_NULL(test, prealloc); + KUNIT_EXPECT_EQ(test, id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE + 1); +} + +static void stackdepot_trie_side_table_bytes(struct kunit *test) +{ + size_t before; + size_t after; + void *prealloc; + u32 id; + + stackdepot_trie_side_table_init_or_skip(test); + before = __stack_depot_trie_side_table_bytes(); + KUNIT_EXPECT_GT(test, before, 0UL); + prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, prealloc); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_bytes(), before); + + id = __stack_depot_trie_side_table_alloc_id(&prealloc); + KUNIT_EXPECT_NULL(test, prealloc); + KUNIT_ASSERT_EQ(test, id, 1U); + after = __stack_depot_trie_side_table_bytes(); + KUNIT_EXPECT_GT(test, after, before); +} + static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0xffff888000001000UL; @@ -4127,6 +4306,14 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), KUNIT_CASE(stackdepot_count_helpers), KUNIT_CASE(stackdepot_trie_handle_namespace), + KUNIT_CASE(stackdepot_trie_side_table_destroy_uninit), + KUNIT_CASE(stackdepot_trie_side_table_alloc_store_lookup), + KUNIT_CASE(stackdepot_trie_side_table_rejects_invalid_ids), + KUNIT_CASE(stackdepot_trie_side_table_revoke_latest), + KUNIT_CASE(stackdepot_trie_side_table_revoke_keeps_chunk), + KUNIT_CASE(stackdepot_trie_side_table_restore), + KUNIT_CASE(stackdepot_trie_side_table_chunk_boundary), + KUNIT_CASE(stackdepot_trie_side_table_bytes), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), From 946ce65615f588019e3af86fbc9a07cef7a34740 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 5 Jun 2026 09:36:29 +0100 Subject: [PATCH 043/129] KRN-1117: Harden stackdepot trie COW publication Reparenting existing trie descendants points already-reachable nodes at newly initialized COW parents. Publish those parent pointers with release ordering and use acquire loads in lockless parent-chain readers so the new parent contents are visible before they are dereferenced. Also keep prepare-failure staging consistent with the intended all-or-none publication model: build promote replacement arrays only after prepare succeeds, and clear split child-array staging when prepare rejects the update. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 48fa6acfd5cc9..d6d4b05ad0f53 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1945,6 +1945,21 @@ trie_child_array_subtree_overlaps(const struct stack_depot_trie_child_array *arr } } +static const struct stack_depot_trie_node * +trie_load_parent(const struct stack_depot_trie_node *node) +{ + /* Pairs with trie_publish_parent(). */ + return smp_load_acquire(&node->parent); +} + +static void +trie_publish_parent(struct stack_depot_trie_node *child, + const struct stack_depot_trie_node *parent) +{ + /* Pairs with trie_load_parent(). */ + smp_store_release(&child->parent, parent); +} + static bool trie_node_slots_subtree_overlap(const struct stack_depot_trie_child_array *array, const struct stack_depot_trie_node *parent, @@ -2239,7 +2254,7 @@ static void trie_reparent_children(struct stack_depot_trie_node *parent) /* Child arrays are const for readers; writers serialize reparenting. */ child = (struct stack_depot_trie_node *)children->children[i]; - WRITE_ONCE(child->parent, parent); + trie_publish_parent(child, parent); } } @@ -2319,7 +2334,6 @@ trie_promote_child(struct stack_depot_trie_root *root, ret = trie_clone_promoted_node(child, leaf_id, slot); if (ret) return ret; - trie_child_array_replace_at(old_array, slot->node, new_storage, pos); if (prepare) { if (!prepare->fn) return -EINVAL; @@ -2329,6 +2343,7 @@ trie_promote_child(struct stack_depot_trie_root *root, if (ret) return ret; } + trie_child_array_replace_at(old_array, slot->node, new_storage, pos); trie_reparent_children(slot->node); publish_slot = trie_publish_slot(root, parent); @@ -2657,7 +2672,7 @@ __stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, if (node) { const struct stack_depot_trie_node *node_parent; - node_parent = READ_ONCE(node->parent); + node_parent = trie_load_parent(node); if (node->stack_len != pos + lookup.matched) return NULL; if (node_parent != parent && @@ -2925,7 +2940,7 @@ trie_parent_chain_matches_prefix(const struct stack_depot_trie_node *node, if (depth++ >= CONFIG_STACKDEPOT_MAX_FRAMES) return false; - parent = READ_ONCE(cur->parent); + parent = trie_load_parent(cur); if (stack_depot_frame_run_validate(&cur->run)) return false; if (!cur->stack_len || cur->run.nr_entries > cur->stack_len) @@ -3136,7 +3151,7 @@ __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, return 0; pos = total; - for (node = leaf; node; node = READ_ONCE(node->parent)) { + for (node = leaf; node; node = trie_load_parent(node)) { if (stack_depot_frame_run_validate(&node->run)) return 0; if (node->stack_len != pos || node->run.nr_entries > pos) @@ -3609,8 +3624,10 @@ static int trie_split_subtree_prepare(const void *child_ptr, unsigned int matche if (!prepare->fn) return -EINVAL; ret = prepare->fn(updates, nr_updates, prepare->ctx); - if (ret) + if (ret) { + memset(split_array, 0, split_array_size); return ret; + } } old_tail->children = child->children; From e6f8b98bd1731903ec007ef7c175005e15d28235 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 5 Jun 2026 10:52:10 +0100 Subject: [PATCH 044/129] KRN-1117: Add stackdepot trie side-table prepare Add private side-table prepare state for future trie insertions. The prepare helper checkpoints each leaf ID's previous side-table value before publishing the requested replacement and rolls back partial updates on failure. Keep rollback LIFO so duplicate leaf-ID updates restore the original entry correctly. This preserves the side-table-before-structural-publish contract without wiring trie storage into the public save path yet. Cover successful prepare/rollback, invalid IDs, duplicate leaf IDs, capacity overflow, and NULL leaf updates in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 61 +++++++++++++ lib/stackdepot_internal.h | 17 ++++ lib/tests/stackdepot_kunit.c | 163 +++++++++++++++++++++++++++++++++++ 3 files changed, 241 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index d6d4b05ad0f53..dba067b915602 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -523,6 +523,67 @@ size_t __stack_depot_trie_side_table_bytes(void) return bytes; } +void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state) +{ + if (state) + memset(state, 0, sizeof(*state)); +} + +void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *state) +{ + if (!state) + return; + + while (state->nr_updates) { + struct stack_depot_trie_side_checkpoint *update; + + state->nr_updates--; + update = &state->updates[state->nr_updates]; + __stack_depot_trie_side_table_restore(update->leaf_id, update->old_leaf); + } +} + +int +__stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, void *ctx) +{ + struct stack_depot_trie_side_prepare *state = ctx; + unsigned int start; + unsigned int i; + int ret; + + if (!state || (!updates && nr_updates)) + return -EINVAL; + + start = state->nr_updates; + for (i = 0; i < nr_updates; i++) { + if (state->nr_updates >= STACK_DEPOT_TRIE_MAX_LEAF_UPDATES) { + ret = -EINVAL; + goto rollback; + } + + state->updates[state->nr_updates].leaf_id = updates[i].leaf_id; + state->updates[state->nr_updates].old_leaf = + __stack_depot_trie_side_table_lookup(updates[i].leaf_id); + state->nr_updates++; + ret = __stack_depot_trie_side_table_store(updates[i].leaf_id, updates[i].leaf); + if (ret) + goto rollback; + } + + return 0; + +rollback: + while (state->nr_updates > start) { + struct stack_depot_trie_side_checkpoint *update; + + state->nr_updates--; + update = &state->updates[state->nr_updates]; + __stack_depot_trie_side_table_restore(update->leaf_id, update->old_leaf); + } + return ret; +} + static int __init disable_stack_depot(char *str) { return kstrtobool(str, &stack_depot_disabled); diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 1d0ba8317ca0e..f07cfd168a293 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -59,6 +59,18 @@ struct stack_depot_trie_publish_prepare { void *ctx; }; +#define STACK_DEPOT_TRIE_MAX_LEAF_UPDATES 2 + +struct stack_depot_trie_side_checkpoint { + u32 leaf_id; + const void *old_leaf; +}; + +struct stack_depot_trie_side_prepare { + struct stack_depot_trie_side_checkpoint updates[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; + unsigned int nr_updates; +}; + #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) @@ -84,6 +96,11 @@ int __stack_depot_trie_side_table_store(u32 id, const void *entry); const void *__stack_depot_trie_side_table_lookup(u32 id); size_t __stack_depot_trie_side_table_entries(void); size_t __stack_depot_trie_side_table_bytes(void); +void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state); +int +__stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, void *ctx); +void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *state); bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, u32 *low); bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 24edf6865a142..1a3315c3eb0fd 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -753,6 +753,164 @@ static void stackdepot_trie_side_table_bytes(struct kunit *test) KUNIT_EXPECT_GT(test, after, before); } +static void stackdepot_trie_side_prepare_updates(struct kunit *test) +{ + struct stack_depot_trie_side_prepare state; + struct stack_depot_trie_leaf_update updates[2]; + const void *old1 = (const void *)0x1111UL; + const void *old2 = (const void *)0x2222UL; + const void *new1 = (const void *)0xaaaaUL; + const void *new2 = (const void *)0xbbbbUL; + int ret; + u32 id1; + u32 id2; + + stackdepot_trie_side_table_init_or_skip(test); + id1 = stackdepot_trie_side_table_alloc(test); + id2 = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, id1, 1U); + KUNIT_ASSERT_EQ(test, id2, 2U); + KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id1, old1), 0); + KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id2, old2), 0); + updates[0].leaf_id = id1; + updates[0].leaf = new1; + updates[1].leaf_id = id2; + updates[1].leaf = new2; + + __stack_depot_trie_side_prepare_init(&state); + ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &state); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, state.nr_updates, (unsigned int)ARRAY_SIZE(updates)); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), new1); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id2), new2); + + __stack_depot_trie_side_rollback(&state); + KUNIT_EXPECT_EQ(test, state.nr_updates, 0U); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), old1); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id2), old2); +} + +static void stackdepot_trie_side_prepare_failure(struct kunit *test) +{ + struct stack_depot_trie_side_prepare state; + struct stack_depot_trie_leaf_update updates[2]; + const void *old1 = (const void *)0x1111UL; + const void *new1 = (const void *)0xaaaaUL; + int ret; + u32 id; + + stackdepot_trie_side_table_init_or_skip(test); + id = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, id, 1U); + KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, old1), 0); + updates[0].leaf_id = id; + updates[0].leaf = new1; + updates[1].leaf_id = id + 1; + updates[1].leaf = (const void *)0xbbbbUL; + + __stack_depot_trie_side_prepare_init(&state); + ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &state); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, state.nr_updates, 0U); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id), old1); +} + +static void stackdepot_trie_side_prepare_duplicate_id(struct kunit *test) +{ + struct stack_depot_trie_side_prepare state; + struct stack_depot_trie_leaf_update updates[2]; + const void *old = (const void *)0x1111UL; + const void *mid = (const void *)0x2222UL; + const void *new = (const void *)0x3333UL; + int ret; + u32 id; + + stackdepot_trie_side_table_init_or_skip(test); + id = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, id, 1U); + KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, old), 0); + updates[0].leaf_id = id; + updates[0].leaf = mid; + updates[1].leaf_id = id; + updates[1].leaf = new; + + __stack_depot_trie_side_prepare_init(&state); + ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &state); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id), new); + __stack_depot_trie_side_rollback(&state); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id), old); +} + +static void stackdepot_trie_side_prepare_rejects_extra_update(struct kunit *test) +{ + struct stack_depot_trie_side_prepare state; + struct stack_depot_trie_leaf_update updates[3]; + const void *old[] = { + (const void *)0x1111UL, + (const void *)0x2222UL, + (const void *)0x3333UL, + }; + const void *new[] = { + (const void *)0xaaaaUL, + (const void *)0xbbbbUL, + (const void *)0xccccUL, + }; + u32 id[ARRAY_SIZE(updates)]; + unsigned int i; + int ret; + + stackdepot_trie_side_table_init_or_skip(test); + for (i = 0; i < ARRAY_SIZE(updates); i++) { + id[i] = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, id[i], i + 1); + KUNIT_ASSERT_EQ(test, + __stack_depot_trie_side_table_store(id[i], old[i]), + 0); + updates[i].leaf_id = id[i]; + updates[i].leaf = new[i]; + } + + __stack_depot_trie_side_prepare_init(&state); + ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &state); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, state.nr_updates, 0U); + for (i = 0; i < ARRAY_SIZE(updates); i++) + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id[i]), + old[i]); +} + +static void stackdepot_trie_side_prepare_rejects_null_leaf(struct kunit *test) +{ + struct stack_depot_trie_side_prepare state; + struct stack_depot_trie_leaf_update updates[2]; + const void *old1 = (const void *)0x1111UL; + const void *old2 = (const void *)0x2222UL; + const void *new1 = (const void *)0xaaaaUL; + u32 id1; + u32 id2; + int ret; + + stackdepot_trie_side_table_init_or_skip(test); + id1 = stackdepot_trie_side_table_alloc(test); + id2 = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, id1, 1U); + KUNIT_ASSERT_EQ(test, id2, 2U); + KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id1, old1), 0); + KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id2, old2), 0); + updates[0].leaf_id = id1; + updates[0].leaf = new1; + updates[1].leaf_id = id2; + updates[1].leaf = NULL; + + __stack_depot_trie_side_prepare_init(&state); + ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &state); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, state.nr_updates, 0U); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), old1); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id2), old2); +} + static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0xffff888000001000UL; @@ -4314,6 +4472,11 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_side_table_restore), KUNIT_CASE(stackdepot_trie_side_table_chunk_boundary), KUNIT_CASE(stackdepot_trie_side_table_bytes), + KUNIT_CASE(stackdepot_trie_side_prepare_updates), + KUNIT_CASE(stackdepot_trie_side_prepare_failure), + KUNIT_CASE(stackdepot_trie_side_prepare_duplicate_id), + KUNIT_CASE(stackdepot_trie_side_prepare_rejects_extra_update), + KUNIT_CASE(stackdepot_trie_side_prepare_rejects_null_leaf), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), From a5d195cd8c1c01814711a0aa8a0e3c770278e19c Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 5 Jun 2026 11:42:40 +0100 Subject: [PATCH 045/129] KRN-1117: Add stackdepot trie pool sizing Add a private helper to validate and align future trie pool allocation requests. Keep zero-sized and oversized requests invalid, round valid sizes to unsigned-long alignment, and reject anything that would exceed a stackdepot pool after alignment. This keeps the pool-carving policy centralized before adding the actual pool checkpoint and carve helpers. Cover zero, alignment, pool-boundary, oversized, and overflow-sized requests in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 12 ++++++++++++ lib/stackdepot_internal.h | 1 + lib/tests/stackdepot_kunit.c | 22 ++++++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index dba067b915602..399e5f206cc46 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -523,6 +523,18 @@ size_t __stack_depot_trie_side_table_bytes(void) return bytes; } +size_t __stack_depot_trie_pool_alloc_size(size_t size) +{ + size_t aligned; + + if (!size || size > DEPOT_POOL_SIZE) + return 0; + if (check_add_overflow(size, sizeof(unsigned long) - 1, &aligned)) + return 0; + aligned = ALIGN(size, sizeof(unsigned long)); + return aligned <= DEPOT_POOL_SIZE ? aligned : 0; +} + void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state) { if (state) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index f07cfd168a293..8e87f750ae06b 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -96,6 +96,7 @@ int __stack_depot_trie_side_table_store(u32 id, const void *entry); const void *__stack_depot_trie_side_table_lookup(u32 id); size_t __stack_depot_trie_side_table_entries(void); size_t __stack_depot_trie_side_table_bytes(void); +size_t __stack_depot_trie_pool_alloc_size(size_t size); void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state); int __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 1a3315c3eb0fd..24eb9e713888c 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -911,6 +911,27 @@ static void stackdepot_trie_side_prepare_rejects_null_leaf(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id2), old2); } +static void stackdepot_trie_pool_alloc_size(struct kunit *test) +{ + size_t align = sizeof(unsigned long); + + KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(0), 0UL); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(1), align); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(align), align); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(align + 1), + align * 2); + KUNIT_EXPECT_EQ(test, + __stack_depot_trie_pool_alloc_size(DEPOT_POOL_SIZE - 1), + (size_t)DEPOT_POOL_SIZE); + KUNIT_EXPECT_EQ(test, + __stack_depot_trie_pool_alloc_size(DEPOT_POOL_SIZE), + (size_t)DEPOT_POOL_SIZE); + KUNIT_EXPECT_EQ(test, + __stack_depot_trie_pool_alloc_size(DEPOT_POOL_SIZE + 1), + 0UL); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(SIZE_MAX), 0UL); +} + static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0xffff888000001000UL; @@ -4477,6 +4498,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_side_prepare_duplicate_id), KUNIT_CASE(stackdepot_trie_side_prepare_rejects_extra_update), KUNIT_CASE(stackdepot_trie_side_prepare_rejects_null_leaf), + KUNIT_CASE(stackdepot_trie_pool_alloc_size), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), From 403bd563dfb5805f8b5afdcc2147d15f1ab347e7 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 5 Jun 2026 12:08:51 +0100 Subject: [PATCH 046/129] KRN-1117: Align stackdepot trie pool allocations Use stackdepot's pool-offset alignment for future trie allocations instead of unsigned-long alignment. Trie storage will share the existing stackdepot pools with hash stack records, whose handles encode offsets using DEPOT_STACK_ALIGN. Keeping trie allocations on the same boundary preserves the invariant that later hash records can still encode their pool offsets correctly. Cover the stricter alignment in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 5 +++-- lib/tests/stackdepot_kunit.c | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 399e5f206cc46..e5d6a0e7d05e0 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -525,13 +525,14 @@ size_t __stack_depot_trie_side_table_bytes(void) size_t __stack_depot_trie_pool_alloc_size(size_t size) { + size_t align = 1UL << DEPOT_STACK_ALIGN; size_t aligned; if (!size || size > DEPOT_POOL_SIZE) return 0; - if (check_add_overflow(size, sizeof(unsigned long) - 1, &aligned)) + if (check_add_overflow(size, align - 1, &aligned)) return 0; - aligned = ALIGN(size, sizeof(unsigned long)); + aligned = ALIGN(size, align); return aligned <= DEPOT_POOL_SIZE ? aligned : 0; } diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 24eb9e713888c..1a4ac095e9180 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -913,10 +913,12 @@ static void stackdepot_trie_side_prepare_rejects_null_leaf(struct kunit *test) static void stackdepot_trie_pool_alloc_size(struct kunit *test) { - size_t align = sizeof(unsigned long); + size_t align = 1UL << DEPOT_STACK_ALIGN; KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(0), 0UL); KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(1), align); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(sizeof(unsigned long)), + align); KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(align), align); KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(align + 1), align * 2); From 6f19677a773170bc5f0d899e498f901292b472db Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 5 Jun 2026 13:58:45 +0100 Subject: [PATCH 047/129] KRN-1117: Add stackdepot trie pool carve helper Add private helpers to reserve bytes from the current stackdepot pool and roll back the most recent reservation. Keep this first pool-backed step best-effort: the helpers never allocate, never roll over to a new pool, and use trylock so constrained contexts fail instead of blocking. Record enough mark state to enforce LIFO rollback and preserve the pool offset when rollback is no longer the top reservation. Cover successful carve/rollback, LIFO behavior, alignment, and invalid requests in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 62 ++++++++++++++++++++++++++++++++ lib/stackdepot_internal.h | 14 ++++++++ lib/tests/stackdepot_kunit.c | 70 ++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index e5d6a0e7d05e0..62d8552bbf119 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -536,6 +536,68 @@ size_t __stack_depot_trie_pool_alloc_size(size_t size) return aligned <= DEPOT_POOL_SIZE ? aligned : 0; } +void * +__stack_depot_trie_pool_carve_current(size_t size, + struct stack_depot_trie_pool_mark *mark) +{ + unsigned long flags; + size_t alloc_size; + void *pool; + void *ptr = NULL; + + if (!mark) + return NULL; + memset(mark, 0, sizeof(*mark)); + + alloc_size = __stack_depot_trie_pool_alloc_size(size); + if (!alloc_size) + return NULL; + + if (!raw_spin_trylock_irqsave(&pool_lock, flags)) + return NULL; + if (!stack_pools || pools_num < 1) + goto out; + if (WARN_ON_ONCE(pool_offset > DEPOT_POOL_SIZE)) + goto out; + if (alloc_size > DEPOT_POOL_SIZE - pool_offset) + goto out; + + mark->pool_index = pools_num - 1; + pool = stack_pools[mark->pool_index]; + if (WARN_ON_ONCE(!pool)) + goto out; + + mark->offset = pool_offset; + mark->size = alloc_size; + ptr = pool + pool_offset; + pool_offset += alloc_size; +out: + raw_spin_unlock_irqrestore(&pool_lock, flags); + return ptr; +} + +bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mark *mark) +{ + unsigned long flags; + size_t end; + bool ret = false; + + if (!mark || !mark->size) + return false; + if (check_add_overflow(mark->offset, mark->size, &end)) + return false; + + if (!raw_spin_trylock_irqsave(&pool_lock, flags)) + return false; + if (mark->pool_index == pools_num - 1 && pool_offset == end) { + pool_offset = mark->offset; + ret = true; + } + raw_spin_unlock_irqrestore(&pool_lock, flags); + + return ret; +} + void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state) { if (state) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 8e87f750ae06b..2d4ee31cb093c 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -71,6 +71,12 @@ struct stack_depot_trie_side_prepare { unsigned int nr_updates; }; +struct stack_depot_trie_pool_mark { + unsigned int pool_index; + size_t offset; + size_t size; +}; + #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) @@ -97,6 +103,14 @@ const void *__stack_depot_trie_side_table_lookup(u32 id); size_t __stack_depot_trie_side_table_entries(void); size_t __stack_depot_trie_side_table_bytes(void); size_t __stack_depot_trie_pool_alloc_size(size_t size); +/* + * Best-effort current-pool helpers. They never allocate or roll over to a new + * pool, and they use trylock so constrained contexts fail instead of blocking. + */ +void * +__stack_depot_trie_pool_carve_current(size_t size, + struct stack_depot_trie_pool_mark *mark); +bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mark *mark); void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state); int __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 1a4ac095e9180..b9999cfc970ff 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -934,6 +934,73 @@ static void stackdepot_trie_pool_alloc_size(struct kunit *test) KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(SIZE_MAX), 0UL); } +static void stackdepot_trie_pool_seed_current_pool(struct kunit *test) +{ + unsigned long entries[] = { 0x1234567800990000UL }; + depot_stack_handle_t handle; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); +} + +static void stackdepot_trie_pool_carve_current(struct kunit *test) +{ + struct stack_depot_trie_pool_mark first; + struct stack_depot_trie_pool_mark second; + void *ptr1; + void *ptr2; + size_t align = 1UL << DEPOT_STACK_ALIGN; + + stackdepot_trie_pool_seed_current_pool(test); + ptr1 = __stack_depot_trie_pool_carve_current(1, &first); + KUNIT_ASSERT_NOT_NULL(test, ptr1); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)ptr1, align)); + KUNIT_EXPECT_EQ(test, first.size, align); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&first)); + + ptr2 = __stack_depot_trie_pool_carve_current(1, &second); + KUNIT_ASSERT_NOT_NULL(test, ptr2); + KUNIT_EXPECT_PTR_EQ(test, ptr2, ptr1); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&second)); + KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(&first)); +} + +static void stackdepot_trie_pool_rollback_requires_lifo(struct kunit *test) +{ + struct stack_depot_trie_pool_mark first; + struct stack_depot_trie_pool_mark second; + void *ptr1; + void *ptr2; + + stackdepot_trie_pool_seed_current_pool(test); + ptr1 = __stack_depot_trie_pool_carve_current(1, &first); + KUNIT_ASSERT_NOT_NULL(test, ptr1); + ptr2 = __stack_depot_trie_pool_carve_current(1, &second); + KUNIT_ASSERT_NOT_NULL(test, ptr2); + + KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(&first)); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&second)); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&first)); +} + +static void stackdepot_trie_pool_carve_current_rejects_bad_inputs(struct kunit *test) +{ + struct stack_depot_trie_pool_mark mark; + void *ptr; + + stackdepot_trie_pool_seed_current_pool(test); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_pool_carve_current(0, &mark)); + KUNIT_EXPECT_EQ(test, mark.size, 0UL); + ptr = __stack_depot_trie_pool_carve_current(DEPOT_POOL_SIZE + 1, &mark); + KUNIT_EXPECT_NULL(test, ptr); + KUNIT_EXPECT_EQ(test, mark.size, 0UL); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_pool_carve_current(1, NULL)); + KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(NULL)); + memset(&mark, 0, sizeof(mark)); + KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(&mark)); +} + static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0xffff888000001000UL; @@ -4501,6 +4568,9 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_side_prepare_rejects_extra_update), KUNIT_CASE(stackdepot_trie_side_prepare_rejects_null_leaf), KUNIT_CASE(stackdepot_trie_pool_alloc_size), + KUNIT_CASE(stackdepot_trie_pool_carve_current), + KUNIT_CASE(stackdepot_trie_pool_rollback_requires_lifo), + KUNIT_CASE(stackdepot_trie_pool_carve_current_rejects_bad_inputs), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), From 20027ae8a902caf1ac4446def4b0a5c0819c08ec Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 5 Jun 2026 16:32:48 +0100 Subject: [PATCH 048/129] KRN-1117: Add stackdepot trie pool batch carve Add a private request-based helper to reserve a planned set of trie nodes, child arrays, and publish storage from the current stackdepot pool. Keep the helper best-effort: it does not allocate, does not roll over to a new pool, and uses trylock so constrained contexts fail instead of blocking. Return one pool mark for the whole batch and require LIFO rollback through the existing current-pool rollback helper. Cover contiguous layout, rollback reuse, and invalid requests in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 104 +++++++++++++++++++++++++++++++---- lib/stackdepot_internal.h | 11 ++++ lib/tests/stackdepot_kunit.c | 92 +++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 12 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 62d8552bbf119..2391f65bfa904 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -275,7 +275,7 @@ int __stack_depot_trie_side_table_init(gfp_t gfp_flags) { u32 max_leaf_id; - if (trie_side_table_initialized) + if (READ_ONCE(trie_side_table_initialized)) return 0; max_leaf_id = __stack_depot_trie_max_leaf_id(); @@ -293,7 +293,7 @@ int __stack_depot_trie_side_table_init(gfp_t gfp_flags) trie_side_table_high_water = 0; trie_side_table_nr_chunks = 0; trie_side_table_next_id = 0; - trie_side_table_initialized = true; + WRITE_ONCE(trie_side_table_initialized, true); return 0; } @@ -301,7 +301,7 @@ void __stack_depot_trie_side_table_destroy(void) { unsigned int i; - if (!trie_side_table_initialized) + if (!READ_ONCE(trie_side_table_initialized)) return; for (i = 0; i < trie_side_table_high_water; i++) @@ -312,7 +312,7 @@ void __stack_depot_trie_side_table_destroy(void) trie_side_table_nr_chunks = 0; trie_side_table_top_size = 0; trie_side_table_next_id = 0; - trie_side_table_initialized = false; + WRITE_ONCE(trie_side_table_initialized, false); } bool __stack_depot_trie_side_table_prealloc_needed(void) @@ -322,7 +322,7 @@ bool __stack_depot_trie_side_table_prealloc_needed(void) u32 id; unsigned int top; - if (!trie_side_table_initialized) + if (!READ_ONCE(trie_side_table_initialized)) return false; raw_spin_lock_irqsave(&trie_side_table_lock, flags); @@ -362,7 +362,7 @@ u32 __stack_depot_trie_side_table_alloc_id(void **prealloc) u32 id; unsigned int top; - if (!trie_side_table_initialized) + if (!READ_ONCE(trie_side_table_initialized)) return 0; raw_spin_lock_irqsave(&trie_side_table_lock, flags); @@ -401,7 +401,7 @@ void __stack_depot_trie_side_table_revoke_latest(u32 id) unsigned int slot; unsigned int top; - if (!trie_side_table_initialized || !id || + if (!READ_ONCE(trie_side_table_initialized) || !id || id != READ_ONCE(trie_side_table_next_id)) return; @@ -429,7 +429,7 @@ void __stack_depot_trie_side_table_restore(u32 id, const void *entry) unsigned long flags; unsigned int top; - if (!trie_side_table_initialized || !id) + if (!READ_ONCE(trie_side_table_initialized) || !id) return; raw_spin_lock_irqsave(&trie_side_table_lock, flags); @@ -455,7 +455,7 @@ int __stack_depot_trie_side_table_store(u32 id, const void *entry) unsigned int top; int ret = -EINVAL; - if (!trie_side_table_initialized || !id || !entry) + if (!READ_ONCE(trie_side_table_initialized) || !id || !entry) return -EINVAL; raw_spin_lock_irqsave(&trie_side_table_lock, flags); @@ -481,7 +481,7 @@ const void *__stack_depot_trie_side_table_lookup(u32 id) const void **chunk; unsigned int top; - if (!trie_side_table_initialized || !id) + if (!READ_ONCE(trie_side_table_initialized) || !id) return NULL; top = trie_side_table_top_index(id); @@ -497,7 +497,8 @@ const void *__stack_depot_trie_side_table_lookup(u32 id) size_t __stack_depot_trie_side_table_entries(void) { - return trie_side_table_initialized ? READ_ONCE(trie_side_table_next_id) : 0; + return READ_ONCE(trie_side_table_initialized) ? + READ_ONCE(trie_side_table_next_id) : 0; } size_t __stack_depot_trie_side_table_bytes(void) @@ -506,7 +507,7 @@ size_t __stack_depot_trie_side_table_bytes(void) size_t bytes; size_t top_bytes; - if (!trie_side_table_initialized) + if (!READ_ONCE(trie_side_table_initialized)) return 0; if (check_mul_overflow((size_t)trie_side_table_top_size, sizeof(*trie_side_table_chunks), &top_bytes)) @@ -598,6 +599,85 @@ bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mar return ret; } +static int trie_pool_add_size(size_t size, size_t *total) +{ + size_t alloc_size; + + alloc_size = __stack_depot_trie_pool_alloc_size(size); + if (!alloc_size) + return -EINVAL; + if (check_add_overflow(*total, alloc_size, total)) + return -EINVAL; + return *total <= DEPOT_POOL_SIZE ? 0 : -EINVAL; +} + +int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) +{ + unsigned long flags; + unsigned int i; + size_t offset; + size_t total = 0; + void *pool; + int ret = -EINVAL; + + if (!req || !req->mark) + return -EINVAL; + memset(req->mark, 0, sizeof(*req->mark)); + if (!req->storage || *req->storage || + (!req->node_slots && req->nr_node_slots) || + (!req->child_slots && req->nr_child_slots)) + return -EINVAL; + + for (i = 0; i < req->nr_node_slots; i++) { + if (req->node_slots[i].node || + trie_pool_add_size(req->node_slots[i].size, &total)) + return -EINVAL; + } + for (i = 0; i < req->nr_child_slots; i++) { + if (req->child_slots[i].array || + trie_pool_add_size(req->child_slots[i].size, &total)) + return -EINVAL; + } + if (trie_pool_add_size(req->storage_size, &total)) + return -EINVAL; + + if (!raw_spin_trylock_irqsave(&pool_lock, flags)) + return -EBUSY; + if (!stack_pools || pools_num < 1) { + ret = -ENOSPC; + goto out; + } + if (WARN_ON_ONCE(pool_offset > DEPOT_POOL_SIZE)) + goto out; + if (total > DEPOT_POOL_SIZE - pool_offset) { + ret = -ENOSPC; + goto out; + } + + req->mark->pool_index = pools_num - 1; + pool = stack_pools[req->mark->pool_index]; + if (WARN_ON_ONCE(!pool)) + goto out; + + req->mark->offset = pool_offset; + req->mark->size = total; + offset = pool_offset; + for (i = 0; i < req->nr_node_slots; i++) { + req->node_slots[i].node = pool + offset; + offset += __stack_depot_trie_pool_alloc_size(req->node_slots[i].size); + } + for (i = 0; i < req->nr_child_slots; i++) { + req->child_slots[i].array = pool + offset; + offset += __stack_depot_trie_pool_alloc_size(req->child_slots[i].size); + } + *req->storage = pool + offset; + pool_offset += total; + ret = 0; +out: + raw_spin_unlock_irqrestore(&pool_lock, flags); + return ret; +} + void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state) { if (state) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 2d4ee31cb093c..1d4ce44bc285c 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -77,6 +77,16 @@ struct stack_depot_trie_pool_mark { size_t size; }; +struct stack_depot_trie_pool_request { + struct stack_depot_trie_node_slot *node_slots; + unsigned int nr_node_slots; + struct stack_depot_trie_child_array_slot *child_slots; + unsigned int nr_child_slots; + void **storage; + size_t storage_size; + struct stack_depot_trie_pool_mark *mark; +}; + #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) @@ -111,6 +121,7 @@ void * __stack_depot_trie_pool_carve_current(size_t size, struct stack_depot_trie_pool_mark *mark); bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mark *mark); +int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req); void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state); int __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index b9999cfc970ff..79a14c295ebbb 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1001,6 +1001,96 @@ static void stackdepot_trie_pool_carve_current_rejects_bad_inputs(struct kunit * KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(&mark)); } +static void stackdepot_trie_pool_carve_slots(struct kunit *test) +{ + struct stack_depot_trie_child_array_slot child_slots[1] = { + { .size = 1 }, + }; + struct stack_depot_trie_node_slot node_slots[2] = { + { .size = 1 }, + { .size = (1UL << DEPOT_STACK_ALIGN) + 1 }, + }; + struct stack_depot_trie_pool_mark mark; + void *storage = NULL; + struct stack_depot_trie_pool_request req = { + .node_slots = node_slots, + .nr_node_slots = ARRAY_SIZE(node_slots), + .child_slots = child_slots, + .nr_child_slots = ARRAY_SIZE(child_slots), + .storage = &storage, + .storage_size = 1, + .mark = &mark, + }; + size_t child_size; + size_t node0_size; + size_t node1_size; + size_t total; + void *again; + int ret; + + stackdepot_trie_pool_seed_current_pool(test); + ret = __stack_depot_trie_pool_carve(&req); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_NULL(test, node_slots[0].node); + KUNIT_ASSERT_NOT_NULL(test, node_slots[1].node); + KUNIT_ASSERT_NOT_NULL(test, child_slots[0].array); + KUNIT_ASSERT_NOT_NULL(test, storage); + + node0_size = __stack_depot_trie_pool_alloc_size(node_slots[0].size); + node1_size = __stack_depot_trie_pool_alloc_size(node_slots[1].size); + child_size = __stack_depot_trie_pool_alloc_size(child_slots[0].size); + KUNIT_EXPECT_PTR_EQ(test, node_slots[1].node, + (char *)node_slots[0].node + node0_size); + KUNIT_EXPECT_PTR_EQ(test, child_slots[0].array, + (char *)node_slots[1].node + node1_size); + KUNIT_EXPECT_PTR_EQ(test, storage, + (char *)child_slots[0].array + child_size); + total = node0_size + node1_size + child_size + + __stack_depot_trie_pool_alloc_size(1); + KUNIT_EXPECT_EQ(test, mark.size, total); + + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&mark)); + again = __stack_depot_trie_pool_carve_current(1, &mark); + KUNIT_ASSERT_NOT_NULL(test, again); + KUNIT_EXPECT_PTR_EQ(test, again, node_slots[0].node); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&mark)); +} + +static void stackdepot_trie_pool_carve_slots_rejects_bad_inputs(struct kunit *test) +{ + struct stack_depot_trie_child_array_slot child_slot = { .size = 1 }; + struct stack_depot_trie_node_slot node_slot = { .size = 1 }; + struct stack_depot_trie_pool_mark mark; + void *storage = (void *)0x1UL; + struct stack_depot_trie_pool_request req = { + .node_slots = &node_slot, + .nr_node_slots = 1, + .child_slots = &child_slot, + .nr_child_slots = 1, + .storage = &storage, + .storage_size = 1, + .mark = &mark, + }; + int ret; + + stackdepot_trie_pool_seed_current_pool(test); + ret = __stack_depot_trie_pool_carve(&req); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, mark.size, 0UL); + + storage = NULL; + node_slot.node = (void *)0x1UL; + ret = __stack_depot_trie_pool_carve(&req); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, mark.size, 0UL); + + node_slot.node = NULL; + req.storage_size = DEPOT_POOL_SIZE; + ret = __stack_depot_trie_pool_carve(&req); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, mark.size, 0UL); +} + static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0xffff888000001000UL; @@ -4571,6 +4661,8 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_pool_carve_current), KUNIT_CASE(stackdepot_trie_pool_rollback_requires_lifo), KUNIT_CASE(stackdepot_trie_pool_carve_current_rejects_bad_inputs), + KUNIT_CASE(stackdepot_trie_pool_carve_slots), + KUNIT_CASE(stackdepot_trie_pool_carve_slots_rejects_bad_inputs), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), From 562848562c0988c1cf605a3f85f2524c35f2850d Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 5 Jun 2026 16:34:37 +0100 Subject: [PATCH 049/129] KRN-1117: Clarify page_owner count transition Clarify that a first counted stack transition stores the page count plus the stack-list marker and requires the list node allocated beforehand. This makes the defensive WARN_ON_ONCE path in inc_stack_record_count() easier to audit. Signed-off-by: Caleb Kan --- mm/page_owner.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index 811dc09edd630..c497427629525 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -240,7 +240,7 @@ static bool inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, free_stack_record(stack); return false; } - /* new_count is only possible after allocating the list node above. */ + /* new_count includes the list marker, and requires the node allocated above. */ if (new_count) { if (WARN_ON_ONCE(!stack)) { __stack_depot_dec_count_and_test(handle, nr_base_pages + 1); From 3c2ee85442354995c0325715c57b084c826fe86e Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 8 Jun 2026 09:08:47 +0100 Subject: [PATCH 050/129] KRN-1117: Add stackdepot trie pool prealloc Add private helpers to preallocate and release a future trie pool. Keep the policy aligned with the existing stackdepot pool reserve path by rejecting no-spin GFP masks and allocating with gfp_nested_mask(). This prepares for pool rollover without wiring trie storage into the public save path. Cover no-spin rejection, successful allocation, alignment, and freeing in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 17 +++++++++++++++++ lib/stackdepot_internal.h | 2 ++ lib/tests/stackdepot_kunit.c | 13 +++++++++++++ 3 files changed, 32 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 2391f65bfa904..202e787e97150 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -537,6 +537,23 @@ size_t __stack_depot_trie_pool_alloc_size(size_t size) return aligned <= DEPOT_POOL_SIZE ? aligned : 0; } +void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags) +{ + struct page *page; + + if (!gfpflags_allow_spinning(gfp_flags)) + return NULL; + + page = alloc_pages(gfp_nested_mask(gfp_flags), DEPOT_POOL_ORDER); + return page ? page_address(page) : NULL; +} + +void __stack_depot_trie_pool_free_prealloc(void *prealloc) +{ + if (prealloc) + free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); +} + void * __stack_depot_trie_pool_carve_current(size_t size, struct stack_depot_trie_pool_mark *mark) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 1d4ce44bc285c..351254cef2cf9 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -113,6 +113,8 @@ const void *__stack_depot_trie_side_table_lookup(u32 id); size_t __stack_depot_trie_side_table_entries(void); size_t __stack_depot_trie_side_table_bytes(void); size_t __stack_depot_trie_pool_alloc_size(size_t size); +void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags); +void __stack_depot_trie_pool_free_prealloc(void *prealloc); /* * Best-effort current-pool helpers. They never allocate or roll over to a new * pool, and they use trylock so constrained contexts fail instead of blocking. diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 79a14c295ebbb..2fe1516ca1667 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -934,6 +934,18 @@ static void stackdepot_trie_pool_alloc_size(struct kunit *test) KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(SIZE_MAX), 0UL); } +static void stackdepot_trie_pool_prealloc(struct kunit *test) +{ + void *prealloc; + + KUNIT_EXPECT_NULL(test, __stack_depot_trie_pool_prealloc(0)); + prealloc = __stack_depot_trie_pool_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, prealloc); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)prealloc, PAGE_SIZE)); + __stack_depot_trie_pool_free_prealloc(prealloc); + __stack_depot_trie_pool_free_prealloc(NULL); +} + static void stackdepot_trie_pool_seed_current_pool(struct kunit *test) { unsigned long entries[] = { 0x1234567800990000UL }; @@ -4658,6 +4670,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_side_prepare_rejects_extra_update), KUNIT_CASE(stackdepot_trie_side_prepare_rejects_null_leaf), KUNIT_CASE(stackdepot_trie_pool_alloc_size), + KUNIT_CASE(stackdepot_trie_pool_prealloc), KUNIT_CASE(stackdepot_trie_pool_carve_current), KUNIT_CASE(stackdepot_trie_pool_rollback_requires_lifo), KUNIT_CASE(stackdepot_trie_pool_carve_current_rejects_bad_inputs), From 8c8c0bc6fe744bb2190731343cf0f7bf712f525d Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 8 Jun 2026 10:00:10 +0100 Subject: [PATCH 051/129] KRN-1117: Add stackdepot trie pool rollover Teach the private trie pool carve helper to consume a caller-owned preallocated pool when the current pool is empty or lacks enough space. Keep the helper best-effort and private: it still uses trylock, performs no allocation itself, and does not route the public save path. Extend the pool mark so rollback can restore a newly-added pool to the reserve slot when the batch is still the top allocation. Cover rollover, rollback, and reserve reuse in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 36 +++++++++++++++++++++++++++++---- lib/stackdepot_internal.h | 4 ++++ lib/tests/stackdepot_kunit.c | 39 ++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 202e787e97150..6943865f02260 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -153,6 +153,8 @@ static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); /* Count helpers rely on saturated refcounts looking negative. */ static_assert(REFCOUNT_SATURATED < 0); +static bool depot_init_pool(void **prealloc); + static u32 stack_depot_pool_index_mask(void) { return (1U << DEPOT_POOL_INDEX_BITS) - 1; @@ -607,10 +609,23 @@ bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mar if (!raw_spin_trylock_irqsave(&pool_lock, flags)) return false; - if (mark->pool_index == pools_num - 1 && pool_offset == end) { + if (mark->pool_index != pools_num - 1 || pool_offset != end) + goto out; + if (mark->added_pool) { + if (mark->offset || stack_pools[mark->pool_index] != mark->pool) + goto out; + if (new_pool && new_pool != STACK_DEPOT_POISON) + goto out; + stack_pools[mark->pool_index] = NULL; + WRITE_ONCE(pools_num, mark->pool_index); + pool_offset = mark->prev_offset; + WRITE_ONCE(new_pool, mark->pool); + ret = true; + } else { pool_offset = mark->offset; ret = true; } +out: raw_spin_unlock_irqrestore(&pool_lock, flags); return ret; @@ -660,15 +675,27 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) if (!raw_spin_trylock_irqsave(&pool_lock, flags)) return -EBUSY; - if (!stack_pools || pools_num < 1) { + if (!stack_pools) { ret = -ENOSPC; goto out; } + if (pools_num < 1) { + req->mark->prev_offset = pool_offset; + if (!depot_init_pool(req->prealloc)) { + ret = -ENOSPC; + goto out; + } + req->mark->added_pool = true; + } if (WARN_ON_ONCE(pool_offset > DEPOT_POOL_SIZE)) goto out; if (total > DEPOT_POOL_SIZE - pool_offset) { - ret = -ENOSPC; - goto out; + req->mark->prev_offset = pool_offset; + if (!depot_init_pool(req->prealloc)) { + ret = -ENOSPC; + goto out; + } + req->mark->added_pool = true; } req->mark->pool_index = pools_num - 1; @@ -677,6 +704,7 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) goto out; req->mark->offset = pool_offset; + req->mark->pool = pool; req->mark->size = total; offset = pool_offset; for (i = 0; i < req->nr_node_slots; i++) { diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 351254cef2cf9..4a46ec2decfba 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -72,9 +72,12 @@ struct stack_depot_trie_side_prepare { }; struct stack_depot_trie_pool_mark { + void *pool; unsigned int pool_index; + size_t prev_offset; size_t offset; size_t size; + bool added_pool; }; struct stack_depot_trie_pool_request { @@ -84,6 +87,7 @@ struct stack_depot_trie_pool_request { unsigned int nr_child_slots; void **storage; size_t storage_size; + void **prealloc; struct stack_depot_trie_pool_mark *mark; }; diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 2fe1516ca1667..793b9b63f5ad9 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1103,6 +1103,44 @@ static void stackdepot_trie_pool_carve_slots_rejects_bad_inputs(struct kunit *te KUNIT_EXPECT_EQ(test, mark.size, 0UL); } +static void stackdepot_trie_pool_carve_uses_prealloc(struct kunit *test) +{ + struct stack_depot_trie_pool_mark first_mark; + struct stack_depot_trie_pool_mark second_mark; + void *first_storage = NULL; + void *second_storage = NULL; + void *prealloc; + struct stack_depot_trie_pool_request first = { + .storage = &first_storage, + .storage_size = DEPOT_POOL_SIZE, + .prealloc = &prealloc, + .mark = &first_mark, + }; + struct stack_depot_trie_pool_request second = { + .storage = &second_storage, + .storage_size = DEPOT_POOL_SIZE, + .mark = &second_mark, + }; + int ret; + + stackdepot_trie_pool_seed_current_pool(test); + prealloc = __stack_depot_trie_pool_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, prealloc); + + ret = __stack_depot_trie_pool_carve(&first); + KUNIT_ASSERT_EQ(test, ret, 0); + __stack_depot_trie_pool_free_prealloc(prealloc); + KUNIT_ASSERT_NOT_NULL(test, first_storage); + KUNIT_ASSERT_TRUE(test, first_mark.added_pool); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&first_mark)); + + ret = __stack_depot_trie_pool_carve(&second); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, second_storage, first_storage); + KUNIT_ASSERT_TRUE(test, second_mark.added_pool); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&second_mark)); +} + static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0xffff888000001000UL; @@ -4676,6 +4714,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_pool_carve_current_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_pool_carve_slots), KUNIT_CASE(stackdepot_trie_pool_carve_slots_rejects_bad_inputs), + KUNIT_CASE(stackdepot_trie_pool_carve_uses_prealloc), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), From 72e99dbe1dbec7730260bf84150510ba4cd457b9 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 8 Jun 2026 11:08:24 +0100 Subject: [PATCH 052/129] KRN-1117: Add stackdepot trie allocation rollback Add private transaction state for future trie allocation failures. The rollback helper unwinds side-table updates first, then revokes a newly allocated leaf ID, then rolls back pool storage. This preserves the publication contract that side-table entries never point at storage after that storage has been reclaimed. Cover the rollback order and state reset in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 20 ++++++++++++++++ lib/stackdepot_internal.h | 8 +++++++ lib/tests/stackdepot_kunit.c | 46 ++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 6943865f02260..3057abb01363b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -723,6 +723,26 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) return ret; } +void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn) +{ + if (txn) + memset(txn, 0, sizeof(*txn)); +} + +void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn) +{ + if (!txn) + return; + + __stack_depot_trie_side_rollback(&txn->side); + if (txn->leaf_id) { + __stack_depot_trie_side_table_revoke_latest(txn->leaf_id); + txn->leaf_id = 0; + } + __stack_depot_trie_pool_try_rollback(&txn->pool); + memset(&txn->pool, 0, sizeof(txn->pool)); +} + void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state) { if (state) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 4a46ec2decfba..cdb500695e7f2 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -91,6 +91,12 @@ struct stack_depot_trie_pool_request { struct stack_depot_trie_pool_mark *mark; }; +struct stack_depot_trie_alloc_txn { + struct stack_depot_trie_side_prepare side; + struct stack_depot_trie_pool_mark pool; + u32 leaf_id; +}; + #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) @@ -128,6 +134,8 @@ __stack_depot_trie_pool_carve_current(size_t size, struct stack_depot_trie_pool_mark *mark); bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mark *mark); int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req); +void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); +void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state); int __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 793b9b63f5ad9..2ae26b6da96d4 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1141,6 +1141,51 @@ static void stackdepot_trie_pool_carve_uses_prealloc(struct kunit *test) KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&second_mark)); } +static void stackdepot_trie_alloc_txn_rollback(struct kunit *test) +{ + struct stack_depot_trie_leaf_update updates[2]; + struct stack_depot_trie_alloc_txn txn; + const void *old_leaf = (const void *)0x1111UL; + void *pool_leaf; + u32 old_id; + int ret; + + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + __stack_depot_trie_alloc_txn_init(&txn); + old_id = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, old_id, 1U); + txn.leaf_id = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, txn.leaf_id, 2U); + ret = __stack_depot_trie_side_table_store(old_id, old_leaf); + KUNIT_ASSERT_EQ(test, ret, 0); + + pool_leaf = __stack_depot_trie_pool_carve_current(1, &txn.pool); + KUNIT_ASSERT_NOT_NULL(test, pool_leaf); + updates[0].leaf_id = old_id; + updates[0].leaf = pool_leaf; + updates[1].leaf_id = txn.leaf_id; + updates[1].leaf = pool_leaf; + ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &txn.side); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(old_id), + pool_leaf); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(txn.leaf_id), + pool_leaf); + + __stack_depot_trie_alloc_txn_rollback(&txn); + KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); + KUNIT_EXPECT_EQ(test, txn.side.nr_updates, 0U); + KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(old_id), + old_leaf); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(2)); + pool_leaf = __stack_depot_trie_pool_carve_current(1, &txn.pool); + KUNIT_ASSERT_NOT_NULL(test, pool_leaf); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&txn.pool)); +} + static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0xffff888000001000UL; @@ -4715,6 +4760,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_pool_carve_slots), KUNIT_CASE(stackdepot_trie_pool_carve_slots_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_pool_carve_uses_prealloc), + KUNIT_CASE(stackdepot_trie_alloc_txn_rollback), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), From 258129dec5f1b303fc9aff6ab12ff09e774afd19 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 8 Jun 2026 11:51:26 +0100 Subject: [PATCH 053/129] KRN-1117: Add stackdepot trie transaction IDs Add a private helper to allocate a side-table leaf ID into trie allocation transaction state. Keep leaf ownership in the transaction so future failure paths can revoke the ID through the existing rollback helper. Cover successful ID allocation, preallocated side-table chunk consumption, duplicate allocation rejection, and rollback in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 16 ++++++++++++++++ lib/stackdepot_internal.h | 2 ++ lib/tests/stackdepot_kunit.c | 27 +++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 3057abb01363b..51a9aba994c0d 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -729,6 +729,22 @@ void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn) memset(txn, 0, sizeof(*txn)); } +int +__stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, void **prealloc) +{ + u32 leaf_id; + + if (!txn || txn->leaf_id) + return -EINVAL; + + leaf_id = __stack_depot_trie_side_table_alloc_id(prealloc); + if (!leaf_id) + return -ENOSPC; + + txn->leaf_id = leaf_id; + return 0; +} + void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn) { if (!txn) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index cdb500695e7f2..03e63374c890c 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -135,6 +135,8 @@ __stack_depot_trie_pool_carve_current(size_t size, bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mark *mark); int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req); void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); +int +__stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, void **prealloc); void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state); int diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 2ae26b6da96d4..9b71320a51cc7 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1141,6 +1141,32 @@ static void stackdepot_trie_pool_carve_uses_prealloc(struct kunit *test) KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&second_mark)); } +static void stackdepot_trie_alloc_txn_id(struct kunit *test) +{ + struct stack_depot_trie_alloc_txn txn; + void *prealloc = NULL; + int ret; + + stackdepot_trie_side_table_init_or_skip(test); + if (__stack_depot_trie_side_table_prealloc_needed()) { + prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, prealloc); + } + + __stack_depot_trie_alloc_txn_init(&txn); + ret = __stack_depot_trie_alloc_txn_id(&txn, &prealloc); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_NULL(test, prealloc); + KUNIT_EXPECT_EQ(test, txn.leaf_id, 1U); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); + ret = __stack_depot_trie_alloc_txn_id(&txn, NULL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + + __stack_depot_trie_alloc_txn_rollback(&txn); + KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); +} + static void stackdepot_trie_alloc_txn_rollback(struct kunit *test) { struct stack_depot_trie_leaf_update updates[2]; @@ -4760,6 +4786,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_pool_carve_slots), KUNIT_CASE(stackdepot_trie_pool_carve_slots_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_pool_carve_uses_prealloc), + KUNIT_CASE(stackdepot_trie_alloc_txn_id), KUNIT_CASE(stackdepot_trie_alloc_txn_rollback), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 From 8c2112df07c790e445496fc15295f41d499e6eaf Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 8 Jun 2026 12:42:34 +0100 Subject: [PATCH 054/129] KRN-1117: Add stackdepot trie allocation reserve Add a private helper that reserves both pool storage and a side-table leaf ID into trie allocation transaction state. If either step fails, the helper rolls back any partial state and clears caller-visible storage outputs. This composes the existing pool carve and leaf-ID helpers without publishing trie nodes or routing the public save path. Cover successful reservation and leaf-ID failure rollback in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 48 ++++++++++++++++++++++++ lib/stackdepot_internal.h | 23 +++++++++--- lib/tests/stackdepot_kunit.c | 72 ++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 5 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 51a9aba994c0d..9b96941a3e571 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -745,6 +745,54 @@ __stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, void **p return 0; } +static void trie_alloc_request_clear_outputs(struct stack_depot_trie_alloc_request *req) +{ + unsigned int i; + + if (!req) + return; + + if (req->storage) + *req->storage = NULL; + for (i = 0; req->node_slots && i < req->nr_node_slots; i++) + req->node_slots[i].node = NULL; + for (i = 0; req->child_slots && i < req->nr_child_slots; i++) + req->child_slots[i].array = NULL; +} + +int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req) +{ + struct stack_depot_trie_pool_request pool_req = {}; + int ret; + + if (!req || !req->txn) + return -EINVAL; + if (req->txn->leaf_id || req->txn->pool.size || req->txn->side.nr_updates) + return -EINVAL; + + pool_req.node_slots = req->node_slots; + pool_req.nr_node_slots = req->nr_node_slots; + pool_req.child_slots = req->child_slots; + pool_req.nr_child_slots = req->nr_child_slots; + pool_req.storage = req->storage; + pool_req.storage_size = req->storage_size; + pool_req.prealloc = req->pool_prealloc; + pool_req.mark = &req->txn->pool; + + ret = __stack_depot_trie_pool_carve(&pool_req); + if (ret) + return ret; + + ret = __stack_depot_trie_alloc_txn_id(req->txn, req->side_prealloc); + if (ret) { + __stack_depot_trie_alloc_txn_rollback(req->txn); + trie_alloc_request_clear_outputs(req); + return ret; + } + + return 0; +} + void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn) { if (!txn) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 03e63374c890c..9991a15ec9a07 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -42,9 +42,9 @@ struct stack_depot_trie_root { }; struct stack_depot_trie_lookup { - enum stack_depot_trie_lookup_status status; const void *parent; const void *node; + enum stack_depot_trie_lookup_status status; unsigned int matched; }; @@ -73,22 +73,22 @@ struct stack_depot_trie_side_prepare { struct stack_depot_trie_pool_mark { void *pool; - unsigned int pool_index; size_t prev_offset; size_t offset; size_t size; + unsigned int pool_index; bool added_pool; }; struct stack_depot_trie_pool_request { struct stack_depot_trie_node_slot *node_slots; - unsigned int nr_node_slots; struct stack_depot_trie_child_array_slot *child_slots; - unsigned int nr_child_slots; void **storage; - size_t storage_size; void **prealloc; struct stack_depot_trie_pool_mark *mark; + size_t storage_size; + unsigned int nr_node_slots; + unsigned int nr_child_slots; }; struct stack_depot_trie_alloc_txn { @@ -97,6 +97,18 @@ struct stack_depot_trie_alloc_txn { u32 leaf_id; }; +struct stack_depot_trie_alloc_request { + struct stack_depot_trie_alloc_txn *txn; + struct stack_depot_trie_node_slot *node_slots; + unsigned int nr_node_slots; + struct stack_depot_trie_child_array_slot *child_slots; + unsigned int nr_child_slots; + void **storage; + size_t storage_size; + void **pool_prealloc; + void **side_prealloc; +}; + #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) @@ -137,6 +149,7 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req); void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); int __stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, void **prealloc); +int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state); int diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 9b71320a51cc7..0416f40e2a33f 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1167,6 +1167,76 @@ static void stackdepot_trie_alloc_txn_id(struct kunit *test) KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); } +static void stackdepot_trie_alloc_txn_reserve(struct kunit *test) +{ + struct stack_depot_trie_node_slot node_slot = { .size = 1 }; + struct stack_depot_trie_alloc_txn txn; + struct stack_depot_trie_alloc_request req; + void *side_prealloc = NULL; + void *storage = NULL; + int ret; + + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + if (__stack_depot_trie_side_table_prealloc_needed()) { + side_prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, side_prealloc); + } + + __stack_depot_trie_alloc_txn_init(&txn); + req = (struct stack_depot_trie_alloc_request) { + .txn = &txn, + .node_slots = &node_slot, + .nr_node_slots = 1, + .storage = &storage, + .storage_size = 1, + .side_prealloc = &side_prealloc, + }; + ret = __stack_depot_trie_alloc_txn_reserve(&req); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_NULL(test, side_prealloc); + KUNIT_EXPECT_EQ(test, txn.leaf_id, 1U); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); + KUNIT_EXPECT_NOT_NULL(test, node_slot.node); + KUNIT_EXPECT_NOT_NULL(test, storage); + + __stack_depot_trie_alloc_txn_rollback(&txn); + KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); + KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); +} + +static void stackdepot_trie_alloc_txn_reserve_id_failure(struct kunit *test) +{ + struct stack_depot_trie_node_slot node_slot = { .size = 1 }; + struct stack_depot_trie_alloc_txn txn; + struct stack_depot_trie_pool_mark mark; + struct stack_depot_trie_alloc_request req; + void *storage = NULL; + void *again; + int ret; + + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + __stack_depot_trie_alloc_txn_init(&txn); + req = (struct stack_depot_trie_alloc_request) { + .txn = &txn, + .node_slots = &node_slot, + .nr_node_slots = 1, + .storage = &storage, + .storage_size = 1, + }; + ret = __stack_depot_trie_alloc_txn_reserve(&req); + KUNIT_EXPECT_EQ(test, ret, -ENOSPC); + KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); + KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); + KUNIT_EXPECT_NULL(test, node_slot.node); + KUNIT_EXPECT_NULL(test, storage); + again = __stack_depot_trie_pool_carve_current(1, &mark); + KUNIT_ASSERT_NOT_NULL(test, again); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&mark)); +} + static void stackdepot_trie_alloc_txn_rollback(struct kunit *test) { struct stack_depot_trie_leaf_update updates[2]; @@ -4787,6 +4857,8 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_pool_carve_slots_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_pool_carve_uses_prealloc), KUNIT_CASE(stackdepot_trie_alloc_txn_id), + KUNIT_CASE(stackdepot_trie_alloc_txn_reserve), + KUNIT_CASE(stackdepot_trie_alloc_txn_reserve_id_failure), KUNIT_CASE(stackdepot_trie_alloc_txn_rollback), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 From e8820ad52dac0844d411598ab3e179a9e362b183 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 8 Jun 2026 14:31:30 +0100 Subject: [PATCH 055/129] KRN-1117: Add stackdepot trie allocation commit Add a private transaction commit helper that returns the allocated trie leaf ID and clears rollback state after publication succeeds. This prevents later error paths from revoking a live leaf ID or rolling back side-table and pool state that has become part of the published trie. Cover commit semantics in KUnit with non-empty side-table and pool rollback state, and keep rollback after commit as a no-op. While validating this path, harden the supporting private trie helpers by deferring printk while holding pool_lock, matching append-side acquire/release ordering, and packing trie allocation structs to reduce node overhead. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 44 ++++++++++++++++++++---------- lib/stackdepot_internal.h | 13 +++++---- lib/tests/stackdepot_kunit.c | 53 ++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 21 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 9b96941a3e571..154ff5126dda9 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -575,6 +575,7 @@ __stack_depot_trie_pool_carve_current(size_t size, if (!raw_spin_trylock_irqsave(&pool_lock, flags)) return NULL; + printk_deferred_enter(); if (!stack_pools || pools_num < 1) goto out; if (WARN_ON_ONCE(pool_offset > DEPOT_POOL_SIZE)) @@ -592,6 +593,7 @@ __stack_depot_trie_pool_carve_current(size_t size, ptr = pool + pool_offset; pool_offset += alloc_size; out: + printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); return ptr; } @@ -675,6 +677,7 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) if (!raw_spin_trylock_irqsave(&pool_lock, flags)) return -EBUSY; + printk_deferred_enter(); if (!stack_pools) { ret = -ENOSPC; goto out; @@ -719,6 +722,7 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) pool_offset += total; ret = 0; out: + printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); return ret; } @@ -793,6 +797,18 @@ int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request * return 0; } +u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) +{ + u32 leaf_id; + + if (!txn) + return 0; + + leaf_id = txn->leaf_id; + __stack_depot_trie_alloc_txn_init(txn); + return leaf_id; +} + void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn) { if (!txn) @@ -923,6 +939,8 @@ static void init_stack_table(unsigned long entries) int __init stack_depot_early_init(void) { unsigned long entries = 0; + unsigned long min = 1UL << STACK_BUCKET_NUMBER_ORDER_MIN; + unsigned long max = 1UL << STACK_BUCKET_NUMBER_ORDER_MAX; /* This function must be called only once, from mm_init(). */ if (WARN_ON(__stack_depot_early_init_passed)) @@ -960,15 +978,10 @@ int __init stack_depot_early_init(void) if (stack_bucket_number_order) entries = 1UL << stack_bucket_number_order; pr_info("allocating hash table via alloc_large_system_hash\n"); - stack_table = alloc_large_system_hash("stackdepot", - sizeof(struct list_head), - entries, - STACK_HASH_TABLE_SCALE, - HASH_EARLY, - NULL, - &stack_hash_mask, - 1UL << STACK_BUCKET_NUMBER_ORDER_MIN, - 1UL << STACK_BUCKET_NUMBER_ORDER_MAX); + stack_table = alloc_large_system_hash("stackdepot", sizeof(*stack_table), + entries, STACK_HASH_TABLE_SCALE, + HASH_EARLY, NULL, &stack_hash_mask, + min, max); if (!stack_table) { pr_err("hash table allocation failed, disabling\n"); stack_depot_disabled = true; @@ -1200,7 +1213,8 @@ static inline size_t depot_stack_record_size(struct stack_record *s, unsigned in /* Allocates a new stack in a stack depot pool. */ static struct stack_record * -depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, depot_flags_t flags, void **prealloc) +depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, + depot_flags_t flags, void **prealloc) { struct stack_record *stack = NULL; size_t record_size; @@ -1342,9 +1356,8 @@ static inline u32 hash_stack(unsigned long *entries, unsigned int size) * Non-instrumented version of memcmp(). * Does not check the lexicographical order, only the equality. */ -static inline -int stackdepot_memcmp(const unsigned long *u1, const unsigned long *u2, - unsigned int n) +static inline int stackdepot_memcmp(const unsigned long *u1, + const unsigned long *u2, unsigned int n) { for ( ; n-- ; u1++, u2++) { if (*u1 != *u2) @@ -1630,7 +1643,7 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, underflow = count > (unsigned int)old; if (underflow) { - WARN_RATELIMIT(1, "stack depot count underflow\n"); + WARN_RATELIMIT(underflow, "stack depot count underflow\n"); return false; } @@ -2876,7 +2889,8 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, slot = &parent->children; } - old_array = READ_ONCE(*slot); + /* Pairs with append publication's smp_store_release(). */ + old_array = smp_load_acquire(slot); old_size = old_array ? __stack_depot_trie_child_array_size(old_array->nr_children) : 0; new_size = old_array ? old_array->nr_children + 1 : 1; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 9991a15ec9a07..0546aaa3c19c6 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -19,10 +19,10 @@ enum stack_depot_trie_lookup_status { }; struct stack_depot_frame_run { - enum stack_depot_frame_mode mode; - u8 prefix_id; - unsigned int nr_entries; size_t bytes; + unsigned int nr_entries; + u8 mode; + u8 prefix_id; }; struct stack_depot_trie_node_slot { @@ -100,13 +100,13 @@ struct stack_depot_trie_alloc_txn { struct stack_depot_trie_alloc_request { struct stack_depot_trie_alloc_txn *txn; struct stack_depot_trie_node_slot *node_slots; - unsigned int nr_node_slots; struct stack_depot_trie_child_array_slot *child_slots; - unsigned int nr_child_slots; void **storage; - size_t storage_size; void **pool_prealloc; void **side_prealloc; + size_t storage_size; + unsigned int nr_node_slots; + unsigned int nr_child_slots; }; #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 @@ -150,6 +150,7 @@ void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); int __stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, void **prealloc); int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); +u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn); void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state); int diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 0416f40e2a33f..17f048232ce8c 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1237,6 +1237,58 @@ static void stackdepot_trie_alloc_txn_reserve_id_failure(struct kunit *test) KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&mark)); } +static void stackdepot_trie_alloc_txn_commit(struct kunit *test) +{ + struct stack_depot_trie_leaf_update updates[1]; + struct stack_depot_trie_alloc_txn txn; + const void *old_leaf = (const void *)0x1111UL; + void *pool_leaf; + void *prealloc = NULL; + u32 old_id; + u32 leaf_id; + int ret; + + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + if (__stack_depot_trie_side_table_prealloc_needed()) { + prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, prealloc); + } + + __stack_depot_trie_alloc_txn_init(&txn); + old_id = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, old_id, 1U); + ret = __stack_depot_trie_side_table_store(old_id, old_leaf); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = __stack_depot_trie_alloc_txn_id(&txn, &prealloc); + KUNIT_ASSERT_EQ(test, ret, 0); + pool_leaf = __stack_depot_trie_pool_carve_current(1, &txn.pool); + KUNIT_ASSERT_NOT_NULL(test, pool_leaf); + updates[0].leaf_id = old_id; + updates[0].leaf = pool_leaf; + ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &txn.side); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(old_id), + pool_leaf); + KUNIT_EXPECT_NE(test, txn.pool.size, 0UL); + KUNIT_EXPECT_NE(test, txn.side.nr_updates, 0U); + leaf_id = __stack_depot_trie_alloc_txn_commit(&txn); + KUNIT_EXPECT_EQ(test, leaf_id, 2U); + KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); + KUNIT_EXPECT_EQ(test, txn.side.nr_updates, 0U); + KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 2UL); + + __stack_depot_trie_alloc_txn_rollback(&txn); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 2UL); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(old_id), + pool_leaf); + pool_leaf = __stack_depot_trie_pool_carve_current(1, &txn.pool); + KUNIT_ASSERT_NOT_NULL(test, pool_leaf); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&txn.pool)); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_alloc_txn_commit(NULL), 0U); +} + static void stackdepot_trie_alloc_txn_rollback(struct kunit *test) { struct stack_depot_trie_leaf_update updates[2]; @@ -4859,6 +4911,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_alloc_txn_id), KUNIT_CASE(stackdepot_trie_alloc_txn_reserve), KUNIT_CASE(stackdepot_trie_alloc_txn_reserve_id_failure), + KUNIT_CASE(stackdepot_trie_alloc_txn_commit), KUNIT_CASE(stackdepot_trie_alloc_txn_rollback), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 From 8248768d0e2d412cb034c4bbe7f99f2fa2593b04 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 8 Jun 2026 15:09:43 +0100 Subject: [PATCH 056/129] KRN-1117: Add stackdepot trie allocation insert Add a private helper that composes trie allocation transactions with the existing insertion path. The helper reserves pool storage and a side-table leaf ID, inserts through the existing append/promote/split machinery, and commits only after structural publication succeeds. On failure, unwind side-table updates, revoke the allocated leaf ID, roll back pool storage, and clear caller-visible storage outputs. Cover successful insertion and stale-plan rollback in KUnit without routing the public save path to trie storage yet. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 48 ++++++++++ lib/stackdepot_internal.h | 7 ++ lib/tests/stackdepot_kunit.c | 167 +++++++++++++++++++++++++++++++++++ 3 files changed, 222 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 154ff5126dda9..e7d7952397647 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -809,6 +809,54 @@ u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) return leaf_id; } +int +__stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, + struct stack_depot_trie_alloc_request *req, + const unsigned long *entries, + unsigned int nr_entries, u32 *scratch, + unsigned int nr_scratch, const void **tail, + u32 *leaf_id) +{ + struct stack_depot_trie_publish_prepare prepare; + struct stack_depot_trie_alloc_txn *txn; + u32 id; + void *storage; + unsigned int nr_used; + int ret; + + if (!root || !req || !req->txn || !tail || !leaf_id) + return -EINVAL; + txn = req->txn; + *tail = NULL; + *leaf_id = 0; + + ret = __stack_depot_trie_alloc_txn_reserve(req); + if (ret) + return ret; + storage = req->storage ? *req->storage : NULL; + + prepare.fn = __stack_depot_trie_side_prepare; + prepare.ctx = &txn->side; + id = txn->leaf_id; + ret = __stack_depot_trie_insert_append_prepare(root, NULL, id, entries, + nr_entries, req->node_slots, + req->nr_node_slots, req->child_slots, + req->nr_child_slots, scratch, + nr_scratch, storage, req->storage_size, + &prepare, tail, &nr_used); + if (ret) + goto rollback; + + *leaf_id = __stack_depot_trie_alloc_txn_commit(txn); + return 0; + +rollback: + __stack_depot_trie_alloc_txn_rollback(req->txn); + trie_alloc_request_clear_outputs(req); + *tail = NULL; + return ret; +} + void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn) { if (!txn) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 0546aaa3c19c6..bb981c2a12c5d 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -151,6 +151,13 @@ int __stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, void **prealloc); int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn); +int +__stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, + struct stack_depot_trie_alloc_request *req, + const unsigned long *entries, + unsigned int nr_entries, u32 *scratch, + unsigned int nr_scratch, const void **tail, + u32 *leaf_id); void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state); int diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 17f048232ce8c..50dfe26d2f952 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1334,6 +1334,171 @@ static void stackdepot_trie_alloc_txn_rollback(struct kunit *test) KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&txn.pool)); } +static int txn_insert_plan(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + struct stack_depot_trie_alloc_txn *txn, + void **storage, void **side_prealloc, + struct stack_depot_trie_alloc_request *req) +{ + unsigned int child_used; + unsigned int used; + size_t storage_size; + int ret; + + ret = insert_plan(root, NULL, entries, nr_entries, node_slots, + nr_node_slots, child_slots, nr_child_slots, &storage_size, + &used, &child_used); + if (ret) + return ret; + + __stack_depot_trie_alloc_txn_init(txn); + *storage = NULL; + *req = (struct stack_depot_trie_alloc_request) { + .txn = txn, + .node_slots = node_slots, + .nr_node_slots = used, + .child_slots = child_slots, + .nr_child_slots = child_used, + .storage = storage, + .storage_size = storage_size, + .side_prealloc = side_prealloc, + }; + return 0; +} + +static int txn_insert(struct stack_depot_trie_root *root, + struct stack_depot_trie_alloc_request *req, + const unsigned long *entries, unsigned int nr_entries, + const void **tail, u32 *leaf_id) +{ + return __stack_depot_trie_alloc_txn_insert(root, req, entries, nr_entries, + NULL, 0, tail, leaf_id); +} + +static void stackdepot_trie_alloc_txn_insert(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_child_array_slot child_slots[1]; + struct stack_depot_trie_node_slot node_slots[1]; + struct stack_depot_trie_alloc_request req; + struct stack_depot_trie_alloc_txn txn; + struct stack_depot_trie_root root = {}; + unsigned long scratch[ARRAY_SIZE(entries)]; + unsigned long out[ARRAY_SIZE(entries)] = {}; + void *side_prealloc = NULL; + const void *tail = NULL; + void *storage = NULL; + unsigned int fetched; + u32 leaf_id = 0; + int ret; + + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + if (__stack_depot_trie_side_table_prealloc_needed()) { + side_prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, side_prealloc); + } + + ret = txn_insert_plan(&root, entries, ARRAY_SIZE(entries), node_slots, + ARRAY_SIZE(node_slots), child_slots, + ARRAY_SIZE(child_slots), &txn, &storage, + &side_prealloc, &req); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = txn_insert(&root, &req, entries, ARRAY_SIZE(entries), &tail, &leaf_id); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_EXPECT_EQ(test, leaf_id, 1U); + KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); + KUNIT_EXPECT_EQ(test, txn.side.nr_updates, 0U); + KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(leaf_id), + tail); + KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), + tail); + fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); + + __stack_depot_trie_alloc_txn_rollback(&txn); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(leaf_id), + tail); +} + +static void stackdepot_trie_alloc_txn_insert_stale_plan(struct kunit *test) +{ + unsigned long first[] = { 0x1000UL }; + unsigned long second[] = { 0x2000UL }; + struct stack_depot_trie_child_array_slot child_slots[1]; + struct stack_depot_trie_child_array_slot fresh_child_slots[1]; + struct stack_depot_trie_node_slot node_slots[1]; + struct stack_depot_trie_node_slot fresh_node_slots[1]; + struct stack_depot_trie_alloc_request req; + struct stack_depot_trie_alloc_request fresh_req; + struct stack_depot_trie_alloc_txn txn; + struct stack_depot_trie_alloc_txn fresh_txn; + struct stack_depot_trie_root root = {}; + void *side_prealloc = NULL; + const void *tail = NULL; + const void *fresh_tail = NULL; + void *storage = NULL; + void *fresh_storage = NULL; + u32 leaf_id = 0; + u32 fresh_leaf_id = 0; + int ret; + + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + if (__stack_depot_trie_side_table_prealloc_needed()) { + side_prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, side_prealloc); + } + + ret = txn_insert_plan(&root, first, ARRAY_SIZE(first), fresh_node_slots, + ARRAY_SIZE(fresh_node_slots), fresh_child_slots, + ARRAY_SIZE(fresh_child_slots), &fresh_txn, + &fresh_storage, &side_prealloc, &fresh_req); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = txn_insert(&root, &fresh_req, first, ARRAY_SIZE(first), &fresh_tail, + &fresh_leaf_id); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, fresh_leaf_id, 1U); + + ret = txn_insert_plan(&root, second, ARRAY_SIZE(second), node_slots, + ARRAY_SIZE(node_slots), child_slots, + ARRAY_SIZE(child_slots), &txn, &storage, + &side_prealloc, &req); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = txn_insert_plan(&root, second, ARRAY_SIZE(second), fresh_node_slots, + ARRAY_SIZE(fresh_node_slots), fresh_child_slots, + ARRAY_SIZE(fresh_child_slots), &fresh_txn, + &fresh_storage, &side_prealloc, &fresh_req); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = txn_insert(&root, &fresh_req, second, ARRAY_SIZE(second), &fresh_tail, + &fresh_leaf_id); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, fresh_leaf_id, 2U); + + ret = txn_insert(&root, &req, second, ARRAY_SIZE(second), &tail, &leaf_id); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, leaf_id, 0U); + KUNIT_EXPECT_NULL(test, tail); + KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); + KUNIT_EXPECT_EQ(test, txn.side.nr_updates, 0U); + KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 2UL); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(1), + find_leaf(&root, first, ARRAY_SIZE(first))); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(2), + find_leaf(&root, second, ARRAY_SIZE(second))); + KUNIT_EXPECT_NULL(test, storage); + KUNIT_EXPECT_NULL(test, node_slots[0].node); +} + static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0xffff888000001000UL; @@ -4913,6 +5078,8 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_alloc_txn_reserve_id_failure), KUNIT_CASE(stackdepot_trie_alloc_txn_commit), KUNIT_CASE(stackdepot_trie_alloc_txn_rollback), + KUNIT_CASE(stackdepot_trie_alloc_txn_insert), + KUNIT_CASE(stackdepot_trie_alloc_txn_insert_stale_plan), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), From 4aeddaebe1795cc486a014923c494c191dc20c7f Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 8 Jun 2026 15:56:18 +0100 Subject: [PATCH 057/129] KRN-1117: Serialize stackdepot trie allocation insert Add a private raw-spin trylock around trie allocation insertion so future writers cannot interleave side-table leaf ID allocation, structural publish, and rollback. This keeps failed stale-plan inserts from racing with later leaf IDs and leaves constrained callers able to fail instead of blocking. The helper remains private and does not route the public save path to trie storage yet. Lockdep-enabled KUnit covers the new raw-lock nesting. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index e7d7952397647..f16d901427d8b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -226,6 +226,7 @@ u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) static const void ***trie_side_table_chunks; static DEFINE_RAW_SPINLOCK(trie_side_table_lock); +static DEFINE_RAW_SPINLOCK(trie_alloc_lock); static unsigned int trie_side_table_high_water; static unsigned int trie_side_table_nr_chunks; static unsigned int trie_side_table_top_size; @@ -819,6 +820,7 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, { struct stack_depot_trie_publish_prepare prepare; struct stack_depot_trie_alloc_txn *txn; + unsigned long flags; u32 id; void *storage; unsigned int nr_used; @@ -830,9 +832,12 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, *tail = NULL; *leaf_id = 0; + if (!raw_spin_trylock_irqsave(&trie_alloc_lock, flags)) + return -EBUSY; + ret = __stack_depot_trie_alloc_txn_reserve(req); if (ret) - return ret; + goto out_unlock; storage = req->storage ? *req->storage : NULL; prepare.fn = __stack_depot_trie_side_prepare; @@ -848,12 +853,15 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, goto rollback; *leaf_id = __stack_depot_trie_alloc_txn_commit(txn); - return 0; + ret = 0; + goto out_unlock; rollback: __stack_depot_trie_alloc_txn_rollback(req->txn); trie_alloc_request_clear_outputs(req); *tail = NULL; +out_unlock: + raw_spin_unlock_irqrestore(&trie_alloc_lock, flags); return ret; } From 0ee779cd57d611d2f53962146be55eada6b65f14 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 8 Jun 2026 17:30:32 +0100 Subject: [PATCH 058/129] KRN-1117: Add stackdepot trie feature flag Add a default-off built-in module parameter backed by a static key for the future trie storage route. The parameter supports boot-time control through the normal stackdepot module parameter namespace and runtime control through the generated sysfs parameter. Keep the flag inert for now and leave the public save path hash-backed. While validating the flag plumbing, fix trie pool rollover with no preallocated pool and cover that failure path in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 34 +++++++++++++++++++++++++++- lib/tests/stackdepot_kunit.c | 43 ++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index f16d901427d8b..c6c2cd3c69bfa 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -18,10 +18,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -53,6 +55,36 @@ static bool stack_depot_disabled; static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT); static bool __stack_depot_early_init_passed __initdata; +static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); +static bool stack_depot_trie_enabled_param; + +static int +stack_depot_trie_enabled_param_set(const char *val, + const struct kernel_param *kp) +{ + int ret; + + ret = param_set_bool(val, kp); + if (ret) + return ret; + + if (stack_depot_trie_enabled_param) + static_branch_enable(&stack_depot_trie_enabled); + else + static_branch_disable(&stack_depot_trie_enabled); + + return 0; +} + +static const struct kernel_param_ops stack_depot_trie_enabled_param_ops = { + .flags = KERNEL_PARAM_OPS_FL_NOARG, + .set = stack_depot_trie_enabled_param_set, + .get = param_get_bool, +}; +module_param_cb(trie_enabled, &stack_depot_trie_enabled_param_ops, + &stack_depot_trie_enabled_param, 0644); +MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage"); + /* Use one hash table bucket per 16 KB of memory. */ #define STACK_HASH_TABLE_SCALE 14 /* Limit the number of buckets between 4K and 1M. */ @@ -1144,7 +1176,7 @@ static bool depot_init_pool(void **prealloc) return false; } - if (!new_pool && *prealloc) { + if (!new_pool && prealloc && *prealloc) { /* We have preallocated memory, use it. */ WRITE_ONCE(new_pool, *prealloc); *prealloc = NULL; diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 50dfe26d2f952..d16ba4956631c 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1141,6 +1141,48 @@ static void stackdepot_trie_pool_carve_uses_prealloc(struct kunit *test) KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&second_mark)); } +static void stackdepot_trie_pool_carve_no_prealloc_rollover(struct kunit *test) +{ + struct stack_depot_trie_pool_mark marks[2]; + void *storage[ARRAY_SIZE(marks)]; + unsigned int consumed = 0; + void *failed_storage = NULL; + struct stack_depot_trie_pool_mark failed_mark; + struct stack_depot_trie_pool_request failed = { + .storage = &failed_storage, + .storage_size = 1, + .mark = &failed_mark, + }; + unsigned int i; + int ret; + + stackdepot_trie_pool_seed_current_pool(test); + for (i = 0; i < ARRAY_SIZE(marks); i++) { + struct stack_depot_trie_pool_request req = { + .storage = &storage[i], + .storage_size = DEPOT_POOL_SIZE, + .mark = &marks[i], + }; + + storage[i] = NULL; + ret = __stack_depot_trie_pool_carve(&req); + if (ret) + break; + KUNIT_ASSERT_NOT_NULL(test, storage[i]); + consumed++; + } + + failed.storage_size = DEPOT_POOL_SIZE; + ret = __stack_depot_trie_pool_carve(&failed); + KUNIT_EXPECT_EQ(test, ret, -ENOSPC); + KUNIT_EXPECT_NULL(test, failed_storage); + KUNIT_EXPECT_EQ(test, failed_mark.size, 0UL); + + while (consumed--) + KUNIT_ASSERT_TRUE(test, + __stack_depot_trie_pool_try_rollback(&marks[consumed])); +} + static void stackdepot_trie_alloc_txn_id(struct kunit *test) { struct stack_depot_trie_alloc_txn txn; @@ -5073,6 +5115,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_pool_carve_slots), KUNIT_CASE(stackdepot_trie_pool_carve_slots_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_pool_carve_uses_prealloc), + KUNIT_CASE(stackdepot_trie_pool_carve_no_prealloc_rollover), KUNIT_CASE(stackdepot_trie_alloc_txn_id), KUNIT_CASE(stackdepot_trie_alloc_txn_reserve), KUNIT_CASE(stackdepot_trie_alloc_txn_reserve_id_failure), From a405d3d98141b508f8f6d51f2bc0b6fbcdc857cf Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 9 Jun 2026 09:33:06 +0100 Subject: [PATCH 059/129] KRN-1117: Remove stackdepot trie feature flag Remove the default-off trie module parameter while the rollout strategy is still under discussion. Keeping the gate out of the port for now avoids adding an extra public routing mode before we confirm the kill-switch tradeoff is worth the validation cost. The private trie helpers remain unchanged and the public save path remains hash-backed. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index c6c2cd3c69bfa..c9b64e642bb6b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -18,12 +18,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -55,36 +53,6 @@ static bool stack_depot_disabled; static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT); static bool __stack_depot_early_init_passed __initdata; -static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); -static bool stack_depot_trie_enabled_param; - -static int -stack_depot_trie_enabled_param_set(const char *val, - const struct kernel_param *kp) -{ - int ret; - - ret = param_set_bool(val, kp); - if (ret) - return ret; - - if (stack_depot_trie_enabled_param) - static_branch_enable(&stack_depot_trie_enabled); - else - static_branch_disable(&stack_depot_trie_enabled); - - return 0; -} - -static const struct kernel_param_ops stack_depot_trie_enabled_param_ops = { - .flags = KERNEL_PARAM_OPS_FL_NOARG, - .set = stack_depot_trie_enabled_param_set, - .get = param_get_bool, -}; -module_param_cb(trie_enabled, &stack_depot_trie_enabled_param_ops, - &stack_depot_trie_enabled_param, 0644); -MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage"); - /* Use one hash table bucket per 16 KB of memory. */ #define STACK_HASH_TABLE_SCALE 14 /* Limit the number of buckets between 4K and 1M. */ From 69b854a693969df7f2f2cc544fb13caa850ab220 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 9 Jun 2026 10:17:15 +0100 Subject: [PATCH 060/129] KRN-1117: Add stackdepot trie allocation planning Add a private helper that turns a trie insertion plan into allocation transaction request state. The helper keeps caller-owned planner storage explicit, initializes the transaction, clears the storage output, and records the node, child-array, publish-storage, and preallocation inputs needed by the reserve and insert helpers. This moves the plan-to-request composition out of KUnit scaffolding and into the private stackdepot implementation without routing the public save path to trie storage yet. Cover the populated request state and bad-input rejection in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 44 ++++++++++++++++++++ lib/stackdepot_internal.h | 12 ++++++ lib/tests/stackdepot_kunit.c | 78 ++++++++++++++++++++++-------------- 3 files changed, 105 insertions(+), 29 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index c9b64e642bb6b..e5c5d71eba555 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -798,6 +798,50 @@ int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request * return 0; } +int +__stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, + const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + struct stack_depot_trie_alloc_txn *txn, + void **storage, void **pool_prealloc, + void **side_prealloc, + struct stack_depot_trie_alloc_request *req) +{ + unsigned int nr_child_used; + unsigned int nr_used; + size_t storage_size; + int ret; + + if (!root || !txn || !storage || !req) + return -EINVAL; + + ret = __stack_depot_trie_insert_plan(root, NULL, entries, nr_entries, + node_slots, nr_node_slots, child_slots, + nr_child_slots, &storage_size, &nr_used, + &nr_child_used); + if (ret) + return ret; + + __stack_depot_trie_alloc_txn_init(txn); + *storage = NULL; + *req = (struct stack_depot_trie_alloc_request) { + .txn = txn, + .node_slots = node_slots, + .child_slots = child_slots, + .storage = storage, + .pool_prealloc = pool_prealloc, + .side_prealloc = side_prealloc, + .storage_size = storage_size, + .nr_node_slots = nr_used, + .nr_child_slots = nr_child_used, + }; + return 0; +} + u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) { u32 leaf_id; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index bb981c2a12c5d..5a12a9620a0f9 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -149,6 +149,18 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req); void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); int __stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, void **prealloc); +int +__stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, + const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + struct stack_depot_trie_alloc_txn *txn, + void **storage, void **pool_prealloc, + void **side_prealloc, + struct stack_depot_trie_alloc_request *req); int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn); int diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index d16ba4956631c..c9a1c0a25badb 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1383,33 +1383,15 @@ static int txn_insert_plan(struct stack_depot_trie_root *root, struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, struct stack_depot_trie_alloc_txn *txn, - void **storage, void **side_prealloc, + void **storage, void **pool_prealloc, + void **side_prealloc, struct stack_depot_trie_alloc_request *req) { - unsigned int child_used; - unsigned int used; - size_t storage_size; - int ret; - - ret = insert_plan(root, NULL, entries, nr_entries, node_slots, - nr_node_slots, child_slots, nr_child_slots, &storage_size, - &used, &child_used); - if (ret) - return ret; - - __stack_depot_trie_alloc_txn_init(txn); - *storage = NULL; - *req = (struct stack_depot_trie_alloc_request) { - .txn = txn, - .node_slots = node_slots, - .nr_node_slots = used, - .child_slots = child_slots, - .nr_child_slots = child_used, - .storage = storage, - .storage_size = storage_size, - .side_prealloc = side_prealloc, - }; - return 0; + return __stack_depot_trie_alloc_txn_plan(root, entries, nr_entries, + node_slots, nr_node_slots, + child_slots, nr_child_slots, txn, + storage, pool_prealloc, side_prealloc, + req); } static int txn_insert(struct stack_depot_trie_root *root, @@ -1421,6 +1403,43 @@ static int txn_insert(struct stack_depot_trie_root *root, NULL, 0, tail, leaf_id); } +static void stackdepot_trie_alloc_txn_plan(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_child_array_slot child_slot; + struct stack_depot_trie_node_slot node_slot; + struct stack_depot_trie_alloc_request req; + struct stack_depot_trie_alloc_txn txn; + struct stack_depot_trie_root root = {}; + void *pool_prealloc = (void *)0x1111UL; + void *side_prealloc = (void *)0x2222UL; + void *storage = (void *)0x3333UL; + int ret; + + ret = txn_insert_plan(&root, entries, ARRAY_SIZE(entries), &node_slot, 1, + &child_slot, 1, &txn, &storage, &pool_prealloc, + &side_prealloc, &req); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, req.txn, &txn); + KUNIT_EXPECT_PTR_EQ(test, req.node_slots, &node_slot); + KUNIT_EXPECT_EQ(test, req.nr_node_slots, 1U); + KUNIT_EXPECT_PTR_EQ(test, req.child_slots, &child_slot); + KUNIT_EXPECT_EQ(test, req.nr_child_slots, 0U); + KUNIT_EXPECT_PTR_EQ(test, req.storage, &storage); + KUNIT_EXPECT_PTR_EQ(test, req.pool_prealloc, &pool_prealloc); + KUNIT_EXPECT_PTR_EQ(test, req.side_prealloc, &side_prealloc); + KUNIT_EXPECT_NE(test, req.storage_size, 0UL); + KUNIT_EXPECT_NULL(test, storage); + KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); + KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); + KUNIT_EXPECT_EQ(test, txn.side.nr_updates, 0U); + + ret = txn_insert_plan(NULL, entries, ARRAY_SIZE(entries), &node_slot, 1, + &child_slot, 1, &txn, &storage, &pool_prealloc, + &side_prealloc, &req); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + static void stackdepot_trie_alloc_txn_insert(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; @@ -1448,7 +1467,7 @@ static void stackdepot_trie_alloc_txn_insert(struct kunit *test) ret = txn_insert_plan(&root, entries, ARRAY_SIZE(entries), node_slots, ARRAY_SIZE(node_slots), child_slots, ARRAY_SIZE(child_slots), &txn, &storage, - &side_prealloc, &req); + NULL, &side_prealloc, &req); KUNIT_ASSERT_EQ(test, ret, 0); ret = txn_insert(&root, &req, entries, ARRAY_SIZE(entries), &tail, &leaf_id); KUNIT_ASSERT_EQ(test, ret, 0); @@ -1503,7 +1522,7 @@ static void stackdepot_trie_alloc_txn_insert_stale_plan(struct kunit *test) ret = txn_insert_plan(&root, first, ARRAY_SIZE(first), fresh_node_slots, ARRAY_SIZE(fresh_node_slots), fresh_child_slots, ARRAY_SIZE(fresh_child_slots), &fresh_txn, - &fresh_storage, &side_prealloc, &fresh_req); + &fresh_storage, NULL, &side_prealloc, &fresh_req); KUNIT_ASSERT_EQ(test, ret, 0); ret = txn_insert(&root, &fresh_req, first, ARRAY_SIZE(first), &fresh_tail, &fresh_leaf_id); @@ -1513,12 +1532,12 @@ static void stackdepot_trie_alloc_txn_insert_stale_plan(struct kunit *test) ret = txn_insert_plan(&root, second, ARRAY_SIZE(second), node_slots, ARRAY_SIZE(node_slots), child_slots, ARRAY_SIZE(child_slots), &txn, &storage, - &side_prealloc, &req); + NULL, &side_prealloc, &req); KUNIT_ASSERT_EQ(test, ret, 0); ret = txn_insert_plan(&root, second, ARRAY_SIZE(second), fresh_node_slots, ARRAY_SIZE(fresh_node_slots), fresh_child_slots, ARRAY_SIZE(fresh_child_slots), &fresh_txn, - &fresh_storage, &side_prealloc, &fresh_req); + &fresh_storage, NULL, &side_prealloc, &fresh_req); KUNIT_ASSERT_EQ(test, ret, 0); ret = txn_insert(&root, &fresh_req, second, ARRAY_SIZE(second), &fresh_tail, &fresh_leaf_id); @@ -5121,6 +5140,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_alloc_txn_reserve_id_failure), KUNIT_CASE(stackdepot_trie_alloc_txn_commit), KUNIT_CASE(stackdepot_trie_alloc_txn_rollback), + KUNIT_CASE(stackdepot_trie_alloc_txn_plan), KUNIT_CASE(stackdepot_trie_alloc_txn_insert), KUNIT_CASE(stackdepot_trie_alloc_txn_insert_stale_plan), KUNIT_CASE(stackdepot_frame_raw_fallback), From 3720e5f01a9239d304e916df0096701b4bb8c67f Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 9 Jun 2026 11:21:31 +0100 Subject: [PATCH 061/129] KRN-1117: Add stackdepot trie allocation workspace Add a private writer workspace for trie allocation planning. The workspace groups the transaction state, allocation request, node slots, child-array slots, compression scratch space, and publish storage output that the future save-miss path needs without placing CONFIG_STACKDEPOT_MAX_FRAMES-sized arrays on the kernel stack. Provide a helper that plans an insert into the workspace while keeping preallocation ownership explicit. Cover heap-backed workspace planning and insertion in KUnit without routing the public save path to trie storage yet. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 20 +++++++++++ lib/stackdepot_internal.h | 18 ++++++++++ lib/tests/stackdepot_kunit.c | 68 ++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index e5c5d71eba555..8bd3726a0a9b5 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -842,6 +842,25 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, return 0; } +int +__stack_depot_trie_alloc_workspace_plan(const struct stack_depot_trie_root *root, + const unsigned long *entries, + unsigned int nr_entries, + void **pool_prealloc, + void **side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace) +{ + if (!workspace) + return -EINVAL; + + memset(workspace, 0, sizeof(*workspace)); + return __stack_depot_trie_alloc_txn_plan(root, entries, nr_entries, + workspace->node_slots, ARRAY_SIZE(workspace->node_slots), + workspace->child_slots, ARRAY_SIZE(workspace->child_slots), + &workspace->txn, &workspace->storage, pool_prealloc, + side_prealloc, &workspace->req); +} + u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) { u32 leaf_id; @@ -1188,6 +1207,7 @@ static bool depot_init_pool(void **prealloc) return false; } + /* Trie allocation probes may intentionally call without a preallocation. */ if (!new_pool && prealloc && *prealloc) { /* We have preallocated memory, use it. */ WRITE_ONCE(new_pool, *prealloc); diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 5a12a9620a0f9..f39c323bcea62 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -60,6 +60,8 @@ struct stack_depot_trie_publish_prepare { }; #define STACK_DEPOT_TRIE_MAX_LEAF_UPDATES 2 +#define STACK_DEPOT_TRIE_MAX_NODE_SLOTS (CONFIG_STACKDEPOT_MAX_FRAMES + 1) +#define STACK_DEPOT_TRIE_MAX_CHILD_SLOTS CONFIG_STACKDEPOT_MAX_FRAMES struct stack_depot_trie_side_checkpoint { u32 leaf_id; @@ -109,6 +111,15 @@ struct stack_depot_trie_alloc_request { unsigned int nr_child_slots; }; +struct stack_depot_trie_alloc_workspace { + struct stack_depot_trie_alloc_txn txn; + struct stack_depot_trie_alloc_request req; + struct stack_depot_trie_node_slot node_slots[STACK_DEPOT_TRIE_MAX_NODE_SLOTS]; + struct stack_depot_trie_child_array_slot child_slots[STACK_DEPOT_TRIE_MAX_CHILD_SLOTS]; + u32 scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; + void *storage; +}; + #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) @@ -161,6 +172,13 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, void **storage, void **pool_prealloc, void **side_prealloc, struct stack_depot_trie_alloc_request *req); +int +__stack_depot_trie_alloc_workspace_plan(const struct stack_depot_trie_root *root, + const unsigned long *entries, + unsigned int nr_entries, + void **pool_prealloc, + void **side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace); int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn); int diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index c9a1c0a25badb..390331695f3f3 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1403,6 +1403,73 @@ static int txn_insert(struct stack_depot_trie_root *root, NULL, 0, tail, leaf_id); } +static int workspace_plan(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, void **side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace) +{ + return __stack_depot_trie_alloc_workspace_plan(root, entries, nr_entries, + pool_prealloc, side_prealloc, + workspace); +} + +static int workspace_insert(struct stack_depot_trie_root *root, + struct stack_depot_trie_alloc_workspace *workspace, + const unsigned long *entries, unsigned int nr_entries, + const void **tail, u32 *leaf_id) +{ + return __stack_depot_trie_alloc_txn_insert(root, &workspace->req, entries, + nr_entries, workspace->scratch, + ARRAY_SIZE(workspace->scratch), tail, + leaf_id); +} + +static void stackdepot_trie_alloc_workspace_plan(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_root root = {}; + void *pool_prealloc = (void *)0x1111UL; + void *side_prealloc = (void *)0x2222UL; + const void *tail = NULL; + u32 leaf_id = 0; + int ret; + + workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workspace); + + ret = workspace_plan(&root, entries, ARRAY_SIZE(entries), &pool_prealloc, + &side_prealloc, workspace); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, workspace->req.txn, &workspace->txn); + KUNIT_EXPECT_PTR_EQ(test, workspace->req.node_slots, + &workspace->node_slots[0]); + KUNIT_EXPECT_PTR_EQ(test, workspace->req.child_slots, + &workspace->child_slots[0]); + KUNIT_EXPECT_PTR_EQ(test, workspace->req.storage, &workspace->storage); + KUNIT_EXPECT_PTR_EQ(test, workspace->req.pool_prealloc, &pool_prealloc); + KUNIT_EXPECT_PTR_EQ(test, workspace->req.side_prealloc, &side_prealloc); + KUNIT_EXPECT_NE(test, workspace->req.storage_size, 0UL); + KUNIT_EXPECT_EQ(test, workspace->req.nr_node_slots, 1U); + KUNIT_EXPECT_EQ(test, workspace->req.nr_child_slots, 0U); + + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + if (__stack_depot_trie_side_table_prealloc_needed()) { + side_prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, side_prealloc); + } + ret = workspace_plan(&root, entries, ARRAY_SIZE(entries), NULL, + &side_prealloc, workspace); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = workspace_insert(&root, workspace, entries, ARRAY_SIZE(entries), &tail, + &leaf_id); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, leaf_id, 1U); + KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), + tail); +} + static void stackdepot_trie_alloc_txn_plan(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -5140,6 +5207,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_alloc_txn_reserve_id_failure), KUNIT_CASE(stackdepot_trie_alloc_txn_commit), KUNIT_CASE(stackdepot_trie_alloc_txn_rollback), + KUNIT_CASE(stackdepot_trie_alloc_workspace_plan), KUNIT_CASE(stackdepot_trie_alloc_txn_plan), KUNIT_CASE(stackdepot_trie_alloc_txn_insert), KUNIT_CASE(stackdepot_trie_alloc_txn_insert_stale_plan), From f93eb03dc914b83b003288f0d476a84d2cc71601 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 9 Jun 2026 12:08:05 +0100 Subject: [PATCH 062/129] KRN-1117: Add stackdepot trie workspace insert Add a private workspace insert helper that plans and inserts using the writer-owned trie allocation workspace. This keeps the future save-miss path from placing CONFIG_STACKDEPOT_MAX_FRAMES-sized planner arrays on the kernel stack while preserving explicit preallocation ownership. Cover heap-backed workspace insertion in KUnit without routing the public save path to trie storage yet. Also make side-table entry clearing use release-store ordering to match lockless acquire-load lookups. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 45 ++++++++++++++++++++----- lib/stackdepot_internal.h | 14 +++++--- lib/tests/stackdepot_kunit.c | 65 ++++++++++++++++++++++++++++-------- 3 files changed, 98 insertions(+), 26 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 8bd3726a0a9b5..51b09745a40d6 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -271,7 +271,8 @@ trie_side_table_store_entry(const void **chunk, unsigned int slot, const void *e static void trie_side_table_clear_entry(const void **chunk, unsigned int slot) { - WRITE_ONCE(chunk[slot], NULL); + /* Pairs with trie_side_table_load_entry(). */ + smp_store_release(&chunk[slot], NULL); } int __stack_depot_trie_side_table_init(gfp_t gfp_flags) @@ -842,13 +843,11 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, return 0; } -int -__stack_depot_trie_alloc_workspace_plan(const struct stack_depot_trie_root *root, - const unsigned long *entries, - unsigned int nr_entries, - void **pool_prealloc, - void **side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace) +static int +trie_ws_plan(const struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, void **side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace) { if (!workspace) return -EINVAL; @@ -861,6 +860,36 @@ __stack_depot_trie_alloc_workspace_plan(const struct stack_depot_trie_root *root side_prealloc, &workspace->req); } +int __stack_depot_trie_workspace_plan(const struct stack_depot_trie_root *root, + const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + void **side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace) +{ + return trie_ws_plan(root, entries, nr_entries, pool_prealloc, side_prealloc, + workspace); +} + +int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, + const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + void **side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace, + const void **tail, u32 *leaf_id) +{ + void **pool = pool_prealloc; + void **side = side_prealloc; + int ret; + + ret = trie_ws_plan(root, entries, nr_entries, pool, side, workspace); + if (ret) + return ret; + + return __stack_depot_trie_alloc_txn_insert(root, &workspace->req, entries, + nr_entries, workspace->scratch, ARRAY_SIZE(workspace->scratch), + tail, leaf_id); +} + u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) { u32 leaf_id; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index f39c323bcea62..263802a44f83a 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -172,13 +172,17 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, void **storage, void **pool_prealloc, void **side_prealloc, struct stack_depot_trie_alloc_request *req); -int -__stack_depot_trie_alloc_workspace_plan(const struct stack_depot_trie_root *root, +int __stack_depot_trie_workspace_plan(const struct stack_depot_trie_root *root, + const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + void **side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace); +int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, const unsigned long *entries, - unsigned int nr_entries, - void **pool_prealloc, + unsigned int nr_entries, void **pool_prealloc, void **side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace); + struct stack_depot_trie_alloc_workspace *workspace, + const void **tail, u32 *leaf_id); int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn); int diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 390331695f3f3..474ea73bacfc0 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1408,20 +1408,19 @@ static int workspace_plan(struct stack_depot_trie_root *root, void **pool_prealloc, void **side_prealloc, struct stack_depot_trie_alloc_workspace *workspace) { - return __stack_depot_trie_alloc_workspace_plan(root, entries, nr_entries, - pool_prealloc, side_prealloc, - workspace); + return __stack_depot_trie_workspace_plan(root, entries, nr_entries, + pool_prealloc, side_prealloc, workspace); } -static int workspace_insert(struct stack_depot_trie_root *root, - struct stack_depot_trie_alloc_workspace *workspace, - const unsigned long *entries, unsigned int nr_entries, - const void **tail, u32 *leaf_id) +static int ws_insert_prealloc(struct stack_depot_trie_root *root, + struct stack_depot_trie_alloc_workspace *workspace, + const unsigned long *entries, unsigned int nr_entries, + void **side_prealloc, const void **tail, u32 *leaf_id) { - return __stack_depot_trie_alloc_txn_insert(root, &workspace->req, entries, - nr_entries, workspace->scratch, - ARRAY_SIZE(workspace->scratch), tail, - leaf_id); + void **side = side_prealloc; + + return __stack_depot_trie_workspace_insert(root, entries, nr_entries, NULL, + side, workspace, tail, leaf_id); } static void stackdepot_trie_alloc_workspace_plan(struct kunit *test) @@ -1462,14 +1461,53 @@ static void stackdepot_trie_alloc_workspace_plan(struct kunit *test) ret = workspace_plan(&root, entries, ARRAY_SIZE(entries), NULL, &side_prealloc, workspace); KUNIT_ASSERT_EQ(test, ret, 0); - ret = workspace_insert(&root, workspace, entries, ARRAY_SIZE(entries), &tail, - &leaf_id); + ret = ws_insert_prealloc(&root, workspace, entries, ARRAY_SIZE(entries), + &side_prealloc, &tail, &leaf_id); KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_EXPECT_EQ(test, leaf_id, 1U); KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), tail); } +static void stackdepot_trie_alloc_workspace_insert(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_root root = {}; + unsigned long out[ARRAY_SIZE(entries)] = {}; + unsigned long scratch[ARRAY_SIZE(entries)]; + void *side_prealloc = NULL; + const void *tail = NULL; + unsigned int fetched; + u32 leaf_id = 0; + int ret; + + workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workspace); + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + if (__stack_depot_trie_side_table_prealloc_needed()) { + side_prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, side_prealloc); + } + + ret = ws_insert_prealloc(&root, workspace, entries, ARRAY_SIZE(entries), + &side_prealloc, &tail, &leaf_id); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, leaf_id, 1U); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(leaf_id), + tail); + KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), + tail); + fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); + + ret = ws_insert_prealloc(NULL, workspace, entries, ARRAY_SIZE(entries), + &side_prealloc, &tail, &leaf_id); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + static void stackdepot_trie_alloc_txn_plan(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -5208,6 +5246,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_alloc_txn_commit), KUNIT_CASE(stackdepot_trie_alloc_txn_rollback), KUNIT_CASE(stackdepot_trie_alloc_workspace_plan), + KUNIT_CASE(stackdepot_trie_alloc_workspace_insert), KUNIT_CASE(stackdepot_trie_alloc_txn_plan), KUNIT_CASE(stackdepot_trie_alloc_txn_insert), KUNIT_CASE(stackdepot_trie_alloc_txn_insert_stale_plan), From 237bf201b68b46daa6d2d33b5d9f8cf0ae347c65 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 9 Jun 2026 12:50:42 +0100 Subject: [PATCH 063/129] KRN-1117: Add stackdepot trie allocation prealloc Add a private helper that prepares optional pool storage and required side-table chunk reserves before trie allocation transactions consume them. This mirrors the userspace save-miss preallocation step while keeping ownership of caller-provided reserves explicit. Return ENOSPC when a side-table chunk is still required but could not be preallocated, so later insert paths can fail without publishing partial state. Cover alloc/noalloc behaviour in KUnit and use WRITE_ONCE for side-table metadata reset stores that pair with lockless readers. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 36 +++++++++++++++++++++------ lib/stackdepot_internal.h | 4 +++ lib/tests/stackdepot_kunit.c | 48 ++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 51b09745a40d6..1c8aacc182cc7 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -294,9 +294,9 @@ int __stack_depot_trie_side_table_init(gfp_t gfp_flags) if (!trie_side_table_chunks) return -ENOMEM; - trie_side_table_high_water = 0; - trie_side_table_nr_chunks = 0; - trie_side_table_next_id = 0; + WRITE_ONCE(trie_side_table_high_water, 0); + WRITE_ONCE(trie_side_table_nr_chunks, 0); + WRITE_ONCE(trie_side_table_next_id, 0); WRITE_ONCE(trie_side_table_initialized, true); return 0; } @@ -312,10 +312,10 @@ void __stack_depot_trie_side_table_destroy(void) kfree(trie_side_table_chunks[i]); kvfree(trie_side_table_chunks); trie_side_table_chunks = NULL; - trie_side_table_high_water = 0; - trie_side_table_nr_chunks = 0; - trie_side_table_top_size = 0; - trie_side_table_next_id = 0; + WRITE_ONCE(trie_side_table_high_water, 0); + WRITE_ONCE(trie_side_table_nr_chunks, 0); + WRITE_ONCE(trie_side_table_top_size, 0); + WRITE_ONCE(trie_side_table_next_id, 0); WRITE_ONCE(trie_side_table_initialized, false); } @@ -558,6 +558,28 @@ void __stack_depot_trie_pool_free_prealloc(void *prealloc) free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); } +int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, + depot_flags_t depot_flags, + void **pool_prealloc, + void **side_prealloc) +{ + bool can_alloc; + + if (!pool_prealloc || !side_prealloc || *pool_prealloc || *side_prealloc) + return -EINVAL; + + can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && + gfpflags_allow_spinning(alloc_flags); + if (can_alloc && !READ_ONCE(new_pool)) + *pool_prealloc = __stack_depot_trie_pool_prealloc(alloc_flags); + if (can_alloc && __stack_depot_trie_side_table_prealloc_needed()) + *side_prealloc = __stack_depot_trie_side_table_prealloc(alloc_flags); + + if (__stack_depot_trie_side_table_prealloc_needed() && !*side_prealloc) + return -ENOSPC; + return 0; +} + void * __stack_depot_trie_pool_carve_current(size_t size, struct stack_depot_trie_pool_mark *mark) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 263802a44f83a..0eba14615a684 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -148,6 +148,10 @@ size_t __stack_depot_trie_side_table_bytes(void); size_t __stack_depot_trie_pool_alloc_size(size_t size); void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags); void __stack_depot_trie_pool_free_prealloc(void *prealloc); +int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, + depot_flags_t depot_flags, + void **pool_prealloc, + void **side_prealloc); /* * Best-effort current-pool helpers. They never allocate or roll over to a new * pool, and they use trylock so constrained contexts fail instead of blocking. diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 474ea73bacfc0..6fa8011677a27 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -946,6 +946,53 @@ static void stackdepot_trie_pool_prealloc(struct kunit *test) __stack_depot_trie_pool_free_prealloc(NULL); } +static int alloc_prealloc_flags(gfp_t gfp_flags, depot_flags_t depot_flags, + void **pool_prealloc, void **side_prealloc) +{ + return __stack_depot_trie_alloc_prealloc(gfp_flags, depot_flags, + pool_prealloc, side_prealloc); +} + +static int alloc_prealloc(gfp_t gfp_flags, void **pool_prealloc, + void **side_prealloc) +{ + return alloc_prealloc_flags(gfp_flags, STACK_DEPOT_FLAG_CAN_ALLOC, + pool_prealloc, side_prealloc); +} + +static void stackdepot_trie_alloc_prealloc(struct kunit *test) +{ + void *pool_prealloc = NULL; + void *side_prealloc = NULL; + u32 id; + int ret; + + stackdepot_trie_side_table_init_or_skip(test); + ret = alloc_prealloc_flags(GFP_NOWAIT, 0, &pool_prealloc, &side_prealloc); + KUNIT_EXPECT_EQ(test, ret, -ENOSPC); + KUNIT_EXPECT_NULL(test, pool_prealloc); + KUNIT_EXPECT_NULL(test, side_prealloc); + + ret = alloc_prealloc(GFP_KERNEL, &pool_prealloc, &side_prealloc); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_NOT_NULL(test, side_prealloc); + __stack_depot_trie_pool_free_prealloc(pool_prealloc); + __stack_depot_trie_side_table_free_prealloc(side_prealloc); + pool_prealloc = NULL; + side_prealloc = NULL; + + id = stackdepot_trie_side_table_alloc(test); + KUNIT_ASSERT_EQ(test, id, 1U); + ret = alloc_prealloc(GFP_NOWAIT, &pool_prealloc, &side_prealloc); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_NULL(test, pool_prealloc); + KUNIT_EXPECT_NULL(test, side_prealloc); + + pool_prealloc = (void *)0x1111UL; + ret = alloc_prealloc(GFP_KERNEL, &pool_prealloc, &side_prealloc); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + static void stackdepot_trie_pool_seed_current_pool(struct kunit *test) { unsigned long entries[] = { 0x1234567800990000UL }; @@ -5233,6 +5280,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_side_prepare_rejects_null_leaf), KUNIT_CASE(stackdepot_trie_pool_alloc_size), KUNIT_CASE(stackdepot_trie_pool_prealloc), + KUNIT_CASE(stackdepot_trie_alloc_prealloc), KUNIT_CASE(stackdepot_trie_pool_carve_current), KUNIT_CASE(stackdepot_trie_pool_rollback_requires_lifo), KUNIT_CASE(stackdepot_trie_pool_carve_current_rejects_bad_inputs), From 9908ea664843d9f882bbf0bd4c5327d7e82eb3c7 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 9 Jun 2026 14:03:24 +0100 Subject: [PATCH 064/129] KRN-1117: Add stackdepot trie save miss Add a private trie save-miss helper that combines preallocation, workspace planning, transaction-backed insertion, and trie handle encoding. The helper rejects refcounted and overlong saves so the future public route can keep those records hash-backed until trie eviction and original-depth identity are designed. Keep the helper private and leave the public save path unchanged. Cover successful saves, GET rejection, bad input, and no-allocation failure in KUnit, and tighten side-table/init cleanup ordering found during review. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 74 +++++++++++++++++++++++++++++++++++- lib/stackdepot_internal.h | 5 +++ lib/tests/stackdepot_kunit.c | 68 +++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 2 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 1c8aacc182cc7..292b3549ff8ab 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -563,6 +563,7 @@ int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, void **pool_prealloc, void **side_prealloc) { + bool needs_side_prealloc; bool can_alloc; if (!pool_prealloc || !side_prealloc || *pool_prealloc || *side_prealloc) @@ -570,12 +571,13 @@ int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && gfpflags_allow_spinning(alloc_flags); + needs_side_prealloc = __stack_depot_trie_side_table_prealloc_needed(); if (can_alloc && !READ_ONCE(new_pool)) *pool_prealloc = __stack_depot_trie_pool_prealloc(alloc_flags); - if (can_alloc && __stack_depot_trie_side_table_prealloc_needed()) + if (can_alloc && needs_side_prealloc) *side_prealloc = __stack_depot_trie_side_table_prealloc(alloc_flags); - if (__stack_depot_trie_side_table_prealloc_needed() && !*side_prealloc) + if (needs_side_prealloc && !*side_prealloc) return -ENOSPC; return 0; } @@ -912,6 +914,72 @@ int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, tail, leaf_id); } +static int +trie_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, + void **pool_prealloc, void **side_prealloc) +{ + return __stack_depot_trie_alloc_prealloc(alloc_flags, depot_flags, + pool_prealloc, side_prealloc); +} + +static int trie_ws_insert(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, void **side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace, + const void **tail, u32 *leaf_id) +{ + return __stack_depot_trie_workspace_insert(root, entries, nr_entries, + pool_prealloc, side_prealloc, + workspace, tail, leaf_id); +} + +static depot_stack_handle_t +trie_save_miss(struct stack_depot_trie_root *root, const unsigned long *entries, + unsigned int nr_entries, gfp_t alloc_flags, + depot_flags_t depot_flags, + struct stack_depot_trie_alloc_workspace *workspace) +{ + depot_stack_handle_t handle = 0; + void *pool_prealloc = NULL; + void *side_prealloc = NULL; + const void *tail; + u32 leaf_id; + int ret; + + if (!root || !entries || !nr_entries || !workspace) + return 0; + if (depot_flags & STACK_DEPOT_FLAG_GET) + return 0; + if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) + return 0; + + ret = trie_prealloc(alloc_flags, depot_flags, &pool_prealloc, + &side_prealloc); + if (ret) + goto out; + + ret = trie_ws_insert(root, entries, nr_entries, &pool_prealloc, + &side_prealloc, workspace, &tail, &leaf_id); + if (ret) + goto out; + + handle = __stack_depot_trie_handle(leaf_id); +out: + __stack_depot_trie_pool_free_prealloc(pool_prealloc); + __stack_depot_trie_side_table_free_prealloc(side_prealloc); + return handle; +} + +depot_stack_handle_t +__stack_depot_trie_save_miss(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags, depot_flags_t depot_flags, + struct stack_depot_trie_alloc_workspace *workspace) +{ + return trie_save_miss(root, entries, nr_entries, alloc_flags, depot_flags, + workspace); +} + u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) { u32 leaf_id; @@ -1232,6 +1300,8 @@ int stack_depot_init(void) if (!stack_pools) { pr_err("stack pools allocation failed, disabling\n"); kvfree(stack_table); + stack_table = NULL; + stack_hash_mask = 0; stack_depot_disabled = true; ret = -ENOMEM; } diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 0eba14615a684..c8fc999c321c8 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -187,6 +187,11 @@ int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, void **side_prealloc, struct stack_depot_trie_alloc_workspace *workspace, const void **tail, u32 *leaf_id); +depot_stack_handle_t +__stack_depot_trie_save_miss(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags, depot_flags_t depot_flags, + struct stack_depot_trie_alloc_workspace *workspace); int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn); int diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 6fa8011677a27..139e6642543f9 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1470,6 +1470,16 @@ static int ws_insert_prealloc(struct stack_depot_trie_root *root, side, workspace, tail, leaf_id); } +static depot_stack_handle_t save_miss(struct stack_depot_trie_root *root, + const unsigned long *entries, + unsigned int nr_entries, gfp_t gfp_flags, + depot_flags_t depot_flags, + struct stack_depot_trie_alloc_workspace *workspace) +{ + return __stack_depot_trie_save_miss(root, entries, nr_entries, gfp_flags, + depot_flags, workspace); +} + static void stackdepot_trie_alloc_workspace_plan(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; @@ -1555,6 +1565,62 @@ static void stackdepot_trie_alloc_workspace_insert(struct kunit *test) KUNIT_EXPECT_EQ(test, ret, -EINVAL); } +static void stackdepot_trie_save_miss(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_root root = {}; + unsigned long out[ARRAY_SIZE(entries)] = {}; + unsigned long scratch[ARRAY_SIZE(entries)]; + depot_stack_handle_t handle; + const void *tail; + unsigned int fetched; + u32 leaf_id; + + workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workspace); + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + + handle = save_miss(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC, workspace); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + leaf_id = __stack_depot_trie_leaf_id(handle); + KUNIT_EXPECT_EQ(test, leaf_id, 1U); + tail = __stack_depot_trie_side_table_lookup(leaf_id); + KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), + tail); + fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); + + handle = save_miss(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_GET, workspace); + KUNIT_EXPECT_EQ(test, handle, (depot_stack_handle_t)0); + handle = save_miss(NULL, entries, ARRAY_SIZE(entries), GFP_KERNEL, 0, + workspace); + KUNIT_EXPECT_EQ(test, handle, (depot_stack_handle_t)0); +} + +static void stackdepot_trie_save_miss_noalloc(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL }; + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_root root = {}; + depot_stack_handle_t handle; + + workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workspace); + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + + handle = save_miss(&root, entries, ARRAY_SIZE(entries), GFP_NOWAIT, 0, + workspace); + KUNIT_EXPECT_EQ(test, handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_NULL(test, find_leaf(&root, entries, ARRAY_SIZE(entries))); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); +} + static void stackdepot_trie_alloc_txn_plan(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -5295,6 +5361,8 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_alloc_txn_rollback), KUNIT_CASE(stackdepot_trie_alloc_workspace_plan), KUNIT_CASE(stackdepot_trie_alloc_workspace_insert), + KUNIT_CASE(stackdepot_trie_save_miss), + KUNIT_CASE(stackdepot_trie_save_miss_noalloc), KUNIT_CASE(stackdepot_trie_alloc_txn_plan), KUNIT_CASE(stackdepot_trie_alloc_txn_insert), KUNIT_CASE(stackdepot_trie_alloc_txn_insert_stale_plan), From f693df3b4fcc2020d5a375b4ae4610a356e457b9 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 9 Jun 2026 14:30:25 +0100 Subject: [PATCH 065/129] KRN-1117: Add stackdepot trie save helper Add a private trie save helper that performs the persistent-stack lookup path before falling back to the private save-miss helper. Existing trie leaves are looked up under sched-RCU and returned as trie handles without allocating new storage. Keep the helper private and leave the public save path unchanged. Reject refcounted and overlong saves so those records remain hash-backed when public routing is added, and cover hit reuse and rejection paths in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 28 ++++++++++++++++++++++++ lib/stackdepot_internal.h | 5 +++++ lib/tests/stackdepot_kunit.c | 41 ++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 292b3549ff8ab..a00b95bcdee50 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -980,6 +980,34 @@ __stack_depot_trie_save_miss(struct stack_depot_trie_root *root, workspace); } +depot_stack_handle_t +__stack_depot_trie_save(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags, depot_flags_t depot_flags, + struct stack_depot_trie_alloc_workspace *workspace) +{ + depot_stack_handle_t handle = 0; + const struct stack_depot_trie_node *leaf; + + if (!root || !entries || !nr_entries || !workspace) + return 0; + if (depot_flags & STACK_DEPOT_FLAG_GET) + return 0; + if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) + return 0; + + rcu_read_lock_sched_notrace(); + leaf = __stack_depot_trie_find_leaf(root, entries, nr_entries); + if (leaf) + handle = __stack_depot_trie_handle(leaf->leaf_id); + rcu_read_unlock_sched_notrace(); + + if (handle) + return handle; + return trie_save_miss(root, entries, nr_entries, alloc_flags, depot_flags, + workspace); +} + u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) { u32 leaf_id; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index c8fc999c321c8..d33b4c4ddd0cd 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -192,6 +192,11 @@ __stack_depot_trie_save_miss(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags, depot_flags_t depot_flags, struct stack_depot_trie_alloc_workspace *workspace); +depot_stack_handle_t +__stack_depot_trie_save(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags, depot_flags_t depot_flags, + struct stack_depot_trie_alloc_workspace *workspace); int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn); int diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 139e6642543f9..2258a34b2c5df 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1480,6 +1480,15 @@ static depot_stack_handle_t save_miss(struct stack_depot_trie_root *root, depot_flags, workspace); } +static depot_stack_handle_t tsave(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + gfp_t gfp_flags, depot_flags_t depot_flags, + struct stack_depot_trie_alloc_workspace *workspace) +{ + return __stack_depot_trie_save(root, entries, nr_entries, gfp_flags, + depot_flags, workspace); +} + static void stackdepot_trie_alloc_workspace_plan(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; @@ -1621,6 +1630,37 @@ static void stackdepot_trie_save_miss_noalloc(struct kunit *test) KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); } +static void stackdepot_trie_save(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_root root = {}; + depot_stack_handle_t first; + depot_stack_handle_t invalid; + depot_stack_handle_t second; + + workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workspace); + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + + first = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC, workspace); + KUNIT_ASSERT_NE(test, first, (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); + + second = tsave(&root, entries, ARRAY_SIZE(entries), GFP_NOWAIT, 0, + workspace); + KUNIT_EXPECT_EQ(test, second, first); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); + + invalid = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_GET, workspace); + KUNIT_EXPECT_EQ(test, invalid, (depot_stack_handle_t)0); + invalid = tsave(NULL, entries, ARRAY_SIZE(entries), GFP_KERNEL, 0, workspace); + KUNIT_EXPECT_EQ(test, invalid, (depot_stack_handle_t)0); +} + static void stackdepot_trie_alloc_txn_plan(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -5363,6 +5403,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_alloc_workspace_insert), KUNIT_CASE(stackdepot_trie_save_miss), KUNIT_CASE(stackdepot_trie_save_miss_noalloc), + KUNIT_CASE(stackdepot_trie_save), KUNIT_CASE(stackdepot_trie_alloc_txn_plan), KUNIT_CASE(stackdepot_trie_alloc_txn_insert), KUNIT_CASE(stackdepot_trie_alloc_txn_insert_stale_plan), From a1afe168d2f2548b8bf0b857c977aaf87a906eca Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 9 Jun 2026 16:13:35 +0100 Subject: [PATCH 066/129] KRN-1117: Add stackdepot trie handle fetch helper Add a private helper that decodes trie handles through the leaf side table and materializes their frames into caller-owned storage. This gives KUnit coverage for trie handle fetch semantics before public stackdepot routing starts returning trie handles. Keep materialized frame buffers visible to KMSAN, validate reparented trie ancestors through acquire-loaded parent pointers, and shrink frame-run metadata so trie node headers stay compact. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 51 ++++++++++++++++++++++++++++++++--- lib/stackdepot_internal.h | 10 ++++++- lib/tests/stackdepot_kunit.c | 52 ++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 4 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index a00b95bcdee50..d22293b94be7c 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -3518,9 +3518,13 @@ static int trie_plan_append_chain(unsigned int base_stack_len, static bool trie_node_depth_invalid(const struct stack_depot_trie_node *parent, const struct stack_depot_trie_node *node) { + const struct stack_depot_trie_node *node_parent; u32 base = parent ? parent->stack_len : 0; - if (!node || node->parent != parent || !node->stack_len) + if (!node || !node->stack_len) + return true; + node_parent = trie_load_parent(node); + if (node_parent != parent) return true; if (parent && !parent->stack_len) return true; @@ -3537,11 +3541,16 @@ static bool trie_node_chain_depth_invalid(const struct stack_depot_trie_node *no { unsigned int depth = 0; - for (; node; node = node->parent, depth++) { + while (node) { + const struct stack_depot_trie_node *parent; + if (depth >= CONFIG_STACKDEPOT_MAX_FRAMES) return true; - if (trie_node_depth_invalid(node->parent, node)) + parent = trie_load_parent(node); + if (trie_node_depth_invalid(parent, node)) return true; + node = parent; + depth++; } return false; @@ -3801,9 +3810,44 @@ __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, return 0; memcpy(entries, scratch, total * sizeof(*entries)); + kmsan_unpoison_memory(entries, total * sizeof(*entries)); return total; } +static unsigned int trie_fetch_leaf(const void *leaf, unsigned long *entries, + unsigned int max_entries, unsigned long *scratch, + unsigned int nr_scratch) +{ + return __stack_depot_trie_fetch_into(leaf, entries, max_entries, scratch, + nr_scratch); +} + +unsigned int +__stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries, + unsigned long *scratch, + unsigned int nr_scratch) +{ + const void *leaf; + u32 leaf_id; + unsigned int nr_entries; + + if (!handle || !entries || !scratch || !max_entries || !nr_scratch) + return 0; + + leaf_id = __stack_depot_trie_leaf_id(handle); + if (!leaf_id) + return 0; + + rcu_read_lock_sched_notrace(); + leaf = __stack_depot_trie_side_table_lookup(leaf_id); + nr_entries = trie_fetch_leaf(leaf, entries, max_entries, scratch, nr_scratch); + rcu_read_unlock_sched_notrace(); + + return nr_entries; +} + size_t __stack_depot_trie_child_array_size(unsigned int nr_children) { size_t size; @@ -4511,6 +4555,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, * keep the handle valid while this helper copies from it. */ memcpy(entries, stack_entries, nr_entries * sizeof(*entries)); + kmsan_unpoison_memory(entries, nr_entries * sizeof(*entries)); copied = nr_entries; out: diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index d33b4c4ddd0cd..f3d88cde01501 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -2,6 +2,7 @@ #ifndef _STACKDEPOT_INTERNAL_H #define _STACKDEPOT_INTERNAL_H +#include #include #include @@ -19,12 +20,14 @@ enum stack_depot_trie_lookup_status { }; struct stack_depot_frame_run { - size_t bytes; unsigned int nr_entries; + u16 bytes; u8 mode; u8 prefix_id; }; +static_assert(CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(unsigned long) <= U16_MAX); + struct stack_depot_trie_node_slot { void *node; size_t size; @@ -299,6 +302,11 @@ unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned int max_entries, unsigned long *scratch, unsigned int nr_scratch); +unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries, + unsigned long *scratch, + unsigned int nr_scratch); size_t __stack_depot_trie_child_array_size(unsigned int nr_children); int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, const void * const *children, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 2258a34b2c5df..9739b4d6b73ab 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1489,6 +1489,14 @@ static depot_stack_handle_t tsave(struct stack_depot_trie_root *root, depot_flags, workspace); } +static unsigned int tfetch_handle(depot_stack_handle_t handle, + unsigned long *entries, unsigned int max_entries, + unsigned long *scratch, unsigned int nr_scratch) +{ + return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries, + scratch, nr_scratch); +} + static void stackdepot_trie_alloc_workspace_plan(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; @@ -1661,6 +1669,49 @@ static void stackdepot_trie_save(struct kunit *test) KUNIT_EXPECT_EQ(test, invalid, (depot_stack_handle_t)0); } +static void stackdepot_trie_fetch_handle_into(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_root root = {}; + unsigned long scratch[ARRAY_SIZE(entries)]; + unsigned long small[1] = { 0xdeadUL }; + unsigned long out[ARRAY_SIZE(entries)] = {}; + depot_stack_handle_t hash_handle; + depot_stack_handle_t handle; + unsigned int invalid; + unsigned int fetched; + + workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workspace); + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + + handle = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC, workspace); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + fetched = tfetch_handle(handle, out, ARRAY_SIZE(out), scratch, + ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); + + fetched = tfetch_handle(handle, small, ARRAY_SIZE(small), scratch, + ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, fetched, 0U); + KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); + invalid = tfetch_handle(0, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, invalid, 0U); + invalid = tfetch_handle(handle, NULL, 0, scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, invalid, 0U); + invalid = tfetch_handle(handle, out, ARRAY_SIZE(out), NULL, 0); + KUNIT_EXPECT_EQ(test, invalid, 0U); + + hash_handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); + invalid = tfetch_handle(hash_handle, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + KUNIT_EXPECT_EQ(test, invalid, 0U); +} + static void stackdepot_trie_alloc_txn_plan(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -5404,6 +5455,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_save_miss), KUNIT_CASE(stackdepot_trie_save_miss_noalloc), KUNIT_CASE(stackdepot_trie_save), + KUNIT_CASE(stackdepot_trie_fetch_handle_into), KUNIT_CASE(stackdepot_trie_alloc_txn_plan), KUNIT_CASE(stackdepot_trie_alloc_txn_insert), KUNIT_CASE(stackdepot_trie_alloc_txn_insert_stale_plan), From 7e823e0c395f3638048a554637917c5cc5c016ef Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 9 Jun 2026 16:42:36 +0100 Subject: [PATCH 067/129] KRN-1117: Re-add stackdepot trie feature flag Re-add a default-off static-key backed module parameter for the future trie storage path. The flag exposes boot-time control through stackdepot.trie_enabled and runtime control through the built-in module parameter sysfs file, but does not route public saves or fetches yet. Keep the flag covered by KUnit so default-off, enable, idempotent enable, and disable behavior stay explicit before public routing is wired in. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 46 ++++++++++++++++++++++++++++++++++++ lib/stackdepot_internal.h | 3 +++ lib/tests/stackdepot_kunit.c | 30 +++++++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index d22293b94be7c..bb43d70e63905 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -18,10 +18,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -52,6 +54,50 @@ static unsigned int stack_max_pools __read_mostly = static bool stack_depot_disabled; static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT); static bool __stack_depot_early_init_passed __initdata; +static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); +static bool stack_depot_trie_enabled_param; + +bool __stack_depot_trie_enabled(void) +{ + return static_branch_unlikely(&stack_depot_trie_enabled); +} + +void __stack_depot_trie_set_enabled(bool enabled) +{ + if (READ_ONCE(stack_depot_trie_enabled_param) == enabled) + return; + + WRITE_ONCE(stack_depot_trie_enabled_param, enabled); + if (enabled) + static_branch_enable(&stack_depot_trie_enabled); + else + static_branch_disable(&stack_depot_trie_enabled); +} + +static int stack_depot_trie_enabled_param_set(const char *val, + const struct kernel_param *kp) +{ + struct kernel_param tmp = *kp; + bool enabled; + int ret; + + tmp.arg = &enabled; + ret = param_set_bool(val, &tmp); + if (ret) + return ret; + + __stack_depot_trie_set_enabled(enabled); + return 0; +} + +static const struct kernel_param_ops stack_depot_trie_enabled_param_ops = { + .flags = KERNEL_PARAM_OPS_FL_NOARG, + .set = stack_depot_trie_enabled_param_set, + .get = param_get_bool, +}; +module_param_cb(trie_enabled, &stack_depot_trie_enabled_param_ops, + &stack_depot_trie_enabled_param, 0644); +MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage"); /* Use one hash table bucket per 16 KB of memory. */ #define STACK_HASH_TABLE_SCALE 14 diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index f3d88cde01501..a188d13d5540f 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -28,6 +28,9 @@ struct stack_depot_frame_run { static_assert(CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(unsigned long) <= U16_MAX); +bool __stack_depot_trie_enabled(void); +void __stack_depot_trie_set_enabled(bool enabled); + struct stack_depot_trie_node_slot { void *node; size_t size; diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 9739b4d6b73ab..55acea73495f8 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -574,6 +574,35 @@ static void stackdepot_trie_handle_namespace(struct kunit *test) (depot_stack_handle_t)0); } +static void stackdepot_trie_disable_action(void *data) +{ + __stack_depot_trie_set_enabled(false); +} + +static void stackdepot_trie_add_disable_action(struct kunit *test) +{ + int ret; + + ret = kunit_add_action_or_reset(test, stackdepot_trie_disable_action, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); +} + +static void stackdepot_trie_feature_flag(struct kunit *test) +{ + stackdepot_trie_add_disable_action(test); + + KUNIT_EXPECT_FALSE(test, __stack_depot_trie_enabled()); + + __stack_depot_trie_set_enabled(true); + KUNIT_EXPECT_TRUE(test, __stack_depot_trie_enabled()); + + __stack_depot_trie_set_enabled(true); + KUNIT_EXPECT_TRUE(test, __stack_depot_trie_enabled()); + + __stack_depot_trie_set_enabled(false); + KUNIT_EXPECT_FALSE(test, __stack_depot_trie_enabled()); +} + static void stackdepot_trie_side_table_destroy_action(void *data) { __stack_depot_trie_side_table_destroy(); @@ -5422,6 +5451,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), KUNIT_CASE(stackdepot_count_helpers), KUNIT_CASE(stackdepot_trie_handle_namespace), + KUNIT_CASE(stackdepot_trie_feature_flag), KUNIT_CASE(stackdepot_trie_side_table_destroy_uninit), KUNIT_CASE(stackdepot_trie_side_table_alloc_store_lookup), KUNIT_CASE(stackdepot_trie_side_table_rejects_invalid_ids), From 76c26db38efc26609ca8d6e32d136ea4eda3f2d6 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 9 Jun 2026 17:24:16 +0100 Subject: [PATCH 068/129] KRN-1117: Materialize stackdepot trie handles directly Materialize trie-backed stacks directly into caller-owned output after a validation pass instead of decoding through a second full-size scratch buffer. This matches the userspace prototype shape more closely and avoids carrying extra stack-sized scratch storage into future public trie fetch routing. Keep the exact-or-nothing fetch contract by validating the parent chain, payloads, and output/source aliasing before writing the caller buffer. Update KUnit coverage and wrappers so the private trie fetch API no longer accepts unused scratch arguments. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 96 +++++++++++++++++---------- lib/stackdepot_internal.h | 8 +-- lib/tests/stackdepot_kunit.c | 124 +++++++++++++---------------------- 3 files changed, 111 insertions(+), 117 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index bb43d70e63905..550b94b4f7f81 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2180,13 +2180,49 @@ stack_depot_frame_run_read_compressed(const struct stack_depot_frame_run *run, return 0; } -static int frame_run_read_to_scratch(const struct stack_depot_frame_run *run, - const void *src, unsigned long *scratch, - unsigned int nr_scratch) +static int frame_run_validate_payload(const struct stack_depot_frame_run *run, + const void *src) { - /* Private staging helper: the public frame-run read API rejects aliasing. */ - return stack_depot_frame_run_read_compressed(run, src, scratch, scratch, - nr_scratch); + unsigned long frame; + unsigned int i; + + if (!src || stack_depot_frame_run_validate(run)) + return -EINVAL; + if (run->mode == STACK_DEPOT_FRAME_RAW) + return 0; + + for (i = 0; i < run->nr_entries; i++) { + u32 low; + + memcpy(&low, (const char *)src + i * sizeof(low), sizeof(low)); + if (!frame_decompress(run->prefix_id, low, &frame)) + return -EINVAL; + } + + return 0; +} + +static int frame_run_read_direct(const struct stack_depot_frame_run *run, + const void *src, unsigned long *entries) +{ + unsigned int i; + + if (!entries || frame_run_validate_payload(run, src)) + return -EINVAL; + if (run->mode == STACK_DEPOT_FRAME_RAW) { + memcpy(entries, src, run->bytes); + return 0; + } + + for (i = 0; i < run->nr_entries; i++) { + u32 low; + + memcpy(&low, (const char *)src + i * sizeof(low), sizeof(low)); + if (!frame_decompress(run->prefix_id, low, &entries[i])) + return -EINVAL; + } + + return 0; } static bool stack_depot_ranges_overlap(const void *a, size_t a_size, @@ -3815,71 +3851,65 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, - unsigned int max_entries, unsigned long *scratch, - unsigned int nr_scratch) + unsigned int max_entries) { const struct stack_depot_trie_node *node = leaf; unsigned int pos; unsigned int total; int ret; - if (!node || !entries || !scratch || !node->stack_len || !node->leaf_id) + if (!node || !entries || !node->stack_len || !node->leaf_id) return 0; total = node->stack_len; - if (max_entries < total || nr_scratch < total) - return 0; - if (stack_depot_ranges_overlap(entries, total * sizeof(*entries), scratch, - total * sizeof(*scratch))) + if (max_entries < total) return 0; + /* Validate first so failure cannot partially write the caller buffer. */ pos = total; for (node = leaf; node; node = trie_load_parent(node)) { - if (stack_depot_frame_run_validate(&node->run)) + if (frame_run_validate_payload(&node->run, node->data)) + return 0; + if (stack_depot_ranges_overlap(entries, total * sizeof(*entries), + node->data, node->run.bytes)) return 0; if (node->stack_len != pos || node->run.nr_entries > pos) return 0; pos -= node->run.nr_entries; - /* nr_scratch >= total, and pos tracks the remaining prefix length. */ - /* Decode directly into the staged output; node->data is separate. */ - if (node->run.mode == STACK_DEPOT_FRAME_COMPRESSED) - ret = frame_run_read_to_scratch(&node->run, node->data, - &scratch[pos], nr_scratch - pos); - else - ret = frame_run_read(&node->run, node->data, - node->run.bytes, &scratch[pos], - nr_scratch - pos, NULL, 0); + } + if (pos) + return 0; + + pos = total; + for (node = leaf; node; node = trie_load_parent(node)) { + pos -= node->run.nr_entries; + ret = frame_run_read_direct(&node->run, node->data, &entries[pos]); if (ret) return 0; } if (pos) return 0; - memcpy(entries, scratch, total * sizeof(*entries)); kmsan_unpoison_memory(entries, total * sizeof(*entries)); return total; } static unsigned int trie_fetch_leaf(const void *leaf, unsigned long *entries, - unsigned int max_entries, unsigned long *scratch, - unsigned int nr_scratch) + unsigned int max_entries) { - return __stack_depot_trie_fetch_into(leaf, entries, max_entries, scratch, - nr_scratch); + return __stack_depot_trie_fetch_into(leaf, entries, max_entries); } unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, unsigned long *entries, - unsigned int max_entries, - unsigned long *scratch, - unsigned int nr_scratch) + unsigned int max_entries) { const void *leaf; u32 leaf_id; unsigned int nr_entries; - if (!handle || !entries || !scratch || !max_entries || !nr_scratch) + if (!handle || !entries || !max_entries) return 0; leaf_id = __stack_depot_trie_leaf_id(handle); @@ -3888,7 +3918,7 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, rcu_read_lock_sched_notrace(); leaf = __stack_depot_trie_side_table_lookup(leaf_id); - nr_entries = trie_fetch_leaf(leaf, entries, max_entries, scratch, nr_scratch); + nr_entries = trie_fetch_leaf(leaf, entries, max_entries); rcu_read_unlock_sched_notrace(); return nr_entries; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index a188d13d5540f..e8ad28c28057a 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -302,14 +302,10 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, unsigned int *nr_used, unsigned int *nr_child_used); unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, - unsigned int max_entries, - unsigned long *scratch, - unsigned int nr_scratch); + unsigned int max_entries); unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, unsigned long *entries, - unsigned int max_entries, - unsigned long *scratch, - unsigned int nr_scratch); + unsigned int max_entries); size_t __stack_depot_trie_child_array_size(unsigned int nr_children); int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, const void * const *children, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 55acea73495f8..d697d8d60f352 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -58,11 +58,9 @@ tnode_init_slice(void *storage, size_t storage_size, const void *parent, } static unsigned int tfetch(const void *leaf, unsigned long *entries, - unsigned int max_entries, unsigned long *scratch, - unsigned int nr_scratch) + unsigned int max_entries) { - return __stack_depot_trie_fetch_into(leaf, entries, max_entries, scratch, - nr_scratch); + return __stack_depot_trie_fetch_into(leaf, entries, max_entries); } static unsigned int tmatch(const void *node, const unsigned long *entries, @@ -1519,11 +1517,9 @@ static depot_stack_handle_t tsave(struct stack_depot_trie_root *root, } static unsigned int tfetch_handle(depot_stack_handle_t handle, - unsigned long *entries, unsigned int max_entries, - unsigned long *scratch, unsigned int nr_scratch) + unsigned long *entries, unsigned int max_entries) { - return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries, - scratch, nr_scratch); + return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); } static void stackdepot_trie_alloc_workspace_plan(struct kunit *test) @@ -1578,7 +1574,6 @@ static void stackdepot_trie_alloc_workspace_insert(struct kunit *test) struct stack_depot_trie_alloc_workspace *workspace; struct stack_depot_trie_root root = {}; unsigned long out[ARRAY_SIZE(entries)] = {}; - unsigned long scratch[ARRAY_SIZE(entries)]; void *side_prealloc = NULL; const void *tail = NULL; unsigned int fetched; @@ -1602,7 +1597,7 @@ static void stackdepot_trie_alloc_workspace_insert(struct kunit *test) tail); KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), tail); - fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); @@ -1617,7 +1612,6 @@ static void stackdepot_trie_save_miss(struct kunit *test) struct stack_depot_trie_alloc_workspace *workspace; struct stack_depot_trie_root root = {}; unsigned long out[ARRAY_SIZE(entries)] = {}; - unsigned long scratch[ARRAY_SIZE(entries)]; depot_stack_handle_t handle; const void *tail; unsigned int fetched; @@ -1636,7 +1630,7 @@ static void stackdepot_trie_save_miss(struct kunit *test) tail = __stack_depot_trie_side_table_lookup(leaf_id); KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), tail); - fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); @@ -1703,7 +1697,6 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) unsigned long entries[] = { 0x1000UL, 0x2000UL }; struct stack_depot_trie_alloc_workspace *workspace; struct stack_depot_trie_root root = {}; - unsigned long scratch[ARRAY_SIZE(entries)]; unsigned long small[1] = { 0xdeadUL }; unsigned long out[ARRAY_SIZE(entries)] = {}; depot_stack_handle_t hash_handle; @@ -1719,25 +1712,24 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) handle = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, STACK_DEPOT_FLAG_CAN_ALLOC, workspace); KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - fetched = tfetch_handle(handle, out, ARRAY_SIZE(out), scratch, - ARRAY_SIZE(scratch)); + fetched = tfetch_handle(handle, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); - fetched = tfetch_handle(handle, small, ARRAY_SIZE(small), scratch, - ARRAY_SIZE(scratch)); + fetched = tfetch_handle(handle, small, ARRAY_SIZE(small)); KUNIT_EXPECT_EQ(test, fetched, 0U); KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); - invalid = tfetch_handle(0, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); - KUNIT_EXPECT_EQ(test, invalid, 0U); - invalid = tfetch_handle(handle, NULL, 0, scratch, ARRAY_SIZE(scratch)); + invalid = tfetch_handle(0, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, invalid, 0U); - invalid = tfetch_handle(handle, out, ARRAY_SIZE(out), NULL, 0); + invalid = tfetch_handle(handle, NULL, 0); KUNIT_EXPECT_EQ(test, invalid, 0U); + fetched = tfetch_handle(handle, out, ARRAY_SIZE(out)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); hash_handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); - invalid = tfetch_handle(hash_handle, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + invalid = tfetch_handle(hash_handle, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, invalid, 0U); } @@ -1786,7 +1778,6 @@ static void stackdepot_trie_alloc_txn_insert(struct kunit *test) struct stack_depot_trie_alloc_request req; struct stack_depot_trie_alloc_txn txn; struct stack_depot_trie_root root = {}; - unsigned long scratch[ARRAY_SIZE(entries)]; unsigned long out[ARRAY_SIZE(entries)] = {}; void *side_prealloc = NULL; const void *tail = NULL; @@ -1818,7 +1809,7 @@ static void stackdepot_trie_alloc_txn_insert(struct kunit *test) tail); KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), tail); - fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); @@ -2206,13 +2197,12 @@ static void stackdepot_frame_run_invalid_inputs(struct kunit *test) static void stackdepot_trie_node_raw_roundtrip(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - unsigned long scratch[ARRAY_SIZE(entries)]; unsigned long out[ARRAY_SIZE(entries)] = {}; unsigned int fetched; void *node; trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 7, &node); - fetched = tfetch(node, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(node, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); } @@ -2222,7 +2212,6 @@ static void stackdepot_trie_node_parent_chain(struct kunit *test) unsigned long root_entries[] = { 0x1000UL, 0x2000UL }; unsigned long child_entries[] = { 0x3000UL, 0x4000UL }; unsigned long expected[] = { 0x1000UL, 0x2000UL, 0x3000UL, 0x4000UL }; - unsigned long scratch[ARRAY_SIZE(expected)]; unsigned long out[ARRAY_SIZE(expected)] = {}; unsigned int fetched; void *root; @@ -2232,7 +2221,7 @@ static void stackdepot_trie_node_parent_chain(struct kunit *test) &root); trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), root, 9, &child); - fetched = tfetch(child, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(child, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } @@ -2242,7 +2231,6 @@ static void stackdepot_trie_node_slice_raw(struct kunit *test) unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; unsigned long expected[] = { 0x2000UL, 0x3000UL }; struct stack_depot_frame_run run; - unsigned long scratch[ARRAY_SIZE(expected)]; unsigned long out[ARRAY_SIZE(expected)] = {}; unsigned int fetched; void *source; @@ -2260,7 +2248,7 @@ static void stackdepot_trie_node_slice_raw(struct kunit *test) ret = tnode_init_slice(slice, size, NULL, 10, source, 1, ARRAY_SIZE(expected)); KUNIT_ASSERT_EQ(test, ret, 0); - fetched = tfetch(slice, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(slice, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); KUNIT_EXPECT_EQ(test, tmatch(slice, expected, ARRAY_SIZE(expected)), @@ -2273,7 +2261,6 @@ static void stackdepot_trie_node_slice_parent_chain(struct kunit *test) unsigned long entries[] = { 0x2000UL, 0x3000UL, 0x4000UL }; unsigned long expected[] = { 0x1000UL, 0x3000UL, 0x4000UL }; struct stack_depot_frame_run run; - unsigned long scratch[ARRAY_SIZE(expected)]; unsigned long out[ARRAY_SIZE(expected)] = {}; unsigned int fetched; void *root; @@ -2293,7 +2280,7 @@ static void stackdepot_trie_node_slice_parent_chain(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, slice); ret = tnode_init_slice(slice, size, root, 11, source, 1, 2); KUNIT_ASSERT_EQ(test, ret, 0); - fetched = tfetch(slice, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(slice, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } @@ -2314,7 +2301,6 @@ static void stackdepot_trie_node_slice_compressed(struct kunit *test) }; unsigned long expected[] = { entries[1], entries[2] }; struct stack_depot_frame_run run; - unsigned long scratch[ARRAY_SIZE(expected)]; unsigned long out[ARRAY_SIZE(expected)] = {}; unsigned int fetched; void *source; @@ -2333,7 +2319,7 @@ static void stackdepot_trie_node_slice_compressed(struct kunit *test) ret = tnode_init_slice(slice, size, NULL, 12, source, 1, ARRAY_SIZE(expected)); KUNIT_ASSERT_EQ(test, ret, 0); - fetched = tfetch(slice, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(slice, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } @@ -2450,7 +2436,6 @@ static void stackdepot_trie_append_chain_raw(struct kunit *test) unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; struct stack_depot_frame_run run; struct stack_depot_trie_node_slot node_slots[1]; - unsigned long scratch[ARRAY_SIZE(entries)]; unsigned long out[ARRAY_SIZE(entries)] = {}; const void *head = NULL; const void *tail = NULL; @@ -2472,7 +2457,7 @@ static void stackdepot_trie_append_chain_raw(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, head, node_slots[0].node); KUNIT_EXPECT_PTR_EQ(test, tail, node_slots[0].node); KUNIT_EXPECT_EQ(test, used, 1U); - fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); } @@ -2484,7 +2469,6 @@ static void stackdepot_trie_append_chain_parent(struct kunit *test) unsigned long expected[] = { 0x1000UL, 0x2000UL, 0x3000UL }; struct stack_depot_frame_run run; struct stack_depot_trie_node_slot node_slots[1]; - unsigned long scratch[ARRAY_SIZE(expected)]; unsigned long out[ARRAY_SIZE(expected)] = {}; const void *head = NULL; const void *tail = NULL; @@ -2508,7 +2492,7 @@ static void stackdepot_trie_append_chain_parent(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_EXPECT_PTR_EQ(test, head, tail); KUNIT_EXPECT_EQ(test, used, 1U); - fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } @@ -3444,7 +3428,6 @@ static void stackdepot_trie_insert_append_descends_one_level(struct kunit *test) struct stack_depot_trie_node_slot child_slot; struct stack_depot_trie_lookup lookup; struct stack_depot_trie_root root = {}; - unsigned long scratch[ARRAY_SIZE(stack_entries)]; unsigned long out[ARRAY_SIZE(stack_entries)] = {}; const void *prefix = NULL; const void *tail = NULL; @@ -3479,7 +3462,7 @@ static void stackdepot_trie_insert_append_descends_one_level(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); KUNIT_EXPECT_PTR_EQ(test, lookup.node, tail); - fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(stack_entries)); KUNIT_EXPECT_MEMEQ(test, out, stack_entries, sizeof(stack_entries)); } @@ -3499,7 +3482,6 @@ static void stackdepot_trie_insert_append_descends_multiple_levels(struct kunit struct stack_depot_trie_node_slot third_slot; struct stack_depot_trie_lookup lookup; struct stack_depot_trie_root root = {}; - unsigned long scratch[ARRAY_SIZE(full_stack)]; unsigned long out[ARRAY_SIZE(full_stack)] = {}; const void *first = NULL; const void *second = NULL; @@ -3543,7 +3525,7 @@ static void stackdepot_trie_insert_append_descends_multiple_levels(struct kunit KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); KUNIT_EXPECT_PTR_EQ(test, lookup.node, tail); - fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(full_stack)); KUNIT_EXPECT_MEMEQ(test, out, full_stack, sizeof(full_stack)); } @@ -3614,7 +3596,6 @@ static void stackdepot_trie_insert_append_promotes_internal(struct kunit *test) struct stack_depot_trie_node_slot promote_slot; struct stack_depot_trie_lookup lookup; struct stack_depot_trie_root root = {}; - unsigned long scratch[ARRAY_SIZE(entries)]; unsigned long out[ARRAY_SIZE(entries)] = {}; const void *children[1]; const void *tail = (const void *)1; @@ -3645,7 +3626,7 @@ static void stackdepot_trie_insert_append_promotes_internal(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, entries[0]), tail); KUNIT_EXPECT_PTR_EQ(test, tail, promote_slot.node); KUNIT_EXPECT_EQ(test, used, 1U); - fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); ret = lookup_step(&root, NULL, entries, ARRAY_SIZE(entries), &lookup); @@ -3665,7 +3646,6 @@ static void stackdepot_trie_insert_append_descends_to_promote(struct kunit *test struct stack_depot_trie_node_slot promote_slot; struct stack_depot_trie_lookup lookup; struct stack_depot_trie_root root = {}; - unsigned long scratch[ARRAY_SIZE(stack_entries)]; unsigned long out[ARRAY_SIZE(stack_entries)] = {}; const void *root_children[1]; const void *tail = (const void *)1; @@ -3710,7 +3690,7 @@ static void stackdepot_trie_insert_append_descends_to_promote(struct kunit *test KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); KUNIT_EXPECT_PTR_EQ(test, lookup.node, tail); KUNIT_EXPECT_EQ(test, used, 1U); - fetched = tfetch(tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(stack_entries)); KUNIT_EXPECT_MEMEQ(test, out, stack_entries, sizeof(stack_entries)); } @@ -3725,7 +3705,6 @@ static void stackdepot_trie_insert_append_promotes_with_children(struct kunit *t struct stack_depot_trie_lookup lookup; struct stack_depot_trie_root root = {}; unsigned long expected[] = { 0x1000UL, 0x2000UL }; - unsigned long scratch[ARRAY_SIZE(expected)]; unsigned long out[ARRAY_SIZE(expected)] = {}; const void *root_children[1]; const void *tail = NULL; @@ -3770,7 +3749,7 @@ static void stackdepot_trie_insert_append_promotes_with_children(struct kunit *t KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); KUNIT_EXPECT_PTR_EQ(test, lookup.node, child); KUNIT_EXPECT_EQ(test, used, 1U); - fetched = tfetch(child, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(child, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } @@ -3795,7 +3774,6 @@ static void stackdepot_trie_insert_append_splits_frame_runs(struct kunit *test) struct stack_depot_trie_child_array_slot child_slots[2]; struct stack_depot_trie_child_array_slot root_array; struct stack_depot_trie_root root = {}; - unsigned long read_scratch[ARRAY_SIZE(entries)]; unsigned long out[ARRAY_SIZE(entries)] = {}; u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; const void *tail = NULL; @@ -3827,8 +3805,7 @@ static void stackdepot_trie_insert_append_splits_frame_runs(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, tail, node_slots[2].node); KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, entries[0]), node_slots[0].node); - fetched = tfetch(tail, out, ARRAY_SIZE(out), read_scratch, - ARRAY_SIZE(read_scratch)); + fetched = tfetch(tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); } @@ -3845,7 +3822,6 @@ static void stackdepot_trie_insert_append_splits_child(struct kunit *test) struct stack_depot_trie_node_slot node_slots[3]; struct stack_depot_trie_lookup lookup; struct stack_depot_trie_root root = {}; - unsigned long scratch[ARRAY_SIZE(old_entries)]; unsigned long out[ARRAY_SIZE(old_entries)] = {}; const void *old_head = NULL; const void *old_tail; @@ -3896,7 +3872,7 @@ static void stackdepot_trie_insert_append_splits_child(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); old_tail = lookup.node; - fetched = tfetch(old_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(old_tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, 2U); KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); @@ -3909,7 +3885,7 @@ static void stackdepot_trie_insert_append_splits_child(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); KUNIT_EXPECT_PTR_EQ(test, lookup.node, new_tail); - fetched = tfetch(new_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(new_tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, 2U); KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); } @@ -3925,7 +3901,6 @@ static void stackdepot_trie_insert_append_splits_prefix_leaf(struct kunit *test) struct stack_depot_trie_node_slot node_slots[2]; struct stack_depot_trie_lookup lookup; struct stack_depot_trie_root root = {}; - unsigned long scratch[ARRAY_SIZE(old_entries)]; unsigned long out[ARRAY_SIZE(old_entries)] = {}; const void *old_head = NULL; const void *old_tail; @@ -3969,7 +3944,7 @@ static void stackdepot_trie_insert_append_splits_prefix_leaf(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); KUNIT_EXPECT_PTR_EQ(test, lookup.node, new_tail); - fetched = tfetch(new_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(new_tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, 1U); KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); @@ -3981,8 +3956,7 @@ static void stackdepot_trie_insert_append_splits_prefix_leaf(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); memset(out, 0, sizeof(out)); - fetched = tfetch(lookup.node, out, ARRAY_SIZE(out), scratch, - ARRAY_SIZE(scratch)); + fetched = tfetch(lookup.node, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, 2U); KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); } @@ -4624,13 +4598,12 @@ static void stackdepot_trie_node_compressed_roundtrip(struct kunit *test) 0xffffffff81002000UL, #endif }; - unsigned long scratch[ARRAY_SIZE(entries)]; unsigned long out[ARRAY_SIZE(entries)] = {}; unsigned int fetched; void *node; trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 11, &node); - fetched = tfetch(node, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(node, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); } @@ -4679,7 +4652,6 @@ static void stackdepot_trie_append_chain_splits_frame_runs(struct kunit *test) }; struct stack_depot_trie_node_slot node_slots[3]; struct stack_depot_trie_child_array_slot child_slots[2]; - unsigned long read_scratch[ARRAY_SIZE(entries)]; unsigned long out[ARRAY_SIZE(entries)] = {}; const void *child; u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; @@ -4715,8 +4687,7 @@ static void stackdepot_trie_append_chain_splits_frame_runs(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, child, node_slots[1].node); child = child_array_find(child_slots[1].array, entries[3]); KUNIT_EXPECT_PTR_EQ(test, child, node_slots[2].node); - fetched = tfetch(tail, out, ARRAY_SIZE(out), read_scratch, - ARRAY_SIZE(read_scratch)); + fetched = tfetch(tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); } @@ -4830,7 +4801,6 @@ static void stackdepot_trie_fetch_rejects_bad_inputs(struct kunit *test) unsigned long root_entries[] = { 0x3000UL }; unsigned long out[ARRAY_SIZE(entries)] = { 0xa5a5UL, 0xb6b6UL }; unsigned long expected[ARRAY_SIZE(out)]; - unsigned long scratch[ARRAY_SIZE(entries)]; unsigned int fetched; void *node; void *root; @@ -4838,18 +4808,19 @@ static void stackdepot_trie_fetch_rejects_bad_inputs(struct kunit *test) memcpy(expected, out, sizeof(expected)); trie_node_alloc(test, root_entries, ARRAY_SIZE(root_entries), NULL, 0, &root); - fetched = tfetch(root, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(root, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, 0); KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 5, &node); - fetched = tfetch(node, out, ARRAY_SIZE(out) - 1, scratch, ARRAY_SIZE(scratch)); - KUNIT_EXPECT_EQ(test, fetched, 0); - KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); - fetched = tfetch(node, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch) - 1); + fetched = tfetch(node, out, ARRAY_SIZE(out) - 1); KUNIT_EXPECT_EQ(test, fetched, 0); KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); - fetched = tfetch(NULL, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(node, out, ARRAY_SIZE(out)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); + memcpy(out, expected, sizeof(out)); + fetched = tfetch(NULL, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, 0); KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); } @@ -5280,7 +5251,6 @@ static void stackdepot_trie_split_subtree_divergent_tail(struct kunit *test) struct stack_depot_trie_child_array_slot child_slot; struct stack_depot_trie_node_slot node_slots[3]; struct stack_depot_trie_lookup lookup; - unsigned long scratch[ARRAY_SIZE(old_entries)]; unsigned long out[ARRAY_SIZE(old_entries)] = {}; const void *new_tail = NULL; const void *old_tail; @@ -5309,11 +5279,11 @@ static void stackdepot_trie_split_subtree_divergent_tail(struct kunit *test) KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); old_tail = lookup.node; KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[2].node); - fetched = tfetch(old_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(old_tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, 2U); KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); memset(out, 0, sizeof(out)); - fetched = tfetch(new_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(new_tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, 2U); KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); } @@ -5325,7 +5295,6 @@ static void stackdepot_trie_split_subtree_prefix_leaf(struct kunit *test) struct stack_depot_trie_child_array_slot child_slot; struct stack_depot_trie_node_slot node_slots[2]; struct stack_depot_trie_lookup lookup; - unsigned long scratch[ARRAY_SIZE(old_entries)]; unsigned long out[ARRAY_SIZE(old_entries)] = {}; const void *new_tail = NULL; const void *old_tail; @@ -5350,7 +5319,7 @@ static void stackdepot_trie_split_subtree_prefix_leaf(struct kunit *test) KUNIT_EXPECT_EQ(test, used, 2U); KUNIT_EXPECT_PTR_EQ(test, new_tail, prefix); - fetched = tfetch(prefix, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(prefix, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, 1U); KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); @@ -5359,7 +5328,7 @@ static void stackdepot_trie_split_subtree_prefix_leaf(struct kunit *test) KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); old_tail = lookup.node; memset(out, 0, sizeof(out)); - fetched = tfetch(old_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(old_tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, 2U); KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); } @@ -5376,7 +5345,6 @@ static void stackdepot_trie_split_subtree_preserves_children(struct kunit *test) struct stack_depot_trie_node_slot desc_slot; struct stack_depot_trie_node_slot node_slots[3]; struct stack_depot_trie_lookup lookup; - unsigned long scratch[ARRAY_SIZE(expected)]; unsigned long out[ARRAY_SIZE(expected)] = {}; const void *desc_head = NULL; const void *desc_tail = NULL; @@ -5424,7 +5392,7 @@ static void stackdepot_trie_split_subtree_preserves_children(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); KUNIT_EXPECT_PTR_EQ(test, lookup.node, desc_tail); - fetched = tfetch(desc_tail, out, ARRAY_SIZE(out), scratch, ARRAY_SIZE(scratch)); + fetched = tfetch(desc_tail, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); } From c030eaffa2c3d0a1a9ab0f107f506705087b73dd Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 10 Jun 2026 09:26:03 +0100 Subject: [PATCH 069/129] KRN-1117: Route stackdepot fetch_into trie handles Teach stack_depot_fetch_into() to recognize trie handles and materialize them through the private trie side-table path. Public saves still return hash handles, so this only removes a fetch-routing blocker before trie saves are enabled behind the feature flag. Warn on corrupt trie handles that decode to a missing side-table entry, matching the existing hash handle behavior. Cover public fetch_into materialization and too-small output buffers for trie handles in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 9 +++++++++ lib/tests/stackdepot_kunit.c | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 550b94b4f7f81..fd287dc9fc970 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -3918,6 +3918,10 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, rcu_read_lock_sched_notrace(); leaf = __stack_depot_trie_side_table_lookup(leaf_id); + if (WARN(!leaf, "corrupt trie handle %08x\n", handle)) { + rcu_read_unlock_sched_notrace(); + return 0; + } nr_entries = trie_fetch_leaf(leaf, entries, max_entries); rcu_read_unlock_sched_notrace(); @@ -4619,6 +4623,11 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, if (!handle || !entries || !max_entries) return 0; + if (stack_depot_disabled) + return 0; + if (__stack_depot_trie_leaf_id(handle)) + return __stack_depot_trie_fetch_handle_into(handle, entries, + max_entries); /* Extend the lookup RCU section so the fetched record cannot be reused. */ rcu_read_lock_sched_notrace(); diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index d697d8d60f352..810fcb38b5655 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1719,6 +1719,12 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) fetched = tfetch_handle(handle, small, ARRAY_SIZE(small)); KUNIT_EXPECT_EQ(test, fetched, 0U); KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); + fetched = stack_depot_fetch_into(handle, out, ARRAY_SIZE(out)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); + fetched = stack_depot_fetch_into(handle, small, ARRAY_SIZE(small)); + KUNIT_EXPECT_EQ(test, fetched, 0U); + KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); invalid = tfetch_handle(0, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, invalid, 0U); invalid = tfetch_handle(handle, NULL, 0); From 7a8781f7b813dbb31452c6df93bba1b4be4a9e11 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 10 Jun 2026 11:26:55 +0100 Subject: [PATCH 070/129] KRN-1117: Add stackdepot trie materialization slots Extend trie side-table entries so each leaf id can hold both the published trie leaf and a future stackdepot-owned materialized frame array. This keeps the compatibility state centralized in stackdepot instead of forcing callers to know which backend stored a handle. Keep materialized frame pointers one-shot so concurrent fetch compatibility work can publish a stable buffer without later overwrites or leaks. Cover the leaf and frame slot invariants in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 124 +++++++++++++++++++++++++++++------ lib/stackdepot_internal.h | 10 ++- lib/tests/stackdepot_kunit.c | 29 ++++++++ 3 files changed, 140 insertions(+), 23 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index fd287dc9fc970..465ce9f12f5e6 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -270,7 +270,12 @@ u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) return leaf_id > U32_MAX ? 0 : leaf_id; } -static const void ***trie_side_table_chunks; +struct stack_depot_trie_side_entry { + const void *leaf; + const unsigned long *frames; +}; + +static struct stack_depot_trie_side_entry **trie_side_table_chunks; static DEFINE_RAW_SPINLOCK(trie_side_table_lock); static DEFINE_RAW_SPINLOCK(trie_alloc_lock); static unsigned int trie_side_table_high_water; @@ -289,36 +294,58 @@ static unsigned int trie_side_table_slot_index(u32 id) return (id - 1) & (STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE - 1); } -static const void **trie_side_table_load_chunk(unsigned int top) +static struct stack_depot_trie_side_entry *trie_side_table_load_chunk(unsigned int top) { /* Pairs with trie_side_table_publish_chunk(). */ return smp_load_acquire(&trie_side_table_chunks[top]); } -static void trie_side_table_publish_chunk(unsigned int top, const void **chunk) +static void +trie_side_table_publish_chunk(unsigned int top, + struct stack_depot_trie_side_entry *chunk) { /* Pairs with trie_side_table_load_chunk(). */ smp_store_release(&trie_side_table_chunks[top], chunk); } static const void * -trie_side_table_load_entry(const void **chunk, unsigned int slot) +trie_side_table_load_leaf(struct stack_depot_trie_side_entry *chunk, + unsigned int slot) { - /* Pairs with trie_side_table_store_entry(). */ - return smp_load_acquire(&chunk[slot]); + /* Pairs with trie_side_table_store_leaf(). */ + return smp_load_acquire(&chunk[slot].leaf); } static void -trie_side_table_store_entry(const void **chunk, unsigned int slot, const void *entry) +trie_side_table_store_leaf(struct stack_depot_trie_side_entry *chunk, + unsigned int slot, const void *leaf) { - /* Pairs with trie_side_table_load_entry(). */ - smp_store_release(&chunk[slot], entry); + /* Pairs with trie_side_table_load_leaf(). */ + smp_store_release(&chunk[slot].leaf, leaf); } -static void trie_side_table_clear_entry(const void **chunk, unsigned int slot) +static const unsigned long * +trie_side_table_load_frames(struct stack_depot_trie_side_entry *chunk, + unsigned int slot) +{ + /* Pairs with trie_side_table_store_frames(). */ + return smp_load_acquire(&chunk[slot].frames); +} + +static void +trie_side_table_store_frames(struct stack_depot_trie_side_entry *chunk, + unsigned int slot, const unsigned long *frames) { - /* Pairs with trie_side_table_load_entry(). */ - smp_store_release(&chunk[slot], NULL); + /* Pairs with trie_side_table_load_frames(). */ + smp_store_release(&chunk[slot].frames, frames); +} + +static void +trie_side_table_clear_entry(struct stack_depot_trie_side_entry *chunk, + unsigned int slot) +{ + trie_side_table_store_frames(chunk, slot, NULL); + trie_side_table_store_leaf(chunk, slot, NULL); } int __stack_depot_trie_side_table_init(gfp_t gfp_flags) @@ -407,7 +434,7 @@ void __stack_depot_trie_side_table_free_prealloc(void *prealloc) u32 __stack_depot_trie_side_table_alloc_id(void **prealloc) { - const void **chunk; + struct stack_depot_trie_side_entry *chunk; unsigned long flags; u32 id; unsigned int top; @@ -446,7 +473,7 @@ u32 __stack_depot_trie_side_table_alloc_id(void **prealloc) void __stack_depot_trie_side_table_revoke_latest(u32 id) { - const void **chunk; + struct stack_depot_trie_side_entry *chunk; unsigned long flags; unsigned int slot; unsigned int top; @@ -475,7 +502,7 @@ void __stack_depot_trie_side_table_revoke_latest(u32 id) void __stack_depot_trie_side_table_restore(u32 id, const void *entry) { - const void **chunk; + struct stack_depot_trie_side_entry *chunk; unsigned long flags; unsigned int top; @@ -493,14 +520,17 @@ void __stack_depot_trie_side_table_restore(u32 id, const void *entry) if (!chunk) goto out; - trie_side_table_store_entry(chunk, trie_side_table_slot_index(id), entry); + if (entry) + trie_side_table_store_leaf(chunk, trie_side_table_slot_index(id), entry); + else + trie_side_table_clear_entry(chunk, trie_side_table_slot_index(id)); out: raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); } int __stack_depot_trie_side_table_store(u32 id, const void *entry) { - const void **chunk; + struct stack_depot_trie_side_entry *chunk; unsigned long flags; unsigned int top; int ret = -EINVAL; @@ -519,7 +549,7 @@ int __stack_depot_trie_side_table_store(u32 id, const void *entry) if (!chunk) goto out; - trie_side_table_store_entry(chunk, trie_side_table_slot_index(id), entry); + trie_side_table_store_leaf(chunk, trie_side_table_slot_index(id), entry); ret = 0; out: raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); @@ -528,7 +558,26 @@ int __stack_depot_trie_side_table_store(u32 id, const void *entry) const void *__stack_depot_trie_side_table_lookup(u32 id) { - const void **chunk; + struct stack_depot_trie_side_entry *chunk; + unsigned int top; + + if (!READ_ONCE(trie_side_table_initialized) || !id) + return NULL; + + top = trie_side_table_top_index(id); + if (top >= trie_side_table_top_size) + return NULL; + + chunk = trie_side_table_load_chunk(top); + if (!chunk) + return NULL; + + return trie_side_table_load_leaf(chunk, trie_side_table_slot_index(id)); +} + +const unsigned long *__stack_depot_trie_side_table_frames(u32 id) +{ + struct stack_depot_trie_side_entry *chunk; unsigned int top; if (!READ_ONCE(trie_side_table_initialized) || !id) @@ -542,7 +591,42 @@ const void *__stack_depot_trie_side_table_lookup(u32 id) if (!chunk) return NULL; - return trie_side_table_load_entry(chunk, trie_side_table_slot_index(id)); + return trie_side_table_load_frames(chunk, trie_side_table_slot_index(id)); +} + +int +__stack_depot_trie_side_table_store_frames(u32 id, const unsigned long *frames) +{ + struct stack_depot_trie_side_entry *chunk; + unsigned long flags; + unsigned int top; + int ret = -EINVAL; + + if (!READ_ONCE(trie_side_table_initialized) || !id || !frames) + return -EINVAL; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + if (id > trie_side_table_next_id) + goto out; + top = trie_side_table_top_index(id); + if (top >= trie_side_table_top_size) + goto out; + + chunk = trie_side_table_load_chunk(top); + if (!chunk) + goto out; + if (!trie_side_table_load_leaf(chunk, trie_side_table_slot_index(id))) + goto out; + if (trie_side_table_load_frames(chunk, trie_side_table_slot_index(id))) { + ret = -EEXIST; + goto out; + } + + trie_side_table_store_frames(chunk, trie_side_table_slot_index(id), frames); + ret = 0; +out: + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return ret; } size_t __stack_depot_trie_side_table_entries(void) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index e8ad28c28057a..be326a2560756 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -135,9 +135,10 @@ u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle); u32 __stack_depot_trie_max_leaf_id(void); /* - * Private trie leaf side table. Writers serialize internally; lookup is - * lockless. Init and destroy are controlled setup/teardown operations and must - * not race with lookup. + * Private trie side table. Writers serialize internally; lookups are lockless. + * Leaf slots are populated before trie publication, while frame slots are for + * future stable materialization of trie handles. Init and destroy are + * controlled setup/teardown operations and must not race with lookup. */ int __stack_depot_trie_side_table_init(gfp_t gfp_flags); void __stack_depot_trie_side_table_destroy(void); @@ -149,6 +150,9 @@ void __stack_depot_trie_side_table_revoke_latest(u32 id); void __stack_depot_trie_side_table_restore(u32 id, const void *entry); int __stack_depot_trie_side_table_store(u32 id, const void *entry); const void *__stack_depot_trie_side_table_lookup(u32 id); +const unsigned long *__stack_depot_trie_side_table_frames(u32 id); +int +__stack_depot_trie_side_table_store_frames(u32 id, const unsigned long *frames); size_t __stack_depot_trie_side_table_entries(void); size_t __stack_depot_trie_side_table_bytes(void); size_t __stack_depot_trie_pool_alloc_size(size_t size); diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 810fcb38b5655..9424dde38af4f 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -643,6 +643,11 @@ static void stackdepot_trie_side_table_alloc_store_lookup(struct kunit *test) { const void *entry1 = (const void *)0x1111UL; const void *entry2 = (const void *)0x2222UL; + const unsigned long frames[] = { 0xaaaaUL, 0xbbbbUL }; + const unsigned long other_frames[] = { 0xccccUL }; + const unsigned long *frames_ptr = frames; + const unsigned long *other_frames_ptr = other_frames; + int ret; u32 id1; u32 id2; @@ -656,11 +661,18 @@ static void stackdepot_trie_side_table_alloc_store_lookup(struct kunit *test) KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store(id2, entry2), 0); KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), entry1); KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id2), entry2); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(id1)); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store_frames(id1, frames_ptr), 0); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(id1), frames_ptr); + ret = __stack_depot_trie_side_table_store_frames(id1, other_frames_ptr); + KUNIT_EXPECT_EQ(test, ret, -EEXIST); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(id1), frames_ptr); KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 2UL); } static void stackdepot_trie_side_table_rejects_invalid_ids(struct kunit *test) { + const unsigned long *frames = (const unsigned long *)0x1UL; int ret; u32 id; @@ -670,14 +682,23 @@ static void stackdepot_trie_side_table_rejects_invalid_ids(struct kunit *test) id = stackdepot_trie_side_table_alloc(test); KUNIT_ASSERT_EQ(test, id, 1U); KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id + 1)); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(id + 1)); + ret = __stack_depot_trie_side_table_store_frames(id, frames); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store(id, NULL), -EINVAL); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store_frames(id, NULL), + -EINVAL); ret = __stack_depot_trie_side_table_store(id + 1, (const void *)0x1UL); KUNIT_EXPECT_EQ(test, ret, -EINVAL); + ret = __stack_depot_trie_side_table_store_frames(id + 1, frames); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); } static void stackdepot_trie_side_table_revoke_latest(struct kunit *test) { const void *entry = (const void *)0xaaaaUL; + const unsigned long frames[] = { 0x1234UL }; + const unsigned long *frames_ptr = frames; size_t bytes; int ret; u32 id; @@ -687,6 +708,8 @@ static void stackdepot_trie_side_table_revoke_latest(struct kunit *test) KUNIT_ASSERT_EQ(test, id, 1U); ret = __stack_depot_trie_side_table_store(id, entry); KUNIT_ASSERT_EQ(test, ret, 0); + ret = __stack_depot_trie_side_table_store_frames(id, frames_ptr); + KUNIT_ASSERT_EQ(test, ret, 0); bytes = __stack_depot_trie_side_table_bytes(); KUNIT_EXPECT_GT(test, bytes, 0UL); @@ -694,6 +717,7 @@ static void stackdepot_trie_side_table_revoke_latest(struct kunit *test) KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_bytes(), bytes); KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id)); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(id)); } static void stackdepot_trie_side_table_revoke_keeps_chunk(struct kunit *test) @@ -724,6 +748,8 @@ static void stackdepot_trie_side_table_restore(struct kunit *test) { const void *entry1 = (const void *)0xaaaaUL; const void *entry2 = (const void *)0xbbbbUL; + const unsigned long frames[] = { 0xccccUL }; + const unsigned long *frames_ptr = frames; u32 id; stackdepot_trie_side_table_init_or_skip(test); @@ -731,11 +757,14 @@ static void stackdepot_trie_side_table_restore(struct kunit *test) KUNIT_ASSERT_EQ(test, id, 1U); KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, entry1), 0); KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, entry2), 0); + KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store_frames(id, frames_ptr), 0); __stack_depot_trie_side_table_restore(id, entry1); KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id), entry1); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(id), frames_ptr); __stack_depot_trie_side_table_restore(id, NULL); KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id)); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(id)); } static void stackdepot_trie_side_table_chunk_boundary(struct kunit *test) From 2cd8dace75ae2bb260db68fa42642ccfffd4f64d Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 10 Jun 2026 12:08:09 +0100 Subject: [PATCH 071/129] KRN-1117: Publish trie materialization slots locklessly Use a release cmpxchg to publish the future trie materialized-frame pointer without taking trie_side_table_lock. This keeps the compatibility path aligned with stackdepot's lockless materialization constraints while preserving the one-shot publication rule for each trie handle. Document that the side table reserves only a pointer per handle up front, while the backing frame array is allocated lazily for the rare fetch path. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 31 ++++++++++++------------------- lib/stackdepot_internal.h | 6 ++++-- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 465ce9f12f5e6..1cd2ab60b0e47 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -598,35 +598,28 @@ int __stack_depot_trie_side_table_store_frames(u32 id, const unsigned long *frames) { struct stack_depot_trie_side_entry *chunk; - unsigned long flags; + const unsigned long *old; + unsigned int slot; unsigned int top; - int ret = -EINVAL; if (!READ_ONCE(trie_side_table_initialized) || !id || !frames) return -EINVAL; - raw_spin_lock_irqsave(&trie_side_table_lock, flags); - if (id > trie_side_table_next_id) - goto out; + if (id > READ_ONCE(trie_side_table_next_id)) + return -EINVAL; top = trie_side_table_top_index(id); if (top >= trie_side_table_top_size) - goto out; + return -EINVAL; chunk = trie_side_table_load_chunk(top); if (!chunk) - goto out; - if (!trie_side_table_load_leaf(chunk, trie_side_table_slot_index(id))) - goto out; - if (trie_side_table_load_frames(chunk, trie_side_table_slot_index(id))) { - ret = -EEXIST; - goto out; - } + return -EINVAL; + slot = trie_side_table_slot_index(id); + if (!trie_side_table_load_leaf(chunk, slot)) + return -EINVAL; - trie_side_table_store_frames(chunk, trie_side_table_slot_index(id), frames); - ret = 0; -out: - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return ret; + old = cmpxchg_release(&chunk[slot].frames, NULL, frames); + return old ? -EEXIST : 0; } size_t __stack_depot_trie_side_table_entries(void) @@ -4713,7 +4706,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); - /* Extend the lookup RCU section so the fetched record cannot be reused. */ + /* Hold RCU so the fetched record cannot be reused during the copy. */ rcu_read_lock_sched_notrace(); nr_entries = stack_depot_fetch(handle, &stack_entries); if (!nr_entries || nr_entries > max_entries) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index be326a2560756..d5707bdd79548 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -137,8 +137,10 @@ u32 __stack_depot_trie_max_leaf_id(void); /* * Private trie side table. Writers serialize internally; lookups are lockless. * Leaf slots are populated before trie publication, while frame slots are for - * future stable materialization of trie handles. Init and destroy are - * controlled setup/teardown operations and must not race with lookup. + * future stable materialization of trie handles. The side table reserves the + * frame pointer per handle, but the backing materialized frame array is + * allocated lazily only for the rare fetch path. Init and destroy are controlled + * setup/teardown operations and must not race with readers or writers. */ int __stack_depot_trie_side_table_init(gfp_t gfp_flags); void __stack_depot_trie_side_table_destroy(void); From ed8ae1fe8b07587ed5f5bd48b02c776e016210a7 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 10 Jun 2026 13:09:50 +0100 Subject: [PATCH 072/129] KRN-1117: Add stackdepot trie frame walker Add a private validated trie frame walker that emits indexed frames in stack order. This centralizes trie frame traversal so future fetch_into, print, snprint, and lazy fetch materialization can share one decode path. Refactor private trie fetch_into to use the walker while preserving the validate-before-write contract. Cover iterator ordering and invalid leaf handling in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 122 +++++++++++++++++++++++------------ lib/stackdepot_internal.h | 5 ++ lib/tests/stackdepot_kunit.c | 36 +++++++++++ 3 files changed, 123 insertions(+), 40 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 1cd2ab60b0e47..06eb061abe363 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2279,29 +2279,6 @@ static int frame_run_validate_payload(const struct stack_depot_frame_run *run, return 0; } -static int frame_run_read_direct(const struct stack_depot_frame_run *run, - const void *src, unsigned long *entries) -{ - unsigned int i; - - if (!entries || frame_run_validate_payload(run, src)) - return -EINVAL; - if (run->mode == STACK_DEPOT_FRAME_RAW) { - memcpy(entries, src, run->bytes); - return 0; - } - - for (i = 0; i < run->nr_entries; i++) { - u32 low; - - memcpy(&low, (const char *)src + i * sizeof(low), sizeof(low)); - if (!frame_decompress(run->prefix_id, low, &entries[i])) - return -EINVAL; - } - - return 0; -} - static bool stack_depot_ranges_overlap(const void *a, size_t a_size, const void *b, size_t b_size) { @@ -3926,45 +3903,110 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, } } -unsigned int -__stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, - unsigned int max_entries) +struct stack_depot_trie_fetch_ctx { + unsigned long *entries; + unsigned int nr_entries; +}; + +static unsigned int trie_validate_leaf(const void *leaf, + const unsigned long *entries) { const struct stack_depot_trie_node *node = leaf; + size_t entries_size; unsigned int pos; unsigned int total; - int ret; - if (!node || !entries || !node->stack_len || !node->leaf_id) + if (!node || !node->stack_len || !node->leaf_id) return 0; total = node->stack_len; - if (max_entries < total) - return 0; - - /* Validate first so failure cannot partially write the caller buffer. */ + entries_size = total * sizeof(*entries); pos = total; for (node = leaf; node; node = trie_load_parent(node)) { + bool overlap; + if (frame_run_validate_payload(&node->run, node->data)) return 0; - if (stack_depot_ranges_overlap(entries, total * sizeof(*entries), - node->data, node->run.bytes)) + overlap = entries && stack_depot_ranges_overlap(entries, + entries_size, node->data, + node->run.bytes); + if (overlap) return 0; if (node->stack_len != pos || node->run.nr_entries > pos) return 0; pos -= node->run.nr_entries; } - if (pos) + + return pos ? 0 : total; +} + +static unsigned int trie_walk_frames(const void *leaf, unsigned int total, + trie_frame_fn_t fn, void *data) +{ + const struct stack_depot_trie_node *node; + unsigned int seen = 0; + unsigned int i; + + if (!fn) return 0; - pos = total; for (node = leaf; node; node = trie_load_parent(node)) { - pos -= node->run.nr_entries; - ret = frame_run_read_direct(&node->run, node->data, &entries[pos]); - if (ret) + unsigned int start; + + if (node->run.nr_entries > node->stack_len) return 0; + start = node->stack_len - node->run.nr_entries; + for (i = 0; i < node->run.nr_entries; i++) { + unsigned long frame; + + if (stack_depot_trie_node_frame(node, i, &frame)) + return 0; + fn(start + i, frame, data); + seen++; + } } - if (pos) + + return seen == total ? total : 0; +} + +static void trie_fetch_frame(unsigned int index, unsigned long frame, void *data) +{ + struct stack_depot_trie_fetch_ctx *ctx = data; + + ctx->entries[index] = frame; + ctx->nr_entries++; +} + +unsigned int +__stack_depot_trie_walk_frames(const void *leaf, trie_frame_fn_t fn, void *data) +{ + unsigned int total; + + total = trie_validate_leaf(leaf, NULL); + if (!total) + return 0; + + return trie_walk_frames(leaf, total, fn, data); +} + +unsigned int +__stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, + unsigned int max_entries) +{ + struct stack_depot_trie_fetch_ctx ctx; + unsigned int total; + + if (!entries) + return 0; + total = trie_validate_leaf(leaf, entries); + if (!total) + return 0; + if (max_entries < total) + return 0; + + ctx.entries = entries; + ctx.nr_entries = 0; + if (trie_walk_frames(leaf, total, trie_fetch_frame, &ctx) != total) return 0; kmsan_unpoison_memory(entries, total * sizeof(*entries)); diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index d5707bdd79548..bf752063e59d4 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -59,6 +59,9 @@ struct stack_depot_trie_leaf_update { const void *leaf; }; +typedef void (*trie_frame_fn_t)(unsigned int index, unsigned long frame, + void *data); + struct stack_depot_trie_publish_prepare { int (*fn)(const struct stack_depot_trie_leaf_update *updates, unsigned int nr_updates, void *ctx); @@ -309,6 +312,8 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries); +unsigned int +__stack_depot_trie_walk_frames(const void *leaf, trie_frame_fn_t fn, void *data); unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, unsigned long *entries, unsigned int max_entries); diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 9424dde38af4f..41b030c26c32e 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1551,6 +1551,25 @@ static unsigned int tfetch_handle(depot_stack_handle_t handle, return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); } +struct trie_frame_iter_ctx { + unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; + unsigned int nr_entries; +}; + +static void trie_frame_iter_record(unsigned int index, unsigned long frame, + void *data) +{ + struct trie_frame_iter_ctx *ctx = data; + + ctx->entries[index] = frame; + ctx->nr_entries++; +} + +static unsigned int twalk_frames(const void *leaf, struct trie_frame_iter_ctx *ctx) +{ + return __stack_depot_trie_walk_frames(leaf, trie_frame_iter_record, ctx); +} + static void stackdepot_trie_alloc_workspace_plan(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; @@ -1725,22 +1744,39 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; struct stack_depot_trie_alloc_workspace *workspace; + struct trie_frame_iter_ctx *iter; struct stack_depot_trie_root root = {}; unsigned long small[1] = { 0xdeadUL }; unsigned long out[ARRAY_SIZE(entries)] = {}; depot_stack_handle_t hash_handle; depot_stack_handle_t handle; + const void *leaf; unsigned int invalid; unsigned int fetched; + u32 leaf_id; workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, workspace); + iter = kunit_kzalloc(test, sizeof(*iter), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, iter); stackdepot_trie_side_table_init_or_skip(test); stackdepot_trie_pool_seed_current_pool(test); handle = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, STACK_DEPOT_FLAG_CAN_ALLOC, workspace); KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + leaf_id = __stack_depot_trie_leaf_id(handle); + KUNIT_ASSERT_NE(test, leaf_id, 0U); + leaf = __stack_depot_trie_side_table_lookup(leaf_id); + KUNIT_ASSERT_NOT_NULL(test, leaf); + fetched = twalk_frames(leaf, iter); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_EQ(test, iter->nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, iter->entries, entries, sizeof(entries)); + iter->nr_entries = 0; + fetched = twalk_frames(NULL, iter); + KUNIT_EXPECT_EQ(test, fetched, 0U); + KUNIT_EXPECT_EQ(test, iter->nr_entries, 0U); fetched = tfetch_handle(handle, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); From 8ee1a7109c255881d59d54d5f42be85c8debde60 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 10 Jun 2026 13:50:42 +0100 Subject: [PATCH 073/129] KRN-1117: Add stackdepot trie materialize helper Add a private helper that materializes a trie handle into caller-prepared storage and publishes that storage through the trie side table. This keeps allocation policy separate while providing the central path needed for future stack_depot_fetch() compatibility. Return an existing cached frame pointer when one is already published, and use the side table's one-shot publication semantics so racing materializers return the winning stackdepot-owned copy. Cover too-small storage, first publication, race-loser, invalid-handle, and cached-hit behavior in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 64 ++++++++++++++++++++++++++++++++++++ lib/stackdepot_internal.h | 5 +++ lib/tests/stackdepot_kunit.c | 34 +++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 06eb061abe363..cc378d09aa034 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -4047,6 +4047,70 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, return nr_entries; } +unsigned int +__stack_depot_trie_materialize_handle(depot_stack_handle_t handle, + unsigned long *storage, + unsigned int max_entries, + const unsigned long **frames) +{ + const unsigned long *cached; + const void *leaf; + unsigned int nr_entries; + u32 leaf_id; + int ret; + + if (!frames) + return 0; + *frames = NULL; + + leaf_id = __stack_depot_trie_leaf_id(handle); + if (!leaf_id) + return 0; + + rcu_read_lock_sched_notrace(); + leaf = __stack_depot_trie_side_table_lookup(leaf_id); + if (WARN(!leaf, "corrupt trie handle %08x\n", handle)) { + rcu_read_unlock_sched_notrace(); + return 0; + } + nr_entries = trie_validate_leaf(leaf, NULL); + if (!nr_entries) { + rcu_read_unlock_sched_notrace(); + return 0; + } + cached = __stack_depot_trie_side_table_frames(leaf_id); + if (cached) { + *frames = cached; + rcu_read_unlock_sched_notrace(); + return nr_entries; + } + if (!storage || max_entries < nr_entries) { + rcu_read_unlock_sched_notrace(); + return 0; + } + + nr_entries = trie_fetch_leaf(leaf, storage, max_entries); + if (!nr_entries) { + rcu_read_unlock_sched_notrace(); + return 0; + } + + ret = __stack_depot_trie_side_table_store_frames(leaf_id, storage); + if (!ret) { + *frames = storage; + rcu_read_unlock_sched_notrace(); + return nr_entries; + } + if (ret == -EEXIST) { + cached = __stack_depot_trie_side_table_frames(leaf_id); + if (cached) + *frames = cached; + } + rcu_read_unlock_sched_notrace(); + + return *frames ? nr_entries : 0; +} + size_t __stack_depot_trie_child_array_size(unsigned int nr_children) { size_t size; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index bf752063e59d4..2efec22240592 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -317,6 +317,11 @@ __stack_depot_trie_walk_frames(const void *leaf, trie_frame_fn_t fn, void *data) unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, unsigned long *entries, unsigned int max_entries); +unsigned int +__stack_depot_trie_materialize_handle(depot_stack_handle_t handle, + unsigned long *storage, + unsigned int max_entries, + const unsigned long **frames); size_t __stack_depot_trie_child_array_size(unsigned int nr_children); int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, const void * const *children, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 41b030c26c32e..95a93cbf47364 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1551,6 +1551,14 @@ static unsigned int tfetch_handle(depot_stack_handle_t handle, return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); } +static unsigned int tmaterialize(depot_stack_handle_t handle, + unsigned long *storage, unsigned int max_entries, + const unsigned long **frames) +{ + return __stack_depot_trie_materialize_handle(handle, storage, max_entries, + frames); +} + struct trie_frame_iter_ctx { unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; unsigned int nr_entries; @@ -1747,9 +1755,13 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) struct trie_frame_iter_ctx *iter; struct stack_depot_trie_root root = {}; unsigned long small[1] = { 0xdeadUL }; + unsigned long cache[ARRAY_SIZE(entries)] = {}; + unsigned long loser[ARRAY_SIZE(entries)] = { 0xdeadUL, 0xbeefUL }; + unsigned long *cache_ptr = cache; unsigned long out[ARRAY_SIZE(entries)] = {}; depot_stack_handle_t hash_handle; depot_stack_handle_t handle; + const unsigned long *frames; const void *leaf; unsigned int invalid; unsigned int fetched; @@ -1790,6 +1802,24 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) fetched = stack_depot_fetch_into(handle, small, ARRAY_SIZE(small)); KUNIT_EXPECT_EQ(test, fetched, 0U); KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); + frames = (const unsigned long *)0x1UL; + fetched = tmaterialize(handle, small, ARRAY_SIZE(small), &frames); + KUNIT_EXPECT_EQ(test, fetched, 0U); + KUNIT_EXPECT_NULL(test, frames); + KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); + fetched = tmaterialize(handle, cache, ARRAY_SIZE(cache), &frames); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_PTR_EQ(test, frames, cache_ptr); + KUNIT_EXPECT_MEMEQ(test, cache, entries, sizeof(entries)); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(leaf_id), + cache_ptr); + fetched = tmaterialize(handle, loser, ARRAY_SIZE(loser), &frames); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_PTR_EQ(test, frames, cache_ptr); + KUNIT_EXPECT_EQ(test, loser[0], 0xdeadUL); + KUNIT_EXPECT_EQ(test, loser[1], 0xbeefUL); + KUNIT_EXPECT_EQ(test, tmaterialize(handle, cache, ARRAY_SIZE(cache), NULL), + 0U); invalid = tfetch_handle(0, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, invalid, 0U); invalid = tfetch_handle(handle, NULL, 0); @@ -1802,6 +1832,10 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); invalid = tfetch_handle(hash_handle, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, invalid, 0U); + frames = (const unsigned long *)0x1UL; + fetched = tmaterialize(hash_handle, cache, ARRAY_SIZE(cache), &frames); + KUNIT_EXPECT_EQ(test, fetched, 0U); + KUNIT_EXPECT_NULL(test, frames); } static void stackdepot_trie_alloc_txn_plan(struct kunit *test) From c76f251e06544d0d3983879a49a283ca043c92a3 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 10 Jun 2026 14:43:37 +0100 Subject: [PATCH 074/129] KRN-1117: Size stackdepot trie materialization buffers Add a private helper that validates a trie handle and reports the exact frame count and byte size needed for stable materialized storage. This keeps future stack_depot_fetch() compatibility allocation separate from trie traversal and avoids allocating CONFIG_STACKDEPOT_MAX_FRAMES for every materialized stack. Cover trie, hash, and invalid handles in KUnit so callers can size storage before using the private materialization helper. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 33 +++++++++++++++++++++++++++++++++ lib/stackdepot_internal.h | 1 + lib/tests/stackdepot_kunit.c | 12 ++++++++++++ 3 files changed, 46 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index cc378d09aa034..10d3388bad9f8 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -4111,6 +4111,39 @@ __stack_depot_trie_materialize_handle(depot_stack_handle_t handle, return *frames ? nr_entries : 0; } +size_t __stack_depot_trie_materialize_bytes(depot_stack_handle_t handle, unsigned int *nr_entries) +{ + const void *leaf; + unsigned int total; + u32 leaf_id; + size_t size; + + if (nr_entries) + *nr_entries = 0; + + leaf_id = __stack_depot_trie_leaf_id(handle); + if (!leaf_id) + return 0; + + rcu_read_lock_sched_notrace(); + leaf = __stack_depot_trie_side_table_lookup(leaf_id); + if (!leaf) + goto out; + total = trie_validate_leaf(leaf, NULL); + if (!total) + goto out; + if (check_mul_overflow((size_t)total, sizeof(unsigned long), &size)) + goto out; + + if (nr_entries) + *nr_entries = total; + rcu_read_unlock_sched_notrace(); + return size; +out: + rcu_read_unlock_sched_notrace(); + return 0; +} + size_t __stack_depot_trie_child_array_size(unsigned int nr_children) { size_t size; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 2efec22240592..78e2e51f195b6 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -322,6 +322,7 @@ __stack_depot_trie_materialize_handle(depot_stack_handle_t handle, unsigned long *storage, unsigned int max_entries, const unsigned long **frames); +size_t __stack_depot_trie_materialize_bytes(depot_stack_handle_t handle, unsigned int *nr_entries); size_t __stack_depot_trie_child_array_size(unsigned int nr_children); int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, const void * const *children, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 95a93cbf47364..23c5e59bdbcfa 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1765,6 +1765,8 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) const void *leaf; unsigned int invalid; unsigned int fetched; + unsigned int nr_sized; + size_t size; u32 leaf_id; workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); @@ -1802,6 +1804,12 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) fetched = stack_depot_fetch_into(handle, small, ARRAY_SIZE(small)); KUNIT_EXPECT_EQ(test, fetched, 0U); KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); + size = __stack_depot_trie_materialize_bytes(handle, &nr_sized); + KUNIT_EXPECT_EQ(test, size, sizeof(entries)); + KUNIT_EXPECT_EQ(test, nr_sized, (unsigned int)ARRAY_SIZE(entries)); + nr_sized = 0xdeadU; + KUNIT_EXPECT_EQ(test, __stack_depot_trie_materialize_bytes(0, &nr_sized), 0UL); + KUNIT_EXPECT_EQ(test, nr_sized, 0U); frames = (const unsigned long *)0x1UL; fetched = tmaterialize(handle, small, ARRAY_SIZE(small), &frames); KUNIT_EXPECT_EQ(test, fetched, 0U); @@ -1836,6 +1844,10 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) fetched = tmaterialize(hash_handle, cache, ARRAY_SIZE(cache), &frames); KUNIT_EXPECT_EQ(test, fetched, 0U); KUNIT_EXPECT_NULL(test, frames); + nr_sized = 0xdeadU; + size = __stack_depot_trie_materialize_bytes(hash_handle, &nr_sized); + KUNIT_EXPECT_EQ(test, size, 0UL); + KUNIT_EXPECT_EQ(test, nr_sized, 0U); } static void stackdepot_trie_alloc_txn_plan(struct kunit *test) From aff868535e5465b2af62c9ad015ded9b63beb46d Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 10 Jun 2026 15:45:09 +0100 Subject: [PATCH 075/129] KRN-1117: Add stackdepot trie materialized records Add a private materialized-record format for stable trie fetch storage and helpers to size, initialize, and recover its frame count. This gives future stack_depot_fetch() trie compatibility a compact stackdepot-owned object without changing public routing yet. Use the existing trie materialization path to populate the record and publish its entries through the side table. Cover sizing, invalid handles, too-small storage, cached reuse, and frame-count recovery in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 45 ++++++++++++++++++++++++++++++++++++ lib/stackdepot_internal.h | 13 +++++++++++ lib/tests/stackdepot_kunit.c | 32 +++++++++++++++++-------- 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 10d3388bad9f8..014d76ec70ac4 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -4144,6 +4144,51 @@ size_t __stack_depot_trie_materialize_bytes(depot_stack_handle_t handle, unsigne return 0; } +size_t __stack_depot_trie_materialized_size(unsigned int nr_entries) +{ + struct stack_depot_trie_materialized *record = NULL; + size_t size; + + if (!nr_entries || nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) + return 0; + size = struct_size(record, entries, nr_entries); + return size <= DEPOT_POOL_SIZE ? __stack_depot_trie_pool_alloc_size(size) : 0; +} + +unsigned int __stack_depot_trie_materialized_count(const unsigned long *frames) +{ + const struct stack_depot_trie_materialized *record; + + if (!frames) + return 0; + record = (const void *)((const char *)frames - + offsetof(struct stack_depot_trie_materialized, entries)); + return READ_ONCE(record->nr_entries); +} + +unsigned int +__stack_depot_trie_materialize_record(depot_stack_handle_t handle, + struct stack_depot_trie_materialized *record, + size_t record_size, + const unsigned long **frames) +{ + unsigned int nr_entries; + size_t size; + + if (!frames) + return 0; + *frames = NULL; + size = __stack_depot_trie_materialize_bytes(handle, &nr_entries); + if (!size) + return 0; + if (!record || record_size < __stack_depot_trie_materialized_size(nr_entries)) + return 0; + + WRITE_ONCE(record->nr_entries, nr_entries); + return __stack_depot_trie_materialize_handle(handle, record->entries, + nr_entries, frames); +} + size_t __stack_depot_trie_child_array_size(unsigned int nr_children) { size_t size; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 78e2e51f195b6..eea466b12f9ff 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -47,6 +47,11 @@ struct stack_depot_trie_root { const struct stack_depot_trie_child_array *children; }; +struct stack_depot_trie_materialized { + unsigned int nr_entries; + unsigned long entries[]; +}; + struct stack_depot_trie_lookup { const void *parent; const void *node; @@ -323,6 +328,14 @@ __stack_depot_trie_materialize_handle(depot_stack_handle_t handle, unsigned int max_entries, const unsigned long **frames); size_t __stack_depot_trie_materialize_bytes(depot_stack_handle_t handle, unsigned int *nr_entries); +size_t __stack_depot_trie_materialized_size(unsigned int nr_entries); +unsigned int +__stack_depot_trie_materialized_count(const unsigned long *frames); +unsigned int +__stack_depot_trie_materialize_record(depot_stack_handle_t handle, + struct stack_depot_trie_materialized *record, + size_t record_size, + const unsigned long **frames); size_t __stack_depot_trie_child_array_size(unsigned int nr_children); int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, const void * const *children, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 23c5e59bdbcfa..11542c6c52677 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1559,6 +1559,9 @@ static unsigned int tmaterialize(depot_stack_handle_t handle, frames); } +#define TMREC(handle, record, size, frames) \ + __stack_depot_trie_materialize_record(handle, record, size, frames) + struct trie_frame_iter_ctx { unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; unsigned int nr_entries; @@ -1752,17 +1755,18 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_materialized *record; struct trie_frame_iter_ctx *iter; struct stack_depot_trie_root root = {}; unsigned long small[1] = { 0xdeadUL }; - unsigned long cache[ARRAY_SIZE(entries)] = {}; unsigned long loser[ARRAY_SIZE(entries)] = { 0xdeadUL, 0xbeefUL }; - unsigned long *cache_ptr = cache; + unsigned long *record_entries; unsigned long out[ARRAY_SIZE(entries)] = {}; depot_stack_handle_t hash_handle; depot_stack_handle_t handle; const unsigned long *frames; const void *leaf; + size_t record_size; unsigned int invalid; unsigned int fetched; unsigned int nr_sized; @@ -1807,6 +1811,12 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) size = __stack_depot_trie_materialize_bytes(handle, &nr_sized); KUNIT_EXPECT_EQ(test, size, sizeof(entries)); KUNIT_EXPECT_EQ(test, nr_sized, (unsigned int)ARRAY_SIZE(entries)); + record_size = __stack_depot_trie_materialized_size(nr_sized); + KUNIT_ASSERT_NE(test, record_size, 0UL); + record = kunit_kzalloc(test, record_size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, record); + record->nr_entries = nr_sized; + record_entries = record->entries; nr_sized = 0xdeadU; KUNIT_EXPECT_EQ(test, __stack_depot_trie_materialize_bytes(0, &nr_sized), 0UL); KUNIT_EXPECT_EQ(test, nr_sized, 0U); @@ -1815,19 +1825,21 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) KUNIT_EXPECT_EQ(test, fetched, 0U); KUNIT_EXPECT_NULL(test, frames); KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); - fetched = tmaterialize(handle, cache, ARRAY_SIZE(cache), &frames); + fetched = TMREC(handle, record, record_size, &frames); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_PTR_EQ(test, frames, cache_ptr); - KUNIT_EXPECT_MEMEQ(test, cache, entries, sizeof(entries)); + KUNIT_EXPECT_PTR_EQ(test, frames, record_entries); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_materialized_count(frames), + (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, record->entries, entries, sizeof(entries)); KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(leaf_id), - cache_ptr); + record_entries); fetched = tmaterialize(handle, loser, ARRAY_SIZE(loser), &frames); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_PTR_EQ(test, frames, cache_ptr); + KUNIT_EXPECT_PTR_EQ(test, frames, record_entries); KUNIT_EXPECT_EQ(test, loser[0], 0xdeadUL); KUNIT_EXPECT_EQ(test, loser[1], 0xbeefUL); - KUNIT_EXPECT_EQ(test, tmaterialize(handle, cache, ARRAY_SIZE(cache), NULL), - 0U); + fetched = TMREC(handle, record, record_size, NULL); + KUNIT_EXPECT_EQ(test, fetched, 0U); invalid = tfetch_handle(0, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, invalid, 0U); invalid = tfetch_handle(handle, NULL, 0); @@ -1841,7 +1853,7 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) invalid = tfetch_handle(hash_handle, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, invalid, 0U); frames = (const unsigned long *)0x1UL; - fetched = tmaterialize(hash_handle, cache, ARRAY_SIZE(cache), &frames); + fetched = TMREC(hash_handle, record, record_size, &frames); KUNIT_EXPECT_EQ(test, fetched, 0U); KUNIT_EXPECT_NULL(test, frames); nr_sized = 0xdeadU; From 5d51a39351e570b1840b9e9473ba632c6af2d139 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 10 Jun 2026 16:41:45 +0100 Subject: [PATCH 076/129] KRN-1117: Add stackdepot trie materialization cache Add a private helper that returns cached materialized trie frames or lazily materializes a trie handle into stackdepot-owned pool storage. Keep the new path best-effort and trylock-only so future stack_depot_fetch() routing can reuse stable storage without sleeping in constrained contexts. Publish the winning frame pointer through the existing side-table cmpxchg and roll back losing current-pool storage before releasing pool_lock. Cover cached reuse, extra-bit handles, invalid handles, and hash handles in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 65 ++++++++++++++++++++++++++++++++- lib/stackdepot_internal.h | 3 ++ lib/tests/stackdepot_kunit.c | 69 ++++++++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 1 deletion(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 014d76ec70ac4..8d65850aa65d9 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -4186,7 +4186,70 @@ __stack_depot_trie_materialize_record(depot_stack_handle_t handle, WRITE_ONCE(record->nr_entries, nr_entries); return __stack_depot_trie_materialize_handle(handle, record->entries, - nr_entries, frames); + nr_entries, frames); +} + +unsigned int +__stack_depot_trie_materialize_cached(depot_stack_handle_t handle, + const unsigned long **frames) +{ + struct stack_depot_trie_materialized *record; + unsigned int nr_entries; + unsigned long flags; + size_t record_size; + size_t offset; + void *pool; + int ret; + + if (!frames) + return 0; + *frames = NULL; + + nr_entries = __stack_depot_trie_materialize_handle(handle, NULL, 0, + frames); + if (nr_entries) + return nr_entries; + + record_size = __stack_depot_trie_materialize_bytes(handle, &nr_entries); + if (!record_size) + return 0; + record_size = __stack_depot_trie_materialized_size(nr_entries); + if (!record_size) + return 0; + + if (!raw_spin_trylock_irqsave(&pool_lock, flags)) + return 0; + printk_deferred_enter(); + + ret = __stack_depot_trie_materialize_handle(handle, NULL, 0, frames); + if (ret) + goto out; + + if (!stack_pools || pools_num < 1) + goto out; + if (WARN_ON_ONCE(pool_offset > DEPOT_POOL_SIZE)) + goto out; + if (record_size > DEPOT_POOL_SIZE - pool_offset) + goto out; + + pool = stack_pools[pools_num - 1]; + if (WARN_ON_ONCE(!pool)) + goto out; + + offset = pool_offset; + record = pool + offset; + pool_offset += record_size; + ret = __stack_depot_trie_materialize_record(handle, record, record_size, frames); + if (ret && *frames == record->entries) + goto out; + + pool_offset = offset; + if (!*frames) + ret = 0; +out: + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return ret; } size_t __stack_depot_trie_child_array_size(unsigned int nr_children) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index eea466b12f9ff..6deed172ca54f 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -336,6 +336,9 @@ __stack_depot_trie_materialize_record(depot_stack_handle_t handle, struct stack_depot_trie_materialized *record, size_t record_size, const unsigned long **frames); +unsigned int +__stack_depot_trie_materialize_cached(depot_stack_handle_t handle, + const unsigned long **frames); size_t __stack_depot_trie_child_array_size(unsigned int nr_children); int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, const void * const *children, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 11542c6c52677..c7872a53f014b 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1559,6 +1559,12 @@ static unsigned int tmaterialize(depot_stack_handle_t handle, frames); } +static unsigned int tmaterialize_cached(depot_stack_handle_t handle, + const unsigned long **frames) +{ + return __stack_depot_trie_materialize_cached(handle, frames); +} + #define TMREC(handle, record, size, frames) \ __stack_depot_trie_materialize_record(handle, record, size, frames) @@ -1862,6 +1868,68 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) KUNIT_EXPECT_EQ(test, nr_sized, 0U); } +static void stackdepot_trie_materialize_cached(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_root root = {}; + depot_stack_handle_t hash_handle; + depot_stack_handle_t extra; + depot_stack_handle_t handle; + const unsigned long *again; + const unsigned long *frames; + unsigned int fetched; + u32 leaf_id; + + workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workspace); + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + + handle = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC, workspace); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + leaf_id = __stack_depot_trie_leaf_id(handle); + KUNIT_ASSERT_NE(test, leaf_id, 0U); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(leaf_id)); + + frames = NULL; + fetched = tmaterialize_cached(handle, &frames); + KUNIT_ASSERT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_ASSERT_NOT_NULL(test, frames); + KUNIT_EXPECT_MEMEQ(test, frames, entries, sizeof(entries)); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_materialized_count(frames), + (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(leaf_id), + frames); + + again = NULL; + fetched = tmaterialize_cached(handle, &again); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_PTR_EQ(test, again, frames); + + extra = stack_depot_set_extra_bits(handle, + (1U << STACK_DEPOT_EXTRA_BITS) - 1); + again = NULL; + fetched = tmaterialize_cached(extra, &again); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_PTR_EQ(test, again, frames); + + frames = (const unsigned long *)0x1UL; + fetched = tmaterialize_cached(0, &frames); + KUNIT_EXPECT_EQ(test, fetched, 0U); + KUNIT_EXPECT_NULL(test, frames); + fetched = tmaterialize_cached(handle, NULL); + KUNIT_EXPECT_EQ(test, fetched, 0U); + + hash_handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); + frames = (const unsigned long *)0x1UL; + fetched = tmaterialize_cached(hash_handle, &frames); + KUNIT_EXPECT_EQ(test, fetched, 0U); + KUNIT_EXPECT_NULL(test, frames); +} + static void stackdepot_trie_alloc_txn_plan(struct kunit *test) { unsigned long entries[] = { 0x1000UL }; @@ -5583,6 +5651,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_save_miss_noalloc), KUNIT_CASE(stackdepot_trie_save), KUNIT_CASE(stackdepot_trie_fetch_handle_into), + KUNIT_CASE(stackdepot_trie_materialize_cached), KUNIT_CASE(stackdepot_trie_alloc_txn_plan), KUNIT_CASE(stackdepot_trie_alloc_txn_insert), KUNIT_CASE(stackdepot_trie_alloc_txn_insert_stale_plan), From b94f73bbccae3491c23f2ac1f5adbacf743c632d Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 10 Jun 2026 17:24:42 +0100 Subject: [PATCH 077/129] KRN-1117: Route stackdepot fetch to trie cache Make stack_depot_fetch() detect trie handles before falling back to legacy pool decoding, and return stackdepot-owned frames from the trie materialization cache. This keeps the pointer-returning frontend API working for trie handles without requiring callers to switch to stack_depot_fetch_into(). Teach stack_depot_put() to ignore trie handles because refcounted GET saves remain hash-backed. Cover public fetch, extra-bit decoding, cached reuse, and the trie put no-op in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 11 +++++++++ lib/tests/stackdepot_kunit.c | 44 ++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 8d65850aa65d9..3c28bc8dbb981 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -4912,7 +4912,9 @@ __stack_depot_trie_child_array_insert(const void *old_storage, const void *child unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries) { + const unsigned long *trie_entries; struct stack_record *stack; + unsigned int nr_entries; *entries = NULL; /* @@ -4923,6 +4925,13 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, if (!handle || stack_depot_disabled) return 0; + if (__stack_depot_trie_leaf_id(handle)) { + nr_entries = __stack_depot_trie_materialize_cached(handle, &trie_entries); + if (!nr_entries) + return 0; + *entries = (unsigned long *)trie_entries; + return nr_entries; + } stack = depot_fetch_stack(handle); /* @@ -4979,6 +4988,8 @@ void stack_depot_put(depot_stack_handle_t handle) if (!handle || stack_depot_disabled) return; + if (__stack_depot_trie_leaf_id(handle)) + return; stack = depot_fetch_stack(handle); /* diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index c7872a53f014b..8691da337f4a1 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1868,6 +1868,49 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) KUNIT_EXPECT_EQ(test, nr_sized, 0U); } +static void stackdepot_trie_fetch_public(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_root root = {}; + depot_stack_handle_t extra; + depot_stack_handle_t handle; + unsigned long *again; + unsigned long *frames; + unsigned int fetched; + u32 leaf_id; + + workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workspace); + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + + handle = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC, workspace); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + leaf_id = __stack_depot_trie_leaf_id(handle); + KUNIT_ASSERT_NE(test, leaf_id, 0U); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(leaf_id)); + + fetched = stack_depot_fetch(handle, &frames); + KUNIT_ASSERT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_ASSERT_NOT_NULL(test, frames); + KUNIT_EXPECT_MEMEQ(test, frames, entries, sizeof(entries)); + KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(leaf_id), + frames); + + extra = stack_depot_set_extra_bits(handle, + (1U << STACK_DEPOT_EXTRA_BITS) - 1); + fetched = stack_depot_fetch(extra, &again); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_PTR_EQ(test, again, frames); + + stack_depot_put(extra); + fetched = stack_depot_fetch(handle, &again); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_PTR_EQ(test, again, frames); +} + static void stackdepot_trie_materialize_cached(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; @@ -5651,6 +5694,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_save_miss_noalloc), KUNIT_CASE(stackdepot_trie_save), KUNIT_CASE(stackdepot_trie_fetch_handle_into), + KUNIT_CASE(stackdepot_trie_fetch_public), KUNIT_CASE(stackdepot_trie_materialize_cached), KUNIT_CASE(stackdepot_trie_alloc_txn_plan), KUNIT_CASE(stackdepot_trie_alloc_txn_insert), From 0a86543b5cd05068361cea0391ed9f4009fd8f42 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 11 Jun 2026 09:33:24 +0100 Subject: [PATCH 078/129] KRN-1117: Count stackdepot trie materialization bytes Expose the lazy materialization cache separately from existing persistent stack storage. This lets DOG validation distinguish trie compatibility storage from hash-backed persistent bytes and from the trie side-table allocation. Count materialized records only when the current CPU publishes a new cached frame pointer, and report both the record count and bytes in debugfs. Cover the first materialization and cached reuse paths in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 20 +++++++++++++++++++- lib/stackdepot_internal.h | 2 ++ lib/tests/stackdepot_kunit.c | 17 +++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 3c28bc8dbb981..611aadfa82f1c 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -184,6 +184,8 @@ enum depot_counter_id { DEPOT_COUNTER_FREELIST_SIZE, DEPOT_COUNTER_PERSIST_COUNT, DEPOT_COUNTER_PERSIST_BYTES, + DEPOT_COUNTER_TRIE_MATERIALIZED_COUNT, + DEPOT_COUNTER_TRIE_MATERIALIZED_BYTES, DEPOT_COUNTER_COUNT, }; static long counters[DEPOT_COUNTER_COUNT]; @@ -194,6 +196,8 @@ static const char *const counter_names[] = { [DEPOT_COUNTER_FREELIST_SIZE] = "freelist_size", [DEPOT_COUNTER_PERSIST_COUNT] = "persistent_count", [DEPOT_COUNTER_PERSIST_BYTES] = "persistent_bytes", + [DEPOT_COUNTER_TRIE_MATERIALIZED_COUNT] = "trie_materialized_count", + [DEPOT_COUNTER_TRIE_MATERIALIZED_BYTES] = "trie_materialized_bytes", }; static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); /* Count helpers rely on saturated refcounts looking negative. */ @@ -4166,6 +4170,15 @@ unsigned int __stack_depot_trie_materialized_count(const unsigned long *frames) return READ_ONCE(record->nr_entries); } +void +__stack_depot_trie_materialized_stats(unsigned long *count, unsigned long *bytes) +{ + if (count) + *count = READ_ONCE(counters[DEPOT_COUNTER_TRIE_MATERIALIZED_COUNT]); + if (bytes) + *bytes = READ_ONCE(counters[DEPOT_COUNTER_TRIE_MATERIALIZED_BYTES]); +} + unsigned int __stack_depot_trie_materialize_record(depot_stack_handle_t handle, struct stack_depot_trie_materialized *record, @@ -4240,8 +4253,11 @@ __stack_depot_trie_materialize_cached(depot_stack_handle_t handle, record = pool + offset; pool_offset += record_size; ret = __stack_depot_trie_materialize_record(handle, record, record_size, frames); - if (ret && *frames == record->entries) + if (ret && *frames == record->entries) { + counters[DEPOT_COUNTER_TRIE_MATERIALIZED_COUNT]++; + counters[DEPOT_COUNTER_TRIE_MATERIALIZED_BYTES] += record_size; goto out; + } pool_offset = offset; if (!*frames) @@ -5058,6 +5074,8 @@ static int stats_show(struct seq_file *seq, void *v) seq_printf(seq, "pools: %d\n", data_race(pools_num)); for (int i = 0; i < DEPOT_COUNTER_COUNT; i++) seq_printf(seq, "%s: %ld\n", counter_names[i], data_race(counters[i])); + seq_printf(seq, "trie_side_table_bytes: %zu\n", + __stack_depot_trie_side_table_bytes()); return 0; } diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 6deed172ca54f..107a7add2e535 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -331,6 +331,8 @@ size_t __stack_depot_trie_materialize_bytes(depot_stack_handle_t handle, unsigne size_t __stack_depot_trie_materialized_size(unsigned int nr_entries); unsigned int __stack_depot_trie_materialized_count(const unsigned long *frames); +void +__stack_depot_trie_materialized_stats(unsigned long *count, unsigned long *bytes); unsigned int __stack_depot_trie_materialize_record(depot_stack_handle_t handle, struct stack_depot_trie_materialized *record, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 8691da337f4a1..2feae08fcd56f 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1921,7 +1921,12 @@ static void stackdepot_trie_materialize_cached(struct kunit *test) depot_stack_handle_t handle; const unsigned long *again; const unsigned long *frames; + unsigned long bytes_after; + unsigned long bytes_before; + unsigned long count_after; + unsigned long count_before; unsigned int fetched; + size_t record_size; u32 leaf_id; workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); @@ -1935,6 +1940,9 @@ static void stackdepot_trie_materialize_cached(struct kunit *test) leaf_id = __stack_depot_trie_leaf_id(handle); KUNIT_ASSERT_NE(test, leaf_id, 0U); KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(leaf_id)); + record_size = __stack_depot_trie_materialized_size(ARRAY_SIZE(entries)); + KUNIT_ASSERT_NE(test, record_size, 0UL); + __stack_depot_trie_materialized_stats(&count_before, &bytes_before); frames = NULL; fetched = tmaterialize_cached(handle, &frames); @@ -1945,11 +1953,17 @@ static void stackdepot_trie_materialize_cached(struct kunit *test) (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(leaf_id), frames); + __stack_depot_trie_materialized_stats(&count_after, &bytes_after); + KUNIT_EXPECT_EQ(test, count_after, count_before + 1); + KUNIT_EXPECT_EQ(test, bytes_after, bytes_before + record_size); again = NULL; fetched = tmaterialize_cached(handle, &again); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_PTR_EQ(test, again, frames); + __stack_depot_trie_materialized_stats(&count_after, &bytes_after); + KUNIT_EXPECT_EQ(test, count_after, count_before + 1); + KUNIT_EXPECT_EQ(test, bytes_after, bytes_before + record_size); extra = stack_depot_set_extra_bits(handle, (1U << STACK_DEPOT_EXTRA_BITS) - 1); @@ -1957,6 +1971,9 @@ static void stackdepot_trie_materialize_cached(struct kunit *test) fetched = tmaterialize_cached(extra, &again); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_PTR_EQ(test, again, frames); + __stack_depot_trie_materialized_stats(&count_after, &bytes_after); + KUNIT_EXPECT_EQ(test, count_after, count_before + 1); + KUNIT_EXPECT_EQ(test, bytes_after, bytes_before + record_size); frames = (const unsigned long *)0x1UL; fetched = tmaterialize_cached(0, &frames); From dd3bf3e95970ce82d6a8f347116c3614663db36f Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 11 Jun 2026 10:31:06 +0100 Subject: [PATCH 079/129] KRN-1117: Add a locked stackdepot trie save helper Add a private trie save helper that serializes access to a caller-provided allocation workspace. Future public save routing needs this because the trie workspace is too large to place on the stack, especially with larger CONFIG_STACKDEPOT_MAX_FRAMES values. Keep preallocation outside the workspace lock, recheck for duplicates while holding the lock, and preserve trylock behavior for NMI and non-spinning contexts. Cover duplicate reuse, GET rejection, and locked insertion in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 126 +++++++++++++++++++++++++++++++++-- lib/stackdepot_internal.h | 7 ++ lib/tests/stackdepot_kunit.c | 50 ++++++++++++++ 3 files changed, 176 insertions(+), 7 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 611aadfa82f1c..8814138c5eb53 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1060,6 +1060,22 @@ static int trie_ws_insert(struct stack_depot_trie_root *root, workspace, tail, leaf_id); } +static depot_stack_handle_t +trie_find_handle(const struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries) +{ + depot_stack_handle_t handle = 0; + const struct stack_depot_trie_node *leaf; + + rcu_read_lock_sched_notrace(); + leaf = __stack_depot_trie_find_leaf(root, entries, nr_entries); + if (leaf) + handle = __stack_depot_trie_handle(leaf->leaf_id); + rcu_read_unlock_sched_notrace(); + + return handle; +} + static depot_stack_handle_t trie_save_miss(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags, @@ -1114,7 +1130,6 @@ __stack_depot_trie_save(struct stack_depot_trie_root *root, struct stack_depot_trie_alloc_workspace *workspace) { depot_stack_handle_t handle = 0; - const struct stack_depot_trie_node *leaf; if (!root || !entries || !nr_entries || !workspace) return 0; @@ -1123,18 +1138,115 @@ __stack_depot_trie_save(struct stack_depot_trie_root *root, if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) return 0; - rcu_read_lock_sched_notrace(); - leaf = __stack_depot_trie_find_leaf(root, entries, nr_entries); - if (leaf) - handle = __stack_depot_trie_handle(leaf->leaf_id); - rcu_read_unlock_sched_notrace(); - + handle = trie_find_handle(root, entries, nr_entries); if (handle) return handle; return trie_save_miss(root, entries, nr_entries, alloc_flags, depot_flags, workspace); } +static depot_stack_handle_t +trie_save_locked_insert(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_trie_alloc_workspace *workspace, + void **pool_prealloc, void **side_prealloc, + bool can_insert) +{ + depot_stack_handle_t handle; + const void *tail; + u32 leaf_id; + int ret; + + handle = trie_find_handle(root, entries, nr_entries); + if (!handle && can_insert) { + ret = trie_ws_insert(root, entries, nr_entries, pool_prealloc, + side_prealloc, workspace, &tail, &leaf_id); + if (!ret) + handle = __stack_depot_trie_handle(leaf_id); + } + + return handle; +} + +static depot_stack_handle_t +trie_save_trylocked(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_trie_alloc_workspace *workspace, + raw_spinlock_t *workspace_lock, void **pool_prealloc, + void **side_prealloc, bool can_insert) +{ + depot_stack_handle_t handle; + unsigned long flags; + + if (!raw_spin_trylock_irqsave(workspace_lock, flags)) + return 0; + handle = trie_save_locked_insert(root, entries, nr_entries, workspace, + pool_prealloc, side_prealloc, + can_insert); + raw_spin_unlock_irqrestore(workspace_lock, flags); + return handle; +} + +static depot_stack_handle_t +trie_save_spinlocked(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_trie_alloc_workspace *workspace, + raw_spinlock_t *workspace_lock, void **pool_prealloc, + void **side_prealloc, bool can_insert) +{ + depot_stack_handle_t handle; + unsigned long flags; + + raw_spin_lock_irqsave(workspace_lock, flags); + handle = trie_save_locked_insert(root, entries, nr_entries, workspace, + pool_prealloc, side_prealloc, + can_insert); + raw_spin_unlock_irqrestore(workspace_lock, flags); + return handle; +} + +depot_stack_handle_t +__stack_depot_trie_save_locked(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags, depot_flags_t depot_flags, + struct stack_depot_trie_alloc_workspace *workspace, + raw_spinlock_t *workspace_lock) +{ + depot_stack_handle_t handle = 0; + void *pool_prealloc = NULL; + void *side_prealloc = NULL; + bool can_insert; + int ret; + + if (!root || !entries || !nr_entries || !workspace || !workspace_lock) + return 0; + if (depot_flags & STACK_DEPOT_FLAG_GET) + return 0; + if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) + return 0; + + handle = trie_find_handle(root, entries, nr_entries); + if (handle) + return handle; + + ret = trie_prealloc(alloc_flags, depot_flags, &pool_prealloc, + &side_prealloc); + can_insert = !ret; + + if (in_nmi() || !gfpflags_allow_spinning(alloc_flags)) + handle = trie_save_trylocked(root, entries, nr_entries, workspace, + workspace_lock, &pool_prealloc, + &side_prealloc, can_insert); + else + handle = trie_save_spinlocked(root, entries, nr_entries, workspace, + workspace_lock, &pool_prealloc, + &side_prealloc, can_insert); + + __stack_depot_trie_pool_free_prealloc(pool_prealloc); + __stack_depot_trie_side_table_free_prealloc(side_prealloc); + return handle; +} + u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) { u32 leaf_id; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 107a7add2e535..42e5abeaec626 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -3,6 +3,7 @@ #define _STACKDEPOT_INTERNAL_H #include +#include #include #include @@ -217,6 +218,12 @@ __stack_depot_trie_save(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags, depot_flags_t depot_flags, struct stack_depot_trie_alloc_workspace *workspace); +depot_stack_handle_t +__stack_depot_trie_save_locked(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags, depot_flags_t depot_flags, + struct stack_depot_trie_alloc_workspace *workspace, + raw_spinlock_t *workspace_lock); int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn); int diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 2feae08fcd56f..1074395004db1 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1545,6 +1545,16 @@ static depot_stack_handle_t tsave(struct stack_depot_trie_root *root, depot_flags, workspace); } +static depot_stack_handle_t +tsave_locked(struct stack_depot_trie_root *root, const unsigned long *entries, + unsigned int nr_entries, gfp_t gfp_flags, depot_flags_t depot_flags, + struct stack_depot_trie_alloc_workspace *workspace, + raw_spinlock_t *workspace_lock) +{ + return __stack_depot_trie_save_locked(root, entries, nr_entries, gfp_flags, + depot_flags, workspace, workspace_lock); +} + static unsigned int tfetch_handle(depot_stack_handle_t handle, unsigned long *entries, unsigned int max_entries) { @@ -1757,6 +1767,45 @@ static void stackdepot_trie_save(struct kunit *test) KUNIT_EXPECT_EQ(test, invalid, (depot_stack_handle_t)0); } +static void stackdepot_trie_save_locked(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL }; + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_root root = {}; + raw_spinlock_t workspace_lock; + depot_stack_handle_t first; + depot_stack_handle_t get; + depot_stack_handle_t second; + unsigned long out[ARRAY_SIZE(entries)] = {}; + unsigned int fetched; + + workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workspace); + raw_spin_lock_init(&workspace_lock); + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + + first = tsave_locked(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC, workspace, &workspace_lock); + KUNIT_ASSERT_NE(test, first, (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); + fetched = tfetch_handle(first, out, ARRAY_SIZE(out)); + KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); + + second = tsave_locked(&root, entries, ARRAY_SIZE(entries), GFP_NOWAIT, 0, + workspace, &workspace_lock); + KUNIT_EXPECT_EQ(test, second, first); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); + + get = tsave_locked(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_GET, workspace, &workspace_lock); + KUNIT_EXPECT_EQ(test, get, (depot_stack_handle_t)0); + get = tsave_locked(NULL, entries, ARRAY_SIZE(entries), GFP_KERNEL, 0, + workspace, &workspace_lock); + KUNIT_EXPECT_EQ(test, get, (depot_stack_handle_t)0); +} + static void stackdepot_trie_fetch_handle_into(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; @@ -5710,6 +5759,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_save_miss), KUNIT_CASE(stackdepot_trie_save_miss_noalloc), KUNIT_CASE(stackdepot_trie_save), + KUNIT_CASE(stackdepot_trie_save_locked), KUNIT_CASE(stackdepot_trie_fetch_handle_into), KUNIT_CASE(stackdepot_trie_fetch_public), KUNIT_CASE(stackdepot_trie_materialize_cached), From 12d3d4936d77cd6ae4987b36667a1f1486a6aa9f Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 11 Jun 2026 11:07:32 +0100 Subject: [PATCH 080/129] KRN-1117: Initialize stackdepot trie side table Add side-table initialization for trie-backed stackdepot handles. Allocate the top-level side-table array during normal stackdepot init when trie storage is enabled, and add an early memblock path for boot-time trie enablement. Preallocate the first early side-table chunk so boot-time saves can allocate initial trie IDs before slab allocation is available. Fall back by disabling trie storage if side-table setup fails, leaving hash-backed stackdepot usable. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 137 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 118 insertions(+), 19 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 8814138c5eb53..720be647a9a65 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -287,6 +287,7 @@ static unsigned int trie_side_table_nr_chunks; static unsigned int trie_side_table_top_size; static u32 trie_side_table_next_id; static bool trie_side_table_initialized; +static bool trie_side_table_memblock; static unsigned int trie_side_table_top_index(u32 id) { @@ -312,6 +313,88 @@ trie_side_table_publish_chunk(unsigned int top, smp_store_release(&trie_side_table_chunks[top], chunk); } +static size_t trie_side_table_top_bytes(unsigned int top_size) +{ + size_t bytes; + + if (check_mul_overflow((size_t)top_size, + sizeof(*trie_side_table_chunks), &bytes)) + return 0; + return bytes; +} + +static size_t trie_side_table_chunk_bytes(void) +{ + return STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE * + sizeof(*trie_side_table_chunks[0]); +} + +static int +trie_side_table_install(struct stack_depot_trie_side_entry **chunks, + unsigned int top_size, + struct stack_depot_trie_side_entry *first_chunk, + bool memblock) +{ + if (READ_ONCE(trie_side_table_initialized)) + return 0; + if (!chunks || !top_size) + return -EINVAL; + + trie_side_table_chunks = chunks; + WRITE_ONCE(trie_side_table_top_size, top_size); + WRITE_ONCE(trie_side_table_high_water, 0); + WRITE_ONCE(trie_side_table_nr_chunks, 0); + WRITE_ONCE(trie_side_table_next_id, 0); + WRITE_ONCE(trie_side_table_memblock, memblock); + if (first_chunk) { + trie_side_table_publish_chunk(0, first_chunk); + WRITE_ONCE(trie_side_table_high_water, 1); + WRITE_ONCE(trie_side_table_nr_chunks, 1); + } + WRITE_ONCE(trie_side_table_initialized, true); + return 0; +} + +static unsigned int trie_side_table_top_size_for_max_id(u32 max_leaf_id) +{ + return DIV_ROUND_UP(max_leaf_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); +} + +static int __init __stack_depot_trie_side_table_init_memblock(void) +{ + struct stack_depot_trie_side_entry **chunks; + struct stack_depot_trie_side_entry *first_chunk; + size_t chunk_bytes; + size_t top_bytes; + u32 max_leaf_id; + unsigned int top_size; + + if (READ_ONCE(trie_side_table_initialized)) + return 0; + + max_leaf_id = __stack_depot_trie_max_leaf_id(); + if (!max_leaf_id) + return -EINVAL; + top_size = trie_side_table_top_size_for_max_id(max_leaf_id); + top_bytes = trie_side_table_top_bytes(top_size); + chunk_bytes = trie_side_table_chunk_bytes(); + if (!top_bytes || !chunk_bytes) + return -ENOMEM; + + chunks = memblock_alloc(top_bytes, PAGE_SIZE); + if (!chunks) + return -ENOMEM; + memset(chunks, 0, top_bytes); + first_chunk = memblock_alloc(chunk_bytes, PAGE_SIZE); + if (!first_chunk) { + memblock_free(chunks, top_bytes); + return -ENOMEM; + } + memset(first_chunk, 0, chunk_bytes); + + return trie_side_table_install(chunks, top_size, first_chunk, true); +} + static const void * trie_side_table_load_leaf(struct stack_depot_trie_side_entry *chunk, unsigned int slot) @@ -354,6 +437,9 @@ trie_side_table_clear_entry(struct stack_depot_trie_side_entry *chunk, int __stack_depot_trie_side_table_init(gfp_t gfp_flags) { + struct stack_depot_trie_side_entry **chunks; + unsigned int top_size; + size_t top_bytes; u32 max_leaf_id; if (READ_ONCE(trie_side_table_initialized)) @@ -363,36 +449,37 @@ int __stack_depot_trie_side_table_init(gfp_t gfp_flags) if (!max_leaf_id) return -EINVAL; - trie_side_table_top_size = - DIV_ROUND_UP(max_leaf_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); - trie_side_table_chunks = - kvcalloc(trie_side_table_top_size, sizeof(*trie_side_table_chunks), - gfp_flags); - if (!trie_side_table_chunks) + top_size = trie_side_table_top_size_for_max_id(max_leaf_id); + top_bytes = trie_side_table_top_bytes(top_size); + if (!top_bytes) + return -ENOMEM; + chunks = kvcalloc(top_size, sizeof(*chunks), gfp_flags); + if (!chunks) return -ENOMEM; - WRITE_ONCE(trie_side_table_high_water, 0); - WRITE_ONCE(trie_side_table_nr_chunks, 0); - WRITE_ONCE(trie_side_table_next_id, 0); - WRITE_ONCE(trie_side_table_initialized, true); - return 0; + return trie_side_table_install(chunks, top_size, NULL, false); } void __stack_depot_trie_side_table_destroy(void) { + unsigned int high_water; unsigned int i; if (!READ_ONCE(trie_side_table_initialized)) return; - for (i = 0; i < trie_side_table_high_water; i++) - kfree(trie_side_table_chunks[i]); - kvfree(trie_side_table_chunks); + high_water = READ_ONCE(trie_side_table_high_water); + if (!READ_ONCE(trie_side_table_memblock)) { + for (i = 0; i < high_water; i++) + kfree(trie_side_table_chunks[i]); + kvfree(trie_side_table_chunks); + } trie_side_table_chunks = NULL; WRITE_ONCE(trie_side_table_high_water, 0); WRITE_ONCE(trie_side_table_nr_chunks, 0); WRITE_ONCE(trie_side_table_top_size, 0); WRITE_ONCE(trie_side_table_next_id, 0); + WRITE_ONCE(trie_side_table_memblock, false); WRITE_ONCE(trie_side_table_initialized, false); } @@ -640,14 +727,13 @@ size_t __stack_depot_trie_side_table_bytes(void) if (!READ_ONCE(trie_side_table_initialized)) return 0; - if (check_mul_overflow((size_t)trie_side_table_top_size, - sizeof(*trie_side_table_chunks), &top_bytes)) + top_bytes = trie_side_table_top_bytes(trie_side_table_top_size); + if (!top_bytes) return SIZE_MAX; nr_chunks = READ_ONCE(trie_side_table_nr_chunks); - if (check_mul_overflow((size_t)nr_chunks, - STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE * - sizeof(*trie_side_table_chunks[0]), &bytes)) + if (check_mul_overflow((size_t)nr_chunks, trie_side_table_chunk_bytes(), + &bytes)) return SIZE_MAX; if (check_add_overflow(top_bytes, bytes, &bytes)) return SIZE_MAX; @@ -1511,6 +1597,11 @@ int __init stack_depot_early_init(void) stack_depot_disabled = true; return -ENOMEM; } + if (__stack_depot_trie_enabled() && + __stack_depot_trie_side_table_init_memblock()) { + pr_warn("trie side table allocation failed, disabling trie storage\n"); + __stack_depot_trie_set_enabled(false); + } return 0; } @@ -1572,6 +1663,14 @@ int stack_depot_init(void) stack_depot_disabled = true; ret = -ENOMEM; } + if (!ret && __stack_depot_trie_enabled()) { + ret = __stack_depot_trie_side_table_init(GFP_KERNEL); + if (ret) { + pr_warn("trie side table allocation failed, disabling trie storage\n"); + __stack_depot_trie_set_enabled(false); + ret = 0; + } + } out_unlock: mutex_unlock(&stack_depot_init_mutex); From a2ecf7b2fc824e3a33dd59237a5ddffdd2d52981 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 11 Jun 2026 11:50:27 +0100 Subject: [PATCH 081/129] KRN-1117: Initialize stackdepot trie workspace Add a global trie allocation workspace and initialize it with stackdepot. The public save path cannot place this workspace on the stack, so future trie save routing needs stable storage before it can return trie handles. Allocate the workspace from memblock for early trie enablement and from kvzalloc for normal init. Gate future routing with a private trie-ready helper and cover late trie initialization in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 115 +++++++++++++++++++++++++++++++++-- lib/stackdepot_internal.h | 1 + lib/tests/stackdepot_kunit.c | 14 +++++ 3 files changed, 124 insertions(+), 6 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 720be647a9a65..44f8a28969f43 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -56,6 +56,8 @@ static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_ST static bool __stack_depot_early_init_passed __initdata; static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); static bool stack_depot_trie_enabled_param; +static struct stack_depot_trie_alloc_workspace *stack_depot_trie_workspace; +static bool stack_depot_trie_ready; bool __stack_depot_trie_enabled(void) { @@ -289,6 +291,19 @@ static u32 trie_side_table_next_id; static bool trie_side_table_initialized; static bool trie_side_table_memblock; +static void stack_depot_trie_mark_not_ready(void) +{ + WRITE_ONCE(stack_depot_trie_ready, false); +} + +bool __stack_depot_trie_ready(void) +{ + return __stack_depot_trie_enabled() && + READ_ONCE(stack_depot_trie_ready) && + READ_ONCE(stack_depot_trie_workspace) && + READ_ONCE(trie_side_table_initialized); +} + static unsigned int trie_side_table_top_index(u32 id) { return (id - 1) >> STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS; @@ -395,6 +410,90 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) return trie_side_table_install(chunks, top_size, first_chunk, true); } +static size_t stack_depot_trie_workspace_size(void) +{ + return sizeof(*stack_depot_trie_workspace); +} + +static int +stack_depot_trie_install_workspace(struct stack_depot_trie_alloc_workspace *workspace) +{ + if (READ_ONCE(stack_depot_trie_workspace)) + return 0; + if (!workspace) + return -EINVAL; + + WRITE_ONCE(stack_depot_trie_workspace, workspace); + return 0; +} + +static int __init stack_depot_trie_init_workspace_memblock(void) +{ + struct stack_depot_trie_alloc_workspace *workspace; + size_t size; + + if (READ_ONCE(stack_depot_trie_workspace)) + return 0; + + size = stack_depot_trie_workspace_size(); + workspace = memblock_alloc(size, __alignof__(*workspace)); + if (!workspace) + return -ENOMEM; + memset(workspace, 0, size); + + return stack_depot_trie_install_workspace(workspace); +} + +static int stack_depot_trie_init_workspace(gfp_t gfp_flags) +{ + struct stack_depot_trie_alloc_workspace *workspace; + + if (READ_ONCE(stack_depot_trie_workspace)) + return 0; + + workspace = kvzalloc(stack_depot_trie_workspace_size(), gfp_flags); + if (!workspace) + return -ENOMEM; + + return stack_depot_trie_install_workspace(workspace); +} + +static int __init stack_depot_trie_init_memblock(void) +{ + int ret; + + if (!__stack_depot_trie_enabled()) + return 0; + + ret = stack_depot_trie_init_workspace_memblock(); + if (ret) + return ret; + ret = __stack_depot_trie_side_table_init_memblock(); + if (ret) + return ret; + + WRITE_ONCE(stack_depot_trie_ready, true); + return 0; +} + +static int stack_depot_trie_init(gfp_t gfp_flags) +{ + int ret; + + if (!__stack_depot_trie_enabled()) + return 0; + + ret = stack_depot_trie_init_workspace(gfp_flags); + if (ret) + return ret; + ret = __stack_depot_trie_side_table_init(gfp_flags); + if (ret) + return ret; + + WRITE_ONCE(stack_depot_trie_ready, true); + return 0; +} + static const void * trie_side_table_load_leaf(struct stack_depot_trie_side_entry *chunk, unsigned int slot) @@ -467,6 +566,7 @@ void __stack_depot_trie_side_table_destroy(void) if (!READ_ONCE(trie_side_table_initialized)) return; + stack_depot_trie_mark_not_ready(); high_water = READ_ONCE(trie_side_table_high_water); if (!READ_ONCE(trie_side_table_memblock)) { @@ -1597,9 +1697,8 @@ int __init stack_depot_early_init(void) stack_depot_disabled = true; return -ENOMEM; } - if (__stack_depot_trie_enabled() && - __stack_depot_trie_side_table_init_memblock()) { - pr_warn("trie side table allocation failed, disabling trie storage\n"); + if (__stack_depot_trie_enabled() && stack_depot_trie_init_memblock()) { + pr_warn("trie storage initialization failed, disabling trie storage\n"); __stack_depot_trie_set_enabled(false); } @@ -1615,8 +1714,10 @@ int stack_depot_init(void) mutex_lock(&stack_depot_init_mutex); - if (stack_depot_disabled || stack_table) + if (stack_depot_disabled) goto out_unlock; + if (stack_table) + goto init_trie; /* * Similarly to stack_depot_early_init, use stack_bucket_number_order @@ -1663,10 +1764,12 @@ int stack_depot_init(void) stack_depot_disabled = true; ret = -ENOMEM; } + +init_trie: if (!ret && __stack_depot_trie_enabled()) { - ret = __stack_depot_trie_side_table_init(GFP_KERNEL); + ret = stack_depot_trie_init(GFP_KERNEL); if (ret) { - pr_warn("trie side table allocation failed, disabling trie storage\n"); + pr_warn("trie storage initialization failed, disabling trie storage\n"); __stack_depot_trie_set_enabled(false); ret = 0; } diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 42e5abeaec626..8b99645077202 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -30,6 +30,7 @@ struct stack_depot_frame_run { static_assert(CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(unsigned long) <= U16_MAX); bool __stack_depot_trie_enabled(void); +bool __stack_depot_trie_ready(void); void __stack_depot_trie_set_enabled(bool enabled); struct stack_depot_trie_node_slot { diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 1074395004db1..d01662f465e02 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -601,6 +601,19 @@ static void stackdepot_trie_feature_flag(struct kunit *test) KUNIT_EXPECT_FALSE(test, __stack_depot_trie_enabled()); } +static void stackdepot_trie_late_init(struct kunit *test) +{ + stackdepot_trie_add_disable_action(test); + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + KUNIT_EXPECT_FALSE(test, __stack_depot_trie_ready()); + if (!__stack_depot_trie_max_leaf_id()) + kunit_skip(test, "trie handle namespace unavailable"); + + __stack_depot_trie_set_enabled(true); + KUNIT_EXPECT_EQ(test, stack_depot_init(), 0); + KUNIT_EXPECT_TRUE(test, __stack_depot_trie_ready()); +} + static void stackdepot_trie_side_table_destroy_action(void *data) { __stack_depot_trie_side_table_destroy(); @@ -5726,6 +5739,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_count_helpers), KUNIT_CASE(stackdepot_trie_handle_namespace), KUNIT_CASE(stackdepot_trie_feature_flag), + KUNIT_CASE(stackdepot_trie_late_init), KUNIT_CASE(stackdepot_trie_side_table_destroy_uninit), KUNIT_CASE(stackdepot_trie_side_table_alloc_store_lookup), KUNIT_CASE(stackdepot_trie_side_table_rejects_invalid_ids), From b60ebcb25440fff358aeda56f0d135c4de2af01e Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 11 Jun 2026 12:49:23 +0100 Subject: [PATCH 082/129] KRN-1117: Gate stackdepot trie allocation readiness Add a private readiness helper for future trie save routing. The public save path must only use trie storage when the trie workspace and side table are initialized, and when the current context can safely preallocate any needed side table chunks. Publish trie-ready and side-table-initialized state with release/acquire ordering so later routing can safely dereference the initialized state. Cover ready, not-ready, GET, noalloc, and non-spinning cases in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 77 +++++++++++++++++++++++++++--------- lib/stackdepot_internal.h | 1 + lib/tests/stackdepot_kunit.c | 14 +++++++ 3 files changed, 73 insertions(+), 19 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 44f8a28969f43..ce570a042033b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -291,6 +291,30 @@ static u32 trie_side_table_next_id; static bool trie_side_table_initialized; static bool trie_side_table_memblock; +static bool stack_depot_trie_is_ready(void) +{ + /* Pairs with stack_depot_trie_publish_ready(). */ + return smp_load_acquire(&stack_depot_trie_ready); +} + +static bool trie_side_table_is_initialized(void) +{ + /* Pairs with trie_side_table_publish_initialized(). */ + return smp_load_acquire(&trie_side_table_initialized); +} + +static void stack_depot_trie_publish_ready(void) +{ + /* Pairs with stack_depot_trie_is_ready(). */ + smp_store_release(&stack_depot_trie_ready, true); +} + +static void trie_side_table_publish_initialized(void) +{ + /* Pairs with trie_side_table_is_initialized(). */ + smp_store_release(&trie_side_table_initialized, true); +} + static void stack_depot_trie_mark_not_ready(void) { WRITE_ONCE(stack_depot_trie_ready, false); @@ -299,9 +323,9 @@ static void stack_depot_trie_mark_not_ready(void) bool __stack_depot_trie_ready(void) { return __stack_depot_trie_enabled() && - READ_ONCE(stack_depot_trie_ready) && + stack_depot_trie_is_ready() && READ_ONCE(stack_depot_trie_workspace) && - READ_ONCE(trie_side_table_initialized); + trie_side_table_is_initialized(); } static unsigned int trie_side_table_top_index(u32 id) @@ -350,7 +374,7 @@ trie_side_table_install(struct stack_depot_trie_side_entry **chunks, struct stack_depot_trie_side_entry *first_chunk, bool memblock) { - if (READ_ONCE(trie_side_table_initialized)) + if (trie_side_table_is_initialized()) return 0; if (!chunks || !top_size) return -EINVAL; @@ -366,7 +390,7 @@ trie_side_table_install(struct stack_depot_trie_side_entry **chunks, WRITE_ONCE(trie_side_table_high_water, 1); WRITE_ONCE(trie_side_table_nr_chunks, 1); } - WRITE_ONCE(trie_side_table_initialized, true); + trie_side_table_publish_initialized(); return 0; } @@ -384,7 +408,7 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) u32 max_leaf_id; unsigned int top_size; - if (READ_ONCE(trie_side_table_initialized)) + if (trie_side_table_is_initialized()) return 0; max_leaf_id = __stack_depot_trie_max_leaf_id(); @@ -472,7 +496,7 @@ static int __init stack_depot_trie_init_memblock(void) if (ret) return ret; - WRITE_ONCE(stack_depot_trie_ready, true); + stack_depot_trie_publish_ready(); return 0; } @@ -490,7 +514,7 @@ static int stack_depot_trie_init(gfp_t gfp_flags) if (ret) return ret; - WRITE_ONCE(stack_depot_trie_ready, true); + stack_depot_trie_publish_ready(); return 0; } @@ -541,7 +565,7 @@ int __stack_depot_trie_side_table_init(gfp_t gfp_flags) size_t top_bytes; u32 max_leaf_id; - if (READ_ONCE(trie_side_table_initialized)) + if (trie_side_table_is_initialized()) return 0; max_leaf_id = __stack_depot_trie_max_leaf_id(); @@ -564,7 +588,7 @@ void __stack_depot_trie_side_table_destroy(void) unsigned int high_water; unsigned int i; - if (!READ_ONCE(trie_side_table_initialized)) + if (!trie_side_table_is_initialized()) return; stack_depot_trie_mark_not_ready(); @@ -590,7 +614,7 @@ bool __stack_depot_trie_side_table_prealloc_needed(void) u32 id; unsigned int top; - if (!READ_ONCE(trie_side_table_initialized)) + if (!trie_side_table_is_initialized()) return false; raw_spin_lock_irqsave(&trie_side_table_lock, flags); @@ -612,6 +636,21 @@ bool __stack_depot_trie_side_table_prealloc_needed(void) return needed; } +bool __stack_depot_trie_can_alloc(gfp_t alloc_flags, depot_flags_t depot_flags) +{ + if (!__stack_depot_trie_ready()) + return false; + if (depot_flags & STACK_DEPOT_FLAG_GET) + return false; + if (!(depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC)) + return false; + if (!gfpflags_allow_spinning(alloc_flags)) + return false; + if (!__stack_depot_trie_side_table_prealloc_needed()) + return true; + return slab_is_available(); +} + void *__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags) { return kcalloc(STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE, @@ -630,7 +669,7 @@ u32 __stack_depot_trie_side_table_alloc_id(void **prealloc) u32 id; unsigned int top; - if (!READ_ONCE(trie_side_table_initialized)) + if (!trie_side_table_is_initialized()) return 0; raw_spin_lock_irqsave(&trie_side_table_lock, flags); @@ -669,7 +708,7 @@ void __stack_depot_trie_side_table_revoke_latest(u32 id) unsigned int slot; unsigned int top; - if (!READ_ONCE(trie_side_table_initialized) || !id || + if (!trie_side_table_is_initialized() || !id || id != READ_ONCE(trie_side_table_next_id)) return; @@ -697,7 +736,7 @@ void __stack_depot_trie_side_table_restore(u32 id, const void *entry) unsigned long flags; unsigned int top; - if (!READ_ONCE(trie_side_table_initialized) || !id) + if (!trie_side_table_is_initialized() || !id) return; raw_spin_lock_irqsave(&trie_side_table_lock, flags); @@ -726,7 +765,7 @@ int __stack_depot_trie_side_table_store(u32 id, const void *entry) unsigned int top; int ret = -EINVAL; - if (!READ_ONCE(trie_side_table_initialized) || !id || !entry) + if (!trie_side_table_is_initialized() || !id || !entry) return -EINVAL; raw_spin_lock_irqsave(&trie_side_table_lock, flags); @@ -752,7 +791,7 @@ const void *__stack_depot_trie_side_table_lookup(u32 id) struct stack_depot_trie_side_entry *chunk; unsigned int top; - if (!READ_ONCE(trie_side_table_initialized) || !id) + if (!trie_side_table_is_initialized() || !id) return NULL; top = trie_side_table_top_index(id); @@ -771,7 +810,7 @@ const unsigned long *__stack_depot_trie_side_table_frames(u32 id) struct stack_depot_trie_side_entry *chunk; unsigned int top; - if (!READ_ONCE(trie_side_table_initialized) || !id) + if (!trie_side_table_is_initialized() || !id) return NULL; top = trie_side_table_top_index(id); @@ -793,7 +832,7 @@ __stack_depot_trie_side_table_store_frames(u32 id, const unsigned long *frames) unsigned int slot; unsigned int top; - if (!READ_ONCE(trie_side_table_initialized) || !id || !frames) + if (!trie_side_table_is_initialized() || !id || !frames) return -EINVAL; if (id > READ_ONCE(trie_side_table_next_id)) @@ -815,7 +854,7 @@ __stack_depot_trie_side_table_store_frames(u32 id, const unsigned long *frames) size_t __stack_depot_trie_side_table_entries(void) { - return READ_ONCE(trie_side_table_initialized) ? + return trie_side_table_is_initialized() ? READ_ONCE(trie_side_table_next_id) : 0; } @@ -825,7 +864,7 @@ size_t __stack_depot_trie_side_table_bytes(void) size_t bytes; size_t top_bytes; - if (!READ_ONCE(trie_side_table_initialized)) + if (!trie_side_table_is_initialized()) return 0; top_bytes = trie_side_table_top_bytes(trie_side_table_top_size); if (!top_bytes) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 8b99645077202..facb094e36315 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -31,6 +31,7 @@ static_assert(CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(unsigned long) <= U16_MAX); bool __stack_depot_trie_enabled(void); bool __stack_depot_trie_ready(void); +bool __stack_depot_trie_can_alloc(gfp_t alloc_flags, depot_flags_t depot_flags); void __stack_depot_trie_set_enabled(bool enabled); struct stack_depot_trie_node_slot { diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index d01662f465e02..818f2e1685d31 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -603,15 +603,29 @@ static void stackdepot_trie_feature_flag(struct kunit *test) static void stackdepot_trie_late_init(struct kunit *test) { + depot_flags_t can_alloc_flag = STACK_DEPOT_FLAG_CAN_ALLOC; + gfp_t no_spin = GFP_NOWAIT & ~__GFP_RECLAIM; + bool can_alloc; + stackdepot_trie_add_disable_action(test); KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); KUNIT_EXPECT_FALSE(test, __stack_depot_trie_ready()); + can_alloc = __stack_depot_trie_can_alloc(GFP_KERNEL, can_alloc_flag); + KUNIT_EXPECT_FALSE(test, can_alloc); if (!__stack_depot_trie_max_leaf_id()) kunit_skip(test, "trie handle namespace unavailable"); __stack_depot_trie_set_enabled(true); KUNIT_EXPECT_EQ(test, stack_depot_init(), 0); KUNIT_EXPECT_TRUE(test, __stack_depot_trie_ready()); + can_alloc = __stack_depot_trie_can_alloc(GFP_KERNEL, can_alloc_flag); + KUNIT_EXPECT_TRUE(test, can_alloc); + can_alloc = __stack_depot_trie_can_alloc(GFP_KERNEL, 0); + KUNIT_EXPECT_FALSE(test, can_alloc); + can_alloc = __stack_depot_trie_can_alloc(no_spin, can_alloc_flag); + KUNIT_EXPECT_FALSE(test, can_alloc); + can_alloc = __stack_depot_trie_can_alloc(GFP_KERNEL, STACK_DEPOT_FLAG_GET); + KUNIT_EXPECT_FALSE(test, can_alloc); } static void stackdepot_trie_side_table_destroy_action(void *data) From 2a085a4ed1afc1d1bbfbfd6619864c7d278f0fcc Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 11 Jun 2026 13:31:37 +0100 Subject: [PATCH 083/129] KRN-1117: Print stackdepot trie handles directly Teach stack_depot_print() and stack_depot_snprint() to handle trie-backed handles without first materializing a contiguous frame array. This keeps these frontend APIs working while avoiding unnecessary persistent cache growth on the cold print path. Print trie frames in original stack order by looking up each absolute frame index under RCU. Cover stack_depot_snprint() output parity with stack_trace_snprint() in KUnit. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 120 +++++++++++++++++++++++++++++++++++ lib/tests/stackdepot_kunit.c | 36 +++++++++++ 2 files changed, 156 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index ce570a042033b..9a8819dcf473c 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -4326,6 +4326,118 @@ static unsigned int trie_walk_frames(const void *leaf, unsigned int total, return seen == total ? total : 0; } +static int trie_frame_at(const void *leaf, unsigned int index, + unsigned long *frame) +{ + const struct stack_depot_trie_node *node; + + if (!leaf || !frame) + return -EINVAL; + + for (node = leaf; node; node = trie_load_parent(node)) { + unsigned int start; + + if (node->run.nr_entries > node->stack_len) + return -EINVAL; + start = node->stack_len - node->run.nr_entries; + if (index < start || index >= node->stack_len) + continue; + return stack_depot_trie_node_frame(node, index - start, frame); + } + + return -EINVAL; +} + +static void trie_print_frames(const void *leaf, unsigned int nr_entries, + int spaces) +{ + unsigned int i; + + for (i = 0; i < nr_entries; i++) { + unsigned long frame; + + if (trie_frame_at(leaf, i, &frame)) + return; + pr_info("%*c%pS\n", 1 + spaces, ' ', (void *)frame); + } +} + +static int +trie_snprint_frames(char *buf, size_t size, const void *leaf, + unsigned int nr_entries, int spaces) +{ + unsigned int generated; + unsigned int total = 0; + unsigned int i; + + for (i = 0; i < nr_entries && size; i++) { + unsigned long frame; + + if (trie_frame_at(leaf, i, &frame)) + break; + generated = snprintf(buf, size, "%*c%pS\n", 1 + spaces, ' ', + (void *)frame); + total += generated; + if (generated >= size) { + buf += size; + size = 0; + } else { + buf += generated; + size -= generated; + } + } + + return total; +} + +static unsigned int trie_handle_leaf(depot_stack_handle_t handle, + const void **leaf) +{ + u32 leaf_id; + + if (!leaf) + return 0; + *leaf = NULL; + leaf_id = __stack_depot_trie_leaf_id(handle); + if (!leaf_id) + return 0; + *leaf = __stack_depot_trie_side_table_lookup(leaf_id); + if (WARN(!*leaf, "corrupt trie handle %08x\n", handle)) + return 0; + return trie_validate_leaf(*leaf, NULL); +} + +static unsigned int trie_print_handle(depot_stack_handle_t handle, int spaces) +{ + unsigned int nr_entries; + const void *leaf; + + rcu_read_lock_sched_notrace(); + nr_entries = trie_handle_leaf(handle, &leaf); + if (nr_entries) + trie_print_frames(leaf, nr_entries, spaces); + rcu_read_unlock_sched_notrace(); + + return nr_entries; +} + +static int +trie_snprint_handle(depot_stack_handle_t handle, char *buf, size_t size, + int spaces) +{ + unsigned int nr_entries; + const void *leaf; + int ret = 0; + + rcu_read_lock_sched_notrace(); + nr_entries = trie_handle_leaf(handle, &leaf); + if (nr_entries) + ret = trie_snprint_frames(buf, size, leaf, nr_entries, spaces); + rcu_read_unlock_sched_notrace(); + + return ret; +} + static void trie_fetch_frame(unsigned int index, unsigned long frame, void *data) { struct stack_depot_trie_fetch_ctx *ctx = data; @@ -5378,6 +5490,11 @@ void stack_depot_print(depot_stack_handle_t stack) unsigned long *entries; unsigned int nr_entries; + if (__stack_depot_trie_leaf_id(stack)) { + trie_print_handle(stack, 0); + return; + } + nr_entries = stack_depot_fetch(stack, &entries); if (nr_entries > 0) stack_trace_print(entries, nr_entries, 0); @@ -5390,6 +5507,9 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, unsigned long *entries; unsigned int nr_entries; + if (__stack_depot_trie_leaf_id(handle)) + return trie_snprint_handle(handle, buf, size, spaces); + nr_entries = stack_depot_fetch(handle, &entries); return nr_entries ? stack_trace_snprint(buf, size, entries, nr_entries, spaces) : 0; diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 818f2e1685d31..80b7b8d060d1e 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "../stackdepot_internal.h" @@ -1987,6 +1988,40 @@ static void stackdepot_trie_fetch_public(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, again, frames); } +static void stackdepot_trie_snprint_public(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + struct stack_depot_trie_alloc_workspace *workspace; + char expected[256]; + char actual[256]; + struct stack_depot_trie_root root = {}; + depot_stack_handle_t extra; + depot_stack_handle_t handle; + unsigned int expected_len; + int actual_len; + u32 leaf_id; + + workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workspace); + stackdepot_trie_side_table_init_or_skip(test); + stackdepot_trie_pool_seed_current_pool(test); + + handle = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC, workspace); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + leaf_id = __stack_depot_trie_leaf_id(handle); + KUNIT_ASSERT_NE(test, leaf_id, 0U); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(leaf_id)); + + expected_len = stack_trace_snprint(expected, sizeof(expected), entries, + ARRAY_SIZE(entries), 2); + extra = stack_depot_set_extra_bits(handle, 7); + actual_len = stack_depot_snprint(extra, actual, sizeof(actual), 2); + KUNIT_EXPECT_EQ(test, actual_len, (int)expected_len); + KUNIT_EXPECT_STREQ(test, actual, expected); + KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(leaf_id)); +} + static void stackdepot_trie_materialize_cached(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; @@ -5790,6 +5825,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_save_locked), KUNIT_CASE(stackdepot_trie_fetch_handle_into), KUNIT_CASE(stackdepot_trie_fetch_public), + KUNIT_CASE(stackdepot_trie_snprint_public), KUNIT_CASE(stackdepot_trie_materialize_cached), KUNIT_CASE(stackdepot_trie_alloc_txn_plan), KUNIT_CASE(stackdepot_trie_alloc_txn_insert), From ab20790c4c8c1f0ab91f27aa714ea5fdbbb9d132 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 11 Jun 2026 15:31:08 +0100 Subject: [PATCH 084/129] KRN-1117: Route stackdepot saves to trie storage Route eligible public save misses to trie storage once the trie workspace and side table are ready. Keep hash lookup first so stacks saved before trie enablement retain their existing handles, and return 0 instead of falling back to hash when an eligible trie miss cannot preallocate safely. Add STACK_DEPOT_FLAG_HASH for internal callers that must keep hash-backed records because they use stackdepot count helpers. Use it for page_owner, whose page accounting depends on struct stack_record counts, and cover hash-first, trie, GET, HASH, overlong, and noalloc routing in KUnit. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 14 +++++-- lib/stackdepot.c | 40 +++++++++++++++---- lib/tests/stackdepot_kunit.c | 75 ++++++++++++++++++++++++++++++++++++ mm/page_owner.c | 15 ++++---- 4 files changed, 126 insertions(+), 18 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index a460cf0407abd..255cae3d74a78 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -47,8 +47,9 @@ typedef u32 depot_flags_t; */ #define STACK_DEPOT_FLAG_CAN_ALLOC ((depot_flags_t)0x0001) #define STACK_DEPOT_FLAG_GET ((depot_flags_t)0x0002) +#define STACK_DEPOT_FLAG_HASH ((depot_flags_t)0x0004) -#define STACK_DEPOT_FLAGS_NUM 2 +#define STACK_DEPOT_FLAGS_NUM 3 #define STACK_DEPOT_FLAGS_MASK ((depot_flags_t)((1 << STACK_DEPOT_FLAGS_NUM) - 1)) /* @@ -107,6 +108,11 @@ static inline int stack_depot_early_init(void) { return 0; } * Users of this flag must also call stack_depot_put() when keeping the stack * trace is no longer required to avoid overflowing the refcount. * + * If STACK_DEPOT_FLAG_HASH is set in @depot_flags, stack depot stores the stack + * trace in legacy hash storage even when trie storage is enabled. This is for + * internal callers that depend on stackdepot count helpers. This flag does not + * imply %STACK_DEPOT_FLAG_CAN_ALLOC. + * * If the provided stack trace comes from the interrupt context, only the part * up to the interrupt entry is saved. * @@ -273,7 +279,7 @@ void stack_depot_print(depot_stack_handle_t stack); * Return: Number of bytes printed */ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, - int spaces); + int spaces); /** * stack_depot_put - Drop a reference to a stack trace from stack depot @@ -298,8 +304,8 @@ void stack_depot_put(depot_stack_handle_t handle); * Stack depot handles have a few unused bits, which can be used for storing * user-specific information. These bits are transparent to the stack depot. */ -depot_stack_handle_t __must_check stack_depot_set_extra_bits( - depot_stack_handle_t handle, unsigned int extra_bits); +depot_stack_handle_t __must_check stack_depot_set_extra_bits(depot_stack_handle_t handle, + unsigned int extra_bits); /** * stack_depot_get_extra_bits - Retrieve extra bits from a stack depot handle diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 9a8819dcf473c..4ff94ca2941eb 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -52,11 +52,14 @@ static unsigned int stack_max_pools __read_mostly = MIN((1LL << DEPOT_POOL_INDEX_BITS) - 1, 8192); static bool stack_depot_disabled; -static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT); +static bool __stack_depot_early_init_requested __initdata = + IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT); static bool __stack_depot_early_init_passed __initdata; static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); static bool stack_depot_trie_enabled_param; +static struct stack_depot_trie_root stack_depot_trie_root; static struct stack_depot_trie_alloc_workspace *stack_depot_trie_workspace; +static DEFINE_RAW_SPINLOCK(stack_depot_trie_workspace_lock); static bool stack_depot_trie_ready; bool __stack_depot_trie_enabled(void) @@ -190,6 +193,7 @@ enum depot_counter_id { DEPOT_COUNTER_TRIE_MATERIALIZED_BYTES, DEPOT_COUNTER_COUNT, }; + static long counters[DEPOT_COUNTER_COUNT]; static const char *const counter_names[] = { [DEPOT_COUNTER_REFD_ALLOCS] = "refcounted_allocations", @@ -201,6 +205,7 @@ static const char *const counter_names[] = { [DEPOT_COUNTER_TRIE_MATERIALIZED_COUNT] = "trie_materialized_count", [DEPOT_COUNTER_TRIE_MATERIALIZED_BYTES] = "trie_materialized_bytes", }; + static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); /* Count helpers rely on saturated refcounts looking negative. */ static_assert(REFCOUNT_SATURATED < 0); @@ -640,7 +645,7 @@ bool __stack_depot_trie_can_alloc(gfp_t alloc_flags, depot_flags_t depot_flags) { if (!__stack_depot_trie_ready()) return false; - if (depot_flags & STACK_DEPOT_FLAG_GET) + if (depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH)) return false; if (!(depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC)) return false; @@ -2165,6 +2170,16 @@ static inline struct stack_record *find_stack(struct list_head *bucket, return ret; } +static depot_stack_handle_t +stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags, depot_flags_t depot_flags) +{ + return __stack_depot_trie_save_locked(&stack_depot_trie_root, entries, + nr_entries, alloc_flags, depot_flags, + stack_depot_trie_workspace, + &stack_depot_trie_workspace_lock); +} + depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags, @@ -2203,6 +2218,17 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, found = find_stack(bucket, entries, nr_entries, hash, depot_flags); if (found) goto exit; + if (__stack_depot_trie_ready() && + !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH)) && + nr_entries <= CONFIG_STACKDEPOT_MAX_FRAMES) { + handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); + if (handle) + return handle; + if (!__stack_depot_trie_can_alloc(alloc_flags, depot_flags)) + return 0; + handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, depot_flags); + return handle; + } /* * Allocate memory for a new pool if required now: @@ -5477,7 +5503,7 @@ void stack_depot_put(depot_stack_handle_t handle) * Should always be able to find the stack record, otherwise this is an * unbalanced put attempt (or corrupt handle). */ - if (WARN(!stack, "corrupt handle or unbalanced stack_depot_put()")) + if (WARN(!stack, "corrupt handle or unbalanced %s()", __func__)) return; if (refcount_dec_and_test(&stack->count)) @@ -5502,7 +5528,7 @@ void stack_depot_print(depot_stack_handle_t stack) EXPORT_SYMBOL_GPL(stack_depot_print); int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, - int spaces) + int spaces) { unsigned long *entries; unsigned int nr_entries; @@ -5516,8 +5542,8 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, } EXPORT_SYMBOL_GPL(stack_depot_snprint); -depot_stack_handle_t __must_check stack_depot_set_extra_bits( - depot_stack_handle_t handle, unsigned int extra_bits) +depot_stack_handle_t __must_check stack_depot_set_extra_bits(depot_stack_handle_t handle, + unsigned int extra_bits) { union handle_parts parts = { .handle = handle }; @@ -5546,7 +5572,7 @@ static int stats_show(struct seq_file *seq, void *v) */ seq_printf(seq, "pools: %d\n", data_race(pools_num)); for (int i = 0; i < DEPOT_COUNTER_COUNT; i++) - seq_printf(seq, "%s: %ld\n", counter_names[i], data_race(counters[i])); + seq_printf(seq, "%s: %ld\n", counter_names[i], READ_ONCE(counters[i])); seq_printf(seq, "trie_side_table_bytes: %zu\n", __stack_depot_trie_side_table_bytes()); diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 80b7b8d060d1e..58c7624579d9c 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -5782,6 +5782,80 @@ static void stackdepot_trie_child_array_insert_empty(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array, 0x1000UL), child); } +static void stackdepot_trie_public_save_route(struct kunit *test) +{ + unsigned long hash_entries[] = { 0x401000UL, 0x402000UL }; + unsigned long trie_entries[] = { 0x501000UL, 0x502000UL, 0x503000UL }; + unsigned long get_entries[] = { 0x601000UL, 0x602000UL }; + unsigned long noalloc_entries[] = { 0x701000UL, 0x702000UL }; + unsigned long hash_flag_entries[] = { 0x901000UL, 0x902000UL }; + unsigned long fetched[ARRAY_SIZE(trie_entries)] = {}; + depot_stack_handle_t get_handle; + depot_stack_handle_t hash_flag; + depot_stack_handle_t hash_again; + depot_stack_handle_t hash_handle; + depot_stack_handle_t noalloc_handle; + depot_stack_handle_t overlong_handle; + depot_stack_handle_t trie_again; + depot_stack_handle_t trie_handle; + depot_flags_t get_flags; + unsigned int hash_flag_nr = ARRAY_SIZE(hash_flag_entries); + gfp_t no_spin = GFP_NOWAIT & ~__GFP_RECLAIM; + unsigned int get_nr = ARRAY_SIZE(get_entries); + unsigned int noalloc_nr = ARRAY_SIZE(noalloc_entries); + unsigned int nr_entries; + unsigned long *overlong_entries; + unsigned int overlong_nr = CONFIG_STACKDEPOT_MAX_FRAMES + 1; + unsigned int i; + + stackdepot_trie_add_disable_action(test); + overlong_entries = kunit_kcalloc(test, overlong_nr, sizeof(*overlong_entries), + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, overlong_entries); + for (i = 0; i < overlong_nr; i++) + overlong_entries[i] = 0x800000UL + i * 0x1000UL; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + hash_handle = stack_depot_save(hash_entries, ARRAY_SIZE(hash_entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(hash_handle), 0U); + if (!__stack_depot_trie_max_leaf_id()) + kunit_skip(test, "trie handle namespace unavailable"); + + __stack_depot_trie_set_enabled(true); + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + KUNIT_EXPECT_TRUE(test, __stack_depot_trie_ready()); + hash_again = stack_depot_save(hash_entries, ARRAY_SIZE(hash_entries), GFP_KERNEL); + KUNIT_EXPECT_EQ(test, hash_again, hash_handle); + + trie_handle = stack_depot_save(trie_entries, ARRAY_SIZE(trie_entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, trie_handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_NE(test, __stack_depot_trie_leaf_id(trie_handle), 0U); + trie_again = stack_depot_save(trie_entries, ARRAY_SIZE(trie_entries), GFP_KERNEL); + KUNIT_EXPECT_EQ(test, trie_again, trie_handle); + nr_entries = stack_depot_fetch_into(trie_handle, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(trie_entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, trie_entries, sizeof(trie_entries)); + noalloc_handle = stack_depot_save_flags(noalloc_entries, noalloc_nr, no_spin, 0); + KUNIT_EXPECT_EQ(test, noalloc_handle, (depot_stack_handle_t)0); + noalloc_handle = stack_depot_save(noalloc_entries, noalloc_nr, GFP_KERNEL); + KUNIT_ASSERT_NE(test, noalloc_handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_NE(test, __stack_depot_trie_leaf_id(noalloc_handle), 0U); + + get_flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; + get_handle = stack_depot_save_flags(get_entries, get_nr, GFP_KERNEL, get_flags); + KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(get_handle), 0U); + stack_depot_put(get_handle); + get_flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_HASH; + hash_flag = stack_depot_save_flags(hash_flag_entries, hash_flag_nr, GFP_KERNEL, get_flags); + KUNIT_ASSERT_NE(test, hash_flag, (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(hash_flag), 0U); + overlong_handle = stack_depot_save(overlong_entries, overlong_nr, GFP_KERNEL); + KUNIT_ASSERT_NE(test, overlong_handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(overlong_handle), 0U); +} + static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), @@ -5945,6 +6019,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_split_subtree_prefix_leaf), KUNIT_CASE(stackdepot_trie_split_subtree_preserves_children), KUNIT_CASE(stackdepot_trie_child_array_insert_empty), + KUNIT_CASE(stackdepot_trie_public_save_route), {} }; diff --git a/mm/page_owner.c b/mm/page_owner.c index c497427629525..dbdb7c9fe4903 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -99,7 +99,8 @@ static __always_inline depot_stack_handle_t create_dummy_stack(void) unsigned int nr_entries; nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); - return stack_depot_save(entries, nr_entries, GFP_KERNEL); + return stack_depot_save_flags(entries, nr_entries, GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_HASH); } static noinline void register_dummy_stack(void) @@ -163,7 +164,8 @@ static noinline depot_stack_handle_t save_stack(gfp_t flags) set_current_in_page_owner(); nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); - handle = stack_depot_save(entries, nr_entries, flags); + handle = stack_depot_save_flags(entries, nr_entries, flags, + STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_HASH); if (!handle) handle = failure_handle; unset_current_in_page_owner(); @@ -580,8 +582,8 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, static ssize_t print_page_owner(char __user *buf, size_t count, unsigned long pfn, - struct page *page, struct page_owner *page_owner, - depot_stack_handle_t handle) + struct page *page, struct page_owner *page_owner, + depot_stack_handle_t handle) { int ret, pageblock_mt, page_mt; char *kbuf; @@ -680,13 +682,13 @@ void __dump_page_owner(const struct page *page) pr_alert("page_owner free stack trace missing\n"); } else { pr_alert("page last free pid %d tgid %d stack trace:\n", - page_owner->free_pid, page_owner->free_tgid); + page_owner->free_pid, page_owner->free_tgid); stack_depot_print(handle); } if (page_owner->last_migrate_reason != -1) pr_alert("page has been migrated, last migrate reason: %s\n", - migrate_reason_names[page_owner->last_migrate_reason]); + migrate_reason_names[page_owner->last_migrate_reason]); page_ext_put(page_ext); } @@ -1014,7 +1016,6 @@ static int page_owner_threshold_set(void *data, u64 val) DEFINE_SIMPLE_ATTRIBUTE(proc_page_owner_threshold, &page_owner_threshold_get, &page_owner_threshold_set, "%llu"); - static int __init pageowner_init(void) { struct dentry *dir; From 2c373ef22c309a712ec3230b5beee8826cbe5837 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 12 Jun 2026 11:16:59 +0100 Subject: [PATCH 085/129] KRN-1117: Migrate persistent stackdepot fetches to copy-out Move persistent stackdepot users away from the pointer-returning stack_depot_fetch() API and onto stack_depot_fetch_into(). Trie-backed persistent stacks do not have contiguous frame storage, so returning a stable unsigned long pointer would require a flat materialization cache that gives back memory savings or can fail later under pool pressure. Keep stack_depot_fetch() as the legacy hash-backed pointer API, remove the trie materialization cache, and walk trie handles directly for print and snprint. This leaves refcounted and explicitly hash-backed users on the original hash storage while allowing persistent users to use trie storage without extra flat copies. Signed-off-by: Caleb Kan --- drivers/gpu/drm/drm_modeset_lock.c | 4 +- include/linux/stackdepot.h | 5 + lib/stackdepot.c | 499 ++++++----------------------- lib/stackdepot_internal.h | 36 +-- lib/tests/stackdepot_kunit.c | 240 +------------- mm/kmemleak.c | 4 +- mm/kmsan/kmsan_test.c | 7 +- mm/kmsan/report.c | 17 +- mm/slub.c | 30 +- 9 files changed, 136 insertions(+), 706 deletions(-) diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c index beb91a13a3124..77b3d71e35fd5 100644 --- a/drivers/gpu/drm/drm_modeset_lock.c +++ b/drivers/gpu/drm/drm_modeset_lock.c @@ -94,7 +94,7 @@ static noinline depot_stack_handle_t __drm_stack_depot_save(void) static void __drm_stack_depot_print(depot_stack_handle_t stack_depot) { struct drm_printer p = drm_dbg_printer(NULL, DRM_UT_KMS, "drm_modeset_lock"); - unsigned long *entries; + unsigned long entries[8]; unsigned int nr_entries; char *buf; @@ -102,7 +102,7 @@ static void __drm_stack_depot_print(depot_stack_handle_t stack_depot) if (!buf) return; - nr_entries = stack_depot_fetch(stack_depot, &entries); + nr_entries = stack_depot_fetch_into(stack_depot, entries, ARRAY_SIZE(entries)); stack_trace_snprint(buf, PAGE_SIZE, entries, nr_entries, 2); drm_printf(&p, "attempting to lock a contended lock without backoff:\n%s", buf); diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 255cae3d74a78..288943fc23b04 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -227,6 +227,11 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, * @handle: Stack depot handle returned from stack_depot_save() * @entries: Pointer to store the address of the stack trace * + * This helper returns a pointer to stackdepot-owned contiguous storage for + * legacy hash-backed handles. Callers that need backend-independent access to + * stack contents should use stack_depot_fetch_into(), stack_depot_print(), or + * stack_depot_snprint(). + * * Return: Number of frames for the fetched stack */ unsigned int stack_depot_fetch(depot_stack_handle_t handle, diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 4ff94ca2941eb..3dac95e1086ab 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -189,8 +189,6 @@ enum depot_counter_id { DEPOT_COUNTER_FREELIST_SIZE, DEPOT_COUNTER_PERSIST_COUNT, DEPOT_COUNTER_PERSIST_BYTES, - DEPOT_COUNTER_TRIE_MATERIALIZED_COUNT, - DEPOT_COUNTER_TRIE_MATERIALIZED_BYTES, DEPOT_COUNTER_COUNT, }; @@ -202,8 +200,6 @@ static const char *const counter_names[] = { [DEPOT_COUNTER_FREELIST_SIZE] = "freelist_size", [DEPOT_COUNTER_PERSIST_COUNT] = "persistent_count", [DEPOT_COUNTER_PERSIST_BYTES] = "persistent_bytes", - [DEPOT_COUNTER_TRIE_MATERIALIZED_COUNT] = "trie_materialized_count", - [DEPOT_COUNTER_TRIE_MATERIALIZED_BYTES] = "trie_materialized_bytes", }; static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); @@ -211,6 +207,7 @@ static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); static_assert(REFCOUNT_SATURATED < 0); static bool depot_init_pool(void **prealloc); +static void depot_try_keep_new_pool(void **prealloc); static u32 stack_depot_pool_index_mask(void) { @@ -283,7 +280,6 @@ u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) struct stack_depot_trie_side_entry { const void *leaf; - const unsigned long *frames; }; static struct stack_depot_trie_side_entry **trie_side_table_chunks; @@ -369,8 +365,19 @@ static size_t trie_side_table_top_bytes(unsigned int top_size) static size_t trie_side_table_chunk_bytes(void) { - return STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE * - sizeof(*trie_side_table_chunks[0]); + return PAGE_ALIGN(STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE * + sizeof(*trie_side_table_chunks[0])); +} + +static unsigned int trie_side_table_chunk_order(void) +{ + return get_order(trie_side_table_chunk_bytes()); +} + +static void trie_side_table_free_chunk(struct stack_depot_trie_side_entry *chunk) +{ + if (chunk) + free_pages((unsigned long)chunk, trie_side_table_chunk_order()); } static int @@ -539,27 +546,10 @@ trie_side_table_store_leaf(struct stack_depot_trie_side_entry *chunk, smp_store_release(&chunk[slot].leaf, leaf); } -static const unsigned long * -trie_side_table_load_frames(struct stack_depot_trie_side_entry *chunk, - unsigned int slot) -{ - /* Pairs with trie_side_table_store_frames(). */ - return smp_load_acquire(&chunk[slot].frames); -} - -static void -trie_side_table_store_frames(struct stack_depot_trie_side_entry *chunk, - unsigned int slot, const unsigned long *frames) -{ - /* Pairs with trie_side_table_load_frames(). */ - smp_store_release(&chunk[slot].frames, frames); -} - static void trie_side_table_clear_entry(struct stack_depot_trie_side_entry *chunk, unsigned int slot) { - trie_side_table_store_frames(chunk, slot, NULL); trie_side_table_store_leaf(chunk, slot, NULL); } @@ -600,7 +590,7 @@ void __stack_depot_trie_side_table_destroy(void) high_water = READ_ONCE(trie_side_table_high_water); if (!READ_ONCE(trie_side_table_memblock)) { for (i = 0; i < high_water; i++) - kfree(trie_side_table_chunks[i]); + trie_side_table_free_chunk(trie_side_table_chunks[i]); kvfree(trie_side_table_chunks); } trie_side_table_chunks = NULL; @@ -641,30 +631,18 @@ bool __stack_depot_trie_side_table_prealloc_needed(void) return needed; } -bool __stack_depot_trie_can_alloc(gfp_t alloc_flags, depot_flags_t depot_flags) -{ - if (!__stack_depot_trie_ready()) - return false; - if (depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH)) - return false; - if (!(depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC)) - return false; - if (!gfpflags_allow_spinning(alloc_flags)) - return false; - if (!__stack_depot_trie_side_table_prealloc_needed()) - return true; - return slab_is_available(); -} - void *__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags) { - return kcalloc(STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE, - sizeof(*trie_side_table_chunks[0]), gfp_flags); + struct page *page; + + page = alloc_pages(gfp_nested_mask(gfp_flags) | __GFP_ZERO, + trie_side_table_chunk_order()); + return page ? page_address(page) : NULL; } void __stack_depot_trie_side_table_free_prealloc(void *prealloc) { - kfree(prealloc); + trie_side_table_free_chunk(prealloc); } u32 __stack_depot_trie_side_table_alloc_id(void **prealloc) @@ -810,53 +788,6 @@ const void *__stack_depot_trie_side_table_lookup(u32 id) return trie_side_table_load_leaf(chunk, trie_side_table_slot_index(id)); } -const unsigned long *__stack_depot_trie_side_table_frames(u32 id) -{ - struct stack_depot_trie_side_entry *chunk; - unsigned int top; - - if (!trie_side_table_is_initialized() || !id) - return NULL; - - top = trie_side_table_top_index(id); - if (top >= trie_side_table_top_size) - return NULL; - - chunk = trie_side_table_load_chunk(top); - if (!chunk) - return NULL; - - return trie_side_table_load_frames(chunk, trie_side_table_slot_index(id)); -} - -int -__stack_depot_trie_side_table_store_frames(u32 id, const unsigned long *frames) -{ - struct stack_depot_trie_side_entry *chunk; - const unsigned long *old; - unsigned int slot; - unsigned int top; - - if (!trie_side_table_is_initialized() || !id || !frames) - return -EINVAL; - - if (id > READ_ONCE(trie_side_table_next_id)) - return -EINVAL; - top = trie_side_table_top_index(id); - if (top >= trie_side_table_top_size) - return -EINVAL; - - chunk = trie_side_table_load_chunk(top); - if (!chunk) - return -EINVAL; - slot = trie_side_table_slot_index(id); - if (!trie_side_table_load_leaf(chunk, slot)) - return -EINVAL; - - old = cmpxchg_release(&chunk[slot].frames, NULL, frames); - return old ? -EEXIST : 0; -} - size_t __stack_depot_trie_side_table_entries(void) { return trie_side_table_is_initialized() ? @@ -1338,6 +1269,7 @@ trie_save_miss(struct stack_depot_trie_root *root, const unsigned long *entries, handle = __stack_depot_trie_handle(leaf_id); out: + depot_try_keep_new_pool(&pool_prealloc); __stack_depot_trie_pool_free_prealloc(pool_prealloc); __stack_depot_trie_side_table_free_prealloc(side_prealloc); return handle; @@ -1892,6 +1824,19 @@ static void depot_keep_new_pool(void **prealloc) *prealloc = NULL; } +static void depot_try_keep_new_pool(void **prealloc) +{ + unsigned long flags; + + if (!prealloc || !*prealloc) + return; + + if (!raw_spin_trylock_irqsave(&pool_lock, flags)) + return; + depot_keep_new_pool(prealloc); + raw_spin_unlock_irqrestore(&pool_lock, flags); +} + /* * Try to initialize a new stack record from the current pool, a cached pool, or * the current pre-allocation. @@ -2224,8 +2169,6 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); if (handle) return handle; - if (!__stack_depot_trie_can_alloc(alloc_flags, depot_flags)) - return 0; handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, depot_flags); return handle; } @@ -4352,116 +4295,90 @@ static unsigned int trie_walk_frames(const void *leaf, unsigned int total, return seen == total ? total : 0; } -static int trie_frame_at(const void *leaf, unsigned int index, - unsigned long *frame) +static unsigned int +trie_walk_handle(depot_stack_handle_t handle, trie_frame_fn_t fn, void *data) { - const struct stack_depot_trie_node *node; - - if (!leaf || !frame) - return -EINVAL; + const void *leaf; + unsigned int walked = 0; + unsigned int total; + u32 leaf_id; - for (node = leaf; node; node = trie_load_parent(node)) { - unsigned int start; + leaf_id = __stack_depot_trie_leaf_id(handle); + if (!leaf_id) + return 0; - if (node->run.nr_entries > node->stack_len) - return -EINVAL; - start = node->stack_len - node->run.nr_entries; - if (index < start || index >= node->stack_len) - continue; - return stack_depot_trie_node_frame(node, index - start, frame); - } + rcu_read_lock_sched_notrace(); + leaf = __stack_depot_trie_side_table_lookup(leaf_id); + if (WARN(!leaf, "corrupt trie handle %08x\n", handle)) + goto out; + total = trie_validate_leaf(leaf, NULL); + if (total) + walked = trie_walk_frames(leaf, total, fn, data); +out: + rcu_read_unlock_sched_notrace(); - return -EINVAL; + return walked; } -static void trie_print_frames(const void *leaf, unsigned int nr_entries, - int spaces) -{ - unsigned int i; - - for (i = 0; i < nr_entries; i++) { - unsigned long frame; - - if (trie_frame_at(leaf, i, &frame)) - return; - pr_info("%*c%pS\n", 1 + spaces, ' ', (void *)frame); - } -} +struct trie_print_ctx { + int spaces; +}; -static int -trie_snprint_frames(char *buf, size_t size, const void *leaf, - unsigned int nr_entries, int spaces) +static void trie_print_frame(unsigned int index, unsigned long frame, void *data) { - unsigned int generated; - unsigned int total = 0; - unsigned int i; - - for (i = 0; i < nr_entries && size; i++) { - unsigned long frame; - - if (trie_frame_at(leaf, i, &frame)) - break; - generated = snprintf(buf, size, "%*c%pS\n", 1 + spaces, ' ', - (void *)frame); - total += generated; - if (generated >= size) { - buf += size; - size = 0; - } else { - buf += generated; - size -= generated; - } - } + struct trie_print_ctx *ctx = data; - return total; + (void)index; + pr_info("%*c%pS\n", 1 + ctx->spaces, ' ', (void *)frame); } -static unsigned int trie_handle_leaf(depot_stack_handle_t handle, - const void **leaf) +static unsigned int trie_print_handle(depot_stack_handle_t handle, int spaces) { - u32 leaf_id; + struct trie_print_ctx ctx = { .spaces = spaces }; - if (!leaf) - return 0; - *leaf = NULL; - leaf_id = __stack_depot_trie_leaf_id(handle); - if (!leaf_id) - return 0; - *leaf = __stack_depot_trie_side_table_lookup(leaf_id); - if (WARN(!*leaf, "corrupt trie handle %08x\n", handle)) - return 0; - return trie_validate_leaf(*leaf, NULL); + return trie_walk_handle(handle, trie_print_frame, &ctx); } -static unsigned int trie_print_handle(depot_stack_handle_t handle, int spaces) +struct trie_snprint_ctx { + char *buf; + size_t size; + unsigned int total; + int spaces; +}; + +static void trie_snprint_frame(unsigned int index, unsigned long frame, void *data) { - unsigned int nr_entries; - const void *leaf; + struct trie_snprint_ctx *ctx = data; + unsigned int generated; - rcu_read_lock_sched_notrace(); - nr_entries = trie_handle_leaf(handle, &leaf); - if (nr_entries) - trie_print_frames(leaf, nr_entries, spaces); - rcu_read_unlock_sched_notrace(); + (void)index; + if (!ctx->size) + return; - return nr_entries; + generated = snprintf(ctx->buf, ctx->size, "%*c%pS\n", + 1 + ctx->spaces, ' ', (void *)frame); + ctx->total += generated; + if (generated >= ctx->size) { + ctx->buf += ctx->size; + ctx->size = 0; + } else { + ctx->buf += generated; + ctx->size -= generated; + } } static int trie_snprint_handle(depot_stack_handle_t handle, char *buf, size_t size, int spaces) { - unsigned int nr_entries; - const void *leaf; - int ret = 0; - - rcu_read_lock_sched_notrace(); - nr_entries = trie_handle_leaf(handle, &leaf); - if (nr_entries) - ret = trie_snprint_frames(buf, size, leaf, nr_entries, spaces); - rcu_read_unlock_sched_notrace(); + struct trie_snprint_ctx ctx = { + .buf = buf, + .size = size, + .spaces = spaces, + }; - return ret; + trie_walk_handle(handle, trie_snprint_frame, &ctx); + return ctx.total; } static void trie_fetch_frame(unsigned int index, unsigned long frame, void *data) @@ -4542,223 +4459,6 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, return nr_entries; } -unsigned int -__stack_depot_trie_materialize_handle(depot_stack_handle_t handle, - unsigned long *storage, - unsigned int max_entries, - const unsigned long **frames) -{ - const unsigned long *cached; - const void *leaf; - unsigned int nr_entries; - u32 leaf_id; - int ret; - - if (!frames) - return 0; - *frames = NULL; - - leaf_id = __stack_depot_trie_leaf_id(handle); - if (!leaf_id) - return 0; - - rcu_read_lock_sched_notrace(); - leaf = __stack_depot_trie_side_table_lookup(leaf_id); - if (WARN(!leaf, "corrupt trie handle %08x\n", handle)) { - rcu_read_unlock_sched_notrace(); - return 0; - } - nr_entries = trie_validate_leaf(leaf, NULL); - if (!nr_entries) { - rcu_read_unlock_sched_notrace(); - return 0; - } - cached = __stack_depot_trie_side_table_frames(leaf_id); - if (cached) { - *frames = cached; - rcu_read_unlock_sched_notrace(); - return nr_entries; - } - if (!storage || max_entries < nr_entries) { - rcu_read_unlock_sched_notrace(); - return 0; - } - - nr_entries = trie_fetch_leaf(leaf, storage, max_entries); - if (!nr_entries) { - rcu_read_unlock_sched_notrace(); - return 0; - } - - ret = __stack_depot_trie_side_table_store_frames(leaf_id, storage); - if (!ret) { - *frames = storage; - rcu_read_unlock_sched_notrace(); - return nr_entries; - } - if (ret == -EEXIST) { - cached = __stack_depot_trie_side_table_frames(leaf_id); - if (cached) - *frames = cached; - } - rcu_read_unlock_sched_notrace(); - - return *frames ? nr_entries : 0; -} - -size_t __stack_depot_trie_materialize_bytes(depot_stack_handle_t handle, unsigned int *nr_entries) -{ - const void *leaf; - unsigned int total; - u32 leaf_id; - size_t size; - - if (nr_entries) - *nr_entries = 0; - - leaf_id = __stack_depot_trie_leaf_id(handle); - if (!leaf_id) - return 0; - - rcu_read_lock_sched_notrace(); - leaf = __stack_depot_trie_side_table_lookup(leaf_id); - if (!leaf) - goto out; - total = trie_validate_leaf(leaf, NULL); - if (!total) - goto out; - if (check_mul_overflow((size_t)total, sizeof(unsigned long), &size)) - goto out; - - if (nr_entries) - *nr_entries = total; - rcu_read_unlock_sched_notrace(); - return size; -out: - rcu_read_unlock_sched_notrace(); - return 0; -} - -size_t __stack_depot_trie_materialized_size(unsigned int nr_entries) -{ - struct stack_depot_trie_materialized *record = NULL; - size_t size; - - if (!nr_entries || nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) - return 0; - size = struct_size(record, entries, nr_entries); - return size <= DEPOT_POOL_SIZE ? __stack_depot_trie_pool_alloc_size(size) : 0; -} - -unsigned int __stack_depot_trie_materialized_count(const unsigned long *frames) -{ - const struct stack_depot_trie_materialized *record; - - if (!frames) - return 0; - record = (const void *)((const char *)frames - - offsetof(struct stack_depot_trie_materialized, entries)); - return READ_ONCE(record->nr_entries); -} - -void -__stack_depot_trie_materialized_stats(unsigned long *count, unsigned long *bytes) -{ - if (count) - *count = READ_ONCE(counters[DEPOT_COUNTER_TRIE_MATERIALIZED_COUNT]); - if (bytes) - *bytes = READ_ONCE(counters[DEPOT_COUNTER_TRIE_MATERIALIZED_BYTES]); -} - -unsigned int -__stack_depot_trie_materialize_record(depot_stack_handle_t handle, - struct stack_depot_trie_materialized *record, - size_t record_size, - const unsigned long **frames) -{ - unsigned int nr_entries; - size_t size; - - if (!frames) - return 0; - *frames = NULL; - size = __stack_depot_trie_materialize_bytes(handle, &nr_entries); - if (!size) - return 0; - if (!record || record_size < __stack_depot_trie_materialized_size(nr_entries)) - return 0; - - WRITE_ONCE(record->nr_entries, nr_entries); - return __stack_depot_trie_materialize_handle(handle, record->entries, - nr_entries, frames); -} - -unsigned int -__stack_depot_trie_materialize_cached(depot_stack_handle_t handle, - const unsigned long **frames) -{ - struct stack_depot_trie_materialized *record; - unsigned int nr_entries; - unsigned long flags; - size_t record_size; - size_t offset; - void *pool; - int ret; - - if (!frames) - return 0; - *frames = NULL; - - nr_entries = __stack_depot_trie_materialize_handle(handle, NULL, 0, - frames); - if (nr_entries) - return nr_entries; - - record_size = __stack_depot_trie_materialize_bytes(handle, &nr_entries); - if (!record_size) - return 0; - record_size = __stack_depot_trie_materialized_size(nr_entries); - if (!record_size) - return 0; - - if (!raw_spin_trylock_irqsave(&pool_lock, flags)) - return 0; - printk_deferred_enter(); - - ret = __stack_depot_trie_materialize_handle(handle, NULL, 0, frames); - if (ret) - goto out; - - if (!stack_pools || pools_num < 1) - goto out; - if (WARN_ON_ONCE(pool_offset > DEPOT_POOL_SIZE)) - goto out; - if (record_size > DEPOT_POOL_SIZE - pool_offset) - goto out; - - pool = stack_pools[pools_num - 1]; - if (WARN_ON_ONCE(!pool)) - goto out; - - offset = pool_offset; - record = pool + offset; - pool_offset += record_size; - ret = __stack_depot_trie_materialize_record(handle, record, record_size, frames); - if (ret && *frames == record->entries) { - counters[DEPOT_COUNTER_TRIE_MATERIALIZED_COUNT]++; - counters[DEPOT_COUNTER_TRIE_MATERIALIZED_BYTES] += record_size; - goto out; - } - - pool_offset = offset; - if (!*frames) - ret = 0; -out: - printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); - return ret; -} - size_t __stack_depot_trie_child_array_size(unsigned int nr_children) { size_t size; @@ -5419,9 +5119,7 @@ __stack_depot_trie_child_array_insert(const void *old_storage, const void *child unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries) { - const unsigned long *trie_entries; struct stack_record *stack; - unsigned int nr_entries; *entries = NULL; /* @@ -5432,13 +5130,8 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, if (!handle || stack_depot_disabled) return 0; - if (__stack_depot_trie_leaf_id(handle)) { - nr_entries = __stack_depot_trie_materialize_cached(handle, &trie_entries); - if (!nr_entries) - return 0; - *entries = (unsigned long *)trie_entries; - return nr_entries; - } + if (__stack_depot_trie_leaf_id(handle)) + return 0; stack = depot_fetch_stack(handle); /* diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index facb094e36315..85b6173e406d6 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -31,7 +31,6 @@ static_assert(CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(unsigned long) <= U16_MAX); bool __stack_depot_trie_enabled(void); bool __stack_depot_trie_ready(void); -bool __stack_depot_trie_can_alloc(gfp_t alloc_flags, depot_flags_t depot_flags); void __stack_depot_trie_set_enabled(bool enabled); struct stack_depot_trie_node_slot { @@ -50,11 +49,6 @@ struct stack_depot_trie_root { const struct stack_depot_trie_child_array *children; }; -struct stack_depot_trie_materialized { - unsigned int nr_entries; - unsigned long entries[]; -}; - struct stack_depot_trie_lookup { const void *parent; const void *node; @@ -147,11 +141,9 @@ u32 __stack_depot_trie_max_leaf_id(void); /* * Private trie side table. Writers serialize internally; lookups are lockless. - * Leaf slots are populated before trie publication, while frame slots are for - * future stable materialization of trie handles. The side table reserves the - * frame pointer per handle, but the backing materialized frame array is - * allocated lazily only for the rare fetch path. Init and destroy are controlled - * setup/teardown operations and must not race with readers or writers. + * Leaf slots are populated before trie publication. Init and destroy are + * controlled setup/teardown operations and must not race with readers or + * writers. */ int __stack_depot_trie_side_table_init(gfp_t gfp_flags); void __stack_depot_trie_side_table_destroy(void); @@ -163,9 +155,6 @@ void __stack_depot_trie_side_table_revoke_latest(u32 id); void __stack_depot_trie_side_table_restore(u32 id, const void *entry); int __stack_depot_trie_side_table_store(u32 id, const void *entry); const void *__stack_depot_trie_side_table_lookup(u32 id); -const unsigned long *__stack_depot_trie_side_table_frames(u32 id); -int -__stack_depot_trie_side_table_store_frames(u32 id, const unsigned long *frames); size_t __stack_depot_trie_side_table_entries(void); size_t __stack_depot_trie_side_table_bytes(void); size_t __stack_depot_trie_pool_alloc_size(size_t size); @@ -331,25 +320,6 @@ __stack_depot_trie_walk_frames(const void *leaf, trie_frame_fn_t fn, void *data) unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, unsigned long *entries, unsigned int max_entries); -unsigned int -__stack_depot_trie_materialize_handle(depot_stack_handle_t handle, - unsigned long *storage, - unsigned int max_entries, - const unsigned long **frames); -size_t __stack_depot_trie_materialize_bytes(depot_stack_handle_t handle, unsigned int *nr_entries); -size_t __stack_depot_trie_materialized_size(unsigned int nr_entries); -unsigned int -__stack_depot_trie_materialized_count(const unsigned long *frames); -void -__stack_depot_trie_materialized_stats(unsigned long *count, unsigned long *bytes); -unsigned int -__stack_depot_trie_materialize_record(depot_stack_handle_t handle, - struct stack_depot_trie_materialized *record, - size_t record_size, - const unsigned long **frames); -unsigned int -__stack_depot_trie_materialize_cached(depot_stack_handle_t handle, - const unsigned long **frames); size_t __stack_depot_trie_child_array_size(unsigned int nr_children); int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, const void * const *children, diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 58c7624579d9c..ab37f0596e15e 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -604,29 +604,15 @@ static void stackdepot_trie_feature_flag(struct kunit *test) static void stackdepot_trie_late_init(struct kunit *test) { - depot_flags_t can_alloc_flag = STACK_DEPOT_FLAG_CAN_ALLOC; - gfp_t no_spin = GFP_NOWAIT & ~__GFP_RECLAIM; - bool can_alloc; - stackdepot_trie_add_disable_action(test); KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); KUNIT_EXPECT_FALSE(test, __stack_depot_trie_ready()); - can_alloc = __stack_depot_trie_can_alloc(GFP_KERNEL, can_alloc_flag); - KUNIT_EXPECT_FALSE(test, can_alloc); if (!__stack_depot_trie_max_leaf_id()) kunit_skip(test, "trie handle namespace unavailable"); __stack_depot_trie_set_enabled(true); KUNIT_EXPECT_EQ(test, stack_depot_init(), 0); KUNIT_EXPECT_TRUE(test, __stack_depot_trie_ready()); - can_alloc = __stack_depot_trie_can_alloc(GFP_KERNEL, can_alloc_flag); - KUNIT_EXPECT_TRUE(test, can_alloc); - can_alloc = __stack_depot_trie_can_alloc(GFP_KERNEL, 0); - KUNIT_EXPECT_FALSE(test, can_alloc); - can_alloc = __stack_depot_trie_can_alloc(no_spin, can_alloc_flag); - KUNIT_EXPECT_FALSE(test, can_alloc); - can_alloc = __stack_depot_trie_can_alloc(GFP_KERNEL, STACK_DEPOT_FLAG_GET); - KUNIT_EXPECT_FALSE(test, can_alloc); } static void stackdepot_trie_side_table_destroy_action(void *data) @@ -671,11 +657,6 @@ static void stackdepot_trie_side_table_alloc_store_lookup(struct kunit *test) { const void *entry1 = (const void *)0x1111UL; const void *entry2 = (const void *)0x2222UL; - const unsigned long frames[] = { 0xaaaaUL, 0xbbbbUL }; - const unsigned long other_frames[] = { 0xccccUL }; - const unsigned long *frames_ptr = frames; - const unsigned long *other_frames_ptr = other_frames; - int ret; u32 id1; u32 id2; @@ -689,18 +670,11 @@ static void stackdepot_trie_side_table_alloc_store_lookup(struct kunit *test) KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store(id2, entry2), 0); KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), entry1); KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id2), entry2); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(id1)); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store_frames(id1, frames_ptr), 0); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(id1), frames_ptr); - ret = __stack_depot_trie_side_table_store_frames(id1, other_frames_ptr); - KUNIT_EXPECT_EQ(test, ret, -EEXIST); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(id1), frames_ptr); KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 2UL); } static void stackdepot_trie_side_table_rejects_invalid_ids(struct kunit *test) { - const unsigned long *frames = (const unsigned long *)0x1UL; int ret; u32 id; @@ -710,23 +684,14 @@ static void stackdepot_trie_side_table_rejects_invalid_ids(struct kunit *test) id = stackdepot_trie_side_table_alloc(test); KUNIT_ASSERT_EQ(test, id, 1U); KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id + 1)); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(id + 1)); - ret = __stack_depot_trie_side_table_store_frames(id, frames); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store(id, NULL), -EINVAL); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store_frames(id, NULL), - -EINVAL); ret = __stack_depot_trie_side_table_store(id + 1, (const void *)0x1UL); KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = __stack_depot_trie_side_table_store_frames(id + 1, frames); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); } static void stackdepot_trie_side_table_revoke_latest(struct kunit *test) { const void *entry = (const void *)0xaaaaUL; - const unsigned long frames[] = { 0x1234UL }; - const unsigned long *frames_ptr = frames; size_t bytes; int ret; u32 id; @@ -736,8 +701,6 @@ static void stackdepot_trie_side_table_revoke_latest(struct kunit *test) KUNIT_ASSERT_EQ(test, id, 1U); ret = __stack_depot_trie_side_table_store(id, entry); KUNIT_ASSERT_EQ(test, ret, 0); - ret = __stack_depot_trie_side_table_store_frames(id, frames_ptr); - KUNIT_ASSERT_EQ(test, ret, 0); bytes = __stack_depot_trie_side_table_bytes(); KUNIT_EXPECT_GT(test, bytes, 0UL); @@ -745,7 +708,6 @@ static void stackdepot_trie_side_table_revoke_latest(struct kunit *test) KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_bytes(), bytes); KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id)); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(id)); } static void stackdepot_trie_side_table_revoke_keeps_chunk(struct kunit *test) @@ -776,8 +738,6 @@ static void stackdepot_trie_side_table_restore(struct kunit *test) { const void *entry1 = (const void *)0xaaaaUL; const void *entry2 = (const void *)0xbbbbUL; - const unsigned long frames[] = { 0xccccUL }; - const unsigned long *frames_ptr = frames; u32 id; stackdepot_trie_side_table_init_or_skip(test); @@ -785,14 +745,11 @@ static void stackdepot_trie_side_table_restore(struct kunit *test) KUNIT_ASSERT_EQ(test, id, 1U); KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, entry1), 0); KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, entry2), 0); - KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store_frames(id, frames_ptr), 0); __stack_depot_trie_side_table_restore(id, entry1); KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id), entry1); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(id), frames_ptr); __stack_depot_trie_side_table_restore(id, NULL); KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id)); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(id)); } static void stackdepot_trie_side_table_chunk_boundary(struct kunit *test) @@ -1589,23 +1546,6 @@ static unsigned int tfetch_handle(depot_stack_handle_t handle, return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); } -static unsigned int tmaterialize(depot_stack_handle_t handle, - unsigned long *storage, unsigned int max_entries, - const unsigned long **frames) -{ - return __stack_depot_trie_materialize_handle(handle, storage, max_entries, - frames); -} - -static unsigned int tmaterialize_cached(depot_stack_handle_t handle, - const unsigned long **frames) -{ - return __stack_depot_trie_materialize_cached(handle, frames); -} - -#define TMREC(handle, record, size, frames) \ - __stack_depot_trie_materialize_record(handle, record, size, frames) - struct trie_frame_iter_ctx { unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; unsigned int nr_entries; @@ -1838,22 +1778,15 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; struct stack_depot_trie_alloc_workspace *workspace; - struct stack_depot_trie_materialized *record; struct trie_frame_iter_ctx *iter; struct stack_depot_trie_root root = {}; unsigned long small[1] = { 0xdeadUL }; - unsigned long loser[ARRAY_SIZE(entries)] = { 0xdeadUL, 0xbeefUL }; - unsigned long *record_entries; unsigned long out[ARRAY_SIZE(entries)] = {}; depot_stack_handle_t hash_handle; depot_stack_handle_t handle; - const unsigned long *frames; const void *leaf; - size_t record_size; unsigned int invalid; unsigned int fetched; - unsigned int nr_sized; - size_t size; u32 leaf_id; workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); @@ -1891,38 +1824,6 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) fetched = stack_depot_fetch_into(handle, small, ARRAY_SIZE(small)); KUNIT_EXPECT_EQ(test, fetched, 0U); KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); - size = __stack_depot_trie_materialize_bytes(handle, &nr_sized); - KUNIT_EXPECT_EQ(test, size, sizeof(entries)); - KUNIT_EXPECT_EQ(test, nr_sized, (unsigned int)ARRAY_SIZE(entries)); - record_size = __stack_depot_trie_materialized_size(nr_sized); - KUNIT_ASSERT_NE(test, record_size, 0UL); - record = kunit_kzalloc(test, record_size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, record); - record->nr_entries = nr_sized; - record_entries = record->entries; - nr_sized = 0xdeadU; - KUNIT_EXPECT_EQ(test, __stack_depot_trie_materialize_bytes(0, &nr_sized), 0UL); - KUNIT_EXPECT_EQ(test, nr_sized, 0U); - frames = (const unsigned long *)0x1UL; - fetched = tmaterialize(handle, small, ARRAY_SIZE(small), &frames); - KUNIT_EXPECT_EQ(test, fetched, 0U); - KUNIT_EXPECT_NULL(test, frames); - KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); - fetched = TMREC(handle, record, record_size, &frames); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_PTR_EQ(test, frames, record_entries); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_materialized_count(frames), - (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, record->entries, entries, sizeof(entries)); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(leaf_id), - record_entries); - fetched = tmaterialize(handle, loser, ARRAY_SIZE(loser), &frames); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_PTR_EQ(test, frames, record_entries); - KUNIT_EXPECT_EQ(test, loser[0], 0xdeadUL); - KUNIT_EXPECT_EQ(test, loser[1], 0xbeefUL); - fetched = TMREC(handle, record, record_size, NULL); - KUNIT_EXPECT_EQ(test, fetched, 0U); invalid = tfetch_handle(0, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, invalid, 0U); invalid = tfetch_handle(handle, NULL, 0); @@ -1935,57 +1836,6 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); invalid = tfetch_handle(hash_handle, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, invalid, 0U); - frames = (const unsigned long *)0x1UL; - fetched = TMREC(hash_handle, record, record_size, &frames); - KUNIT_EXPECT_EQ(test, fetched, 0U); - KUNIT_EXPECT_NULL(test, frames); - nr_sized = 0xdeadU; - size = __stack_depot_trie_materialize_bytes(hash_handle, &nr_sized); - KUNIT_EXPECT_EQ(test, size, 0UL); - KUNIT_EXPECT_EQ(test, nr_sized, 0U); -} - -static void stackdepot_trie_fetch_public(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - struct stack_depot_trie_alloc_workspace *workspace; - struct stack_depot_trie_root root = {}; - depot_stack_handle_t extra; - depot_stack_handle_t handle; - unsigned long *again; - unsigned long *frames; - unsigned int fetched; - u32 leaf_id; - - workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, workspace); - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - - handle = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_CAN_ALLOC, workspace); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - leaf_id = __stack_depot_trie_leaf_id(handle); - KUNIT_ASSERT_NE(test, leaf_id, 0U); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(leaf_id)); - - fetched = stack_depot_fetch(handle, &frames); - KUNIT_ASSERT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_ASSERT_NOT_NULL(test, frames); - KUNIT_EXPECT_MEMEQ(test, frames, entries, sizeof(entries)); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(leaf_id), - frames); - - extra = stack_depot_set_extra_bits(handle, - (1U << STACK_DEPOT_EXTRA_BITS) - 1); - fetched = stack_depot_fetch(extra, &again); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_PTR_EQ(test, again, frames); - - stack_depot_put(extra); - fetched = stack_depot_fetch(handle, &again); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_PTR_EQ(test, again, frames); } static void stackdepot_trie_snprint_public(struct kunit *test) @@ -1999,7 +1849,6 @@ static void stackdepot_trie_snprint_public(struct kunit *test) depot_stack_handle_t handle; unsigned int expected_len; int actual_len; - u32 leaf_id; workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, workspace); @@ -2009,9 +1858,6 @@ static void stackdepot_trie_snprint_public(struct kunit *test) handle = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, STACK_DEPOT_FLAG_CAN_ALLOC, workspace); KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - leaf_id = __stack_depot_trie_leaf_id(handle); - KUNIT_ASSERT_NE(test, leaf_id, 0U); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(leaf_id)); expected_len = stack_trace_snprint(expected, sizeof(expected), entries, ARRAY_SIZE(entries), 2); @@ -2019,86 +1865,6 @@ static void stackdepot_trie_snprint_public(struct kunit *test) actual_len = stack_depot_snprint(extra, actual, sizeof(actual), 2); KUNIT_EXPECT_EQ(test, actual_len, (int)expected_len); KUNIT_EXPECT_STREQ(test, actual, expected); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(leaf_id)); -} - -static void stackdepot_trie_materialize_cached(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - struct stack_depot_trie_alloc_workspace *workspace; - struct stack_depot_trie_root root = {}; - depot_stack_handle_t hash_handle; - depot_stack_handle_t extra; - depot_stack_handle_t handle; - const unsigned long *again; - const unsigned long *frames; - unsigned long bytes_after; - unsigned long bytes_before; - unsigned long count_after; - unsigned long count_before; - unsigned int fetched; - size_t record_size; - u32 leaf_id; - - workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, workspace); - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - - handle = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_CAN_ALLOC, workspace); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - leaf_id = __stack_depot_trie_leaf_id(handle); - KUNIT_ASSERT_NE(test, leaf_id, 0U); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_frames(leaf_id)); - record_size = __stack_depot_trie_materialized_size(ARRAY_SIZE(entries)); - KUNIT_ASSERT_NE(test, record_size, 0UL); - __stack_depot_trie_materialized_stats(&count_before, &bytes_before); - - frames = NULL; - fetched = tmaterialize_cached(handle, &frames); - KUNIT_ASSERT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_ASSERT_NOT_NULL(test, frames); - KUNIT_EXPECT_MEMEQ(test, frames, entries, sizeof(entries)); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_materialized_count(frames), - (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_frames(leaf_id), - frames); - __stack_depot_trie_materialized_stats(&count_after, &bytes_after); - KUNIT_EXPECT_EQ(test, count_after, count_before + 1); - KUNIT_EXPECT_EQ(test, bytes_after, bytes_before + record_size); - - again = NULL; - fetched = tmaterialize_cached(handle, &again); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_PTR_EQ(test, again, frames); - __stack_depot_trie_materialized_stats(&count_after, &bytes_after); - KUNIT_EXPECT_EQ(test, count_after, count_before + 1); - KUNIT_EXPECT_EQ(test, bytes_after, bytes_before + record_size); - - extra = stack_depot_set_extra_bits(handle, - (1U << STACK_DEPOT_EXTRA_BITS) - 1); - again = NULL; - fetched = tmaterialize_cached(extra, &again); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_PTR_EQ(test, again, frames); - __stack_depot_trie_materialized_stats(&count_after, &bytes_after); - KUNIT_EXPECT_EQ(test, count_after, count_before + 1); - KUNIT_EXPECT_EQ(test, bytes_after, bytes_before + record_size); - - frames = (const unsigned long *)0x1UL; - fetched = tmaterialize_cached(0, &frames); - KUNIT_EXPECT_EQ(test, fetched, 0U); - KUNIT_EXPECT_NULL(test, frames); - fetched = tmaterialize_cached(handle, NULL); - KUNIT_EXPECT_EQ(test, fetched, 0U); - - hash_handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); - KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); - frames = (const unsigned long *)0x1UL; - fetched = tmaterialize_cached(hash_handle, &frames); - KUNIT_EXPECT_EQ(test, fetched, 0U); - KUNIT_EXPECT_NULL(test, frames); } static void stackdepot_trie_alloc_txn_plan(struct kunit *test) @@ -5837,10 +5603,10 @@ static void stackdepot_trie_public_save_route(struct kunit *test) KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(trie_entries)); KUNIT_EXPECT_MEMEQ(test, fetched, trie_entries, sizeof(trie_entries)); noalloc_handle = stack_depot_save_flags(noalloc_entries, noalloc_nr, no_spin, 0); - KUNIT_EXPECT_EQ(test, noalloc_handle, (depot_stack_handle_t)0); - noalloc_handle = stack_depot_save(noalloc_entries, noalloc_nr, GFP_KERNEL); KUNIT_ASSERT_NE(test, noalloc_handle, (depot_stack_handle_t)0); KUNIT_EXPECT_NE(test, __stack_depot_trie_leaf_id(noalloc_handle), 0U); + KUNIT_EXPECT_EQ(test, stack_depot_save(noalloc_entries, noalloc_nr, GFP_KERNEL), + noalloc_handle); get_flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; get_handle = stack_depot_save_flags(get_entries, get_nr, GFP_KERNEL, get_flags); @@ -5898,9 +5664,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_save), KUNIT_CASE(stackdepot_trie_save_locked), KUNIT_CASE(stackdepot_trie_fetch_handle_into), - KUNIT_CASE(stackdepot_trie_fetch_public), KUNIT_CASE(stackdepot_trie_snprint_public), - KUNIT_CASE(stackdepot_trie_materialize_cached), KUNIT_CASE(stackdepot_trie_alloc_txn_plan), KUNIT_CASE(stackdepot_trie_alloc_txn_insert), KUNIT_CASE(stackdepot_trie_alloc_txn_insert_stale_plan), diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 1ac56ceb29b6b..d7fbcc9f121c9 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -364,10 +364,10 @@ static void print_unreferenced(struct seq_file *seq, struct kmemleak_object *object) { int i; - unsigned long *entries; + unsigned long entries[MAX_TRACE]; unsigned int nr_entries; - nr_entries = stack_depot_fetch(object->trace_handle, &entries); + nr_entries = stack_depot_fetch_into(object->trace_handle, entries, ARRAY_SIZE(entries)); warn_or_seq_printf(seq, "unreferenced object%s 0x%08lx (size %zu):\n", __object_type_str(object), object->pointer, object->size); diff --git a/mm/kmsan/kmsan_test.c b/mm/kmsan/kmsan_test.c index 902ec48b1e3e6..123bcc9565721 100644 --- a/mm/kmsan/kmsan_test.c +++ b/mm/kmsan/kmsan_test.c @@ -610,7 +610,7 @@ static void test_long_origin_chain(struct kunit *test) */ static void test_stackdepot_roundtrip(struct kunit *test) { - unsigned long src_entries[16], *dst_entries; + unsigned long src_entries[16], dst_entries[16]; unsigned int src_nentries, dst_nentries; EXPECTATION_NO_REPORT(expect); depot_stack_handle_t handle; @@ -621,11 +621,10 @@ static void test_stackdepot_roundtrip(struct kunit *test) stack_trace_save(src_entries, ARRAY_SIZE(src_entries), 1); handle = stack_depot_save(src_entries, src_nentries, GFP_KERNEL); stack_depot_print(handle); - dst_nentries = stack_depot_fetch(handle, &dst_entries); + dst_nentries = stack_depot_fetch_into(handle, dst_entries, ARRAY_SIZE(dst_entries)); KUNIT_EXPECT_TRUE(test, src_nentries == dst_nentries); - kmsan_check_memory((void *)dst_entries, - sizeof(*dst_entries) * dst_nentries); + kmsan_check_memory(dst_entries, sizeof(*dst_entries) * dst_nentries); KUNIT_EXPECT_TRUE(test, report_matches(&expect)); } diff --git a/mm/kmsan/report.c b/mm/kmsan/report.c index d6853ce089541..94746ebe97e74 100644 --- a/mm/kmsan/report.c +++ b/mm/kmsan/report.c @@ -85,7 +85,8 @@ static char *pretty_descr(char *descr) void kmsan_print_origin(depot_stack_handle_t origin) { - unsigned long *entries = NULL, *chained_entries = NULL; + unsigned long entries[KMSAN_STACK_DEPTH]; + unsigned long chain[KMSAN_STACK_DEPTH]; unsigned int nr_entries, chained_nr_entries, skipnr; void *pc1 = NULL, *pc2 = NULL; depot_stack_handle_t head; @@ -97,7 +98,7 @@ void kmsan_print_origin(depot_stack_handle_t origin) return; while (true) { - nr_entries = stack_depot_fetch(origin, &entries); + nr_entries = stack_depot_fetch_into(origin, entries, ARRAY_SIZE(entries)); depth = kmsan_depth_from_eb(stack_depot_get_extra_bits(origin)); magic = nr_entries ? entries[0] : 0; if ((nr_entries == 4) && (magic == KMSAN_ALLOCA_MAGIC_ORIGIN)) { @@ -122,15 +123,13 @@ void kmsan_print_origin(depot_stack_handle_t origin) head = entries[1]; origin = entries[2]; pr_err("Uninit was stored to memory at:\n"); - chained_nr_entries = - stack_depot_fetch(head, &chained_entries); + chained_nr_entries = stack_depot_fetch_into(head, chain, ARRAY_SIZE(chain)); kmsan_internal_unpoison_memory( - chained_entries, - chained_nr_entries * sizeof(*chained_entries), + chain, + chained_nr_entries * sizeof(*chain), /*checked*/ false); - skipnr = get_stack_skipnr(chained_entries, - chained_nr_entries); - stack_trace_print(chained_entries + skipnr, + skipnr = get_stack_skipnr(chain, chained_nr_entries); + stack_trace_print(chain + skipnr, chained_nr_entries - skipnr, 0); pr_err("\n"); continue; diff --git a/mm/slub.c b/mm/slub.c index a89df6ddcc587..a12ab73b7558c 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -8147,12 +8147,12 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) #ifdef CONFIG_STACKDEPOT { depot_stack_handle_t handle; - unsigned long *entries; + unsigned long entries[TRACK_ADDRS_COUNT]; unsigned int nr_entries; handle = READ_ONCE(trackp->handle); if (handle) { - nr_entries = stack_depot_fetch(handle, &entries); + nr_entries = stack_depot_fetch_into(handle, entries, ARRAY_SIZE(entries)); for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) kpp->kp_stack[i] = (void *)entries[i]; } @@ -8160,7 +8160,7 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) trackp = get_track(s, objp, TRACK_FREE); handle = READ_ONCE(trackp->handle); if (handle) { - nr_entries = stack_depot_fetch(handle, &entries); + nr_entries = stack_depot_fetch_into(handle, entries, ARRAY_SIZE(entries)); for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) kpp->kp_free_stack[i] = (void *)entries[i]; } @@ -9928,19 +9928,19 @@ static int slab_debugfs_show(struct seq_file *seq, void *v) nodemask_pr_args(&l->nodes)); #ifdef CONFIG_STACKDEPOT - { - depot_stack_handle_t handle; - unsigned long *entries; - unsigned int nr_entries, j; - - handle = READ_ONCE(l->handle); - if (handle) { - nr_entries = stack_depot_fetch(handle, &entries); - seq_puts(seq, "\n"); - for (j = 0; j < nr_entries; j++) - seq_printf(seq, " %pS\n", (void *)entries[j]); - } + { + depot_stack_handle_t handle; + unsigned long entries[TRACK_ADDRS_COUNT]; + unsigned int nr_entries, j; + + handle = READ_ONCE(l->handle); + if (handle) { + nr_entries = stack_depot_fetch_into(handle, entries, ARRAY_SIZE(entries)); + seq_puts(seq, "\n"); + for (j = 0; j < nr_entries; j++) + seq_printf(seq, " %pS\n", (void *)entries[j]); } + } #endif seq_puts(seq, "\n"); } From 40070059e98b2c97a719faf2665a3724f19e3db6 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 12 Jun 2026 15:41:43 +0100 Subject: [PATCH 086/129] KRN-1117: Print stackdepot trie frames in stack order Restore absolute-index lookup for stack_depot_print() and stack_depot_snprint() on trie-backed handles. The callback walker emits frames leaf-to-root, which reversed allocation stacks in KASAN reports after trie materialization was removed. Keep this on the cold print path rather than reintroducing flat materialized storage. This preserves the memory-saving backend design while making printed trie stacks match the hash-backed stack order. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 136 ++++++++++++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 55 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 3dac95e1086ab..536bb7d7ccee5 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -4295,90 +4295,116 @@ static unsigned int trie_walk_frames(const void *leaf, unsigned int total, return seen == total ? total : 0; } -static unsigned int -trie_walk_handle(depot_stack_handle_t handle, trie_frame_fn_t fn, void *data) +static int trie_frame_at(const void *leaf, unsigned int index, + unsigned long *frame) +{ + const struct stack_depot_trie_node *node; + + if (!leaf || !frame) + return -EINVAL; + + for (node = leaf; node; node = trie_load_parent(node)) { + unsigned int start; + + if (node->run.nr_entries > node->stack_len) + return -EINVAL; + start = node->stack_len - node->run.nr_entries; + if (index < start || index >= node->stack_len) + continue; + return stack_depot_trie_node_frame(node, index - start, frame); + } + + return -EINVAL; +} + +static unsigned int trie_handle_leaf(depot_stack_handle_t handle, + const void **leaf) { - const void *leaf; - unsigned int walked = 0; - unsigned int total; u32 leaf_id; + if (!leaf) + return 0; + *leaf = NULL; leaf_id = __stack_depot_trie_leaf_id(handle); if (!leaf_id) return 0; - - rcu_read_lock_sched_notrace(); - leaf = __stack_depot_trie_side_table_lookup(leaf_id); - if (WARN(!leaf, "corrupt trie handle %08x\n", handle)) - goto out; - total = trie_validate_leaf(leaf, NULL); - if (total) - walked = trie_walk_frames(leaf, total, fn, data); -out: - rcu_read_unlock_sched_notrace(); - - return walked; + *leaf = __stack_depot_trie_side_table_lookup(leaf_id); + if (WARN(!*leaf, "corrupt trie handle %08x\n", handle)) + return 0; + return trie_validate_leaf(*leaf, NULL); } -struct trie_print_ctx { - int spaces; -}; - -static void trie_print_frame(unsigned int index, unsigned long frame, void *data) +static void trie_print_frames(const void *leaf, unsigned int nr_entries, + int spaces) { - struct trie_print_ctx *ctx = data; + unsigned int i; - (void)index; - pr_info("%*c%pS\n", 1 + ctx->spaces, ' ', (void *)frame); + for (i = 0; i < nr_entries; i++) { + unsigned long frame; + + if (trie_frame_at(leaf, i, &frame)) + return; + pr_info("%*c%pS\n", 1 + spaces, ' ', (void *)frame); + } } static unsigned int trie_print_handle(depot_stack_handle_t handle, int spaces) { - struct trie_print_ctx ctx = { .spaces = spaces }; + unsigned int nr_entries; + const void *leaf; - return trie_walk_handle(handle, trie_print_frame, &ctx); -} + rcu_read_lock_sched_notrace(); + nr_entries = trie_handle_leaf(handle, &leaf); + if (nr_entries) + trie_print_frames(leaf, nr_entries, spaces); + rcu_read_unlock_sched_notrace(); -struct trie_snprint_ctx { - char *buf; - size_t size; - unsigned int total; - int spaces; -}; + return nr_entries; +} -static void trie_snprint_frame(unsigned int index, unsigned long frame, void *data) +static int +trie_snprint_frames(char *buf, size_t size, const void *leaf, + unsigned int nr_entries, int spaces) { - struct trie_snprint_ctx *ctx = data; unsigned int generated; + unsigned int total = 0; + unsigned int i; - (void)index; - if (!ctx->size) - return; + for (i = 0; i < nr_entries && size; i++) { + unsigned long frame; - generated = snprintf(ctx->buf, ctx->size, "%*c%pS\n", - 1 + ctx->spaces, ' ', (void *)frame); - ctx->total += generated; - if (generated >= ctx->size) { - ctx->buf += ctx->size; - ctx->size = 0; - } else { - ctx->buf += generated; - ctx->size -= generated; + if (trie_frame_at(leaf, i, &frame)) + break; + generated = snprintf(buf, size, "%*c%pS\n", 1 + spaces, ' ', + (void *)frame); + total += generated; + if (generated >= size) { + buf += size; + size = 0; + } else { + buf += generated; + size -= generated; + } } + + return total; } static int trie_snprint_handle(depot_stack_handle_t handle, char *buf, size_t size, int spaces) { - struct trie_snprint_ctx ctx = { - .buf = buf, - .size = size, - .spaces = spaces, - }; + unsigned int nr_entries; + const void *leaf; + int ret = 0; - trie_walk_handle(handle, trie_snprint_frame, &ctx); - return ctx.total; + rcu_read_lock_sched_notrace(); + nr_entries = trie_handle_leaf(handle, &leaf); + if (nr_entries) + ret = trie_snprint_frames(buf, size, leaf, nr_entries, spaces); + rcu_read_unlock_sched_notrace(); + + return ret; } static void trie_fetch_frame(unsigned int index, unsigned long frame, void *data) From 50a0c846d2ed3d4adce85866a61ae8d6ab27883a Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 15 Jun 2026 08:29:53 +0100 Subject: [PATCH 087/129] KRN-1117: Stress stackdepot trie saves concurrently Add a KUnit stress case that drives the public persistent save path from multiple kernel threads. Each worker saves unique trie-backed stacks, fetches them with stack_depot_fetch_into(), verifies copied frames, checks duplicate no-spin saves return the same handle, and exercises stack_depot_snprint(). This gives KCSAN and lockdep a targeted concurrent workload for trie save, lookup, and fetch paths without relying on heavyweight full-system debug boots. Signed-off-by: Caleb Kan --- lib/tests/stackdepot_kunit.c | 130 +++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index ab37f0596e15e..883b67bdbc71e 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -2,8 +2,13 @@ #include #include +#include +#include +#include #include #include +#include +#include #include #include #include @@ -5622,6 +5627,130 @@ static void stackdepot_trie_public_save_route(struct kunit *test) KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(overlong_handle), 0U); } +#define STACKDEPOT_STRESS_THREADS 4 +#define STACKDEPOT_STRESS_ITERS 64 +#define STACKDEPOT_STRESS_DEPTH 8 + +struct stackdepot_stress_ctx { + struct completion ready; + struct completion done; + struct completion *start; + atomic_t *failures; + unsigned int id; +}; + +static void stackdepot_stress_entries(unsigned int id, unsigned int iter, + unsigned long *entries) +{ + unsigned int i; + + for (i = 0; i < STACKDEPOT_STRESS_DEPTH; i++) + entries[i] = 0xa0000000UL + id * 0x100000UL + iter * 0x1000UL + + i * 0x10UL; +} + +static int stackdepot_trie_stress_worker(void *data) +{ + struct stackdepot_stress_ctx *ctx = data; + unsigned long entries[STACKDEPOT_STRESS_DEPTH]; + unsigned long fetched[STACKDEPOT_STRESS_DEPTH]; + depot_stack_handle_t again; + depot_stack_handle_t handle; + unsigned int nr_entries; + unsigned int iter; + gfp_t no_spin; + char buf[256]; + + complete(&ctx->ready); + wait_for_completion(ctx->start); + + for (iter = 0; iter < STACKDEPOT_STRESS_ITERS; iter++) { + stackdepot_stress_entries(ctx->id, iter, entries); + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + if (!handle || !__stack_depot_trie_leaf_id(handle)) { + atomic_inc(ctx->failures); + continue; + } + + nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); + if (nr_entries != ARRAY_SIZE(entries) || + memcmp(fetched, entries, sizeof(entries))) { + atomic_inc(ctx->failures); + continue; + } + + no_spin = GFP_NOWAIT & ~__GFP_RECLAIM; + again = stack_depot_save_flags(entries, ARRAY_SIZE(entries), no_spin, 0); + if (again != handle) + atomic_inc(ctx->failures); + + if (!(iter % 8) && !stack_depot_snprint(handle, buf, sizeof(buf), 0)) + atomic_inc(ctx->failures); + } + + complete(&ctx->done); + return 0; +} + +static void stackdepot_trie_concurrent_save_fetch(struct kunit *test) +{ + struct stackdepot_stress_ctx *ctx; + struct task_struct *task; + atomic_t failures = ATOMIC_INIT(0); + struct completion start; + unsigned int created = 0; + unsigned int i; + size_t size; + long timeout; + int err = 0; + + stackdepot_trie_add_disable_action(test); + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + if (!__stack_depot_trie_max_leaf_id()) + kunit_skip(test, "trie handle namespace unavailable"); + + __stack_depot_trie_set_enabled(true); + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + KUNIT_ASSERT_TRUE(test, __stack_depot_trie_ready()); + init_completion(&start); + + size = sizeof(*ctx); + ctx = kunit_kcalloc(test, STACKDEPOT_STRESS_THREADS, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + + for (i = 0; i < STACKDEPOT_STRESS_THREADS; i++) { + init_completion(&ctx[i].ready); + init_completion(&ctx[i].done); + ctx[i].start = &start; + ctx[i].failures = &failures; + ctx[i].id = i + 1; + + task = kthread_run(stackdepot_trie_stress_worker, &ctx[i], + "stackdepot_stress/%u", i); + if (IS_ERR(task)) { + err = PTR_ERR(task); + break; + } + created++; + } + + for (i = 0; i < created; i++) { + timeout = wait_for_completion_timeout(&ctx[i].ready, + msecs_to_jiffies(10000)); + KUNIT_EXPECT_GT(test, timeout, 0L); + } + complete_all(&start); + + for (i = 0; i < created; i++) { + timeout = wait_for_completion_timeout(&ctx[i].done, + msecs_to_jiffies(10000)); + KUNIT_EXPECT_GT(test, timeout, 0L); + } + + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, atomic_read(&failures), 0); +} + static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), @@ -5784,6 +5913,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_split_subtree_preserves_children), KUNIT_CASE(stackdepot_trie_child_array_insert_empty), KUNIT_CASE(stackdepot_trie_public_save_route), + KUNIT_CASE(stackdepot_trie_concurrent_save_fetch), {} }; From 72761b3b0a59d4e0eeca62b7a19776f7f77f5980 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 15 Jun 2026 13:42:18 +0100 Subject: [PATCH 088/129] KRN-1117: Make stackdepot trie side table sparse Replace the flat trie side-table chunk array with a sparse root, directory, and chunk layout so unused handle namespace space is not allocated up front. This keeps stable leaf IDs and lockless acquire lookups while making side table memory scale with the number of trie-backed stacks that were actually saved. Carry separate directory and chunk preallocations through trie transactions, and keep no-spin or NMI-context trie misses on the hash backend because side table growth still requires blocking writer-side serialization. Trie hits remain lockless and can still return existing trie handles in those contexts. Add KUnit coverage for the sparse preallocation paths, side-table accounting, constrained-context public save routing, and extra-only handle rejection. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 461 +++++++++++++++++++++++++---------- lib/stackdepot_internal.h | 28 ++- lib/tests/stackdepot_kunit.c | 142 ++++++----- 3 files changed, 427 insertions(+), 204 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 536bb7d7ccee5..bc7225536a7dd 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -282,12 +282,22 @@ struct stack_depot_trie_side_entry { const void *leaf; }; -static struct stack_depot_trie_side_entry **trie_side_table_chunks; +#define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS 9 +#define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE \ + (1U << STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS) + +struct stack_depot_trie_side_dir { + struct stack_depot_trie_side_entry *chunks[STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE]; +}; + +static struct stack_depot_trie_side_dir **trie_side_table_dirs; static DEFINE_RAW_SPINLOCK(trie_side_table_lock); static DEFINE_RAW_SPINLOCK(trie_alloc_lock); static unsigned int trie_side_table_high_water; +static unsigned int trie_side_table_nr_dirs; static unsigned int trie_side_table_nr_chunks; -static unsigned int trie_side_table_top_size; +static unsigned int trie_side_table_root_size; +static u32 trie_side_table_max_id; static u32 trie_side_table_next_id; static bool trie_side_table_initialized; static bool trie_side_table_memblock; @@ -334,39 +344,76 @@ static unsigned int trie_side_table_top_index(u32 id) return (id - 1) >> STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS; } +static unsigned int trie_side_table_root_index(u32 id) +{ + return trie_side_table_top_index(id) >> STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS; +} + +static unsigned int trie_side_table_dir_index(u32 id) +{ + return trie_side_table_top_index(id) & + (STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE - 1); +} + static unsigned int trie_side_table_slot_index(u32 id) { return (id - 1) & (STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE - 1); } -static struct stack_depot_trie_side_entry *trie_side_table_load_chunk(unsigned int top) +static struct stack_depot_trie_side_dir *trie_side_table_load_dir(unsigned int root) { - /* Pairs with trie_side_table_publish_chunk(). */ - return smp_load_acquire(&trie_side_table_chunks[top]); + /* Pairs with trie_side_table_publish_dir(). */ + return smp_load_acquire(&trie_side_table_dirs[root]); +} + +static void trie_side_table_publish_dir(unsigned int root, + struct stack_depot_trie_side_dir *dir) +{ + /* Pairs with trie_side_table_load_dir(). */ + smp_store_release(&trie_side_table_dirs[root], dir); +} + +static struct stack_depot_trie_side_entry * +trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, + unsigned int idx) +{ + /* Pairs with trie_side_table_dir_publish_chunk(). */ + return smp_load_acquire(&dir->chunks[idx]); } static void -trie_side_table_publish_chunk(unsigned int top, - struct stack_depot_trie_side_entry *chunk) +trie_side_table_dir_publish_chunk(struct stack_depot_trie_side_dir *dir, + unsigned int idx, + struct stack_depot_trie_side_entry *chunk) { - /* Pairs with trie_side_table_load_chunk(). */ - smp_store_release(&trie_side_table_chunks[top], chunk); + /* Pairs with trie_side_table_dir_load_chunk(). */ + smp_store_release(&dir->chunks[idx], chunk); } -static size_t trie_side_table_top_bytes(unsigned int top_size) +static size_t trie_side_table_root_bytes(unsigned int root_size) { size_t bytes; - if (check_mul_overflow((size_t)top_size, - sizeof(*trie_side_table_chunks), &bytes)) + if (check_mul_overflow((size_t)root_size, + sizeof(*trie_side_table_dirs), &bytes)) return 0; - return bytes; + return PAGE_ALIGN(bytes); +} + +static size_t trie_side_table_dir_bytes(void) +{ + return PAGE_ALIGN(sizeof(struct stack_depot_trie_side_dir)); +} + +static unsigned int trie_side_table_dir_order(void) +{ + return get_order(trie_side_table_dir_bytes()); } static size_t trie_side_table_chunk_bytes(void) { return PAGE_ALIGN(STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE * - sizeof(*trie_side_table_chunks[0])); + sizeof(struct stack_depot_trie_side_entry)); } static unsigned int trie_side_table_chunk_order(void) @@ -380,45 +427,63 @@ static void trie_side_table_free_chunk(struct stack_depot_trie_side_entry *chunk free_pages((unsigned long)chunk, trie_side_table_chunk_order()); } +static void trie_side_table_free_dir(struct stack_depot_trie_side_dir *dir) +{ + if (dir) + free_pages((unsigned long)dir, trie_side_table_dir_order()); +} + static int -trie_side_table_install(struct stack_depot_trie_side_entry **chunks, - unsigned int top_size, +trie_side_table_install(struct stack_depot_trie_side_dir **dirs, + unsigned int root_size, u32 max_id, + struct stack_depot_trie_side_dir *first_dir, struct stack_depot_trie_side_entry *first_chunk, bool memblock) { if (trie_side_table_is_initialized()) return 0; - if (!chunks || !top_size) + if (!dirs || !root_size || !max_id) return -EINVAL; - trie_side_table_chunks = chunks; - WRITE_ONCE(trie_side_table_top_size, top_size); + WRITE_ONCE(trie_side_table_dirs, dirs); + WRITE_ONCE(trie_side_table_root_size, root_size); WRITE_ONCE(trie_side_table_high_water, 0); + WRITE_ONCE(trie_side_table_nr_dirs, 0); WRITE_ONCE(trie_side_table_nr_chunks, 0); + WRITE_ONCE(trie_side_table_max_id, max_id); WRITE_ONCE(trie_side_table_next_id, 0); WRITE_ONCE(trie_side_table_memblock, memblock); - if (first_chunk) { - trie_side_table_publish_chunk(0, first_chunk); + if (first_dir) { + trie_side_table_publish_dir(0, first_dir); WRITE_ONCE(trie_side_table_high_water, 1); + WRITE_ONCE(trie_side_table_nr_dirs, 1); + } + if (first_dir && first_chunk) { + trie_side_table_dir_publish_chunk(first_dir, 0, first_chunk); WRITE_ONCE(trie_side_table_nr_chunks, 1); } trie_side_table_publish_initialized(); return 0; } -static unsigned int trie_side_table_top_size_for_max_id(u32 max_leaf_id) +static unsigned int trie_side_table_root_size_for_max_id(u32 max_leaf_id) { - return DIV_ROUND_UP(max_leaf_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); + unsigned int top_size; + + top_size = DIV_ROUND_UP(max_leaf_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); + return DIV_ROUND_UP(top_size, STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE); } static int __init __stack_depot_trie_side_table_init_memblock(void) { - struct stack_depot_trie_side_entry **chunks; + struct stack_depot_trie_side_dir **dirs; + struct stack_depot_trie_side_dir *first_dir; struct stack_depot_trie_side_entry *first_chunk; + size_t dir_bytes; size_t chunk_bytes; - size_t top_bytes; + size_t root_bytes; u32 max_leaf_id; - unsigned int top_size; + unsigned int root_size; if (trie_side_table_is_initialized()) return 0; @@ -426,24 +491,33 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) max_leaf_id = __stack_depot_trie_max_leaf_id(); if (!max_leaf_id) return -EINVAL; - top_size = trie_side_table_top_size_for_max_id(max_leaf_id); - top_bytes = trie_side_table_top_bytes(top_size); + root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_bytes = trie_side_table_root_bytes(root_size); + dir_bytes = trie_side_table_dir_bytes(); chunk_bytes = trie_side_table_chunk_bytes(); - if (!top_bytes || !chunk_bytes) + if (!root_bytes || !dir_bytes || !chunk_bytes) return -ENOMEM; - chunks = memblock_alloc(top_bytes, PAGE_SIZE); - if (!chunks) + dirs = memblock_alloc(root_bytes, PAGE_SIZE); + if (!dirs) + return -ENOMEM; + memset(dirs, 0, root_bytes); + first_dir = memblock_alloc(dir_bytes, PAGE_SIZE); + if (!first_dir) { + memblock_free(dirs, root_bytes); return -ENOMEM; - memset(chunks, 0, top_bytes); + } + memset(first_dir, 0, dir_bytes); first_chunk = memblock_alloc(chunk_bytes, PAGE_SIZE); if (!first_chunk) { - memblock_free(chunks, top_bytes); + memblock_free(first_dir, dir_bytes); + memblock_free(dirs, root_bytes); return -ENOMEM; } memset(first_chunk, 0, chunk_bytes); - return trie_side_table_install(chunks, top_size, first_chunk, true); + return trie_side_table_install(dirs, root_size, max_leaf_id, first_dir, + first_chunk, true); } static size_t stack_depot_trie_workspace_size(void) @@ -555,9 +629,9 @@ trie_side_table_clear_entry(struct stack_depot_trie_side_entry *chunk, int __stack_depot_trie_side_table_init(gfp_t gfp_flags) { - struct stack_depot_trie_side_entry **chunks; - unsigned int top_size; - size_t top_bytes; + struct stack_depot_trie_side_dir **dirs; + unsigned int root_size; + size_t root_bytes; u32 max_leaf_id; if (trie_side_table_is_initialized()) @@ -567,20 +641,23 @@ int __stack_depot_trie_side_table_init(gfp_t gfp_flags) if (!max_leaf_id) return -EINVAL; - top_size = trie_side_table_top_size_for_max_id(max_leaf_id); - top_bytes = trie_side_table_top_bytes(top_size); - if (!top_bytes) + root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_bytes = trie_side_table_root_bytes(root_size); + if (!root_bytes) return -ENOMEM; - chunks = kvcalloc(top_size, sizeof(*chunks), gfp_flags); - if (!chunks) + dirs = kvcalloc(root_size, sizeof(*dirs), gfp_flags); + if (!dirs) return -ENOMEM; - return trie_side_table_install(chunks, top_size, NULL, false); + return trie_side_table_install(dirs, root_size, max_leaf_id, NULL, NULL, + false); } void __stack_depot_trie_side_table_destroy(void) { + struct stack_depot_trie_side_dir *dir; unsigned int high_water; + unsigned int j; unsigned int i; if (!trie_side_table_is_initialized()) @@ -589,14 +666,22 @@ void __stack_depot_trie_side_table_destroy(void) high_water = READ_ONCE(trie_side_table_high_water); if (!READ_ONCE(trie_side_table_memblock)) { - for (i = 0; i < high_water; i++) - trie_side_table_free_chunk(trie_side_table_chunks[i]); - kvfree(trie_side_table_chunks); + for (i = 0; i < high_water; i++) { + dir = trie_side_table_dirs[i]; + if (!dir) + continue; + for (j = 0; j < STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE; j++) + trie_side_table_free_chunk(dir->chunks[j]); + trie_side_table_free_dir(dir); + } + kvfree(trie_side_table_dirs); } - trie_side_table_chunks = NULL; + trie_side_table_dirs = NULL; WRITE_ONCE(trie_side_table_high_water, 0); + WRITE_ONCE(trie_side_table_nr_dirs, 0); WRITE_ONCE(trie_side_table_nr_chunks, 0); - WRITE_ONCE(trie_side_table_top_size, 0); + WRITE_ONCE(trie_side_table_root_size, 0); + WRITE_ONCE(trie_side_table_max_id, 0); WRITE_ONCE(trie_side_table_next_id, 0); WRITE_ONCE(trie_side_table_memblock, false); WRITE_ONCE(trie_side_table_initialized, false); @@ -604,76 +689,153 @@ void __stack_depot_trie_side_table_destroy(void) bool __stack_depot_trie_side_table_prealloc_needed(void) { + struct stack_depot_trie_side_dir *dir; unsigned long flags; bool needed; u32 id; - unsigned int top; + unsigned int root; if (!trie_side_table_is_initialized()) return false; raw_spin_lock_irqsave(&trie_side_table_lock, flags); id = READ_ONCE(trie_side_table_next_id) + 1; - if (!id || id > __stack_depot_trie_max_leaf_id()) { + if (!id || id > READ_ONCE(trie_side_table_max_id)) { needed = false; goto out; } - top = trie_side_table_top_index(id); - if (top >= trie_side_table_top_size) { + root = trie_side_table_root_index(id); + if (root >= trie_side_table_root_size) { needed = false; goto out; } - needed = !trie_side_table_load_chunk(top); + dir = trie_side_table_load_dir(root); + needed = !dir || !trie_side_table_dir_load_chunk(dir, + trie_side_table_dir_index(id)); out: raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); return needed; } -void *__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags) +static void *trie_side_table_alloc_page(gfp_t gfp_flags, unsigned int order) { struct page *page; page = alloc_pages(gfp_nested_mask(gfp_flags) | __GFP_ZERO, - trie_side_table_chunk_order()); + order); return page ? page_address(page) : NULL; } -void __stack_depot_trie_side_table_free_prealloc(void *prealloc) +int +__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, + struct stack_depot_trie_side_prealloc *prealloc) +{ + struct stack_depot_trie_side_dir *dir; + unsigned long flags; + bool need_chunk; + bool need_dir; + u32 id; + unsigned int root; + + if (!prealloc || prealloc->dir || prealloc->chunk) + return -EINVAL; + if (!trie_side_table_is_initialized()) + return 0; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + id = READ_ONCE(trie_side_table_next_id) + 1; + if (!id || id > READ_ONCE(trie_side_table_max_id)) { + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return 0; + } + root = trie_side_table_root_index(id); + if (root >= trie_side_table_root_size) { + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return 0; + } + dir = trie_side_table_load_dir(root); + need_dir = !dir; + need_chunk = need_dir || !trie_side_table_dir_load_chunk(dir, + trie_side_table_dir_index(id)); + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + + if (need_dir) { + unsigned int order = trie_side_table_dir_order(); + + prealloc->dir = trie_side_table_alloc_page(gfp_flags, order); + if (!prealloc->dir) + return -ENOMEM; + } + if (need_chunk) { + unsigned int order = trie_side_table_chunk_order(); + + prealloc->chunk = trie_side_table_alloc_page(gfp_flags, order); + if (!prealloc->chunk) { + trie_side_table_free_dir(prealloc->dir); + prealloc->dir = NULL; + return -ENOMEM; + } + } + + return 0; +} + +void +__stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_prealloc *prealloc) { - trie_side_table_free_chunk(prealloc); + if (!prealloc) + return; + trie_side_table_free_dir(prealloc->dir); + trie_side_table_free_chunk(prealloc->chunk); + prealloc->dir = NULL; + prealloc->chunk = NULL; } -u32 __stack_depot_trie_side_table_alloc_id(void **prealloc) +u32 +__stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc) { struct stack_depot_trie_side_entry *chunk; + struct stack_depot_trie_side_dir *dir; unsigned long flags; + unsigned int root; + unsigned int idx; u32 id; - unsigned int top; if (!trie_side_table_is_initialized()) return 0; raw_spin_lock_irqsave(&trie_side_table_lock, flags); id = trie_side_table_next_id + 1; - if (!id || id > __stack_depot_trie_max_leaf_id()) + if (!id || id > trie_side_table_max_id) goto fail; - top = trie_side_table_top_index(id); - if (top >= trie_side_table_top_size) + root = trie_side_table_root_index(id); + if (root >= trie_side_table_root_size) goto fail; - chunk = trie_side_table_load_chunk(top); + dir = trie_side_table_load_dir(root); + if (!dir) { + if (!prealloc || !prealloc->dir) + goto fail; + dir = prealloc->dir; + prealloc->dir = NULL; + trie_side_table_publish_dir(root, dir); + trie_side_table_nr_dirs++; + if (trie_side_table_high_water < root + 1) + trie_side_table_high_water = root + 1; + } + + idx = trie_side_table_dir_index(id); + chunk = trie_side_table_dir_load_chunk(dir, idx); if (!chunk) { - if (!prealloc || !*prealloc) + if (!prealloc || !prealloc->chunk) goto fail; - chunk = *prealloc; - *prealloc = NULL; - trie_side_table_publish_chunk(top, chunk); + chunk = prealloc->chunk; + prealloc->chunk = NULL; + trie_side_table_dir_publish_chunk(dir, idx, chunk); trie_side_table_nr_chunks++; - if (trie_side_table_high_water < top + 1) - trie_side_table_high_water = top + 1; } WRITE_ONCE(trie_side_table_next_id, id); @@ -687,9 +849,10 @@ u32 __stack_depot_trie_side_table_alloc_id(void **prealloc) void __stack_depot_trie_side_table_revoke_latest(u32 id) { struct stack_depot_trie_side_entry *chunk; + struct stack_depot_trie_side_dir *dir; unsigned long flags; unsigned int slot; - unsigned int top; + unsigned int root; if (!trie_side_table_is_initialized() || !id || id != READ_ONCE(trie_side_table_next_id)) @@ -698,11 +861,14 @@ void __stack_depot_trie_side_table_revoke_latest(u32 id) raw_spin_lock_irqsave(&trie_side_table_lock, flags); if (id != trie_side_table_next_id) goto out; - top = trie_side_table_top_index(id); - if (top >= trie_side_table_top_size) + root = trie_side_table_root_index(id); + if (root >= trie_side_table_root_size) goto out; - chunk = trie_side_table_load_chunk(top); + dir = trie_side_table_load_dir(root); + if (!dir) + goto out; + chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); if (!chunk) goto out; @@ -716,8 +882,9 @@ void __stack_depot_trie_side_table_revoke_latest(u32 id) void __stack_depot_trie_side_table_restore(u32 id, const void *entry) { struct stack_depot_trie_side_entry *chunk; + struct stack_depot_trie_side_dir *dir; unsigned long flags; - unsigned int top; + unsigned int root; if (!trie_side_table_is_initialized() || !id) return; @@ -725,11 +892,14 @@ void __stack_depot_trie_side_table_restore(u32 id, const void *entry) raw_spin_lock_irqsave(&trie_side_table_lock, flags); if (id > trie_side_table_next_id) goto out; - top = trie_side_table_top_index(id); - if (top >= trie_side_table_top_size) + root = trie_side_table_root_index(id); + if (root >= trie_side_table_root_size) goto out; - chunk = trie_side_table_load_chunk(top); + dir = trie_side_table_load_dir(root); + if (!dir) + goto out; + chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); if (!chunk) goto out; @@ -744,8 +914,9 @@ void __stack_depot_trie_side_table_restore(u32 id, const void *entry) int __stack_depot_trie_side_table_store(u32 id, const void *entry) { struct stack_depot_trie_side_entry *chunk; + struct stack_depot_trie_side_dir *dir; unsigned long flags; - unsigned int top; + unsigned int root; int ret = -EINVAL; if (!trie_side_table_is_initialized() || !id || !entry) @@ -754,11 +925,14 @@ int __stack_depot_trie_side_table_store(u32 id, const void *entry) raw_spin_lock_irqsave(&trie_side_table_lock, flags); if (id > trie_side_table_next_id) goto out; - top = trie_side_table_top_index(id); - if (top >= trie_side_table_top_size) + root = trie_side_table_root_index(id); + if (root >= trie_side_table_root_size) goto out; - chunk = trie_side_table_load_chunk(top); + dir = trie_side_table_load_dir(root); + if (!dir) + goto out; + chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); if (!chunk) goto out; @@ -772,16 +946,20 @@ int __stack_depot_trie_side_table_store(u32 id, const void *entry) const void *__stack_depot_trie_side_table_lookup(u32 id) { struct stack_depot_trie_side_entry *chunk; - unsigned int top; + struct stack_depot_trie_side_dir *dir; + unsigned int root; if (!trie_side_table_is_initialized() || !id) return NULL; - top = trie_side_table_top_index(id); - if (top >= trie_side_table_top_size) + root = trie_side_table_root_index(id); + if (root >= trie_side_table_root_size) return NULL; - chunk = trie_side_table_load_chunk(top); + dir = trie_side_table_load_dir(root); + if (!dir) + return NULL; + chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); if (!chunk) return NULL; @@ -796,21 +974,29 @@ size_t __stack_depot_trie_side_table_entries(void) size_t __stack_depot_trie_side_table_bytes(void) { + unsigned int nr_dirs; unsigned int nr_chunks; + size_t dir_bytes; size_t bytes; - size_t top_bytes; + size_t root_bytes; if (!trie_side_table_is_initialized()) return 0; - top_bytes = trie_side_table_top_bytes(trie_side_table_top_size); - if (!top_bytes) + root_bytes = trie_side_table_root_bytes(trie_side_table_root_size); + if (!root_bytes) return SIZE_MAX; + nr_dirs = READ_ONCE(trie_side_table_nr_dirs); nr_chunks = READ_ONCE(trie_side_table_nr_chunks); + if (check_mul_overflow((size_t)nr_dirs, trie_side_table_dir_bytes(), + &dir_bytes)) + return SIZE_MAX; if (check_mul_overflow((size_t)nr_chunks, trie_side_table_chunk_bytes(), &bytes)) return SIZE_MAX; - if (check_add_overflow(top_bytes, bytes, &bytes)) + if (check_add_overflow(root_bytes, dir_bytes, &dir_bytes)) + return SIZE_MAX; + if (check_add_overflow(dir_bytes, bytes, &bytes)) return SIZE_MAX; return bytes; @@ -849,12 +1035,14 @@ void __stack_depot_trie_pool_free_prealloc(void *prealloc) int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, void **pool_prealloc, - void **side_prealloc) + struct stack_depot_trie_side_prealloc *side_prealloc) { bool needs_side_prealloc; bool can_alloc; + int ret = 0; - if (!pool_prealloc || !side_prealloc || *pool_prealloc || *side_prealloc) + if (!pool_prealloc || !side_prealloc || *pool_prealloc || + side_prealloc->dir || side_prealloc->chunk) return -EINVAL; can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && @@ -862,10 +1050,12 @@ int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, needs_side_prealloc = __stack_depot_trie_side_table_prealloc_needed(); if (can_alloc && !READ_ONCE(new_pool)) *pool_prealloc = __stack_depot_trie_pool_prealloc(alloc_flags); + if (needs_side_prealloc && !can_alloc) + return -ENOSPC; if (can_alloc && needs_side_prealloc) - *side_prealloc = __stack_depot_trie_side_table_prealloc(alloc_flags); + ret = __stack_depot_trie_side_table_prealloc(alloc_flags, side_prealloc); - if (needs_side_prealloc && !*side_prealloc) + if (needs_side_prealloc && ret) return -ENOSPC; return 0; } @@ -1048,7 +1238,8 @@ void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn) } int -__stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, void **prealloc) +__stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, + struct stack_depot_trie_side_prealloc *prealloc) { u32 leaf_id; @@ -1121,7 +1312,7 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, unsigned int nr_child_slots, struct stack_depot_trie_alloc_txn *txn, void **storage, void **pool_prealloc, - void **side_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_request *req) { unsigned int nr_child_used; @@ -1158,7 +1349,8 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, static int trie_ws_plan(const struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, - void **pool_prealloc, void **side_prealloc, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_workspace *workspace) { if (!workspace) @@ -1175,7 +1367,7 @@ trie_ws_plan(const struct stack_depot_trie_root *root, int __stack_depot_trie_workspace_plan(const struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, - void **side_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_workspace *workspace) { return trie_ws_plan(root, entries, nr_entries, pool_prealloc, side_prealloc, @@ -1185,15 +1377,14 @@ int __stack_depot_trie_workspace_plan(const struct stack_depot_trie_root *root, int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, - void **side_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_workspace *workspace, const void **tail, u32 *leaf_id) { void **pool = pool_prealloc; - void **side = side_prealloc; int ret; - ret = trie_ws_plan(root, entries, nr_entries, pool, side, workspace); + ret = trie_ws_plan(root, entries, nr_entries, pool, side_prealloc, workspace); if (ret) return ret; @@ -1204,7 +1395,8 @@ int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, static int trie_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, - void **pool_prealloc, void **side_prealloc) + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { return __stack_depot_trie_alloc_prealloc(alloc_flags, depot_flags, pool_prealloc, side_prealloc); @@ -1212,7 +1404,8 @@ trie_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, static int trie_ws_insert(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, - void **pool_prealloc, void **side_prealloc, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_workspace *workspace, const void **tail, u32 *leaf_id) { @@ -1245,7 +1438,7 @@ trie_save_miss(struct stack_depot_trie_root *root, const unsigned long *entries, { depot_stack_handle_t handle = 0; void *pool_prealloc = NULL; - void *side_prealloc = NULL; + struct stack_depot_trie_side_prealloc side_prealloc = {}; const void *tail; u32 leaf_id; int ret; @@ -1271,7 +1464,7 @@ trie_save_miss(struct stack_depot_trie_root *root, const unsigned long *entries, out: depot_try_keep_new_pool(&pool_prealloc); __stack_depot_trie_pool_free_prealloc(pool_prealloc); - __stack_depot_trie_side_table_free_prealloc(side_prealloc); + __stack_depot_trie_side_table_free_prealloc(&side_prealloc); return handle; } @@ -1311,7 +1504,8 @@ static depot_stack_handle_t trie_save_locked_insert(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_alloc_workspace *workspace, - void **pool_prealloc, void **side_prealloc, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, bool can_insert) { depot_stack_handle_t handle; @@ -1335,7 +1529,8 @@ trie_save_trylocked(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_alloc_workspace *workspace, raw_spinlock_t *workspace_lock, void **pool_prealloc, - void **side_prealloc, bool can_insert) + struct stack_depot_trie_side_prealloc *side_prealloc, + bool can_insert) { depot_stack_handle_t handle; unsigned long flags; @@ -1354,7 +1549,8 @@ trie_save_spinlocked(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_alloc_workspace *workspace, raw_spinlock_t *workspace_lock, void **pool_prealloc, - void **side_prealloc, bool can_insert) + struct stack_depot_trie_side_prealloc *side_prealloc, + bool can_insert) { depot_stack_handle_t handle; unsigned long flags; @@ -1376,7 +1572,7 @@ __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, { depot_stack_handle_t handle = 0; void *pool_prealloc = NULL; - void *side_prealloc = NULL; + struct stack_depot_trie_side_prealloc side_prealloc = {}; bool can_insert; int ret; @@ -1405,7 +1601,7 @@ __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, &side_prealloc, can_insert); __stack_depot_trie_pool_free_prealloc(pool_prealloc); - __stack_depot_trie_side_table_free_prealloc(side_prealloc); + __stack_depot_trie_side_table_free_prealloc(&side_prealloc); return handle; } @@ -2137,6 +2333,7 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, void *prealloc = NULL; bool allow_spin = gfpflags_allow_spinning(alloc_flags); bool can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && allow_spin; + bool trie_candidate; unsigned long flags; u32 hash; @@ -2163,14 +2360,22 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, found = find_stack(bucket, entries, nr_entries, hash, depot_flags); if (found) goto exit; - if (__stack_depot_trie_ready() && - !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH)) && - nr_entries <= CONFIG_STACKDEPOT_MAX_FRAMES) { - handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); - if (handle) - return handle; - handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, depot_flags); - return handle; + + trie_candidate = __stack_depot_trie_ready() && + !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH)) && + nr_entries <= CONFIG_STACKDEPOT_MAX_FRAMES; + if (trie_candidate) { + if (in_nmi() || !allow_spin) { + handle = trie_find_handle(&stack_depot_trie_root, entries, + nr_entries); + if (handle) + return handle; + } else { + handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, + depot_flags); + if (handle) + return handle; + } } /* @@ -5178,7 +5383,6 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, { unsigned long *stack_entries; unsigned int nr_entries; - unsigned int copied = 0; if (!handle || !entries || !max_entries) return 0; @@ -5186,13 +5390,11 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, return 0; if (__stack_depot_trie_leaf_id(handle)) return __stack_depot_trie_fetch_handle_into(handle, entries, - max_entries); + max_entries); - /* Hold RCU so the fetched record cannot be reused during the copy. */ - rcu_read_lock_sched_notrace(); nr_entries = stack_depot_fetch(handle, &stack_entries); if (!nr_entries || nr_entries > max_entries) - goto out; + return 0; /* * stack_depot_fetch() returns stackdepot-owned storage; the caller must @@ -5200,11 +5402,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, */ memcpy(entries, stack_entries, nr_entries * sizeof(*entries)); kmsan_unpoison_memory(entries, nr_entries * sizeof(*entries)); - copied = nr_entries; - -out: - rcu_read_unlock_sched_notrace(); - return copied; + return nr_entries; } EXPORT_SYMBOL_GPL(stack_depot_fetch_into); @@ -5266,8 +5464,9 @@ depot_stack_handle_t __must_check stack_depot_set_extra_bits(depot_stack_handle_ { union handle_parts parts = { .handle = handle }; - /* Don't set extra bits on empty handles. */ - if (!handle) + /* Do not set extra bits on empty handles. */ + parts.extra = 0; + if (!parts.handle) return 0; parts.extra = extra_bits; @@ -5290,8 +5489,10 @@ static int stats_show(struct seq_file *seq, void *v) * statistics are ok for debugging. */ seq_printf(seq, "pools: %d\n", data_race(pools_num)); + /* data race ok: counters are approximate debugfs statistics. */ for (int i = 0; i < DEPOT_COUNTER_COUNT; i++) - seq_printf(seq, "%s: %ld\n", counter_names[i], READ_ONCE(counters[i])); + seq_printf(seq, "%s: %ld\n", counter_names[i], + data_race(counters[i])); /* Statistic. */ seq_printf(seq, "trie_side_table_bytes: %zu\n", __stack_depot_trie_side_table_bytes()); diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 85b6173e406d6..14f0b6c77bcb3 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -84,6 +84,11 @@ struct stack_depot_trie_side_prepare { unsigned int nr_updates; }; +struct stack_depot_trie_side_prealloc { + void *dir; + void *chunk; +}; + struct stack_depot_trie_pool_mark { void *pool; size_t prev_offset; @@ -116,7 +121,7 @@ struct stack_depot_trie_alloc_request { struct stack_depot_trie_child_array_slot *child_slots; void **storage; void **pool_prealloc; - void **side_prealloc; + struct stack_depot_trie_side_prealloc *side_prealloc; size_t storage_size; unsigned int nr_node_slots; unsigned int nr_child_slots; @@ -148,9 +153,13 @@ u32 __stack_depot_trie_max_leaf_id(void); int __stack_depot_trie_side_table_init(gfp_t gfp_flags); void __stack_depot_trie_side_table_destroy(void); bool __stack_depot_trie_side_table_prealloc_needed(void); -void *__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags); -void __stack_depot_trie_side_table_free_prealloc(void *prealloc); -u32 __stack_depot_trie_side_table_alloc_id(void **prealloc); +int +__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, + struct stack_depot_trie_side_prealloc *prealloc); +void +__stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_prealloc *prealloc); +u32 +__stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc); void __stack_depot_trie_side_table_revoke_latest(u32 id); void __stack_depot_trie_side_table_restore(u32 id, const void *entry); int __stack_depot_trie_side_table_store(u32 id, const void *entry); @@ -163,7 +172,7 @@ void __stack_depot_trie_pool_free_prealloc(void *prealloc); int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, void **pool_prealloc, - void **side_prealloc); + struct stack_depot_trie_side_prealloc *side_prealloc); /* * Best-effort current-pool helpers. They never allocate or roll over to a new * pool, and they use trylock so constrained contexts fail instead of blocking. @@ -175,7 +184,8 @@ bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mar int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req); void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); int -__stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, void **prealloc); +__stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, + struct stack_depot_trie_side_prealloc *prealloc); int __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, const unsigned long *entries, @@ -186,17 +196,17 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, unsigned int nr_child_slots, struct stack_depot_trie_alloc_txn *txn, void **storage, void **pool_prealloc, - void **side_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_request *req); int __stack_depot_trie_workspace_plan(const struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, - void **side_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_workspace *workspace); int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, - void **side_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_workspace *workspace, const void **tail, u32 *leaf_id); depot_stack_handle_t diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 883b67bdbc71e..05106c5345d38 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -540,6 +540,7 @@ static void stackdepot_trie_handle_namespace(struct kunit *test) 0x1234567800730000UL, }; depot_stack_handle_t boundary_handle; + depot_stack_handle_t extra_only; depot_stack_handle_t hash_handle; depot_stack_handle_t tagged; depot_stack_handle_t trie; @@ -553,6 +554,10 @@ static void stackdepot_trie_handle_namespace(struct kunit *test) KUNIT_EXPECT_NE(test, hash_handle, (depot_stack_handle_t)0); KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(hash_handle), 0U); KUNIT_EXPECT_EQ(test, __stack_depot_trie_handle(0), (depot_stack_handle_t)0); + extra_only = (depot_stack_handle_t)7 << + (DEPOT_HANDLE_BITS - STACK_DEPOT_EXTRA_BITS); + KUNIT_EXPECT_EQ(test, stack_depot_set_extra_bits(extra_only, 1), + (depot_stack_handle_t)0); if (!trie) { KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(0), 0U); @@ -637,18 +642,28 @@ static void stackdepot_trie_side_table_init_or_skip(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); } +static void +stackdepot_trie_side_table_prealloc_or_fail(struct kunit *test, + struct stack_depot_trie_side_prealloc *prealloc) +{ + int ret; + + ret = __stack_depot_trie_side_table_prealloc(GFP_KERNEL, prealloc); + KUNIT_ASSERT_EQ(test, ret, 0); +} + static u32 stackdepot_trie_side_table_alloc(struct kunit *test) { - void *prealloc = NULL; + struct stack_depot_trie_side_prealloc prealloc = {}; u32 id; if (__stack_depot_trie_side_table_prealloc_needed()) { - prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, prealloc); + stackdepot_trie_side_table_prealloc_or_fail(test, &prealloc); + KUNIT_ASSERT_TRUE(test, prealloc.dir || prealloc.chunk); } id = __stack_depot_trie_side_table_alloc_id(&prealloc); - __stack_depot_trie_side_table_free_prealloc(prealloc); + __stack_depot_trie_side_table_free_prealloc(&prealloc); return id; } @@ -759,7 +774,7 @@ static void stackdepot_trie_side_table_restore(struct kunit *test) static void stackdepot_trie_side_table_chunk_boundary(struct kunit *test) { - void *prealloc = NULL; + struct stack_depot_trie_side_prealloc prealloc = {}; u32 id = 0; u32 i; @@ -771,29 +786,31 @@ static void stackdepot_trie_side_table_chunk_boundary(struct kunit *test) KUNIT_ASSERT_TRUE(test, __stack_depot_trie_side_table_prealloc_needed()); KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_alloc_id(NULL), 0U); - prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, prealloc); + stackdepot_trie_side_table_prealloc_or_fail(test, &prealloc); + KUNIT_ASSERT_NOT_NULL(test, prealloc.chunk); id = __stack_depot_trie_side_table_alloc_id(&prealloc); - KUNIT_EXPECT_NULL(test, prealloc); + KUNIT_EXPECT_NULL(test, prealloc.dir); + KUNIT_EXPECT_NULL(test, prealloc.chunk); KUNIT_EXPECT_EQ(test, id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE + 1); } static void stackdepot_trie_side_table_bytes(struct kunit *test) { + struct stack_depot_trie_side_prealloc prealloc = {}; size_t before; size_t after; - void *prealloc; u32 id; stackdepot_trie_side_table_init_or_skip(test); before = __stack_depot_trie_side_table_bytes(); KUNIT_EXPECT_GT(test, before, 0UL); - prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, prealloc); + stackdepot_trie_side_table_prealloc_or_fail(test, &prealloc); + KUNIT_ASSERT_TRUE(test, prealloc.dir || prealloc.chunk); KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_bytes(), before); id = __stack_depot_trie_side_table_alloc_id(&prealloc); - KUNIT_EXPECT_NULL(test, prealloc); + KUNIT_EXPECT_NULL(test, prealloc.dir); + KUNIT_EXPECT_NULL(test, prealloc.chunk); KUNIT_ASSERT_EQ(test, id, 1U); after = __stack_depot_trie_side_table_bytes(); KUNIT_EXPECT_GT(test, after, before); @@ -993,14 +1010,15 @@ static void stackdepot_trie_pool_prealloc(struct kunit *test) } static int alloc_prealloc_flags(gfp_t gfp_flags, depot_flags_t depot_flags, - void **pool_prealloc, void **side_prealloc) + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { return __stack_depot_trie_alloc_prealloc(gfp_flags, depot_flags, pool_prealloc, side_prealloc); } static int alloc_prealloc(gfp_t gfp_flags, void **pool_prealloc, - void **side_prealloc) + struct stack_depot_trie_side_prealloc *side_prealloc) { return alloc_prealloc_flags(gfp_flags, STACK_DEPOT_FLAG_CAN_ALLOC, pool_prealloc, side_prealloc); @@ -1008,8 +1026,8 @@ static int alloc_prealloc(gfp_t gfp_flags, void **pool_prealloc, static void stackdepot_trie_alloc_prealloc(struct kunit *test) { + struct stack_depot_trie_side_prealloc side_prealloc = {}; void *pool_prealloc = NULL; - void *side_prealloc = NULL; u32 id; int ret; @@ -1017,22 +1035,23 @@ static void stackdepot_trie_alloc_prealloc(struct kunit *test) ret = alloc_prealloc_flags(GFP_NOWAIT, 0, &pool_prealloc, &side_prealloc); KUNIT_EXPECT_EQ(test, ret, -ENOSPC); KUNIT_EXPECT_NULL(test, pool_prealloc); - KUNIT_EXPECT_NULL(test, side_prealloc); + KUNIT_EXPECT_NULL(test, side_prealloc.dir); + KUNIT_EXPECT_NULL(test, side_prealloc.chunk); ret = alloc_prealloc(GFP_KERNEL, &pool_prealloc, &side_prealloc); KUNIT_EXPECT_EQ(test, ret, 0); - KUNIT_EXPECT_NOT_NULL(test, side_prealloc); + KUNIT_EXPECT_TRUE(test, side_prealloc.dir || side_prealloc.chunk); __stack_depot_trie_pool_free_prealloc(pool_prealloc); - __stack_depot_trie_side_table_free_prealloc(side_prealloc); + __stack_depot_trie_side_table_free_prealloc(&side_prealloc); pool_prealloc = NULL; - side_prealloc = NULL; id = stackdepot_trie_side_table_alloc(test); KUNIT_ASSERT_EQ(test, id, 1U); ret = alloc_prealloc(GFP_NOWAIT, &pool_prealloc, &side_prealloc); KUNIT_EXPECT_EQ(test, ret, 0); KUNIT_EXPECT_NULL(test, pool_prealloc); - KUNIT_EXPECT_NULL(test, side_prealloc); + KUNIT_EXPECT_NULL(test, side_prealloc.dir); + KUNIT_EXPECT_NULL(test, side_prealloc.chunk); pool_prealloc = (void *)0x1111UL; ret = alloc_prealloc(GFP_KERNEL, &pool_prealloc, &side_prealloc); @@ -1278,20 +1297,19 @@ static void stackdepot_trie_pool_carve_no_prealloc_rollover(struct kunit *test) static void stackdepot_trie_alloc_txn_id(struct kunit *test) { + struct stack_depot_trie_side_prealloc prealloc = {}; struct stack_depot_trie_alloc_txn txn; - void *prealloc = NULL; int ret; stackdepot_trie_side_table_init_or_skip(test); - if (__stack_depot_trie_side_table_prealloc_needed()) { - prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, prealloc); - } + if (__stack_depot_trie_side_table_prealloc_needed()) + stackdepot_trie_side_table_prealloc_or_fail(test, &prealloc); __stack_depot_trie_alloc_txn_init(&txn); ret = __stack_depot_trie_alloc_txn_id(&txn, &prealloc); KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_NULL(test, prealloc); + KUNIT_EXPECT_NULL(test, prealloc.dir); + KUNIT_EXPECT_NULL(test, prealloc.chunk); KUNIT_EXPECT_EQ(test, txn.leaf_id, 1U); KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); ret = __stack_depot_trie_alloc_txn_id(&txn, NULL); @@ -1305,18 +1323,16 @@ static void stackdepot_trie_alloc_txn_id(struct kunit *test) static void stackdepot_trie_alloc_txn_reserve(struct kunit *test) { struct stack_depot_trie_node_slot node_slot = { .size = 1 }; + struct stack_depot_trie_side_prealloc side_prealloc = {}; struct stack_depot_trie_alloc_txn txn; struct stack_depot_trie_alloc_request req; - void *side_prealloc = NULL; void *storage = NULL; int ret; stackdepot_trie_side_table_init_or_skip(test); stackdepot_trie_pool_seed_current_pool(test); - if (__stack_depot_trie_side_table_prealloc_needed()) { - side_prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, side_prealloc); - } + if (__stack_depot_trie_side_table_prealloc_needed()) + stackdepot_trie_side_table_prealloc_or_fail(test, &side_prealloc); __stack_depot_trie_alloc_txn_init(&txn); req = (struct stack_depot_trie_alloc_request) { @@ -1329,7 +1345,8 @@ static void stackdepot_trie_alloc_txn_reserve(struct kunit *test) }; ret = __stack_depot_trie_alloc_txn_reserve(&req); KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_NULL(test, side_prealloc); + KUNIT_EXPECT_NULL(test, side_prealloc.dir); + KUNIT_EXPECT_NULL(test, side_prealloc.chunk); KUNIT_EXPECT_EQ(test, txn.leaf_id, 1U); KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); KUNIT_EXPECT_NOT_NULL(test, node_slot.node); @@ -1375,20 +1392,18 @@ static void stackdepot_trie_alloc_txn_reserve_id_failure(struct kunit *test) static void stackdepot_trie_alloc_txn_commit(struct kunit *test) { struct stack_depot_trie_leaf_update updates[1]; + struct stack_depot_trie_side_prealloc prealloc = {}; struct stack_depot_trie_alloc_txn txn; const void *old_leaf = (const void *)0x1111UL; void *pool_leaf; - void *prealloc = NULL; u32 old_id; u32 leaf_id; int ret; stackdepot_trie_side_table_init_or_skip(test); stackdepot_trie_pool_seed_current_pool(test); - if (__stack_depot_trie_side_table_prealloc_needed()) { - prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, prealloc); - } + if (__stack_depot_trie_side_table_prealloc_needed()) + stackdepot_trie_side_table_prealloc_or_fail(test, &prealloc); __stack_depot_trie_alloc_txn_init(&txn); old_id = stackdepot_trie_side_table_alloc(test); @@ -1477,7 +1492,7 @@ static int txn_insert_plan(struct stack_depot_trie_root *root, unsigned int nr_child_slots, struct stack_depot_trie_alloc_txn *txn, void **storage, void **pool_prealloc, - void **side_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_request *req) { return __stack_depot_trie_alloc_txn_plan(root, entries, nr_entries, @@ -1498,7 +1513,8 @@ static int txn_insert(struct stack_depot_trie_root *root, static int workspace_plan(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, - void **pool_prealloc, void **side_prealloc, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_workspace *workspace) { return __stack_depot_trie_workspace_plan(root, entries, nr_entries, @@ -1508,12 +1524,11 @@ static int workspace_plan(struct stack_depot_trie_root *root, static int ws_insert_prealloc(struct stack_depot_trie_root *root, struct stack_depot_trie_alloc_workspace *workspace, const unsigned long *entries, unsigned int nr_entries, - void **side_prealloc, const void **tail, u32 *leaf_id) + struct stack_depot_trie_side_prealloc *side_prealloc, + const void **tail, u32 *leaf_id) { - void **side = side_prealloc; - return __stack_depot_trie_workspace_insert(root, entries, nr_entries, NULL, - side, workspace, tail, leaf_id); + side_prealloc, workspace, tail, leaf_id); } static depot_stack_handle_t save_miss(struct stack_depot_trie_root *root, @@ -1574,9 +1589,11 @@ static void stackdepot_trie_alloc_workspace_plan(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_side_prealloc side_prealloc = { + .chunk = (void *)0x2222UL, + }; struct stack_depot_trie_root root = {}; void *pool_prealloc = (void *)0x1111UL; - void *side_prealloc = (void *)0x2222UL; const void *tail = NULL; u32 leaf_id = 0; int ret; @@ -1601,10 +1618,9 @@ static void stackdepot_trie_alloc_workspace_plan(struct kunit *test) stackdepot_trie_side_table_init_or_skip(test); stackdepot_trie_pool_seed_current_pool(test); - if (__stack_depot_trie_side_table_prealloc_needed()) { - side_prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, side_prealloc); - } + side_prealloc = (struct stack_depot_trie_side_prealloc) {}; + if (__stack_depot_trie_side_table_prealloc_needed()) + stackdepot_trie_side_table_prealloc_or_fail(test, &side_prealloc); ret = workspace_plan(&root, entries, ARRAY_SIZE(entries), NULL, &side_prealloc, workspace); KUNIT_ASSERT_EQ(test, ret, 0); @@ -1620,9 +1636,9 @@ static void stackdepot_trie_alloc_workspace_insert(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_side_prealloc side_prealloc = {}; struct stack_depot_trie_root root = {}; unsigned long out[ARRAY_SIZE(entries)] = {}; - void *side_prealloc = NULL; const void *tail = NULL; unsigned int fetched; u32 leaf_id = 0; @@ -1632,10 +1648,8 @@ static void stackdepot_trie_alloc_workspace_insert(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, workspace); stackdepot_trie_side_table_init_or_skip(test); stackdepot_trie_pool_seed_current_pool(test); - if (__stack_depot_trie_side_table_prealloc_needed()) { - side_prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, side_prealloc); - } + if (__stack_depot_trie_side_table_prealloc_needed()) + stackdepot_trie_side_table_prealloc_or_fail(test, &side_prealloc); ret = ws_insert_prealloc(&root, workspace, entries, ARRAY_SIZE(entries), &side_prealloc, &tail, &leaf_id); @@ -1880,8 +1894,10 @@ static void stackdepot_trie_alloc_txn_plan(struct kunit *test) struct stack_depot_trie_alloc_request req; struct stack_depot_trie_alloc_txn txn; struct stack_depot_trie_root root = {}; + struct stack_depot_trie_side_prealloc side_prealloc = { + .chunk = (void *)0x2222UL, + }; void *pool_prealloc = (void *)0x1111UL; - void *side_prealloc = (void *)0x2222UL; void *storage = (void *)0x3333UL; int ret; @@ -1917,8 +1933,8 @@ static void stackdepot_trie_alloc_txn_insert(struct kunit *test) struct stack_depot_trie_alloc_request req; struct stack_depot_trie_alloc_txn txn; struct stack_depot_trie_root root = {}; + struct stack_depot_trie_side_prealloc side_prealloc = {}; unsigned long out[ARRAY_SIZE(entries)] = {}; - void *side_prealloc = NULL; const void *tail = NULL; void *storage = NULL; unsigned int fetched; @@ -1927,10 +1943,8 @@ static void stackdepot_trie_alloc_txn_insert(struct kunit *test) stackdepot_trie_side_table_init_or_skip(test); stackdepot_trie_pool_seed_current_pool(test); - if (__stack_depot_trie_side_table_prealloc_needed()) { - side_prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, side_prealloc); - } + if (__stack_depot_trie_side_table_prealloc_needed()) + stackdepot_trie_side_table_prealloc_or_fail(test, &side_prealloc); ret = txn_insert_plan(&root, entries, ARRAY_SIZE(entries), node_slots, ARRAY_SIZE(node_slots), child_slots, @@ -1971,7 +1985,7 @@ static void stackdepot_trie_alloc_txn_insert_stale_plan(struct kunit *test) struct stack_depot_trie_alloc_txn txn; struct stack_depot_trie_alloc_txn fresh_txn; struct stack_depot_trie_root root = {}; - void *side_prealloc = NULL; + struct stack_depot_trie_side_prealloc side_prealloc = {}; const void *tail = NULL; const void *fresh_tail = NULL; void *storage = NULL; @@ -1982,10 +1996,8 @@ static void stackdepot_trie_alloc_txn_insert_stale_plan(struct kunit *test) stackdepot_trie_side_table_init_or_skip(test); stackdepot_trie_pool_seed_current_pool(test); - if (__stack_depot_trie_side_table_prealloc_needed()) { - side_prealloc = __stack_depot_trie_side_table_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, side_prealloc); - } + if (__stack_depot_trie_side_table_prealloc_needed()) + stackdepot_trie_side_table_prealloc_or_fail(test, &side_prealloc); ret = txn_insert_plan(&root, first, ARRAY_SIZE(first), fresh_node_slots, ARRAY_SIZE(fresh_node_slots), fresh_child_slots, @@ -5609,7 +5621,7 @@ static void stackdepot_trie_public_save_route(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, fetched, trie_entries, sizeof(trie_entries)); noalloc_handle = stack_depot_save_flags(noalloc_entries, noalloc_nr, no_spin, 0); KUNIT_ASSERT_NE(test, noalloc_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_NE(test, __stack_depot_trie_leaf_id(noalloc_handle), 0U); + KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(noalloc_handle), 0U); KUNIT_EXPECT_EQ(test, stack_depot_save(noalloc_entries, noalloc_nr, GFP_KERNEL), noalloc_handle); From d8774f18366ae28916c20c11ea80096f43995e39 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 15 Jun 2026 17:22:24 +0100 Subject: [PATCH 089/129] KRN-1117: Reuse retired stackdepot trie child arrays Reuse replaced trie child arrays after their RCU grace period has elapsed. Trie insertion publishes replacement child arrays with copy-on-write, so the old arrays are no longer reachable from new trie walks but may still be held by stale RCU readers. Keep those arrays on size-bucketed free lists with RCU state cookies, then recycle them for later child-array allocations once polling confirms the grace period completed. Store the free-list metadata in a small header before the child-array payload so retired arrays can be tracked without modifying payload bytes that stale readers may still observe. Return reused arrays immediately on transaction rollback with a pre-completed RCU cookie, and keep trie node reuse out of this patch because nodes do not have a private metadata header. This reduces pool churn from child-array COW while preserving the existing trie publication and rollback rules. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 280 +++++++++++++++++++++++++++++++++-- lib/tests/stackdepot_kunit.c | 28 ++-- 2 files changed, 287 insertions(+), 21 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index bc7225536a7dd..9ce087d52155f 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -161,6 +162,12 @@ struct stack_depot_trie_child_array { const struct stack_depot_trie_node *children[]; }; +struct stack_depot_trie_free_object { + struct list_head list; + unsigned long rcu_state; + size_t size; +}; + /* Hash table of stored stack records. */ static struct list_head *stack_table; /* Fixed order of the number of table buckets. Used when KASAN is enabled. */ @@ -178,6 +185,11 @@ static int pools_num; static size_t pool_offset = DEPOT_POOL_SIZE; /* Freelist of stack records within stack_pools. */ static LIST_HEAD(free_stacks); + +#define STACK_DEPOT_TRIE_FREE_OBJECT_BUCKETS (DEPOT_POOL_ORDER + PAGE_SHIFT + 1) + +static struct list_head free_trie_objects[STACK_DEPOT_TRIE_FREE_OBJECT_BUCKETS]; +static bool free_trie_objects_initialized; /* The lock must be held when performing pool or freelist modifications. */ static DEFINE_RAW_SPINLOCK(pool_lock); @@ -1015,6 +1027,169 @@ size_t __stack_depot_trie_pool_alloc_size(size_t size) return aligned <= DEPOT_POOL_SIZE ? aligned : 0; } +static size_t trie_object_header_size(void) +{ + return ALIGN(sizeof(struct stack_depot_trie_free_object), + 1UL << DEPOT_STACK_ALIGN); +} + +static size_t trie_object_alloc_size(size_t size) +{ + size_t alloc_size; + + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return 0; + if (check_add_overflow(trie_object_header_size(), size, + &alloc_size)) + return 0; + return alloc_size <= DEPOT_POOL_SIZE ? alloc_size : 0; +} + +static struct stack_depot_trie_free_object *trie_object_header(const void *ptr) +{ + return (void *)ptr - trie_object_header_size(); +} + +static void *trie_object_payload(struct stack_depot_trie_free_object *free) +{ + return (void *)free + trie_object_header_size(); +} + +static void trie_free_object_buckets_init_locked(void) +{ + unsigned int i; + + lockdep_assert_held(&pool_lock); + + if (free_trie_objects_initialized) + return; + for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) + INIT_LIST_HEAD(&free_trie_objects[i]); + free_trie_objects_initialized = true; +} + +static unsigned int trie_free_object_bucket(size_t size) +{ + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return 0; + return min_t(unsigned int, order_base_2(size), + ARRAY_SIZE(free_trie_objects) - 1); +} + +static void *trie_object_init_fresh(void *ptr, size_t size) +{ + struct stack_depot_trie_free_object *free = ptr; + + free->size = __stack_depot_trie_pool_alloc_size(size); + free->rcu_state = 0; + INIT_LIST_HEAD(&free->list); + return trie_object_payload(free); +} + +static bool trie_pool_contains_locked(const void *ptr) +{ + unsigned int pools = READ_ONCE(pools_num); + unsigned int i; + + lockdep_assert_held(&pool_lock); + + if (!stack_pools) + return false; + for (i = 0; i < pools; i++) { + void *pool = stack_pools[i]; + + if (!pool) + continue; + if (ptr >= pool && ptr < pool + DEPOT_POOL_SIZE) + return true; + } + + return false; +} + +static bool trie_pool_mark_contains(const struct stack_depot_trie_pool_mark *mark, + const void *ptr) +{ + const void *start; + const void *end; + + if (!mark || !mark->pool) + return false; + start = mark->pool + mark->offset; + end = start + mark->size; + return ptr >= start && ptr < end; +} + +static void trie_free_object_locked(const void *ptr, unsigned long rcu_state) +{ + struct stack_depot_trie_free_object *free; + unsigned int bucket; + + lockdep_assert_held(&pool_lock); + + if (!ptr) + return; + trie_free_object_buckets_init_locked(); + free = trie_object_header(ptr); + if (!trie_pool_contains_locked(free)) + return; + free->rcu_state = rcu_state; + bucket = trie_free_object_bucket(free->size); + if (poll_state_synchronize_rcu(rcu_state)) + list_add(&free->list, &free_trie_objects[bucket]); + else + list_add_tail(&free->list, &free_trie_objects[bucket]); +} + +static void trie_retire_object(const void *ptr) +{ + struct stack_depot_trie_free_object *free; + unsigned long flags; + + if (!ptr) + return; + + raw_spin_lock_irqsave(&pool_lock, flags); + trie_free_object_buckets_init_locked(); + free = trie_object_header(ptr); + if (trie_pool_contains_locked(free)) { + free->rcu_state = get_state_synchronize_rcu(); + list_add_tail(&free->list, + &free_trie_objects[trie_free_object_bucket(free->size)]); + } + raw_spin_unlock_irqrestore(&pool_lock, flags); +} + +static void *trie_pop_free_object(size_t size) +{ + struct stack_depot_trie_free_object *free; + unsigned int bucket; + unsigned int i; + + lockdep_assert_held(&pool_lock); + + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return NULL; + trie_free_object_buckets_init_locked(); + bucket = trie_free_object_bucket(size); + for (i = bucket; i < ARRAY_SIZE(free_trie_objects); i++) { + list_for_each_entry(free, &free_trie_objects[i], list) { + if (free->size < size) + continue; + if (!poll_state_synchronize_rcu(free->rcu_state)) + break; + list_del_init(&free->list); + free->rcu_state = 0; + return trie_object_payload(free); + } + } + + return NULL; +} + void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags) { struct page *page; @@ -1137,6 +1312,18 @@ bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mar return ret; } +static int trie_pool_add_object_size(size_t size, size_t *total) +{ + size_t alloc_size; + + alloc_size = trie_object_alloc_size(size); + if (!alloc_size) + return -EINVAL; + if (check_add_overflow(*total, alloc_size, total)) + return -EINVAL; + return *total <= DEPOT_POOL_SIZE ? 0 : -EINVAL; +} + static int trie_pool_add_size(size_t size, size_t *total) { size_t alloc_size; @@ -1149,6 +1336,43 @@ static int trie_pool_add_size(size_t size, size_t *total) return *total <= DEPOT_POOL_SIZE ? 0 : -EINVAL; } +static void trie_pool_release_reused_objects_locked(struct stack_depot_trie_pool_request *req) +{ + unsigned long completed; + unsigned int i; + + lockdep_assert_held(&pool_lock); + + if (!req) + return; + completed = get_completed_synchronize_rcu(); + for (i = 0; req->child_slots && i < req->nr_child_slots; i++) { + void *array = req->child_slots[i].array; + + if (array && + !trie_pool_mark_contains(req->mark, + trie_object_header(array))) { + trie_free_object_locked(array, completed); + req->child_slots[i].array = NULL; + } + } + if (req->storage && *req->storage && + !trie_pool_mark_contains(req->mark, + trie_object_header(*req->storage))) { + trie_free_object_locked(*req->storage, completed); + *req->storage = NULL; + } +} + +static void trie_pool_release_reused_objects(struct stack_depot_trie_pool_request *req) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&pool_lock, flags); + trie_pool_release_reused_objects_locked(req); + raw_spin_unlock_irqrestore(&pool_lock, flags); +} + int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) { unsigned long flags; @@ -1173,10 +1397,10 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) } for (i = 0; i < req->nr_child_slots; i++) { if (req->child_slots[i].array || - trie_pool_add_size(req->child_slots[i].size, &total)) + !trie_object_alloc_size(req->child_slots[i].size)) return -EINVAL; } - if (trie_pool_add_size(req->storage_size, &total)) + if (!trie_object_alloc_size(req->storage_size)) return -EINVAL; if (!raw_spin_trylock_irqsave(&pool_lock, flags)) @@ -1186,21 +1410,32 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) ret = -ENOSPC; goto out; } + for (i = 0; i < req->nr_child_slots; i++) { + req->child_slots[i].array = + trie_pop_free_object(req->child_slots[i].size); + if (!req->child_slots[i].array && + trie_pool_add_object_size(req->child_slots[i].size, &total)) + goto out_release_reused; + } + *req->storage = trie_pop_free_object(req->storage_size); + if (!*req->storage && trie_pool_add_object_size(req->storage_size, &total)) + goto out_release_reused; + if (pools_num < 1) { req->mark->prev_offset = pool_offset; if (!depot_init_pool(req->prealloc)) { ret = -ENOSPC; - goto out; + goto out_release_reused; } req->mark->added_pool = true; } if (WARN_ON_ONCE(pool_offset > DEPOT_POOL_SIZE)) - goto out; + goto out_release_reused; if (total > DEPOT_POOL_SIZE - pool_offset) { req->mark->prev_offset = pool_offset; if (!depot_init_pool(req->prealloc)) { ret = -ENOSPC; - goto out; + goto out_release_reused; } req->mark->added_pool = true; } @@ -1208,7 +1443,7 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) req->mark->pool_index = pools_num - 1; pool = stack_pools[req->mark->pool_index]; if (WARN_ON_ONCE(!pool)) - goto out; + goto out_release_reused; req->mark->offset = pool_offset; req->mark->pool = pool; @@ -1219,12 +1454,21 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) offset += __stack_depot_trie_pool_alloc_size(req->node_slots[i].size); } for (i = 0; i < req->nr_child_slots; i++) { - req->child_slots[i].array = pool + offset; - offset += __stack_depot_trie_pool_alloc_size(req->child_slots[i].size); + if (req->child_slots[i].array) + continue; + req->child_slots[i].array = + trie_object_init_fresh(pool + offset, + req->child_slots[i].size); + offset += trie_object_alloc_size(req->child_slots[i].size); } - *req->storage = pool + offset; + if (!*req->storage) + *req->storage = + trie_object_init_fresh(pool + offset, req->storage_size); pool_offset += total; ret = 0; + goto out; +out_release_reused: + trie_pool_release_reused_objects_locked(req); out: printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -1269,6 +1513,19 @@ static void trie_alloc_request_clear_outputs(struct stack_depot_trie_alloc_reque req->child_slots[i].array = NULL; } +static void trie_alloc_request_release_reused_objects(struct stack_depot_trie_alloc_request *req) +{ + struct stack_depot_trie_pool_request pool_req = {}; + + if (!req || !req->txn) + return; + pool_req.child_slots = req->child_slots; + pool_req.nr_child_slots = req->nr_child_slots; + pool_req.storage = req->storage; + pool_req.mark = &req->txn->pool; + trie_pool_release_reused_objects(&pool_req); +} + int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req) { struct stack_depot_trie_pool_request pool_req = {}; @@ -1294,6 +1551,7 @@ int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request * ret = __stack_depot_trie_alloc_txn_id(req->txn, req->side_prealloc); if (ret) { + trie_alloc_request_release_reused_objects(req); __stack_depot_trie_alloc_txn_rollback(req->txn); trie_alloc_request_clear_outputs(req); return ret; @@ -1664,6 +1922,7 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, goto out_unlock; rollback: + trie_alloc_request_release_reused_objects(req); __stack_depot_trie_alloc_txn_rollback(req->txn); trie_alloc_request_clear_outputs(req); *tail = NULL; @@ -3643,6 +3902,7 @@ trie_promote_child(struct stack_depot_trie_root *root, publish_slot = trie_publish_slot(root, parent); /* Publish the fully initialized replacement array last. */ smp_store_release(publish_slot, new_storage); + trie_retire_object(old_array); return 0; } @@ -3859,6 +4119,7 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, /* Publish the fully initialized replacement array last. */ smp_store_release(slot, new_array); + trie_retire_object(old_array); return 0; } @@ -5226,6 +5487,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, trie_child_array_replace_at(old_array, prefix, new_storage, pos); /* Publish the fully initialized replacement array last. */ smp_store_release(publish_slot, new_storage); + trie_retire_object(old_array); *nr_used = used; return 0; } diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 05106c5345d38..8c4032f4e29a1 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1148,7 +1148,7 @@ static void stackdepot_trie_pool_carve_slots(struct kunit *test) size_t child_size; size_t node0_size; size_t node1_size; - size_t total; + size_t old_total; void *again; int ret; @@ -1163,15 +1163,15 @@ static void stackdepot_trie_pool_carve_slots(struct kunit *test) node0_size = __stack_depot_trie_pool_alloc_size(node_slots[0].size); node1_size = __stack_depot_trie_pool_alloc_size(node_slots[1].size); child_size = __stack_depot_trie_pool_alloc_size(child_slots[0].size); + old_total = node0_size + node1_size + child_size + + __stack_depot_trie_pool_alloc_size(1); KUNIT_EXPECT_PTR_EQ(test, node_slots[1].node, (char *)node_slots[0].node + node0_size); - KUNIT_EXPECT_PTR_EQ(test, child_slots[0].array, - (char *)node_slots[1].node + node1_size); - KUNIT_EXPECT_PTR_EQ(test, storage, - (char *)child_slots[0].array + child_size); - total = node0_size + node1_size + child_size + - __stack_depot_trie_pool_alloc_size(1); - KUNIT_EXPECT_EQ(test, mark.size, total); + KUNIT_EXPECT_GT(test, (unsigned long)child_slots[0].array, + (unsigned long)node_slots[1].node + node1_size); + KUNIT_EXPECT_GT(test, (unsigned long)storage, + (unsigned long)child_slots[0].array + child_size); + KUNIT_EXPECT_GT(test, mark.size, old_total); KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&mark)); again = __stack_depot_trie_pool_carve_current(1, &mark); @@ -1222,20 +1222,22 @@ static void stackdepot_trie_pool_carve_uses_prealloc(struct kunit *test) void *first_storage = NULL; void *second_storage = NULL; void *prealloc; + size_t storage_size = DEPOT_POOL_SIZE - 64; struct stack_depot_trie_pool_request first = { .storage = &first_storage, - .storage_size = DEPOT_POOL_SIZE, + .storage_size = DEPOT_POOL_SIZE - 64, .prealloc = &prealloc, .mark = &first_mark, }; struct stack_depot_trie_pool_request second = { .storage = &second_storage, - .storage_size = DEPOT_POOL_SIZE, + .storage_size = DEPOT_POOL_SIZE - 64, .mark = &second_mark, }; int ret; stackdepot_trie_pool_seed_current_pool(test); + KUNIT_ASSERT_GT(test, storage_size, 0UL); prealloc = __stack_depot_trie_pool_prealloc(GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, prealloc); @@ -1270,12 +1272,14 @@ static void stackdepot_trie_pool_carve_no_prealloc_rollover(struct kunit *test) stackdepot_trie_pool_seed_current_pool(test); for (i = 0; i < ARRAY_SIZE(marks); i++) { + size_t storage_size = DEPOT_POOL_SIZE - 64; struct stack_depot_trie_pool_request req = { .storage = &storage[i], - .storage_size = DEPOT_POOL_SIZE, + .storage_size = DEPOT_POOL_SIZE - 64, .mark = &marks[i], }; + KUNIT_ASSERT_GT(test, storage_size, 0UL); storage[i] = NULL; ret = __stack_depot_trie_pool_carve(&req); if (ret) @@ -1284,7 +1288,7 @@ static void stackdepot_trie_pool_carve_no_prealloc_rollover(struct kunit *test) consumed++; } - failed.storage_size = DEPOT_POOL_SIZE; + failed.storage_size = DEPOT_POOL_SIZE - 64; ret = __stack_depot_trie_pool_carve(&failed); KUNIT_EXPECT_EQ(test, ret, -ENOSPC); KUNIT_EXPECT_NULL(test, failed_storage); From 222556bb396a168060705bd1a5dcb40a29f5b0d6 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 16 Jun 2026 11:48:02 +0100 Subject: [PATCH 090/129] KRN-1117: Assert x86 stackdepot frame prefix is nonzero The x86 stackdepot frame codec uses prefix ID 0 for compressed kernel-text frames. Make that single-prefix contract explicit so future changes cannot accidentally make compressed frames indistinguishable from the raw fallback prefix. Signed-off-by: Caleb Kan --- arch/x86/include/asm/stackdepot.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/include/asm/stackdepot.h b/arch/x86/include/asm/stackdepot.h index d61eef4dc1f3f..21c43dec9ad2a 100644 --- a/arch/x86/include/asm/stackdepot.h +++ b/arch/x86/include/asm/stackdepot.h @@ -2,12 +2,15 @@ #ifndef _ASM_X86_STACKDEPOT_H #define _ASM_X86_STACKDEPOT_H +#include #include #ifdef CONFIG_X86_64 #define STACK_DEPOT_X86_64_FRAME_PREFIX 0xffffffff00000000UL #define STACK_DEPOT_X86_64_FRAME_LOW_MASK 0x00000000ffffffffUL +static_assert(STACK_DEPOT_X86_64_FRAME_PREFIX != 0); + static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, u32 *low) { From 3358ba74bb53e0087981c4ca49a7eec7ad8a10c8 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 16 Jun 2026 11:48:39 +0100 Subject: [PATCH 091/129] KRN-1117: Reuse retired stackdepot trie nodes Child-array reuse alone still leaves retired COW trie nodes consuming pool space until the pool allocator advances. Track the retired node in the private child-array header and move it to a separate node free list only after the retired array has passed the existing RCU cookie poll. Also make side-table prepare validate all target slots before publishing any update. That keeps rollback from seeing partially exposed trie nodes and preserves the rule that no RCU-visible node memory is reused before a grace period has completed. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 235 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 210 insertions(+), 25 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 9ce087d52155f..96c578aebada1 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -162,12 +162,22 @@ struct stack_depot_trie_child_array { const struct stack_depot_trie_node *children[]; }; +struct stack_depot_trie_free_node { + struct list_head list; + size_t size; +}; + struct stack_depot_trie_free_object { struct list_head list; unsigned long rcu_state; size_t size; + void *pending_node; + size_t pending_node_size; }; +static_assert(sizeof(struct stack_depot_trie_node) >= + sizeof(struct stack_depot_trie_free_node)); + /* Hash table of stored stack records. */ static struct list_head *stack_table; /* Fixed order of the number of table buckets. Used when KASAN is enabled. */ @@ -181,6 +191,8 @@ static void **stack_pools; static void *new_pool; /* Number of pools in stack_pools. */ static int pools_num; +static unsigned long pools_min_addr; +static unsigned long pools_max_addr; /* Offset to the unused space in the currently used pool. */ static size_t pool_offset = DEPOT_POOL_SIZE; /* Freelist of stack records within stack_pools. */ @@ -189,6 +201,8 @@ static LIST_HEAD(free_stacks); #define STACK_DEPOT_TRIE_FREE_OBJECT_BUCKETS (DEPOT_POOL_ORDER + PAGE_SHIFT + 1) static struct list_head free_trie_objects[STACK_DEPOT_TRIE_FREE_OBJECT_BUCKETS]; +static struct list_head free_trie_nodes[STACK_DEPOT_TRIE_FREE_OBJECT_BUCKETS]; +static unsigned int free_trie_pending_nodes; static bool free_trie_objects_initialized; /* The lock must be held when performing pool or freelist modifications. */ static DEFINE_RAW_SPINLOCK(pool_lock); @@ -955,6 +969,33 @@ int __stack_depot_trie_side_table_store(u32 id, const void *entry) return ret; } +static struct stack_depot_trie_side_entry * +trie_side_table_chunk_locked(u32 id, unsigned int *slot) +{ + struct stack_depot_trie_side_entry *chunk; + struct stack_depot_trie_side_dir *dir; + unsigned int root; + + lockdep_assert_held(&trie_side_table_lock); + + if (!trie_side_table_is_initialized() || !id || id > trie_side_table_next_id) + return NULL; + + root = trie_side_table_root_index(id); + if (root >= trie_side_table_root_size) + return NULL; + + dir = trie_side_table_load_dir(root); + if (!dir) + return NULL; + chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); + if (!chunk) + return NULL; + + *slot = trie_side_table_slot_index(id); + return chunk; +} + const void *__stack_depot_trie_side_table_lookup(u32 id) { struct stack_depot_trie_side_entry *chunk; @@ -1064,8 +1105,10 @@ static void trie_free_object_buckets_init_locked(void) if (free_trie_objects_initialized) return; - for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) + for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) { INIT_LIST_HEAD(&free_trie_objects[i]); + INIT_LIST_HEAD(&free_trie_nodes[i]); + } free_trie_objects_initialized = true; } @@ -1084,31 +1127,62 @@ static void *trie_object_init_fresh(void *ptr, size_t size) free->size = __stack_depot_trie_pool_alloc_size(size); free->rcu_state = 0; + free->pending_node = NULL; + free->pending_node_size = 0; INIT_LIST_HEAD(&free->list); return trie_object_payload(free); } -static bool trie_pool_contains_locked(const void *ptr) +static void depot_record_pool_range_locked(void *pool) +{ + unsigned long start = (unsigned long)pool; + unsigned long end; + + lockdep_assert_held(&pool_lock); + if (check_add_overflow(start, DEPOT_POOL_SIZE, &end)) + return; + + if (!pools_min_addr || start < pools_min_addr) + pools_min_addr = start; + if (end > pools_max_addr) + pools_max_addr = end; +} + +static bool trie_pool_range_contains_locked(const void *ptr, size_t size) { + unsigned long start = (unsigned long)ptr; + unsigned long end; unsigned int pools = READ_ONCE(pools_num); unsigned int i; lockdep_assert_held(&pool_lock); + if (!ptr || !size || check_add_overflow(start, size, &end)) + return false; if (!stack_pools) return false; + if (pools_min_addr && (start < pools_min_addr || end > pools_max_addr)) + return false; + for (i = 0; i < pools; i++) { + unsigned long pool_start; void *pool = stack_pools[i]; if (!pool) continue; - if (ptr >= pool && ptr < pool + DEPOT_POOL_SIZE) + pool_start = (unsigned long)pool; + if (start >= pool_start && end <= pool_start + DEPOT_POOL_SIZE) return true; } return false; } +static bool trie_pool_contains_locked(const void *ptr) +{ + return trie_pool_range_contains_locked(ptr, 1); +} + static bool trie_pool_mark_contains(const struct stack_depot_trie_pool_mark *mark, const void *ptr) { @@ -1143,9 +1217,81 @@ static void trie_free_object_locked(const void *ptr, unsigned long rcu_state) list_add_tail(&free->list, &free_trie_objects[bucket]); } -static void trie_retire_object(const void *ptr) +static void trie_add_free_node_locked(void *ptr, size_t size) +{ + struct stack_depot_trie_free_node *free = ptr; + unsigned int bucket; + + lockdep_assert_held(&pool_lock); + + size = __stack_depot_trie_pool_alloc_size(size); + if (!ptr || size < sizeof(*free)) + return; + + free->size = size; + INIT_LIST_HEAD(&free->list); + bucket = trie_free_object_bucket(size); + list_add(&free->list, &free_trie_nodes[bucket]); +} + +static void trie_drain_free_object_node_locked(struct stack_depot_trie_free_object *free) +{ + lockdep_assert_held(&pool_lock); + + if (!free->pending_node) + return; + trie_add_free_node_locked(free->pending_node, free->pending_node_size); + free->pending_node = NULL; + free->pending_node_size = 0; + free_trie_pending_nodes--; +} + +static void trie_drain_pending_nodes_locked(void) +{ + struct stack_depot_trie_free_object *free; + unsigned int i; + + lockdep_assert_held(&pool_lock); + + if (!free_trie_pending_nodes) + return; + for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) { + list_for_each_entry(free, &free_trie_objects[i], list) { + if (!poll_state_synchronize_rcu(free->rcu_state)) + continue; + trie_drain_free_object_node_locked(free); + } + } +} + +static void *trie_pop_free_node(size_t size) +{ + struct stack_depot_trie_free_node *free; + unsigned int bucket; + + lockdep_assert_held(&pool_lock); + + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return NULL; + trie_free_object_buckets_init_locked(); + trie_drain_pending_nodes_locked(); + bucket = trie_free_object_bucket(size); + list_for_each_entry(free, &free_trie_nodes[bucket], list) { + if (free->size != size) + continue; + list_del_init(&free->list); + return free; + } + + return NULL; +} + +static void trie_retire_object_node(const void *ptr, const void *node, + size_t node_size) { struct stack_depot_trie_free_object *free; + size_t size; unsigned long flags; if (!ptr) @@ -1155,6 +1301,15 @@ static void trie_retire_object(const void *ptr) trie_free_object_buckets_init_locked(); free = trie_object_header(ptr); if (trie_pool_contains_locked(free)) { + free->pending_node = NULL; + free->pending_node_size = 0; + size = __stack_depot_trie_pool_alloc_size(node_size); + if (node && size >= sizeof(struct stack_depot_trie_free_node) && + trie_pool_range_contains_locked(node, size)) { + free->pending_node = (void *)node; + free->pending_node_size = size; + free_trie_pending_nodes++; + } free->rcu_state = get_state_synchronize_rcu(); list_add_tail(&free->list, &free_trie_objects[trie_free_object_bucket(free->size)]); @@ -1162,6 +1317,11 @@ static void trie_retire_object(const void *ptr) raw_spin_unlock_irqrestore(&pool_lock, flags); } +static void trie_retire_object(const void *ptr) +{ + trie_retire_object_node(ptr, NULL, 0); +} + static void *trie_pop_free_object(size_t size) { struct stack_depot_trie_free_object *free; @@ -1181,6 +1341,7 @@ static void *trie_pop_free_object(size_t size) continue; if (!poll_state_synchronize_rcu(free->rcu_state)) break; + trie_drain_free_object_node_locked(free); list_del_init(&free->list); free->rcu_state = 0; return trie_object_payload(free); @@ -1346,6 +1507,14 @@ static void trie_pool_release_reused_objects_locked(struct stack_depot_trie_pool if (!req) return; completed = get_completed_synchronize_rcu(); + for (i = 0; req->node_slots && i < req->nr_node_slots; i++) { + void *node = req->node_slots[i].node; + + if (node && !trie_pool_mark_contains(req->mark, node)) { + trie_add_free_node_locked(node, req->node_slots[i].size); + req->node_slots[i].node = NULL; + } + } for (i = 0; req->child_slots && i < req->nr_child_slots; i++) { void *array = req->child_slots[i].array; @@ -1392,7 +1561,7 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) for (i = 0; i < req->nr_node_slots; i++) { if (req->node_slots[i].node || - trie_pool_add_size(req->node_slots[i].size, &total)) + !__stack_depot_trie_pool_alloc_size(req->node_slots[i].size)) return -EINVAL; } for (i = 0; i < req->nr_child_slots; i++) { @@ -1410,6 +1579,12 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) ret = -ENOSPC; goto out; } + for (i = 0; i < req->nr_node_slots; i++) { + req->node_slots[i].node = trie_pop_free_node(req->node_slots[i].size); + if (!req->node_slots[i].node && + trie_pool_add_size(req->node_slots[i].size, &total)) + goto out_release_reused; + } for (i = 0; i < req->nr_child_slots; i++) { req->child_slots[i].array = trie_pop_free_object(req->child_slots[i].size); @@ -1450,6 +1625,8 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) req->mark->size = total; offset = pool_offset; for (i = 0; i < req->nr_node_slots; i++) { + if (req->node_slots[i].node) + continue; req->node_slots[i].node = pool + offset; offset += __stack_depot_trie_pool_alloc_size(req->node_slots[i].size); } @@ -1519,6 +1696,8 @@ static void trie_alloc_request_release_reused_objects(struct stack_depot_trie_al if (!req || !req->txn) return; + pool_req.node_slots = req->node_slots; + pool_req.nr_node_slots = req->nr_node_slots; pool_req.child_slots = req->child_slots; pool_req.nr_child_slots = req->nr_child_slots; pool_req.storage = req->storage; @@ -1970,40 +2149,42 @@ __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updat unsigned int nr_updates, void *ctx) { struct stack_depot_trie_side_prepare *state = ctx; - unsigned int start; + struct stack_depot_trie_side_entry *chunks[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; + unsigned int slots[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; + unsigned long flags; unsigned int i; - int ret; if (!state || (!updates && nr_updates)) return -EINVAL; + if (nr_updates > ARRAY_SIZE(chunks) || + state->nr_updates > ARRAY_SIZE(state->updates) - nr_updates) + return -EINVAL; - start = state->nr_updates; + raw_spin_lock_irqsave(&trie_side_table_lock, flags); for (i = 0; i < nr_updates; i++) { - if (state->nr_updates >= STACK_DEPOT_TRIE_MAX_LEAF_UPDATES) { - ret = -EINVAL; + u32 leaf_id = updates[i].leaf_id; + + if (!updates[i].leaf) goto rollback; - } + chunks[i] = trie_side_table_chunk_locked(leaf_id, &slots[i]); + if (!chunks[i]) + goto rollback; + } + for (i = 0; i < nr_updates; i++) { state->updates[state->nr_updates].leaf_id = updates[i].leaf_id; state->updates[state->nr_updates].old_leaf = - __stack_depot_trie_side_table_lookup(updates[i].leaf_id); + trie_side_table_load_leaf(chunks[i], slots[i]); state->nr_updates++; - ret = __stack_depot_trie_side_table_store(updates[i].leaf_id, updates[i].leaf); - if (ret) - goto rollback; + trie_side_table_store_leaf(chunks[i], slots[i], updates[i].leaf); } + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); return 0; rollback: - while (state->nr_updates > start) { - struct stack_depot_trie_side_checkpoint *update; - - state->nr_updates--; - update = &state->updates[state->nr_updates]; - __stack_depot_trie_side_table_restore(update->leaf_id, update->old_leaf); - } - return ret; + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return -EINVAL; } static int __init disable_stack_depot(char *str) @@ -2240,6 +2421,7 @@ static bool depot_init_pool(void **prealloc) /* Save reference to the pool to be used by depot_fetch_stack(). */ stack_pools[pools_num] = new_pool; + depot_record_pool_range_locked(new_pool); /* * Stack depot tries to keep an extra pool allocated even before it runs @@ -3875,6 +4057,7 @@ trie_promote_child(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array **publish_slot; const struct stack_depot_trie_child_array *old_array; struct stack_depot_trie_leaf_update update; + size_t child_size; unsigned int pos; int ret; @@ -3902,7 +4085,8 @@ trie_promote_child(struct stack_depot_trie_root *root, publish_slot = trie_publish_slot(root, parent); /* Publish the fully initialized replacement array last. */ smp_store_release(publish_slot, new_storage); - trie_retire_object(old_array); + child_size = __stack_depot_trie_node_size(&child->run); + trie_retire_object_node(old_array, child, child_size); return 0; } @@ -5487,7 +5671,8 @@ static int trie_split_child(struct stack_depot_trie_root *root, trie_child_array_replace_at(old_array, prefix, new_storage, pos); /* Publish the fully initialized replacement array last. */ smp_store_release(publish_slot, new_storage); - trie_retire_object(old_array); + trie_retire_object_node(old_array, child, + __stack_depot_trie_node_size(&child->run)); *nr_used = used; return 0; } From e356b33f84a024da8242ab6142e01773ad201dea Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 16 Jun 2026 21:24:18 +0100 Subject: [PATCH 092/129] KRN-1117: Harden arm64 stackdepot prefix scan Use an unsigned int for the arm64 stackdepot prefix-id loop so the scan cannot wrap if the prefix id range grows in the future. Signed-off-by: Caleb Kan --- arch/arm64/include/asm/stackdepot.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h index d718b4722de1e..079dcaf9154a6 100644 --- a/arch/arm64/include/asm/stackdepot.h +++ b/arch/arm64/include/asm/stackdepot.h @@ -59,7 +59,7 @@ static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, { unsigned long prefix = frame & STACK_DEPOT_ARM64_FRAME_PREFIX_MASK; unsigned long candidate; - u8 i; + unsigned int i; if (!prefix_id || !low) return false; From 0df12cc2c48491b1a58026c140d9f15a67a9c293 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 16 Jun 2026 21:24:48 +0100 Subject: [PATCH 093/129] KRN-1117: Reduce stackdepot trie pool usage Keep trie-eligible persistent saves in trie storage instead of mixing them back into hash storage, while preserving hash-backed records for refcounted, count-helper, and overlong users. Reduce trie pool pressure by reusing exact-size free objects, splitting oversized free object tails, allowing in-place child-array appends, and shrinking trie node allocations to the actual flexible-array payload offset. Preserve the RCU copy-on-write publication rules by keeping pending objects separate until their RCU cookie has completed. Also tighten hash compatibility and copy-out ordering so late-enabled persistent hash records are observed only after publication, and hash fetch_into() copies under the stackdepot RCU read-side section. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 473 +++++++++++++++++++++++++---------- lib/stackdepot_internal.h | 2 +- lib/tests/stackdepot_kunit.c | 2 +- 3 files changed, 342 insertions(+), 135 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 96c578aebada1..203f0b61c08a1 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -14,6 +14,7 @@ #define pr_fmt(fmt) "stackdepot: " fmt +#include #include #include #include @@ -152,13 +153,14 @@ struct stack_depot_trie_node { const struct stack_depot_trie_node *parent; const struct stack_depot_trie_child_array *children; u32 leaf_id; - u32 stack_len; + u16 stack_len; struct stack_depot_frame_run run; unsigned char data[]; }; struct stack_depot_trie_child_array { unsigned int nr_children; + unsigned int capacity; const struct stack_depot_trie_node *children[]; }; @@ -189,6 +191,8 @@ static unsigned int stack_hash_mask; static void **stack_pools; /* Newly allocated pool that is not yet added to stack_pools. */ static void *new_pool; +/* Whether legacy hash storage may contain normal persistent records. */ +static bool stack_depot_persistent_hash_record_seen; /* Number of pools in stack_pools. */ static int pools_num; static unsigned long pools_min_addr; @@ -198,10 +202,15 @@ static size_t pool_offset = DEPOT_POOL_SIZE; /* Freelist of stack records within stack_pools. */ static LIST_HEAD(free_stacks); -#define STACK_DEPOT_TRIE_FREE_OBJECT_BUCKETS (DEPOT_POOL_ORDER + PAGE_SHIFT + 1) +#define STACK_DEPOT_TRIE_FREE_CLASSES \ + ((DEPOT_POOL_SIZE >> DEPOT_STACK_ALIGN) + 1) -static struct list_head free_trie_objects[STACK_DEPOT_TRIE_FREE_OBJECT_BUCKETS]; -static struct list_head free_trie_nodes[STACK_DEPOT_TRIE_FREE_OBJECT_BUCKETS]; +static struct list_head free_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; +static struct list_head pending_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; +static struct list_head free_trie_nodes[STACK_DEPOT_TRIE_FREE_CLASSES]; +static DECLARE_BITMAP(free_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); +static DECLARE_BITMAP(pending_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); +static DECLARE_BITMAP(free_trie_node_map, STACK_DEPOT_TRIE_FREE_CLASSES); static unsigned int free_trie_pending_nodes; static bool free_trie_objects_initialized; /* The lock must be held when performing pool or freelist modifications. */ @@ -328,6 +337,8 @@ static u32 trie_side_table_next_id; static bool trie_side_table_initialized; static bool trie_side_table_memblock; +/* Lock order: trie_alloc_lock -> pool_lock -> trie_side_table_lock. */ + static bool stack_depot_trie_is_ready(void) { /* Pairs with stack_depot_trie_publish_ready(). */ @@ -689,6 +700,7 @@ void __stack_depot_trie_side_table_destroy(void) if (!trie_side_table_is_initialized()) return; stack_depot_trie_mark_not_ready(); + WRITE_ONCE(trie_side_table_initialized, false); high_water = READ_ONCE(trie_side_table_high_water); if (!READ_ONCE(trie_side_table_memblock)) { @@ -710,7 +722,6 @@ void __stack_depot_trie_side_table_destroy(void) WRITE_ONCE(trie_side_table_max_id, 0); WRITE_ONCE(trie_side_table_next_id, 0); WRITE_ONCE(trie_side_table_memblock, false); - WRITE_ONCE(trie_side_table_initialized, false); } bool __stack_depot_trie_side_table_prealloc_needed(void) @@ -1107,18 +1118,45 @@ static void trie_free_object_buckets_init_locked(void) return; for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) { INIT_LIST_HEAD(&free_trie_objects[i]); + INIT_LIST_HEAD(&pending_trie_objects[i]); INIT_LIST_HEAD(&free_trie_nodes[i]); } free_trie_objects_initialized = true; } -static unsigned int trie_free_object_bucket(size_t size) +static unsigned int trie_free_class(size_t size) { size = __stack_depot_trie_pool_alloc_size(size); if (!size) return 0; - return min_t(unsigned int, order_base_2(size), - ARRAY_SIZE(free_trie_objects) - 1); + + return size >> DEPOT_STACK_ALIGN; +} + +static void trie_free_list_add(struct list_head *entry, struct list_head *heads, + unsigned long *map, unsigned int class, bool tail) +{ + lockdep_assert_held(&pool_lock); + + if (WARN_ON_ONCE(!class || class >= STACK_DEPOT_TRIE_FREE_CLASSES)) + return; + if (tail) + list_add_tail(entry, &heads[class]); + else + list_add(entry, &heads[class]); + __set_bit(class, map); +} + +static void trie_free_list_del(struct list_head *entry, struct list_head *heads, + unsigned long *map, unsigned int class) +{ + lockdep_assert_held(&pool_lock); + + if (WARN_ON_ONCE(!class || class >= STACK_DEPOT_TRIE_FREE_CLASSES)) + return; + list_del_init(entry); + if (list_empty(&heads[class])) + __clear_bit(class, map); } static void *trie_object_init_fresh(void *ptr, size_t size) @@ -1199,7 +1237,7 @@ static bool trie_pool_mark_contains(const struct stack_depot_trie_pool_mark *mar static void trie_free_object_locked(const void *ptr, unsigned long rcu_state) { struct stack_depot_trie_free_object *free; - unsigned int bucket; + unsigned int class; lockdep_assert_held(&pool_lock); @@ -1210,17 +1248,20 @@ static void trie_free_object_locked(const void *ptr, unsigned long rcu_state) if (!trie_pool_contains_locked(free)) return; free->rcu_state = rcu_state; - bucket = trie_free_object_bucket(free->size); + class = trie_free_class(free->size); + INIT_LIST_HEAD(&free->list); if (poll_state_synchronize_rcu(rcu_state)) - list_add(&free->list, &free_trie_objects[bucket]); + trie_free_list_add(&free->list, free_trie_objects, + free_trie_object_map, class, false); else - list_add_tail(&free->list, &free_trie_objects[bucket]); + trie_free_list_add(&free->list, pending_trie_objects, + pending_trie_object_map, class, true); } static void trie_add_free_node_locked(void *ptr, size_t size) { struct stack_depot_trie_free_node *free = ptr; - unsigned int bucket; + unsigned int class; lockdep_assert_held(&pool_lock); @@ -1230,8 +1271,9 @@ static void trie_add_free_node_locked(void *ptr, size_t size) free->size = size; INIT_LIST_HEAD(&free->list); - bucket = trie_free_object_bucket(size); - list_add(&free->list, &free_trie_nodes[bucket]); + class = trie_free_class(size); + trie_free_list_add(&free->list, free_trie_nodes, free_trie_node_map, + class, false); } static void trie_drain_free_object_node_locked(struct stack_depot_trie_free_object *free) @@ -1246,20 +1288,28 @@ static void trie_drain_free_object_node_locked(struct stack_depot_trie_free_obje free_trie_pending_nodes--; } -static void trie_drain_pending_nodes_locked(void) +static void trie_drain_pending_objects_locked(void) { struct stack_depot_trie_free_object *free; - unsigned int i; + struct stack_depot_trie_free_object *tmp; + unsigned int class; lockdep_assert_held(&pool_lock); - if (!free_trie_pending_nodes) + if (!free_trie_pending_nodes && + bitmap_empty(pending_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES)) return; - for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) { - list_for_each_entry(free, &free_trie_objects[i], list) { + for_each_set_bit(class, pending_trie_object_map, + STACK_DEPOT_TRIE_FREE_CLASSES) { + list_for_each_entry_safe(free, tmp, &pending_trie_objects[class], list) { if (!poll_state_synchronize_rcu(free->rcu_state)) - continue; + break; trie_drain_free_object_node_locked(free); + trie_free_list_del(&free->list, pending_trie_objects, + pending_trie_object_map, class); + free->rcu_state = 0; + trie_free_list_add(&free->list, free_trie_objects, + free_trie_object_map, class, false); } } } @@ -1267,24 +1317,63 @@ static void trie_drain_pending_nodes_locked(void) static void *trie_pop_free_node(size_t size) { struct stack_depot_trie_free_node *free; - unsigned int bucket; + unsigned int class; lockdep_assert_held(&pool_lock); size = __stack_depot_trie_pool_alloc_size(size); if (!size) return NULL; - trie_free_object_buckets_init_locked(); - trie_drain_pending_nodes_locked(); - bucket = trie_free_object_bucket(size); - list_for_each_entry(free, &free_trie_nodes[bucket], list) { - if (free->size != size) - continue; - list_del_init(&free->list); - return free; + class = trie_free_class(size); + if (!test_bit(class, free_trie_node_map)) + return NULL; + free = list_first_entry(&free_trie_nodes[class], typeof(*free), list); + if (WARN_ON_ONCE(free->size != size)) + return NULL; + trie_free_list_del(&free->list, free_trie_nodes, free_trie_node_map, + class); + return free; +} + +static void trie_free_object_tail_locked(void *ptr, size_t size) +{ + struct stack_depot_trie_free_object *free = ptr; + size_t header_size = trie_object_header_size(); + size_t tail_size; + + lockdep_assert_held(&pool_lock); + + if (size >= header_size + (1UL << DEPOT_STACK_ALIGN)) { + tail_size = size - header_size; + free->size = tail_size; + free->rcu_state = get_completed_synchronize_rcu(); + free->pending_node = NULL; + free->pending_node_size = 0; + INIT_LIST_HEAD(&free->list); + trie_free_list_add(&free->list, free_trie_objects, + free_trie_object_map, trie_free_class(tail_size), + false); + return; } - return NULL; + trie_add_free_node_locked(ptr, size); +} + +static void trie_split_free_object_locked(struct stack_depot_trie_free_object *free, + size_t size) +{ + void *tail; + size_t old_size = free->size; + size_t tail_size; + + lockdep_assert_held(&pool_lock); + + if (old_size <= size) + return; + free->size = size; + tail = trie_object_payload(free) + size; + tail_size = old_size - size; + trie_free_object_tail_locked(tail, tail_size); } static void trie_retire_object_node(const void *ptr, const void *node, @@ -1311,8 +1400,9 @@ static void trie_retire_object_node(const void *ptr, const void *node, free_trie_pending_nodes++; } free->rcu_state = get_state_synchronize_rcu(); - list_add_tail(&free->list, - &free_trie_objects[trie_free_object_bucket(free->size)]); + trie_free_list_add(&free->list, pending_trie_objects, + pending_trie_object_map, + trie_free_class(free->size), true); } raw_spin_unlock_irqrestore(&pool_lock, flags); } @@ -1325,30 +1415,28 @@ static void trie_retire_object(const void *ptr) static void *trie_pop_free_object(size_t size) { struct stack_depot_trie_free_object *free; - unsigned int bucket; - unsigned int i; + unsigned int class; lockdep_assert_held(&pool_lock); size = __stack_depot_trie_pool_alloc_size(size); if (!size) return NULL; - trie_free_object_buckets_init_locked(); - bucket = trie_free_object_bucket(size); - for (i = bucket; i < ARRAY_SIZE(free_trie_objects); i++) { - list_for_each_entry(free, &free_trie_objects[i], list) { - if (free->size < size) - continue; - if (!poll_state_synchronize_rcu(free->rcu_state)) - break; - trie_drain_free_object_node_locked(free); - list_del_init(&free->list); - free->rcu_state = 0; - return trie_object_payload(free); - } - } + class = trie_free_class(size); + class = find_next_bit(free_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES, + class); + if (class >= STACK_DEPOT_TRIE_FREE_CLASSES) + return NULL; - return NULL; + free = list_first_entry(&free_trie_objects[class], typeof(*free), list); + if (WARN_ON_ONCE(free->size < size)) + return NULL; + trie_drain_free_object_node_locked(free); + trie_free_list_del(&free->list, free_trie_objects, + free_trie_object_map, class); + free->rcu_state = 0; + trie_split_free_object_locked(free, size); + return trie_object_payload(free); } void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags) @@ -1569,7 +1657,7 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) !trie_object_alloc_size(req->child_slots[i].size)) return -EINVAL; } - if (!trie_object_alloc_size(req->storage_size)) + if (req->storage_size && !trie_object_alloc_size(req->storage_size)) return -EINVAL; if (!raw_spin_trylock_irqsave(&pool_lock, flags)) @@ -1579,6 +1667,8 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) ret = -ENOSPC; goto out; } + trie_free_object_buckets_init_locked(); + trie_drain_pending_objects_locked(); for (i = 0; i < req->nr_node_slots; i++) { req->node_slots[i].node = trie_pop_free_node(req->node_slots[i].size); if (!req->node_slots[i].node && @@ -1592,9 +1682,11 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) trie_pool_add_object_size(req->child_slots[i].size, &total)) goto out_release_reused; } - *req->storage = trie_pop_free_object(req->storage_size); - if (!*req->storage && trie_pool_add_object_size(req->storage_size, &total)) - goto out_release_reused; + if (req->storage_size) { + *req->storage = trie_pop_free_object(req->storage_size); + if (!*req->storage && trie_pool_add_object_size(req->storage_size, &total)) + goto out_release_reused; + } if (pools_num < 1) { req->mark->prev_offset = pool_offset; @@ -1638,7 +1730,7 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) req->child_slots[i].size); offset += trie_object_alloc_size(req->child_slots[i].size); } - if (!*req->storage) + if (req->storage_size && !*req->storage) *req->storage = trie_object_init_fresh(pool + offset, req->storage_size); pool_offset += total; @@ -1899,7 +1991,6 @@ trie_save_miss(struct stack_depot_trie_root *root, const unsigned long *entries, handle = __stack_depot_trie_handle(leaf_id); out: - depot_try_keep_new_pool(&pool_prealloc); __stack_depot_trie_pool_free_prealloc(pool_prealloc); __stack_depot_trie_side_table_free_prealloc(&side_prealloc); return handle; @@ -2011,6 +2102,7 @@ __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, void *pool_prealloc = NULL; struct stack_depot_trie_side_prealloc side_prealloc = {}; bool can_insert; + bool no_spin; int ret; if (!root || !entries || !nr_entries || !workspace || !workspace_lock) @@ -2023,12 +2115,13 @@ __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, handle = trie_find_handle(root, entries, nr_entries); if (handle) return handle; + no_spin = in_nmi() || !gfpflags_allow_spinning(alloc_flags); ret = trie_prealloc(alloc_flags, depot_flags, &pool_prealloc, &side_prealloc); can_insert = !ret; - if (in_nmi() || !gfpflags_allow_spinning(alloc_flags)) + if (no_spin) handle = trie_save_trylocked(root, entries, nr_entries, workspace, workspace_lock, &pool_prealloc, &side_prealloc, can_insert); @@ -2037,6 +2130,7 @@ __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, workspace_lock, &pool_prealloc, &side_prealloc, can_insert); + depot_try_keep_new_pool(&pool_prealloc); __stack_depot_trie_pool_free_prealloc(pool_prealloc); __stack_depot_trie_side_table_free_prealloc(&side_prealloc); return handle; @@ -2774,6 +2868,7 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, void *prealloc = NULL; bool allow_spin = gfpflags_allow_spinning(alloc_flags); bool can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && allow_spin; + bool normal_persistent; bool trie_candidate; unsigned long flags; u32 hash; @@ -2793,6 +2888,35 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, if (unlikely(nr_entries == 0) || stack_depot_disabled) return 0; + normal_persistent = !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH)) && + nr_entries <= CONFIG_STACKDEPOT_MAX_FRAMES; + + trie_candidate = __stack_depot_trie_ready() && + normal_persistent; + if (trie_candidate) { + handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); + if (handle) + return handle; + + if (READ_ONCE(stack_depot_persistent_hash_record_seen)) { + /* + * Trie storage may be enabled after stack depot has already saved + * hash records. Preserve the same-handle contract by checking hash + * only on trie misses and only when hash records may exist. + */ + hash = hash_stack(entries, nr_entries); + bucket = &stack_table[hash & stack_hash_mask]; + found = find_stack(bucket, entries, nr_entries, hash, depot_flags); + if (found) + goto exit; + } + + handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, + depot_flags); + if (handle) + return handle; + return 0; + } hash = hash_stack(entries, nr_entries); bucket = &stack_table[hash & stack_hash_mask]; @@ -2801,24 +2925,6 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, found = find_stack(bucket, entries, nr_entries, hash, depot_flags); if (found) goto exit; - - trie_candidate = __stack_depot_trie_ready() && - !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH)) && - nr_entries <= CONFIG_STACKDEPOT_MAX_FRAMES; - if (trie_candidate) { - if (in_nmi() || !allow_spin) { - handle = trie_find_handle(&stack_depot_trie_root, entries, - nr_entries); - if (handle) - return handle; - } else { - handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, - depot_flags); - if (handle) - return handle; - } - } - /* * Allocate memory for a new pool if required now: * we won't be able to do that under the lock. @@ -2853,6 +2959,8 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, * makes it visible to readers in find_stack(). */ list_add_rcu(&new->hash_list, bucket); + if (normal_persistent) + WRITE_ONCE(stack_depot_persistent_hash_record_seen, true); found = new; } } @@ -2871,10 +2979,7 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, exit: if (prealloc) { /* Stack depot didn't use this memory, free it. */ - if (!allow_spin) - free_pages_nolock(virt_to_page(prealloc), DEPOT_POOL_ORDER); - else - free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); + free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); } if (found) handle = found->handle.handle; @@ -2992,7 +3097,7 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, * the existing count unchanged. */ /* A stale read is harmless: cmpxchg reloads @old before retry checks. */ - old = refcount_read(&stack->count); + old = data_race(refcount_read(&stack->count)); /* See above. */ do { bool underflow; @@ -3055,6 +3160,8 @@ static int stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *array, unsigned long frame, unsigned int *pos, bool *found); +static unsigned int trie_child_array_storage_capacity(size_t storage_size); +static size_t trie_child_array_size_for_capacity(unsigned int capacity); static bool trie_parent_chain_matches_prefix(const struct stack_depot_trie_node *node, const unsigned long *entries, @@ -3322,7 +3429,7 @@ size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) if (stack_depot_frame_run_validate(run)) return 0; - size = sizeof(struct stack_depot_trie_node); + size = offsetof(struct stack_depot_trie_node, data); if (check_add_overflow(size, run->bytes, &size)) return 0; @@ -3535,7 +3642,7 @@ static bool trie_ancestor_overlaps(const struct stack_depot_trie_node *node, if (!node->children) continue; - child_size = __stack_depot_trie_child_array_size(node->children->nr_children); + child_size = trie_child_array_size_for_capacity(node->children->capacity); if (!child_size) return true; if (stack_depot_ranges_overlap(ptr, size, node->children, @@ -3572,7 +3679,7 @@ static bool trie_chain_overlaps(const struct stack_depot_trie_node *node, break; if (children->nr_children != 1) return true; - child_size = __stack_depot_trie_child_array_size(1); + child_size = trie_child_array_size_for_capacity(children->capacity); if (!child_size) return true; if (stack_depot_ranges_overlap(ptr, size, children, child_size)) @@ -3598,7 +3705,7 @@ trie_child_array_subtree_overlaps(const struct stack_depot_trie_child_array *arr if (!array) return false; - array_size = __stack_depot_trie_child_array_size(array->nr_children); + array_size = trie_child_array_size_for_capacity(array->capacity); if (!array_size) return true; if (stack_depot_ranges_overlap(ptr, size, array, array_size)) @@ -3632,7 +3739,7 @@ trie_child_array_subtree_overlaps(const struct stack_depot_trie_child_array *arr children = node->children; if (children) { - child_size = __stack_depot_trie_child_array_size(children->nr_children); + child_size = trie_child_array_size_for_capacity(children->capacity); if (!child_size) return true; if (stack_depot_ranges_overlap(ptr, size, children, @@ -3768,6 +3875,13 @@ trie_publish_slot(struct stack_depot_trie_root *root, return &parent->children; } +static bool +trie_child_array_can_append(const struct stack_depot_trie_child_array *array, + unsigned int pos) +{ + return array && pos == array->nr_children && array->nr_children < array->capacity; +} + static int trie_insert_append_precheck(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, const unsigned long *entries, @@ -3784,21 +3898,23 @@ static int trie_insert_append_precheck(struct stack_depot_trie_root *root, unsigned int pos; bool found; - if (!entries || !nr_entries || !new_storage) + if (!entries || !nr_entries) return -EINVAL; if (!entries[0]) return -EINVAL; if ((nr_node_slots && !node_slots) || (nr_child_slots && !child_slots)) return -EINVAL; - if (!IS_ALIGNED((unsigned long)new_storage, - __alignof__(struct stack_depot_trie_child_array))) + if (new_storage && + !IS_ALIGNED((unsigned long)new_storage, + __alignof__(struct stack_depot_trie_child_array))) return -EINVAL; slot = trie_publish_slot(root, parent); if (!slot) return -EINVAL; - if (stack_depot_ranges_overlap(new_storage, new_storage_size, slot, - sizeof(*slot))) + if (new_storage && + stack_depot_ranges_overlap(new_storage, new_storage_size, + slot, sizeof(*slot))) return -EINVAL; if (root) { if (trie_node_slot_overlaps(node_slots, nr_node_slots, slot, @@ -3808,22 +3924,34 @@ static int trie_insert_append_precheck(struct stack_depot_trie_root *root, sizeof(*slot))) return -EINVAL; } - if (parent && trie_ancestor_overlaps(parent, new_storage, new_storage_size)) + if (new_storage && parent && + trie_ancestor_overlaps(parent, new_storage, new_storage_size)) return -EINVAL; - if (trie_node_slot_overlaps(node_slots, nr_node_slots, new_storage, - new_storage_size) || - trie_child_slot_overlaps(child_slots, nr_child_slots, new_storage, - new_storage_size)) + if (new_storage && + (trie_node_slot_overlaps(node_slots, nr_node_slots, new_storage, + new_storage_size) || + trie_child_slot_overlaps(child_slots, nr_child_slots, new_storage, + new_storage_size))) return -EINVAL; /* Pairs with append publication's smp_store_release(). */ children = smp_load_acquire(slot); + if (!new_storage) { + if (!children) + return -EINVAL; + if (stack_depot_trie_child_lower_bound(children, entries[0], &pos, + &found)) + return -EINVAL; + if (found || !trie_child_array_can_append(children, pos)) + return -EINVAL; + return 0; + } size = __stack_depot_trie_child_array_size(children ? - children->nr_children + 1 : 1); + children->nr_children + 1 : 1); if (!size || new_storage_size < size) return -EINVAL; if (children) { - size = __stack_depot_trie_child_array_size(children->nr_children); + size = trie_child_array_size_for_capacity(children->capacity); if (!size) return -EINVAL; if (stack_depot_ranges_overlap(children, size, new_storage, @@ -3854,18 +3982,21 @@ static int trie_insert_descend_precheck(struct stack_depot_trie_root *root, { const struct stack_depot_trie_child_array **slot; const struct stack_depot_trie_child_array *children; + bool overlap; size_t size; slot = trie_publish_slot(root, parent); - if (!slot || !new_storage) + if (!slot) return -EINVAL; if ((nr_node_slots && !node_slots) || (nr_child_slots && !child_slots)) return -EINVAL; - if (!IS_ALIGNED((unsigned long)new_storage, - __alignof__(struct stack_depot_trie_child_array))) + if (new_storage && + !IS_ALIGNED((unsigned long)new_storage, + __alignof__(struct stack_depot_trie_child_array))) return -EINVAL; - if (stack_depot_ranges_overlap(new_storage, new_storage_size, slot, - sizeof(*slot))) + if (new_storage && + stack_depot_ranges_overlap(new_storage, new_storage_size, + slot, sizeof(*slot))) return -EINVAL; if (trie_node_slot_overlaps(node_slots, nr_node_slots, slot, sizeof(*slot))) @@ -3878,10 +4009,11 @@ static int trie_insert_descend_precheck(struct stack_depot_trie_root *root, children = smp_load_acquire(slot); if (!children) return -EINVAL; - size = __stack_depot_trie_child_array_size(children->nr_children); + size = trie_child_array_size_for_capacity(children->capacity); if (!size) return -EINVAL; - if (stack_depot_ranges_overlap(children, size, new_storage, + if (new_storage && + stack_depot_ranges_overlap(children, size, new_storage, new_storage_size)) return -EINVAL; if (trie_node_slot_overlaps(node_slots, nr_node_slots, children, size)) @@ -3894,7 +4026,10 @@ static int trie_insert_descend_precheck(struct stack_depot_trie_root *root, return -EINVAL; if (trie_child_slots_subtree_overlap(children, parent, child_slots, nr_child_slots)) return -EINVAL; - if (trie_child_array_subtree_overlaps(children, parent, new_storage, new_storage_size)) + overlap = new_storage && + trie_child_array_subtree_overlaps(children, parent, new_storage, + new_storage_size); + if (overlap) return -EINVAL; return 0; @@ -3919,7 +4054,7 @@ trie_child_array_replace_precheck(const struct stack_depot_trie_child_array *old if (old_array == new_array) return -EINVAL; - size = __stack_depot_trie_child_array_size(old_array->nr_children); + size = trie_child_array_size_for_capacity(old_array->capacity); if (!size || new_storage_size < size) return -EINVAL; if (stack_depot_ranges_overlap(old_array, size, new_array, new_storage_size)) @@ -3936,12 +4071,14 @@ trie_child_array_replace_precheck(const struct stack_depot_trie_child_array *old static void trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array, const struct stack_depot_trie_node *new_child, - void *new_storage, unsigned int pos) + void *new_storage, size_t new_storage_size, + unsigned int pos) { struct stack_depot_trie_child_array *new_array = new_storage; unsigned int i; new_array->nr_children = old_array->nr_children; + new_array->capacity = trie_child_array_storage_capacity(new_storage_size); for (i = 0; i < old_array->nr_children; i++) new_array->children[i] = old_array->children[i]; new_array->children[pos] = new_child; @@ -4079,7 +4216,8 @@ trie_promote_child(struct stack_depot_trie_root *root, if (ret) return ret; } - trie_child_array_replace_at(old_array, slot->node, new_storage, pos); + trie_child_array_replace_at(old_array, slot->node, new_storage, + new_storage_size, pos); trie_reparent_children(slot->node); publish_slot = trie_publish_slot(root, parent); @@ -4247,24 +4385,29 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array *old_array; const struct stack_depot_trie_node *head = head_ptr; const struct stack_depot_trie_child_array **slot; + struct stack_depot_trie_child_array *append_array = NULL; struct stack_depot_trie_node *parent = parent_ptr; struct stack_depot_trie_child_array *new_array = new_storage; + unsigned int append_pos = 0; size_t storage_size = new_storage_size; size_t new_size; size_t old_size; - if ((root && parent) || (!root && !parent) || !head || !new_array) + if ((root && parent) || (!root && !parent) || !head) return -EINVAL; if (head->parent != parent) return -EINVAL; if (root) { - if (stack_depot_ranges_overlap(new_array, storage_size, - &root->children, sizeof(root->children))) + if (new_array && + stack_depot_ranges_overlap(new_array, storage_size, + &root->children, + sizeof(root->children))) return -EINVAL; slot = &root->children; } else { - if (trie_ancestor_overlaps(parent, new_array, storage_size)) + if (new_array && + trie_ancestor_overlaps(parent, new_array, storage_size)) return -EINVAL; slot = &parent->children; } @@ -4272,18 +4415,37 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, /* Pairs with append publication's smp_store_release(). */ old_array = smp_load_acquire(slot); old_size = old_array ? - __stack_depot_trie_child_array_size(old_array->nr_children) : 0; + trie_child_array_size_for_capacity(old_array->capacity) : 0; new_size = old_array ? old_array->nr_children + 1 : 1; new_size = __stack_depot_trie_child_array_size(new_size); - if (!new_size || storage_size < new_size) + if (!new_size) return -EINVAL; - if (old_array && + if (!new_array) { + unsigned long frame; + unsigned int pos; + bool found; + + if (!old_array) + return -EINVAL; + if (stack_depot_trie_node_first_frame(head, &frame)) + return -EINVAL; + if (stack_depot_trie_child_lower_bound(old_array, frame, &pos, &found)) + return -EINVAL; + if (found || !trie_child_array_can_append(old_array, pos)) + return -EINVAL; + append_array = (struct stack_depot_trie_child_array *)old_array; + append_pos = pos; + } + if (new_array && storage_size < new_size) + return -EINVAL; + if (new_array && old_array && stack_depot_ranges_overlap(old_array, old_size, new_array, storage_size)) return -EINVAL; - if (trie_chain_overlaps(head, new_array, storage_size)) + if (new_array && trie_chain_overlaps(head, new_array, storage_size)) return -EINVAL; - if (__stack_depot_trie_child_array_insert(old_array, head, new_array, storage_size)) + if (new_array && + __stack_depot_trie_child_array_insert(old_array, head, new_array, storage_size)) return -EINVAL; if (prepare) { struct stack_depot_trie_leaf_update update = { @@ -4300,6 +4462,12 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, if (ret) return ret; } + if (!new_array) { + append_array->children[append_pos] = head; + /* Pairs with child lookup's smp_load_acquire(). */ + smp_store_release(&append_array->nr_children, append_pos + 1); + return 0; + } /* Publish the fully initialized replacement array last. */ smp_store_release(slot, new_array); @@ -4778,7 +4946,7 @@ static int trie_plan_split(const struct stack_depot_trie_child_array *children, child_slots[0].size = __stack_depot_trie_child_array_size(has_new_tail ? 2 : 1); *new_storage_size = - __stack_depot_trie_child_array_size(children->nr_children); + trie_child_array_size_for_capacity(children->capacity); if (!child_slots[0].size || !*new_storage_size) return -EINVAL; *nr_used = 2 + new_used; @@ -4839,6 +5007,10 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, nr_child_slots, nr_used, nr_child_used)) return -EINVAL; + if (trie_child_array_can_append(children, pos)) { + *new_storage_size = 0; + return 0; + } *new_storage_size = __stack_depot_trie_child_array_size(children->nr_children + 1); return *new_storage_size ? 0 : -EINVAL; @@ -4864,7 +5036,7 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, node_slots[0].node = NULL; node_slots[0].size = __stack_depot_trie_node_size(&child->run); *new_storage_size = - __stack_depot_trie_child_array_size(children->nr_children); + trie_child_array_size_for_capacity(children->capacity); if (!node_slots[0].size || !*new_storage_size) return -EINVAL; *nr_used = 1; @@ -5135,12 +5307,27 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, return nr_entries; } -size_t __stack_depot_trie_child_array_size(unsigned int nr_children) +static unsigned int trie_child_array_capacity(unsigned int nr_children) +{ + if (!nr_children) + return 0; + return roundup_pow_of_two(nr_children); +} + +static unsigned int trie_child_array_storage_capacity(size_t storage_size) +{ + if (storage_size < sizeof(struct stack_depot_trie_child_array)) + return 0; + storage_size -= sizeof(struct stack_depot_trie_child_array); + return storage_size / sizeof(struct stack_depot_trie_node *); +} + +static size_t trie_child_array_size_for_capacity(unsigned int capacity) { size_t size; size_t bytes; - if (check_mul_overflow((size_t)nr_children, + if (check_mul_overflow((size_t)capacity, sizeof(struct stack_depot_trie_node *), &bytes)) return 0; size = sizeof(struct stack_depot_trie_child_array); @@ -5150,6 +5337,13 @@ size_t __stack_depot_trie_child_array_size(unsigned int nr_children) return ALIGN(size, sizeof(unsigned long)); } +size_t __stack_depot_trie_child_array_size(unsigned int nr_children) +{ + unsigned int capacity = trie_child_array_capacity(nr_children); + + return trie_child_array_size_for_capacity(capacity); +} + int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, const void * const *children, unsigned int nr_children) @@ -5157,6 +5351,7 @@ int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, struct stack_depot_trie_child_array *array = storage; const struct stack_depot_trie_node * const *nodes = (const struct stack_depot_trie_node * const *)children; + unsigned int capacity; unsigned long last = 0; unsigned int i; @@ -5166,6 +5361,9 @@ int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, return -EINVAL; if (nr_children && !nodes) return -EINVAL; + capacity = trie_child_array_storage_capacity(storage_size); + if (capacity < nr_children) + return -EINVAL; for (i = 0; i < nr_children; i++) { unsigned long frame; @@ -5185,6 +5383,7 @@ int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, } array->nr_children = nr_children; + array->capacity = capacity; for (i = 0; i < nr_children; i++) array->children[i] = nodes[i]; @@ -5363,7 +5562,7 @@ int __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, children = smp_load_acquire(slot); if (!children) return -EINVAL; - size = __stack_depot_trie_child_array_size(children->nr_children); + size = trie_child_array_size_for_capacity(children->capacity); if (!size || new_storage_size < size) return -EINVAL; if (stack_depot_ranges_overlap(children, size, new_storage, @@ -5668,7 +5867,8 @@ static int trie_split_child(struct stack_depot_trie_root *root, if (ret) return ret; - trie_child_array_replace_at(old_array, prefix, new_storage, pos); + trie_child_array_replace_at(old_array, prefix, new_storage, + new_storage_size, pos); /* Publish the fully initialized replacement array last. */ smp_store_release(publish_slot, new_storage); trie_retire_object_node(old_array, child, @@ -5689,7 +5889,8 @@ stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *ar *pos = 0; *found = false; - right = array->nr_children; + /* Pairs with in-place append publication's smp_store_release(). */ + right = smp_load_acquire(&array->nr_children); while (left < right) { unsigned int mid = left + (right - left) / 2; unsigned long mid_frame; @@ -5763,7 +5964,7 @@ __stack_depot_trie_child_array_insert(const void *old_storage, const void *child nr_old = old_array ? old_array->nr_children : 0; if (new_storage_size < __stack_depot_trie_child_array_size(nr_old + 1)) return -EINVAL; - old_size = __stack_depot_trie_child_array_size(nr_old); + old_size = old_array ? trie_child_array_size_for_capacity(old_array->capacity) : 0; overlaps = old_array && stack_depot_ranges_overlap(old_array, old_size, new_array, new_storage_size); if (overlaps) @@ -5781,6 +5982,7 @@ __stack_depot_trie_child_array_insert(const void *old_storage, const void *child } new_array->nr_children = nr_old + 1; + new_array->capacity = trie_child_array_storage_capacity(new_storage_size); if (old_array) { for (i = 0; i < pos; i++) new_array->children[i] = old_array->children[i]; @@ -5828,7 +6030,7 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, unsigned long *entries, unsigned int max_entries) { - unsigned long *stack_entries; + struct stack_record *stack; unsigned int nr_entries; if (!handle || !entries || !max_entries) @@ -5839,15 +6041,20 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); - nr_entries = stack_depot_fetch(handle, &stack_entries); - if (!nr_entries || nr_entries > max_entries) + rcu_read_lock_sched_notrace(); + stack = depot_fetch_stack(handle); + if (!stack) { + rcu_read_unlock_sched_notrace(); + return 0; + } + nr_entries = stack->size; + if (!nr_entries || nr_entries > max_entries) { + rcu_read_unlock_sched_notrace(); return 0; + } - /* - * stack_depot_fetch() returns stackdepot-owned storage; the caller must - * keep the handle valid while this helper copies from it. - */ - memcpy(entries, stack_entries, nr_entries * sizeof(*entries)); + memcpy(entries, stack->entries, nr_entries * sizeof(*entries)); + rcu_read_unlock_sched_notrace(); kmsan_unpoison_memory(entries, nr_entries * sizeof(*entries)); return nr_entries; } diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 14f0b6c77bcb3..92ed2c3f8fbe1 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -21,8 +21,8 @@ enum stack_depot_trie_lookup_status { }; struct stack_depot_frame_run { - unsigned int nr_entries; u16 bytes; + u16 nr_entries; u8 mode; u8 prefix_id; }; diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 8c4032f4e29a1..baaec7bc012e7 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -5625,7 +5625,7 @@ static void stackdepot_trie_public_save_route(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, fetched, trie_entries, sizeof(trie_entries)); noalloc_handle = stack_depot_save_flags(noalloc_entries, noalloc_nr, no_spin, 0); KUNIT_ASSERT_NE(test, noalloc_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(noalloc_handle), 0U); + KUNIT_EXPECT_NE(test, __stack_depot_trie_leaf_id(noalloc_handle), 0U); KUNIT_EXPECT_EQ(test, stack_depot_save(noalloc_entries, noalloc_nr, GFP_KERNEL), noalloc_handle); From 961acf09ad97dddc121881034bb2ec5a2ace4d07 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 17 Jun 2026 11:48:40 +0100 Subject: [PATCH 094/129] KRN-1117: Warn on direct fetch of trie stackdepot handles Direct stack_depot_fetch() only returns stackdepot-owned contiguous storage for legacy hash-backed handles. Trie-backed stacks must be materialized through stack_depot_fetch_into(), stack_depot_print(), or stack_depot_snprint(). Warn once when a trie handle reaches stack_depot_fetch() so unmigrated callers are visible during validation instead of silently losing the stack trace. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 203f0b61c08a1..67ff3bf8aca6f 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -6010,7 +6010,7 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, if (!handle || stack_depot_disabled) return 0; - if (__stack_depot_trie_leaf_id(handle)) + if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) return 0; stack = depot_fetch_stack(handle); From 3fddeba31141b248b741b60d680810c351b00d4c Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 17 Jun 2026 12:37:59 +0100 Subject: [PATCH 095/129] KRN-1117: Clean up stackdepot save control flow Move the locked hash-backed save path into a small helper so sparse can see the lock contexts in stack_depot_save_flags() clearly. This removes the stackdepot context-imbalance warning and keeps preallocated pool cleanup on the trylock-failure path. Also let the trie save helper own trie lookup on trie-eligible saves, avoiding a redundant outer trie walk on misses while preserving the late-enable hash compatibility check. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 99 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 30 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 67ff3bf8aca6f..ca1ce0fe5e296 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2856,12 +2856,53 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, &stack_depot_trie_workspace_lock); } +struct stack_depot_hash_save { + struct list_head *bucket; + unsigned long *entries; + unsigned int nr_entries; + u32 hash; + depot_flags_t depot_flags; + void **prealloc; + bool normal_persistent; +}; + +static depot_stack_handle_t +depot_save_stack_locked(struct stack_depot_hash_save *save) +{ + struct stack_record *found; + struct stack_record *new; + + lockdep_assert_held(&pool_lock); + + /* Try to find again, to avoid concurrently inserting duplicates. */ + found = find_stack(save->bucket, save->entries, save->nr_entries, + save->hash, save->depot_flags); + if (found) + return found->handle.handle; + + new = depot_alloc_stack(save->entries, save->nr_entries, save->hash, + save->depot_flags, save->prealloc); + if (!new) + return 0; + + /* + * This releases the stack record into the bucket and makes it visible to + * readers in find_stack(). + */ + list_add_rcu(&new->hash_list, save->bucket); + if (save->normal_persistent) + WRITE_ONCE(stack_depot_persistent_hash_record_seen, true); + + return new->handle.handle; +} + depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags, depot_flags_t depot_flags) { struct list_head *bucket; + struct stack_depot_hash_save save; struct stack_record *found = NULL; depot_stack_handle_t handle = 0; struct page *page = NULL; @@ -2894,21 +2935,17 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, trie_candidate = __stack_depot_trie_ready() && normal_persistent; if (trie_candidate) { - handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); - if (handle) - return handle; - if (READ_ONCE(stack_depot_persistent_hash_record_seen)) { /* * Trie storage may be enabled after stack depot has already saved * hash records. Preserve the same-handle contract by checking hash - * only on trie misses and only when hash records may exist. + * only when such records may exist. */ hash = hash_stack(entries, nr_entries); bucket = &stack_table[hash & stack_hash_mask]; found = find_stack(bucket, entries, nr_entries, hash, depot_flags); if (found) - goto exit; + return found->handle.handle; } handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, @@ -2920,11 +2957,20 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, hash = hash_stack(entries, nr_entries); bucket = &stack_table[hash & stack_hash_mask]; + save = (struct stack_depot_hash_save) { + .bucket = bucket, + .entries = entries, + .nr_entries = nr_entries, + .hash = hash, + .depot_flags = depot_flags, + .prealloc = &prealloc, + .normal_persistent = normal_persistent, + }; /* Fast path: look the stack trace up without locking. */ found = find_stack(bucket, entries, nr_entries, hash, depot_flags); if (found) - goto exit; + return found->handle.handle; /* * Allocate memory for a new pool if required now: * we won't be able to do that under the lock. @@ -2941,30 +2987,25 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, WARN_ON_ONCE(can_alloc); /* Best effort; bail if we fail to take the lock. */ if (!raw_spin_trylock_irqsave(&pool_lock, flags)) - goto exit; - } else { - raw_spin_lock_irqsave(&pool_lock, flags); - } - printk_deferred_enter(); - - /* Try to find again, to avoid concurrently inserting duplicates. */ - found = find_stack(bucket, entries, nr_entries, hash, depot_flags); - if (!found) { - struct stack_record *new = - depot_alloc_stack(entries, nr_entries, hash, depot_flags, &prealloc); - - if (new) { + goto out_free; + printk_deferred_enter(); + handle = depot_save_stack_locked(&save); + if (prealloc) { /* - * This releases the stack record into the bucket and - * makes it visible to readers in find_stack(). + * Either stack depot already contains this stack trace, or + * depot_alloc_stack() did not consume the preallocated memory. + * Try to keep the preallocated memory for future. */ - list_add_rcu(&new->hash_list, bucket); - if (normal_persistent) - WRITE_ONCE(stack_depot_persistent_hash_record_seen, true); - found = new; + depot_keep_new_pool(&prealloc); } + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + goto out_free; } + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + handle = depot_save_stack_locked(&save); if (prealloc) { /* * Either stack depot already contains this stack trace, or @@ -2973,16 +3014,14 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, */ depot_keep_new_pool(&prealloc); } - printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); -exit: + +out_free: if (prealloc) { /* Stack depot didn't use this memory, free it. */ free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); } - if (found) - handle = found->handle.handle; return handle; } EXPORT_SYMBOL_GPL(stack_depot_save_flags); From 6f255eb56537bb195c585ae4db89966745260c4c Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 17 Jun 2026 21:16:35 +0100 Subject: [PATCH 096/129] KRN-1117: Close stackdepot trie save-path gaps Make trie-backed persistent saves behave more like the legacy hash path in constrained contexts by using trylock-only insertion when all required resources are already available. Keep blocking side-table and pool work out of no-spin paths and document that constrained trie saves remain best effort. Reduce kernel-only overhead that hid the trie storage win by dynamically sizing the sorted pool index instead of charging stack_max_pools up front. Reuse retired trie nodes with best-fit tail recycling so reusable pool memory is not stranded by exact-size classes. Keep the helper contracts explicit with a retire_locked publish flag, clean up the counted-decrement cmpxchg seed, and extend KUnit coverage for no-spin trie hits. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 4 + lib/stackdepot.c | 704 +++++++++++++++++++++++++++++------ lib/stackdepot_internal.h | 1 + lib/tests/stackdepot_kunit.c | 2 + 4 files changed, 599 insertions(+), 112 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 288943fc23b04..bc3b4319ed4b6 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -113,6 +113,10 @@ static inline int stack_depot_early_init(void) { return 0; } * internal callers that depend on stackdepot count helpers. This flag does not * imply %STACK_DEPOT_FLAG_CAN_ALLOC. * + * When trie storage is enabled, persistent non-refcounted saves use trie + * storage. Constrained contexts remain best effort and can return 0 if a + * required trylock or reserved resource is unavailable. + * * If the provided stack trace comes from the interrupt context, only the part * up to the interrupt entry is saved. * diff --git a/lib/stackdepot.c b/lib/stackdepot.c index ca1ce0fe5e296..2f09c7956f746 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -98,6 +99,7 @@ static int stack_depot_trie_enabled_param_set(const char *val, } static const struct kernel_param_ops stack_depot_trie_enabled_param_ops = { + /* param_set_bool() treats a missing value as true. */ .flags = KERNEL_PARAM_OPS_FL_NOARG, .set = stack_depot_trie_enabled_param_set, .get = param_get_bool, @@ -189,6 +191,10 @@ static unsigned int stack_hash_mask; /* Array of memory regions that store stack records. */ static void **stack_pools; +/* Stack pools sorted by address for fast membership checks. */ +static void **stack_pools_sorted; +static unsigned int stack_pools_sorted_capacity; +static bool stack_pools_sorted_memblock; /* Newly allocated pool that is not yet added to stack_pools. */ static void *new_pool; /* Whether legacy hash storage may contain normal persistent records. */ @@ -368,6 +374,163 @@ static void stack_depot_trie_mark_not_ready(void) WRITE_ONCE(stack_depot_trie_ready, false); } +static int stack_pool_addr_cmp(const void *a, const void *b) +{ + unsigned long ap = (unsigned long)*(void * const *)a; + unsigned long bp = (unsigned long)*(void * const *)b; + + return (ap > bp) - (ap < bp); +} + +static unsigned int stack_pools_sorted_round_capacity(unsigned int pools) +{ + unsigned int step = PAGE_SIZE / sizeof(*stack_pools_sorted); + + if (!pools) + pools = 1; + if (pools > stack_max_pools) + return 0; + return min(round_up(pools, step), stack_max_pools); +} + +static unsigned int stack_pools_sorted_lower_bound(unsigned long addr, + unsigned int pools) +{ + unsigned int left = 0; + unsigned int right = pools; + + while (left < right) { + unsigned int mid = left + (right - left) / 2; + unsigned long mid_start = (unsigned long)stack_pools_sorted[mid]; + + if (mid_start < addr) + left = mid + 1; + else + right = mid; + } + + return left; +} + +static int __init stack_depot_trie_init_sorted_pools_memblock(void) +{ + unsigned int capacity; + size_t bytes; + + if (READ_ONCE(stack_pools_sorted)) + return 0; + capacity = stack_pools_sorted_round_capacity(READ_ONCE(pools_num) + 1); + if (!capacity) + return -ENOMEM; + bytes = capacity * sizeof(*stack_pools_sorted); + stack_pools_sorted = memblock_alloc(bytes, PAGE_SIZE); + if (!stack_pools_sorted) + return -ENOMEM; + memset(stack_pools_sorted, 0, bytes); + WRITE_ONCE(stack_pools_sorted_capacity, capacity); + WRITE_ONCE(stack_pools_sorted_memblock, true); + return 0; +} + +static int stack_depot_trie_init_sorted_pools(gfp_t gfp_flags) +{ + unsigned long flags; + unsigned int capacity; + unsigned int pools; + void **sorted; + + if (READ_ONCE(stack_pools_sorted)) + return 0; + capacity = stack_pools_sorted_round_capacity(READ_ONCE(pools_num) + 1); + if (!capacity) + return -ENOMEM; + sorted = kvcalloc(capacity, sizeof(*sorted), gfp_flags); + if (!sorted) + return -ENOMEM; + + while (sorted) { + raw_spin_lock_irqsave(&pool_lock, flags); + if (stack_pools_sorted) { + raw_spin_unlock_irqrestore(&pool_lock, flags); + break; + } + pools = READ_ONCE(pools_num); + if (pools > capacity) { + raw_spin_unlock_irqrestore(&pool_lock, flags); + break; + } + memcpy(sorted, stack_pools, pools * sizeof(*sorted)); + raw_spin_unlock_irqrestore(&pool_lock, flags); + + sort(sorted, pools, sizeof(*sorted), stack_pool_addr_cmp, NULL); + + raw_spin_lock_irqsave(&pool_lock, flags); + if (!stack_pools_sorted && pools == READ_ONCE(pools_num)) { + WRITE_ONCE(stack_pools_sorted_capacity, capacity); + WRITE_ONCE(stack_pools_sorted_memblock, false); + WRITE_ONCE(stack_pools_sorted, sorted); + sorted = NULL; + } + raw_spin_unlock_irqrestore(&pool_lock, flags); + } + + kvfree(sorted); + return 0; +} + +static void stack_pools_sorted_grow(gfp_t gfp_flags) +{ + unsigned int old_capacity; + unsigned int capacity; + unsigned long flags; + unsigned int pools; + void **old; + bool old_memblock = false; + void **sorted; + + old_capacity = READ_ONCE(stack_pools_sorted_capacity); + if (old_capacity > READ_ONCE(pools_num)) + return; + capacity = stack_pools_sorted_round_capacity(READ_ONCE(pools_num) + 1); + if (capacity <= old_capacity) + return; + + sorted = kvcalloc(capacity, sizeof(*sorted), gfp_flags); + if (!sorted) + return; + + for (;;) { + raw_spin_lock_irqsave(&pool_lock, flags); + old = stack_pools_sorted; + if (capacity <= stack_pools_sorted_capacity) { + raw_spin_unlock_irqrestore(&pool_lock, flags); + break; + } + pools = READ_ONCE(pools_num); + memcpy(sorted, stack_pools, pools * sizeof(*sorted)); + raw_spin_unlock_irqrestore(&pool_lock, flags); + + sort(sorted, pools, sizeof(*sorted), stack_pool_addr_cmp, NULL); + + raw_spin_lock_irqsave(&pool_lock, flags); + old = stack_pools_sorted; + if (capacity > stack_pools_sorted_capacity && + pools == READ_ONCE(pools_num)) { + old_memblock = stack_pools_sorted_memblock; + WRITE_ONCE(stack_pools_sorted_capacity, capacity); + WRITE_ONCE(stack_pools_sorted_memblock, false); + WRITE_ONCE(stack_pools_sorted, sorted); + sorted = old; + raw_spin_unlock_irqrestore(&pool_lock, flags); + break; + } + raw_spin_unlock_irqrestore(&pool_lock, flags); + } + + if (!old_memblock) + kvfree(sorted); +} + bool __stack_depot_trie_ready(void) { return __stack_depot_trie_enabled() && @@ -427,6 +590,54 @@ trie_side_table_dir_publish_chunk(struct stack_depot_trie_side_dir *dir, smp_store_release(&dir->chunks[idx], chunk); } +static u32 +trie_side_table_alloc_id_locked(struct stack_depot_trie_side_prealloc *prealloc) +{ + struct stack_depot_trie_side_entry *chunk; + struct stack_depot_trie_side_dir *dir; + unsigned int root; + unsigned int idx; + u32 id; + + lockdep_assert_held(&trie_side_table_lock); + if (!trie_side_table_is_initialized()) + return 0; + + id = trie_side_table_next_id + 1; + if (!id || id > trie_side_table_max_id) + return 0; + + root = trie_side_table_root_index(id); + if (root >= trie_side_table_root_size) + return 0; + + dir = trie_side_table_load_dir(root); + if (!dir) { + if (!prealloc || !prealloc->dir) + return 0; + dir = prealloc->dir; + prealloc->dir = NULL; + trie_side_table_publish_dir(root, dir); + trie_side_table_nr_dirs++; + if (trie_side_table_high_water < root + 1) + trie_side_table_high_water = root + 1; + } + + idx = trie_side_table_dir_index(id); + chunk = trie_side_table_dir_load_chunk(dir, idx); + if (!chunk) { + if (!prealloc || !prealloc->chunk) + return 0; + chunk = prealloc->chunk; + prealloc->chunk = NULL; + trie_side_table_dir_publish_chunk(dir, idx, chunk); + trie_side_table_nr_chunks++; + } + + WRITE_ONCE(trie_side_table_next_id, id); + return id; +} + static size_t trie_side_table_root_bytes(unsigned int root_size) { size_t bytes; @@ -613,6 +824,10 @@ static int __init stack_depot_trie_init_memblock(void) return 0; ret = stack_depot_trie_init_workspace_memblock(); + if (ret) + return ret; + /* Memblock allocations are permanent; keep successful pieces reusable. */ + ret = stack_depot_trie_init_sorted_pools_memblock(); if (ret) return ret; ret = __stack_depot_trie_side_table_init_memblock(); @@ -631,6 +846,9 @@ static int stack_depot_trie_init(gfp_t gfp_flags) return 0; ret = stack_depot_trie_init_workspace(gfp_flags); + if (ret) + return ret; + ret = stack_depot_trie_init_sorted_pools(gfp_flags); if (ret) return ret; ret = __stack_depot_trie_side_table_init(gfp_flags); @@ -832,70 +1050,76 @@ __stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_preallo u32 __stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc) +{ + unsigned long flags; + u32 id; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + id = trie_side_table_alloc_id_locked(prealloc); + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return id; +} + +static u32 trie_side_table_alloc_id_trylock(void) +{ + unsigned long flags; + u32 id; + + if (!raw_spin_trylock_irqsave(&trie_side_table_lock, flags)) + return 0; + id = trie_side_table_alloc_id_locked(NULL); + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return id; +} + +void __stack_depot_trie_side_table_revoke_latest(u32 id) { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; unsigned long flags; + unsigned int slot; unsigned int root; - unsigned int idx; - u32 id; - if (!trie_side_table_is_initialized()) - return 0; + if (!trie_side_table_is_initialized() || !id || + id != READ_ONCE(trie_side_table_next_id)) + return; raw_spin_lock_irqsave(&trie_side_table_lock, flags); - id = trie_side_table_next_id + 1; - if (!id || id > trie_side_table_max_id) - goto fail; - + if (id != trie_side_table_next_id) + goto out; root = trie_side_table_root_index(id); if (root >= trie_side_table_root_size) - goto fail; + goto out; dir = trie_side_table_load_dir(root); - if (!dir) { - if (!prealloc || !prealloc->dir) - goto fail; - dir = prealloc->dir; - prealloc->dir = NULL; - trie_side_table_publish_dir(root, dir); - trie_side_table_nr_dirs++; - if (trie_side_table_high_water < root + 1) - trie_side_table_high_water = root + 1; - } - - idx = trie_side_table_dir_index(id); - chunk = trie_side_table_dir_load_chunk(dir, idx); - if (!chunk) { - if (!prealloc || !prealloc->chunk) - goto fail; - chunk = prealloc->chunk; - prealloc->chunk = NULL; - trie_side_table_dir_publish_chunk(dir, idx, chunk); - trie_side_table_nr_chunks++; - } + if (!dir) + goto out; + chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); + if (!chunk) + goto out; - WRITE_ONCE(trie_side_table_next_id, id); - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return id; -fail: + slot = trie_side_table_slot_index(id); + trie_side_table_clear_entry(chunk, slot); + WRITE_ONCE(trie_side_table_next_id, id - 1); +out: raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return 0; } -void __stack_depot_trie_side_table_revoke_latest(u32 id) +static bool trie_side_table_revoke_latest_trylock(u32 id) { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; unsigned long flags; unsigned int slot; unsigned int root; + bool ret = false; if (!trie_side_table_is_initialized() || !id || id != READ_ONCE(trie_side_table_next_id)) - return; + return false; - raw_spin_lock_irqsave(&trie_side_table_lock, flags); + if (!raw_spin_trylock_irqsave(&trie_side_table_lock, flags)) + return false; if (id != trie_side_table_next_id) goto out; root = trie_side_table_root_index(id); @@ -912,8 +1136,10 @@ void __stack_depot_trie_side_table_revoke_latest(u32 id) slot = trie_side_table_slot_index(id); trie_side_table_clear_entry(chunk, slot); WRITE_ONCE(trie_side_table_next_id, id - 1); + ret = true; out: raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return ret; } void __stack_depot_trie_side_table_restore(u32 id, const void *entry) @@ -1171,19 +1397,45 @@ static void *trie_object_init_fresh(void *ptr, size_t size) return trie_object_payload(free); } -static void depot_record_pool_range_locked(void *pool) +static void depot_record_pool_locked(void *pool) { unsigned long start = (unsigned long)pool; unsigned long end; + unsigned int pools = READ_ONCE(pools_num); + unsigned int pos; lockdep_assert_held(&pool_lock); - if (check_add_overflow(start, DEPOT_POOL_SIZE, &end)) + if (!pool || check_add_overflow(start, DEPOT_POOL_SIZE, &end)) return; if (!pools_min_addr || start < pools_min_addr) pools_min_addr = start; if (end > pools_max_addr) pools_max_addr = end; + + if (!stack_pools_sorted || READ_ONCE(stack_pools_sorted_capacity) <= pools) + return; + pos = stack_pools_sorted_lower_bound(start, pools); + memmove(&stack_pools_sorted[pos + 1], &stack_pools_sorted[pos], + (pools - pos) * sizeof(*stack_pools_sorted)); + stack_pools_sorted[pos] = pool; +} + +static void depot_forget_pool_locked(void *pool) +{ + unsigned int pools = READ_ONCE(pools_num); + unsigned int i; + + lockdep_assert_held(&pool_lock); + if (!pool || !stack_pools_sorted || READ_ONCE(stack_pools_sorted_capacity) < pools) + return; + + i = stack_pools_sorted_lower_bound((unsigned long)pool, pools); + if (i >= pools || stack_pools_sorted[i] != pool) + return; + memmove(&stack_pools_sorted[i], &stack_pools_sorted[i + 1], + (pools - i - 1) * sizeof(*stack_pools_sorted)); + stack_pools_sorted[pools - 1] = NULL; } static bool trie_pool_range_contains_locked(const void *ptr, size_t size) @@ -1191,29 +1443,39 @@ static bool trie_pool_range_contains_locked(const void *ptr, size_t size) unsigned long start = (unsigned long)ptr; unsigned long end; unsigned int pools = READ_ONCE(pools_num); - unsigned int i; + unsigned int left = 0; + unsigned int right = pools; + unsigned int pos; + unsigned long pool_start; lockdep_assert_held(&pool_lock); if (!ptr || !size || check_add_overflow(start, size, &end)) return false; - if (!stack_pools) + if (!stack_pools_sorted || READ_ONCE(stack_pools_sorted_capacity) < pools) { + unsigned int i; + + if (!stack_pools) + return false; + for (i = 0; i < pools; i++) { + if (!stack_pools[i]) + continue; + pool_start = (unsigned long)stack_pools[i]; + if (start >= pool_start && end <= pool_start + DEPOT_POOL_SIZE) + return true; + } return false; + } if (pools_min_addr && (start < pools_min_addr || end > pools_max_addr)) return false; - for (i = 0; i < pools; i++) { - unsigned long pool_start; - void *pool = stack_pools[i]; - - if (!pool) - continue; - pool_start = (unsigned long)pool; - if (start >= pool_start && end <= pool_start + DEPOT_POOL_SIZE) - return true; - } + left = stack_pools_sorted_lower_bound(start + 1, right); + if (!left) + return false; - return false; + pos = left - 1; + pool_start = (unsigned long)stack_pools_sorted[pos]; + return end <= pool_start + DEPOT_POOL_SIZE; } static bool trie_pool_contains_locked(const void *ptr) @@ -1314,10 +1576,13 @@ static void trie_drain_pending_objects_locked(void) } } +static void trie_free_object_tail_locked(void *ptr, size_t size); + static void *trie_pop_free_node(size_t size) { struct stack_depot_trie_free_node *free; unsigned int class; + size_t old_size; lockdep_assert_held(&pool_lock); @@ -1325,13 +1590,18 @@ static void *trie_pop_free_node(size_t size) if (!size) return NULL; class = trie_free_class(size); - if (!test_bit(class, free_trie_node_map)) + class = find_next_bit(free_trie_node_map, STACK_DEPOT_TRIE_FREE_CLASSES, + class); + if (class >= STACK_DEPOT_TRIE_FREE_CLASSES) return NULL; free = list_first_entry(&free_trie_nodes[class], typeof(*free), list); - if (WARN_ON_ONCE(free->size != size)) + if (WARN_ON_ONCE(free->size < size)) return NULL; + old_size = free->size; trie_free_list_del(&free->list, free_trie_nodes, free_trie_node_map, class); + if (old_size > size) + trie_free_object_tail_locked((void *)free + size, old_size - size); return free; } @@ -1376,17 +1646,16 @@ static void trie_split_free_object_locked(struct stack_depot_trie_free_object *f trie_free_object_tail_locked(tail, tail_size); } -static void trie_retire_object_node(const void *ptr, const void *node, - size_t node_size) +static void trie_retire_object_node_locked(const void *ptr, const void *node, + size_t node_size) { struct stack_depot_trie_free_object *free; size_t size; - unsigned long flags; + lockdep_assert_held(&pool_lock); if (!ptr) return; - raw_spin_lock_irqsave(&pool_lock, flags); trie_free_object_buckets_init_locked(); free = trie_object_header(ptr); if (trie_pool_contains_locked(free)) { @@ -1404,6 +1673,15 @@ static void trie_retire_object_node(const void *ptr, const void *node, pending_trie_object_map, trie_free_class(free->size), true); } +} + +static void trie_retire_object_node(const void *ptr, const void *node, + size_t node_size) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&pool_lock, flags); + trie_retire_object_node_locked(ptr, node, node_size); raw_spin_unlock_irqrestore(&pool_lock, flags); } @@ -1471,6 +1749,8 @@ int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && gfpflags_allow_spinning(alloc_flags); + if (can_alloc) + stack_pools_sorted_grow(alloc_flags); needs_side_prealloc = __stack_depot_trie_side_table_prealloc_needed(); if (can_alloc && !READ_ONCE(new_pool)) *pool_prealloc = __stack_depot_trie_pool_prealloc(alloc_flags); @@ -1546,6 +1826,7 @@ bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mar goto out; if (new_pool && new_pool != STACK_DEPOT_POISON) goto out; + depot_forget_pool_locked(mark->pool); stack_pools[mark->pool_index] = NULL; WRITE_ONCE(pools_num, mark->pool_index); pool_offset = mark->prev_offset; @@ -1630,6 +1911,16 @@ static void trie_pool_release_reused_objects(struct stack_depot_trie_pool_reques raw_spin_unlock_irqrestore(&pool_lock, flags); } +static void trie_pool_release_reused_objects_trylock(struct stack_depot_trie_pool_request *req) +{ + unsigned long flags; + + if (!raw_spin_trylock_irqsave(&pool_lock, flags)) + return; + trie_pool_release_reused_objects_locked(req); + raw_spin_unlock_irqrestore(&pool_lock, flags); +} + int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) { unsigned long flags; @@ -1767,6 +2058,21 @@ __stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, return 0; } +static int trie_alloc_txn_id_trylock(struct stack_depot_trie_alloc_txn *txn) +{ + u32 leaf_id; + + if (!txn || txn->leaf_id) + return -EINVAL; + + leaf_id = trie_side_table_alloc_id_trylock(); + if (!leaf_id) + return -ENOSPC; + + txn->leaf_id = leaf_id; + return 0; +} + static void trie_alloc_request_clear_outputs(struct stack_depot_trie_alloc_request *req) { unsigned int i; @@ -1831,6 +2137,69 @@ int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request * return 0; } +static void +trie_alloc_request_release_reused_objects_trylock(struct stack_depot_trie_alloc_request *req) +{ + struct stack_depot_trie_pool_request pool_req = {}; + + if (!req || !req->txn) + return; + pool_req.node_slots = req->node_slots; + pool_req.nr_node_slots = req->nr_node_slots; + pool_req.child_slots = req->child_slots; + pool_req.nr_child_slots = req->nr_child_slots; + pool_req.storage = req->storage; + pool_req.mark = &req->txn->pool; + trie_pool_release_reused_objects_trylock(&pool_req); +} + +static void trie_alloc_txn_rollback_trylock(struct stack_depot_trie_alloc_txn *txn) +{ + if (!txn) + return; + + if (txn->side.nr_updates) + return; + if (txn->leaf_id && trie_side_table_revoke_latest_trylock(txn->leaf_id)) + txn->leaf_id = 0; + __stack_depot_trie_pool_try_rollback(&txn->pool); + memset(&txn->pool, 0, sizeof(txn->pool)); +} + +static int trie_alloc_txn_reserve_trylock(struct stack_depot_trie_alloc_request *req) +{ + struct stack_depot_trie_pool_request pool_req = {}; + int ret; + + if (!req || !req->txn) + return -EINVAL; + if (req->txn->leaf_id || req->txn->pool.size || req->txn->side.nr_updates) + return -EINVAL; + + pool_req.node_slots = req->node_slots; + pool_req.nr_node_slots = req->nr_node_slots; + pool_req.child_slots = req->child_slots; + pool_req.nr_child_slots = req->nr_child_slots; + pool_req.storage = req->storage; + pool_req.storage_size = req->storage_size; + pool_req.prealloc = req->pool_prealloc; + pool_req.mark = &req->txn->pool; + + ret = __stack_depot_trie_pool_carve(&pool_req); + if (ret) + return ret; + + ret = trie_alloc_txn_id_trylock(req->txn); + if (ret) { + trie_alloc_request_release_reused_objects_trylock(req); + trie_alloc_txn_rollback_trylock(req->txn); + trie_alloc_request_clear_outputs(req); + return ret; + } + + return 0; +} + int __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, const unsigned long *entries, @@ -1943,6 +2312,78 @@ static int trie_ws_insert(struct stack_depot_trie_root *root, workspace, tail, leaf_id); } +static int trie_side_prepare_trylock(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, void *ctx); + +static int trie_ws_insert_trylock(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_trie_alloc_workspace *workspace, + const void **tail, u32 *leaf_id) +{ + struct stack_depot_trie_alloc_request *req = &workspace->req; + struct stack_depot_trie_publish_prepare prepare; + struct stack_depot_trie_side_prealloc side_prealloc = {}; + void *pool_prealloc = NULL; + unsigned long flags; + unsigned long retire_flags; + unsigned int nr_used; + bool retire_locked = false; + u32 id; + int ret; + + if (!raw_spin_trylock_irqsave(&trie_alloc_lock, flags)) + return -EBUSY; + + ret = trie_ws_plan(root, entries, nr_entries, &pool_prealloc, + &side_prealloc, workspace); + if (ret) + goto out_unlock; + if (pool_prealloc || side_prealloc.dir || side_prealloc.chunk) { + ret = -EINVAL; + goto out_unlock; + } + + ret = trie_alloc_txn_reserve_trylock(req); + if (ret) + goto out_unlock; + if (!raw_spin_trylock_irqsave(&pool_lock, retire_flags)) { + ret = -EBUSY; + goto rollback; + } + retire_locked = true; + + prepare.fn = trie_side_prepare_trylock; + prepare.ctx = &workspace->txn.side; + prepare.retire_locked = true; + id = workspace->txn.leaf_id; + ret = __stack_depot_trie_insert_append_prepare(root, NULL, id, entries, + nr_entries, workspace->node_slots, + req->nr_node_slots, workspace->child_slots, + req->nr_child_slots, workspace->scratch, + ARRAY_SIZE(workspace->scratch), + workspace->storage, req->storage_size, + &prepare, tail, &nr_used); + raw_spin_unlock_irqrestore(&pool_lock, retire_flags); + retire_locked = false; + if (ret) + goto rollback; + + *leaf_id = __stack_depot_trie_alloc_txn_commit(&workspace->txn); + ret = 0; + goto out_unlock; + +rollback: + if (retire_locked) + raw_spin_unlock_irqrestore(&pool_lock, retire_flags); + trie_alloc_request_release_reused_objects_trylock(req); + trie_alloc_txn_rollback_trylock(&workspace->txn); + trie_alloc_request_clear_outputs(req); + *tail = NULL; +out_unlock: + raw_spin_unlock_irqrestore(&trie_alloc_lock, flags); + return ret; +} + static depot_stack_handle_t trie_find_handle(const struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries) @@ -1978,6 +2419,8 @@ trie_save_miss(struct stack_depot_trie_root *root, const unsigned long *entries, return 0; if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) return 0; + if (in_nmi() || !gfpflags_allow_spinning(alloc_flags)) + return 0; ret = trie_prealloc(alloc_flags, depot_flags, &pool_prealloc, &side_prealloc); @@ -2053,18 +2496,17 @@ trie_save_locked_insert(struct stack_depot_trie_root *root, } static depot_stack_handle_t -trie_save_trylocked(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_alloc_workspace *workspace, - raw_spinlock_t *workspace_lock, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - bool can_insert) +trie_save_spinlocked(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_trie_alloc_workspace *workspace, + raw_spinlock_t *workspace_lock, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, + bool can_insert) { depot_stack_handle_t handle; unsigned long flags; - if (!raw_spin_trylock_irqsave(workspace_lock, flags)) - return 0; + raw_spin_lock_irqsave(workspace_lock, flags); handle = trie_save_locked_insert(root, entries, nr_entries, workspace, pool_prealloc, side_prealloc, can_insert); @@ -2073,20 +2515,25 @@ trie_save_trylocked(struct stack_depot_trie_root *root, } static depot_stack_handle_t -trie_save_spinlocked(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_alloc_workspace *workspace, - raw_spinlock_t *workspace_lock, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - bool can_insert) +trie_save_trylocked(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_trie_alloc_workspace *workspace, + raw_spinlock_t *workspace_lock) { - depot_stack_handle_t handle; + depot_stack_handle_t handle = 0; unsigned long flags; + const void *tail; + u32 leaf_id; - raw_spin_lock_irqsave(workspace_lock, flags); - handle = trie_save_locked_insert(root, entries, nr_entries, workspace, - pool_prealloc, side_prealloc, - can_insert); + if (!raw_spin_trylock_irqsave(workspace_lock, flags)) + return 0; + handle = trie_find_handle(root, entries, nr_entries); + if (handle) + goto out; + if (!trie_ws_insert_trylock(root, entries, nr_entries, workspace, &tail, + &leaf_id)) + handle = __stack_depot_trie_handle(leaf_id); +out: raw_spin_unlock_irqrestore(workspace_lock, flags); return handle; } @@ -2116,19 +2563,16 @@ __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, if (handle) return handle; no_spin = in_nmi() || !gfpflags_allow_spinning(alloc_flags); + if (no_spin) + return trie_save_trylocked(root, entries, nr_entries, workspace, + workspace_lock); ret = trie_prealloc(alloc_flags, depot_flags, &pool_prealloc, &side_prealloc); can_insert = !ret; - - if (no_spin) - handle = trie_save_trylocked(root, entries, nr_entries, workspace, - workspace_lock, &pool_prealloc, - &side_prealloc, can_insert); - else - handle = trie_save_spinlocked(root, entries, nr_entries, workspace, - workspace_lock, &pool_prealloc, - &side_prealloc, can_insert); + handle = trie_save_spinlocked(root, entries, nr_entries, workspace, + workspace_lock, &pool_prealloc, + &side_prealloc, can_insert); depot_try_keep_new_pool(&pool_prealloc); __stack_depot_trie_pool_free_prealloc(pool_prealloc); @@ -2180,6 +2624,7 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, prepare.fn = __stack_depot_trie_side_prepare; prepare.ctx = &txn->side; + prepare.retire_locked = false; id = txn->leaf_id; ret = __stack_depot_trie_insert_append_prepare(root, NULL, id, entries, nr_entries, req->node_slots, @@ -2238,31 +2683,30 @@ void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *stat } } -int -__stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, void *ctx) +static int +trie_side_prepare_locked(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, + struct stack_depot_trie_side_prepare *state) { - struct stack_depot_trie_side_prepare *state = ctx; struct stack_depot_trie_side_entry *chunks[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; unsigned int slots[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; - unsigned long flags; unsigned int i; + lockdep_assert_held(&trie_side_table_lock); if (!state || (!updates && nr_updates)) return -EINVAL; if (nr_updates > ARRAY_SIZE(chunks) || state->nr_updates > ARRAY_SIZE(state->updates) - nr_updates) return -EINVAL; - raw_spin_lock_irqsave(&trie_side_table_lock, flags); for (i = 0; i < nr_updates; i++) { u32 leaf_id = updates[i].leaf_id; if (!updates[i].leaf) - goto rollback; + return -EINVAL; chunks[i] = trie_side_table_chunk_locked(leaf_id, &slots[i]); if (!chunks[i]) - goto rollback; + return -EINVAL; } for (i = 0; i < nr_updates; i++) { @@ -2272,13 +2716,35 @@ __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updat state->nr_updates++; trie_side_table_store_leaf(chunks[i], slots[i], updates[i].leaf); } - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); return 0; +} -rollback: +int +__stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, void *ctx) +{ + unsigned long flags; + int ret; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + ret = trie_side_prepare_locked(updates, nr_updates, ctx); raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return -EINVAL; + return ret; +} + +static int +trie_side_prepare_trylock(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, void *ctx) +{ + unsigned long flags; + int ret; + + if (!raw_spin_trylock_irqsave(&trie_side_table_lock, flags)) + return -EBUSY; + ret = trie_side_prepare_locked(updates, nr_updates, ctx); + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return ret; } static int __init disable_stack_depot(char *str) @@ -2469,8 +2935,8 @@ int stack_depot_init(void) stack_hash_mask = 0; stack_depot_disabled = true; ret = -ENOMEM; + goto out_unlock; } - init_trie: if (!ret && __stack_depot_trie_enabled()) { ret = stack_depot_trie_init(GFP_KERNEL); @@ -2515,7 +2981,7 @@ static bool depot_init_pool(void **prealloc) /* Save reference to the pool to be used by depot_fetch_stack(). */ stack_pools[pools_num] = new_pool; - depot_record_pool_range_locked(new_pool); + depot_record_pool_locked(new_pool); /* * Stack depot tries to keep an extra pool allocated even before it runs @@ -2932,8 +3398,7 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, normal_persistent = !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH)) && nr_entries <= CONFIG_STACKDEPOT_MAX_FRAMES; - trie_candidate = __stack_depot_trie_ready() && - normal_persistent; + trie_candidate = normal_persistent && __stack_depot_trie_ready(); if (trie_candidate) { if (READ_ONCE(stack_depot_persistent_hash_record_seen)) { /* @@ -3135,8 +3600,8 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, * saturate on underflow, but page_owner accounting must warn and leave * the existing count unchanged. */ - /* A stale read is harmless: cmpxchg reloads @old before retry checks. */ - old = data_race(refcount_read(&stack->count)); /* See above. */ + /* The first cmpxchg failure reloads @old before retry checks. */ + old = INT_MAX; do { bool underflow; @@ -4246,6 +4711,9 @@ trie_promote_child(struct stack_depot_trie_root *root, ret = trie_clone_promoted_node(child, leaf_id, slot); if (ret) return ret; + child_size = __stack_depot_trie_node_size(&child->run); + if (!child_size) + return -EINVAL; if (prepare) { if (!prepare->fn) return -EINVAL; @@ -4262,8 +4730,10 @@ trie_promote_child(struct stack_depot_trie_root *root, publish_slot = trie_publish_slot(root, parent); /* Publish the fully initialized replacement array last. */ smp_store_release(publish_slot, new_storage); - child_size = __stack_depot_trie_node_size(&child->run); - trie_retire_object_node(old_array, child, child_size); + if (prepare && prepare->retire_locked) + trie_retire_object_node_locked(old_array, child, child_size); + else + trie_retire_object_node(old_array, child, child_size); return 0; } @@ -4431,6 +4901,7 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, size_t storage_size = new_storage_size; size_t new_size; size_t old_size; + int ret; if ((root && parent) || (!root && !parent) || !head) return -EINVAL; @@ -4491,7 +4962,6 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, .leaf_id = leaf_id, .leaf = leaf, }; - int ret; if (!prepare->fn) return -EINVAL; @@ -4510,7 +4980,10 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, /* Publish the fully initialized replacement array last. */ smp_store_release(slot, new_array); - trie_retire_object(old_array); + if (prepare && prepare->retire_locked) + trie_retire_object_node_locked(old_array, NULL, 0); + else + trie_retire_object(old_array); return 0; } @@ -5872,6 +6345,9 @@ static int trie_split_child(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array **publish_slot; const struct stack_depot_trie_child_array *old_array; const void *prefix; + void *storage = new_storage; + size_t child_size; + size_t storage_size = new_storage_size; unsigned int pos; unsigned int used; int ret; @@ -5893,10 +6369,12 @@ static int trie_split_child(struct stack_depot_trie_root *root, /* Pairs with append, promote, and split publication. */ old_array = smp_load_acquire(publish_slot); - ret = trie_child_array_replace_precheck(old_array, child, new_storage, - new_storage_size, &pos); + ret = trie_child_array_replace_precheck(old_array, child, storage, storage_size, &pos); if (ret) return ret; + child_size = __stack_depot_trie_node_size(&child->run); + if (!child_size) + return -EINVAL; ret = trie_split_subtree_prepare(child, matched, leaf_id, entries, nr_entries, node_slots, nr_node_slots, @@ -5910,8 +6388,10 @@ static int trie_split_child(struct stack_depot_trie_root *root, new_storage_size, pos); /* Publish the fully initialized replacement array last. */ smp_store_release(publish_slot, new_storage); - trie_retire_object_node(old_array, child, - __stack_depot_trie_node_size(&child->run)); + if (prepare && prepare->retire_locked) + trie_retire_object_node_locked(old_array, child, child_size); + else + trie_retire_object_node(old_array, child, child_size); *nr_used = used; return 0; } diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 92ed2c3f8fbe1..a1b0abc19fadf 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -68,6 +68,7 @@ struct stack_depot_trie_publish_prepare { int (*fn)(const struct stack_depot_trie_leaf_update *updates, unsigned int nr_updates, void *ctx); void *ctx; + bool retire_locked; }; #define STACK_DEPOT_TRIE_MAX_LEAF_UPDATES 2 diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index baaec7bc012e7..e4507b3b479a9 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -5623,6 +5623,8 @@ static void stackdepot_trie_public_save_route(struct kunit *test) nr_entries = stack_depot_fetch_into(trie_handle, fetched, ARRAY_SIZE(fetched)); KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(trie_entries)); KUNIT_EXPECT_MEMEQ(test, fetched, trie_entries, sizeof(trie_entries)); + noalloc_handle = stack_depot_save_flags(trie_entries, ARRAY_SIZE(trie_entries), no_spin, 0); + KUNIT_EXPECT_EQ(test, noalloc_handle, trie_handle); noalloc_handle = stack_depot_save_flags(noalloc_entries, noalloc_nr, no_spin, 0); KUNIT_ASSERT_NE(test, noalloc_handle, (depot_stack_handle_t)0); KUNIT_EXPECT_NE(test, __stack_depot_trie_leaf_id(noalloc_handle), 0U); From 428f519d5b3bf1861c97b264dfeaff661e404433 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 17 Jun 2026 21:18:50 +0100 Subject: [PATCH 097/129] KRN-1117: Preserve page owner handles without stack list nodes Keep applying page-owner stack counts even when allocating the optional show_stacks list node fails. The stack list remains best effort, but the page handle and count stay symmetric so free-time accounting can still decrement the allocation handle. Also document that a zero handle means no allocation stack count was applied. Signed-off-by: Caleb Kan --- mm/page_owner.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index dbdb7c9fe4903..f1a67a06d9a59 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -229,12 +229,8 @@ static bool inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, /* Snapshot only avoids allocation when the stack is already counted. */ /* If this races a final decrement to zero, inc_count() fails safely. */ - if (!__stack_depot_get_count(handle, &count)) { + if (!__stack_depot_get_count(handle, &count)) stack = alloc_stack_record(gfp_mask); - /* Leave saturated stacks retryable for future tracked allocations. */ - if (!stack) - return false; - } /* Racing transition losers free their unused list node below. */ if (!__stack_depot_inc_count(handle, nr_base_pages, &new_count)) { @@ -242,13 +238,13 @@ static bool inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, free_stack_record(stack); return false; } - /* new_count includes the list marker, and requires the node allocated above. */ + /* + * new_count includes the list marker. If list allocation failed, keep + * the count and handle anyway; show_stacks remains best effort. + */ if (new_count) { - if (WARN_ON_ONCE(!stack)) { - __stack_depot_dec_count_and_test(handle, nr_base_pages + 1); - return false; - } - add_stack_record_to_list(handle, stack); + if (stack) + add_stack_record_to_list(handle, stack); } else if (stack) { free_stack_record(stack); } @@ -345,6 +341,7 @@ void __reset_page_owner(struct page *page, unsigned short order) __update_page_owner_free_handle(page, handle, order, current->pid, current->tgid, free_ts_nsec); + /* A zero handle means no allocation stack count was applied. */ if (alloc_handle && alloc_handle != early_handle) /* * early_handle is being set as a handle for all those From 88ac7bab2812b678de63b7685a5dde053f9997ff Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 17 Jun 2026 23:04:33 +0100 Subject: [PATCH 098/129] KRN-1117: Harden stackdepot trie teardown ordering Mark trie and side-table state unavailable with release stores before freeing side-table storage, then wait for pre-existing RCU readers to drain. This keeps test teardown and reinitialization from racing lockless side-table lookups. Also route trie_enabled sysfs reads through a READ_ONCE-backed getter so the runtime toggle parameter matches the setter's WRITE_ONCE discipline. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 2f09c7956f746..75c71a5e8ac74 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -98,11 +98,21 @@ static int stack_depot_trie_enabled_param_set(const char *val, return 0; } +static int stack_depot_trie_enabled_param_get(char *buffer, + const struct kernel_param *kp) +{ + struct kernel_param tmp = *kp; + bool enabled = READ_ONCE(*(bool *)kp->arg); + + tmp.arg = &enabled; + return param_get_bool(buffer, &tmp); +} + static const struct kernel_param_ops stack_depot_trie_enabled_param_ops = { /* param_set_bool() treats a missing value as true. */ .flags = KERNEL_PARAM_OPS_FL_NOARG, .set = stack_depot_trie_enabled_param_set, - .get = param_get_bool, + .get = stack_depot_trie_enabled_param_get, }; module_param_cb(trie_enabled, &stack_depot_trie_enabled_param_ops, &stack_depot_trie_enabled_param, 0644); @@ -371,7 +381,8 @@ static void trie_side_table_publish_initialized(void) static void stack_depot_trie_mark_not_ready(void) { - WRITE_ONCE(stack_depot_trie_ready, false); + /* Pairs with stack_depot_trie_is_ready(). */ + smp_store_release(&stack_depot_trie_ready, false); } static int stack_pool_addr_cmp(const void *a, const void *b) @@ -918,7 +929,9 @@ void __stack_depot_trie_side_table_destroy(void) if (!trie_side_table_is_initialized()) return; stack_depot_trie_mark_not_ready(); - WRITE_ONCE(trie_side_table_initialized, false); + /* Pairs with trie_side_table_is_initialized(). */ + smp_store_release(&trie_side_table_initialized, false); + synchronize_rcu(); high_water = READ_ONCE(trie_side_table_high_water); if (!READ_ONCE(trie_side_table_memblock)) { From b8ee429e2291955975a37d2a0f601bd2dd17f229 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 18 Jun 2026 11:56:44 +0100 Subject: [PATCH 099/129] KRN-1117: Initialize stackdepot trie on runtime enable Make runtime writes to stackdepot.trie_enabled fully enable trie storage after boot by calling stack_depot_init() once the system is running. If initialization fails, disable the static key again and return the failure to the sysfs writer. This keeps the emergency/runtime toggle path consistent with the boot-time stackdepot.trie_enabled=1 path. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 75c71a5e8ac74..4249a45c93c7c 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -95,6 +95,13 @@ static int stack_depot_trie_enabled_param_set(const char *val, return ret; __stack_depot_trie_set_enabled(enabled); + if (enabled && system_state >= SYSTEM_RUNNING) { + ret = stack_depot_init(); + if (ret || !__stack_depot_trie_ready()) { + __stack_depot_trie_set_enabled(false); + return ret ?: -ENOMEM; + } + } return 0; } From 4f5a99bddc422f5cb111ae757c29fc122afbc347 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 18 Jun 2026 14:27:47 +0100 Subject: [PATCH 100/129] KRN-1117: Drop stackdepot MAINTAINERS entry Keep the Cloudflare stackdepot one-off focused on the trie backend and caller migration. A dedicated upstream MAINTAINERS entry can be discussed later if the work is prepared for upstream submission. Signed-off-by: Caleb Kan --- MAINTAINERS | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 6adb143d363d2..554e881b05bea 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -24615,18 +24615,6 @@ S: Supported F: Documentation/devicetree/bindings/interrupt-controller/starfive,jh8100-intc.yaml F: drivers/irqchip/irq-starfive-jh8100-intc.c -STACK DEPOT -M: Andrew Morton -L: linux-kernel@vger.kernel.org -S: Supported -T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-nonmm-unstable -F: arch/*/include/asm/stackdepot.h -F: include/asm-generic/stackdepot.h -F: include/linux/stackdepot.h -F: lib/stackdepot.c -F: lib/stackdepot_internal.h -F: lib/tests/stackdepot* - STATIC BRANCH/CALL M: Peter Zijlstra M: Josh Poimboeuf From 5bdac6c0573eae922a2099c2719b9708024230fa Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 18 Jun 2026 14:56:42 +0100 Subject: [PATCH 101/129] KRN-1117: Reduce KMSAN stackdepot report stack usage Reuse the existing origin stack buffer when fetching chained KMSAN origins instead of keeping a second KMSAN_STACK_DEPTH array on the reporting stack. The chained origin path consumes the previous head and origin values before refilling the buffer, so the materialized trace can share the same storage safely. Signed-off-by: Caleb Kan --- mm/kmsan/report.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mm/kmsan/report.c b/mm/kmsan/report.c index 94746ebe97e74..df4b70a757a6a 100644 --- a/mm/kmsan/report.c +++ b/mm/kmsan/report.c @@ -86,7 +86,7 @@ static char *pretty_descr(char *descr) void kmsan_print_origin(depot_stack_handle_t origin) { unsigned long entries[KMSAN_STACK_DEPTH]; - unsigned long chain[KMSAN_STACK_DEPTH]; + const unsigned int max_entries = ARRAY_SIZE(entries); unsigned int nr_entries, chained_nr_entries, skipnr; void *pc1 = NULL, *pc2 = NULL; depot_stack_handle_t head; @@ -98,7 +98,7 @@ void kmsan_print_origin(depot_stack_handle_t origin) return; while (true) { - nr_entries = stack_depot_fetch_into(origin, entries, ARRAY_SIZE(entries)); + nr_entries = stack_depot_fetch_into(origin, entries, max_entries); depth = kmsan_depth_from_eb(stack_depot_get_extra_bits(origin)); magic = nr_entries ? entries[0] : 0; if ((nr_entries == 4) && (magic == KMSAN_ALLOCA_MAGIC_ORIGIN)) { @@ -123,13 +123,14 @@ void kmsan_print_origin(depot_stack_handle_t origin) head = entries[1]; origin = entries[2]; pr_err("Uninit was stored to memory at:\n"); - chained_nr_entries = stack_depot_fetch_into(head, chain, ARRAY_SIZE(chain)); + chained_nr_entries = + stack_depot_fetch_into(head, entries, max_entries); kmsan_internal_unpoison_memory( - chain, - chained_nr_entries * sizeof(*chain), + entries, + chained_nr_entries * sizeof(*entries), /*checked*/ false); - skipnr = get_stack_skipnr(chain, chained_nr_entries); - stack_trace_print(chain + skipnr, + skipnr = get_stack_skipnr(entries, chained_nr_entries); + stack_trace_print(entries + skipnr, chained_nr_entries - skipnr, 0); pr_err("\n"); continue; From 9fb876c2e74791aa6ce06ea308d7bad4351e8635 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 18 Jun 2026 14:57:10 +0100 Subject: [PATCH 102/129] KRN-1117: Clarify stackdepot trie review details Make the stackdepot internal header include the static_assert() provider directly instead of relying on includer order. Also document that trie insert failures intentionally return zero rather than falling back to hash storage, keeping trie pool pressure visible during validation. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 1 + lib/stackdepot_internal.h | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 4249a45c93c7c..13567171b062f 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -3437,6 +3437,7 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, depot_flags); if (handle) return handle; + /* Keep trie failures visible; hash fallback hides trie pool pressure. */ return 0; } diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index a1b0abc19fadf..a1061e368149a 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -2,6 +2,7 @@ #ifndef _STACKDEPOT_INTERNAL_H #define _STACKDEPOT_INTERNAL_H +#include #include #include #include From 842163f21b8cbbab607dd6902ddfed84d82ddb28 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 18 Jun 2026 15:11:35 +0100 Subject: [PATCH 103/129] KRN-1117: Read page owner threshold consistently Match the page owner stack display path with the debugfs threshold setter and getter by reading page_owner_pages_threshold through READ_ONCE(). This avoids a racy plain load while preserving the best-effort nature of show_stacks output. Signed-off-by: Caleb Kan --- mm/page_owner.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index f1a67a06d9a59..cdda22ccb44b0 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -954,7 +954,7 @@ static int stack_print(struct seq_file *m, void *v) nr_base_pages--; /* Drop the list marker before applying the page-count threshold. */ - if (nr_base_pages < page_owner_pages_threshold) + if (nr_base_pages < READ_ONCE(page_owner_pages_threshold)) return 0; /* Keep show_stacks independent of stackdepot's internal storage layout. */ From 0f1de1c0bef20db17367e744662d6fb61e87f8f6 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 18 Jun 2026 15:24:31 +0100 Subject: [PATCH 104/129] KRN-1117: Clarify page owner count marker comment Make the page owner decrement comment describe the counted-handle marker rather than tying it to successful show_stacks list insertion. The marker is part of the stackdepot counted value even when the optional list node was not allocated. Signed-off-by: Caleb Kan --- mm/page_owner.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index cdda22ccb44b0..30f715f75cc77 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -255,7 +255,7 @@ static bool inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, static void dec_stack_record_count(depot_stack_handle_t handle, unsigned int nr_base_pages) { - /* Successful list insertion leaves a marker; zero means it was decremented. */ + /* Counted handles keep a marker unit; zero means it was decremented. */ if (__stack_depot_dec_count_and_test(handle, nr_base_pages)) pr_warn("%s: refcount went to 0 for %u handle\n", __func__, handle); From 6d9ee439f932c8235bf38f8fe97dc5117b81218c Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 18 Jun 2026 15:36:15 +0100 Subject: [PATCH 105/129] KRN-1117: Clarify stackdepot trie buffer contracts Document that stack_depot_fetch_into() callers should size buffers to the save-side stack depth cap when dropping diagnostics would be surprising. Also spell out why the trie side-table update budget is two leaf updates: a split can repoint the old leaf and publish one new leaf. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 4 ++++ lib/stackdepot_internal.h | 1 + 2 files changed, 5 insertions(+) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index bc3b4319ed4b6..848fef1994cf2 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -253,6 +253,10 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, * is returned. If more frames are stored than @max_entries, the copy is skipped * entirely and 0 is returned. * + * Callers should size @entries to match the save-side stack depth cap (for + * example, %CONFIG_STACKDEPOT_MAX_FRAMES or the local stack_trace_save() limit) + * when losing diagnostics on an undersized buffer would be surprising. + * * A non-zero invalid or post-put @handle is treated like stack_depot_fetch(): it * returns 0 and may WARN because such handles indicate a corrupt caller state. * diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index a1061e368149a..d91cb7bb4e211 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -72,6 +72,7 @@ struct stack_depot_trie_publish_prepare { bool retire_locked; }; +/* A split can repoint the old leaf and publish one new leaf. */ #define STACK_DEPOT_TRIE_MAX_LEAF_UPDATES 2 #define STACK_DEPOT_TRIE_MAX_NODE_SLOTS (CONFIG_STACKDEPOT_MAX_FRAMES + 1) #define STACK_DEPOT_TRIE_MAX_CHILD_SLOTS CONFIG_STACKDEPOT_MAX_FRAMES From c435880a90cc19bfdb68c1697ec4c5411ec48cb3 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 18 Jun 2026 17:13:22 +0100 Subject: [PATCH 106/129] KRN-1117: Drop test-only stackdepot pool carve helper Remove the current-pool carve helper from the production stackdepot trie API now that the save path uses the allocation transaction machinery. Rewrite the low-level pool KUnit coverage to exercise the real pool carve request path instead, keeping rollback coverage without exposing a test-only helper from lib/stackdepot.c. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 42 ------------------------------ lib/stackdepot_internal.h | 7 ----- lib/tests/stackdepot_kunit.c | 50 ++++++++++++++++++++++++------------ 3 files changed, 33 insertions(+), 66 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 13567171b062f..58b5f0cd013ba 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -1784,48 +1784,6 @@ int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, return 0; } -void * -__stack_depot_trie_pool_carve_current(size_t size, - struct stack_depot_trie_pool_mark *mark) -{ - unsigned long flags; - size_t alloc_size; - void *pool; - void *ptr = NULL; - - if (!mark) - return NULL; - memset(mark, 0, sizeof(*mark)); - - alloc_size = __stack_depot_trie_pool_alloc_size(size); - if (!alloc_size) - return NULL; - - if (!raw_spin_trylock_irqsave(&pool_lock, flags)) - return NULL; - printk_deferred_enter(); - if (!stack_pools || pools_num < 1) - goto out; - if (WARN_ON_ONCE(pool_offset > DEPOT_POOL_SIZE)) - goto out; - if (alloc_size > DEPOT_POOL_SIZE - pool_offset) - goto out; - - mark->pool_index = pools_num - 1; - pool = stack_pools[mark->pool_index]; - if (WARN_ON_ONCE(!pool)) - goto out; - - mark->offset = pool_offset; - mark->size = alloc_size; - ptr = pool + pool_offset; - pool_offset += alloc_size; -out: - printk_deferred_exit(); - raw_spin_unlock_irqrestore(&pool_lock, flags); - return ptr; -} - bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mark *mark) { unsigned long flags; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index d91cb7bb4e211..10f449ccc7f41 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -176,13 +176,6 @@ int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc); -/* - * Best-effort current-pool helpers. They never allocate or roll over to a new - * pool, and they use trylock so constrained contexts fail instead of blocking. - */ -void * -__stack_depot_trie_pool_carve_current(size_t size, - struct stack_depot_trie_pool_mark *mark); bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mark *mark); int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req); void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index e4507b3b479a9..406560553f6aa 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -1068,7 +1068,23 @@ static void stackdepot_trie_pool_seed_current_pool(struct kunit *test) KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); } -static void stackdepot_trie_pool_carve_current(struct kunit *test) +static void * +stackdepot_trie_pool_carve_node(size_t size, + struct stack_depot_trie_pool_mark *mark) +{ + struct stack_depot_trie_node_slot slot = { .size = size }; + void *storage = NULL; + struct stack_depot_trie_pool_request req = { + .node_slots = &slot, + .nr_node_slots = 1, + .storage = &storage, + .mark = mark, + }; + + return __stack_depot_trie_pool_carve(&req) ? NULL : slot.node; +} + +static void stackdepot_trie_pool_carve_node_test(struct kunit *test) { struct stack_depot_trie_pool_mark first; struct stack_depot_trie_pool_mark second; @@ -1077,13 +1093,13 @@ static void stackdepot_trie_pool_carve_current(struct kunit *test) size_t align = 1UL << DEPOT_STACK_ALIGN; stackdepot_trie_pool_seed_current_pool(test); - ptr1 = __stack_depot_trie_pool_carve_current(1, &first); + ptr1 = stackdepot_trie_pool_carve_node(1, &first); KUNIT_ASSERT_NOT_NULL(test, ptr1); KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)ptr1, align)); KUNIT_EXPECT_EQ(test, first.size, align); KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&first)); - ptr2 = __stack_depot_trie_pool_carve_current(1, &second); + ptr2 = stackdepot_trie_pool_carve_node(1, &second); KUNIT_ASSERT_NOT_NULL(test, ptr2); KUNIT_EXPECT_PTR_EQ(test, ptr2, ptr1); KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&second)); @@ -1098,9 +1114,9 @@ static void stackdepot_trie_pool_rollback_requires_lifo(struct kunit *test) void *ptr2; stackdepot_trie_pool_seed_current_pool(test); - ptr1 = __stack_depot_trie_pool_carve_current(1, &first); + ptr1 = stackdepot_trie_pool_carve_node(1, &first); KUNIT_ASSERT_NOT_NULL(test, ptr1); - ptr2 = __stack_depot_trie_pool_carve_current(1, &second); + ptr2 = stackdepot_trie_pool_carve_node(1, &second); KUNIT_ASSERT_NOT_NULL(test, ptr2); KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(&first)); @@ -1108,18 +1124,18 @@ static void stackdepot_trie_pool_rollback_requires_lifo(struct kunit *test) KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&first)); } -static void stackdepot_trie_pool_carve_current_rejects_bad_inputs(struct kunit *test) +static void stackdepot_trie_pool_carve_node_rejects_bad_inputs(struct kunit *test) { struct stack_depot_trie_pool_mark mark; void *ptr; stackdepot_trie_pool_seed_current_pool(test); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_pool_carve_current(0, &mark)); + KUNIT_EXPECT_NULL(test, stackdepot_trie_pool_carve_node(0, &mark)); KUNIT_EXPECT_EQ(test, mark.size, 0UL); - ptr = __stack_depot_trie_pool_carve_current(DEPOT_POOL_SIZE + 1, &mark); + ptr = stackdepot_trie_pool_carve_node(DEPOT_POOL_SIZE + 1, &mark); KUNIT_EXPECT_NULL(test, ptr); KUNIT_EXPECT_EQ(test, mark.size, 0UL); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_pool_carve_current(1, NULL)); + KUNIT_EXPECT_NULL(test, stackdepot_trie_pool_carve_node(1, NULL)); KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(NULL)); memset(&mark, 0, sizeof(mark)); KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(&mark)); @@ -1174,7 +1190,7 @@ static void stackdepot_trie_pool_carve_slots(struct kunit *test) KUNIT_EXPECT_GT(test, mark.size, old_total); KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&mark)); - again = __stack_depot_trie_pool_carve_current(1, &mark); + again = stackdepot_trie_pool_carve_node(1, &mark); KUNIT_ASSERT_NOT_NULL(test, again); KUNIT_EXPECT_PTR_EQ(test, again, node_slots[0].node); KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&mark)); @@ -1388,7 +1404,7 @@ static void stackdepot_trie_alloc_txn_reserve_id_failure(struct kunit *test) KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); KUNIT_EXPECT_NULL(test, node_slot.node); KUNIT_EXPECT_NULL(test, storage); - again = __stack_depot_trie_pool_carve_current(1, &mark); + again = stackdepot_trie_pool_carve_node(1, &mark); KUNIT_ASSERT_NOT_NULL(test, again); KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&mark)); } @@ -1416,7 +1432,7 @@ static void stackdepot_trie_alloc_txn_commit(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); ret = __stack_depot_trie_alloc_txn_id(&txn, &prealloc); KUNIT_ASSERT_EQ(test, ret, 0); - pool_leaf = __stack_depot_trie_pool_carve_current(1, &txn.pool); + pool_leaf = stackdepot_trie_pool_carve_node(1, &txn.pool); KUNIT_ASSERT_NOT_NULL(test, pool_leaf); updates[0].leaf_id = old_id; updates[0].leaf = pool_leaf; @@ -1437,7 +1453,7 @@ static void stackdepot_trie_alloc_txn_commit(struct kunit *test) KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 2UL); KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(old_id), pool_leaf); - pool_leaf = __stack_depot_trie_pool_carve_current(1, &txn.pool); + pool_leaf = stackdepot_trie_pool_carve_node(1, &txn.pool); KUNIT_ASSERT_NOT_NULL(test, pool_leaf); KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&txn.pool)); KUNIT_EXPECT_EQ(test, __stack_depot_trie_alloc_txn_commit(NULL), 0U); @@ -1462,7 +1478,7 @@ static void stackdepot_trie_alloc_txn_rollback(struct kunit *test) ret = __stack_depot_trie_side_table_store(old_id, old_leaf); KUNIT_ASSERT_EQ(test, ret, 0); - pool_leaf = __stack_depot_trie_pool_carve_current(1, &txn.pool); + pool_leaf = stackdepot_trie_pool_carve_node(1, &txn.pool); KUNIT_ASSERT_NOT_NULL(test, pool_leaf); updates[0].leaf_id = old_id; updates[0].leaf = pool_leaf; @@ -1483,7 +1499,7 @@ static void stackdepot_trie_alloc_txn_rollback(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(old_id), old_leaf); KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(2)); - pool_leaf = __stack_depot_trie_pool_carve_current(1, &txn.pool); + pool_leaf = stackdepot_trie_pool_carve_node(1, &txn.pool); KUNIT_ASSERT_NOT_NULL(test, pool_leaf); KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&txn.pool)); } @@ -5792,9 +5808,9 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_pool_alloc_size), KUNIT_CASE(stackdepot_trie_pool_prealloc), KUNIT_CASE(stackdepot_trie_alloc_prealloc), - KUNIT_CASE(stackdepot_trie_pool_carve_current), + KUNIT_CASE(stackdepot_trie_pool_carve_node_test), KUNIT_CASE(stackdepot_trie_pool_rollback_requires_lifo), - KUNIT_CASE(stackdepot_trie_pool_carve_current_rejects_bad_inputs), + KUNIT_CASE(stackdepot_trie_pool_carve_node_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_pool_carve_slots), KUNIT_CASE(stackdepot_trie_pool_carve_slots_rejects_bad_inputs), KUNIT_CASE(stackdepot_trie_pool_carve_uses_prealloc), From 0c5302024fd4a8390d53ab68c5527cd17a188033 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 19 Jun 2026 09:03:19 +0100 Subject: [PATCH 107/129] KRN-1117: Simplify stackdepot trie internals Remove production-visible KUnit-only wrappers and duplicate trie unit-test scaffolding that made the stackdepot port harder to review. Keep the tests focused on the real save, insert, fetch, and publication paths while preserving coverage for the RCU-safe trie operations. Also serialize runtime trie flag updates, bound sorted-pool retry loops, and cap trie stack printing lengths so the cleanup does not weaken the runtime safety checks. Signed-off-by: Caleb Kan --- arch/arm64/include/asm/stackdepot.h | 3 + lib/stackdepot.c | 461 +++++----------------------- lib/stackdepot_internal.h | 24 -- lib/tests/Makefile | 1 + lib/tests/stackdepot_kunit.c | 248 ++------------- 5 files changed, 106 insertions(+), 631 deletions(-) diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h index 079dcaf9154a6..a87955d49515b 100644 --- a/arch/arm64/include/asm/stackdepot.h +++ b/arch/arm64/include/asm/stackdepot.h @@ -16,6 +16,9 @@ * each frame. If that window crosses a 4 GB high-bit boundary, module text * may have the previous or next prefix even though it is still within * relocation range of _text. + * + * Prefix IDs are arch-local metadata; trie storage is per boot and is never + * interpreted by another architecture's decompressor. */ #define STACK_DEPOT_ARM64_PREV_PREFIX_ID 0 #define STACK_DEPOT_ARM64_TEXT_PREFIX_ID 1 diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 58b5f0cd013ba..6fe238daaa6df 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -60,6 +60,7 @@ static bool __stack_depot_early_init_requested __initdata = static bool __stack_depot_early_init_passed __initdata; static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); static bool stack_depot_trie_enabled_param; +static DEFINE_MUTEX(stack_depot_trie_param_lock); static struct stack_depot_trie_root stack_depot_trie_root; static struct stack_depot_trie_alloc_workspace *stack_depot_trie_workspace; static DEFINE_RAW_SPINLOCK(stack_depot_trie_workspace_lock); @@ -94,15 +95,18 @@ static int stack_depot_trie_enabled_param_set(const char *val, if (ret) return ret; + /* Keep runtime toggles serialized outside stack_depot_init_mutex. */ + mutex_lock(&stack_depot_trie_param_lock); __stack_depot_trie_set_enabled(enabled); if (enabled && system_state >= SYSTEM_RUNNING) { ret = stack_depot_init(); if (ret || !__stack_depot_trie_ready()) { __stack_depot_trie_set_enabled(false); - return ret ?: -ENOMEM; + ret = ret ?: -ENOMEM; } } - return 0; + mutex_unlock(&stack_depot_trie_param_lock); + return ret; } static int stack_depot_trie_enabled_param_get(char *buffer, @@ -132,6 +136,7 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage"); #define STACK_BUCKET_NUMBER_ORDER_MAX 20 /* Initial seed for jhash2. */ #define STACK_HASH_SEED 0x9747b28c +#define STACK_POOLS_SORTED_RETRIES 8 /* Compact structure that stores a reference to a stack. */ union handle_parts { @@ -455,6 +460,7 @@ static int stack_depot_trie_init_sorted_pools(gfp_t gfp_flags) unsigned long flags; unsigned int capacity; unsigned int pools; + unsigned int retry; void **sorted; if (READ_ONCE(stack_pools_sorted)) @@ -466,7 +472,7 @@ static int stack_depot_trie_init_sorted_pools(gfp_t gfp_flags) if (!sorted) return -ENOMEM; - while (sorted) { + for (retry = 0; sorted && retry < STACK_POOLS_SORTED_RETRIES; retry++) { raw_spin_lock_irqsave(&pool_lock, flags); if (stack_pools_sorted) { raw_spin_unlock_irqrestore(&pool_lock, flags); @@ -502,6 +508,7 @@ static void stack_pools_sorted_grow(gfp_t gfp_flags) unsigned int capacity; unsigned long flags; unsigned int pools; + unsigned int retry; void **old; bool old_memblock = false; void **sorted; @@ -517,7 +524,7 @@ static void stack_pools_sorted_grow(gfp_t gfp_flags) if (!sorted) return; - for (;;) { + for (retry = 0; retry < STACK_POOLS_SORTED_RETRIES; retry++) { raw_spin_lock_irqsave(&pool_lock, flags); old = stack_pools_sorted; if (capacity <= stack_pools_sorted_capacity) { @@ -873,6 +880,10 @@ static int stack_depot_trie_init(gfp_t gfp_flags) if (ret) return ret; + /* Runtime enable is not a live migration from hash to trie storage. */ + if (system_state >= SYSTEM_RUNNING) + WRITE_ONCE(stack_depot_persistent_hash_record_seen, true); + stack_depot_trie_publish_ready(); return 0; } @@ -935,6 +946,7 @@ void __stack_depot_trie_side_table_destroy(void) if (!trie_side_table_is_initialized()) return; + /* KUnit/teardown helper only; callers must exclude readers and writers. */ stack_depot_trie_mark_not_ready(); /* Pairs with trie_side_table_is_initialized(). */ smp_store_release(&trie_side_table_initialized, false); @@ -1080,18 +1092,6 @@ __stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *pr return id; } -static u32 trie_side_table_alloc_id_trylock(void) -{ - unsigned long flags; - u32 id; - - if (!raw_spin_trylock_irqsave(&trie_side_table_lock, flags)) - return 0; - id = trie_side_table_alloc_id_locked(NULL); - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return id; -} - void __stack_depot_trie_side_table_revoke_latest(u32 id) { struct stack_depot_trie_side_entry *chunk; @@ -1125,43 +1125,6 @@ void __stack_depot_trie_side_table_revoke_latest(u32 id) raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); } -static bool trie_side_table_revoke_latest_trylock(u32 id) -{ - struct stack_depot_trie_side_entry *chunk; - struct stack_depot_trie_side_dir *dir; - unsigned long flags; - unsigned int slot; - unsigned int root; - bool ret = false; - - if (!trie_side_table_is_initialized() || !id || - id != READ_ONCE(trie_side_table_next_id)) - return false; - - if (!raw_spin_trylock_irqsave(&trie_side_table_lock, flags)) - return false; - if (id != trie_side_table_next_id) - goto out; - root = trie_side_table_root_index(id); - if (root >= trie_side_table_root_size) - goto out; - - dir = trie_side_table_load_dir(root); - if (!dir) - goto out; - chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); - if (!chunk) - goto out; - - slot = trie_side_table_slot_index(id); - trie_side_table_clear_entry(chunk, slot); - WRITE_ONCE(trie_side_table_next_id, id - 1); - ret = true; -out: - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return ret; -} - void __stack_depot_trie_side_table_restore(u32 id, const void *entry) { struct stack_depot_trie_side_entry *chunk; @@ -1784,26 +1747,25 @@ int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, return 0; } -bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mark *mark) +static bool stack_depot_trie_pool_rollback_locked(const struct stack_depot_trie_pool_mark *mark) { - unsigned long flags; size_t end; bool ret = false; + lockdep_assert_held(&pool_lock); + if (!mark || !mark->size) return false; if (check_add_overflow(mark->offset, mark->size, &end)) return false; - if (!raw_spin_trylock_irqsave(&pool_lock, flags)) - return false; if (mark->pool_index != pools_num - 1 || pool_offset != end) - goto out; + return false; if (mark->added_pool) { if (mark->offset || stack_pools[mark->pool_index] != mark->pool) - goto out; + return false; if (new_pool && new_pool != STACK_DEPOT_POISON) - goto out; + return false; depot_forget_pool_locked(mark->pool); stack_pools[mark->pool_index] = NULL; WRITE_ONCE(pools_num, mark->pool_index); @@ -1814,7 +1776,30 @@ bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mar pool_offset = mark->offset; ret = true; } -out: + + return ret; +} + +static bool stack_depot_trie_pool_rollback(const struct stack_depot_trie_pool_mark *mark) +{ + unsigned long flags; + bool ret; + + raw_spin_lock_irqsave(&pool_lock, flags); + ret = stack_depot_trie_pool_rollback_locked(mark); + raw_spin_unlock_irqrestore(&pool_lock, flags); + + return ret; +} + +bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mark *mark) +{ + unsigned long flags; + bool ret; + + if (!raw_spin_trylock_irqsave(&pool_lock, flags)) + return false; + ret = stack_depot_trie_pool_rollback_locked(mark); raw_spin_unlock_irqrestore(&pool_lock, flags); return ret; @@ -1889,16 +1874,6 @@ static void trie_pool_release_reused_objects(struct stack_depot_trie_pool_reques raw_spin_unlock_irqrestore(&pool_lock, flags); } -static void trie_pool_release_reused_objects_trylock(struct stack_depot_trie_pool_request *req) -{ - unsigned long flags; - - if (!raw_spin_trylock_irqsave(&pool_lock, flags)) - return; - trie_pool_release_reused_objects_locked(req); - raw_spin_unlock_irqrestore(&pool_lock, flags); -} - int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) { unsigned long flags; @@ -1929,8 +1904,7 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) if (req->storage_size && !trie_object_alloc_size(req->storage_size)) return -EINVAL; - if (!raw_spin_trylock_irqsave(&pool_lock, flags)) - return -EBUSY; + raw_spin_lock_irqsave(&pool_lock, flags); printk_deferred_enter(); if (!stack_pools) { ret = -ENOSPC; @@ -2036,21 +2010,6 @@ __stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, return 0; } -static int trie_alloc_txn_id_trylock(struct stack_depot_trie_alloc_txn *txn) -{ - u32 leaf_id; - - if (!txn || txn->leaf_id) - return -EINVAL; - - leaf_id = trie_side_table_alloc_id_trylock(); - if (!leaf_id) - return -ENOSPC; - - txn->leaf_id = leaf_id; - return 0; -} - static void trie_alloc_request_clear_outputs(struct stack_depot_trie_alloc_request *req) { unsigned int i; @@ -2115,69 +2074,6 @@ int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request * return 0; } -static void -trie_alloc_request_release_reused_objects_trylock(struct stack_depot_trie_alloc_request *req) -{ - struct stack_depot_trie_pool_request pool_req = {}; - - if (!req || !req->txn) - return; - pool_req.node_slots = req->node_slots; - pool_req.nr_node_slots = req->nr_node_slots; - pool_req.child_slots = req->child_slots; - pool_req.nr_child_slots = req->nr_child_slots; - pool_req.storage = req->storage; - pool_req.mark = &req->txn->pool; - trie_pool_release_reused_objects_trylock(&pool_req); -} - -static void trie_alloc_txn_rollback_trylock(struct stack_depot_trie_alloc_txn *txn) -{ - if (!txn) - return; - - if (txn->side.nr_updates) - return; - if (txn->leaf_id && trie_side_table_revoke_latest_trylock(txn->leaf_id)) - txn->leaf_id = 0; - __stack_depot_trie_pool_try_rollback(&txn->pool); - memset(&txn->pool, 0, sizeof(txn->pool)); -} - -static int trie_alloc_txn_reserve_trylock(struct stack_depot_trie_alloc_request *req) -{ - struct stack_depot_trie_pool_request pool_req = {}; - int ret; - - if (!req || !req->txn) - return -EINVAL; - if (req->txn->leaf_id || req->txn->pool.size || req->txn->side.nr_updates) - return -EINVAL; - - pool_req.node_slots = req->node_slots; - pool_req.nr_node_slots = req->nr_node_slots; - pool_req.child_slots = req->child_slots; - pool_req.nr_child_slots = req->nr_child_slots; - pool_req.storage = req->storage; - pool_req.storage_size = req->storage_size; - pool_req.prealloc = req->pool_prealloc; - pool_req.mark = &req->txn->pool; - - ret = __stack_depot_trie_pool_carve(&pool_req); - if (ret) - return ret; - - ret = trie_alloc_txn_id_trylock(req->txn); - if (ret) { - trie_alloc_request_release_reused_objects_trylock(req); - trie_alloc_txn_rollback_trylock(req->txn); - trie_alloc_request_clear_outputs(req); - return ret; - } - - return 0; -} - int __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, const unsigned long *entries, @@ -2199,10 +2095,12 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, if (!root || !txn || !storage || !req) return -EINVAL; + rcu_read_lock_sched_notrace(); ret = __stack_depot_trie_insert_plan(root, NULL, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, &storage_size, &nr_used, &nr_child_used); + rcu_read_unlock_sched_notrace(); if (ret) return ret; @@ -2240,16 +2138,6 @@ trie_ws_plan(const struct stack_depot_trie_root *root, side_prealloc, &workspace->req); } -int __stack_depot_trie_workspace_plan(const struct stack_depot_trie_root *root, - const unsigned long *entries, - unsigned int nr_entries, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace) -{ - return trie_ws_plan(root, entries, nr_entries, pool_prealloc, side_prealloc, - workspace); -} - int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, @@ -2269,99 +2157,6 @@ int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, tail, leaf_id); } -static int -trie_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) -{ - return __stack_depot_trie_alloc_prealloc(alloc_flags, depot_flags, - pool_prealloc, side_prealloc); -} - -static int trie_ws_insert(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace, - const void **tail, u32 *leaf_id) -{ - return __stack_depot_trie_workspace_insert(root, entries, nr_entries, - pool_prealloc, side_prealloc, - workspace, tail, leaf_id); -} - -static int trie_side_prepare_trylock(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, void *ctx); - -static int trie_ws_insert_trylock(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_alloc_workspace *workspace, - const void **tail, u32 *leaf_id) -{ - struct stack_depot_trie_alloc_request *req = &workspace->req; - struct stack_depot_trie_publish_prepare prepare; - struct stack_depot_trie_side_prealloc side_prealloc = {}; - void *pool_prealloc = NULL; - unsigned long flags; - unsigned long retire_flags; - unsigned int nr_used; - bool retire_locked = false; - u32 id; - int ret; - - if (!raw_spin_trylock_irqsave(&trie_alloc_lock, flags)) - return -EBUSY; - - ret = trie_ws_plan(root, entries, nr_entries, &pool_prealloc, - &side_prealloc, workspace); - if (ret) - goto out_unlock; - if (pool_prealloc || side_prealloc.dir || side_prealloc.chunk) { - ret = -EINVAL; - goto out_unlock; - } - - ret = trie_alloc_txn_reserve_trylock(req); - if (ret) - goto out_unlock; - if (!raw_spin_trylock_irqsave(&pool_lock, retire_flags)) { - ret = -EBUSY; - goto rollback; - } - retire_locked = true; - - prepare.fn = trie_side_prepare_trylock; - prepare.ctx = &workspace->txn.side; - prepare.retire_locked = true; - id = workspace->txn.leaf_id; - ret = __stack_depot_trie_insert_append_prepare(root, NULL, id, entries, - nr_entries, workspace->node_slots, - req->nr_node_slots, workspace->child_slots, - req->nr_child_slots, workspace->scratch, - ARRAY_SIZE(workspace->scratch), - workspace->storage, req->storage_size, - &prepare, tail, &nr_used); - raw_spin_unlock_irqrestore(&pool_lock, retire_flags); - retire_locked = false; - if (ret) - goto rollback; - - *leaf_id = __stack_depot_trie_alloc_txn_commit(&workspace->txn); - ret = 0; - goto out_unlock; - -rollback: - if (retire_locked) - raw_spin_unlock_irqrestore(&pool_lock, retire_flags); - trie_alloc_request_release_reused_objects_trylock(req); - trie_alloc_txn_rollback_trylock(&workspace->txn); - trie_alloc_request_clear_outputs(req); - *tail = NULL; -out_unlock: - raw_spin_unlock_irqrestore(&trie_alloc_lock, flags); - return ret; -} - static depot_stack_handle_t trie_find_handle(const struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries) @@ -2378,77 +2173,6 @@ trie_find_handle(const struct stack_depot_trie_root *root, return handle; } -static depot_stack_handle_t -trie_save_miss(struct stack_depot_trie_root *root, const unsigned long *entries, - unsigned int nr_entries, gfp_t alloc_flags, - depot_flags_t depot_flags, - struct stack_depot_trie_alloc_workspace *workspace) -{ - depot_stack_handle_t handle = 0; - void *pool_prealloc = NULL; - struct stack_depot_trie_side_prealloc side_prealloc = {}; - const void *tail; - u32 leaf_id; - int ret; - - if (!root || !entries || !nr_entries || !workspace) - return 0; - if (depot_flags & STACK_DEPOT_FLAG_GET) - return 0; - if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) - return 0; - if (in_nmi() || !gfpflags_allow_spinning(alloc_flags)) - return 0; - - ret = trie_prealloc(alloc_flags, depot_flags, &pool_prealloc, - &side_prealloc); - if (ret) - goto out; - - ret = trie_ws_insert(root, entries, nr_entries, &pool_prealloc, - &side_prealloc, workspace, &tail, &leaf_id); - if (ret) - goto out; - - handle = __stack_depot_trie_handle(leaf_id); -out: - __stack_depot_trie_pool_free_prealloc(pool_prealloc); - __stack_depot_trie_side_table_free_prealloc(&side_prealloc); - return handle; -} - -depot_stack_handle_t -__stack_depot_trie_save_miss(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - gfp_t alloc_flags, depot_flags_t depot_flags, - struct stack_depot_trie_alloc_workspace *workspace) -{ - return trie_save_miss(root, entries, nr_entries, alloc_flags, depot_flags, - workspace); -} - -depot_stack_handle_t -__stack_depot_trie_save(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - gfp_t alloc_flags, depot_flags_t depot_flags, - struct stack_depot_trie_alloc_workspace *workspace) -{ - depot_stack_handle_t handle = 0; - - if (!root || !entries || !nr_entries || !workspace) - return 0; - if (depot_flags & STACK_DEPOT_FLAG_GET) - return 0; - if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) - return 0; - - handle = trie_find_handle(root, entries, nr_entries); - if (handle) - return handle; - return trie_save_miss(root, entries, nr_entries, alloc_flags, depot_flags, - workspace); -} - static depot_stack_handle_t trie_save_locked_insert(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, @@ -2464,8 +2188,9 @@ trie_save_locked_insert(struct stack_depot_trie_root *root, handle = trie_find_handle(root, entries, nr_entries); if (!handle && can_insert) { - ret = trie_ws_insert(root, entries, nr_entries, pool_prealloc, - side_prealloc, workspace, &tail, &leaf_id); + ret = __stack_depot_trie_workspace_insert(root, entries, nr_entries, + pool_prealloc, side_prealloc, + workspace, &tail, &leaf_id); if (!ret) handle = __stack_depot_trie_handle(leaf_id); } @@ -2500,18 +2225,10 @@ trie_save_trylocked(struct stack_depot_trie_root *root, { depot_stack_handle_t handle = 0; unsigned long flags; - const void *tail; - u32 leaf_id; if (!raw_spin_trylock_irqsave(workspace_lock, flags)) return 0; handle = trie_find_handle(root, entries, nr_entries); - if (handle) - goto out; - if (!trie_ws_insert_trylock(root, entries, nr_entries, workspace, &tail, - &leaf_id)) - handle = __stack_depot_trie_handle(leaf_id); -out: raw_spin_unlock_irqrestore(workspace_lock, flags); return handle; } @@ -2545,8 +2262,9 @@ __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, return trie_save_trylocked(root, entries, nr_entries, workspace, workspace_lock); - ret = trie_prealloc(alloc_flags, depot_flags, &pool_prealloc, - &side_prealloc); + ret = __stack_depot_trie_alloc_prealloc(alloc_flags, depot_flags, + &pool_prealloc, + &side_prealloc); can_insert = !ret; handle = trie_save_spinlocked(root, entries, nr_entries, workspace, workspace_lock, &pool_prealloc, @@ -2592,8 +2310,7 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, *tail = NULL; *leaf_id = 0; - if (!raw_spin_trylock_irqsave(&trie_alloc_lock, flags)) - return -EBUSY; + raw_spin_lock_irqsave(&trie_alloc_lock, flags); ret = __stack_depot_trie_alloc_txn_reserve(req); if (ret) @@ -2637,7 +2354,7 @@ void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *tx __stack_depot_trie_side_table_revoke_latest(txn->leaf_id); txn->leaf_id = 0; } - __stack_depot_trie_pool_try_rollback(&txn->pool); + stack_depot_trie_pool_rollback(&txn->pool); memset(&txn->pool, 0, sizeof(txn->pool)); } @@ -2711,20 +2428,6 @@ __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updat return ret; } -static int -trie_side_prepare_trylock(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, void *ctx) -{ - unsigned long flags; - int ret; - - if (!raw_spin_trylock_irqsave(&trie_side_table_lock, flags)) - return -EBUSY; - ret = trie_side_prepare_locked(updates, nr_updates, ctx); - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return ret; -} - static int __init disable_stack_depot(char *str) { return kstrtobool(str, &stack_depot_disabled); @@ -3323,6 +3026,8 @@ depot_save_stack_locked(struct stack_depot_hash_save *save) save->hash, save->depot_flags); if (found) return found->handle.handle; + if (save->normal_persistent && __stack_depot_trie_ready()) + return 0; new = depot_alloc_stack(save->entries, save->nr_entries, save->hash, save->depot_flags, save->prealloc); @@ -3466,6 +3171,9 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, /* Stack depot didn't use this memory, free it. */ free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); } + if (!handle && normal_persistent && __stack_depot_trie_ready()) + return stack_depot_trie_save(entries, nr_entries, alloc_flags, + depot_flags); return handle; } EXPORT_SYMBOL_GPL(stack_depot_save_flags); @@ -3617,12 +3325,6 @@ static bool frame_try_compress(unsigned long frame, u8 *prefix_id, u32 *low) return arch_stack_depot_frame_try_compress(frame, prefix_id, low); } -bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, - u32 *low) -{ - return frame_try_compress(frame, prefix_id, low); -} - static bool frame_decompress(u8 prefix_id, u32 low, unsigned long *frame) { if (!frame) @@ -3631,12 +3333,6 @@ static bool frame_decompress(u8 prefix_id, u32 low, unsigned long *frame) return arch_stack_depot_frame_decompress(prefix_id, low, frame); } -bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, - unsigned long *frame) -{ - return frame_decompress(prefix_id, low, frame); -} - static bool stack_depot_ranges_overlap(const void *a, size_t a_size, const void *b, size_t b_size); static int @@ -5580,7 +5276,10 @@ static unsigned int trie_validate_leaf(const void *leaf, } static unsigned int trie_walk_frames(const void *leaf, unsigned int total, - trie_frame_fn_t fn, void *data) + void (*fn)(unsigned int index, + unsigned long frame, + void *data), + void *data) { const struct stack_depot_trie_node *node; unsigned int seen = 0; @@ -5679,8 +5378,8 @@ static int trie_snprint_frames(char *buf, size_t size, const void *leaf, unsigned int nr_entries, int spaces) { - unsigned int generated; - unsigned int total = 0; + int generated; + int total = 0; unsigned int i; for (i = 0; i < nr_entries && size; i++) { @@ -5690,6 +5389,10 @@ trie_snprint_frames(char *buf, size_t size, const void *leaf, break; generated = snprintf(buf, size, "%*c%pS\n", 1 + spaces, ' ', (void *)frame); + if (generated < 0) + break; + if (generated > INT_MAX - total) + return INT_MAX; total += generated; if (generated >= size) { buf += size; @@ -5728,18 +5431,6 @@ static void trie_fetch_frame(unsigned int index, unsigned long frame, void *data ctx->nr_entries++; } -unsigned int -__stack_depot_trie_walk_frames(const void *leaf, trie_frame_fn_t fn, void *data) -{ - unsigned int total; - - total = trie_validate_leaf(leaf, NULL); - if (!total) - return 0; - - return trie_walk_frames(leaf, total, fn, data); -} - unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries) @@ -5764,12 +5455,6 @@ __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, return total; } -static unsigned int trie_fetch_leaf(const void *leaf, unsigned long *entries, - unsigned int max_entries) -{ - return __stack_depot_trie_fetch_into(leaf, entries, max_entries); -} - unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, unsigned long *entries, @@ -5792,7 +5477,7 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, rcu_read_unlock_sched_notrace(); return 0; } - nr_entries = trie_fetch_leaf(leaf, entries, max_entries); + nr_entries = __stack_depot_trie_fetch_into(leaf, entries, max_entries); rcu_read_unlock_sched_notrace(); return nr_entries; diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 10f449ccc7f41..7ab55cbb3ec61 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -62,9 +62,6 @@ struct stack_depot_trie_leaf_update { const void *leaf; }; -typedef void (*trie_frame_fn_t)(unsigned int index, unsigned long frame, - void *data); - struct stack_depot_trie_publish_prepare { int (*fn)(const struct stack_depot_trie_leaf_update *updates, unsigned int nr_updates, void *ctx); @@ -194,11 +191,6 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, void **storage, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_request *req); -int __stack_depot_trie_workspace_plan(const struct stack_depot_trie_root *root, - const unsigned long *entries, - unsigned int nr_entries, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace); int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, void **pool_prealloc, @@ -206,16 +198,6 @@ int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, struct stack_depot_trie_alloc_workspace *workspace, const void **tail, u32 *leaf_id); depot_stack_handle_t -__stack_depot_trie_save_miss(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - gfp_t alloc_flags, depot_flags_t depot_flags, - struct stack_depot_trie_alloc_workspace *workspace); -depot_stack_handle_t -__stack_depot_trie_save(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - gfp_t alloc_flags, depot_flags_t depot_flags, - struct stack_depot_trie_alloc_workspace *workspace); -depot_stack_handle_t __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags, depot_flags_t depot_flags, @@ -236,10 +218,6 @@ int __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, unsigned int nr_updates, void *ctx); void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *state); -bool __stack_depot_frame_try_compress(unsigned long frame, u8 *prefix_id, - u32 *low); -bool __stack_depot_frame_decompress(u8 prefix_id, u32 low, - unsigned long *frame); int __stack_depot_frame_run_init(const unsigned long *entries, unsigned int nr_entries, struct stack_depot_frame_run *run); @@ -321,8 +299,6 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries); -unsigned int -__stack_depot_trie_walk_frames(const void *leaf, trie_frame_fn_t fn, void *data); unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, unsigned long *entries, unsigned int max_entries); diff --git a/lib/tests/Makefile b/lib/tests/Makefile index 1d0954575ad5d..f4cd8797469fc 100644 --- a/lib/tests/Makefile +++ b/lib/tests/Makefile @@ -40,6 +40,7 @@ obj-$(CONFIG_SCANF_KUNIT_TEST) += scanf_kunit.o obj-$(CONFIG_SEQ_BUF_KUNIT_TEST) += seq_buf_kunit.o obj-$(CONFIG_SIPHASH_KUNIT_TEST) += siphash_kunit.o obj-$(CONFIG_SLUB_KUNIT_TEST) += slub_kunit.o +# STACKDEPOT_KUNIT_TEST uses non-exported helpers, so Kconfig forces built-in. obj-$(CONFIG_STACKDEPOT_KUNIT_TEST) += stackdepot_kunit.o obj-$(CONFIG_TEST_SORT) += test_sort.o CFLAGS_stackinit_kunit.o += $(call cc-disable-warning, switch-unreachable) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 406560553f6aa..1f8dbf3bdf9fc 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -16,9 +16,7 @@ #include "../stackdepot_internal.h" -#ifdef CONFIG_ARM64 #include -#endif static int frame_run_init(const unsigned long *entries, unsigned int nr_entries, @@ -1531,16 +1529,6 @@ static int txn_insert(struct stack_depot_trie_root *root, NULL, 0, tail, leaf_id); } -static int workspace_plan(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace) -{ - return __stack_depot_trie_workspace_plan(root, entries, nr_entries, - pool_prealloc, side_prealloc, workspace); -} - static int ws_insert_prealloc(struct stack_depot_trie_root *root, struct stack_depot_trie_alloc_workspace *workspace, const unsigned long *entries, unsigned int nr_entries, @@ -1551,25 +1539,6 @@ static int ws_insert_prealloc(struct stack_depot_trie_root *root, side_prealloc, workspace, tail, leaf_id); } -static depot_stack_handle_t save_miss(struct stack_depot_trie_root *root, - const unsigned long *entries, - unsigned int nr_entries, gfp_t gfp_flags, - depot_flags_t depot_flags, - struct stack_depot_trie_alloc_workspace *workspace) -{ - return __stack_depot_trie_save_miss(root, entries, nr_entries, gfp_flags, - depot_flags, workspace); -} - -static depot_stack_handle_t tsave(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - gfp_t gfp_flags, depot_flags_t depot_flags, - struct stack_depot_trie_alloc_workspace *workspace) -{ - return __stack_depot_trie_save(root, entries, nr_entries, gfp_flags, - depot_flags, workspace); -} - static depot_stack_handle_t tsave_locked(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, gfp_t gfp_flags, depot_flags_t depot_flags, @@ -1586,72 +1555,6 @@ static unsigned int tfetch_handle(depot_stack_handle_t handle, return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); } -struct trie_frame_iter_ctx { - unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; - unsigned int nr_entries; -}; - -static void trie_frame_iter_record(unsigned int index, unsigned long frame, - void *data) -{ - struct trie_frame_iter_ctx *ctx = data; - - ctx->entries[index] = frame; - ctx->nr_entries++; -} - -static unsigned int twalk_frames(const void *leaf, struct trie_frame_iter_ctx *ctx) -{ - return __stack_depot_trie_walk_frames(leaf, trie_frame_iter_record, ctx); -} - -static void stackdepot_trie_alloc_workspace_plan(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_alloc_workspace *workspace; - struct stack_depot_trie_side_prealloc side_prealloc = { - .chunk = (void *)0x2222UL, - }; - struct stack_depot_trie_root root = {}; - void *pool_prealloc = (void *)0x1111UL; - const void *tail = NULL; - u32 leaf_id = 0; - int ret; - - workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, workspace); - - ret = workspace_plan(&root, entries, ARRAY_SIZE(entries), &pool_prealloc, - &side_prealloc, workspace); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, workspace->req.txn, &workspace->txn); - KUNIT_EXPECT_PTR_EQ(test, workspace->req.node_slots, - &workspace->node_slots[0]); - KUNIT_EXPECT_PTR_EQ(test, workspace->req.child_slots, - &workspace->child_slots[0]); - KUNIT_EXPECT_PTR_EQ(test, workspace->req.storage, &workspace->storage); - KUNIT_EXPECT_PTR_EQ(test, workspace->req.pool_prealloc, &pool_prealloc); - KUNIT_EXPECT_PTR_EQ(test, workspace->req.side_prealloc, &side_prealloc); - KUNIT_EXPECT_NE(test, workspace->req.storage_size, 0UL); - KUNIT_EXPECT_EQ(test, workspace->req.nr_node_slots, 1U); - KUNIT_EXPECT_EQ(test, workspace->req.nr_child_slots, 0U); - - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - side_prealloc = (struct stack_depot_trie_side_prealloc) {}; - if (__stack_depot_trie_side_table_prealloc_needed()) - stackdepot_trie_side_table_prealloc_or_fail(test, &side_prealloc); - ret = workspace_plan(&root, entries, ARRAY_SIZE(entries), NULL, - &side_prealloc, workspace); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = ws_insert_prealloc(&root, workspace, entries, ARRAY_SIZE(entries), - &side_prealloc, &tail, &leaf_id); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, leaf_id, 1U); - KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), - tail); -} - static void stackdepot_trie_alloc_workspace_insert(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; @@ -1688,92 +1591,6 @@ static void stackdepot_trie_alloc_workspace_insert(struct kunit *test) KUNIT_EXPECT_EQ(test, ret, -EINVAL); } -static void stackdepot_trie_save_miss(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_alloc_workspace *workspace; - struct stack_depot_trie_root root = {}; - unsigned long out[ARRAY_SIZE(entries)] = {}; - depot_stack_handle_t handle; - const void *tail; - unsigned int fetched; - u32 leaf_id; - - workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, workspace); - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - - handle = save_miss(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_CAN_ALLOC, workspace); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - leaf_id = __stack_depot_trie_leaf_id(handle); - KUNIT_EXPECT_EQ(test, leaf_id, 1U); - tail = __stack_depot_trie_side_table_lookup(leaf_id); - KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), - tail); - fetched = tfetch(tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); - - handle = save_miss(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_GET, workspace); - KUNIT_EXPECT_EQ(test, handle, (depot_stack_handle_t)0); - handle = save_miss(NULL, entries, ARRAY_SIZE(entries), GFP_KERNEL, 0, - workspace); - KUNIT_EXPECT_EQ(test, handle, (depot_stack_handle_t)0); -} - -static void stackdepot_trie_save_miss_noalloc(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_alloc_workspace *workspace; - struct stack_depot_trie_root root = {}; - depot_stack_handle_t handle; - - workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, workspace); - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - - handle = save_miss(&root, entries, ARRAY_SIZE(entries), GFP_NOWAIT, 0, - workspace); - KUNIT_EXPECT_EQ(test, handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_NULL(test, find_leaf(&root, entries, ARRAY_SIZE(entries))); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); -} - -static void stackdepot_trie_save(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_alloc_workspace *workspace; - struct stack_depot_trie_root root = {}; - depot_stack_handle_t first; - depot_stack_handle_t invalid; - depot_stack_handle_t second; - - workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, workspace); - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - - first = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_CAN_ALLOC, workspace); - KUNIT_ASSERT_NE(test, first, (depot_stack_handle_t)0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); - - second = tsave(&root, entries, ARRAY_SIZE(entries), GFP_NOWAIT, 0, - workspace); - KUNIT_EXPECT_EQ(test, second, first); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); - - invalid = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_GET, workspace); - KUNIT_EXPECT_EQ(test, invalid, (depot_stack_handle_t)0); - invalid = tsave(NULL, entries, ARRAY_SIZE(entries), GFP_KERNEL, 0, workspace); - KUNIT_EXPECT_EQ(test, invalid, (depot_stack_handle_t)0); -} - static void stackdepot_trie_save_locked(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; @@ -1817,8 +1634,8 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) { unsigned long entries[] = { 0x1000UL, 0x2000UL }; struct stack_depot_trie_alloc_workspace *workspace; - struct trie_frame_iter_ctx *iter; struct stack_depot_trie_root root = {}; + raw_spinlock_t workspace_lock; unsigned long small[1] = { 0xdeadUL }; unsigned long out[ARRAY_SIZE(entries)] = {}; depot_stack_handle_t hash_handle; @@ -1830,26 +1647,18 @@ static void stackdepot_trie_fetch_handle_into(struct kunit *test) workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, workspace); - iter = kunit_kzalloc(test, sizeof(*iter), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, iter); + raw_spin_lock_init(&workspace_lock); stackdepot_trie_side_table_init_or_skip(test); stackdepot_trie_pool_seed_current_pool(test); - handle = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_CAN_ALLOC, workspace); + handle = tsave_locked(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC, workspace, + &workspace_lock); KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); leaf_id = __stack_depot_trie_leaf_id(handle); KUNIT_ASSERT_NE(test, leaf_id, 0U); leaf = __stack_depot_trie_side_table_lookup(leaf_id); KUNIT_ASSERT_NOT_NULL(test, leaf); - fetched = twalk_frames(leaf, iter); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_EQ(test, iter->nr_entries, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, iter->entries, entries, sizeof(entries)); - iter->nr_entries = 0; - fetched = twalk_frames(NULL, iter); - KUNIT_EXPECT_EQ(test, fetched, 0U); - KUNIT_EXPECT_EQ(test, iter->nr_entries, 0U); fetched = tfetch_handle(handle, out, ARRAY_SIZE(out)); KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); @@ -1884,6 +1693,7 @@ static void stackdepot_trie_snprint_public(struct kunit *test) char expected[256]; char actual[256]; struct stack_depot_trie_root root = {}; + raw_spinlock_t workspace_lock; depot_stack_handle_t extra; depot_stack_handle_t handle; unsigned int expected_len; @@ -1891,11 +1701,13 @@ static void stackdepot_trie_snprint_public(struct kunit *test) workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, workspace); + raw_spin_lock_init(&workspace_lock); stackdepot_trie_side_table_init_or_skip(test); stackdepot_trie_pool_seed_current_pool(test); - handle = tsave(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_CAN_ALLOC, workspace); + handle = tsave_locked(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC, workspace, + &workspace_lock); KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); expected_len = stack_trace_snprint(expected, sizeof(expected), entries, @@ -2078,16 +1890,16 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) /* Arch hooks may exist, but this frame is chosen to stay raw. */ KUNIT_EXPECT_FALSE(test, - __stack_depot_frame_try_compress(frame, &prefix_id, &low)); + arch_stack_depot_frame_try_compress(frame, &prefix_id, &low)); KUNIT_EXPECT_EQ(test, prefix_id, (u8)0xaa); KUNIT_EXPECT_EQ(test, low, (u32)0xfeedbeef); KUNIT_EXPECT_FALSE(test, - __stack_depot_frame_decompress(0xff, 0x81234567, &out)); + arch_stack_depot_frame_decompress(0xff, 0x81234567, &out)); KUNIT_EXPECT_EQ(test, out, 0x12345678UL); KUNIT_EXPECT_FALSE(test, - __stack_depot_frame_decompress(0, 0x81234567, NULL)); + arch_stack_depot_frame_decompress(0, 0x81234567, NULL)); } #ifdef CONFIG_X86_64 @@ -2101,17 +1913,17 @@ static void stackdepot_frame_x86_64(struct kunit *test) u8 prefix_id; KUNIT_EXPECT_TRUE(test, - __stack_depot_frame_try_compress(frame, &prefix_id, &low)); + arch_stack_depot_frame_try_compress(frame, &prefix_id, &low)); KUNIT_EXPECT_EQ(test, prefix_id, (u8)0); KUNIT_EXPECT_EQ(test, low, (u32)0x81234567); KUNIT_EXPECT_TRUE(test, - __stack_depot_frame_decompress(prefix_id, low, &out)); + arch_stack_depot_frame_decompress(prefix_id, low, &out)); KUNIT_EXPECT_EQ(test, out, frame); - compressed = __stack_depot_frame_try_compress(direct_map, &prefix_id, &low); + compressed = arch_stack_depot_frame_try_compress(direct_map, &prefix_id, &low); KUNIT_EXPECT_FALSE(test, compressed); KUNIT_EXPECT_FALSE(test, - __stack_depot_frame_decompress(1, low, &out)); + arch_stack_depot_frame_decompress(1, low, &out)); } #endif /* CONFIG_X86_64 */ @@ -2126,42 +1938,42 @@ static void stackdepot_frame_arm64(struct kunit *test) u8 prefix_id; KUNIT_EXPECT_TRUE(test, - __stack_depot_frame_try_compress(frame, &prefix_id, &low)); + arch_stack_depot_frame_try_compress(frame, &prefix_id, &low)); KUNIT_EXPECT_EQ(test, low, (u32)frame); KUNIT_EXPECT_TRUE(test, - __stack_depot_frame_decompress(prefix_id, low, &out)); + arch_stack_depot_frame_decompress(prefix_id, low, &out)); KUNIT_EXPECT_EQ(test, out, frame); if (text_prefix > SZ_4G) { frame = (text_prefix - SZ_4G) | 0x12345678UL; KUNIT_EXPECT_TRUE(test, - __stack_depot_frame_try_compress(frame, &prefix_id, &low)); + arch_stack_depot_frame_try_compress(frame, &prefix_id, &low)); KUNIT_EXPECT_EQ(test, prefix_id, (u8)STACK_DEPOT_ARM64_PREV_PREFIX_ID); KUNIT_EXPECT_TRUE(test, - __stack_depot_frame_decompress(prefix_id, low, &out)); + arch_stack_depot_frame_decompress(prefix_id, low, &out)); KUNIT_EXPECT_EQ(test, out, frame); } else { prefix_id = STACK_DEPOT_ARM64_PREV_PREFIX_ID; - decoded = __stack_depot_frame_decompress(prefix_id, 0, &out); + decoded = arch_stack_depot_frame_decompress(prefix_id, 0, &out); KUNIT_EXPECT_FALSE(test, decoded); } if (text_prefix <= ~0UL - SZ_4G) { frame = (text_prefix + SZ_4G) | 0x87654321UL; KUNIT_EXPECT_TRUE(test, - __stack_depot_frame_try_compress(frame, &prefix_id, &low)); + arch_stack_depot_frame_try_compress(frame, &prefix_id, &low)); KUNIT_EXPECT_EQ(test, prefix_id, (u8)STACK_DEPOT_ARM64_NEXT_PREFIX_ID); KUNIT_EXPECT_TRUE(test, - __stack_depot_frame_decompress(prefix_id, low, &out)); + arch_stack_depot_frame_decompress(prefix_id, low, &out)); KUNIT_EXPECT_EQ(test, out, frame); } else { prefix_id = STACK_DEPOT_ARM64_NEXT_PREFIX_ID; - decoded = __stack_depot_frame_decompress(prefix_id, 0, &out); + decoded = arch_stack_depot_frame_decompress(prefix_id, 0, &out); KUNIT_EXPECT_FALSE(test, decoded); } KUNIT_EXPECT_FALSE(test, - __stack_depot_frame_decompress(3, low, &out)); + arch_stack_depot_frame_decompress(3, low, &out)); } #endif /* CONFIG_ARM64 */ @@ -5642,9 +5454,11 @@ static void stackdepot_trie_public_save_route(struct kunit *test) noalloc_handle = stack_depot_save_flags(trie_entries, ARRAY_SIZE(trie_entries), no_spin, 0); KUNIT_EXPECT_EQ(test, noalloc_handle, trie_handle); noalloc_handle = stack_depot_save_flags(noalloc_entries, noalloc_nr, no_spin, 0); + KUNIT_EXPECT_EQ(test, noalloc_handle, (depot_stack_handle_t)0); + noalloc_handle = stack_depot_save(noalloc_entries, noalloc_nr, GFP_KERNEL); KUNIT_ASSERT_NE(test, noalloc_handle, (depot_stack_handle_t)0); KUNIT_EXPECT_NE(test, __stack_depot_trie_leaf_id(noalloc_handle), 0U); - KUNIT_EXPECT_EQ(test, stack_depot_save(noalloc_entries, noalloc_nr, GFP_KERNEL), + KUNIT_EXPECT_EQ(test, stack_depot_save_flags(noalloc_entries, noalloc_nr, no_spin, 0), noalloc_handle); get_flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; @@ -5820,11 +5634,7 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_trie_alloc_txn_reserve_id_failure), KUNIT_CASE(stackdepot_trie_alloc_txn_commit), KUNIT_CASE(stackdepot_trie_alloc_txn_rollback), - KUNIT_CASE(stackdepot_trie_alloc_workspace_plan), KUNIT_CASE(stackdepot_trie_alloc_workspace_insert), - KUNIT_CASE(stackdepot_trie_save_miss), - KUNIT_CASE(stackdepot_trie_save_miss_noalloc), - KUNIT_CASE(stackdepot_trie_save), KUNIT_CASE(stackdepot_trie_save_locked), KUNIT_CASE(stackdepot_trie_fetch_handle_into), KUNIT_CASE(stackdepot_trie_snprint_public), From 8ab08c2f1734d5a5a5068e82f83423100839d3e5 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 19 Jun 2026 09:04:10 +0100 Subject: [PATCH 108/129] KRN-1117: Clarify stackdepot caller assumptions Tighten caller-side assumptions after switching more users to stack_depot_fetch_into(). Make DRM use one shared stack-depth cap for both save and fetch, document KMSAN's buffer reuse ordering, and make chained origin printing handle empty fetches explicitly. Document the page_owner count-transition invariant so future refactors keep the single-winner saturated-to-counted cmpxchg behavior intact. Signed-off-by: Caleb Kan --- drivers/gpu/drm/drm_modeset_lock.c | 7 +++++-- mm/kmsan/report.c | 20 +++++++++++--------- mm/page_owner.c | 1 + 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c index 77b3d71e35fd5..731bc1cf5fd04 100644 --- a/drivers/gpu/drm/drm_modeset_lock.c +++ b/drivers/gpu/drm/drm_modeset_lock.c @@ -81,9 +81,12 @@ static DEFINE_WW_CLASS(crtc_ww_class); #if IS_ENABLED(CONFIG_DRM_DEBUG_MODESET_LOCK) +/* Save and fetch use the same cap so fetch_into() cannot reject saved stacks. */ +#define DRM_STACK_DEPOT_MAX_FRAMES 8 + static noinline depot_stack_handle_t __drm_stack_depot_save(void) { - unsigned long entries[8]; + unsigned long entries[DRM_STACK_DEPOT_MAX_FRAMES]; unsigned int n; n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); @@ -94,7 +97,7 @@ static noinline depot_stack_handle_t __drm_stack_depot_save(void) static void __drm_stack_depot_print(depot_stack_handle_t stack_depot) { struct drm_printer p = drm_dbg_printer(NULL, DRM_UT_KMS, "drm_modeset_lock"); - unsigned long entries[8]; + unsigned long entries[DRM_STACK_DEPOT_MAX_FRAMES]; unsigned int nr_entries; char *buf; diff --git a/mm/kmsan/report.c b/mm/kmsan/report.c index df4b70a757a6a..8e62e98916a5f 100644 --- a/mm/kmsan/report.c +++ b/mm/kmsan/report.c @@ -88,6 +88,7 @@ void kmsan_print_origin(depot_stack_handle_t origin) unsigned long entries[KMSAN_STACK_DEPTH]; const unsigned int max_entries = ARRAY_SIZE(entries); unsigned int nr_entries, chained_nr_entries, skipnr; + size_t chained_size; void *pc1 = NULL, *pc2 = NULL; depot_stack_handle_t head; unsigned long magic; @@ -101,7 +102,7 @@ void kmsan_print_origin(depot_stack_handle_t origin) nr_entries = stack_depot_fetch_into(origin, entries, max_entries); depth = kmsan_depth_from_eb(stack_depot_get_extra_bits(origin)); magic = nr_entries ? entries[0] : 0; - if ((nr_entries == 4) && (magic == KMSAN_ALLOCA_MAGIC_ORIGIN)) { + if (nr_entries == 4 && magic == KMSAN_ALLOCA_MAGIC_ORIGIN) { descr = (char *)entries[1]; pc1 = (void *)entries[2]; pc2 = (void *)entries[3]; @@ -113,7 +114,7 @@ void kmsan_print_origin(depot_stack_handle_t origin) pr_err(" %pSb\n", pc2); break; } - if ((nr_entries == 3) && (magic == KMSAN_CHAIN_MAGIC_ORIGIN)) { + if (nr_entries == 3 && magic == KMSAN_CHAIN_MAGIC_ORIGIN) { /* * Origin chains deeper than KMSAN_MAX_ORIGIN_DEPTH are * not stored, so the output may be incomplete. @@ -123,15 +124,16 @@ void kmsan_print_origin(depot_stack_handle_t origin) head = entries[1]; origin = entries[2]; pr_err("Uninit was stored to memory at:\n"); + /* Save head/origin locally before reusing entries below. */ chained_nr_entries = stack_depot_fetch_into(head, entries, max_entries); - kmsan_internal_unpoison_memory( - entries, - chained_nr_entries * sizeof(*entries), - /*checked*/ false); - skipnr = get_stack_skipnr(entries, chained_nr_entries); - stack_trace_print(entries + skipnr, - chained_nr_entries - skipnr, 0); + chained_size = chained_nr_entries * sizeof(*entries); + kmsan_internal_unpoison_memory(entries, chained_size, false); + if (chained_nr_entries) { + skipnr = get_stack_skipnr(entries, chained_nr_entries); + stack_trace_print(entries + skipnr, + chained_nr_entries - skipnr, 0); + } pr_err("\n"); continue; } diff --git a/mm/page_owner.c b/mm/page_owner.c index 30f715f75cc77..49f496a963d6d 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -232,6 +232,7 @@ static bool inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, if (!__stack_depot_get_count(handle, &count)) stack = alloc_stack_record(gfp_mask); + /* Only one caller can win the saturated-to-counted cmpxchg transition. */ /* Racing transition losers free their unused list node below. */ if (!__stack_depot_inc_count(handle, nr_base_pages, &new_count)) { if (stack) From 09a15a552e09357e1219f876f6149633c6d61d81 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 19 Jun 2026 20:02:24 +0100 Subject: [PATCH 109/129] KRN-1117: Remove test-driven stackdepot trie indirection Simplify the trie backend around review feedback. Drop the remaining production entry points that existed only for white-box KUnit tests, remove the local frame-compression prefix abstraction, and call the arch compression hooks through a single low-bit payload. Keep KUnit coverage on public stackdepot behavior instead of exporting internal trie operations just for tests. Add comments for the remaining opaque trie storage, side-table, handle namespace, and pool reuse invariants so the production data structures are reviewable without extra wrappers. Signed-off-by: Caleb Kan --- arch/arm64/include/asm/stackdepot.h | 86 +- arch/x86/include/asm/stackdepot.h | 14 +- include/asm-generic/stackdepot.h | 8 +- lib/stackdepot.c | 411 +- lib/stackdepot_internal.h | 58 +- lib/tests/stackdepot_kunit.c | 5832 ++------------------------- 6 files changed, 383 insertions(+), 6026 deletions(-) diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h index a87955d49515b..eea824c293caf 100644 --- a/arch/arm64/include/asm/stackdepot.h +++ b/arch/arm64/include/asm/stackdepot.h @@ -2,98 +2,38 @@ #ifndef __ASM_STACKDEPOT_H #define __ASM_STACKDEPOT_H -#include #include #include #include -#define STACK_DEPOT_ARM64_FRAME_LOW_MASK 0x00000000ffffffffUL -#define STACK_DEPOT_ARM64_FRAME_PREFIX_MASK (~STACK_DEPOT_ARM64_FRAME_LOW_MASK) - /* * Modules are allocated inside a 2 GB relocation window containing the - * kernel image, but stackdepot compression stores only the low 32 bits of - * each frame. If that window crosses a 4 GB high-bit boundary, module text - * may have the previous or next prefix even though it is still within - * relocation range of _text. - * - * Prefix IDs are arch-local metadata; trie storage is per boot and is never - * interpreted by another architecture's decompressor. + * kernel image. Store a signed 32-bit offset from _text so compression is + * independent of 4 GB high-bit boundaries crossed by that window. */ -#define STACK_DEPOT_ARM64_PREV_PREFIX_ID 0 -#define STACK_DEPOT_ARM64_TEXT_PREFIX_ID 1 -#define STACK_DEPOT_ARM64_NEXT_PREFIX_ID 2 - -static inline unsigned long arch_stack_depot_frame_text_prefix(void) +static inline bool +arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) { - return (unsigned long)_text & STACK_DEPOT_ARM64_FRAME_PREFIX_MASK; -} + long offset; -static inline bool arch_stack_depot_frame_prefix(u8 prefix_id, - unsigned long *prefix) -{ - unsigned long text_prefix = arch_stack_depot_frame_text_prefix(); - - switch (prefix_id) { - case STACK_DEPOT_ARM64_PREV_PREFIX_ID: - /* Reject < SZ_4G for underflow and == SZ_4G for prefix value 0. */ - if (text_prefix <= SZ_4G) - return false; - *prefix = text_prefix - SZ_4G; - return true; - case STACK_DEPOT_ARM64_TEXT_PREFIX_ID: - /* Prefix zero is reserved for the raw fallback. */ - if (!text_prefix) - return false; - *prefix = text_prefix; - return true; - case STACK_DEPOT_ARM64_NEXT_PREFIX_ID: - if (text_prefix > ULONG_MAX - SZ_4G) - return false; - *prefix = text_prefix + SZ_4G; - return true; - default: + if (!low) return false; - } -} - -static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, - u8 *prefix_id, u32 *low) -{ - unsigned long prefix = frame & STACK_DEPOT_ARM64_FRAME_PREFIX_MASK; - unsigned long candidate; - unsigned int i; - if (!prefix_id || !low) + offset = (long)frame - (long)_text; + if (offset < S32_MIN || offset > S32_MAX) return false; - for (i = STACK_DEPOT_ARM64_PREV_PREFIX_ID; - i <= STACK_DEPOT_ARM64_NEXT_PREFIX_ID; i++) { - if (!arch_stack_depot_frame_prefix(i, &candidate)) - continue; - if (prefix != candidate) - continue; - - *prefix_id = i; - *low = (u32)frame; - return true; - } - - return false; + *low = (u32)(s32)offset; + return true; } -static inline bool arch_stack_depot_frame_decompress(u8 prefix_id, u32 low, - unsigned long *frame) +static inline bool +arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) { - unsigned long prefix; - if (!frame) return false; - if (!arch_stack_depot_frame_prefix(prefix_id, &prefix)) - return false; - - *frame = prefix | low; + *frame = (unsigned long)((long)_text + (s32)low); return true; } diff --git a/arch/x86/include/asm/stackdepot.h b/arch/x86/include/asm/stackdepot.h index 21c43dec9ad2a..457516ac9bf39 100644 --- a/arch/x86/include/asm/stackdepot.h +++ b/arch/x86/include/asm/stackdepot.h @@ -11,30 +11,26 @@ static_assert(STACK_DEPOT_X86_64_FRAME_PREFIX != 0); -static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, - u8 *prefix_id, u32 *low) +static inline bool +arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) { - if (!prefix_id || !low) + if (!low) return false; if ((frame & ~STACK_DEPOT_X86_64_FRAME_LOW_MASK) != STACK_DEPOT_X86_64_FRAME_PREFIX) return false; - *prefix_id = 0; *low = (u32)frame; return true; } -static inline bool arch_stack_depot_frame_decompress(u8 prefix_id, u32 low, - unsigned long *frame) +static inline bool +arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) { if (!frame) return false; - if (prefix_id) - return false; - *frame = STACK_DEPOT_X86_64_FRAME_PREFIX | low; return true; } diff --git a/include/asm-generic/stackdepot.h b/include/asm-generic/stackdepot.h index f9bacfaf72ad8..26c2073ccda03 100644 --- a/include/asm-generic/stackdepot.h +++ b/include/asm-generic/stackdepot.h @@ -4,14 +4,14 @@ #include -static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, - u8 *prefix_id, u32 *low) +static inline bool +arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) { return false; } -static inline bool arch_stack_depot_frame_decompress(u8 prefix_id, u32 low, - unsigned long *frame) +static inline bool +arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) { return false; } diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 6fe238daaa6df..35edd849a3118 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -15,6 +15,7 @@ #define pr_fmt(fmt) "stackdepot: " fmt #include +#include #include #include #include @@ -59,7 +60,6 @@ static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT); static bool __stack_depot_early_init_passed __initdata; static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); -static bool stack_depot_trie_enabled_param; static DEFINE_MUTEX(stack_depot_trie_param_lock); static struct stack_depot_trie_root stack_depot_trie_root; static struct stack_depot_trie_alloc_workspace *stack_depot_trie_workspace; @@ -73,10 +73,9 @@ bool __stack_depot_trie_enabled(void) void __stack_depot_trie_set_enabled(bool enabled) { - if (READ_ONCE(stack_depot_trie_enabled_param) == enabled) + if (__stack_depot_trie_enabled() == enabled) return; - WRITE_ONCE(stack_depot_trie_enabled_param, enabled); if (enabled) static_branch_enable(&stack_depot_trie_enabled); else @@ -97,6 +96,7 @@ static int stack_depot_trie_enabled_param_set(const char *val, /* Keep runtime toggles serialized outside stack_depot_init_mutex. */ mutex_lock(&stack_depot_trie_param_lock); + /* stack_depot_init() sees this key; save routing still waits for ready. */ __stack_depot_trie_set_enabled(enabled); if (enabled && system_state >= SYSTEM_RUNNING) { ret = stack_depot_init(); @@ -113,7 +113,7 @@ static int stack_depot_trie_enabled_param_get(char *buffer, const struct kernel_param *kp) { struct kernel_param tmp = *kp; - bool enabled = READ_ONCE(*(bool *)kp->arg); + bool enabled = __stack_depot_trie_enabled(); tmp.arg = &enabled; return param_get_bool(buffer, &tmp); @@ -126,7 +126,7 @@ static const struct kernel_param_ops stack_depot_trie_enabled_param_ops = { .get = stack_depot_trie_enabled_param_get, }; module_param_cb(trie_enabled, &stack_depot_trie_enabled_param_ops, - &stack_depot_trie_enabled_param, 0644); + NULL, 0644); MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage"); /* Use one hash table bucket per 16 KB of memory. */ @@ -174,7 +174,9 @@ struct stack_record { }; struct stack_depot_trie_node { + /* Parent links let fetch rebuild a full stack from a leaf to the root. */ const struct stack_depot_trie_node *parent; + /* Child arrays are separate RCU/COW generations; nodes stay immutable. */ const struct stack_depot_trie_child_array *children; u32 leaf_id; u16 stack_len; @@ -183,16 +185,23 @@ struct stack_depot_trie_node { }; struct stack_depot_trie_child_array { + /* nr_children/capacity must live with the pointer array readers index. */ unsigned int nr_children; unsigned int capacity; const struct stack_depot_trie_node *children[]; }; +/* Headerless reusable storage for trie nodes. */ struct stack_depot_trie_free_node { struct list_head list; size_t size; }; +/* + * Reusable object storage for child arrays and other payloads that need an + * object header. A retired child array can carry the old child node that was + * replaced with it; both become reusable after the array's RCU grace period. + */ struct stack_depot_trie_free_object { struct list_head list; unsigned long rcu_state; @@ -230,9 +239,19 @@ static size_t pool_offset = DEPOT_POOL_SIZE; /* Freelist of stack records within stack_pools. */ static LIST_HEAD(free_stacks); +/* Size classes bucket reusable trie storage by aligned allocation size. */ #define STACK_DEPOT_TRIE_FREE_CLASSES \ ((DEPOT_POOL_SIZE >> DEPOT_STACK_ALIGN) + 1) +/* + * Trie storage is suballocated from stackdepot pools, not slab caches, so pool + * pressure stays visible through stack_depot_max_pools and no-spin callers can + * fail without allocator recursion. Trie COW insertion retires child arrays + * and sometimes the node they replaced. Objects carry the RCU cookie for + * child-array payloads; headerless node fragments either live directly on + * free_trie_nodes or are attached to a pending object until that object's grace + * period has elapsed. + */ static struct list_head free_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; static struct list_head pending_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; static struct list_head free_trie_nodes[STACK_DEPOT_TRIE_FREE_CLASSES]; @@ -266,8 +285,6 @@ static const char *const counter_names[] = { }; static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); -/* Count helpers rely on saturated refcounts looking negative. */ -static_assert(REFCOUNT_SATURATED < 0); static bool depot_init_pool(void **prealloc); static void depot_try_keep_new_pool(void **prealloc); @@ -288,6 +305,13 @@ static bool stack_depot_trie_namespace_available(void) return stack_max_pools < stack_depot_pool_index_mask() - 1; } +/* + * Hash handles encode pool_index_plus_1 and offset. Trie handles reserve the + * pool-index values above stack_max_pools and reinterpret the offset bits as a + * dense leaf_id, which the side table maps to a trie leaf. Init treats an + * unavailable namespace as a hard trie failure; the checks below are defensive + * because handle helpers can be reached from tests and disabled configurations. + */ u32 __stack_depot_trie_max_leaf_id(void) { if (!stack_depot_trie_namespace_available()) @@ -345,6 +369,14 @@ struct stack_depot_trie_side_entry { const void *leaf; }; +/* + * Trie handles encode a dense leaf ID. The side table maps that ID to a leaf + * pointer for lockless fetch/print paths, which can run from diagnostic + * contexts where taking trie_side_table_lock would be unsafe. Init installs the + * root and first chunk only; additional directories/chunks are preallocated and + * published lazily as leaf IDs grow. Release/acquire pairs publish fully + * initialized dirs, chunks, and leaves to those lockless readers. + */ #define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS 9 #define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE \ (1U << STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS) @@ -365,18 +397,17 @@ static u32 trie_side_table_next_id; static bool trie_side_table_initialized; static bool trie_side_table_memblock; -/* Lock order: trie_alloc_lock -> pool_lock -> trie_side_table_lock. */ +/* Lock order: workspace_lock -> trie_alloc_lock -> pool_lock -> trie_side_table_lock. */ static bool stack_depot_trie_is_ready(void) { - /* Pairs with stack_depot_trie_publish_ready(). */ + /* Pairs with stack_depot_trie_publish_ready(); publishes all trie init. */ return smp_load_acquire(&stack_depot_trie_ready); } static bool trie_side_table_is_initialized(void) { - /* Pairs with trie_side_table_publish_initialized(). */ - return smp_load_acquire(&trie_side_table_initialized); + return READ_ONCE(trie_side_table_initialized); } static void stack_depot_trie_publish_ready(void) @@ -387,14 +418,7 @@ static void stack_depot_trie_publish_ready(void) static void trie_side_table_publish_initialized(void) { - /* Pairs with trie_side_table_is_initialized(). */ - smp_store_release(&trie_side_table_initialized, true); -} - -static void stack_depot_trie_mark_not_ready(void) -{ - /* Pairs with stack_depot_trie_is_ready(). */ - smp_store_release(&stack_depot_trie_ready, false); + WRITE_ONCE(trie_side_table_initialized, true); } static int stack_pool_addr_cmp(const void *a, const void *b) @@ -587,14 +611,14 @@ static unsigned int trie_side_table_slot_index(u32 id) static struct stack_depot_trie_side_dir *trie_side_table_load_dir(unsigned int root) { - /* Pairs with trie_side_table_publish_dir(). */ + /* Pairs with trie_side_table_publish_dir(); lookup is lockless. */ return smp_load_acquire(&trie_side_table_dirs[root]); } static void trie_side_table_publish_dir(unsigned int root, struct stack_depot_trie_side_dir *dir) { - /* Pairs with trie_side_table_load_dir(). */ + /* Publish the zeroed directory before readers can load it locklessly. */ smp_store_release(&trie_side_table_dirs[root], dir); } @@ -602,7 +626,7 @@ static struct stack_depot_trie_side_entry * trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, unsigned int idx) { - /* Pairs with trie_side_table_dir_publish_chunk(). */ + /* Pairs with trie_side_table_dir_publish_chunk(); lookup is lockless. */ return smp_load_acquire(&dir->chunks[idx]); } @@ -625,19 +649,23 @@ trie_side_table_alloc_id_locked(struct stack_depot_trie_side_prealloc *prealloc) u32 id; lockdep_assert_held(&trie_side_table_lock); + /* Failed or disabled trie init means no leaf IDs can be allocated. */ if (!trie_side_table_is_initialized()) return 0; id = trie_side_table_next_id + 1; + /* ID zero wraps the 32-bit counter; max_id is handle namespace capacity. */ if (!id || id > trie_side_table_max_id) return 0; root = trie_side_table_root_index(id); + /* Should be impossible when trie_side_table_max_id/root_size agree. */ if (root >= trie_side_table_root_size) return 0; dir = trie_side_table_load_dir(root); if (!dir) { + /* Sparse growth needs a preallocated dir before taking this lock. */ if (!prealloc || !prealloc->dir) return 0; dir = prealloc->dir; @@ -651,6 +679,7 @@ trie_side_table_alloc_id_locked(struct stack_depot_trie_side_prealloc *prealloc) idx = trie_side_table_dir_index(id); chunk = trie_side_table_dir_load_chunk(dir, idx); if (!chunk) { + /* Sparse growth needs a preallocated chunk before taking this lock. */ if (!prealloc || !prealloc->chunk) return 0; chunk = prealloc->chunk; @@ -713,6 +742,7 @@ trie_side_table_install(struct stack_depot_trie_side_dir **dirs, struct stack_depot_trie_side_entry *first_chunk, bool memblock) { + /* Init installs only the root and first chunk; later chunks grow lazily. */ if (trie_side_table_is_initialized()) return 0; if (!dirs || !root_size || !max_id) @@ -937,43 +967,6 @@ int __stack_depot_trie_side_table_init(gfp_t gfp_flags) false); } -void __stack_depot_trie_side_table_destroy(void) -{ - struct stack_depot_trie_side_dir *dir; - unsigned int high_water; - unsigned int j; - unsigned int i; - - if (!trie_side_table_is_initialized()) - return; - /* KUnit/teardown helper only; callers must exclude readers and writers. */ - stack_depot_trie_mark_not_ready(); - /* Pairs with trie_side_table_is_initialized(). */ - smp_store_release(&trie_side_table_initialized, false); - synchronize_rcu(); - - high_water = READ_ONCE(trie_side_table_high_water); - if (!READ_ONCE(trie_side_table_memblock)) { - for (i = 0; i < high_water; i++) { - dir = trie_side_table_dirs[i]; - if (!dir) - continue; - for (j = 0; j < STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE; j++) - trie_side_table_free_chunk(dir->chunks[j]); - trie_side_table_free_dir(dir); - } - kvfree(trie_side_table_dirs); - } - trie_side_table_dirs = NULL; - WRITE_ONCE(trie_side_table_high_water, 0); - WRITE_ONCE(trie_side_table_nr_dirs, 0); - WRITE_ONCE(trie_side_table_nr_chunks, 0); - WRITE_ONCE(trie_side_table_root_size, 0); - WRITE_ONCE(trie_side_table_max_id, 0); - WRITE_ONCE(trie_side_table_next_id, 0); - WRITE_ONCE(trie_side_table_memblock, false); -} - bool __stack_depot_trie_side_table_prealloc_needed(void) { struct stack_depot_trie_side_dir *dir; @@ -1157,38 +1150,6 @@ void __stack_depot_trie_side_table_restore(u32 id, const void *entry) raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); } -int __stack_depot_trie_side_table_store(u32 id, const void *entry) -{ - struct stack_depot_trie_side_entry *chunk; - struct stack_depot_trie_side_dir *dir; - unsigned long flags; - unsigned int root; - int ret = -EINVAL; - - if (!trie_side_table_is_initialized() || !id || !entry) - return -EINVAL; - - raw_spin_lock_irqsave(&trie_side_table_lock, flags); - if (id > trie_side_table_next_id) - goto out; - root = trie_side_table_root_index(id); - if (root >= trie_side_table_root_size) - goto out; - - dir = trie_side_table_load_dir(root); - if (!dir) - goto out; - chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); - if (!chunk) - goto out; - - trie_side_table_store_leaf(chunk, trie_side_table_slot_index(id), entry); - ret = 0; -out: - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return ret; -} - static struct stack_depot_trie_side_entry * trie_side_table_chunk_locked(u32 id, unsigned int *slot) { @@ -1239,12 +1200,6 @@ const void *__stack_depot_trie_side_table_lookup(u32 id) return trie_side_table_load_leaf(chunk, trie_side_table_slot_index(id)); } -size_t __stack_depot_trie_side_table_entries(void) -{ - return trie_side_table_is_initialized() ? - READ_ONCE(trie_side_table_next_id) : 0; -} - size_t __stack_depot_trie_side_table_bytes(void) { unsigned int nr_dirs; @@ -1433,6 +1388,7 @@ static bool trie_pool_range_contains_locked(const void *ptr, size_t size) lockdep_assert_held(&pool_lock); + /* Reject stale/non-pool storage before putting COW-retired bytes on freelists. */ if (!ptr || !size || check_add_overflow(start, size, &end)) return false; if (!stack_pools_sorted || READ_ONCE(stack_pools_sorted_capacity) < pools) { @@ -1547,6 +1503,7 @@ static void trie_drain_pending_objects_locked(void) for_each_set_bit(class, pending_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES) { list_for_each_entry_safe(free, tmp, &pending_trie_objects[class], list) { + /* Pending lists are FIFO; later entries cannot be ready yet. */ if (!poll_state_synchronize_rcu(free->rcu_state)) break; trie_drain_free_object_node_locked(free); @@ -1761,6 +1718,11 @@ static bool stack_depot_trie_pool_rollback_locked(const struct stack_depot_trie_ if (mark->pool_index != pools_num - 1 || pool_offset != end) return false; + /* + * mark->offset is the reservation start in the active pool. If the + * reservation added a new pool, mark->prev_offset is the offset to restore + * in the previous pool after making the new pool available for reuse. + */ if (mark->added_pool) { if (mark->offset || stack_pools[mark->pool_index] != mark->pool) return false; @@ -1792,19 +1754,6 @@ static bool stack_depot_trie_pool_rollback(const struct stack_depot_trie_pool_ma return ret; } -bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mark *mark) -{ - unsigned long flags; - bool ret; - - if (!raw_spin_trylock_irqsave(&pool_lock, flags)) - return false; - ret = stack_depot_trie_pool_rollback_locked(mark); - raw_spin_unlock_irqrestore(&pool_lock, flags); - - return ret; -} - static int trie_pool_add_object_size(size_t size, size_t *total) { size_t alloc_size; @@ -2358,12 +2307,6 @@ void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *tx memset(&txn->pool, 0, sizeof(txn->pool)); } -void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state) -{ - if (state) - memset(state, 0, sizeof(*state)); -} - void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *state) { if (!state) @@ -3026,9 +2969,6 @@ depot_save_stack_locked(struct stack_depot_hash_save *save) save->hash, save->depot_flags); if (found) return found->handle.handle; - if (save->normal_persistent && __stack_depot_trie_ready()) - return 0; - new = depot_alloc_stack(save->entries, save->nr_entries, save->hash, save->depot_flags, save->prealloc); if (!new) @@ -3171,9 +3111,6 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, /* Stack depot didn't use this memory, free it. */ free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); } - if (!handle && normal_persistent && __stack_depot_trie_ready()) - return stack_depot_trie_save(entries, nr_entries, alloc_flags, - depot_flags); return handle; } EXPORT_SYMBOL_GPL(stack_depot_save_flags); @@ -3317,22 +3254,6 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, return !new; } -static bool frame_try_compress(unsigned long frame, u8 *prefix_id, u32 *low) -{ - if (!prefix_id || !low) - return false; - - return arch_stack_depot_frame_try_compress(frame, prefix_id, low); -} - -static bool frame_decompress(u8 prefix_id, u32 low, unsigned long *frame) -{ - if (!frame) - return false; - - return arch_stack_depot_frame_decompress(prefix_id, low, frame); -} - static bool stack_depot_ranges_overlap(const void *a, size_t a_size, const void *b, size_t b_size); static int @@ -3381,7 +3302,6 @@ static int frame_run_init_lows(const unsigned long *entries, struct stack_depot_frame_run *run, u32 *lows, unsigned int nr_lows) { - u8 first_prefix = 0; u32 low; unsigned int i; bool compressed; @@ -3394,26 +3314,21 @@ static int frame_run_init_lows(const unsigned long *entries, return -EINVAL; /* On compressed success, only lows[0..run->nr_entries - 1] are initialized. */ - /* Only prefix ids classify a run; low bits are scratch for the arch hook. */ - compressed = frame_try_compress(entries[0], &first_prefix, &low); + compressed = arch_stack_depot_frame_try_compress(entries[0], &low); if (compressed && lows) lows[0] = low; for (i = 1; i < nr_entries; i++) { - u8 prefix_id; bool next; - next = frame_try_compress(entries[i], &prefix_id, &low); + next = arch_stack_depot_frame_try_compress(entries[i], &low); if (next != compressed) break; - if (compressed && prefix_id != first_prefix) - break; if (compressed && lows) lows[i] = low; } /* @i is the first non-matching frame, or @nr_entries if all matched. */ run->mode = compressed ? STACK_DEPOT_FRAME_COMPRESSED : STACK_DEPOT_FRAME_RAW; - run->prefix_id = compressed ? first_prefix : 0; run->nr_entries = i; run->bytes = i * stack_depot_frame_run_entry_bytes(run->mode); @@ -3427,94 +3342,6 @@ int __stack_depot_frame_run_init(const unsigned long *entries, return frame_run_init_lows(entries, nr_entries, run, NULL, 0); } -static int -stack_depot_frame_run_write_compressed(const struct stack_depot_frame_run *run, - const unsigned long *entries, void *dst, - u32 *scratch, unsigned int nr_scratch) -{ - unsigned int i; - - if (!scratch || nr_scratch < run->nr_entries) - return -EINVAL; - if (stack_depot_ranges_overlap(dst, run->bytes, scratch, run->bytes)) - return -EINVAL; - if (stack_depot_ranges_overlap(scratch, run->bytes, entries, - run->nr_entries * sizeof(*entries))) - return -EINVAL; - - for (i = 0; i < run->nr_entries; i++) { - u8 prefix_id; - - if (!frame_try_compress(entries[i], &prefix_id, &scratch[i])) - return -EINVAL; - if (prefix_id != run->prefix_id) - return -EINVAL; - } - - memcpy(dst, scratch, run->bytes); - return 0; -} - -static int frame_run_write(const struct stack_depot_frame_run *run, - const unsigned long *entries, void *dst, size_t dst_size, - u32 *scratch, unsigned int nr_scratch) -{ - int ret; - - if (!entries || !dst) - return -EINVAL; - - ret = stack_depot_frame_run_validate(run); - if (ret) - return ret; - if (dst_size < run->bytes) - return -EINVAL; - if (stack_depot_ranges_overlap(dst, run->bytes, entries, - run->nr_entries * sizeof(*entries))) - return -EINVAL; - - if (run->mode == STACK_DEPOT_FRAME_RAW) { - memcpy(dst, entries, run->bytes); - return 0; - } - - return stack_depot_frame_run_write_compressed(run, entries, dst, scratch, - nr_scratch); -} - -int __stack_depot_frame_run_write(const struct stack_depot_frame_run *run, - const unsigned long *entries, void *dst, - size_t dst_size, u32 *scratch, - unsigned int nr_scratch) -{ - return frame_run_write(run, entries, dst, dst_size, scratch, nr_scratch); -} - -static int -stack_depot_frame_run_read_compressed(const struct stack_depot_frame_run *run, - const void *src, unsigned long *entries, - unsigned long *scratch, - unsigned int nr_scratch) -{ - unsigned int i; - - if (!scratch || nr_scratch < run->nr_entries) - return -EINVAL; - - /* Stage lows first so a bad compressed run cannot leave a partial write. */ - for (i = 0; i < run->nr_entries; i++) { - u32 low; - - memcpy(&low, (const char *)src + i * sizeof(low), sizeof(low)); - if (!frame_decompress(run->prefix_id, low, &scratch[i])) - return -EINVAL; - } - - if (entries != scratch) - memcpy(entries, scratch, run->nr_entries * sizeof(*entries)); - return 0; -} - static int frame_run_validate_payload(const struct stack_depot_frame_run *run, const void *src) { @@ -3530,7 +3357,7 @@ static int frame_run_validate_payload(const struct stack_depot_frame_run *run, u32 low; memcpy(&low, (const char *)src + i * sizeof(low), sizeof(low)); - if (!frame_decompress(run->prefix_id, low, &frame)) + if (!arch_stack_depot_frame_decompress(low, &frame)) return -EINVAL; } @@ -3554,54 +3381,6 @@ static bool stack_depot_ranges_overlap(const void *a, size_t a_size, return a_start < b_end && b_start < a_end; } -static int frame_run_read(const struct stack_depot_frame_run *run, - const void *src, size_t src_size, - unsigned long *entries, unsigned int max_entries, - unsigned long *scratch, unsigned int nr_scratch) -{ - int ret; - - if (!src || !entries) - return -EINVAL; - - ret = stack_depot_frame_run_validate(run); - if (ret) - return ret; - if (src_size < run->bytes || max_entries < run->nr_entries) - return -EINVAL; - if (stack_depot_ranges_overlap(entries, - run->nr_entries * sizeof(*entries), src, - run->bytes)) - return -EINVAL; - - if (run->mode == STACK_DEPOT_FRAME_RAW) { - memcpy(entries, src, run->bytes); - return 0; - } - if (!scratch || nr_scratch < run->nr_entries) - return -EINVAL; - if (stack_depot_ranges_overlap(entries, - run->nr_entries * sizeof(*entries), scratch, - run->nr_entries * sizeof(*scratch))) - return -EINVAL; - if (stack_depot_ranges_overlap(src, run->bytes, scratch, - run->nr_entries * sizeof(*scratch))) - return -EINVAL; - - return stack_depot_frame_run_read_compressed(run, src, entries, scratch, - nr_scratch); -} - -int __stack_depot_frame_run_read(const struct stack_depot_frame_run *run, - const void *src, size_t src_size, - unsigned long *entries, unsigned int max_entries, - unsigned long *scratch, - unsigned int nr_scratch) -{ - return frame_run_read(run, src, src_size, entries, max_entries, scratch, - nr_scratch); -} - size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) { size_t size; @@ -3647,7 +3426,7 @@ stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, } memcpy(&low, node->data + index * sizeof(low), sizeof(low)); - if (!frame_decompress(node->run.prefix_id, low, frame)) + if (!arch_stack_depot_frame_decompress(low, frame)) return -EINVAL; return 0; @@ -4662,15 +4441,6 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, return 0; } -int -__stack_depot_trie_publish_append(struct stack_depot_trie_root *root, - void *parent_ptr, const void *head_ptr, - void *new_storage, size_t new_storage_size) -{ - return trie_publish_append_prepare(root, parent_ptr, head_ptr, new_storage, - new_storage_size, NULL, 0, NULL); -} - int __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, const void *parent_ptr, const unsigned long *entries, @@ -4905,27 +4675,6 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, return 0; } -int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, - void *parent, u32 leaf_id, - const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, void *new_storage, - size_t new_storage_size, const void **tail, - unsigned int *nr_used) -{ - return __stack_depot_trie_insert_append_prepare(root, parent, leaf_id, - entries, nr_entries, node_slots, - nr_node_slots, child_slots, - nr_child_slots, scratch, - nr_scratch, new_storage, - new_storage_size, NULL, tail, - nr_used); -} - static int trie_plan_append_chain(unsigned int base_stack_len, const unsigned long *entries, unsigned int nr_entries, @@ -5973,23 +5722,6 @@ static int trie_split_subtree_prepare(const void *child_ptr, unsigned int matche return 0; } -int __stack_depot_trie_split_subtree(const void *child, unsigned int matched, - u32 leaf_id, const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, const void **prefix, - const void **tail, unsigned int *nr_used) -{ - return trie_split_subtree_prepare(child, matched, leaf_id, entries, - nr_entries, node_slots, nr_node_slots, - child_slots, nr_child_slots, scratch, - nr_scratch, NULL, prefix, tail, - nr_used); -} - static int trie_split_child(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, const struct stack_depot_trie_node *child, @@ -6095,23 +5827,6 @@ stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *ar return 0; } -const void * -__stack_depot_trie_child_array_find(const void *storage, unsigned long frame) -{ - const struct stack_depot_trie_child_array *array = storage; - unsigned int pos; - bool found; - - if (!array) - return NULL; - - if (stack_depot_trie_child_lower_bound(array, frame, &pos, &found) || - !found) - return NULL; - - return array->children[pos]; -} - int __stack_depot_trie_child_array_insert(const void *old_storage, const void *child, void *new_storage, size_t new_storage_size) diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h index 7ab55cbb3ec61..b4d8f99c5b6f1 100644 --- a/lib/stackdepot_internal.h +++ b/lib/stackdepot_internal.h @@ -25,7 +25,6 @@ struct stack_depot_frame_run { u16 bytes; u16 nr_entries; u8 mode; - u8 prefix_id; }; static_assert(CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(unsigned long) <= U16_MAX); @@ -34,11 +33,13 @@ bool __stack_depot_trie_enabled(void); bool __stack_depot_trie_ready(void); void __stack_depot_trie_set_enabled(bool enabled); +/* Opaque trie node storage; node layout stays private to stackdepot.c. */ struct stack_depot_trie_node_slot { void *node; size_t size; }; +/* Opaque child-array storage; child array layout stays private. */ struct stack_depot_trie_child_array_slot { void *array; size_t size; @@ -51,7 +52,9 @@ struct stack_depot_trie_root { }; struct stack_depot_trie_lookup { + /* Opaque parent trie node for the current lookup step. */ const void *parent; + /* Opaque trie node matched at this step, if any. */ const void *node; enum stack_depot_trie_lookup_status status; unsigned int matched; @@ -59,12 +62,14 @@ struct stack_depot_trie_lookup { struct stack_depot_trie_leaf_update { u32 leaf_id; + /* Opaque trie leaf that should become visible for leaf_id. */ const void *leaf; }; struct stack_depot_trie_publish_prepare { int (*fn)(const struct stack_depot_trie_leaf_update *updates, unsigned int nr_updates, void *ctx); + /* Caller-owned state passed to fn. */ void *ctx; bool retire_locked; }; @@ -85,11 +90,14 @@ struct stack_depot_trie_side_prepare { }; struct stack_depot_trie_side_prealloc { + /* Preallocated side-table directory page for sparse growth. */ void *dir; + /* Preallocated side-table leaf chunk for sparse growth. */ void *chunk; }; struct stack_depot_trie_pool_mark { + /* Stackdepot pool backing this transactional reservation. */ void *pool; size_t prev_offset; size_t offset; @@ -101,7 +109,9 @@ struct stack_depot_trie_pool_mark { struct stack_depot_trie_pool_request { struct stack_depot_trie_node_slot *node_slots; struct stack_depot_trie_child_array_slot *child_slots; + /* Optional opaque object storage reserved with the node/child slots. */ void **storage; + /* Optional fresh stackdepot pool page, preallocated outside pool_lock. */ void **prealloc; struct stack_depot_trie_pool_mark *mark; size_t storage_size; @@ -119,7 +129,9 @@ struct stack_depot_trie_alloc_request { struct stack_depot_trie_alloc_txn *txn; struct stack_depot_trie_node_slot *node_slots; struct stack_depot_trie_child_array_slot *child_slots; + /* Optional opaque replacement child-array storage. */ void **storage; + /* Optional fresh stackdepot pool page, preallocated before insertion. */ void **pool_prealloc; struct stack_depot_trie_side_prealloc *side_prealloc; size_t storage_size; @@ -146,12 +158,10 @@ u32 __stack_depot_trie_max_leaf_id(void); /* * Private trie side table. Writers serialize internally; lookups are lockless. - * Leaf slots are populated before trie publication. Init and destroy are - * controlled setup/teardown operations and must not race with readers or - * writers. + * Leaf slots are populated before trie publication. Initialization is one-way + * because trie handles can outlive runtime disabling of new trie saves. */ int __stack_depot_trie_side_table_init(gfp_t gfp_flags); -void __stack_depot_trie_side_table_destroy(void); bool __stack_depot_trie_side_table_prealloc_needed(void); int __stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, @@ -162,9 +172,7 @@ u32 __stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc); void __stack_depot_trie_side_table_revoke_latest(u32 id); void __stack_depot_trie_side_table_restore(u32 id, const void *entry); -int __stack_depot_trie_side_table_store(u32 id, const void *entry); const void *__stack_depot_trie_side_table_lookup(u32 id); -size_t __stack_depot_trie_side_table_entries(void); size_t __stack_depot_trie_side_table_bytes(void); size_t __stack_depot_trie_pool_alloc_size(size_t size); void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags); @@ -173,7 +181,6 @@ int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc); -bool __stack_depot_trie_pool_try_rollback(const struct stack_depot_trie_pool_mark *mark); int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req); void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); int @@ -213,7 +220,6 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, unsigned int nr_scratch, const void **tail, u32 *leaf_id); void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); -void __stack_depot_trie_side_prepare_init(struct stack_depot_trie_side_prepare *state); int __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, unsigned int nr_updates, void *ctx); @@ -221,15 +227,6 @@ void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *stat int __stack_depot_frame_run_init(const unsigned long *entries, unsigned int nr_entries, struct stack_depot_frame_run *run); -int __stack_depot_frame_run_write(const struct stack_depot_frame_run *run, - const unsigned long *entries, void *dst, - size_t dst_size, u32 *scratch, - unsigned int nr_scratch); -int __stack_depot_frame_run_read(const struct stack_depot_frame_run *run, - const void *src, size_t src_size, - unsigned long *entries, unsigned int max_entries, - unsigned long *scratch, - unsigned int nr_scratch); size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run); int __stack_depot_trie_node_init(void *storage, size_t storage_size, const void *parent, u32 leaf_id, @@ -252,9 +249,6 @@ int __stack_depot_trie_append_chain(const void *parent, u32 leaf_id, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, const void **head, const void **tail, unsigned int *nr_used); -int __stack_depot_trie_publish_append(struct stack_depot_trie_root *root, - void *parent, const void *head, - void *new_storage, size_t new_storage_size); int __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, const void *parent, const unsigned long *entries, unsigned int nr_entries, @@ -262,17 +256,6 @@ int __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, const void * __stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries); -int __stack_depot_trie_insert_append(struct stack_depot_trie_root *root, - void *parent, u32 leaf_id, - const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, void *new_storage, - size_t new_storage_size, const void **tail, - unsigned int *nr_used); int __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, void *parent, u32 leaf_id, @@ -323,17 +306,6 @@ int __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, void *new_storage, size_t new_storage_size); -int __stack_depot_trie_split_subtree(const void *child, unsigned int matched, - u32 leaf_id, const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, const void **prefix, - const void **tail, unsigned int *nr_used); -const void *__stack_depot_trie_child_array_find(const void *storage, - unsigned long frame); int __stack_depot_trie_child_array_insert(const void *old_storage, const void *child, void *new_storage, size_t new_storage_size); diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 1f8dbf3bdf9fc..f2614acead548 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -4,5476 +4,365 @@ #include #include #include -#include -#include #include -#include #include #include #include #include #include -#include "../stackdepot_internal.h" - -#include - -static int -frame_run_init(const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_frame_run *run) -{ - return __stack_depot_frame_run_init(entries, nr_entries, run); -} - -static int frame_run_write(const struct stack_depot_frame_run *run, - const unsigned long *entries, void *dst, size_t dst_size, - u32 *scratch, unsigned int nr_scratch) -{ - return __stack_depot_frame_run_write(run, entries, dst, dst_size, - scratch, nr_scratch); -} - -static int frame_run_read(const struct stack_depot_frame_run *run, - const void *src, size_t src_size, - unsigned long *entries, unsigned int max_entries, - unsigned long *scratch, unsigned int nr_scratch) -{ - return __stack_depot_frame_run_read(run, src, src_size, entries, - max_entries, scratch, nr_scratch); -} - -static int tnode_init(void *storage, size_t storage_size, const void *parent, - u32 leaf_id, const unsigned long *entries, - unsigned int nr_entries, u32 *scratch, - unsigned int nr_scratch) -{ - return __stack_depot_trie_node_init(storage, storage_size, parent, leaf_id, - entries, nr_entries, scratch, - nr_scratch); -} - -static int -tnode_init_slice(void *storage, size_t storage_size, const void *parent, - u32 leaf_id, const void *src_node, unsigned int start, - unsigned int nr_entries) -{ - return __stack_depot_trie_node_init_slice(storage, storage_size, parent, - leaf_id, src_node, start, nr_entries); -} - -static unsigned int tfetch(const void *leaf, unsigned long *entries, - unsigned int max_entries) -{ - return __stack_depot_trie_fetch_into(leaf, entries, max_entries); -} - -static unsigned int tmatch(const void *node, const unsigned long *entries, - unsigned int nr_entries) -{ - return __stack_depot_trie_node_match(node, entries, nr_entries); -} - -static int -append_chain(const void *parent, u32 leaf_id, const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, - const void **head, const void **tail, unsigned int *nr_used) -{ - return __stack_depot_trie_append_chain(parent, leaf_id, entries, - nr_entries, node_slots, nr_node_slots, child_slots, - nr_child_slots, scratch, nr_scratch, head, tail, nr_used); -} - -static int publish_append(struct stack_depot_trie_root *root, void *parent, - const void *head, void *storage, size_t storage_size) -{ - return __stack_depot_trie_publish_append(root, parent, head, storage, - storage_size); -} - -static int lookup_step(const struct stack_depot_trie_root *root, - const void *parent, const unsigned long *entries, - unsigned int nr_entries, - struct stack_depot_trie_lookup *lookup) -{ - return __stack_depot_trie_lookup_step(root, parent, entries, nr_entries, - lookup); -} - -static const void *find_leaf(const struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries) -{ - return __stack_depot_trie_find_leaf(root, entries, nr_entries); -} - -static int insert_append(struct stack_depot_trie_root *root, void *parent, - u32 leaf_id, const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, void *storage, size_t storage_size, - const void **tail, unsigned int *nr_used) -{ - return __stack_depot_trie_insert_append(root, parent, leaf_id, entries, - nr_entries, node_slots, nr_node_slots, child_slots, - nr_child_slots, scratch, nr_scratch, storage, - storage_size, tail, nr_used); -} - -static int -insert_append_prepare(struct stack_depot_trie_root *root, void *parent, - u32 leaf_id, const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, void *storage, size_t storage_size, - const struct stack_depot_trie_publish_prepare *prepare, - const void **tail, unsigned int *nr_used) -{ - return __stack_depot_trie_insert_append_prepare(root, parent, leaf_id, - entries, nr_entries, node_slots, - nr_node_slots, child_slots, - nr_child_slots, scratch, - nr_scratch, storage, - storage_size, prepare, tail, - nr_used); -} - -static int insert_plan(const struct stack_depot_trie_root *root, - const void *parent, const unsigned long *entries, - unsigned int nr_entries, - struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, size_t *new_storage_size, - unsigned int *nr_used, unsigned int *nr_child_used) -{ - return __stack_depot_trie_insert_plan(root, parent, entries, nr_entries, - node_slots, nr_node_slots, - child_slots, nr_child_slots, - new_storage_size, nr_used, - nr_child_used); -} - -#define STACKDEPOT_TRIE_PREPARE_MAX_UPDATES 2 - -struct stackdepot_trie_prepare_ctx { - struct kunit *test; - const struct stack_depot_trie_child_array **visible; - const struct stack_depot_trie_child_array *expected_visible; - const void *expected_leaf[STACKDEPOT_TRIE_PREPARE_MAX_UPDATES]; - u32 expected_leaf_id[STACKDEPOT_TRIE_PREPARE_MAX_UPDATES]; - unsigned int nr_expected; - unsigned int calls; - int ret; -}; - -static int -stackdepot_trie_prepare(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, void *data) -{ - struct stackdepot_trie_prepare_ctx *ctx = data; - unsigned int i; - - ctx->calls++; - KUNIT_EXPECT_NOT_NULL(ctx->test, updates); - if (!updates) - return -EINVAL; - KUNIT_EXPECT_EQ(ctx->test, nr_updates, ctx->nr_expected); - for (i = 0; i < nr_updates && i < ctx->nr_expected; i++) { - KUNIT_EXPECT_EQ(ctx->test, updates[i].leaf_id, - ctx->expected_leaf_id[i]); - KUNIT_EXPECT_PTR_EQ(ctx->test, updates[i].leaf, - ctx->expected_leaf[i]); - } - if (ctx->visible) - KUNIT_EXPECT_PTR_EQ(ctx->test, *ctx->visible, - ctx->expected_visible); - - return ctx->ret; -} - -static int split_subtree(const void *child, unsigned int matched, u32 leaf_id, - const unsigned long *entries, unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, const void **prefix, - const void **tail, unsigned int *nr_used) -{ - return __stack_depot_trie_split_subtree(child, matched, leaf_id, entries, - nr_entries, node_slots, nr_node_slots, child_slots, - nr_child_slots, scratch, nr_scratch, prefix, tail, nr_used); -} - -static void -trie_node_slot_alloc(struct kunit *test, - struct stack_depot_trie_node_slot *slot, - const unsigned long *entries, unsigned int nr_entries) -{ - struct stack_depot_frame_run run; - - KUNIT_ASSERT_EQ(test, frame_run_init(entries, nr_entries, &run), 0); - KUNIT_ASSERT_EQ(test, run.nr_entries, nr_entries); - slot->size = __stack_depot_trie_node_size(&run); - KUNIT_ASSERT_GT(test, slot->size, (size_t)0); - slot->node = kunit_kzalloc(test, slot->size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, slot->node); -} - -static int child_array_init(void *storage, size_t storage_size, - const void * const *children, unsigned int nr_children) -{ - return __stack_depot_trie_child_array_init(storage, storage_size, children, - nr_children); -} - -static int split_child_array_init(void *storage, size_t storage_size, - const void *old_tail, const void *new_head) -{ - return __stack_depot_trie_split_child_array_init(storage, storage_size, - old_tail, new_head); -} - -static int split_tail_plan(const unsigned long *entries, unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, unsigned int *nr_runs) -{ - return __stack_depot_trie_split_tail_plan(entries, nr_entries, node_slots, - nr_node_slots, child_slots, - nr_child_slots, nr_runs); -} - -static int split_precheck(struct stack_depot_trie_root *root, const void *parent, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, void *new_storage, - size_t new_storage_size) -{ - return __stack_depot_trie_split_precheck(root, parent, node_slots, - nr_node_slots, child_slots, nr_child_slots, - new_storage, new_storage_size); -} - -static int child_array_insert(const void *old_storage, const void *child, - void *new_storage, size_t new_storage_size) -{ - return __stack_depot_trie_child_array_insert(old_storage, child, - new_storage, new_storage_size); -} - -static const void *child_array_find(const void *storage, unsigned long frame) -{ - return __stack_depot_trie_child_array_find(storage, frame); -} - -static void -trie_node_alloc(struct kunit *test, const unsigned long *entries, - unsigned int nr_entries, const void *parent, u32 leaf_id, - void **node) -{ - u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; - struct stack_depot_frame_run run; - size_t size; - int ret; - - /* This helper allocates one trie node, so @entries must form one run. */ - KUNIT_ASSERT_EQ(test, frame_run_init(entries, nr_entries, &run), 0); - size = __stack_depot_trie_node_size(&run); - KUNIT_ASSERT_GT(test, size, (size_t)0); - *node = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, *node); - ret = tnode_init(*node, size, parent, leaf_id, entries, nr_entries, - write_scratch, ARRAY_SIZE(write_scratch)); - KUNIT_ASSERT_EQ(test, ret, 0); -} - -static void trie_fill_raw_entries(unsigned long *entries, unsigned int nr_entries, - unsigned long base) -{ - unsigned int i; - - for (i = 0; i < nr_entries; i++) - entries[i] = base + i * 0x10UL; -} - -static void stackdepot_fetch_into_roundtrip(struct kunit *test) -{ - unsigned long entries[] = { - 0x1234567800010000UL, - 0x1234567800020000UL, - 0x1234567800030000UL, - }; - unsigned long exact[ARRAY_SIZE(entries)] = {}; - unsigned long fetched[ARRAY_SIZE(entries) + 1] = { - [ARRAY_SIZE(entries)] = 0xa5a5a5a5a5a5a5a5UL, - }; - unsigned long expected_tail = fetched[ARRAY_SIZE(entries)]; - depot_stack_handle_t handle; - unsigned int nr_entries; - - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - - handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - - nr_entries = - stack_depot_fetch_into(handle, exact, ARRAY_SIZE(exact)); - KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, exact, entries, sizeof(entries)); - - nr_entries = - stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); - KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); - KUNIT_EXPECT_EQ(test, fetched[ARRAY_SIZE(entries)], expected_tail); -} - -static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) -{ - unsigned long entries[] = { - 0x1234567800110000UL, - 0x1234567800120000UL, - 0x1234567800130000UL, - }; - unsigned long fetched[ARRAY_SIZE(entries)] = { - 0xa1a1a1a1a1a1a1a1UL, - 0xb2b2b2b2b2b2b2b2UL, - 0xc3c3c3c3c3c3c3c3UL, - }; - unsigned long expected[ARRAY_SIZE(fetched)]; - depot_stack_handle_t handle; - unsigned int nr_entries; - - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - - handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - memcpy(expected, fetched, sizeof(expected)); - - nr_entries = stack_depot_fetch_into(0, fetched, ARRAY_SIZE(fetched)); - KUNIT_EXPECT_EQ(test, nr_entries, 0); - KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); - - nr_entries = stack_depot_fetch_into(0, NULL, 0); - KUNIT_EXPECT_EQ(test, nr_entries, 0); - /* No buffer is supplied for this invalid-input combination. */ - - nr_entries = stack_depot_fetch_into(handle, NULL, ARRAY_SIZE(fetched)); - KUNIT_EXPECT_EQ(test, nr_entries, 0); - KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); - - nr_entries = stack_depot_fetch_into(handle, fetched, 0); - KUNIT_EXPECT_EQ(test, nr_entries, 0); - KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); - - nr_entries = stack_depot_fetch_into(handle, fetched, - ARRAY_SIZE(fetched) - 1); - KUNIT_EXPECT_EQ(test, nr_entries, 0); - KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); -} - -static void stackdepot_count_helpers(struct kunit *test) -{ - unsigned long entries[] = { - 0x1234567800210000UL, - 0x1234567800220000UL, - 0x1234567800230000UL, - }; - unsigned long zero_entries[] = { - 0x1234567800310000UL, - 0x1234567800320000UL, - 0x1234567800330000UL, - }; - unsigned long zeroed_entries[] = { - 0x1234567800610000UL, - 0x1234567800620000UL, - 0x1234567800630000UL, - }; - unsigned long seeded_entries[] = { - 0x1234567800410000UL, - 0x1234567800420000UL, - 0x1234567800430000UL, - }; - unsigned long max_entries[] = { - 0x1234567800510000UL, - 0x1234567800520000UL, - 0x1234567800530000UL, - }; - depot_stack_handle_t handle; - depot_stack_handle_t second_handle; - depot_stack_handle_t seeded_handle; - depot_stack_handle_t max_handle; - depot_stack_handle_t zeroed_handle; - unsigned int zeroed_nr = ARRAY_SIZE(zeroed_entries); - bool new_count; - unsigned int count; - - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - - KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(0, &count)); - KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(0, NULL)); - __stack_depot_set_count(0, 1); - __stack_depot_set_count(0, 0); - __stack_depot_set_count(0, INT_MAX); - KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(0, 1, &new_count)); - KUNIT_EXPECT_FALSE(test, - __stack_depot_inc_count(0, INT_MAX - 2, &new_count)); - KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(0, 1)); - - handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - - KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_FALSE(test, - __stack_depot_inc_count(handle, INT_MAX, &new_count)); - KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 1)); - KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); - - max_handle = stack_depot_save(max_entries, ARRAY_SIZE(max_entries), GFP_KERNEL); - KUNIT_ASSERT_NE(test, max_handle, (depot_stack_handle_t)0); - new_count = false; - KUNIT_EXPECT_TRUE(test, - __stack_depot_inc_count(max_handle, INT_MAX - 1, - &new_count)); - KUNIT_EXPECT_TRUE(test, new_count); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(max_handle, &count)); - KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX); - new_count = true; - KUNIT_EXPECT_FALSE(test, - __stack_depot_inc_count(max_handle, 1, &new_count)); - KUNIT_EXPECT_FALSE(test, new_count); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(max_handle, &count)); - KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX); - KUNIT_EXPECT_TRUE(test, - __stack_depot_dec_count_and_test(max_handle, INT_MAX)); - KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(max_handle, &count)); - - new_count = false; - KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 2, &new_count)); - KUNIT_EXPECT_TRUE(test, new_count); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 3); - - /* Already-counted records increment without needing a list marker. */ - new_count = true; - KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 4, &new_count)); - KUNIT_EXPECT_FALSE(test, new_count); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 7); - - KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 5)); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 2); - KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 3)); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 2); - __stack_depot_set_count(handle, 0); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 2); - __stack_depot_set_count(handle, INT_MAX - 1); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX - 1); - __stack_depot_set_count(handle, 2); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 2); - __stack_depot_set_count(handle, INT_MAX); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX); - __stack_depot_set_count(handle, 2); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 2); - __stack_depot_set_count(handle, 6); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 6); - KUNIT_EXPECT_FALSE(test, - __stack_depot_inc_count(handle, INT_MAX, &new_count)); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 6); - - second_handle = stack_depot_save(zero_entries, ARRAY_SIZE(zero_entries), - GFP_KERNEL); - KUNIT_ASSERT_NE(test, second_handle, (depot_stack_handle_t)0); - new_count = false; - KUNIT_EXPECT_TRUE(test, - __stack_depot_inc_count(second_handle, 1, &new_count)); - KUNIT_EXPECT_TRUE(test, new_count); - KUNIT_EXPECT_FALSE(test, - __stack_depot_dec_count_and_test(second_handle, 1)); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(second_handle, &count)); - KUNIT_EXPECT_EQ(test, count, 1); - - seeded_handle = stack_depot_save(seeded_entries, ARRAY_SIZE(seeded_entries), - GFP_KERNEL); - KUNIT_ASSERT_NE(test, seeded_handle, (depot_stack_handle_t)0); - __stack_depot_set_count(seeded_handle, 3); - KUNIT_EXPECT_FALSE(test, - __stack_depot_dec_count_and_test(seeded_handle, 1)); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(seeded_handle, &count)); - KUNIT_EXPECT_EQ(test, count, 2); - - zeroed_handle = stack_depot_save(zeroed_entries, zeroed_nr, GFP_KERNEL); - KUNIT_ASSERT_NE(test, zeroed_handle, (depot_stack_handle_t)0); - __stack_depot_set_count(zeroed_handle, 3); - KUNIT_EXPECT_TRUE(test, __stack_depot_dec_count_and_test(zeroed_handle, 3)); - KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(zeroed_handle, 1)); -} - -static void stackdepot_trie_handle_namespace(struct kunit *test) -{ - unsigned long entries[] = { - 0x1234567800710000UL, - 0x1234567800720000UL, - 0x1234567800730000UL, - }; - depot_stack_handle_t boundary_handle; - depot_stack_handle_t extra_only; - depot_stack_handle_t hash_handle; - depot_stack_handle_t tagged; - depot_stack_handle_t trie; - u32 boundary_id; - u32 max_id; - - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - - trie = __stack_depot_trie_handle(1); - hash_handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); - KUNIT_EXPECT_NE(test, hash_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(hash_handle), 0U); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_handle(0), (depot_stack_handle_t)0); - extra_only = (depot_stack_handle_t)7 << - (DEPOT_HANDLE_BITS - STACK_DEPOT_EXTRA_BITS); - KUNIT_EXPECT_EQ(test, stack_depot_set_extra_bits(extra_only, 1), - (depot_stack_handle_t)0); - - if (!trie) { - KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(0), 0U); - return; - } - - boundary_id = (1U << DEPOT_OFFSET_BITS) + 2; - max_id = __stack_depot_trie_max_leaf_id(); - boundary_handle = __stack_depot_trie_handle(boundary_id); - tagged = stack_depot_set_extra_bits(trie, 7); - - KUNIT_EXPECT_NE(test, boundary_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(trie), 1U); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(tagged), 1U); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(boundary_handle), - boundary_id); - KUNIT_EXPECT_NE(test, max_id, 0U); - KUNIT_EXPECT_NE(test, __stack_depot_trie_handle(max_id), - (depot_stack_handle_t)0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_handle(max_id + 1), - (depot_stack_handle_t)0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_handle(U32_MAX), - (depot_stack_handle_t)0); -} - -static void stackdepot_trie_disable_action(void *data) -{ - __stack_depot_trie_set_enabled(false); -} - -static void stackdepot_trie_add_disable_action(struct kunit *test) -{ - int ret; - - ret = kunit_add_action_or_reset(test, stackdepot_trie_disable_action, NULL); - KUNIT_ASSERT_EQ(test, ret, 0); -} - -static void stackdepot_trie_feature_flag(struct kunit *test) -{ - stackdepot_trie_add_disable_action(test); - - KUNIT_EXPECT_FALSE(test, __stack_depot_trie_enabled()); - - __stack_depot_trie_set_enabled(true); - KUNIT_EXPECT_TRUE(test, __stack_depot_trie_enabled()); - - __stack_depot_trie_set_enabled(true); - KUNIT_EXPECT_TRUE(test, __stack_depot_trie_enabled()); - - __stack_depot_trie_set_enabled(false); - KUNIT_EXPECT_FALSE(test, __stack_depot_trie_enabled()); -} - -static void stackdepot_trie_late_init(struct kunit *test) -{ - stackdepot_trie_add_disable_action(test); - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - KUNIT_EXPECT_FALSE(test, __stack_depot_trie_ready()); - if (!__stack_depot_trie_max_leaf_id()) - kunit_skip(test, "trie handle namespace unavailable"); - - __stack_depot_trie_set_enabled(true); - KUNIT_EXPECT_EQ(test, stack_depot_init(), 0); - KUNIT_EXPECT_TRUE(test, __stack_depot_trie_ready()); -} - -static void stackdepot_trie_side_table_destroy_action(void *data) -{ - __stack_depot_trie_side_table_destroy(); -} - -static void stackdepot_trie_side_table_init_or_skip(struct kunit *test) -{ - int ret; - - ret = __stack_depot_trie_side_table_init(GFP_KERNEL); - if (ret == -EINVAL && !__stack_depot_trie_max_leaf_id()) - kunit_skip(test, "trie handle namespace unavailable"); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = kunit_add_action_or_reset(test, stackdepot_trie_side_table_destroy_action, NULL); - KUNIT_ASSERT_EQ(test, ret, 0); -} - -static void -stackdepot_trie_side_table_prealloc_or_fail(struct kunit *test, - struct stack_depot_trie_side_prealloc *prealloc) -{ - int ret; - - ret = __stack_depot_trie_side_table_prealloc(GFP_KERNEL, prealloc); - KUNIT_ASSERT_EQ(test, ret, 0); -} - -static u32 stackdepot_trie_side_table_alloc(struct kunit *test) -{ - struct stack_depot_trie_side_prealloc prealloc = {}; - u32 id; - - if (__stack_depot_trie_side_table_prealloc_needed()) { - stackdepot_trie_side_table_prealloc_or_fail(test, &prealloc); - KUNIT_ASSERT_TRUE(test, prealloc.dir || prealloc.chunk); - } - - id = __stack_depot_trie_side_table_alloc_id(&prealloc); - __stack_depot_trie_side_table_free_prealloc(&prealloc); - return id; -} - -static void stackdepot_trie_side_table_destroy_uninit(struct kunit *test) -{ - __stack_depot_trie_side_table_destroy(); - KUNIT_SUCCEED(test); -} - -static void stackdepot_trie_side_table_alloc_store_lookup(struct kunit *test) -{ - const void *entry1 = (const void *)0x1111UL; - const void *entry2 = (const void *)0x2222UL; - u32 id1; - u32 id2; - - stackdepot_trie_side_table_init_or_skip(test); - id1 = stackdepot_trie_side_table_alloc(test); - id2 = stackdepot_trie_side_table_alloc(test); - - KUNIT_ASSERT_EQ(test, id1, 1U); - KUNIT_ASSERT_EQ(test, id2, 2U); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store(id1, entry1), 0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store(id2, entry2), 0); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), entry1); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id2), entry2); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 2UL); -} - -static void stackdepot_trie_side_table_rejects_invalid_ids(struct kunit *test) -{ - int ret; - u32 id; - - stackdepot_trie_side_table_init_or_skip(test); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(0)); - - id = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, id, 1U); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id + 1)); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_store(id, NULL), -EINVAL); - ret = __stack_depot_trie_side_table_store(id + 1, (const void *)0x1UL); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} - -static void stackdepot_trie_side_table_revoke_latest(struct kunit *test) -{ - const void *entry = (const void *)0xaaaaUL; - size_t bytes; - int ret; - u32 id; - - stackdepot_trie_side_table_init_or_skip(test); - id = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, id, 1U); - ret = __stack_depot_trie_side_table_store(id, entry); - KUNIT_ASSERT_EQ(test, ret, 0); - bytes = __stack_depot_trie_side_table_bytes(); - KUNIT_EXPECT_GT(test, bytes, 0UL); - - __stack_depot_trie_side_table_revoke_latest(id); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_bytes(), bytes); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id)); -} - -static void stackdepot_trie_side_table_revoke_keeps_chunk(struct kunit *test) -{ - const void *entry1 = (const void *)0x1111UL; - const void *entry2 = (const void *)0x2222UL; - size_t bytes; - u32 id1; - u32 id2; - - stackdepot_trie_side_table_init_or_skip(test); - id1 = stackdepot_trie_side_table_alloc(test); - id2 = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, id1, 1U); - KUNIT_ASSERT_EQ(test, id2, 2U); - KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id1, entry1), 0); - KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id2, entry2), 0); - bytes = __stack_depot_trie_side_table_bytes(); - - __stack_depot_trie_side_table_revoke_latest(id2); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_bytes(), bytes); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), entry1); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id2)); -} - -static void stackdepot_trie_side_table_restore(struct kunit *test) -{ - const void *entry1 = (const void *)0xaaaaUL; - const void *entry2 = (const void *)0xbbbbUL; - u32 id; - - stackdepot_trie_side_table_init_or_skip(test); - id = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, id, 1U); - KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, entry1), 0); - KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, entry2), 0); - - __stack_depot_trie_side_table_restore(id, entry1); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id), entry1); - __stack_depot_trie_side_table_restore(id, NULL); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(id)); -} - -static void stackdepot_trie_side_table_chunk_boundary(struct kunit *test) -{ - struct stack_depot_trie_side_prealloc prealloc = {}; - u32 id = 0; - u32 i; - - stackdepot_trie_side_table_init_or_skip(test); - for (i = 0; i < STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE; i++) - id = stackdepot_trie_side_table_alloc(test); - - KUNIT_ASSERT_EQ(test, id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_side_table_prealloc_needed()); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_alloc_id(NULL), 0U); - - stackdepot_trie_side_table_prealloc_or_fail(test, &prealloc); - KUNIT_ASSERT_NOT_NULL(test, prealloc.chunk); - id = __stack_depot_trie_side_table_alloc_id(&prealloc); - KUNIT_EXPECT_NULL(test, prealloc.dir); - KUNIT_EXPECT_NULL(test, prealloc.chunk); - KUNIT_EXPECT_EQ(test, id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE + 1); -} - -static void stackdepot_trie_side_table_bytes(struct kunit *test) -{ - struct stack_depot_trie_side_prealloc prealloc = {}; - size_t before; - size_t after; - u32 id; - - stackdepot_trie_side_table_init_or_skip(test); - before = __stack_depot_trie_side_table_bytes(); - KUNIT_EXPECT_GT(test, before, 0UL); - stackdepot_trie_side_table_prealloc_or_fail(test, &prealloc); - KUNIT_ASSERT_TRUE(test, prealloc.dir || prealloc.chunk); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_bytes(), before); - - id = __stack_depot_trie_side_table_alloc_id(&prealloc); - KUNIT_EXPECT_NULL(test, prealloc.dir); - KUNIT_EXPECT_NULL(test, prealloc.chunk); - KUNIT_ASSERT_EQ(test, id, 1U); - after = __stack_depot_trie_side_table_bytes(); - KUNIT_EXPECT_GT(test, after, before); -} - -static void stackdepot_trie_side_prepare_updates(struct kunit *test) -{ - struct stack_depot_trie_side_prepare state; - struct stack_depot_trie_leaf_update updates[2]; - const void *old1 = (const void *)0x1111UL; - const void *old2 = (const void *)0x2222UL; - const void *new1 = (const void *)0xaaaaUL; - const void *new2 = (const void *)0xbbbbUL; - int ret; - u32 id1; - u32 id2; - - stackdepot_trie_side_table_init_or_skip(test); - id1 = stackdepot_trie_side_table_alloc(test); - id2 = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, id1, 1U); - KUNIT_ASSERT_EQ(test, id2, 2U); - KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id1, old1), 0); - KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id2, old2), 0); - updates[0].leaf_id = id1; - updates[0].leaf = new1; - updates[1].leaf_id = id2; - updates[1].leaf = new2; - - __stack_depot_trie_side_prepare_init(&state); - ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &state); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, state.nr_updates, (unsigned int)ARRAY_SIZE(updates)); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), new1); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id2), new2); - - __stack_depot_trie_side_rollback(&state); - KUNIT_EXPECT_EQ(test, state.nr_updates, 0U); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), old1); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id2), old2); -} - -static void stackdepot_trie_side_prepare_failure(struct kunit *test) -{ - struct stack_depot_trie_side_prepare state; - struct stack_depot_trie_leaf_update updates[2]; - const void *old1 = (const void *)0x1111UL; - const void *new1 = (const void *)0xaaaaUL; - int ret; - u32 id; - - stackdepot_trie_side_table_init_or_skip(test); - id = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, id, 1U); - KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, old1), 0); - updates[0].leaf_id = id; - updates[0].leaf = new1; - updates[1].leaf_id = id + 1; - updates[1].leaf = (const void *)0xbbbbUL; - - __stack_depot_trie_side_prepare_init(&state); - ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &state); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_EQ(test, state.nr_updates, 0U); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id), old1); -} - -static void stackdepot_trie_side_prepare_duplicate_id(struct kunit *test) -{ - struct stack_depot_trie_side_prepare state; - struct stack_depot_trie_leaf_update updates[2]; - const void *old = (const void *)0x1111UL; - const void *mid = (const void *)0x2222UL; - const void *new = (const void *)0x3333UL; - int ret; - u32 id; - - stackdepot_trie_side_table_init_or_skip(test); - id = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, id, 1U); - KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id, old), 0); - updates[0].leaf_id = id; - updates[0].leaf = mid; - updates[1].leaf_id = id; - updates[1].leaf = new; - - __stack_depot_trie_side_prepare_init(&state); - ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &state); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id), new); - __stack_depot_trie_side_rollback(&state); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id), old); -} - -static void stackdepot_trie_side_prepare_rejects_extra_update(struct kunit *test) -{ - struct stack_depot_trie_side_prepare state; - struct stack_depot_trie_leaf_update updates[3]; - const void *old[] = { - (const void *)0x1111UL, - (const void *)0x2222UL, - (const void *)0x3333UL, - }; - const void *new[] = { - (const void *)0xaaaaUL, - (const void *)0xbbbbUL, - (const void *)0xccccUL, - }; - u32 id[ARRAY_SIZE(updates)]; - unsigned int i; - int ret; - - stackdepot_trie_side_table_init_or_skip(test); - for (i = 0; i < ARRAY_SIZE(updates); i++) { - id[i] = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, id[i], i + 1); - KUNIT_ASSERT_EQ(test, - __stack_depot_trie_side_table_store(id[i], old[i]), - 0); - updates[i].leaf_id = id[i]; - updates[i].leaf = new[i]; - } - - __stack_depot_trie_side_prepare_init(&state); - ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &state); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_EQ(test, state.nr_updates, 0U); - for (i = 0; i < ARRAY_SIZE(updates); i++) - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id[i]), - old[i]); -} - -static void stackdepot_trie_side_prepare_rejects_null_leaf(struct kunit *test) -{ - struct stack_depot_trie_side_prepare state; - struct stack_depot_trie_leaf_update updates[2]; - const void *old1 = (const void *)0x1111UL; - const void *old2 = (const void *)0x2222UL; - const void *new1 = (const void *)0xaaaaUL; - u32 id1; - u32 id2; - int ret; - - stackdepot_trie_side_table_init_or_skip(test); - id1 = stackdepot_trie_side_table_alloc(test); - id2 = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, id1, 1U); - KUNIT_ASSERT_EQ(test, id2, 2U); - KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id1, old1), 0); - KUNIT_ASSERT_EQ(test, __stack_depot_trie_side_table_store(id2, old2), 0); - updates[0].leaf_id = id1; - updates[0].leaf = new1; - updates[1].leaf_id = id2; - updates[1].leaf = NULL; - - __stack_depot_trie_side_prepare_init(&state); - ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &state); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_EQ(test, state.nr_updates, 0U); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id1), old1); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(id2), old2); -} - -static void stackdepot_trie_pool_alloc_size(struct kunit *test) -{ - size_t align = 1UL << DEPOT_STACK_ALIGN; - - KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(0), 0UL); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(1), align); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(sizeof(unsigned long)), - align); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(align), align); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(align + 1), - align * 2); - KUNIT_EXPECT_EQ(test, - __stack_depot_trie_pool_alloc_size(DEPOT_POOL_SIZE - 1), - (size_t)DEPOT_POOL_SIZE); - KUNIT_EXPECT_EQ(test, - __stack_depot_trie_pool_alloc_size(DEPOT_POOL_SIZE), - (size_t)DEPOT_POOL_SIZE); - KUNIT_EXPECT_EQ(test, - __stack_depot_trie_pool_alloc_size(DEPOT_POOL_SIZE + 1), - 0UL); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_pool_alloc_size(SIZE_MAX), 0UL); -} - -static void stackdepot_trie_pool_prealloc(struct kunit *test) -{ - void *prealloc; - - KUNIT_EXPECT_NULL(test, __stack_depot_trie_pool_prealloc(0)); - prealloc = __stack_depot_trie_pool_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, prealloc); - KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)prealloc, PAGE_SIZE)); - __stack_depot_trie_pool_free_prealloc(prealloc); - __stack_depot_trie_pool_free_prealloc(NULL); -} - -static int alloc_prealloc_flags(gfp_t gfp_flags, depot_flags_t depot_flags, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) -{ - return __stack_depot_trie_alloc_prealloc(gfp_flags, depot_flags, - pool_prealloc, side_prealloc); -} - -static int alloc_prealloc(gfp_t gfp_flags, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) -{ - return alloc_prealloc_flags(gfp_flags, STACK_DEPOT_FLAG_CAN_ALLOC, - pool_prealloc, side_prealloc); -} - -static void stackdepot_trie_alloc_prealloc(struct kunit *test) -{ - struct stack_depot_trie_side_prealloc side_prealloc = {}; - void *pool_prealloc = NULL; - u32 id; - int ret; - - stackdepot_trie_side_table_init_or_skip(test); - ret = alloc_prealloc_flags(GFP_NOWAIT, 0, &pool_prealloc, &side_prealloc); - KUNIT_EXPECT_EQ(test, ret, -ENOSPC); - KUNIT_EXPECT_NULL(test, pool_prealloc); - KUNIT_EXPECT_NULL(test, side_prealloc.dir); - KUNIT_EXPECT_NULL(test, side_prealloc.chunk); - - ret = alloc_prealloc(GFP_KERNEL, &pool_prealloc, &side_prealloc); - KUNIT_EXPECT_EQ(test, ret, 0); - KUNIT_EXPECT_TRUE(test, side_prealloc.dir || side_prealloc.chunk); - __stack_depot_trie_pool_free_prealloc(pool_prealloc); - __stack_depot_trie_side_table_free_prealloc(&side_prealloc); - pool_prealloc = NULL; - - id = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, id, 1U); - ret = alloc_prealloc(GFP_NOWAIT, &pool_prealloc, &side_prealloc); - KUNIT_EXPECT_EQ(test, ret, 0); - KUNIT_EXPECT_NULL(test, pool_prealloc); - KUNIT_EXPECT_NULL(test, side_prealloc.dir); - KUNIT_EXPECT_NULL(test, side_prealloc.chunk); - - pool_prealloc = (void *)0x1111UL; - ret = alloc_prealloc(GFP_KERNEL, &pool_prealloc, &side_prealloc); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} - -static void stackdepot_trie_pool_seed_current_pool(struct kunit *test) -{ - unsigned long entries[] = { 0x1234567800990000UL }; - depot_stack_handle_t handle; - - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); -} - -static void * -stackdepot_trie_pool_carve_node(size_t size, - struct stack_depot_trie_pool_mark *mark) -{ - struct stack_depot_trie_node_slot slot = { .size = size }; - void *storage = NULL; - struct stack_depot_trie_pool_request req = { - .node_slots = &slot, - .nr_node_slots = 1, - .storage = &storage, - .mark = mark, - }; - - return __stack_depot_trie_pool_carve(&req) ? NULL : slot.node; -} - -static void stackdepot_trie_pool_carve_node_test(struct kunit *test) -{ - struct stack_depot_trie_pool_mark first; - struct stack_depot_trie_pool_mark second; - void *ptr1; - void *ptr2; - size_t align = 1UL << DEPOT_STACK_ALIGN; - - stackdepot_trie_pool_seed_current_pool(test); - ptr1 = stackdepot_trie_pool_carve_node(1, &first); - KUNIT_ASSERT_NOT_NULL(test, ptr1); - KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)ptr1, align)); - KUNIT_EXPECT_EQ(test, first.size, align); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&first)); - - ptr2 = stackdepot_trie_pool_carve_node(1, &second); - KUNIT_ASSERT_NOT_NULL(test, ptr2); - KUNIT_EXPECT_PTR_EQ(test, ptr2, ptr1); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&second)); - KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(&first)); -} - -static void stackdepot_trie_pool_rollback_requires_lifo(struct kunit *test) -{ - struct stack_depot_trie_pool_mark first; - struct stack_depot_trie_pool_mark second; - void *ptr1; - void *ptr2; - - stackdepot_trie_pool_seed_current_pool(test); - ptr1 = stackdepot_trie_pool_carve_node(1, &first); - KUNIT_ASSERT_NOT_NULL(test, ptr1); - ptr2 = stackdepot_trie_pool_carve_node(1, &second); - KUNIT_ASSERT_NOT_NULL(test, ptr2); - - KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(&first)); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&second)); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&first)); -} - -static void stackdepot_trie_pool_carve_node_rejects_bad_inputs(struct kunit *test) -{ - struct stack_depot_trie_pool_mark mark; - void *ptr; - - stackdepot_trie_pool_seed_current_pool(test); - KUNIT_EXPECT_NULL(test, stackdepot_trie_pool_carve_node(0, &mark)); - KUNIT_EXPECT_EQ(test, mark.size, 0UL); - ptr = stackdepot_trie_pool_carve_node(DEPOT_POOL_SIZE + 1, &mark); - KUNIT_EXPECT_NULL(test, ptr); - KUNIT_EXPECT_EQ(test, mark.size, 0UL); - KUNIT_EXPECT_NULL(test, stackdepot_trie_pool_carve_node(1, NULL)); - KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(NULL)); - memset(&mark, 0, sizeof(mark)); - KUNIT_EXPECT_FALSE(test, __stack_depot_trie_pool_try_rollback(&mark)); -} - -static void stackdepot_trie_pool_carve_slots(struct kunit *test) -{ - struct stack_depot_trie_child_array_slot child_slots[1] = { - { .size = 1 }, - }; - struct stack_depot_trie_node_slot node_slots[2] = { - { .size = 1 }, - { .size = (1UL << DEPOT_STACK_ALIGN) + 1 }, - }; - struct stack_depot_trie_pool_mark mark; - void *storage = NULL; - struct stack_depot_trie_pool_request req = { - .node_slots = node_slots, - .nr_node_slots = ARRAY_SIZE(node_slots), - .child_slots = child_slots, - .nr_child_slots = ARRAY_SIZE(child_slots), - .storage = &storage, - .storage_size = 1, - .mark = &mark, - }; - size_t child_size; - size_t node0_size; - size_t node1_size; - size_t old_total; - void *again; - int ret; - - stackdepot_trie_pool_seed_current_pool(test); - ret = __stack_depot_trie_pool_carve(&req); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_ASSERT_NOT_NULL(test, node_slots[0].node); - KUNIT_ASSERT_NOT_NULL(test, node_slots[1].node); - KUNIT_ASSERT_NOT_NULL(test, child_slots[0].array); - KUNIT_ASSERT_NOT_NULL(test, storage); - - node0_size = __stack_depot_trie_pool_alloc_size(node_slots[0].size); - node1_size = __stack_depot_trie_pool_alloc_size(node_slots[1].size); - child_size = __stack_depot_trie_pool_alloc_size(child_slots[0].size); - old_total = node0_size + node1_size + child_size + - __stack_depot_trie_pool_alloc_size(1); - KUNIT_EXPECT_PTR_EQ(test, node_slots[1].node, - (char *)node_slots[0].node + node0_size); - KUNIT_EXPECT_GT(test, (unsigned long)child_slots[0].array, - (unsigned long)node_slots[1].node + node1_size); - KUNIT_EXPECT_GT(test, (unsigned long)storage, - (unsigned long)child_slots[0].array + child_size); - KUNIT_EXPECT_GT(test, mark.size, old_total); - - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&mark)); - again = stackdepot_trie_pool_carve_node(1, &mark); - KUNIT_ASSERT_NOT_NULL(test, again); - KUNIT_EXPECT_PTR_EQ(test, again, node_slots[0].node); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&mark)); -} - -static void stackdepot_trie_pool_carve_slots_rejects_bad_inputs(struct kunit *test) -{ - struct stack_depot_trie_child_array_slot child_slot = { .size = 1 }; - struct stack_depot_trie_node_slot node_slot = { .size = 1 }; - struct stack_depot_trie_pool_mark mark; - void *storage = (void *)0x1UL; - struct stack_depot_trie_pool_request req = { - .node_slots = &node_slot, - .nr_node_slots = 1, - .child_slots = &child_slot, - .nr_child_slots = 1, - .storage = &storage, - .storage_size = 1, - .mark = &mark, - }; - int ret; - - stackdepot_trie_pool_seed_current_pool(test); - ret = __stack_depot_trie_pool_carve(&req); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_EQ(test, mark.size, 0UL); - - storage = NULL; - node_slot.node = (void *)0x1UL; - ret = __stack_depot_trie_pool_carve(&req); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_EQ(test, mark.size, 0UL); - - node_slot.node = NULL; - req.storage_size = DEPOT_POOL_SIZE; - ret = __stack_depot_trie_pool_carve(&req); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_EQ(test, mark.size, 0UL); -} - -static void stackdepot_trie_pool_carve_uses_prealloc(struct kunit *test) -{ - struct stack_depot_trie_pool_mark first_mark; - struct stack_depot_trie_pool_mark second_mark; - void *first_storage = NULL; - void *second_storage = NULL; - void *prealloc; - size_t storage_size = DEPOT_POOL_SIZE - 64; - struct stack_depot_trie_pool_request first = { - .storage = &first_storage, - .storage_size = DEPOT_POOL_SIZE - 64, - .prealloc = &prealloc, - .mark = &first_mark, - }; - struct stack_depot_trie_pool_request second = { - .storage = &second_storage, - .storage_size = DEPOT_POOL_SIZE - 64, - .mark = &second_mark, - }; - int ret; - - stackdepot_trie_pool_seed_current_pool(test); - KUNIT_ASSERT_GT(test, storage_size, 0UL); - prealloc = __stack_depot_trie_pool_prealloc(GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, prealloc); - - ret = __stack_depot_trie_pool_carve(&first); - KUNIT_ASSERT_EQ(test, ret, 0); - __stack_depot_trie_pool_free_prealloc(prealloc); - KUNIT_ASSERT_NOT_NULL(test, first_storage); - KUNIT_ASSERT_TRUE(test, first_mark.added_pool); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&first_mark)); - - ret = __stack_depot_trie_pool_carve(&second); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, second_storage, first_storage); - KUNIT_ASSERT_TRUE(test, second_mark.added_pool); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&second_mark)); -} - -static void stackdepot_trie_pool_carve_no_prealloc_rollover(struct kunit *test) -{ - struct stack_depot_trie_pool_mark marks[2]; - void *storage[ARRAY_SIZE(marks)]; - unsigned int consumed = 0; - void *failed_storage = NULL; - struct stack_depot_trie_pool_mark failed_mark; - struct stack_depot_trie_pool_request failed = { - .storage = &failed_storage, - .storage_size = 1, - .mark = &failed_mark, - }; - unsigned int i; - int ret; - - stackdepot_trie_pool_seed_current_pool(test); - for (i = 0; i < ARRAY_SIZE(marks); i++) { - size_t storage_size = DEPOT_POOL_SIZE - 64; - struct stack_depot_trie_pool_request req = { - .storage = &storage[i], - .storage_size = DEPOT_POOL_SIZE - 64, - .mark = &marks[i], - }; - - KUNIT_ASSERT_GT(test, storage_size, 0UL); - storage[i] = NULL; - ret = __stack_depot_trie_pool_carve(&req); - if (ret) - break; - KUNIT_ASSERT_NOT_NULL(test, storage[i]); - consumed++; - } - - failed.storage_size = DEPOT_POOL_SIZE - 64; - ret = __stack_depot_trie_pool_carve(&failed); - KUNIT_EXPECT_EQ(test, ret, -ENOSPC); - KUNIT_EXPECT_NULL(test, failed_storage); - KUNIT_EXPECT_EQ(test, failed_mark.size, 0UL); - - while (consumed--) - KUNIT_ASSERT_TRUE(test, - __stack_depot_trie_pool_try_rollback(&marks[consumed])); -} - -static void stackdepot_trie_alloc_txn_id(struct kunit *test) -{ - struct stack_depot_trie_side_prealloc prealloc = {}; - struct stack_depot_trie_alloc_txn txn; - int ret; - - stackdepot_trie_side_table_init_or_skip(test); - if (__stack_depot_trie_side_table_prealloc_needed()) - stackdepot_trie_side_table_prealloc_or_fail(test, &prealloc); - - __stack_depot_trie_alloc_txn_init(&txn); - ret = __stack_depot_trie_alloc_txn_id(&txn, &prealloc); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_NULL(test, prealloc.dir); - KUNIT_EXPECT_NULL(test, prealloc.chunk); - KUNIT_EXPECT_EQ(test, txn.leaf_id, 1U); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); - ret = __stack_depot_trie_alloc_txn_id(&txn, NULL); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - - __stack_depot_trie_alloc_txn_rollback(&txn); - KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); -} - -static void stackdepot_trie_alloc_txn_reserve(struct kunit *test) -{ - struct stack_depot_trie_node_slot node_slot = { .size = 1 }; - struct stack_depot_trie_side_prealloc side_prealloc = {}; - struct stack_depot_trie_alloc_txn txn; - struct stack_depot_trie_alloc_request req; - void *storage = NULL; - int ret; - - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - if (__stack_depot_trie_side_table_prealloc_needed()) - stackdepot_trie_side_table_prealloc_or_fail(test, &side_prealloc); - - __stack_depot_trie_alloc_txn_init(&txn); - req = (struct stack_depot_trie_alloc_request) { - .txn = &txn, - .node_slots = &node_slot, - .nr_node_slots = 1, - .storage = &storage, - .storage_size = 1, - .side_prealloc = &side_prealloc, - }; - ret = __stack_depot_trie_alloc_txn_reserve(&req); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_NULL(test, side_prealloc.dir); - KUNIT_EXPECT_NULL(test, side_prealloc.chunk); - KUNIT_EXPECT_EQ(test, txn.leaf_id, 1U); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); - KUNIT_EXPECT_NOT_NULL(test, node_slot.node); - KUNIT_EXPECT_NOT_NULL(test, storage); - - __stack_depot_trie_alloc_txn_rollback(&txn); - KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); - KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 0UL); -} - -static void stackdepot_trie_alloc_txn_reserve_id_failure(struct kunit *test) -{ - struct stack_depot_trie_node_slot node_slot = { .size = 1 }; - struct stack_depot_trie_alloc_txn txn; - struct stack_depot_trie_pool_mark mark; - struct stack_depot_trie_alloc_request req; - void *storage = NULL; - void *again; - int ret; - - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - __stack_depot_trie_alloc_txn_init(&txn); - req = (struct stack_depot_trie_alloc_request) { - .txn = &txn, - .node_slots = &node_slot, - .nr_node_slots = 1, - .storage = &storage, - .storage_size = 1, - }; - ret = __stack_depot_trie_alloc_txn_reserve(&req); - KUNIT_EXPECT_EQ(test, ret, -ENOSPC); - KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); - KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); - KUNIT_EXPECT_NULL(test, node_slot.node); - KUNIT_EXPECT_NULL(test, storage); - again = stackdepot_trie_pool_carve_node(1, &mark); - KUNIT_ASSERT_NOT_NULL(test, again); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&mark)); -} - -static void stackdepot_trie_alloc_txn_commit(struct kunit *test) -{ - struct stack_depot_trie_leaf_update updates[1]; - struct stack_depot_trie_side_prealloc prealloc = {}; - struct stack_depot_trie_alloc_txn txn; - const void *old_leaf = (const void *)0x1111UL; - void *pool_leaf; - u32 old_id; - u32 leaf_id; - int ret; - - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - if (__stack_depot_trie_side_table_prealloc_needed()) - stackdepot_trie_side_table_prealloc_or_fail(test, &prealloc); - - __stack_depot_trie_alloc_txn_init(&txn); - old_id = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, old_id, 1U); - ret = __stack_depot_trie_side_table_store(old_id, old_leaf); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = __stack_depot_trie_alloc_txn_id(&txn, &prealloc); - KUNIT_ASSERT_EQ(test, ret, 0); - pool_leaf = stackdepot_trie_pool_carve_node(1, &txn.pool); - KUNIT_ASSERT_NOT_NULL(test, pool_leaf); - updates[0].leaf_id = old_id; - updates[0].leaf = pool_leaf; - ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &txn.side); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(old_id), - pool_leaf); - KUNIT_EXPECT_NE(test, txn.pool.size, 0UL); - KUNIT_EXPECT_NE(test, txn.side.nr_updates, 0U); - leaf_id = __stack_depot_trie_alloc_txn_commit(&txn); - KUNIT_EXPECT_EQ(test, leaf_id, 2U); - KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); - KUNIT_EXPECT_EQ(test, txn.side.nr_updates, 0U); - KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 2UL); - - __stack_depot_trie_alloc_txn_rollback(&txn); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 2UL); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(old_id), - pool_leaf); - pool_leaf = stackdepot_trie_pool_carve_node(1, &txn.pool); - KUNIT_ASSERT_NOT_NULL(test, pool_leaf); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&txn.pool)); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_alloc_txn_commit(NULL), 0U); -} - -static void stackdepot_trie_alloc_txn_rollback(struct kunit *test) -{ - struct stack_depot_trie_leaf_update updates[2]; - struct stack_depot_trie_alloc_txn txn; - const void *old_leaf = (const void *)0x1111UL; - void *pool_leaf; - u32 old_id; - int ret; - - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - __stack_depot_trie_alloc_txn_init(&txn); - old_id = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, old_id, 1U); - txn.leaf_id = stackdepot_trie_side_table_alloc(test); - KUNIT_ASSERT_EQ(test, txn.leaf_id, 2U); - ret = __stack_depot_trie_side_table_store(old_id, old_leaf); - KUNIT_ASSERT_EQ(test, ret, 0); - - pool_leaf = stackdepot_trie_pool_carve_node(1, &txn.pool); - KUNIT_ASSERT_NOT_NULL(test, pool_leaf); - updates[0].leaf_id = old_id; - updates[0].leaf = pool_leaf; - updates[1].leaf_id = txn.leaf_id; - updates[1].leaf = pool_leaf; - ret = __stack_depot_trie_side_prepare(updates, ARRAY_SIZE(updates), &txn.side); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(old_id), - pool_leaf); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(txn.leaf_id), - pool_leaf); - - __stack_depot_trie_alloc_txn_rollback(&txn); - KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); - KUNIT_EXPECT_EQ(test, txn.side.nr_updates, 0U); - KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(old_id), - old_leaf); - KUNIT_EXPECT_NULL(test, __stack_depot_trie_side_table_lookup(2)); - pool_leaf = stackdepot_trie_pool_carve_node(1, &txn.pool); - KUNIT_ASSERT_NOT_NULL(test, pool_leaf); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_pool_try_rollback(&txn.pool)); -} - -static int txn_insert_plan(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, - struct stack_depot_trie_alloc_txn *txn, - void **storage, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_request *req) -{ - return __stack_depot_trie_alloc_txn_plan(root, entries, nr_entries, - node_slots, nr_node_slots, - child_slots, nr_child_slots, txn, - storage, pool_prealloc, side_prealloc, - req); -} - -static int txn_insert(struct stack_depot_trie_root *root, - struct stack_depot_trie_alloc_request *req, - const unsigned long *entries, unsigned int nr_entries, - const void **tail, u32 *leaf_id) -{ - return __stack_depot_trie_alloc_txn_insert(root, req, entries, nr_entries, - NULL, 0, tail, leaf_id); -} - -static int ws_insert_prealloc(struct stack_depot_trie_root *root, - struct stack_depot_trie_alloc_workspace *workspace, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_side_prealloc *side_prealloc, - const void **tail, u32 *leaf_id) -{ - return __stack_depot_trie_workspace_insert(root, entries, nr_entries, NULL, - side_prealloc, workspace, tail, leaf_id); -} - -static depot_stack_handle_t -tsave_locked(struct stack_depot_trie_root *root, const unsigned long *entries, - unsigned int nr_entries, gfp_t gfp_flags, depot_flags_t depot_flags, - struct stack_depot_trie_alloc_workspace *workspace, - raw_spinlock_t *workspace_lock) -{ - return __stack_depot_trie_save_locked(root, entries, nr_entries, gfp_flags, - depot_flags, workspace, workspace_lock); -} - -static unsigned int tfetch_handle(depot_stack_handle_t handle, - unsigned long *entries, unsigned int max_entries) -{ - return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); -} - -static void stackdepot_trie_alloc_workspace_insert(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_alloc_workspace *workspace; - struct stack_depot_trie_side_prealloc side_prealloc = {}; - struct stack_depot_trie_root root = {}; - unsigned long out[ARRAY_SIZE(entries)] = {}; - const void *tail = NULL; - unsigned int fetched; - u32 leaf_id = 0; - int ret; - - workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, workspace); - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - if (__stack_depot_trie_side_table_prealloc_needed()) - stackdepot_trie_side_table_prealloc_or_fail(test, &side_prealloc); - - ret = ws_insert_prealloc(&root, workspace, entries, ARRAY_SIZE(entries), - &side_prealloc, &tail, &leaf_id); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, leaf_id, 1U); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(leaf_id), - tail); - KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), - tail); - fetched = tfetch(tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); - - ret = ws_insert_prealloc(NULL, workspace, entries, ARRAY_SIZE(entries), - &side_prealloc, &tail, &leaf_id); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} - -static void stackdepot_trie_save_locked(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_alloc_workspace *workspace; - struct stack_depot_trie_root root = {}; - raw_spinlock_t workspace_lock; - depot_stack_handle_t first; - depot_stack_handle_t get; - depot_stack_handle_t second; - unsigned long out[ARRAY_SIZE(entries)] = {}; - unsigned int fetched; - - workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, workspace); - raw_spin_lock_init(&workspace_lock); - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - - first = tsave_locked(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_CAN_ALLOC, workspace, &workspace_lock); - KUNIT_ASSERT_NE(test, first, (depot_stack_handle_t)0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); - fetched = tfetch_handle(first, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); - - second = tsave_locked(&root, entries, ARRAY_SIZE(entries), GFP_NOWAIT, 0, - workspace, &workspace_lock); - KUNIT_EXPECT_EQ(test, second, first); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); - - get = tsave_locked(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_GET, workspace, &workspace_lock); - KUNIT_EXPECT_EQ(test, get, (depot_stack_handle_t)0); - get = tsave_locked(NULL, entries, ARRAY_SIZE(entries), GFP_KERNEL, 0, - workspace, &workspace_lock); - KUNIT_EXPECT_EQ(test, get, (depot_stack_handle_t)0); -} - -static void stackdepot_trie_fetch_handle_into(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_alloc_workspace *workspace; - struct stack_depot_trie_root root = {}; - raw_spinlock_t workspace_lock; - unsigned long small[1] = { 0xdeadUL }; - unsigned long out[ARRAY_SIZE(entries)] = {}; - depot_stack_handle_t hash_handle; - depot_stack_handle_t handle; - const void *leaf; - unsigned int invalid; - unsigned int fetched; - u32 leaf_id; - - workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, workspace); - raw_spin_lock_init(&workspace_lock); - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - - handle = tsave_locked(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_CAN_ALLOC, workspace, - &workspace_lock); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - leaf_id = __stack_depot_trie_leaf_id(handle); - KUNIT_ASSERT_NE(test, leaf_id, 0U); - leaf = __stack_depot_trie_side_table_lookup(leaf_id); - KUNIT_ASSERT_NOT_NULL(test, leaf); - fetched = tfetch_handle(handle, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); - - fetched = tfetch_handle(handle, small, ARRAY_SIZE(small)); - KUNIT_EXPECT_EQ(test, fetched, 0U); - KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); - fetched = stack_depot_fetch_into(handle, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); - fetched = stack_depot_fetch_into(handle, small, ARRAY_SIZE(small)); - KUNIT_EXPECT_EQ(test, fetched, 0U); - KUNIT_EXPECT_EQ(test, small[0], 0xdeadUL); - invalid = tfetch_handle(0, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, invalid, 0U); - invalid = tfetch_handle(handle, NULL, 0); - KUNIT_EXPECT_EQ(test, invalid, 0U); - fetched = tfetch_handle(handle, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); - - hash_handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); - KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); - invalid = tfetch_handle(hash_handle, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, invalid, 0U); -} - -static void stackdepot_trie_snprint_public(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - struct stack_depot_trie_alloc_workspace *workspace; - char expected[256]; - char actual[256]; - struct stack_depot_trie_root root = {}; - raw_spinlock_t workspace_lock; - depot_stack_handle_t extra; - depot_stack_handle_t handle; - unsigned int expected_len; - int actual_len; - - workspace = kunit_kzalloc(test, sizeof(*workspace), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, workspace); - raw_spin_lock_init(&workspace_lock); - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - - handle = tsave_locked(&root, entries, ARRAY_SIZE(entries), GFP_KERNEL, - STACK_DEPOT_FLAG_CAN_ALLOC, workspace, - &workspace_lock); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - - expected_len = stack_trace_snprint(expected, sizeof(expected), entries, - ARRAY_SIZE(entries), 2); - extra = stack_depot_set_extra_bits(handle, 7); - actual_len = stack_depot_snprint(extra, actual, sizeof(actual), 2); - KUNIT_EXPECT_EQ(test, actual_len, (int)expected_len); - KUNIT_EXPECT_STREQ(test, actual, expected); -} - -static void stackdepot_trie_alloc_txn_plan(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_slot; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_alloc_request req; - struct stack_depot_trie_alloc_txn txn; - struct stack_depot_trie_root root = {}; - struct stack_depot_trie_side_prealloc side_prealloc = { - .chunk = (void *)0x2222UL, - }; - void *pool_prealloc = (void *)0x1111UL; - void *storage = (void *)0x3333UL; - int ret; - - ret = txn_insert_plan(&root, entries, ARRAY_SIZE(entries), &node_slot, 1, - &child_slot, 1, &txn, &storage, &pool_prealloc, - &side_prealloc, &req); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, req.txn, &txn); - KUNIT_EXPECT_PTR_EQ(test, req.node_slots, &node_slot); - KUNIT_EXPECT_EQ(test, req.nr_node_slots, 1U); - KUNIT_EXPECT_PTR_EQ(test, req.child_slots, &child_slot); - KUNIT_EXPECT_EQ(test, req.nr_child_slots, 0U); - KUNIT_EXPECT_PTR_EQ(test, req.storage, &storage); - KUNIT_EXPECT_PTR_EQ(test, req.pool_prealloc, &pool_prealloc); - KUNIT_EXPECT_PTR_EQ(test, req.side_prealloc, &side_prealloc); - KUNIT_EXPECT_NE(test, req.storage_size, 0UL); - KUNIT_EXPECT_NULL(test, storage); - KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); - KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); - KUNIT_EXPECT_EQ(test, txn.side.nr_updates, 0U); - - ret = txn_insert_plan(NULL, entries, ARRAY_SIZE(entries), &node_slot, 1, - &child_slot, 1, &txn, &storage, &pool_prealloc, - &side_prealloc, &req); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} - -static void stackdepot_trie_alloc_txn_insert(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_child_array_slot child_slots[1]; - struct stack_depot_trie_node_slot node_slots[1]; - struct stack_depot_trie_alloc_request req; - struct stack_depot_trie_alloc_txn txn; - struct stack_depot_trie_root root = {}; - struct stack_depot_trie_side_prealloc side_prealloc = {}; - unsigned long out[ARRAY_SIZE(entries)] = {}; - const void *tail = NULL; - void *storage = NULL; - unsigned int fetched; - u32 leaf_id = 0; - int ret; - - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - if (__stack_depot_trie_side_table_prealloc_needed()) - stackdepot_trie_side_table_prealloc_or_fail(test, &side_prealloc); - - ret = txn_insert_plan(&root, entries, ARRAY_SIZE(entries), node_slots, - ARRAY_SIZE(node_slots), child_slots, - ARRAY_SIZE(child_slots), &txn, &storage, - NULL, &side_prealloc, &req); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = txn_insert(&root, &req, entries, ARRAY_SIZE(entries), &tail, &leaf_id); - KUNIT_ASSERT_EQ(test, ret, 0); - - KUNIT_EXPECT_EQ(test, leaf_id, 1U); - KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); - KUNIT_EXPECT_EQ(test, txn.side.nr_updates, 0U); - KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(leaf_id), - tail); - KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), - tail); - fetched = tfetch(tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); - - __stack_depot_trie_alloc_txn_rollback(&txn); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 1UL); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(leaf_id), - tail); -} - -static void stackdepot_trie_alloc_txn_insert_stale_plan(struct kunit *test) -{ - unsigned long first[] = { 0x1000UL }; - unsigned long second[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot child_slots[1]; - struct stack_depot_trie_child_array_slot fresh_child_slots[1]; - struct stack_depot_trie_node_slot node_slots[1]; - struct stack_depot_trie_node_slot fresh_node_slots[1]; - struct stack_depot_trie_alloc_request req; - struct stack_depot_trie_alloc_request fresh_req; - struct stack_depot_trie_alloc_txn txn; - struct stack_depot_trie_alloc_txn fresh_txn; - struct stack_depot_trie_root root = {}; - struct stack_depot_trie_side_prealloc side_prealloc = {}; - const void *tail = NULL; - const void *fresh_tail = NULL; - void *storage = NULL; - void *fresh_storage = NULL; - u32 leaf_id = 0; - u32 fresh_leaf_id = 0; - int ret; - - stackdepot_trie_side_table_init_or_skip(test); - stackdepot_trie_pool_seed_current_pool(test); - if (__stack_depot_trie_side_table_prealloc_needed()) - stackdepot_trie_side_table_prealloc_or_fail(test, &side_prealloc); - - ret = txn_insert_plan(&root, first, ARRAY_SIZE(first), fresh_node_slots, - ARRAY_SIZE(fresh_node_slots), fresh_child_slots, - ARRAY_SIZE(fresh_child_slots), &fresh_txn, - &fresh_storage, NULL, &side_prealloc, &fresh_req); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = txn_insert(&root, &fresh_req, first, ARRAY_SIZE(first), &fresh_tail, - &fresh_leaf_id); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_ASSERT_EQ(test, fresh_leaf_id, 1U); - - ret = txn_insert_plan(&root, second, ARRAY_SIZE(second), node_slots, - ARRAY_SIZE(node_slots), child_slots, - ARRAY_SIZE(child_slots), &txn, &storage, - NULL, &side_prealloc, &req); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = txn_insert_plan(&root, second, ARRAY_SIZE(second), fresh_node_slots, - ARRAY_SIZE(fresh_node_slots), fresh_child_slots, - ARRAY_SIZE(fresh_child_slots), &fresh_txn, - &fresh_storage, NULL, &side_prealloc, &fresh_req); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = txn_insert(&root, &fresh_req, second, ARRAY_SIZE(second), &fresh_tail, - &fresh_leaf_id); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_ASSERT_EQ(test, fresh_leaf_id, 2U); - - ret = txn_insert(&root, &req, second, ARRAY_SIZE(second), &tail, &leaf_id); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_EQ(test, leaf_id, 0U); - KUNIT_EXPECT_NULL(test, tail); - KUNIT_EXPECT_EQ(test, txn.leaf_id, 0U); - KUNIT_EXPECT_EQ(test, txn.side.nr_updates, 0U); - KUNIT_EXPECT_EQ(test, txn.pool.size, 0UL); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_side_table_entries(), 2UL); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(1), - find_leaf(&root, first, ARRAY_SIZE(first))); - KUNIT_EXPECT_PTR_EQ(test, __stack_depot_trie_side_table_lookup(2), - find_leaf(&root, second, ARRAY_SIZE(second))); - KUNIT_EXPECT_NULL(test, storage); - KUNIT_EXPECT_NULL(test, node_slots[0].node); -} - -static void stackdepot_frame_raw_fallback(struct kunit *test) -{ - unsigned long frame = 0xffff888000001000UL; - unsigned long out = 0x12345678UL; - u32 low = 0xfeedbeef; - u8 prefix_id = 0xaa; - -#ifdef CONFIG_ARM64 - frame = arch_stack_depot_frame_text_prefix(); - if (frame <= ~0UL - 2 * SZ_4G) - frame += 2 * SZ_4G; - else - frame -= 2 * SZ_4G; - frame |= 0x1000UL; -#endif - - /* Arch hooks may exist, but this frame is chosen to stay raw. */ - KUNIT_EXPECT_FALSE(test, - arch_stack_depot_frame_try_compress(frame, &prefix_id, &low)); - KUNIT_EXPECT_EQ(test, prefix_id, (u8)0xaa); - KUNIT_EXPECT_EQ(test, low, (u32)0xfeedbeef); - - KUNIT_EXPECT_FALSE(test, - arch_stack_depot_frame_decompress(0xff, 0x81234567, &out)); - KUNIT_EXPECT_EQ(test, out, 0x12345678UL); - - KUNIT_EXPECT_FALSE(test, - arch_stack_depot_frame_decompress(0, 0x81234567, NULL)); -} - -#ifdef CONFIG_X86_64 -static void stackdepot_frame_x86_64(struct kunit *test) -{ - unsigned long direct_map = 0xffff888000001000UL; - unsigned long frame = 0xffffffff81234567UL; - unsigned long out; - bool compressed; - u32 low; - u8 prefix_id; - - KUNIT_EXPECT_TRUE(test, - arch_stack_depot_frame_try_compress(frame, &prefix_id, &low)); - KUNIT_EXPECT_EQ(test, prefix_id, (u8)0); - KUNIT_EXPECT_EQ(test, low, (u32)0x81234567); - KUNIT_EXPECT_TRUE(test, - arch_stack_depot_frame_decompress(prefix_id, low, &out)); - KUNIT_EXPECT_EQ(test, out, frame); - - compressed = arch_stack_depot_frame_try_compress(direct_map, &prefix_id, &low); - KUNIT_EXPECT_FALSE(test, compressed); - KUNIT_EXPECT_FALSE(test, - arch_stack_depot_frame_decompress(1, low, &out)); -} -#endif /* CONFIG_X86_64 */ - -#ifdef CONFIG_ARM64 -static void stackdepot_frame_arm64(struct kunit *test) -{ - unsigned long frame = (unsigned long)stackdepot_frame_arm64; - unsigned long text_prefix = arch_stack_depot_frame_text_prefix(); - unsigned long out; - bool decoded; - u32 low; - u8 prefix_id; - - KUNIT_EXPECT_TRUE(test, - arch_stack_depot_frame_try_compress(frame, &prefix_id, &low)); - KUNIT_EXPECT_EQ(test, low, (u32)frame); - KUNIT_EXPECT_TRUE(test, - arch_stack_depot_frame_decompress(prefix_id, low, &out)); - KUNIT_EXPECT_EQ(test, out, frame); - - if (text_prefix > SZ_4G) { - frame = (text_prefix - SZ_4G) | 0x12345678UL; - KUNIT_EXPECT_TRUE(test, - arch_stack_depot_frame_try_compress(frame, &prefix_id, &low)); - KUNIT_EXPECT_EQ(test, prefix_id, (u8)STACK_DEPOT_ARM64_PREV_PREFIX_ID); - KUNIT_EXPECT_TRUE(test, - arch_stack_depot_frame_decompress(prefix_id, low, &out)); - KUNIT_EXPECT_EQ(test, out, frame); - } else { - prefix_id = STACK_DEPOT_ARM64_PREV_PREFIX_ID; - decoded = arch_stack_depot_frame_decompress(prefix_id, 0, &out); - KUNIT_EXPECT_FALSE(test, decoded); - } - - if (text_prefix <= ~0UL - SZ_4G) { - frame = (text_prefix + SZ_4G) | 0x87654321UL; - KUNIT_EXPECT_TRUE(test, - arch_stack_depot_frame_try_compress(frame, &prefix_id, &low)); - KUNIT_EXPECT_EQ(test, prefix_id, (u8)STACK_DEPOT_ARM64_NEXT_PREFIX_ID); - KUNIT_EXPECT_TRUE(test, - arch_stack_depot_frame_decompress(prefix_id, low, &out)); - KUNIT_EXPECT_EQ(test, out, frame); - } else { - prefix_id = STACK_DEPOT_ARM64_NEXT_PREFIX_ID; - decoded = arch_stack_depot_frame_decompress(prefix_id, 0, &out); - KUNIT_EXPECT_FALSE(test, decoded); - } - - KUNIT_EXPECT_FALSE(test, - arch_stack_depot_frame_decompress(3, low, &out)); -} -#endif /* CONFIG_ARM64 */ - -static void stackdepot_frame_run_raw_roundtrip(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - unsigned long out[ARRAY_SIZE(entries)] = {}; - struct stack_depot_frame_run run; - unsigned char payload[sizeof(entries)]; - int ret; - - ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); - KUNIT_EXPECT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, run.mode, STACK_DEPOT_FRAME_RAW); - KUNIT_EXPECT_EQ(test, run.nr_entries, - (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_EQ(test, run.bytes, sizeof(entries)); - - ret = frame_run_write(&run, entries, payload, sizeof(payload), NULL, 0); - KUNIT_EXPECT_EQ(test, ret, 0); - ret = frame_run_read(&run, payload, run.bytes, out, ARRAY_SIZE(out), NULL, 0); - KUNIT_EXPECT_EQ(test, ret, 0); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); -} - -#ifdef CONFIG_ARM64 -static void stackdepot_frame_run_arm64_roundtrip(struct kunit *test) -{ - unsigned long entries[] = { - arch_stack_depot_frame_text_prefix() | 0x1000UL, - arch_stack_depot_frame_text_prefix() | 0x2000UL, - }; - unsigned long read_scratch[ARRAY_SIZE(entries)]; - unsigned long out[ARRAY_SIZE(entries)] = {}; - struct stack_depot_frame_run run; - u32 payload[ARRAY_SIZE(entries)]; - u32 write_scratch[ARRAY_SIZE(entries)]; - int ret; - - ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); - KUNIT_EXPECT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, run.mode, STACK_DEPOT_FRAME_COMPRESSED); - KUNIT_EXPECT_EQ(test, run.nr_entries, - (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_EQ(test, run.bytes, sizeof(payload)); - - ret = frame_run_write(&run, entries, payload, sizeof(payload), - write_scratch, ARRAY_SIZE(write_scratch)); - KUNIT_EXPECT_EQ(test, ret, 0); - ret = frame_run_read(&run, payload, run.bytes, out, ARRAY_SIZE(out), - read_scratch, ARRAY_SIZE(read_scratch)); - KUNIT_EXPECT_EQ(test, ret, 0); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); -} -#endif - -#ifdef CONFIG_X86_64 -static void stackdepot_frame_run_x86_64_roundtrip(struct kunit *test) -{ - unsigned long entries[] = { - 0xffffffff81000001UL, - 0xffffffff81000002UL, - 0xffffffff81000003UL, - }; - unsigned long out[ARRAY_SIZE(entries)] = {}; - unsigned long read_scratch[ARRAY_SIZE(entries)]; - struct stack_depot_frame_run run; - u32 payload[ARRAY_SIZE(entries)]; - u32 write_scratch[ARRAY_SIZE(entries)]; - int ret; - - ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); - KUNIT_EXPECT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, run.mode, STACK_DEPOT_FRAME_COMPRESSED); - KUNIT_EXPECT_EQ(test, run.prefix_id, (u8)0); - KUNIT_EXPECT_EQ(test, run.nr_entries, - (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_EQ(test, run.bytes, sizeof(payload)); - - ret = frame_run_write(&run, entries, payload, sizeof(payload), - write_scratch, ARRAY_SIZE(write_scratch)); - KUNIT_EXPECT_EQ(test, ret, 0); - ret = frame_run_read(&run, payload, run.bytes, out, ARRAY_SIZE(out), - read_scratch, ARRAY_SIZE(read_scratch)); - KUNIT_EXPECT_EQ(test, ret, 0); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); -} - -static void stackdepot_frame_run_x86_64_boundary(struct kunit *test) -{ - unsigned long entries[] = { - 0xffffffff81000001UL, - 0xffffffff81000002UL, - 0xffff888000000003UL, - 0xffffffff81000004UL, - }; - struct stack_depot_frame_run run; - int ret; - - ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); - KUNIT_EXPECT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, run.mode, STACK_DEPOT_FRAME_COMPRESSED); - KUNIT_EXPECT_EQ(test, run.nr_entries, 2U); - - ret = frame_run_init(&entries[2], 2, &run); - KUNIT_EXPECT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, run.mode, STACK_DEPOT_FRAME_RAW); - KUNIT_EXPECT_EQ(test, run.nr_entries, 1U); -} - -static void stackdepot_frame_run_x86_64_write_rejects_mismatch(struct kunit *test) -{ - unsigned long good[] = { - 0xffffffff81000001UL, - 0xffffffff81000002UL, - }; - unsigned long bad[] = { - 0xffffffff81000001UL, - 0xffff888000000002UL, - }; - u32 payload[ARRAY_SIZE(good)] = { 0xa5a5a5a5, 0xb6b6b6b6 }; - u32 scratch[ARRAY_SIZE(good)]; - u32 old[ARRAY_SIZE(payload)]; - struct stack_depot_frame_run run; - int ret; - - memcpy(old, payload, sizeof(old)); - ret = frame_run_init(good, ARRAY_SIZE(good), &run); - KUNIT_EXPECT_EQ(test, ret, 0); - ret = frame_run_write(&run, bad, payload, sizeof(payload), scratch, - ARRAY_SIZE(scratch)); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_MEMEQ(test, payload, old, sizeof(payload)); -} -#endif /* CONFIG_X86_64 */ - -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) -static void stackdepot_frame_run_compressed_rejects_src_scratch_overlap(struct kunit *test) -{ - unsigned long entries[] = { -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x1000UL, - arch_stack_depot_frame_text_prefix() | 0x2000UL, -#else - 0xffffffff81001000UL, - 0xffffffff81002000UL, -#endif - }; - unsigned long alias[ARRAY_SIZE(entries)] = {}; - unsigned long out[ARRAY_SIZE(entries)] = { 0xa5a5UL, 0xb6b6UL }; - unsigned long old[ARRAY_SIZE(out)]; - struct stack_depot_frame_run run; - u32 payload[ARRAY_SIZE(entries)]; - u32 write_scratch[ARRAY_SIZE(entries)]; - int ret; - - memcpy(old, out, sizeof(old)); - ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_ASSERT_EQ(test, run.mode, STACK_DEPOT_FRAME_COMPRESSED); - ret = frame_run_write(&run, entries, payload, sizeof(payload), - write_scratch, ARRAY_SIZE(write_scratch)); - KUNIT_ASSERT_EQ(test, ret, 0); - memcpy(alias, payload, run.bytes); - - ret = frame_run_read(&run, alias, run.bytes, out, ARRAY_SIZE(out), - alias, ARRAY_SIZE(alias)); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_MEMEQ(test, out, old, sizeof(out)); -} -#endif - -static void stackdepot_frame_run_invalid_inputs(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - unsigned long out[ARRAY_SIZE(entries)] = { 0xa5a5a5a5UL }; - unsigned long old[ARRAY_SIZE(out)]; - struct stack_depot_frame_run run; - unsigned char payload[sizeof(entries)]; - int ret; - - memcpy(old, out, sizeof(old)); - ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); - KUNIT_EXPECT_EQ(test, ret, 0); - - ret = frame_run_init(NULL, ARRAY_SIZE(entries), &run); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = frame_run_init(entries, 0, &run); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = frame_run_init(entries, ARRAY_SIZE(entries), NULL); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = frame_run_write(&run, entries, NULL, run.bytes, NULL, 0); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = frame_run_write(&run, entries, payload, run.bytes - 1, NULL, 0); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = frame_run_read(&run, payload, run.bytes - 1, out, ARRAY_SIZE(out), - NULL, 0); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = frame_run_read(&run, payload, run.bytes, out, 0, NULL, 0); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_MEMEQ(test, out, old, sizeof(out)); -} - -static void stackdepot_trie_node_raw_roundtrip(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - unsigned long out[ARRAY_SIZE(entries)] = {}; - unsigned int fetched; - void *node; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 7, &node); - fetched = tfetch(node, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); -} - -static void stackdepot_trie_node_parent_chain(struct kunit *test) -{ - unsigned long root_entries[] = { 0x1000UL, 0x2000UL }; - unsigned long child_entries[] = { 0x3000UL, 0x4000UL }; - unsigned long expected[] = { 0x1000UL, 0x2000UL, 0x3000UL, 0x4000UL }; - unsigned long out[ARRAY_SIZE(expected)] = {}; - unsigned int fetched; - void *root; - void *child; - - trie_node_alloc(test, root_entries, ARRAY_SIZE(root_entries), NULL, 0, - &root); - trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), root, 9, - &child); - fetched = tfetch(child, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); - KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); -} - -static void stackdepot_trie_node_slice_raw(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - unsigned long expected[] = { 0x2000UL, 0x3000UL }; - struct stack_depot_frame_run run; - unsigned long out[ARRAY_SIZE(expected)] = {}; - unsigned int fetched; - void *source; - void *slice; - size_t size; - int ret; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &source); - ret = frame_run_init(&entries[1], ARRAY_SIZE(expected), &run); - KUNIT_ASSERT_EQ(test, ret, 0); - size = __stack_depot_trie_node_size(&run); - KUNIT_ASSERT_GT(test, size, (size_t)0); - slice = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, slice); - ret = tnode_init_slice(slice, size, NULL, 10, source, 1, - ARRAY_SIZE(expected)); - KUNIT_ASSERT_EQ(test, ret, 0); - fetched = tfetch(slice, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); - KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); - KUNIT_EXPECT_EQ(test, tmatch(slice, expected, ARRAY_SIZE(expected)), - (unsigned int)ARRAY_SIZE(expected)); -} - -static void stackdepot_trie_node_slice_parent_chain(struct kunit *test) -{ - unsigned long root_entries[] = { 0x1000UL }; - unsigned long entries[] = { 0x2000UL, 0x3000UL, 0x4000UL }; - unsigned long expected[] = { 0x1000UL, 0x3000UL, 0x4000UL }; - struct stack_depot_frame_run run; - unsigned long out[ARRAY_SIZE(expected)] = {}; - unsigned int fetched; - void *root; - void *source; - void *slice; - size_t size; - int ret; - - trie_node_alloc(test, root_entries, ARRAY_SIZE(root_entries), NULL, 0, - &root); - trie_node_alloc(test, entries, ARRAY_SIZE(entries), root, 0, &source); - ret = frame_run_init(&entries[1], 2, &run); - KUNIT_ASSERT_EQ(test, ret, 0); - size = __stack_depot_trie_node_size(&run); - KUNIT_ASSERT_GT(test, size, (size_t)0); - slice = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, slice); - ret = tnode_init_slice(slice, size, root, 11, source, 1, 2); - KUNIT_ASSERT_EQ(test, ret, 0); - fetched = tfetch(slice, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); - KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); -} - -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) -static void stackdepot_trie_node_slice_compressed(struct kunit *test) -{ - unsigned long entries[] = { -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x1000UL, - arch_stack_depot_frame_text_prefix() | 0x2000UL, - arch_stack_depot_frame_text_prefix() | 0x3000UL, -#else - 0xffffffff81001000UL, - 0xffffffff81002000UL, - 0xffffffff81003000UL, -#endif - }; - unsigned long expected[] = { entries[1], entries[2] }; - struct stack_depot_frame_run run; - unsigned long out[ARRAY_SIZE(expected)] = {}; - unsigned int fetched; - void *source; - void *slice; - size_t size; - int ret; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &source); - ret = frame_run_init(&entries[1], ARRAY_SIZE(expected), &run); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_ASSERT_EQ(test, run.mode, STACK_DEPOT_FRAME_COMPRESSED); - size = __stack_depot_trie_node_size(&run); - KUNIT_ASSERT_GT(test, size, (size_t)0); - slice = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, slice); - ret = tnode_init_slice(slice, size, NULL, 12, source, 1, - ARRAY_SIZE(expected)); - KUNIT_ASSERT_EQ(test, ret, 0); - fetched = tfetch(slice, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); - KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); -} -#endif - -static void stackdepot_trie_node_slice_rejects_bad_inputs(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_frame_run run; - void *source; - void *slice; - size_t size; - int ret; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &source); - ret = frame_run_init(entries, 1, &run); - KUNIT_ASSERT_EQ(test, ret, 0); - size = __stack_depot_trie_node_size(&run); - KUNIT_ASSERT_GT(test, size, (size_t)0); - slice = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, slice); - - KUNIT_EXPECT_EQ(test, tnode_init_slice(NULL, size, NULL, 1, source, 0, 1), - -EINVAL); - KUNIT_EXPECT_EQ(test, tnode_init_slice(slice, size, NULL, 1, NULL, 0, 1), - -EINVAL); - KUNIT_EXPECT_EQ(test, tnode_init_slice(slice, size, NULL, 1, source, 0, 0), - -EINVAL); - KUNIT_EXPECT_EQ(test, tnode_init_slice(slice, size, NULL, 1, source, 2, 1), - -EINVAL); - KUNIT_EXPECT_EQ(test, tnode_init_slice(slice, size - 1, NULL, 1, source, 0, 1), - -EINVAL); - KUNIT_EXPECT_EQ(test, tnode_init_slice(source, size, NULL, 1, source, 0, 1), - -EINVAL); -} - -static void stackdepot_trie_node_rejects_stack_len_overflow(struct kunit *test) -{ - unsigned long exact_child[] = { 0x80000000UL }; - unsigned long overflow_child[] = { 0x80001000UL, 0x80002000UL }; - unsigned int parent_len = CONFIG_STACKDEPOT_MAX_FRAMES - 1; - struct stack_depot_frame_run run; - unsigned long *parent_entries; - void *parent; - void *child; - size_t size; - int ret; - - if (CONFIG_STACKDEPOT_MAX_FRAMES < 2) { - kunit_skip(test, "stack length overflow test needs at least two frames"); - return; - } - - parent_entries = kunit_kcalloc(test, parent_len, sizeof(*parent_entries), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, parent_entries); - trie_fill_raw_entries(parent_entries, parent_len, 0x1000UL); - KUNIT_ASSERT_EQ(test, frame_run_init(parent_entries, parent_len, &run), 0); - size = __stack_depot_trie_node_size(&run); - KUNIT_ASSERT_GT(test, size, (size_t)0); - parent = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, parent); - ret = tnode_init(parent, size, NULL, 0, parent_entries, parent_len, - NULL, 0); - KUNIT_ASSERT_EQ(test, ret, 0); - - ret = frame_run_init(exact_child, ARRAY_SIZE(exact_child), &run); - KUNIT_ASSERT_EQ(test, ret, 0); - size = __stack_depot_trie_node_size(&run); - child = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child); - ret = tnode_init(child, size, parent, 1, exact_child, - ARRAY_SIZE(exact_child), NULL, 0); - KUNIT_EXPECT_EQ(test, ret, 0); - - ret = frame_run_init(overflow_child, ARRAY_SIZE(overflow_child), &run); - KUNIT_ASSERT_EQ(test, ret, 0); - size = __stack_depot_trie_node_size(&run); - child = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child); - ret = tnode_init(child, size, parent, 2, overflow_child, - ARRAY_SIZE(overflow_child), NULL, 0); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} - -static void stackdepot_trie_node_match_raw(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - unsigned long mismatch[] = { 0x1000UL, 0x2222UL, 0x3000UL }; - unsigned long short_input[] = { 0x1000UL, 0x2000UL }; - unsigned long long_input[] = { - 0x1000UL, 0x2000UL, 0x3000UL, 0x4000UL, - }; - unsigned long first_mismatch[] = { 0x9000UL, 0x2000UL }; - unsigned int matched; - void *node; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 7, &node); - KUNIT_EXPECT_EQ(test, tmatch(node, entries, ARRAY_SIZE(entries)), - (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_EQ(test, tmatch(node, mismatch, ARRAY_SIZE(mismatch)), 1U); - KUNIT_EXPECT_EQ(test, tmatch(node, short_input, ARRAY_SIZE(short_input)), - (unsigned int)ARRAY_SIZE(short_input)); - KUNIT_EXPECT_EQ(test, tmatch(node, long_input, ARRAY_SIZE(long_input)), - (unsigned int)ARRAY_SIZE(entries)); - matched = tmatch(node, first_mismatch, ARRAY_SIZE(first_mismatch)); - KUNIT_EXPECT_EQ(test, matched, 0U); - KUNIT_EXPECT_EQ(test, tmatch(NULL, entries, ARRAY_SIZE(entries)), 0U); - KUNIT_EXPECT_EQ(test, tmatch(node, NULL, ARRAY_SIZE(entries)), 0U); - KUNIT_EXPECT_EQ(test, tmatch(node, entries, 0), 0U); -} - -static void stackdepot_trie_append_chain_raw(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - struct stack_depot_frame_run run; - struct stack_depot_trie_node_slot node_slots[1]; - unsigned long out[ARRAY_SIZE(entries)] = {}; - const void *head = NULL; - const void *tail = NULL; - unsigned int used = 0; - unsigned int fetched; - size_t size; - int ret; - - KUNIT_ASSERT_EQ(test, frame_run_init(entries, ARRAY_SIZE(entries), &run), 0); - size = __stack_depot_trie_node_size(&run); - node_slots[0].node = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, node_slots[0].node); - node_slots[0].size = size; - - ret = append_chain(NULL, 13, entries, ARRAY_SIZE(entries), node_slots, - ARRAY_SIZE(node_slots), NULL, 0, NULL, 0, &head, &tail, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, head, node_slots[0].node); - KUNIT_EXPECT_PTR_EQ(test, tail, node_slots[0].node); - KUNIT_EXPECT_EQ(test, used, 1U); - fetched = tfetch(tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); -} - -static void stackdepot_trie_append_chain_parent(struct kunit *test) -{ - unsigned long parent_entries[] = { 0x1000UL }; - unsigned long entries[] = { 0x2000UL, 0x3000UL }; - unsigned long expected[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - struct stack_depot_frame_run run; - struct stack_depot_trie_node_slot node_slots[1]; - unsigned long out[ARRAY_SIZE(expected)] = {}; - const void *head = NULL; - const void *tail = NULL; - unsigned int used = 0; - unsigned int fetched; - void *parent; - size_t size; - int ret; - - trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, - &parent); - KUNIT_ASSERT_EQ(test, frame_run_init(entries, ARRAY_SIZE(entries), &run), 0); - size = __stack_depot_trie_node_size(&run); - node_slots[0].node = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, node_slots[0].node); - node_slots[0].size = size; - - ret = append_chain(parent, 14, entries, ARRAY_SIZE(entries), node_slots, - ARRAY_SIZE(node_slots), NULL, 0, NULL, 0, &head, &tail, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, head, tail); - KUNIT_EXPECT_EQ(test, used, 1U); - fetched = tfetch(tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); - KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); -} - -static void stackdepot_trie_append_chain_rejects_stack_len_overflow(struct kunit *test) -{ - unsigned long entries[] = { 0x80001000UL, 0x80002000UL }; - unsigned int parent_len = CONFIG_STACKDEPOT_MAX_FRAMES - 1; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_frame_run run; - unsigned long *parent_entries; - unsigned char *old; - const void *head = NULL; - const void *tail = NULL; - unsigned int used = 99; - void *parent; - size_t size; - int ret; - - if (CONFIG_STACKDEPOT_MAX_FRAMES < 2) { - kunit_skip(test, "stack length overflow test needs at least two frames"); - return; - } - - parent_entries = kunit_kcalloc(test, parent_len, sizeof(*parent_entries), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, parent_entries); - trie_fill_raw_entries(parent_entries, parent_len, 0x1000UL); - KUNIT_ASSERT_EQ(test, frame_run_init(parent_entries, parent_len, &run), 0); - size = __stack_depot_trie_node_size(&run); - parent = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, parent); - ret = tnode_init(parent, size, NULL, 0, parent_entries, parent_len, - NULL, 0); - KUNIT_ASSERT_EQ(test, ret, 0); - - KUNIT_ASSERT_EQ(test, frame_run_init(entries, ARRAY_SIZE(entries), &run), 0); - node_slot.size = __stack_depot_trie_node_size(&run); - node_slot.node = kunit_kzalloc(test, node_slot.size, GFP_KERNEL); - old = kunit_kzalloc(test, node_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, node_slot.node); - KUNIT_ASSERT_NOT_NULL(test, old); - memset(node_slot.node, 0xaa, node_slot.size); - memcpy(old, node_slot.node, node_slot.size); - - ret = append_chain(parent, 18, entries, ARRAY_SIZE(entries), &node_slot, 1, - NULL, 0, NULL, 0, &head, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_MEMEQ(test, node_slot.node, old, node_slot.size); - KUNIT_EXPECT_NULL(test, head); - KUNIT_EXPECT_NULL(test, tail); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_publish_append_root(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *head = NULL; - const void *tail = NULL; - unsigned int used = 0; - int ret; - - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = append_chain(NULL, 17, entries, ARRAY_SIZE(entries), &node_slot, 1, - NULL, 0, NULL, 0, &head, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, head, child_array.array, - child_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, root.children, child_array.array); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, entries[0]), head); - KUNIT_EXPECT_PTR_EQ(test, head, tail); -} - -static void stackdepot_trie_publish_append_parent(struct kunit *test) -{ - unsigned long parent_entries[] = { 0x1000UL }; - unsigned long old_entries[] = { 0x2000UL }; - unsigned long new_entries[] = { 0x3000UL }; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot old_slot; - struct stack_depot_trie_node_slot new_slot; - const void *old_head = NULL; - const void *old_tail = NULL; - const void *new_head = NULL; - const void *new_tail = NULL; - unsigned int used = 0; - void *parent; - int ret; - - trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, - &parent); - trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); - trie_node_slot_alloc(test, &new_slot, new_entries, ARRAY_SIZE(new_entries)); - old_array.size = __stack_depot_trie_child_array_size(1); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - new_array.size = __stack_depot_trie_child_array_size(2); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); - - ret = append_chain(parent, 18, old_entries, ARRAY_SIZE(old_entries), - &old_slot, 1, NULL, 0, NULL, 0, &old_head, &old_tail, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(NULL, parent, old_head, old_array.array, - old_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = append_chain(parent, 19, new_entries, ARRAY_SIZE(new_entries), - &new_slot, 1, NULL, 0, NULL, 0, &new_head, &new_tail, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(NULL, parent, new_head, new_array.array, - new_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array.array, old_entries[0]), - old_head); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array.array, new_entries[0]), - new_head); - KUNIT_EXPECT_PTR_EQ(test, old_head, old_tail); - KUNIT_EXPECT_PTR_EQ(test, new_head, new_tail); -} - -static void stackdepot_trie_publish_append_root_replaces_array(struct kunit *test) -{ - unsigned long old_entries[] = { 0x1000UL }; - unsigned long new_entries[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot dup_array; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot dup_slot; - struct stack_depot_trie_node_slot old_slot; - struct stack_depot_trie_node_slot new_slot; - struct stack_depot_trie_root root = {}; - const void *dup_head = NULL; - const void *dup_tail = NULL; - const void *old_head = NULL; - const void *old_tail = NULL; - const void *new_head = NULL; - const void *new_tail = NULL; - unsigned int used = 0; - int ret; - - trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); - trie_node_slot_alloc(test, &dup_slot, old_entries, ARRAY_SIZE(old_entries)); - trie_node_slot_alloc(test, &new_slot, new_entries, ARRAY_SIZE(new_entries)); - old_array.size = __stack_depot_trie_child_array_size(1); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - dup_array.size = __stack_depot_trie_child_array_size(2); - dup_array.array = kunit_kzalloc(test, dup_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, dup_array.array); - new_array.size = __stack_depot_trie_child_array_size(2); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); - - ret = append_chain(NULL, 21, old_entries, ARRAY_SIZE(old_entries), - &old_slot, 1, NULL, 0, NULL, 0, &old_head, &old_tail, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, old_head, old_array.array, - old_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - ret = append_chain(NULL, 22, old_entries, ARRAY_SIZE(old_entries), - &dup_slot, 1, NULL, 0, NULL, 0, &dup_head, &dup_tail, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, dup_head, dup_array.array, - dup_array.size); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); - - ret = append_chain(NULL, 23, new_entries, ARRAY_SIZE(new_entries), - &new_slot, 1, NULL, 0, NULL, 0, &new_head, &new_tail, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, new_head, new_array.array, - new_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, root.children, new_array.array); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, old_entries[0]), - old_head); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, new_entries[0]), - new_head); - KUNIT_EXPECT_PTR_EQ(test, old_head, old_tail); - KUNIT_EXPECT_PTR_EQ(test, dup_head, dup_tail); - KUNIT_EXPECT_PTR_EQ(test, new_head, new_tail); -} - -static void stackdepot_trie_publish_append_rejects_bad_inputs(struct kunit *test) -{ - unsigned long parent_entries[] = { 0x1000UL }; - unsigned long entries[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *head = NULL; - const void *tail = NULL; - unsigned int used = 0; - void *parent; - int ret; - - trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, - &parent); - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = append_chain(parent, 20, entries, ARRAY_SIZE(entries), &node_slot, 1, - NULL, 0, NULL, 0, &head, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, head, child_array.array, - child_array.size); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = publish_append(NULL, parent, NULL, child_array.array, - child_array.size); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = publish_append(NULL, parent, head, node_slot.node, node_slot.size); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} - -static void stackdepot_trie_lookup_step_root(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - unsigned long longer[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - unsigned long partial[] = { 0x1000UL, 0x2222UL }; - unsigned long missing[] = { 0x9000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_lookup lookup; - struct stack_depot_trie_root root = {}; - const void *head = NULL; - const void *tail = NULL; - unsigned int used = 0; - int ret; - - ret = lookup_step(&root, NULL, missing, ARRAY_SIZE(missing), &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_APPEND); - KUNIT_EXPECT_NULL(test, lookup.node); - KUNIT_EXPECT_EQ(test, lookup.matched, 0U); - - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = append_chain(NULL, 24, entries, ARRAY_SIZE(entries), &node_slot, 1, - NULL, 0, NULL, 0, &head, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, head, child_array.array, - child_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - ret = lookup_step(&root, NULL, entries, ARRAY_SIZE(entries), &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.parent, NULL); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, head); - KUNIT_EXPECT_EQ(test, lookup.matched, (unsigned int)ARRAY_SIZE(entries)); - - ret = lookup_step(&root, NULL, longer, ARRAY_SIZE(longer), &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_DESCEND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, head); - KUNIT_EXPECT_EQ(test, lookup.matched, (unsigned int)ARRAY_SIZE(entries)); - - ret = lookup_step(&root, NULL, partial, ARRAY_SIZE(partial), &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_SPLIT); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, head); - KUNIT_EXPECT_EQ(test, lookup.matched, 1U); - - ret = lookup_step(&root, NULL, missing, ARRAY_SIZE(missing), &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_APPEND); - KUNIT_EXPECT_NULL(test, lookup.node); -} - -static void stackdepot_trie_lookup_step_parent_promote(struct kunit *test) -{ - unsigned long parent_entries[] = { 0x1000UL }; - unsigned long child_entries[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_lookup lookup; - struct stack_depot_trie_root root = {}; - void *child; - void *parent; - int ret; - - trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, - &parent); - trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), parent, 0, - &child); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = publish_append(NULL, parent, child, child_array.array, - child_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - ret = lookup_step(NULL, parent, child_entries, ARRAY_SIZE(child_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_PROMOTE); - KUNIT_EXPECT_PTR_EQ(test, lookup.parent, parent); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, child); - KUNIT_EXPECT_EQ(test, lookup.matched, - (unsigned int)ARRAY_SIZE(child_entries)); - - ret = lookup_step(&root, parent, child_entries, ARRAY_SIZE(child_entries), - &lookup); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = lookup_step(NULL, NULL, child_entries, ARRAY_SIZE(child_entries), - &lookup); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = lookup_step(NULL, parent, NULL, ARRAY_SIZE(child_entries), &lookup); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = lookup_step(NULL, parent, child_entries, 0, &lookup); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = lookup_step(NULL, parent, child_entries, ARRAY_SIZE(child_entries), - NULL); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} - -static void stackdepot_trie_lookup_step_accepts_reparented_child(struct kunit *test) -{ - unsigned long parent_entries[] = { 0x1000UL }; - unsigned long child_entries[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot child_slot; - struct stack_depot_trie_lookup lookup; - void *old_parent; - void *new_parent; - int ret; - - trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, - &old_parent); - trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 8, - &new_parent); - trie_node_slot_alloc(test, &child_slot, child_entries, - ARRAY_SIZE(child_entries)); - KUNIT_ASSERT_EQ(test, - tnode_init(child_slot.node, child_slot.size, old_parent, 9, - child_entries, ARRAY_SIZE(child_entries), NULL, 0), - 0); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = publish_append(NULL, old_parent, child_slot.node, child_array.array, - child_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_ASSERT_EQ(test, - tnode_init(child_slot.node, child_slot.size, new_parent, 9, - child_entries, ARRAY_SIZE(child_entries), NULL, 0), - 0); - - ret = lookup_step(NULL, old_parent, child_entries, ARRAY_SIZE(child_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.parent, old_parent); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, child_slot.node); - KUNIT_EXPECT_EQ(test, lookup.matched, - (unsigned int)ARRAY_SIZE(child_entries)); -} - -static void stackdepot_trie_find_leaf_root(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *head = NULL; - const void *tail = NULL; - unsigned int used = 0; - int ret; - - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = append_chain(NULL, 61, entries, ARRAY_SIZE(entries), &node_slot, 1, - NULL, 0, NULL, 0, &head, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, head, child_array.array, - child_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, entries, ARRAY_SIZE(entries)), - head); - KUNIT_EXPECT_NULL(test, find_leaf(NULL, entries, ARRAY_SIZE(entries))); - KUNIT_EXPECT_NULL(test, find_leaf(&root, NULL, ARRAY_SIZE(entries))); - KUNIT_EXPECT_NULL(test, find_leaf(&root, entries, 0)); -} - -static void stackdepot_trie_find_leaf_descends(struct kunit *test) -{ - unsigned long prefix_entries[] = { 0x1000UL }; - unsigned long full_entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_child_array_slot root_array; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot prefix_slot; - struct stack_depot_trie_node_slot child_slot; - struct stack_depot_trie_root root = {}; - const void *prefix = NULL; - const void *child = NULL; - unsigned int used = 0; - int ret; - - trie_node_slot_alloc(test, &prefix_slot, prefix_entries, - ARRAY_SIZE(prefix_entries)); - trie_node_slot_alloc(test, &child_slot, &full_entries[1], 1); - root_array.size = __stack_depot_trie_child_array_size(1); - root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, root_array.array); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append(&root, NULL, 62, prefix_entries, - ARRAY_SIZE(prefix_entries), &prefix_slot, 1, NULL, 0, - NULL, 0, root_array.array, root_array.size, &prefix, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = insert_append(&root, NULL, 63, full_entries, ARRAY_SIZE(full_entries), - &child_slot, 1, NULL, 0, NULL, 0, child_array.array, - child_array.size, &child, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - - KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, prefix_entries, - ARRAY_SIZE(prefix_entries)), prefix); - KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, full_entries, - ARRAY_SIZE(full_entries)), child); -} - -static void stackdepot_trie_find_leaf_accepts_reparented_child(struct kunit *test) -{ - unsigned long child_entries[] = { 0x1000UL, 0x2000UL }; - unsigned long desc_entries[] = { 0x4000UL }; - unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; - unsigned long old_stack[] = { 0x1000UL, 0x2000UL, 0x4000UL }; - struct stack_depot_trie_child_array_slot root_array; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_child_array_slot split_array; - struct stack_depot_trie_node_slot desc_slot; - struct stack_depot_trie_node_slot node_slots[3]; - struct stack_depot_trie_root root = {}; - const void *desc_head = NULL; - const void *desc_tail = NULL; - const void *new_tail = NULL; - const void *prefix = NULL; - unsigned int used = 0; - void *child; - int ret; - - trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), NULL, 0, - &child); - root_array.size = __stack_depot_trie_child_array_size(1); - root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, root_array.array); - ret = publish_append(&root, NULL, child, root_array.array, - root_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - trie_node_slot_alloc(test, &desc_slot, desc_entries, - ARRAY_SIZE(desc_entries)); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = append_chain(child, 3, desc_entries, ARRAY_SIZE(desc_entries), - &desc_slot, 1, NULL, 0, NULL, 0, &desc_head, - &desc_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(NULL, child, desc_head, child_array.array, - child_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - trie_node_slot_alloc(test, &node_slots[0], child_entries, 1); - trie_node_slot_alloc(test, &node_slots[1], &child_entries[1], 1); - trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); - split_array.size = __stack_depot_trie_child_array_size(2); - split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, split_array.array); - ret = split_subtree(child, 1, 4, new_entries, ARRAY_SIZE(new_entries), - node_slots, ARRAY_SIZE(node_slots), &split_array, 1, - NULL, 0, &prefix, &new_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - - KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, old_stack, ARRAY_SIZE(old_stack)), - desc_tail); - KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[2].node); - KUNIT_EXPECT_PTR_EQ(test, prefix, node_slots[0].node); -} - -static void stackdepot_trie_find_leaf_misses(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - unsigned long split_miss[] = { 0x1000UL, 0x2222UL }; - unsigned long append_miss[] = { 0x3000UL }; - unsigned long prefix_miss[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *head = NULL; - const void *tail = NULL; - unsigned int used = 0; - int ret; - - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = append_chain(NULL, 64, entries, ARRAY_SIZE(entries), &node_slot, 1, - NULL, 0, NULL, 0, &head, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, head, child_array.array, - child_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - KUNIT_EXPECT_NULL(test, find_leaf(&root, split_miss, - ARRAY_SIZE(split_miss))); - KUNIT_EXPECT_NULL(test, find_leaf(&root, append_miss, - ARRAY_SIZE(append_miss))); - KUNIT_EXPECT_NULL(test, find_leaf(&root, prefix_miss, - ARRAY_SIZE(prefix_miss))); -} - -static void stackdepot_trie_find_leaf_rejects_bad_parent(struct kunit *test) -{ - unsigned long parent_entries[] = { 0x1000UL }; - unsigned long wrong_parent_entries[] = { 0x1111UL }; - unsigned long full_entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_child_array_slot root_array; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot parent_slot; - struct stack_depot_trie_node_slot child_slot; - struct stack_depot_trie_root root = {}; - void *wrong_parent; - const void *parent = NULL; - const void *child = NULL; - unsigned int used = 0; - int ret; - - trie_node_slot_alloc(test, &parent_slot, parent_entries, - ARRAY_SIZE(parent_entries)); - trie_node_slot_alloc(test, &child_slot, &full_entries[1], 1); - trie_node_alloc(test, wrong_parent_entries, ARRAY_SIZE(wrong_parent_entries), - NULL, 66, &wrong_parent); - root_array.size = __stack_depot_trie_child_array_size(1); - root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, root_array.array); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append(&root, NULL, 67, parent_entries, - ARRAY_SIZE(parent_entries), &parent_slot, 1, NULL, 0, - NULL, 0, root_array.array, root_array.size, &parent, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = insert_append(&root, NULL, 68, full_entries, ARRAY_SIZE(full_entries), - &child_slot, 1, NULL, 0, NULL, 0, child_array.array, - child_array.size, &child, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = tnode_init(child_slot.node, child_slot.size, wrong_parent, 68, - &full_entries[1], 1, NULL, 0); - KUNIT_ASSERT_EQ(test, ret, 0); - - KUNIT_EXPECT_NULL(test, find_leaf(&root, full_entries, - ARRAY_SIZE(full_entries))); -} - -static void stackdepot_trie_insert_append_root(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *tail = NULL; - unsigned int used = 0; - int ret; - - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append(&root, NULL, 31, entries, ARRAY_SIZE(entries), - &node_slot, 1, NULL, 0, NULL, 0, child_array.array, - child_array.size, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, tail, node_slot.node); - KUNIT_EXPECT_EQ(test, used, 1U); - KUNIT_EXPECT_PTR_EQ(test, root.children, child_array.array); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, entries[0]), - node_slot.node); -} - -static void stackdepot_trie_insert_append_prepare_root(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - struct stackdepot_trie_prepare_ctx ctx = { - .test = test, - .visible = &root.children, - .expected_visible = NULL, - .expected_leaf_id = { 69 }, - .nr_expected = 1, - }; - struct stack_depot_trie_publish_prepare prepare = { - .fn = stackdepot_trie_prepare, - .ctx = &ctx, - }; - const void *tail = NULL; - unsigned int used = 0; - int ret; - - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - ctx.expected_leaf[0] = node_slot.node; - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append_prepare(&root, NULL, 69, entries, ARRAY_SIZE(entries), - &node_slot, 1, NULL, 0, NULL, 0, - child_array.array, child_array.size, - &prepare, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, ctx.calls, 1U); - KUNIT_EXPECT_PTR_EQ(test, root.children, child_array.array); - KUNIT_EXPECT_PTR_EQ(test, tail, node_slot.node); - KUNIT_EXPECT_EQ(test, used, 1U); -} - -static void stackdepot_trie_insert_append_prepare_failure(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - struct stackdepot_trie_prepare_ctx ctx = { - .test = test, - .visible = &root.children, - .expected_visible = NULL, - .expected_leaf_id = { 70 }, - .nr_expected = 1, - .ret = -EAGAIN, - }; - struct stack_depot_trie_publish_prepare prepare = { - .fn = stackdepot_trie_prepare, - .ctx = &ctx, - }; - const void *tail = (const void *)1; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - ctx.expected_leaf[0] = node_slot.node; - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append_prepare(&root, NULL, 70, entries, ARRAY_SIZE(entries), - &node_slot, 1, NULL, 0, NULL, 0, - child_array.array, child_array.size, - &prepare, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EAGAIN); - KUNIT_EXPECT_EQ(test, ctx.calls, 1U); - KUNIT_EXPECT_NULL(test, root.children); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_prepare_promote_failure(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - struct stackdepot_trie_prepare_ctx ctx = { - .test = test, - .visible = &root.children, - .expected_leaf_id = { 72 }, - .nr_expected = 1, - .ret = -EAGAIN, - }; - struct stack_depot_trie_publish_prepare prepare = { - .fn = stackdepot_trie_prepare, - .ctx = &ctx, - }; - const void *children[1]; - const void *tail = (const void *)1; - unsigned int used = 99; - void *child; - int ret; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &child); - children[0] = child; - old_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = child_array_init(old_array.array, old_array.size, children, - ARRAY_SIZE(children)); - KUNIT_ASSERT_EQ(test, ret, 0); - root.children = old_array.array; - ctx.expected_visible = old_array.array; - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - ctx.expected_leaf[0] = node_slot.node; - new_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); - - ret = insert_append_prepare(&root, NULL, 72, entries, ARRAY_SIZE(entries), - &node_slot, 1, NULL, 0, NULL, 0, - new_array.array, new_array.size, - &prepare, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EAGAIN); - KUNIT_EXPECT_EQ(test, ctx.calls, 1U); - KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_prepare_split(struct kunit *test) -{ - unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; - unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot split_array; - struct stack_depot_trie_child_array_slot replace_array; - struct stack_depot_trie_node_slot old_slot; - struct stack_depot_trie_node_slot node_slots[3]; - struct stack_depot_trie_root root = {}; - struct stackdepot_trie_prepare_ctx ctx = { - .test = test, - .visible = &root.children, - .expected_leaf_id = { 73, 74 }, - .nr_expected = 2, - }; - struct stack_depot_trie_publish_prepare prepare = { - .fn = stackdepot_trie_prepare, - .ctx = &ctx, - }; - const void *old_head = NULL; - const void *old_tail = NULL; - const void *new_tail = NULL; - unsigned int used = 0; - int ret; - - trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); - old_array.size = __stack_depot_trie_child_array_size(1); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = append_chain(NULL, 73, old_entries, ARRAY_SIZE(old_entries), - &old_slot, 1, NULL, 0, NULL, 0, &old_head, - &old_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, old_head, old_array.array, - old_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - ctx.expected_visible = old_array.array; - - trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); - trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); - trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); - ctx.expected_leaf[0] = node_slots[1].node; - ctx.expected_leaf[1] = node_slots[2].node; - split_array.size = __stack_depot_trie_child_array_size(2); - split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, split_array.array); - replace_array.size = __stack_depot_trie_child_array_size(1); - replace_array.array = kunit_kzalloc(test, replace_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, replace_array.array); - - ret = insert_append_prepare(&root, NULL, 74, new_entries, - ARRAY_SIZE(new_entries), node_slots, - ARRAY_SIZE(node_slots), &split_array, 1, - NULL, 0, replace_array.array, - replace_array.size, &prepare, &new_tail, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, ctx.calls, 1U); - KUNIT_EXPECT_PTR_EQ(test, root.children, replace_array.array); - KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[2].node); -} - -static void stackdepot_trie_insert_append_prepare_split_failure(struct kunit *test) -{ - unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; - unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot split_array; - struct stack_depot_trie_child_array_slot replace_array; - struct stack_depot_trie_node_slot old_slot; - struct stack_depot_trie_node_slot node_slots[3]; - struct stack_depot_trie_root root = {}; - const void *found; - struct stackdepot_trie_prepare_ctx ctx = { - .test = test, - .visible = &root.children, - .expected_leaf_id = { 75, 76 }, - .nr_expected = 2, - .ret = -EAGAIN, - }; - struct stack_depot_trie_publish_prepare prepare = { - .fn = stackdepot_trie_prepare, - .ctx = &ctx, - }; - const void *old_head = NULL; - const void *old_tail = NULL; - const void *new_tail = (const void *)1; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); - old_array.size = __stack_depot_trie_child_array_size(1); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = append_chain(NULL, 75, old_entries, ARRAY_SIZE(old_entries), - &old_slot, 1, NULL, 0, NULL, 0, &old_head, - &old_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, old_head, old_array.array, - old_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - ctx.expected_visible = old_array.array; - - trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); - trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); - trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); - ctx.expected_leaf[0] = node_slots[1].node; - ctx.expected_leaf[1] = node_slots[2].node; - split_array.size = __stack_depot_trie_child_array_size(2); - split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, split_array.array); - replace_array.size = __stack_depot_trie_child_array_size(1); - replace_array.array = kunit_kzalloc(test, replace_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, replace_array.array); - new_tail = (const void *)1; - used = 99; - - ret = insert_append_prepare(&root, NULL, 76, new_entries, - ARRAY_SIZE(new_entries), node_slots, - ARRAY_SIZE(node_slots), &split_array, 1, - NULL, 0, replace_array.array, - replace_array.size, &prepare, &new_tail, - &used); - KUNIT_EXPECT_EQ(test, ret, -EAGAIN); - KUNIT_EXPECT_EQ(test, ctx.calls, 1U); - KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); - KUNIT_EXPECT_PTR_EQ(test, find_leaf(&root, old_entries, - ARRAY_SIZE(old_entries)), old_tail); - found = find_leaf(&root, new_entries, ARRAY_SIZE(new_entries)); - KUNIT_EXPECT_NULL(test, found); - KUNIT_EXPECT_PTR_EQ(test, new_tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_parent(struct kunit *test) -{ - unsigned long parent_entries[] = { 0x1000UL }; - unsigned long old_entries[] = { 0x2000UL }; - unsigned long new_entries[] = { 0x3000UL }; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot old_slot; - struct stack_depot_trie_node_slot new_slot; - struct stack_depot_trie_lookup lookup; - const void *old_head = NULL; - const void *old_tail = NULL; - const void *new_tail = NULL; - unsigned int used = 0; - void *parent; - int ret; - - trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, - &parent); - trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); - trie_node_slot_alloc(test, &new_slot, new_entries, ARRAY_SIZE(new_entries)); - old_array.size = __stack_depot_trie_child_array_size(1); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - new_array.size = __stack_depot_trie_child_array_size(2); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); - - ret = append_chain(parent, 32, old_entries, ARRAY_SIZE(old_entries), - &old_slot, 1, NULL, 0, NULL, 0, &old_head, &old_tail, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(NULL, parent, old_head, old_array.array, - old_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - ret = insert_append(NULL, parent, 33, new_entries, ARRAY_SIZE(new_entries), - &new_slot, 1, NULL, 0, NULL, 0, new_array.array, - new_array.size, &new_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = lookup_step(NULL, parent, old_entries, ARRAY_SIZE(old_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, old_head); - ret = lookup_step(NULL, parent, new_entries, ARRAY_SIZE(new_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, new_tail); - KUNIT_EXPECT_EQ(test, used, 1U); -} - -static void stackdepot_trie_insert_append_descends_one_level(struct kunit *test) -{ - unsigned long prefix_entries[] = { 0x1000UL }; - unsigned long child_entries[] = { 0x2000UL }; - unsigned long stack_entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_child_array_slot root_array; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot prefix_slot; - struct stack_depot_trie_node_slot child_slot; - struct stack_depot_trie_lookup lookup; - struct stack_depot_trie_root root = {}; - unsigned long out[ARRAY_SIZE(stack_entries)] = {}; - const void *prefix = NULL; - const void *tail = NULL; - unsigned int fetched; - unsigned int used = 0; - int ret; - - trie_node_slot_alloc(test, &prefix_slot, prefix_entries, - ARRAY_SIZE(prefix_entries)); - trie_node_slot_alloc(test, &child_slot, child_entries, - ARRAY_SIZE(child_entries)); - root_array.size = __stack_depot_trie_child_array_size(1); - root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, root_array.array); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append(&root, NULL, 47, prefix_entries, - ARRAY_SIZE(prefix_entries), &prefix_slot, 1, NULL, 0, - NULL, 0, root_array.array, root_array.size, &prefix, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = insert_append(&root, NULL, 48, stack_entries, - ARRAY_SIZE(stack_entries), &child_slot, 1, NULL, 0, - NULL, 0, child_array.array, child_array.size, &tail, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, used, 1U); - ret = lookup_step(NULL, prefix, child_entries, ARRAY_SIZE(child_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, tail); - fetched = tfetch(tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(stack_entries)); - KUNIT_EXPECT_MEMEQ(test, out, stack_entries, sizeof(stack_entries)); -} - -static void stackdepot_trie_insert_append_descends_multiple_levels(struct kunit *test) -{ - unsigned long first_entries[] = { 0x1000UL }; - unsigned long second_entries[] = { 0x2000UL }; - unsigned long tail_entries[] = { 0x3000UL }; - unsigned long second_stack[] = { 0x1000UL, 0x2000UL }; - unsigned long full_stack[] = { 0x1000UL, 0x2000UL, 0x3000UL }; - struct stack_depot_trie_child_array_slot first_array; - struct stack_depot_trie_child_array_slot second_array; - struct stack_depot_trie_child_array_slot third_array; - struct stack_depot_trie_node_slot first_slot; - struct stack_depot_trie_node_slot second_slot; - struct stack_depot_trie_node_slot third_slot; - struct stack_depot_trie_lookup lookup; - struct stack_depot_trie_root root = {}; - unsigned long out[ARRAY_SIZE(full_stack)] = {}; - const void *first = NULL; - const void *second = NULL; - const void *tail = NULL; - unsigned int used = 0; - unsigned int fetched; - int ret; - - trie_node_slot_alloc(test, &first_slot, first_entries, - ARRAY_SIZE(first_entries)); - trie_node_slot_alloc(test, &second_slot, second_entries, - ARRAY_SIZE(second_entries)); - trie_node_slot_alloc(test, &third_slot, tail_entries, - ARRAY_SIZE(tail_entries)); - first_array.size = __stack_depot_trie_child_array_size(1); - first_array.array = kunit_kzalloc(test, first_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, first_array.array); - second_array.size = __stack_depot_trie_child_array_size(1); - second_array.array = kunit_kzalloc(test, second_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, second_array.array); - third_array.size = __stack_depot_trie_child_array_size(1); - third_array.array = kunit_kzalloc(test, third_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, third_array.array); - - ret = insert_append(&root, NULL, 55, first_entries, - ARRAY_SIZE(first_entries), &first_slot, 1, NULL, 0, - NULL, 0, first_array.array, first_array.size, &first, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = insert_append(&root, NULL, 56, second_stack, ARRAY_SIZE(second_stack), - &second_slot, 1, NULL, 0, NULL, 0, - second_array.array, second_array.size, &second, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = insert_append(&root, NULL, 57, full_stack, ARRAY_SIZE(full_stack), - &third_slot, 1, NULL, 0, NULL, 0, third_array.array, - third_array.size, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, used, 1U); - ret = lookup_step(NULL, second, tail_entries, ARRAY_SIZE(tail_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, tail); - fetched = tfetch(tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(full_stack)); - KUNIT_EXPECT_MEMEQ(test, out, full_stack, sizeof(full_stack)); -} - -static void stackdepot_trie_insert_append_descend_rejects_sibling_overlap(struct kunit *test) -{ - unsigned long prefix_entries[] = { 0x1000UL }; - unsigned long sibling_entries[] = { 0x9000UL }; - unsigned long stack_entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_child_array_slot root_array; - struct stack_depot_trie_child_array_slot sibling_array; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot prefix_slot; - struct stack_depot_trie_node_slot sibling_slot; - struct stack_depot_trie_root root = {}; - struct stack_depot_trie_lookup lookup; - const void *prefix = NULL; - const void *sibling = NULL; - const void *tail = (const void *)1; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &prefix_slot, prefix_entries, - ARRAY_SIZE(prefix_entries)); - trie_node_slot_alloc(test, &sibling_slot, sibling_entries, - ARRAY_SIZE(sibling_entries)); - root_array.size = __stack_depot_trie_child_array_size(1); - root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, root_array.array); - sibling_array.size = __stack_depot_trie_child_array_size(2); - sibling_array.array = kunit_kzalloc(test, sibling_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, sibling_array.array); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append(&root, NULL, 49, prefix_entries, - ARRAY_SIZE(prefix_entries), &prefix_slot, 1, NULL, 0, - NULL, 0, root_array.array, root_array.size, &prefix, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = insert_append(&root, NULL, 50, sibling_entries, - ARRAY_SIZE(sibling_entries), &sibling_slot, 1, NULL, 0, - NULL, 0, sibling_array.array, sibling_array.size, - &sibling, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - used = 99; - - ret = insert_append(&root, NULL, 51, stack_entries, - ARRAY_SIZE(stack_entries), &sibling_slot, 1, NULL, 0, - NULL, 0, child_array.array, child_array.size, &tail, - &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); - ret = lookup_step(&root, NULL, sibling_entries, ARRAY_SIZE(sibling_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, sibling); -} - -static void stackdepot_trie_insert_append_promotes_internal(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot promote_slot; - struct stack_depot_trie_lookup lookup; - struct stack_depot_trie_root root = {}; - unsigned long out[ARRAY_SIZE(entries)] = {}; - const void *children[1]; - const void *tail = (const void *)1; - unsigned int fetched; - unsigned int used = 99; - void *old_child; - int ret; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &old_child); - children[0] = old_child; - old_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = child_array_init(old_array.array, old_array.size, children, - ARRAY_SIZE(children)); - KUNIT_ASSERT_EQ(test, ret, 0); - root.children = old_array.array; - trie_node_slot_alloc(test, &promote_slot, entries, ARRAY_SIZE(entries)); - new_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); - - ret = insert_append(&root, NULL, 58, entries, ARRAY_SIZE(entries), - &promote_slot, 1, NULL, 0, NULL, 0, new_array.array, - new_array.size, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, root.children, new_array.array); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, entries[0]), tail); - KUNIT_EXPECT_PTR_EQ(test, tail, promote_slot.node); - KUNIT_EXPECT_EQ(test, used, 1U); - fetched = tfetch(tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); - ret = lookup_step(&root, NULL, entries, ARRAY_SIZE(entries), &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, tail); -} - -static void stackdepot_trie_insert_append_descends_to_promote(struct kunit *test) -{ - unsigned long prefix_entries[] = { 0x1000UL }; - unsigned long child_entries[] = { 0x2000UL }; - unsigned long stack_entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_child_array_slot root_array; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot promote_slot; - struct stack_depot_trie_lookup lookup; - struct stack_depot_trie_root root = {}; - unsigned long out[ARRAY_SIZE(stack_entries)] = {}; - const void *root_children[1]; - const void *tail = (const void *)1; - unsigned int fetched; - unsigned int used = 99; - void *child; - void *prefix; - int ret; - - trie_node_alloc(test, prefix_entries, ARRAY_SIZE(prefix_entries), NULL, 0, - &prefix); - trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), prefix, 0, - &child); - root_children[0] = prefix; - root_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(root_children)); - root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, root_array.array); - ret = child_array_init(root_array.array, root_array.size, root_children, - ARRAY_SIZE(root_children)); - KUNIT_ASSERT_EQ(test, ret, 0); - root.children = root_array.array; - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = publish_append(NULL, prefix, child, child_array.array, - child_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - trie_node_slot_alloc(test, &promote_slot, child_entries, - ARRAY_SIZE(child_entries)); - new_array.size = __stack_depot_trie_child_array_size(1); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); - - ret = insert_append(&root, NULL, 61, stack_entries, ARRAY_SIZE(stack_entries), - &promote_slot, 1, NULL, 0, NULL, 0, new_array.array, - new_array.size, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, root.children, root_array.array); - ret = lookup_step(NULL, prefix, child_entries, ARRAY_SIZE(child_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, tail); - KUNIT_EXPECT_EQ(test, used, 1U); - fetched = tfetch(tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(stack_entries)); - KUNIT_EXPECT_MEMEQ(test, out, stack_entries, sizeof(stack_entries)); -} - -static void stackdepot_trie_insert_append_promotes_with_children(struct kunit *test) -{ - unsigned long parent_entries[] = { 0x1000UL }; - unsigned long child_entries[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot root_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot promote_slot; - struct stack_depot_trie_lookup lookup; - struct stack_depot_trie_root root = {}; - unsigned long expected[] = { 0x1000UL, 0x2000UL }; - unsigned long out[ARRAY_SIZE(expected)] = {}; - const void *root_children[1]; - const void *tail = NULL; - unsigned int fetched; - unsigned int used = 99; - void *child; - void *parent; - int ret; - - trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 0, - &parent); - trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), parent, 59, - &child); - root_children[0] = parent; - root_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(root_children)); - root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, root_array.array); - ret = child_array_init(root_array.array, root_array.size, root_children, - ARRAY_SIZE(root_children)); - KUNIT_ASSERT_EQ(test, ret, 0); - root.children = root_array.array; - new_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(root_children)); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); - ret = publish_append(NULL, parent, child, new_array.array, new_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - trie_node_slot_alloc(test, &promote_slot, parent_entries, - ARRAY_SIZE(parent_entries)); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); - - ret = insert_append(&root, NULL, 60, parent_entries, - ARRAY_SIZE(parent_entries), &promote_slot, 1, NULL, 0, - NULL, 0, new_array.array, new_array.size, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, root.children, new_array.array); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, parent_entries[0]), - tail); - ret = lookup_step(NULL, tail, child_entries, ARRAY_SIZE(child_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, child); - KUNIT_EXPECT_EQ(test, used, 1U); - fetched = tfetch(child, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); - KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); -} - -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) -static void stackdepot_trie_insert_append_splits_frame_runs(struct kunit *test) -{ - unsigned long entries[] = { -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x1000UL, - arch_stack_depot_frame_text_prefix() | 0x2000UL, - 0x1000UL, - arch_stack_depot_frame_text_prefix() | 0x3000UL, -#else - 0xffffffff81001000UL, - 0xffffffff81002000UL, - 0xffff888000001000UL, - 0xffffffff81003000UL, -#endif - }; - struct stack_depot_trie_node_slot node_slots[3]; - struct stack_depot_trie_child_array_slot child_slots[2]; - struct stack_depot_trie_child_array_slot root_array; - struct stack_depot_trie_root root = {}; - unsigned long out[ARRAY_SIZE(entries)] = {}; - u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; - const void *tail = NULL; - unsigned int used = 0; - unsigned int fetched; - unsigned int i; - int ret; - - trie_node_slot_alloc(test, &node_slots[0], entries, 2); - trie_node_slot_alloc(test, &node_slots[1], &entries[2], 1); - trie_node_slot_alloc(test, &node_slots[2], &entries[3], 1); - - for (i = 0; i < ARRAY_SIZE(child_slots); i++) { - child_slots[i].size = __stack_depot_trie_child_array_size(1); - child_slots[i].array = kunit_kzalloc(test, child_slots[i].size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_slots[i].array); - } - root_array.size = __stack_depot_trie_child_array_size(1); - root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, root_array.array); - - ret = insert_append(&root, NULL, 42, entries, ARRAY_SIZE(entries), - node_slots, ARRAY_SIZE(node_slots), child_slots, - ARRAY_SIZE(child_slots), write_scratch, - ARRAY_SIZE(write_scratch), root_array.array, - root_array.size, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, used, 3U); - KUNIT_EXPECT_PTR_EQ(test, tail, node_slots[2].node); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(root.children, entries[0]), - node_slots[0].node); - fetched = tfetch(tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); -} -#endif - -static void stackdepot_trie_insert_append_splits_child(struct kunit *test) -{ - unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; - unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot split_array; - struct stack_depot_trie_child_array_slot replace_array; - struct stack_depot_trie_node_slot old_slot; - struct stack_depot_trie_node_slot node_slots[3]; - struct stack_depot_trie_lookup lookup; - struct stack_depot_trie_root root = {}; - unsigned long out[ARRAY_SIZE(old_entries)] = {}; - const void *old_head = NULL; - const void *old_tail; - const void *new_tail = NULL; - const void *prefix; - unsigned int fetched; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); - old_array.size = __stack_depot_trie_child_array_size(1); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = append_chain(NULL, 1, old_entries, ARRAY_SIZE(old_entries), - &old_slot, 1, NULL, 0, NULL, 0, &old_head, - &old_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, old_head, old_array.array, - old_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); - trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); - trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); - split_array.size = __stack_depot_trie_child_array_size(2); - split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, split_array.array); - replace_array.size = __stack_depot_trie_child_array_size(1); - replace_array.array = kunit_kzalloc(test, replace_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, replace_array.array); - used = 99; - - ret = insert_append(&root, NULL, 2, new_entries, ARRAY_SIZE(new_entries), - node_slots, ARRAY_SIZE(node_slots), &split_array, 1, - NULL, 0, replace_array.array, replace_array.size, - &new_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, used, 3U); - KUNIT_EXPECT_PTR_EQ(test, root.children, replace_array.array); - KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[2].node); - - ret = lookup_step(&root, NULL, old_entries, ARRAY_SIZE(old_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_DESCEND); - prefix = lookup.node; - ret = lookup_step(NULL, prefix, &old_entries[1], 1, &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - old_tail = lookup.node; - fetched = tfetch(old_tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, 2U); - KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); - - memset(out, 0, sizeof(out)); - ret = lookup_step(&root, NULL, new_entries, ARRAY_SIZE(new_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_DESCEND); - ret = lookup_step(NULL, lookup.node, &new_entries[1], 1, &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, new_tail); - fetched = tfetch(new_tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, 2U); - KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); -} - -static void stackdepot_trie_insert_append_splits_prefix_leaf(struct kunit *test) -{ - unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; - unsigned long new_entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot split_array; - struct stack_depot_trie_child_array_slot replace_array; - struct stack_depot_trie_node_slot old_slot; - struct stack_depot_trie_node_slot node_slots[2]; - struct stack_depot_trie_lookup lookup; - struct stack_depot_trie_root root = {}; - unsigned long out[ARRAY_SIZE(old_entries)] = {}; - const void *old_head = NULL; - const void *old_tail; - const void *new_tail = NULL; - unsigned int fetched; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); - old_array.size = __stack_depot_trie_child_array_size(1); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = append_chain(NULL, 1, old_entries, ARRAY_SIZE(old_entries), - &old_slot, 1, NULL, 0, NULL, 0, &old_head, - &old_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, old_head, old_array.array, - old_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); - trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); - split_array.size = __stack_depot_trie_child_array_size(1); - split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, split_array.array); - replace_array.size = __stack_depot_trie_child_array_size(1); - replace_array.array = kunit_kzalloc(test, replace_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, replace_array.array); - used = 99; - - ret = insert_append(&root, NULL, 2, new_entries, ARRAY_SIZE(new_entries), - node_slots, ARRAY_SIZE(node_slots), &split_array, 1, - NULL, 0, replace_array.array, replace_array.size, - &new_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, used, 2U); - KUNIT_EXPECT_PTR_EQ(test, root.children, replace_array.array); - KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[0].node); - ret = lookup_step(&root, NULL, new_entries, ARRAY_SIZE(new_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, new_tail); - fetched = tfetch(new_tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, 1U); - KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); - - ret = lookup_step(&root, NULL, old_entries, ARRAY_SIZE(old_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_DESCEND); - ret = lookup_step(NULL, lookup.node, &old_entries[1], 1, &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - memset(out, 0, sizeof(out)); - fetched = tfetch(lookup.node, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, 2U); - KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); -} - -static void stackdepot_trie_insert_plan_append(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_slot; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_frame_run run; - struct stack_depot_trie_root root = {}; - unsigned int child_used = 99; - unsigned int used = 99; - size_t publish_size = 0; - int ret; - - ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), &node_slot, - 1, &child_slot, 1, &publish_size, &used, - &child_used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_NULL(test, node_slot.node); - KUNIT_EXPECT_EQ(test, node_slot.size, __stack_depot_trie_node_size(&run)); - KUNIT_EXPECT_EQ(test, used, 1U); - KUNIT_EXPECT_EQ(test, child_used, 0U); - KUNIT_EXPECT_EQ(test, publish_size, __stack_depot_trie_child_array_size(1)); -} - -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) -static void stackdepot_trie_insert_plan_mixed_append(struct kunit *test) -{ - unsigned long entries[] = { -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x1000UL, - 0x1000UL, -#else - 0xffffffff81001000UL, - 0xffff888000001000UL, -#endif - }; - struct stack_depot_trie_child_array_slot child_slot; - struct stack_depot_trie_node_slot node_slots[2]; - struct stack_depot_frame_run first_run; - struct stack_depot_frame_run second_run; - struct stack_depot_trie_root root = {}; - unsigned int child_used = 99; - unsigned int used = 99; - size_t publish_size = 0; - int ret; - - ret = frame_run_init(entries, ARRAY_SIZE(entries), &first_run); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = frame_run_init(&entries[first_run.nr_entries], - ARRAY_SIZE(entries) - first_run.nr_entries, - &second_run); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), node_slots, - ARRAY_SIZE(node_slots), &child_slot, 1, &publish_size, - &used, &child_used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, used, 2U); - KUNIT_EXPECT_EQ(test, child_used, 1U); - KUNIT_EXPECT_EQ(test, node_slots[0].size, - __stack_depot_trie_node_size(&first_run)); - KUNIT_EXPECT_EQ(test, node_slots[1].size, - __stack_depot_trie_node_size(&second_run)); - KUNIT_EXPECT_EQ(test, child_slot.size, - __stack_depot_trie_child_array_size(1)); - KUNIT_EXPECT_EQ(test, publish_size, __stack_depot_trie_child_array_size(1)); -} -#endif - -static void stackdepot_trie_insert_plan_promote(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_frame_run run; - struct stack_depot_trie_root root = {}; - const void *children[1]; - unsigned int child_used = 99; - unsigned int used = 99; - size_t publish_size = 0; - void *child; - int ret; - - ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); - KUNIT_ASSERT_EQ(test, ret, 0); - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &child); - children[0] = child; - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = child_array_init(child_array.array, child_array.size, children, - ARRAY_SIZE(children)); - KUNIT_ASSERT_EQ(test, ret, 0); - root.children = child_array.array; - - ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), &node_slot, - 1, NULL, 0, &publish_size, &used, &child_used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_NULL(test, node_slot.node); - KUNIT_EXPECT_EQ(test, node_slot.size, __stack_depot_trie_node_size(&run)); - KUNIT_EXPECT_EQ(test, used, 1U); - KUNIT_EXPECT_EQ(test, child_used, 0U); - KUNIT_EXPECT_EQ(test, publish_size, __stack_depot_trie_child_array_size(1)); -} - -static void stackdepot_trie_insert_plan_promote_rejects_empty_slots(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot = { - .node = (void *)1, - .size = 99, - }; - struct stack_depot_trie_root root = {}; - const void *children[1]; - unsigned int child_used = 99; - unsigned int used = 99; - size_t publish_size = 0; - void *child; - int ret; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 0, &child); - children[0] = child; - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = child_array_init(child_array.array, child_array.size, children, - ARRAY_SIZE(children)); - KUNIT_ASSERT_EQ(test, ret, 0); - root.children = child_array.array; - - ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), &node_slot, - 0, NULL, 0, &publish_size, &used, &child_used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_PTR_EQ(test, node_slot.node, (void *)1); - KUNIT_EXPECT_EQ(test, node_slot.size, (size_t)99); -} - -static void stackdepot_trie_insert_plan_split(struct kunit *test) -{ - unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; - unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; - struct stack_depot_trie_child_array_slot child_slot; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_node_slot node_slots[3]; - struct stack_depot_trie_node_slot old_slot; - struct stack_depot_frame_run run; - struct stack_depot_trie_root root = {}; - const void *old_head = NULL; - const void *old_tail = NULL; - unsigned int child_used = 99; - unsigned int used = 99; - size_t publish_size = 0; - int ret; - - trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); - old_array.size = __stack_depot_trie_child_array_size(1); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = append_chain(NULL, 1, old_entries, ARRAY_SIZE(old_entries), - &old_slot, 1, NULL, 0, NULL, 0, &old_head, - &old_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, old_head, old_array.array, - old_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = frame_run_init(new_entries, 1, &run); - KUNIT_ASSERT_EQ(test, ret, 0); - - ret = insert_plan(&root, NULL, new_entries, ARRAY_SIZE(new_entries), - node_slots, ARRAY_SIZE(node_slots), &child_slot, 1, - &publish_size, &used, &child_used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, used, 3U); - KUNIT_EXPECT_EQ(test, child_used, 1U); - KUNIT_EXPECT_EQ(test, node_slots[0].size, - __stack_depot_trie_node_size(&run)); - KUNIT_EXPECT_EQ(test, node_slots[1].size, - __stack_depot_trie_node_size(&run)); - KUNIT_EXPECT_EQ(test, node_slots[2].size, - __stack_depot_trie_node_size(&run)); - KUNIT_EXPECT_EQ(test, child_slot.size, - __stack_depot_trie_child_array_size(2)); - KUNIT_EXPECT_EQ(test, publish_size, __stack_depot_trie_child_array_size(1)); -} - -static void stackdepot_trie_insert_plan_descends(struct kunit *test) -{ - unsigned long prefix_entries[] = { 0x1000UL }; - unsigned long stack_entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_child_array_slot root_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_node_slot prefix_slot; - struct stack_depot_frame_run run; - struct stack_depot_trie_root root = {}; - const void *prefix = NULL; - unsigned int child_used = 99; - unsigned int used = 99; - size_t publish_size = 0; - int ret; - - trie_node_slot_alloc(test, &prefix_slot, prefix_entries, - ARRAY_SIZE(prefix_entries)); - root_array.size = __stack_depot_trie_child_array_size(1); - root_array.array = kunit_kzalloc(test, root_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, root_array.array); - ret = insert_append(&root, NULL, 75, prefix_entries, - ARRAY_SIZE(prefix_entries), &prefix_slot, 1, NULL, 0, - NULL, 0, root_array.array, root_array.size, &prefix, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = frame_run_init(&stack_entries[1], 1, &run); - KUNIT_ASSERT_EQ(test, ret, 0); - - ret = insert_plan(&root, NULL, stack_entries, ARRAY_SIZE(stack_entries), - &node_slot, 1, NULL, 0, &publish_size, &used, - &child_used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, used, 1U); - KUNIT_EXPECT_EQ(test, child_used, 0U); - KUNIT_EXPECT_EQ(test, node_slot.size, __stack_depot_trie_node_size(&run)); - KUNIT_EXPECT_EQ(test, publish_size, __stack_depot_trie_child_array_size(1)); -} - -static void stackdepot_trie_insert_plan_rejects_existing_leaf(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *tail = NULL; - unsigned int child_used = 99; - unsigned int used = 99; - size_t publish_size = 0; - int ret; - - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = insert_append(&root, NULL, 76, entries, ARRAY_SIZE(entries), - &node_slot, 1, NULL, 0, NULL, 0, child_array.array, - child_array.size, &tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - - ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), &node_slot, - 1, NULL, 0, &publish_size, &used, &child_used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} - -static void stackdepot_trie_insert_plan_rejects_bad_child(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *children[1]; - unsigned int child_used = 99; - unsigned int used = 99; - size_t publish_size = 0; - void *bad_parent; - void *child; - int ret; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 77, &bad_parent); - trie_node_alloc(test, entries, ARRAY_SIZE(entries), bad_parent, 78, &child); - children[0] = child; - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = child_array_init(child_array.array, child_array.size, children, - ARRAY_SIZE(children)); - KUNIT_ASSERT_EQ(test, ret, 0); - root.children = child_array.array; - - ret = insert_plan(&root, NULL, entries, ARRAY_SIZE(entries), &node_slot, - 1, NULL, 0, &publish_size, &used, &child_used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} - -static void stackdepot_trie_insert_append_rejects_existing_child(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot old_slot; - struct stack_depot_trie_node_slot dup_slot; - struct stack_depot_trie_root root = {}; - unsigned char *old; - const void *old_head = NULL; - const void *old_tail = NULL; - const void *tail = (const void *)1; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &old_slot, entries, ARRAY_SIZE(entries)); - trie_node_slot_alloc(test, &dup_slot, entries, ARRAY_SIZE(entries)); - old = kunit_kzalloc(test, dup_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old); - memset(dup_slot.node, 0xaa, dup_slot.size); - memcpy(old, dup_slot.node, dup_slot.size); - old_array.size = __stack_depot_trie_child_array_size(1); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - new_array.size = __stack_depot_trie_child_array_size(2); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); - - ret = append_chain(NULL, 34, entries, ARRAY_SIZE(entries), &old_slot, 1, - NULL, 0, NULL, 0, &old_head, &old_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, old_head, old_array.array, - old_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - used = 99; - - ret = insert_append(&root, NULL, 35, entries, ARRAY_SIZE(entries), - &dup_slot, 1, NULL, 0, NULL, 0, new_array.array, - new_array.size, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); - KUNIT_EXPECT_MEMEQ(test, dup_slot.node, old, dup_slot.size); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_rejects_short_array(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *tail = (const void *)1; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - child_array.size = __stack_depot_trie_child_array_size(0); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append(&root, NULL, 36, entries, ARRAY_SIZE(entries), - &node_slot, 1, NULL, 0, NULL, 0, child_array.array, - child_array.size, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_NULL(test, root.children); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_rejects_zero_frame(struct kunit *test) -{ - unsigned long entries[] = { 0 }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - unsigned char *old; - const void *tail = (const void *)1; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - old = kunit_kzalloc(test, node_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old); - memset(node_slot.node, 0xaa, node_slot.size); - memcpy(old, node_slot.node, node_slot.size); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append(&root, NULL, 39, entries, ARRAY_SIZE(entries), - &node_slot, 1, NULL, 0, NULL, 0, child_array.array, - child_array.size, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_NULL(test, root.children); - KUNIT_EXPECT_MEMEQ(test, node_slot.node, old, node_slot.size); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_rejects_root_with_parent(struct kunit *test) -{ - unsigned long parent_entries[] = { 0x1000UL }; - unsigned long entries[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *tail = (const void *)1; - unsigned int used = 99; - void *parent; - int ret; - - trie_node_alloc(test, parent_entries, ARRAY_SIZE(parent_entries), NULL, 7, - &parent); - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append(&root, parent, 37, entries, ARRAY_SIZE(entries), - &node_slot, 1, NULL, 0, NULL, 0, child_array.array, - child_array.size, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_NULL(test, root.children); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_rejects_root_slot_alias(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *tail = (const void *)1; - unsigned int used = 99; - int ret; - - node_slot.node = &root.children; - node_slot.size = sizeof(root.children); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append(&root, NULL, 40, entries, ARRAY_SIZE(entries), - &node_slot, 1, NULL, 0, NULL, 0, child_array.array, - child_array.size, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_NULL(test, root.children); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_rejects_parent_overlap(struct kunit *test) -{ - unsigned long parent_entries[] = { 0x1000UL }; - unsigned long child_entries[] = { 0x2000UL }; - struct stack_depot_trie_node_slot parent_slot; - struct stack_depot_trie_node_slot child_slot; - struct stack_depot_trie_lookup lookup; - const void *tail = (const void *)1; - unsigned int used = 99; - unsigned char *old; - int ret; - - trie_node_slot_alloc(test, &parent_slot, parent_entries, - ARRAY_SIZE(parent_entries)); - ret = tnode_init(parent_slot.node, parent_slot.size, NULL, 7, - parent_entries, ARRAY_SIZE(parent_entries), NULL, 0); - KUNIT_ASSERT_EQ(test, ret, 0); - trie_node_slot_alloc(test, &child_slot, child_entries, - ARRAY_SIZE(child_entries)); - old = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old); - memset(child_slot.node, 0xaa, child_slot.size); - memcpy(old, child_slot.node, child_slot.size); - - ret = insert_append(NULL, parent_slot.node, 41, child_entries, - ARRAY_SIZE(child_entries), &child_slot, 1, NULL, 0, - NULL, 0, parent_slot.node, parent_slot.size, &tail, - &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_MEMEQ(test, child_slot.node, old, child_slot.size); - ret = lookup_step(NULL, parent_slot.node, child_entries, - ARRAY_SIZE(child_entries), &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_APPEND); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_rejects_parent_cycle(struct kunit *test) -{ - unsigned long parent_entries[] = { 0x1000UL }; - unsigned long entries[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_node_slot parent_slot; - struct stack_depot_trie_node_slot node_slot; - const void *tail = (const void *)1; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &parent_slot, parent_entries, - ARRAY_SIZE(parent_entries)); - ret = tnode_init(parent_slot.node, parent_slot.size, NULL, 7, - parent_entries, ARRAY_SIZE(parent_entries), NULL, 0); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = tnode_init(parent_slot.node, parent_slot.size, parent_slot.node, 7, - parent_entries, ARRAY_SIZE(parent_entries), NULL, 0); - KUNIT_ASSERT_EQ(test, ret, 0); - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - - ret = insert_append(NULL, parent_slot.node, 42, entries, ARRAY_SIZE(entries), - &node_slot, 1, NULL, 0, NULL, 0, child_array.array, - child_array.size, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_rejects_publish_overlap(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - unsigned char *old; - const void *tail = (const void *)1; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - old = kunit_kzalloc(test, node_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old); - memset(node_slot.node, 0xaa, node_slot.size); - memcpy(old, node_slot.node, node_slot.size); - - ret = insert_append(&root, NULL, 38, entries, ARRAY_SIZE(entries), - &node_slot, 1, NULL, 0, NULL, 0, node_slot.node, - node_slot.size, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_NULL(test, root.children); - KUNIT_EXPECT_MEMEQ(test, node_slot.node, old, node_slot.size); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_rejects_child_node_overlap(struct kunit *test) -{ - unsigned long old_entries[] = { 0x1000UL }; - unsigned long new_entries[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot old_slot; - struct stack_depot_trie_node_slot new_slot; - struct stack_depot_trie_root root = {}; - const void *old_head = NULL; - const void *old_tail = NULL; - const void *tail = (const void *)1; - unsigned char *old; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); - old_array.size = __stack_depot_trie_child_array_size(1); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = append_chain(NULL, 43, old_entries, ARRAY_SIZE(old_entries), - &old_slot, 1, NULL, 0, NULL, 0, &old_head, - &old_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, old_head, old_array.array, - old_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - old = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old); - memcpy(old, old_array.array, old_array.size); - new_slot.node = old_array.array; - new_slot.size = old_array.size; - new_array.size = __stack_depot_trie_child_array_size(2); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); - used = 99; - - ret = insert_append(&root, NULL, 44, new_entries, ARRAY_SIZE(new_entries), - &new_slot, 1, NULL, 0, NULL, 0, new_array.array, - new_array.size, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_MEMEQ(test, old_array.array, old, old_array.size); - KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -static void stackdepot_trie_insert_append_rejects_child_array_overlap(struct kunit *test) -{ - unsigned long old_entries[] = { 0x1000UL }; - unsigned long new_entries[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_child_array_slot child_slot; - struct stack_depot_trie_node_slot old_slot; - struct stack_depot_trie_node_slot new_slot; - struct stack_depot_trie_root root = {}; - const void *old_head = NULL; - const void *old_tail = NULL; - const void *tail = (const void *)1; - unsigned char *old; - unsigned int used = 99; - int ret; - - trie_node_slot_alloc(test, &old_slot, old_entries, ARRAY_SIZE(old_entries)); - trie_node_slot_alloc(test, &new_slot, new_entries, ARRAY_SIZE(new_entries)); - old_array.size = __stack_depot_trie_child_array_size(1); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = append_chain(NULL, 45, old_entries, ARRAY_SIZE(old_entries), - &old_slot, 1, NULL, 0, NULL, 0, &old_head, - &old_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(&root, NULL, old_head, old_array.array, - old_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - old = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old); - memcpy(old, old_array.array, old_array.size); - child_slot.array = old_array.array; - child_slot.size = old_array.size; - new_array.size = __stack_depot_trie_child_array_size(2); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); - used = 99; - - ret = insert_append(&root, NULL, 46, new_entries, ARRAY_SIZE(new_entries), - &new_slot, 1, &child_slot, 1, NULL, 0, new_array.array, - new_array.size, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_MEMEQ(test, old_array.array, old, old_array.size); - KUNIT_EXPECT_PTR_EQ(test, root.children, old_array.array); - KUNIT_EXPECT_PTR_EQ(test, tail, (const void *)1); - KUNIT_EXPECT_EQ(test, used, 99U); -} - -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) -static void stackdepot_trie_node_compressed_roundtrip(struct kunit *test) -{ - unsigned long entries[] = { -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x1000UL, - arch_stack_depot_frame_text_prefix() | 0x2000UL, -#else - 0xffffffff81001000UL, - 0xffffffff81002000UL, -#endif - }; - unsigned long out[ARRAY_SIZE(entries)] = {}; - unsigned int fetched; - void *node; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 11, &node); - fetched = tfetch(node, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); -} - -static void stackdepot_trie_node_match_compressed(struct kunit *test) -{ - unsigned long entries[] = { -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x1000UL, - arch_stack_depot_frame_text_prefix() | 0x2000UL, -#else - 0xffffffff81001000UL, - 0xffffffff81002000UL, -#endif - }; - unsigned long mismatch[] = { - entries[0], -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x3000UL, -#else - 0xffffffff81003000UL, -#endif - }; - void *node; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 11, &node); - KUNIT_EXPECT_EQ(test, tmatch(node, entries, ARRAY_SIZE(entries)), - (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_EQ(test, tmatch(node, mismatch, ARRAY_SIZE(mismatch)), 1U); -} - -static void stackdepot_trie_append_chain_splits_frame_runs(struct kunit *test) -{ - unsigned long entries[] = { -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x1000UL, - arch_stack_depot_frame_text_prefix() | 0x2000UL, - 0x1000UL, - arch_stack_depot_frame_text_prefix() | 0x3000UL, -#else - 0xffffffff81001000UL, - 0xffffffff81002000UL, - 0xffff888000001000UL, - 0xffffffff81003000UL, -#endif - }; - struct stack_depot_trie_node_slot node_slots[3]; - struct stack_depot_trie_child_array_slot child_slots[2]; - unsigned long out[ARRAY_SIZE(entries)] = {}; - const void *child; - u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; - const void *head = NULL; - const void *tail = NULL; - unsigned int used = 0; - unsigned int fetched; - unsigned int i; - int ret; - - trie_node_slot_alloc(test, &node_slots[0], entries, 2); - trie_node_slot_alloc(test, &node_slots[1], &entries[2], 1); - trie_node_slot_alloc(test, &node_slots[2], &entries[3], 1); - - for (i = 0; i < ARRAY_SIZE(child_slots); i++) { - size_t size; - - child_slots[i].size = __stack_depot_trie_child_array_size(1); - size = child_slots[i].size; - child_slots[i].array = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_slots[i].array); - } - - ret = append_chain(NULL, 15, entries, ARRAY_SIZE(entries), node_slots, - ARRAY_SIZE(node_slots), child_slots, ARRAY_SIZE(child_slots), - write_scratch, ARRAY_SIZE(write_scratch), &head, &tail, - &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, head, node_slots[0].node); - KUNIT_EXPECT_PTR_EQ(test, tail, node_slots[2].node); - KUNIT_EXPECT_EQ(test, used, 3U); - child = child_array_find(child_slots[0].array, entries[2]); - KUNIT_EXPECT_PTR_EQ(test, child, node_slots[1].node); - child = child_array_find(child_slots[1].array, entries[3]); - KUNIT_EXPECT_PTR_EQ(test, child, node_slots[2].node); - fetched = tfetch(tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); -} - -static void stackdepot_trie_append_chain_rejects_bad_inputs(struct kunit *test) -{ - unsigned long entries[] = { -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x1000UL, - 0x1000UL, -#else - 0xffffffff81001000UL, - 0xffff888000001000UL, -#endif - }; - struct stack_depot_trie_node_slot node_slots[2]; - struct stack_depot_trie_child_array_slot child_slot; - u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; - const void *head = NULL; - const void *tail = NULL; - unsigned int used = 0; - int ret; - - trie_node_slot_alloc(test, &node_slots[0], entries, 1); - trie_node_slot_alloc(test, &node_slots[1], &entries[1], 1); - child_slot.size = __stack_depot_trie_child_array_size(1); - child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_slot.array); - - ret = append_chain(NULL, 16, entries, ARRAY_SIZE(entries), node_slots, 1, - &child_slot, 1, write_scratch, ARRAY_SIZE(write_scratch), - &head, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = append_chain(NULL, 16, entries, ARRAY_SIZE(entries), node_slots, - ARRAY_SIZE(node_slots), NULL, 0, write_scratch, - ARRAY_SIZE(write_scratch), &head, &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = append_chain(NULL, 16, entries, ARRAY_SIZE(entries), node_slots, - ARRAY_SIZE(node_slots), &child_slot, 1, NULL, 0, &head, - &tail, &used); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} - -static void stackdepot_trie_node_rejects_compressed_without_scratch(struct kunit *test) -{ - unsigned long entries[] = { -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x1000UL, -#else - 0xffffffff81001000UL, -#endif - }; - struct stack_depot_frame_run run; - void *storage; - size_t size; - int ret; +#include - KUNIT_ASSERT_EQ(test, frame_run_init(entries, ARRAY_SIZE(entries), &run), 0); - size = __stack_depot_trie_node_size(&run); - KUNIT_ASSERT_GT(test, size, (size_t)0); - storage = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, storage); - ret = tnode_init(storage, size, NULL, 1, entries, ARRAY_SIZE(entries), - NULL, 0); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} -#endif +#ifdef CONFIG_ARM64 +#include -static void stackdepot_trie_node_rejects_short_storage(struct kunit *test) +static unsigned long stackdepot_arm64_frame(long offset) { - unsigned long entries[] = { 0x1000UL }; - struct stack_depot_frame_run run; - unsigned char storage[sizeof(unsigned long)]; - size_t size; - int ret; - - KUNIT_ASSERT_EQ(test, frame_run_init(entries, ARRAY_SIZE(entries), &run), 0); - size = __stack_depot_trie_node_size(&run); - KUNIT_ASSERT_GT(test, size, sizeof(storage)); - ret = tnode_init(storage, sizeof(storage), NULL, 1, entries, - ARRAY_SIZE(entries), NULL, 0); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); + return (unsigned long)((long)_text + offset); } +#endif -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) -static void stackdepot_trie_node_rejects_mixed_run(struct kunit *test) +static void stackdepot_fetch_into_roundtrip(struct kunit *test) { - unsigned long storage[32]; - u32 write_scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; unsigned long entries[] = { -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x1000UL, - 0x1000UL, -#else - 0xffffffff81001000UL, - 0xffff888000001000UL, -#endif + 0x1234567800010000UL, + 0x1234567800020000UL, + 0x1234567800030000UL, }; - int ret; - - ret = tnode_init(storage, sizeof(storage), NULL, 1, entries, - ARRAY_SIZE(entries), write_scratch, - ARRAY_SIZE(write_scratch)); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} -#endif - -static void stackdepot_trie_fetch_rejects_bad_inputs(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - unsigned long root_entries[] = { 0x3000UL }; - unsigned long out[ARRAY_SIZE(entries)] = { 0xa5a5UL, 0xb6b6UL }; - unsigned long expected[ARRAY_SIZE(out)]; - unsigned int fetched; - void *node; - void *root; - - memcpy(expected, out, sizeof(expected)); - trie_node_alloc(test, root_entries, ARRAY_SIZE(root_entries), NULL, 0, - &root); - fetched = tfetch(root, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, 0); - KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); + unsigned long exact[ARRAY_SIZE(entries)] = {}; + unsigned long fetched[ARRAY_SIZE(entries) + 1] = { + [ARRAY_SIZE(entries)] = 0xa5a5a5a5a5a5a5a5UL, + }; + unsigned long expected_tail = fetched[ARRAY_SIZE(entries)]; + depot_stack_handle_t handle; + unsigned int nr_entries; - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 5, &node); - fetched = tfetch(node, out, ARRAY_SIZE(out) - 1); - KUNIT_EXPECT_EQ(test, fetched, 0); - KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); - fetched = tfetch(node, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(entries)); - KUNIT_EXPECT_MEMEQ(test, out, entries, sizeof(entries)); - memcpy(out, expected, sizeof(out)); - fetched = tfetch(NULL, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, 0); - KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(out)); -} + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); -static void stackdepot_trie_child_array_init_find(struct kunit *test) -{ - unsigned long first_entries[] = { 0x1000UL }; - unsigned long second_entries[] = { 0x2000UL }; - unsigned long third_entries[] = { 0x3000UL }; - const void *children[3]; - void *node; - void *array; - size_t size; - int ret; + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - trie_node_alloc(test, first_entries, ARRAY_SIZE(first_entries), NULL, 1, - &node); - children[0] = node; - trie_node_alloc(test, second_entries, ARRAY_SIZE(second_entries), NULL, 2, - &node); - children[1] = node; - trie_node_alloc(test, third_entries, ARRAY_SIZE(third_entries), NULL, 3, - &node); - children[2] = node; + nr_entries = stack_depot_fetch_into(handle, exact, ARRAY_SIZE(exact)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, exact, entries, sizeof(entries)); - size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); - KUNIT_ASSERT_GT(test, size, (size_t)0); - array = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, array); - ret = child_array_init(array, size, children, ARRAY_SIZE(children)); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, 0x1000UL), children[0]); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, 0x2000UL), children[1]); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, 0x3000UL), children[2]); - KUNIT_EXPECT_NULL(test, child_array_find(array, 0x4000UL)); - KUNIT_EXPECT_NULL(test, child_array_find(NULL, 0x1000UL)); + nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); + KUNIT_EXPECT_EQ(test, fetched[ARRAY_SIZE(entries)], expected_tail); } -static void stackdepot_trie_child_array_rejects_unsorted(struct kunit *test) +static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) { - unsigned long first_entries[] = { 0x2000UL }; - unsigned long second_entries[] = { 0x1000UL }; - const void *children[2]; - void *node; - void *array; - size_t size; - int ret; + unsigned long entries[] = { + 0x1234567800110000UL, + 0x1234567800120000UL, + 0x1234567800130000UL, + }; + unsigned long fetched[ARRAY_SIZE(entries)] = { + 0xa1a1a1a1a1a1a1a1UL, + 0xb2b2b2b2b2b2b2b2UL, + 0xc3c3c3c3c3c3c3c3UL, + }; + unsigned long expected[ARRAY_SIZE(fetched)]; + depot_stack_handle_t handle; + unsigned int nr_entries; - trie_node_alloc(test, first_entries, ARRAY_SIZE(first_entries), NULL, 1, - &node); - children[0] = node; - trie_node_alloc(test, second_entries, ARRAY_SIZE(second_entries), NULL, 2, - &node); - children[1] = node; - size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); - array = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, array); - ret = child_array_init(array, size, children, ARRAY_SIZE(children)); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); -static void stackdepot_trie_child_array_init_rejects_child_overlap(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - const void *children[1]; - unsigned char *old; - void *node; - size_t size; - int ret; + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + memcpy(expected, fetched, sizeof(expected)); - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 1, &node); - children[0] = node; - size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); - old = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old); - memcpy(old, node, size); + nr_entries = stack_depot_fetch_into(0, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); - ret = child_array_init(node, size, children, ARRAY_SIZE(children)); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_MEMEQ(test, node, old, size); -} + nr_entries = stack_depot_fetch_into(0, NULL, 0); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); -static void stackdepot_trie_child_array_insert(struct kunit *test) -{ - unsigned long first_entries[] = { 0x1000UL }; - unsigned long second_entries[] = { 0x3000UL }; - unsigned long middle_entries[] = { 0x2000UL }; - const void *children[2]; - void *old_array; - void *new_array; - void *middle; - void *node; - size_t old_size; - size_t new_size; - int ret; + nr_entries = stack_depot_fetch_into(handle, NULL, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); - trie_node_alloc(test, first_entries, ARRAY_SIZE(first_entries), NULL, 1, - &node); - children[0] = node; - trie_node_alloc(test, second_entries, ARRAY_SIZE(second_entries), NULL, 2, - &node); - children[1] = node; - trie_node_alloc(test, middle_entries, ARRAY_SIZE(middle_entries), NULL, 3, - &middle); - old_size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); - new_size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children) + 1); - old_array = kunit_kzalloc(test, old_size, GFP_KERNEL); - new_array = kunit_kzalloc(test, new_size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array); - KUNIT_ASSERT_NOT_NULL(test, new_array); - KUNIT_ASSERT_EQ(test, - child_array_init(old_array, old_size, children, - ARRAY_SIZE(children)), - 0); + nr_entries = stack_depot_fetch_into(handle, fetched, 0); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); - ret = child_array_insert(old_array, middle, new_array, new_size); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array, 0x1000UL), children[0]); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array, 0x2000UL), middle); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array, 0x3000UL), children[1]); - ret = child_array_insert(old_array, children[0], new_array, new_size); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = child_array_insert(old_array, middle, old_array, old_size); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); + nr_entries = stack_depot_fetch_into(handle, fetched, + ARRAY_SIZE(fetched) - 1); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); } -static void stackdepot_trie_child_array_insert_rejects_child_overlap(struct kunit *test) +static depot_stack_handle_t save_hash(unsigned long *entries, unsigned int nr) { - unsigned long entries[] = { 0x1000UL }; - unsigned char *old; - void *child; - size_t size; - int ret; - - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 1, &child); - size = __stack_depot_trie_child_array_size(1); - old = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old); - memcpy(old, child, size); + depot_flags_t flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_HASH; - ret = child_array_insert(NULL, child, child, size); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - KUNIT_EXPECT_MEMEQ(test, child, old, size); + return stack_depot_save_flags(entries, nr, GFP_KERNEL, flags); } -static void stackdepot_trie_split_child_array_init_one_child(struct kunit *test) +static void stackdepot_hash_flag_roundtrip(struct kunit *test) { - unsigned long old_entries[] = { 0x2000UL }; - void *old_tail; - void *array; - size_t size; - int ret; + unsigned long entries[] = { + 0x1234567800210000UL, + 0x1234567800220000UL, + 0x1234567800230000UL, + }; + unsigned long fetched[ARRAY_SIZE(entries)] = {}; + depot_stack_handle_t handle; + unsigned int nr_entries; - trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, - &old_tail); - size = __stack_depot_trie_child_array_size(1); - array = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, array); - ret = split_child_array_init(array, size, old_tail, NULL); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, old_entries[0]), old_tail); -} + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); -static void stackdepot_trie_split_child_array_init_orders_children(struct kunit *test) -{ - unsigned long old_entries[] = { 0x3000UL }; - unsigned long new_entries[] = { 0x1000UL }; - void *new_head; - void *old_tail; - void *array; - size_t size; - int ret; + handle = save_hash(entries, ARRAY_SIZE(entries)); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, - &old_tail); - trie_node_alloc(test, new_entries, ARRAY_SIZE(new_entries), NULL, 2, - &new_head); - size = __stack_depot_trie_child_array_size(2); - array = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, array); - ret = split_child_array_init(array, size, old_tail, new_head); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, new_entries[0]), new_head); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(array, old_entries[0]), old_tail); + nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); } -static void stackdepot_trie_split_child_array_rejects_bad_inputs(struct kunit *test) +static depot_stack_handle_t save_noalloc(unsigned long *entries, unsigned int nr) { - unsigned long old_entries[] = { 0x2000UL }; - unsigned long dup_entries[] = { 0x2000UL }; - unsigned char *old; - void *old_tail; - void *dup_tail; - void *array; - size_t size; - - trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, - &old_tail); - trie_node_alloc(test, dup_entries, ARRAY_SIZE(dup_entries), NULL, 2, - &dup_tail); - size = __stack_depot_trie_child_array_size(2); - array = kunit_kzalloc(test, size, GFP_KERNEL); - old = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, array); - KUNIT_ASSERT_NOT_NULL(test, old); - memset(array, 0xaa, size); - memcpy(old, array, size); + gfp_t no_spin = GFP_NOWAIT & ~__GFP_RECLAIM; - KUNIT_EXPECT_EQ(test, split_child_array_init(array, size, NULL, dup_tail), - -EINVAL); - KUNIT_EXPECT_EQ(test, split_child_array_init(array, size, old_tail, dup_tail), - -EINVAL); - size = __stack_depot_trie_child_array_size(1); - KUNIT_EXPECT_EQ(test, split_child_array_init(array, size, old_tail, dup_tail), - -EINVAL); - KUNIT_EXPECT_EQ(test, split_child_array_init(old_tail, size, old_tail, NULL), - -EINVAL); - KUNIT_EXPECT_MEMEQ(test, array, old, size); + return stack_depot_save_flags(entries, nr, no_spin, 0); } -static void stackdepot_trie_split_tail_plan_raw(struct kunit *test) +static void stackdepot_save_flags_public(struct kunit *test) { - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_node_slot node_slot; - unsigned int nr_runs = 0; - int ret; + unsigned long entries[] = { 0x501000UL, 0x502000UL, 0x503000UL }; + unsigned long get_entries[] = { 0x601000UL, 0x602000UL }; + unsigned long noalloc_entries[] = { 0x701000UL, 0x702000UL }; + unsigned long fetched[ARRAY_SIZE(entries)] = {}; + depot_stack_handle_t noalloc_handle; + depot_stack_handle_t overlong_handle; + depot_stack_handle_t hash_handle; + depot_stack_handle_t get_handle; + depot_stack_handle_t again; + depot_stack_handle_t extra; + depot_flags_t flags; + unsigned long *overlong_entries; + unsigned int noalloc_nr = ARRAY_SIZE(noalloc_entries); + unsigned int overlong_nr = CONFIG_STACKDEPOT_MAX_FRAMES + 1; + unsigned int nr_entries; + unsigned int i; - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - ret = split_tail_plan(entries, ARRAY_SIZE(entries), &node_slot, 1, NULL, 0, - &nr_runs); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, nr_runs, 1U); -} + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + overlong_entries = kunit_kcalloc(test, overlong_nr, + sizeof(*overlong_entries), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, overlong_entries); + for (i = 0; i < overlong_nr; i++) + overlong_entries[i] = 0x800000UL + i * 0x1000UL; -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) -static void stackdepot_trie_split_tail_plan_mixed_runs(struct kunit *test) -{ - unsigned long entries[] = { -#ifdef CONFIG_ARM64 - arch_stack_depot_frame_text_prefix() | 0x1000UL, - 0x1000UL, - arch_stack_depot_frame_text_prefix() | 0x2000UL, -#else - 0xffffffff81001000UL, - 0xffff888000001000UL, - 0xffffffff81002000UL, -#endif - }; - struct stack_depot_trie_child_array_slot child_slots[2]; - struct stack_depot_trie_node_slot node_slots[3]; - unsigned int nr_runs = 0; - unsigned int i; - int ret; + hash_handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); + again = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_EXPECT_EQ(test, again, hash_handle); - trie_node_slot_alloc(test, &node_slots[0], entries, 1); - trie_node_slot_alloc(test, &node_slots[1], &entries[1], 1); - trie_node_slot_alloc(test, &node_slots[2], &entries[2], 1); - for (i = 0; i < ARRAY_SIZE(child_slots); i++) { - size_t size; + nr_entries = stack_depot_fetch_into(hash_handle, fetched, + ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); - child_slots[i].size = __stack_depot_trie_child_array_size(1); - size = child_slots[i].size; - child_slots[i].array = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_slots[i].array); + noalloc_handle = save_noalloc(entries, ARRAY_SIZE(entries)); + KUNIT_EXPECT_EQ(test, noalloc_handle, hash_handle); + noalloc_handle = save_noalloc(noalloc_entries, noalloc_nr); + if (noalloc_handle) { + unsigned long noalloc_fetched[ARRAY_SIZE(noalloc_entries)] = {}; + + nr_entries = stack_depot_fetch_into(noalloc_handle, noalloc_fetched, + ARRAY_SIZE(noalloc_fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, noalloc_nr); + KUNIT_EXPECT_MEMEQ(test, noalloc_fetched, noalloc_entries, + sizeof(noalloc_entries)); } - ret = split_tail_plan(entries, ARRAY_SIZE(entries), node_slots, - ARRAY_SIZE(node_slots), child_slots, - ARRAY_SIZE(child_slots), &nr_runs); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, nr_runs, 3U); -} -#endif + flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; + get_handle = stack_depot_save_flags(get_entries, ARRAY_SIZE(get_entries), + GFP_KERNEL, flags); + KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); + stack_depot_put(get_handle); -static void stackdepot_trie_split_tail_plan_rejects_bad_inputs(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL, 0x2000UL }; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_node_slot short_slot; - struct stack_depot_frame_run run; - unsigned int nr_runs = 99; - int ret; + flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_HASH; + hash_handle = stack_depot_save_flags(entries, ARRAY_SIZE(entries), + GFP_KERNEL, flags); + KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); - trie_node_slot_alloc(test, &node_slot, entries, ARRAY_SIZE(entries)); - ret = frame_run_init(entries, ARRAY_SIZE(entries), &run); - KUNIT_ASSERT_EQ(test, ret, 0); - short_slot = node_slot; - short_slot.size = __stack_depot_trie_node_size(&run) - 1; + overlong_handle = stack_depot_save(overlong_entries, overlong_nr, + GFP_KERNEL); + KUNIT_ASSERT_NE(test, overlong_handle, (depot_stack_handle_t)0); - ret = split_tail_plan(NULL, ARRAY_SIZE(entries), &node_slot, 1, NULL, 0, - &nr_runs); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = split_tail_plan(entries, 0, &node_slot, 1, NULL, 0, &nr_runs); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = split_tail_plan(entries, ARRAY_SIZE(entries), NULL, 1, NULL, 0, - &nr_runs); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = split_tail_plan(entries, ARRAY_SIZE(entries), &node_slot, 1, NULL, 0, - NULL); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - ret = split_tail_plan(entries, ARRAY_SIZE(entries), &short_slot, 1, NULL, 0, - &nr_runs); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); + extra = stack_depot_set_extra_bits(hash_handle, 7); + KUNIT_ASSERT_NE(test, extra, (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, stack_depot_get_extra_bits(extra), 7U); + memset(fetched, 0, sizeof(fetched)); + nr_entries = stack_depot_fetch_into(extra, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); } -static void stackdepot_trie_split_precheck(struct kunit *test) +static void stackdepot_snprint_public(struct kunit *test) { - unsigned long old_entries[] = { 0x1000UL }; - unsigned long new_entries[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot child_slot; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *children[1]; - void *old_child; - int ret; + unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + char expected[256]; + char actual[256]; + depot_stack_handle_t handle; + unsigned int expected_len; + int actual_len; - trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, - &old_child); - trie_node_slot_alloc(test, &node_slot, new_entries, ARRAY_SIZE(new_entries)); - children[0] = old_child; - old_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = child_array_init(old_array.array, old_array.size, children, - ARRAY_SIZE(children)); - KUNIT_ASSERT_EQ(test, ret, 0); - root.children = old_array.array; - child_slot.size = __stack_depot_trie_child_array_size(1); - child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_slot.array); - new_array.size = __stack_depot_trie_child_array_size(1); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - ret = split_precheck(&root, NULL, &node_slot, 1, &child_slot, 1, - new_array.array, new_array.size); - KUNIT_EXPECT_EQ(test, ret, 0); + expected_len = stack_trace_snprint(expected, sizeof(expected), entries, + ARRAY_SIZE(entries), 2); + actual_len = stack_depot_snprint(handle, actual, sizeof(actual), 2); + KUNIT_EXPECT_EQ(test, actual_len, (int)expected_len); + KUNIT_EXPECT_STREQ(test, actual, expected); } -static void stackdepot_trie_split_precheck_rejects_aliases(struct kunit *test) +static void stackdepot_count_helpers(struct kunit *test) { - unsigned long old_entries[] = { 0x1000UL }; - unsigned long new_entries[] = { 0x2000UL }; - struct stack_depot_trie_child_array_slot child_slot; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *children[1]; - void *old_child; - int ret; - - trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, - &old_child); - trie_node_slot_alloc(test, &node_slot, new_entries, ARRAY_SIZE(new_entries)); - children[0] = old_child; - old_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = child_array_init(old_array.array, old_array.size, children, - ARRAY_SIZE(children)); - KUNIT_ASSERT_EQ(test, ret, 0); - root.children = old_array.array; - child_slot.size = __stack_depot_trie_child_array_size(1); - child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_slot.array); - new_array.size = __stack_depot_trie_child_array_size(1); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); + unsigned long entries[] = { + 0x1234567800310000UL, + 0x1234567800320000UL, + 0x1234567800330000UL, + }; + unsigned long second_entries[] = { + 0x1234567800410000UL, + 0x1234567800420000UL, + 0x1234567800430000UL, + }; + depot_stack_handle_t second_handle; + depot_stack_handle_t handle; + unsigned int count; + bool new_count; - ret = split_precheck(&root, NULL, &node_slot, 1, &child_slot, 1, - old_array.array, old_array.size); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); - node_slot.node = old_array.array; - node_slot.size = old_array.size; - ret = split_precheck(&root, NULL, &node_slot, 1, &child_slot, 1, - new_array.array, new_array.size); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); -static void stackdepot_trie_split_precheck_rejects_short_array(struct kunit *test) -{ - unsigned long first_entries[] = { 0x1000UL }; - unsigned long second_entries[] = { 0x2000UL }; - unsigned long new_entries[] = { 0x3000UL }; - struct stack_depot_trie_child_array_slot child_slot; - struct stack_depot_trie_child_array_slot old_array; - struct stack_depot_trie_child_array_slot new_array; - struct stack_depot_trie_node_slot node_slot; - struct stack_depot_trie_root root = {}; - const void *children[2]; - void *first_child; - void *second_child; - int ret; + KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(0, &count)); + KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(0, NULL)); + __stack_depot_set_count(0, 1); + KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(0, 1, &new_count)); + KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(0, 1)); - trie_node_alloc(test, first_entries, ARRAY_SIZE(first_entries), NULL, 1, - &first_child); - trie_node_alloc(test, second_entries, ARRAY_SIZE(second_entries), NULL, 2, - &second_child); - trie_node_slot_alloc(test, &node_slot, new_entries, ARRAY_SIZE(new_entries)); - children[0] = first_child; - children[1] = second_child; - old_array.size = __stack_depot_trie_child_array_size(ARRAY_SIZE(children)); - old_array.array = kunit_kzalloc(test, old_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, old_array.array); - ret = child_array_init(old_array.array, old_array.size, children, - ARRAY_SIZE(children)); - KUNIT_ASSERT_EQ(test, ret, 0); - root.children = old_array.array; - child_slot.size = __stack_depot_trie_child_array_size(1); - child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_slot.array); - new_array.size = __stack_depot_trie_child_array_size(1); - new_array.array = kunit_kzalloc(test, new_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array.array); + handle = save_hash(entries, ARRAY_SIZE(entries)); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); - ret = split_precheck(&root, NULL, &node_slot, 1, &child_slot, 1, - new_array.array, new_array.size); - KUNIT_EXPECT_EQ(test, ret, -EINVAL); -} + new_count = false; + KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 2, &new_count)); + KUNIT_EXPECT_TRUE(test, new_count); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 3U); -static void stackdepot_trie_split_subtree_divergent_tail(struct kunit *test) -{ - unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; - unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; - struct stack_depot_trie_child_array_slot child_slot; - struct stack_depot_trie_node_slot node_slots[3]; - struct stack_depot_trie_lookup lookup; - unsigned long out[ARRAY_SIZE(old_entries)] = {}; - const void *new_tail = NULL; - const void *old_tail; - const void *prefix = NULL; - unsigned int fetched; - unsigned int used = 99; - void *child; - int ret; + new_count = true; + KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 4, &new_count)); + KUNIT_EXPECT_FALSE(test, new_count); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 7U); - trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, - &child); - trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); - trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); - trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); - child_slot.size = __stack_depot_trie_child_array_size(2); - child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_slot.array); + KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 5)); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); + KUNIT_EXPECT_EQ(test, count, 2U); + KUNIT_EXPECT_TRUE(test, __stack_depot_dec_count_and_test(handle, 2)); + KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); - ret = split_subtree(child, 1, 2, new_entries, ARRAY_SIZE(new_entries), - node_slots, ARRAY_SIZE(node_slots), &child_slot, 1, - NULL, 0, &prefix, &new_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, used, 3U); - ret = lookup_step(NULL, prefix, &old_entries[1], 1, &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - old_tail = lookup.node; - KUNIT_EXPECT_PTR_EQ(test, new_tail, node_slots[2].node); - fetched = tfetch(old_tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, 2U); - KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); - memset(out, 0, sizeof(out)); - fetched = tfetch(new_tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, 2U); - KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); + second_handle = save_hash(second_entries, ARRAY_SIZE(second_entries)); + KUNIT_ASSERT_NE(test, second_handle, (depot_stack_handle_t)0); + __stack_depot_set_count(second_handle, INT_MAX - 1); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(second_handle, &count)); + KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX - 1); + KUNIT_EXPECT_FALSE(test, + __stack_depot_inc_count(second_handle, 2, &new_count)); + KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(second_handle, &count)); + KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX - 1); } -static void stackdepot_trie_split_subtree_prefix_leaf(struct kunit *test) +static void stackdepot_frame_raw_fallback(struct kunit *test) { - unsigned long old_entries[] = { 0x1000UL, 0x2000UL }; - unsigned long new_entries[] = { 0x1000UL }; - struct stack_depot_trie_child_array_slot child_slot; - struct stack_depot_trie_node_slot node_slots[2]; - struct stack_depot_trie_lookup lookup; - unsigned long out[ARRAY_SIZE(old_entries)] = {}; - const void *new_tail = NULL; - const void *old_tail; - const void *prefix = NULL; - unsigned int fetched; - unsigned int used = 99; - void *child; - int ret; - - trie_node_alloc(test, old_entries, ARRAY_SIZE(old_entries), NULL, 1, - &child); - trie_node_slot_alloc(test, &node_slots[0], old_entries, 1); - trie_node_slot_alloc(test, &node_slots[1], &old_entries[1], 1); - child_slot.size = __stack_depot_trie_child_array_size(1); - child_slot.array = kunit_kzalloc(test, child_slot.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_slot.array); + unsigned long frame = 0xffff888000001000UL; + unsigned long out = 0x12345678UL; + bool compressed; + bool decoded; + u32 low = 0xfeedbeef; - ret = split_subtree(child, 1, 2, new_entries, ARRAY_SIZE(new_entries), - node_slots, ARRAY_SIZE(node_slots), &child_slot, 1, - NULL, 0, &prefix, &new_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, used, 2U); - KUNIT_EXPECT_PTR_EQ(test, new_tail, prefix); +#ifdef CONFIG_ARM64 + if ((unsigned long)_text <= ULONG_MAX - ((unsigned long)S32_MAX + 1UL)) + frame = (unsigned long)_text + (unsigned long)S32_MAX + 1UL; + else + frame = stackdepot_arm64_frame((long)S32_MIN - 1L); +#endif - fetched = tfetch(prefix, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, 1U); - KUNIT_EXPECT_MEMEQ(test, out, new_entries, sizeof(new_entries)); + compressed = arch_stack_depot_frame_try_compress(frame, &low); + KUNIT_EXPECT_FALSE(test, compressed); + KUNIT_EXPECT_EQ(test, low, (u32)0xfeedbeef); - ret = lookup_step(NULL, prefix, &old_entries[1], 1, &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - old_tail = lookup.node; - memset(out, 0, sizeof(out)); - fetched = tfetch(old_tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, 2U); - KUNIT_EXPECT_MEMEQ(test, out, old_entries, sizeof(old_entries)); + decoded = arch_stack_depot_frame_decompress(0x81234567, NULL); + KUNIT_EXPECT_FALSE(test, decoded); + KUNIT_EXPECT_EQ(test, out, 0x12345678UL); } -static void stackdepot_trie_split_subtree_preserves_children(struct kunit *test) +#ifdef CONFIG_X86_64 +static void stackdepot_frame_x86_64(struct kunit *test) { - unsigned long child_entries[] = { 0x1000UL, 0x2000UL }; - unsigned long desc_entries[] = { 0x4000UL }; - unsigned long new_entries[] = { 0x1000UL, 0x3000UL }; - unsigned long old_tail_lookup[] = { 0x2000UL, 0x4000UL }; - unsigned long expected[] = { 0x1000UL, 0x2000UL, 0x4000UL }; - struct stack_depot_trie_child_array_slot child_array; - struct stack_depot_trie_child_array_slot split_array; - struct stack_depot_trie_node_slot desc_slot; - struct stack_depot_trie_node_slot node_slots[3]; - struct stack_depot_trie_lookup lookup; - unsigned long out[ARRAY_SIZE(expected)] = {}; - const void *desc_head = NULL; - const void *desc_tail = NULL; - const void *new_tail = NULL; - const void *old_tail; - const void *prefix = NULL; - unsigned int fetched; - unsigned int used = 99; - void *child; - int ret; - - trie_node_alloc(test, child_entries, ARRAY_SIZE(child_entries), NULL, 0, - &child); - trie_node_slot_alloc(test, &desc_slot, desc_entries, ARRAY_SIZE(desc_entries)); - child_array.size = __stack_depot_trie_child_array_size(1); - child_array.array = kunit_kzalloc(test, child_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, child_array.array); - ret = append_chain(child, 3, desc_entries, ARRAY_SIZE(desc_entries), - &desc_slot, 1, NULL, 0, NULL, 0, &desc_head, - &desc_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = publish_append(NULL, child, desc_head, child_array.array, - child_array.size); - KUNIT_ASSERT_EQ(test, ret, 0); - - trie_node_slot_alloc(test, &node_slots[0], child_entries, 1); - trie_node_slot_alloc(test, &node_slots[1], &child_entries[1], 1); - trie_node_slot_alloc(test, &node_slots[2], &new_entries[1], 1); - split_array.size = __stack_depot_trie_child_array_size(2); - split_array.array = kunit_kzalloc(test, split_array.size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, split_array.array); - - ret = split_subtree(child, 1, 4, new_entries, ARRAY_SIZE(new_entries), - node_slots, ARRAY_SIZE(node_slots), &split_array, 1, - NULL, 0, &prefix, &new_tail, &used); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, used, 3U); - ret = lookup_step(NULL, prefix, old_tail_lookup, - ARRAY_SIZE(old_tail_lookup), &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_DESCEND); - old_tail = lookup.node; - ret = lookup_step(NULL, old_tail, desc_entries, ARRAY_SIZE(desc_entries), - &lookup); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_EQ(test, lookup.status, STACK_DEPOT_TRIE_LOOKUP_FOUND); - KUNIT_EXPECT_PTR_EQ(test, lookup.node, desc_tail); - fetched = tfetch(desc_tail, out, ARRAY_SIZE(out)); - KUNIT_EXPECT_EQ(test, fetched, (unsigned int)ARRAY_SIZE(expected)); - KUNIT_EXPECT_MEMEQ(test, out, expected, sizeof(expected)); -} + unsigned long direct_map = 0xffff888000001000UL; + unsigned long frame = 0xffffffff81234567UL; + unsigned long out; + bool compressed; + bool decoded; + u32 low; -static void stackdepot_trie_child_array_insert_empty(struct kunit *test) -{ - unsigned long entries[] = { 0x1000UL }; - void *new_array; - void *child; - size_t size; - int ret; + compressed = arch_stack_depot_frame_try_compress(frame, &low); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, low, (u32)0x81234567); + decoded = arch_stack_depot_frame_decompress(low, &out); + KUNIT_EXPECT_TRUE(test, decoded); + KUNIT_EXPECT_EQ(test, out, frame); - trie_node_alloc(test, entries, ARRAY_SIZE(entries), NULL, 1, &child); - size = __stack_depot_trie_child_array_size(1); - new_array = kunit_kzalloc(test, size, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, new_array); - ret = child_array_insert(NULL, child, new_array, size); - KUNIT_ASSERT_EQ(test, ret, 0); - KUNIT_EXPECT_PTR_EQ(test, child_array_find(new_array, 0x1000UL), child); + compressed = arch_stack_depot_frame_try_compress(direct_map, &low); + KUNIT_EXPECT_FALSE(test, compressed); } +#endif /* CONFIG_X86_64 */ -static void stackdepot_trie_public_save_route(struct kunit *test) +#ifdef CONFIG_ARM64 +static void stackdepot_frame_arm64(struct kunit *test) { - unsigned long hash_entries[] = { 0x401000UL, 0x402000UL }; - unsigned long trie_entries[] = { 0x501000UL, 0x502000UL, 0x503000UL }; - unsigned long get_entries[] = { 0x601000UL, 0x602000UL }; - unsigned long noalloc_entries[] = { 0x701000UL, 0x702000UL }; - unsigned long hash_flag_entries[] = { 0x901000UL, 0x902000UL }; - unsigned long fetched[ARRAY_SIZE(trie_entries)] = {}; - depot_stack_handle_t get_handle; - depot_stack_handle_t hash_flag; - depot_stack_handle_t hash_again; - depot_stack_handle_t hash_handle; - depot_stack_handle_t noalloc_handle; - depot_stack_handle_t overlong_handle; - depot_stack_handle_t trie_again; - depot_stack_handle_t trie_handle; - depot_flags_t get_flags; - unsigned int hash_flag_nr = ARRAY_SIZE(hash_flag_entries); - gfp_t no_spin = GFP_NOWAIT & ~__GFP_RECLAIM; - unsigned int get_nr = ARRAY_SIZE(get_entries); - unsigned int noalloc_nr = ARRAY_SIZE(noalloc_entries); - unsigned int nr_entries; - unsigned long *overlong_entries; - unsigned int overlong_nr = CONFIG_STACKDEPOT_MAX_FRAMES + 1; - unsigned int i; - - stackdepot_trie_add_disable_action(test); - overlong_entries = kunit_kcalloc(test, overlong_nr, sizeof(*overlong_entries), - GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, overlong_entries); - for (i = 0; i < overlong_nr; i++) - overlong_entries[i] = 0x800000UL + i * 0x1000UL; - - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - hash_handle = stack_depot_save(hash_entries, ARRAY_SIZE(hash_entries), GFP_KERNEL); - KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(hash_handle), 0U); - if (!__stack_depot_trie_max_leaf_id()) - kunit_skip(test, "trie handle namespace unavailable"); + long negative_offset = S32_MIN; + long positive_offset = S32_MAX; + long offset = 0x123456; + unsigned long frame = stackdepot_arm64_frame(offset); + unsigned long out; + bool compressed; + bool decoded; + u32 low; - __stack_depot_trie_set_enabled(true); - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - KUNIT_EXPECT_TRUE(test, __stack_depot_trie_ready()); - hash_again = stack_depot_save(hash_entries, ARRAY_SIZE(hash_entries), GFP_KERNEL); - KUNIT_EXPECT_EQ(test, hash_again, hash_handle); + compressed = arch_stack_depot_frame_try_compress(frame, &low); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, low, (u32)(s32)offset); + decoded = arch_stack_depot_frame_decompress(low, &out); + KUNIT_EXPECT_TRUE(test, decoded); + KUNIT_EXPECT_EQ(test, out, frame); - trie_handle = stack_depot_save(trie_entries, ARRAY_SIZE(trie_entries), GFP_KERNEL); - KUNIT_ASSERT_NE(test, trie_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_NE(test, __stack_depot_trie_leaf_id(trie_handle), 0U); - trie_again = stack_depot_save(trie_entries, ARRAY_SIZE(trie_entries), GFP_KERNEL); - KUNIT_EXPECT_EQ(test, trie_again, trie_handle); - nr_entries = stack_depot_fetch_into(trie_handle, fetched, ARRAY_SIZE(fetched)); - KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(trie_entries)); - KUNIT_EXPECT_MEMEQ(test, fetched, trie_entries, sizeof(trie_entries)); - noalloc_handle = stack_depot_save_flags(trie_entries, ARRAY_SIZE(trie_entries), no_spin, 0); - KUNIT_EXPECT_EQ(test, noalloc_handle, trie_handle); - noalloc_handle = stack_depot_save_flags(noalloc_entries, noalloc_nr, no_spin, 0); - KUNIT_EXPECT_EQ(test, noalloc_handle, (depot_stack_handle_t)0); - noalloc_handle = stack_depot_save(noalloc_entries, noalloc_nr, GFP_KERNEL); - KUNIT_ASSERT_NE(test, noalloc_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_NE(test, __stack_depot_trie_leaf_id(noalloc_handle), 0U); - KUNIT_EXPECT_EQ(test, stack_depot_save_flags(noalloc_entries, noalloc_nr, no_spin, 0), - noalloc_handle); + frame = stackdepot_arm64_frame(negative_offset); + compressed = arch_stack_depot_frame_try_compress(frame, &low); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, low, (u32)(s32)negative_offset); + decoded = arch_stack_depot_frame_decompress(low, &out); + KUNIT_EXPECT_TRUE(test, decoded); + KUNIT_EXPECT_EQ(test, out, frame); - get_flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; - get_handle = stack_depot_save_flags(get_entries, get_nr, GFP_KERNEL, get_flags); - KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(get_handle), 0U); - stack_depot_put(get_handle); - get_flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_HASH; - hash_flag = stack_depot_save_flags(hash_flag_entries, hash_flag_nr, GFP_KERNEL, get_flags); - KUNIT_ASSERT_NE(test, hash_flag, (depot_stack_handle_t)0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(hash_flag), 0U); - overlong_handle = stack_depot_save(overlong_entries, overlong_nr, GFP_KERNEL); - KUNIT_ASSERT_NE(test, overlong_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_EQ(test, __stack_depot_trie_leaf_id(overlong_handle), 0U); + frame = stackdepot_arm64_frame(positive_offset); + compressed = arch_stack_depot_frame_try_compress(frame, &low); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, low, (u32)(s32)positive_offset); + decoded = arch_stack_depot_frame_decompress(low, &out); + KUNIT_EXPECT_TRUE(test, decoded); + KUNIT_EXPECT_EQ(test, out, frame); } +#endif /* CONFIG_ARM64 */ #define STACKDEPOT_STRESS_THREADS 4 #define STACKDEPOT_STRESS_ITERS 64 @@ -5497,7 +386,7 @@ static void stackdepot_stress_entries(unsigned int id, unsigned int iter, i * 0x10UL; } -static int stackdepot_trie_stress_worker(void *data) +static int stackdepot_stress_worker(void *data) { struct stackdepot_stress_ctx *ctx = data; unsigned long entries[STACKDEPOT_STRESS_DEPTH]; @@ -5506,7 +395,6 @@ static int stackdepot_trie_stress_worker(void *data) depot_stack_handle_t handle; unsigned int nr_entries; unsigned int iter; - gfp_t no_spin; char buf[256]; complete(&ctx->ready); @@ -5515,20 +403,20 @@ static int stackdepot_trie_stress_worker(void *data) for (iter = 0; iter < STACKDEPOT_STRESS_ITERS; iter++) { stackdepot_stress_entries(ctx->id, iter, entries); handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); - if (!handle || !__stack_depot_trie_leaf_id(handle)) { + if (!handle) { atomic_inc(ctx->failures); continue; } - nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); + nr_entries = stack_depot_fetch_into(handle, fetched, + ARRAY_SIZE(fetched)); if (nr_entries != ARRAY_SIZE(entries) || memcmp(fetched, entries, sizeof(entries))) { atomic_inc(ctx->failures); continue; } - no_spin = GFP_NOWAIT & ~__GFP_RECLAIM; - again = stack_depot_save_flags(entries, ARRAY_SIZE(entries), no_spin, 0); + again = save_noalloc(entries, ARRAY_SIZE(entries)); if (again != handle) atomic_inc(ctx->failures); @@ -5540,7 +428,7 @@ static int stackdepot_trie_stress_worker(void *data) return 0; } -static void stackdepot_trie_concurrent_save_fetch(struct kunit *test) +static void stackdepot_concurrent_save_fetch(struct kunit *test) { struct stackdepot_stress_ctx *ctx; struct task_struct *task; @@ -5548,22 +436,12 @@ static void stackdepot_trie_concurrent_save_fetch(struct kunit *test) struct completion start; unsigned int created = 0; unsigned int i; - size_t size; long timeout; int err = 0; - stackdepot_trie_add_disable_action(test); - KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - if (!__stack_depot_trie_max_leaf_id()) - kunit_skip(test, "trie handle namespace unavailable"); - - __stack_depot_trie_set_enabled(true); KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - KUNIT_ASSERT_TRUE(test, __stack_depot_trie_ready()); init_completion(&start); - - size = sizeof(*ctx); - ctx = kunit_kcalloc(test, STACKDEPOT_STRESS_THREADS, size, GFP_KERNEL); + ctx = kunit_kcalloc(test, STACKDEPOT_STRESS_THREADS, sizeof(*ctx), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, ctx); for (i = 0; i < STACKDEPOT_STRESS_THREADS; i++) { @@ -5573,7 +451,7 @@ static void stackdepot_trie_concurrent_save_fetch(struct kunit *test) ctx[i].failures = &failures; ctx[i].id = i + 1; - task = kthread_run(stackdepot_trie_stress_worker, &ctx[i], + task = kthread_run(stackdepot_stress_worker, &ctx[i], "stackdepot_stress/%u", i); if (IS_ERR(task)) { err = PTR_ERR(task); @@ -5602,45 +480,10 @@ static void stackdepot_trie_concurrent_save_fetch(struct kunit *test) static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_fetch_into_roundtrip), KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), + KUNIT_CASE(stackdepot_hash_flag_roundtrip), + KUNIT_CASE(stackdepot_save_flags_public), + KUNIT_CASE(stackdepot_snprint_public), KUNIT_CASE(stackdepot_count_helpers), - KUNIT_CASE(stackdepot_trie_handle_namespace), - KUNIT_CASE(stackdepot_trie_feature_flag), - KUNIT_CASE(stackdepot_trie_late_init), - KUNIT_CASE(stackdepot_trie_side_table_destroy_uninit), - KUNIT_CASE(stackdepot_trie_side_table_alloc_store_lookup), - KUNIT_CASE(stackdepot_trie_side_table_rejects_invalid_ids), - KUNIT_CASE(stackdepot_trie_side_table_revoke_latest), - KUNIT_CASE(stackdepot_trie_side_table_revoke_keeps_chunk), - KUNIT_CASE(stackdepot_trie_side_table_restore), - KUNIT_CASE(stackdepot_trie_side_table_chunk_boundary), - KUNIT_CASE(stackdepot_trie_side_table_bytes), - KUNIT_CASE(stackdepot_trie_side_prepare_updates), - KUNIT_CASE(stackdepot_trie_side_prepare_failure), - KUNIT_CASE(stackdepot_trie_side_prepare_duplicate_id), - KUNIT_CASE(stackdepot_trie_side_prepare_rejects_extra_update), - KUNIT_CASE(stackdepot_trie_side_prepare_rejects_null_leaf), - KUNIT_CASE(stackdepot_trie_pool_alloc_size), - KUNIT_CASE(stackdepot_trie_pool_prealloc), - KUNIT_CASE(stackdepot_trie_alloc_prealloc), - KUNIT_CASE(stackdepot_trie_pool_carve_node_test), - KUNIT_CASE(stackdepot_trie_pool_rollback_requires_lifo), - KUNIT_CASE(stackdepot_trie_pool_carve_node_rejects_bad_inputs), - KUNIT_CASE(stackdepot_trie_pool_carve_slots), - KUNIT_CASE(stackdepot_trie_pool_carve_slots_rejects_bad_inputs), - KUNIT_CASE(stackdepot_trie_pool_carve_uses_prealloc), - KUNIT_CASE(stackdepot_trie_pool_carve_no_prealloc_rollover), - KUNIT_CASE(stackdepot_trie_alloc_txn_id), - KUNIT_CASE(stackdepot_trie_alloc_txn_reserve), - KUNIT_CASE(stackdepot_trie_alloc_txn_reserve_id_failure), - KUNIT_CASE(stackdepot_trie_alloc_txn_commit), - KUNIT_CASE(stackdepot_trie_alloc_txn_rollback), - KUNIT_CASE(stackdepot_trie_alloc_workspace_insert), - KUNIT_CASE(stackdepot_trie_save_locked), - KUNIT_CASE(stackdepot_trie_fetch_handle_into), - KUNIT_CASE(stackdepot_trie_snprint_public), - KUNIT_CASE(stackdepot_trie_alloc_txn_plan), - KUNIT_CASE(stackdepot_trie_alloc_txn_insert), - KUNIT_CASE(stackdepot_trie_alloc_txn_insert_stale_plan), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), @@ -5648,116 +491,7 @@ static struct kunit_case stackdepot_test_cases[] = { #ifdef CONFIG_ARM64 KUNIT_CASE(stackdepot_frame_arm64), #endif - KUNIT_CASE(stackdepot_frame_run_raw_roundtrip), -#ifdef CONFIG_ARM64 - KUNIT_CASE(stackdepot_frame_run_arm64_roundtrip), -#endif -#ifdef CONFIG_X86_64 - KUNIT_CASE(stackdepot_frame_run_x86_64_roundtrip), - KUNIT_CASE(stackdepot_frame_run_x86_64_boundary), - KUNIT_CASE(stackdepot_frame_run_x86_64_write_rejects_mismatch), -#endif -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) - KUNIT_CASE(stackdepot_frame_run_compressed_rejects_src_scratch_overlap), -#endif - KUNIT_CASE(stackdepot_frame_run_invalid_inputs), - KUNIT_CASE(stackdepot_trie_node_raw_roundtrip), - KUNIT_CASE(stackdepot_trie_node_parent_chain), - KUNIT_CASE(stackdepot_trie_node_slice_raw), - KUNIT_CASE(stackdepot_trie_node_slice_parent_chain), -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) - KUNIT_CASE(stackdepot_trie_node_slice_compressed), -#endif - KUNIT_CASE(stackdepot_trie_node_slice_rejects_bad_inputs), - KUNIT_CASE(stackdepot_trie_node_rejects_stack_len_overflow), - KUNIT_CASE(stackdepot_trie_node_match_raw), - KUNIT_CASE(stackdepot_trie_append_chain_raw), - KUNIT_CASE(stackdepot_trie_append_chain_parent), - KUNIT_CASE(stackdepot_trie_append_chain_rejects_stack_len_overflow), - KUNIT_CASE(stackdepot_trie_publish_append_root), - KUNIT_CASE(stackdepot_trie_publish_append_parent), - KUNIT_CASE(stackdepot_trie_publish_append_root_replaces_array), - KUNIT_CASE(stackdepot_trie_publish_append_rejects_bad_inputs), - KUNIT_CASE(stackdepot_trie_lookup_step_root), - KUNIT_CASE(stackdepot_trie_lookup_step_parent_promote), - KUNIT_CASE(stackdepot_trie_lookup_step_accepts_reparented_child), - KUNIT_CASE(stackdepot_trie_find_leaf_root), - KUNIT_CASE(stackdepot_trie_find_leaf_descends), - KUNIT_CASE(stackdepot_trie_find_leaf_accepts_reparented_child), - KUNIT_CASE(stackdepot_trie_find_leaf_misses), - KUNIT_CASE(stackdepot_trie_find_leaf_rejects_bad_parent), - KUNIT_CASE(stackdepot_trie_insert_append_root), - KUNIT_CASE(stackdepot_trie_insert_append_prepare_root), - KUNIT_CASE(stackdepot_trie_insert_append_prepare_failure), - KUNIT_CASE(stackdepot_trie_insert_append_prepare_promote_failure), - KUNIT_CASE(stackdepot_trie_insert_append_prepare_split), - KUNIT_CASE(stackdepot_trie_insert_append_prepare_split_failure), - KUNIT_CASE(stackdepot_trie_insert_append_parent), - KUNIT_CASE(stackdepot_trie_insert_append_descends_one_level), - KUNIT_CASE(stackdepot_trie_insert_append_descends_multiple_levels), - KUNIT_CASE(stackdepot_trie_insert_append_descend_rejects_sibling_overlap), - KUNIT_CASE(stackdepot_trie_insert_append_promotes_internal), - KUNIT_CASE(stackdepot_trie_insert_append_descends_to_promote), - KUNIT_CASE(stackdepot_trie_insert_append_promotes_with_children), -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) - KUNIT_CASE(stackdepot_trie_insert_append_splits_frame_runs), -#endif - KUNIT_CASE(stackdepot_trie_insert_append_splits_child), - KUNIT_CASE(stackdepot_trie_insert_append_splits_prefix_leaf), - KUNIT_CASE(stackdepot_trie_insert_plan_append), -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) - KUNIT_CASE(stackdepot_trie_insert_plan_mixed_append), -#endif - KUNIT_CASE(stackdepot_trie_insert_plan_promote), - KUNIT_CASE(stackdepot_trie_insert_plan_promote_rejects_empty_slots), - KUNIT_CASE(stackdepot_trie_insert_plan_split), - KUNIT_CASE(stackdepot_trie_insert_plan_descends), - KUNIT_CASE(stackdepot_trie_insert_plan_rejects_existing_leaf), - KUNIT_CASE(stackdepot_trie_insert_plan_rejects_bad_child), - KUNIT_CASE(stackdepot_trie_insert_append_rejects_existing_child), - KUNIT_CASE(stackdepot_trie_insert_append_rejects_short_array), - KUNIT_CASE(stackdepot_trie_insert_append_rejects_zero_frame), - KUNIT_CASE(stackdepot_trie_insert_append_rejects_root_with_parent), - KUNIT_CASE(stackdepot_trie_insert_append_rejects_root_slot_alias), - KUNIT_CASE(stackdepot_trie_insert_append_rejects_parent_overlap), - KUNIT_CASE(stackdepot_trie_insert_append_rejects_parent_cycle), - KUNIT_CASE(stackdepot_trie_insert_append_rejects_publish_overlap), - KUNIT_CASE(stackdepot_trie_insert_append_rejects_child_node_overlap), - KUNIT_CASE(stackdepot_trie_insert_append_rejects_child_array_overlap), -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) - KUNIT_CASE(stackdepot_trie_node_compressed_roundtrip), - KUNIT_CASE(stackdepot_trie_node_match_compressed), - KUNIT_CASE(stackdepot_trie_append_chain_splits_frame_runs), - KUNIT_CASE(stackdepot_trie_append_chain_rejects_bad_inputs), - KUNIT_CASE(stackdepot_trie_node_rejects_compressed_without_scratch), -#endif - KUNIT_CASE(stackdepot_trie_node_rejects_short_storage), -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) - KUNIT_CASE(stackdepot_trie_node_rejects_mixed_run), -#endif - KUNIT_CASE(stackdepot_trie_fetch_rejects_bad_inputs), - KUNIT_CASE(stackdepot_trie_child_array_init_find), - KUNIT_CASE(stackdepot_trie_child_array_rejects_unsorted), - KUNIT_CASE(stackdepot_trie_child_array_init_rejects_child_overlap), - KUNIT_CASE(stackdepot_trie_child_array_insert), - KUNIT_CASE(stackdepot_trie_child_array_insert_rejects_child_overlap), - KUNIT_CASE(stackdepot_trie_split_child_array_init_one_child), - KUNIT_CASE(stackdepot_trie_split_child_array_init_orders_children), - KUNIT_CASE(stackdepot_trie_split_child_array_rejects_bad_inputs), - KUNIT_CASE(stackdepot_trie_split_tail_plan_raw), -#if defined(CONFIG_ARM64) || defined(CONFIG_X86_64) - KUNIT_CASE(stackdepot_trie_split_tail_plan_mixed_runs), -#endif - KUNIT_CASE(stackdepot_trie_split_tail_plan_rejects_bad_inputs), - KUNIT_CASE(stackdepot_trie_split_precheck), - KUNIT_CASE(stackdepot_trie_split_precheck_rejects_aliases), - KUNIT_CASE(stackdepot_trie_split_precheck_rejects_short_array), - KUNIT_CASE(stackdepot_trie_split_subtree_divergent_tail), - KUNIT_CASE(stackdepot_trie_split_subtree_prefix_leaf), - KUNIT_CASE(stackdepot_trie_split_subtree_preserves_children), - KUNIT_CASE(stackdepot_trie_child_array_insert_empty), - KUNIT_CASE(stackdepot_trie_public_save_route), - KUNIT_CASE(stackdepot_trie_concurrent_save_fetch), + KUNIT_CASE(stackdepot_concurrent_save_fetch), {} }; From bbf31e7ffd517a9495a5f391883d99284f01f9b3 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 19 Jun 2026 20:03:31 +0100 Subject: [PATCH 110/129] KRN-1117: Clarify stackdepot caller invariants Clarify the remaining caller-side invariants after the trie cleanup. Note that DRM intentionally uses a short stack cap and keeps save/fetch depths in sync, and rewrite the page_owner count comments to describe the fail-closed transition and failure-handle rules. Signed-off-by: Caleb Kan --- drivers/gpu/drm/drm_modeset_lock.c | 2 +- mm/page_owner.c | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c index 731bc1cf5fd04..bd136471b0abe 100644 --- a/drivers/gpu/drm/drm_modeset_lock.c +++ b/drivers/gpu/drm/drm_modeset_lock.c @@ -81,7 +81,7 @@ static DEFINE_WW_CLASS(crtc_ww_class); #if IS_ENABLED(CONFIG_DRM_DEBUG_MODESET_LOCK) -/* Save and fetch use the same cap so fetch_into() cannot reject saved stacks. */ +/* Existing debug output only records a short caller chain; keep save/fetch caps in sync. */ #define DRM_STACK_DEPOT_MAX_FRAMES 8 static noinline depot_stack_handle_t __drm_stack_depot_save(void) diff --git a/mm/page_owner.c b/mm/page_owner.c index 49f496a963d6d..199c580a53cdc 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -227,13 +227,17 @@ static bool inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, if (!handle || !nr_base_pages) return false; - /* Snapshot only avoids allocation when the stack is already counted. */ - /* If this races a final decrement to zero, inc_count() fails safely. */ + /* + * Snapshot only avoids allocation when the stack is already counted. If this + * races a final decrement to zero, inc_count() fails safely. + */ if (!__stack_depot_get_count(handle, &count)) stack = alloc_stack_record(gfp_mask); - /* Only one caller can win the saturated-to-counted cmpxchg transition. */ - /* Racing transition losers free their unused list node below. */ + /* + * Only one caller can win the saturated-to-counted cmpxchg transition. + * Racing transition losers free their unused list node below. + */ if (!__stack_depot_inc_count(handle, nr_base_pages, &new_count)) { if (stack) free_stack_record(stack); @@ -365,7 +369,7 @@ noinline void __set_page_owner(struct page *page, unsigned short order, handle = save_stack(gfp_mask); counted = inc_stack_record_count(handle, gfp_mask, 1 << order); if (!counted && handle != failure_handle) { - /* Attribute to failure_handle only if it can be symmetrically counted. */ + /* Store failure_handle only if the matching count was applied. */ handle = failure_handle; counted = inc_stack_record_count(handle, gfp_mask, 1 << order); } From 8f6d52c37ca8010686c7fa59575c001ef11bf84d Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 22 Jun 2026 09:22:43 +0100 Subject: [PATCH 111/129] KRN-1117: Fold stackdepot trie internals into implementation Remove the single-use stackdepot internal header now that KUnit no longer needs private trie declarations. Keeping the trie types and helpers local to lib/stackdepot.c avoids a fake internal API boundary and makes the backend implementation easier to review. Make the folded trie helpers static so the cleanup does not leave behind unnecessary global symbols. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 469 +++++++++++++++++++++++++++++++------- lib/stackdepot_internal.h | 313 ------------------------- 2 files changed, 385 insertions(+), 397 deletions(-) delete mode 100644 lib/stackdepot_internal.h diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 35edd849a3118..4ce7c788c3b53 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -47,7 +47,300 @@ #include -#include "stackdepot_internal.h" +enum stack_depot_frame_mode { + STACK_DEPOT_FRAME_RAW, + STACK_DEPOT_FRAME_COMPRESSED, +}; + +enum stack_depot_trie_lookup_status { + STACK_DEPOT_TRIE_LOOKUP_APPEND, + STACK_DEPOT_TRIE_LOOKUP_DESCEND, + STACK_DEPOT_TRIE_LOOKUP_FOUND, + STACK_DEPOT_TRIE_LOOKUP_PROMOTE, + STACK_DEPOT_TRIE_LOOKUP_SPLIT, +}; + +struct stack_depot_frame_run { + u16 bytes; + u16 nr_entries; + u8 mode; +}; + +static_assert(CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(unsigned long) <= U16_MAX); + +/* Opaque trie node storage; node layout stays private to stackdepot.c. */ +struct stack_depot_trie_node_slot { + void *node; + size_t size; +}; + +/* Opaque child-array storage; child array layout stays private. */ +struct stack_depot_trie_child_array_slot { + void *array; + size_t size; +}; + +struct stack_depot_trie_child_array; + +struct stack_depot_trie_root { + const struct stack_depot_trie_child_array *children; +}; + +struct stack_depot_trie_lookup { + /* Opaque parent trie node for the current lookup step. */ + const void *parent; + /* Opaque trie node matched at this step, if any. */ + const void *node; + enum stack_depot_trie_lookup_status status; + unsigned int matched; +}; + +struct stack_depot_trie_leaf_update { + u32 leaf_id; + /* Opaque trie leaf that should become visible for leaf_id. */ + const void *leaf; +}; + +struct stack_depot_trie_publish_prepare { + int (*fn)(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, void *ctx); + /* Caller-owned state passed to fn. */ + void *ctx; + bool retire_locked; +}; + +/* A split can repoint the old leaf and publish one new leaf. */ +#define STACK_DEPOT_TRIE_MAX_LEAF_UPDATES 2 +#define STACK_DEPOT_TRIE_MAX_NODE_SLOTS (CONFIG_STACKDEPOT_MAX_FRAMES + 1) +#define STACK_DEPOT_TRIE_MAX_CHILD_SLOTS CONFIG_STACKDEPOT_MAX_FRAMES + +struct stack_depot_trie_side_checkpoint { + u32 leaf_id; + const void *old_leaf; +}; + +struct stack_depot_trie_side_prepare { + struct stack_depot_trie_side_checkpoint updates[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; + unsigned int nr_updates; +}; + +struct stack_depot_trie_side_prealloc { + /* Preallocated side-table directory page for sparse growth. */ + void *dir; + /* Preallocated side-table leaf chunk for sparse growth. */ + void *chunk; +}; + +struct stack_depot_trie_pool_mark { + /* Stackdepot pool backing this transactional reservation. */ + void *pool; + size_t prev_offset; + size_t offset; + size_t size; + unsigned int pool_index; + bool added_pool; +}; + +struct stack_depot_trie_pool_request { + struct stack_depot_trie_node_slot *node_slots; + struct stack_depot_trie_child_array_slot *child_slots; + /* Optional opaque object storage reserved with the node/child slots. */ + void **storage; + /* Optional fresh stackdepot pool page, preallocated outside pool_lock. */ + void **prealloc; + struct stack_depot_trie_pool_mark *mark; + size_t storage_size; + unsigned int nr_node_slots; + unsigned int nr_child_slots; +}; + +struct stack_depot_trie_alloc_txn { + struct stack_depot_trie_side_prepare side; + struct stack_depot_trie_pool_mark pool; + u32 leaf_id; +}; + +struct stack_depot_trie_alloc_request { + struct stack_depot_trie_alloc_txn *txn; + struct stack_depot_trie_node_slot *node_slots; + struct stack_depot_trie_child_array_slot *child_slots; + /* Optional opaque replacement child-array storage. */ + void **storage; + /* Optional fresh stackdepot pool page, preallocated before insertion. */ + void **pool_prealloc; + struct stack_depot_trie_side_prealloc *side_prealloc; + size_t storage_size; + unsigned int nr_node_slots; + unsigned int nr_child_slots; +}; + +struct stack_depot_trie_alloc_workspace { + struct stack_depot_trie_alloc_txn txn; + struct stack_depot_trie_alloc_request req; + struct stack_depot_trie_node_slot node_slots[STACK_DEPOT_TRIE_MAX_NODE_SLOTS]; + struct stack_depot_trie_child_array_slot child_slots[STACK_DEPOT_TRIE_MAX_CHILD_SLOTS]; + u32 scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; + void *storage; +}; + +#define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 +#define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ + (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) + +static bool __stack_depot_trie_ready(void); +static void __stack_depot_trie_set_enabled(bool enabled); +static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id); +static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle); +static u32 __stack_depot_trie_max_leaf_id(void); +static int __stack_depot_trie_side_table_init(gfp_t gfp_flags); +static bool __stack_depot_trie_side_table_prealloc_needed(void); +static int __stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, + struct stack_depot_trie_side_prealloc *prealloc); +static void +__stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_prealloc *prealloc); +static u32 +__stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc); +static void __stack_depot_trie_side_table_revoke_latest(u32 id); +static void __stack_depot_trie_side_table_restore(u32 id, const void *entry); +static const void *__stack_depot_trie_side_table_lookup(u32 id); +static size_t __stack_depot_trie_side_table_bytes(void); +static size_t __stack_depot_trie_pool_alloc_size(size_t size); +static void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags); +static void __stack_depot_trie_pool_free_prealloc(void *prealloc); +static int +__stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, + depot_flags_t depot_flags, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc); +static int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req); +static void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); +static int +__stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, + struct stack_depot_trie_side_prealloc *prealloc); +static int +__stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, + const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + struct stack_depot_trie_alloc_txn *txn, + void **storage, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, + struct stack_depot_trie_alloc_request *req); +static int +__stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, + const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace, + const void **tail, u32 *leaf_id); +static depot_stack_handle_t +__stack_depot_trie_save_locked(struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags, depot_flags_t depot_flags, + struct stack_depot_trie_alloc_workspace *workspace, + raw_spinlock_t *workspace_lock); +static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); +static u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn); +static int +__stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, + struct stack_depot_trie_alloc_request *req, + const unsigned long *entries, unsigned int nr_entries, + u32 *scratch, unsigned int nr_scratch, + const void **tail, u32 *leaf_id); +static void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); +static int +__stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, void *ctx); +static void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *state); +static int __stack_depot_frame_run_init(const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_frame_run *run); +static size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run); +static int __stack_depot_trie_node_init(void *storage, size_t storage_size, + const void *parent, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, u32 *scratch, + unsigned int nr_scratch); +static int __stack_depot_trie_node_init_slice(void *storage, size_t storage_size, + const void *parent, u32 leaf_id, + const void *src_node, unsigned int start, + unsigned int nr_entries); +static unsigned int __stack_depot_trie_node_match(const void *node, + const unsigned long *entries, + unsigned int nr_entries); +static int +__stack_depot_trie_append_chain(const void *parent, u32 leaf_id, + const unsigned long *entries, unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, const void **head, + const void **tail, unsigned int *nr_used); +static int +__stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, + const void *parent, const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_lookup *lookup); +static const void * +__stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, + const unsigned long *entries, unsigned int nr_entries); +static int +__stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, + void *parent, u32 leaf_id, + const unsigned long *entries, unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot + *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, void *new_storage, + size_t new_storage_size, + const struct stack_depot_trie_publish_prepare *prepare, + const void **tail, unsigned int *nr_used); +static int +__stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, + const void *parent, const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, size_t *new_storage_size, + unsigned int *nr_used, unsigned int *nr_child_used); +static unsigned int __stack_depot_trie_fetch_into(const void *leaf, + unsigned long *entries, + unsigned int max_entries); +static unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries); +static size_t __stack_depot_trie_child_array_size(unsigned int nr_children); +static int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, + const void * const *children, + unsigned int nr_children); +static int +__stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, + const void *old_tail, const void *new_head); +static int +__stack_depot_trie_split_tail_plan(const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, unsigned int *nr_runs); +static int +__stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, + const void *parent, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + void *new_storage, size_t new_storage_size); +static int __stack_depot_trie_child_array_insert(const void *old_storage, + const void *child, + void *new_storage, + size_t new_storage_size); /* * The pool_index is offset by 1 so the first record does not have a 0 handle. @@ -66,12 +359,12 @@ static struct stack_depot_trie_alloc_workspace *stack_depot_trie_workspace; static DEFINE_RAW_SPINLOCK(stack_depot_trie_workspace_lock); static bool stack_depot_trie_ready; -bool __stack_depot_trie_enabled(void) +static bool __stack_depot_trie_enabled(void) { return static_branch_unlikely(&stack_depot_trie_enabled); } -void __stack_depot_trie_set_enabled(bool enabled) +static void __stack_depot_trie_set_enabled(bool enabled) { if (__stack_depot_trie_enabled() == enabled) return; @@ -312,7 +605,7 @@ static bool stack_depot_trie_namespace_available(void) * unavailable namespace as a hard trie failure; the checks below are defensive * because handle helpers can be reached from tests and disabled configurations. */ -u32 __stack_depot_trie_max_leaf_id(void) +static u32 __stack_depot_trie_max_leaf_id(void) { if (!stack_depot_trie_namespace_available()) return 0; @@ -321,7 +614,7 @@ u32 __stack_depot_trie_max_leaf_id(void) DEPOT_OFFSET_BITS; } -depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) +static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) { union handle_parts parts = {}; u64 pool_index_plus_1; @@ -344,7 +637,7 @@ depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) return parts.handle; } -u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) +static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) { union handle_parts parts = { .handle = handle }; u64 leaf_id; @@ -580,7 +873,7 @@ static void stack_pools_sorted_grow(gfp_t gfp_flags) kvfree(sorted); } -bool __stack_depot_trie_ready(void) +static bool __stack_depot_trie_ready(void) { return __stack_depot_trie_enabled() && stack_depot_trie_is_ready() && @@ -941,7 +1234,7 @@ trie_side_table_clear_entry(struct stack_depot_trie_side_entry *chunk, trie_side_table_store_leaf(chunk, slot, NULL); } -int __stack_depot_trie_side_table_init(gfp_t gfp_flags) +static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) { struct stack_depot_trie_side_dir **dirs; unsigned int root_size; @@ -967,7 +1260,7 @@ int __stack_depot_trie_side_table_init(gfp_t gfp_flags) false); } -bool __stack_depot_trie_side_table_prealloc_needed(void) +static bool __stack_depot_trie_side_table_prealloc_needed(void) { struct stack_depot_trie_side_dir *dir; unsigned long flags; @@ -1008,7 +1301,7 @@ static void *trie_side_table_alloc_page(gfp_t gfp_flags, unsigned int order) return page ? page_address(page) : NULL; } -int +static int __stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, struct stack_depot_trie_side_prealloc *prealloc) { @@ -1062,7 +1355,7 @@ __stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, return 0; } -void +static void __stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_prealloc *prealloc) { if (!prealloc) @@ -1073,7 +1366,7 @@ __stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_preallo prealloc->chunk = NULL; } -u32 +static u32 __stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc) { unsigned long flags; @@ -1085,7 +1378,7 @@ __stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *pr return id; } -void __stack_depot_trie_side_table_revoke_latest(u32 id) +static void __stack_depot_trie_side_table_revoke_latest(u32 id) { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; @@ -1118,7 +1411,7 @@ void __stack_depot_trie_side_table_revoke_latest(u32 id) raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); } -void __stack_depot_trie_side_table_restore(u32 id, const void *entry) +static void __stack_depot_trie_side_table_restore(u32 id, const void *entry) { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; @@ -1177,7 +1470,7 @@ trie_side_table_chunk_locked(u32 id, unsigned int *slot) return chunk; } -const void *__stack_depot_trie_side_table_lookup(u32 id) +static const void *__stack_depot_trie_side_table_lookup(u32 id) { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; @@ -1200,7 +1493,7 @@ const void *__stack_depot_trie_side_table_lookup(u32 id) return trie_side_table_load_leaf(chunk, trie_side_table_slot_index(id)); } -size_t __stack_depot_trie_side_table_bytes(void) +static size_t __stack_depot_trie_side_table_bytes(void) { unsigned int nr_dirs; unsigned int nr_chunks; @@ -1230,7 +1523,7 @@ size_t __stack_depot_trie_side_table_bytes(void) return bytes; } -size_t __stack_depot_trie_pool_alloc_size(size_t size) +static size_t __stack_depot_trie_pool_alloc_size(size_t size) { size_t align = 1UL << DEPOT_STACK_ALIGN; size_t aligned; @@ -1668,16 +1961,16 @@ void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags) return page ? page_address(page) : NULL; } -void __stack_depot_trie_pool_free_prealloc(void *prealloc) +static void __stack_depot_trie_pool_free_prealloc(void *prealloc) { if (prealloc) free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); } -int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, - depot_flags_t depot_flags, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc) +static int +__stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) { bool needs_side_prealloc; bool can_alloc; @@ -1823,7 +2116,7 @@ static void trie_pool_release_reused_objects(struct stack_depot_trie_pool_reques raw_spin_unlock_irqrestore(&pool_lock, flags); } -int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) +static int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) { unsigned long flags; unsigned int i; @@ -1936,13 +2229,13 @@ int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) return ret; } -void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn) +static void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn) { if (txn) memset(txn, 0, sizeof(*txn)); } -int +static int __stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, struct stack_depot_trie_side_prealloc *prealloc) { @@ -1989,7 +2282,7 @@ static void trie_alloc_request_release_reused_objects(struct stack_depot_trie_al trie_pool_release_reused_objects(&pool_req); } -int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req) +static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req) { struct stack_depot_trie_pool_request pool_req = {}; int ret; @@ -2023,7 +2316,7 @@ int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request * return 0; } -int +static int __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, @@ -2087,12 +2380,13 @@ trie_ws_plan(const struct stack_depot_trie_root *root, side_prealloc, &workspace->req); } -int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, - const unsigned long *entries, - unsigned int nr_entries, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace, - const void **tail, u32 *leaf_id) +static int +__stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, + const unsigned long *entries, + unsigned int nr_entries, void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace, + const void **tail, u32 *leaf_id) { void **pool = pool_prealloc; int ret; @@ -2182,7 +2476,7 @@ trie_save_trylocked(struct stack_depot_trie_root *root, return handle; } -depot_stack_handle_t +static depot_stack_handle_t __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags, depot_flags_t depot_flags, @@ -2225,7 +2519,7 @@ __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, return handle; } -u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) +static u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) { u32 leaf_id; @@ -2237,7 +2531,7 @@ u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) return leaf_id; } -int +static int __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, struct stack_depot_trie_alloc_request *req, const unsigned long *entries, @@ -2293,7 +2587,7 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, return ret; } -void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn) +static void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn) { if (!txn) return; @@ -2307,7 +2601,7 @@ void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *tx memset(&txn->pool, 0, sizeof(txn->pool)); } -void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *state) +static void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *state) { if (!state) return; @@ -2358,7 +2652,7 @@ trie_side_prepare_locked(const struct stack_depot_trie_leaf_update *updates, return 0; } -int +static int __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, unsigned int nr_updates, void *ctx) { @@ -3335,9 +3629,10 @@ static int frame_run_init_lows(const unsigned long *entries, return 0; } -int __stack_depot_frame_run_init(const unsigned long *entries, - unsigned int nr_entries, - struct stack_depot_frame_run *run) +static int +__stack_depot_frame_run_init(const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_frame_run *run) { return frame_run_init_lows(entries, nr_entries, run, NULL, 0); } @@ -3381,7 +3676,7 @@ static bool stack_depot_ranges_overlap(const void *a, size_t a_size, return a_start < b_end && b_start < a_end; } -size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) +static size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) { size_t size; @@ -3439,11 +3734,12 @@ stack_depot_trie_node_first_frame(const struct stack_depot_trie_node *node, return stack_depot_trie_node_frame(node, 0, frame); } -int __stack_depot_trie_node_init(void *storage, size_t storage_size, - const void *parent, u32 leaf_id, - const unsigned long *entries, - unsigned int nr_entries, u32 *scratch, - unsigned int nr_scratch) +static int +__stack_depot_trie_node_init(void *storage, size_t storage_size, + const void *parent, u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, u32 *scratch, + unsigned int nr_scratch) { const struct stack_depot_trie_node *parent_node = parent; struct stack_depot_trie_node *node = storage; @@ -3493,10 +3789,11 @@ int __stack_depot_trie_node_init(void *storage, size_t storage_size, return 0; } -int __stack_depot_trie_node_init_slice(void *storage, size_t storage_size, - const void *parent, u32 leaf_id, - const void *src_node, unsigned int start, - unsigned int nr_entries) +static int +__stack_depot_trie_node_init_slice(void *storage, size_t storage_size, + const void *parent, u32 leaf_id, + const void *src_node, unsigned int start, + unsigned int nr_entries) { const struct stack_depot_trie_node *parent_node = parent; const struct stack_depot_trie_node *src = src_node; @@ -3541,9 +3838,10 @@ int __stack_depot_trie_node_init_slice(void *storage, size_t storage_size, return 0; } -unsigned int __stack_depot_trie_node_match(const void *node_ptr, - const unsigned long *entries, - unsigned int nr_entries) +static unsigned int +__stack_depot_trie_node_match(const void *node_ptr, + const unsigned long *entries, + unsigned int nr_entries) { const struct stack_depot_trie_node *node = node_ptr; unsigned int limit; @@ -4279,7 +4577,7 @@ static int trie_append_chain_validate(const struct stack_depot_trie_node *parent return 0; } -int +static int __stack_depot_trie_append_chain(const void *parent_ptr, u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, @@ -4441,7 +4739,7 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, return 0; } -int +static int __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, const void *parent_ptr, const unsigned long *entries, unsigned int nr_entries, @@ -4514,7 +4812,7 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, return 0; } -const void * +static const void * __stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries) { @@ -4584,7 +4882,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, const void **tail, unsigned int *nr_used); -int +static int __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, void *parent_ptr, u32 leaf_id, const unsigned long *entries, @@ -4890,7 +5188,7 @@ static int trie_plan_split(const struct stack_depot_trie_child_array *children, return 0; } -int +static int __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, const void *parent_ptr, const unsigned long *entries, unsigned int nr_entries, @@ -5180,7 +5478,7 @@ static void trie_fetch_frame(unsigned int index, unsigned long frame, void *data ctx->nr_entries++; } -unsigned int +static unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries) { @@ -5204,7 +5502,7 @@ __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, return total; } -unsigned int +static unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, unsigned long *entries, unsigned int max_entries) @@ -5262,16 +5560,17 @@ static size_t trie_child_array_size_for_capacity(unsigned int capacity) return ALIGN(size, sizeof(unsigned long)); } -size_t __stack_depot_trie_child_array_size(unsigned int nr_children) +static size_t __stack_depot_trie_child_array_size(unsigned int nr_children) { unsigned int capacity = trie_child_array_capacity(nr_children); return trie_child_array_size_for_capacity(capacity); } -int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, - const void * const *children, - unsigned int nr_children) +static int +__stack_depot_trie_child_array_init(void *storage, size_t storage_size, + const void * const *children, + unsigned int nr_children) { struct stack_depot_trie_child_array *array = storage; const struct stack_depot_trie_node * const *nodes = @@ -5315,9 +5614,10 @@ int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, return 0; } -int __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, - const void *old_tail, - const void *new_head) +static int +__stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, + const void *old_tail, + const void *new_head) { const void *children[2]; unsigned long new_frame; @@ -5346,13 +5646,13 @@ int __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size return __stack_depot_trie_child_array_init(storage, storage_size, children, 2); } -int __stack_depot_trie_split_tail_plan(const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, - unsigned int *nr_runs) +static int +__stack_depot_trie_split_tail_plan(const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, unsigned int *nr_runs) { unsigned int pos = 0; unsigned int runs = 0; @@ -5406,13 +5706,14 @@ int __stack_depot_trie_split_tail_plan(const unsigned long *entries, return 0; } -int __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, - const void *parent_ptr, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, - void *new_storage, size_t new_storage_size) +static int +__stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, + const void *parent_ptr, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, + void *new_storage, size_t new_storage_size) { struct stack_depot_trie_node *parent = (void *)parent_ptr; const struct stack_depot_trie_child_array **slot; @@ -5827,7 +6128,7 @@ stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *ar return 0; } -int +static int __stack_depot_trie_child_array_insert(const void *old_storage, const void *child, void *new_storage, size_t new_storage_size) { diff --git a/lib/stackdepot_internal.h b/lib/stackdepot_internal.h deleted file mode 100644 index b4d8f99c5b6f1..0000000000000 --- a/lib/stackdepot_internal.h +++ /dev/null @@ -1,313 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -#ifndef _STACKDEPOT_INTERNAL_H -#define _STACKDEPOT_INTERNAL_H - -#include -#include -#include -#include -#include - -enum stack_depot_frame_mode { - STACK_DEPOT_FRAME_RAW, - STACK_DEPOT_FRAME_COMPRESSED, -}; - -enum stack_depot_trie_lookup_status { - STACK_DEPOT_TRIE_LOOKUP_APPEND, - STACK_DEPOT_TRIE_LOOKUP_DESCEND, - STACK_DEPOT_TRIE_LOOKUP_FOUND, - STACK_DEPOT_TRIE_LOOKUP_PROMOTE, - STACK_DEPOT_TRIE_LOOKUP_SPLIT, -}; - -struct stack_depot_frame_run { - u16 bytes; - u16 nr_entries; - u8 mode; -}; - -static_assert(CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(unsigned long) <= U16_MAX); - -bool __stack_depot_trie_enabled(void); -bool __stack_depot_trie_ready(void); -void __stack_depot_trie_set_enabled(bool enabled); - -/* Opaque trie node storage; node layout stays private to stackdepot.c. */ -struct stack_depot_trie_node_slot { - void *node; - size_t size; -}; - -/* Opaque child-array storage; child array layout stays private. */ -struct stack_depot_trie_child_array_slot { - void *array; - size_t size; -}; - -struct stack_depot_trie_child_array; - -struct stack_depot_trie_root { - const struct stack_depot_trie_child_array *children; -}; - -struct stack_depot_trie_lookup { - /* Opaque parent trie node for the current lookup step. */ - const void *parent; - /* Opaque trie node matched at this step, if any. */ - const void *node; - enum stack_depot_trie_lookup_status status; - unsigned int matched; -}; - -struct stack_depot_trie_leaf_update { - u32 leaf_id; - /* Opaque trie leaf that should become visible for leaf_id. */ - const void *leaf; -}; - -struct stack_depot_trie_publish_prepare { - int (*fn)(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, void *ctx); - /* Caller-owned state passed to fn. */ - void *ctx; - bool retire_locked; -}; - -/* A split can repoint the old leaf and publish one new leaf. */ -#define STACK_DEPOT_TRIE_MAX_LEAF_UPDATES 2 -#define STACK_DEPOT_TRIE_MAX_NODE_SLOTS (CONFIG_STACKDEPOT_MAX_FRAMES + 1) -#define STACK_DEPOT_TRIE_MAX_CHILD_SLOTS CONFIG_STACKDEPOT_MAX_FRAMES - -struct stack_depot_trie_side_checkpoint { - u32 leaf_id; - const void *old_leaf; -}; - -struct stack_depot_trie_side_prepare { - struct stack_depot_trie_side_checkpoint updates[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; - unsigned int nr_updates; -}; - -struct stack_depot_trie_side_prealloc { - /* Preallocated side-table directory page for sparse growth. */ - void *dir; - /* Preallocated side-table leaf chunk for sparse growth. */ - void *chunk; -}; - -struct stack_depot_trie_pool_mark { - /* Stackdepot pool backing this transactional reservation. */ - void *pool; - size_t prev_offset; - size_t offset; - size_t size; - unsigned int pool_index; - bool added_pool; -}; - -struct stack_depot_trie_pool_request { - struct stack_depot_trie_node_slot *node_slots; - struct stack_depot_trie_child_array_slot *child_slots; - /* Optional opaque object storage reserved with the node/child slots. */ - void **storage; - /* Optional fresh stackdepot pool page, preallocated outside pool_lock. */ - void **prealloc; - struct stack_depot_trie_pool_mark *mark; - size_t storage_size; - unsigned int nr_node_slots; - unsigned int nr_child_slots; -}; - -struct stack_depot_trie_alloc_txn { - struct stack_depot_trie_side_prepare side; - struct stack_depot_trie_pool_mark pool; - u32 leaf_id; -}; - -struct stack_depot_trie_alloc_request { - struct stack_depot_trie_alloc_txn *txn; - struct stack_depot_trie_node_slot *node_slots; - struct stack_depot_trie_child_array_slot *child_slots; - /* Optional opaque replacement child-array storage. */ - void **storage; - /* Optional fresh stackdepot pool page, preallocated before insertion. */ - void **pool_prealloc; - struct stack_depot_trie_side_prealloc *side_prealloc; - size_t storage_size; - unsigned int nr_node_slots; - unsigned int nr_child_slots; -}; - -struct stack_depot_trie_alloc_workspace { - struct stack_depot_trie_alloc_txn txn; - struct stack_depot_trie_alloc_request req; - struct stack_depot_trie_node_slot node_slots[STACK_DEPOT_TRIE_MAX_NODE_SLOTS]; - struct stack_depot_trie_child_array_slot child_slots[STACK_DEPOT_TRIE_MAX_CHILD_SLOTS]; - u32 scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; - void *storage; -}; - -#define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 -#define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ - (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) - -depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id); -u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle); -u32 __stack_depot_trie_max_leaf_id(void); - -/* - * Private trie side table. Writers serialize internally; lookups are lockless. - * Leaf slots are populated before trie publication. Initialization is one-way - * because trie handles can outlive runtime disabling of new trie saves. - */ -int __stack_depot_trie_side_table_init(gfp_t gfp_flags); -bool __stack_depot_trie_side_table_prealloc_needed(void); -int -__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, - struct stack_depot_trie_side_prealloc *prealloc); -void -__stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_prealloc *prealloc); -u32 -__stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc); -void __stack_depot_trie_side_table_revoke_latest(u32 id); -void __stack_depot_trie_side_table_restore(u32 id, const void *entry); -const void *__stack_depot_trie_side_table_lookup(u32 id); -size_t __stack_depot_trie_side_table_bytes(void); -size_t __stack_depot_trie_pool_alloc_size(size_t size); -void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags); -void __stack_depot_trie_pool_free_prealloc(void *prealloc); -int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, - depot_flags_t depot_flags, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc); -int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req); -void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); -int -__stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, - struct stack_depot_trie_side_prealloc *prealloc); -int -__stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, - const unsigned long *entries, - unsigned int nr_entries, - struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, - struct stack_depot_trie_alloc_txn *txn, - void **storage, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_request *req); -int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, - const unsigned long *entries, - unsigned int nr_entries, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace, - const void **tail, u32 *leaf_id); -depot_stack_handle_t -__stack_depot_trie_save_locked(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - gfp_t alloc_flags, depot_flags_t depot_flags, - struct stack_depot_trie_alloc_workspace *workspace, - raw_spinlock_t *workspace_lock); -int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); -u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn); -int -__stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, - struct stack_depot_trie_alloc_request *req, - const unsigned long *entries, - unsigned int nr_entries, u32 *scratch, - unsigned int nr_scratch, const void **tail, - u32 *leaf_id); -void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); -int -__stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, void *ctx); -void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *state); -int __stack_depot_frame_run_init(const unsigned long *entries, - unsigned int nr_entries, - struct stack_depot_frame_run *run); -size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run); -int __stack_depot_trie_node_init(void *storage, size_t storage_size, - const void *parent, u32 leaf_id, - const unsigned long *entries, - unsigned int nr_entries, u32 *scratch, - unsigned int nr_scratch); -int __stack_depot_trie_node_init_slice(void *storage, size_t storage_size, - const void *parent, u32 leaf_id, - const void *src_node, unsigned int start, - unsigned int nr_entries); -unsigned int __stack_depot_trie_node_match(const void *node, - const unsigned long *entries, - unsigned int nr_entries); -int __stack_depot_trie_append_chain(const void *parent, u32 leaf_id, - const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, const void **head, - const void **tail, unsigned int *nr_used); -int __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, - const void *parent, const unsigned long *entries, - unsigned int nr_entries, - struct stack_depot_trie_lookup *lookup); -const void * -__stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries); -int -__stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, - void *parent, u32 leaf_id, - const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot - *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, void *new_storage, - size_t new_storage_size, - const struct stack_depot_trie_publish_prepare *prepare, - const void **tail, unsigned int *nr_used); -int -__stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, - const void *parent, const unsigned long *entries, - unsigned int nr_entries, - struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, size_t *new_storage_size, - unsigned int *nr_used, unsigned int *nr_child_used); -unsigned int __stack_depot_trie_fetch_into(const void *leaf, - unsigned long *entries, - unsigned int max_entries); -unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, - unsigned long *entries, - unsigned int max_entries); -size_t __stack_depot_trie_child_array_size(unsigned int nr_children); -int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, - const void * const *children, - unsigned int nr_children); -int __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, - const void *old_tail, - const void *new_head); -int __stack_depot_trie_split_tail_plan(const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, - unsigned int *nr_runs); -int __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, - const void *parent, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, - void *new_storage, size_t new_storage_size); -int __stack_depot_trie_child_array_insert(const void *old_storage, - const void *child, void *new_storage, - size_t new_storage_size); - -#endif /* _STACKDEPOT_INTERNAL_H */ From 3218ba2cc9f1e4221fdd9266fe4149c853e852a6 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 22 Jun 2026 09:56:56 +0100 Subject: [PATCH 112/129] KRN-1117: Document stackdepot trie design invariants Clarify the trie backend invariants that were still implicit. Document the storage backends, frame-run encoding, child array publication, side-table sparse growth, allocation preconditions, pool-carve transactions, and no-spin trie save behavior. Also update stack_depot_put() documentation and explain why x86 frame compression only accepts kernel-text style addresses. Signed-off-by: Caleb Kan --- arch/x86/include/asm/stackdepot.h | 5 +++ include/linux/stackdepot.h | 12 ++++--- lib/stackdepot.c | 54 ++++++++++++++++++++++++++++--- 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/stackdepot.h b/arch/x86/include/asm/stackdepot.h index 457516ac9bf39..46e0cb0e7eb4c 100644 --- a/arch/x86/include/asm/stackdepot.h +++ b/arch/x86/include/asm/stackdepot.h @@ -6,6 +6,11 @@ #include #ifdef CONFIG_X86_64 +/* + * Compress canonical kernel text/module addresses whose upper 32 bits are all + * ones. Other kernel virtual addresses stay raw, so decompression reconstructs + * the original frame by restoring this prefix. + */ #define STACK_DEPOT_X86_64_FRAME_PREFIX 0xffffffff00000000UL #define STACK_DEPOT_X86_64_FRAME_LOW_MASK 0x00000000ffffffffUL diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 848fef1994cf2..07dd288ae5b12 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -299,10 +299,14 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, * * @handle: Stack depot handle returned from stack_depot_save() * - * The stack trace is evicted from stack depot once all references to it have - * been dropped (once the number of stack_depot_evict() calls matches the - * number of stack_depot_save_flags() calls with STACK_DEPOT_FLAG_GET set for - * this stack trace). + * Drop a reference acquired by stack_depot_save_flags() with + * %STACK_DEPOT_FLAG_GET. Calling this for a handle saved without + * %STACK_DEPOT_FLAG_GET is invalid; persistent handles, including trie-backed + * handles, are owned by stack depot for the lifetime of the system. + * + * The stack trace is evicted once the number of stack_depot_put() calls matches + * the number of successful stack_depot_save_flags() calls with + * %STACK_DEPOT_FLAG_GET for this stack trace. */ void stack_depot_put(depot_stack_handle_t handle); diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 4ce7c788c3b53..1bac5277ae9b7 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2,9 +2,11 @@ /* * Stack depot - a stack trace storage that avoids duplication. * - * Internally, stack depot maintains a hash table of unique stacktraces. The - * stack traces themselves are stored contiguously one after another in a set - * of separate page allocations. + * Internally, stack depot has two storage backends. Refcounted entries and + * callers that request STACK_DEPOT_FLAG_HASH use the legacy hash table with + * contiguous stack records in stack pools. Persistent non-refcounted entries + * can use trie storage when enabled; trie nodes share common frame prefixes and + * are published through RCU/COW child arrays. * * Author: Alexander Potapenko * Copyright (C) 2016 Google, Inc. @@ -60,6 +62,12 @@ enum stack_depot_trie_lookup_status { STACK_DEPOT_TRIE_LOOKUP_SPLIT, }; +/* + * A trie node stores one run of frames that all use the same payload format. + * Architectures may compress some frames to 32-bit payloads; mixed raw and + * compressed input is split across multiple trie nodes so each node has one + * decoding mode. + */ struct stack_depot_frame_run { u16 bytes; u16 nr_entries; @@ -106,6 +114,7 @@ struct stack_depot_trie_publish_prepare { unsigned int nr_updates, void *ctx); /* Caller-owned state passed to fn. */ void *ctx; + /* Use when the publish path already holds pool_lock for retirement. */ bool retire_locked; }; @@ -477,8 +486,14 @@ struct stack_depot_trie_node { unsigned char data[]; }; +/* + * Children are sorted by first frame and searched with lower_bound(). + * nr_children is the published element count. It can grow in place only when + * capacity has spare room; otherwise writers build a replacement array and + * publish that pointer. Readers use acquire loads for both the array pointer + * and nr_children. + */ struct stack_depot_trie_child_array { - /* nr_children/capacity must live with the pointer array readers index. */ unsigned int nr_children; unsigned int capacity; const struct stack_depot_trie_node *children[]; @@ -1035,7 +1050,12 @@ trie_side_table_install(struct stack_depot_trie_side_dir **dirs, struct stack_depot_trie_side_entry *first_chunk, bool memblock) { - /* Init installs only the root and first chunk; later chunks grow lazily. */ + /* + * Early init installs the first directory and chunk so early leaf ID + * allocation cannot fail immediately. Runtime init installs only the root + * vector; sparse directories and chunks are preallocated outside + * trie_side_table_lock and published lazily as IDs grow. + */ if (trie_side_table_is_initialized()) return 0; if (!dirs || !root_size || !max_id) @@ -1967,6 +1987,15 @@ static void __stack_depot_trie_pool_free_prealloc(void *prealloc) free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); } +/* + * Preallocate resources that cannot be allocated while trie writers hold raw + * spinlocks. Side-table growth is mandatory before a new leaf ID can be + * reserved, so side-table preallocation failure disables insertion for this + * save. Pool preallocation is opportunistic: reusable trie storage or active + * pool space may still satisfy the reservation, and pool_carve() reports + * -ENOSPC if they do not. Callers without spinning allocation context skip + * insertion and perform only best-effort lookup. + */ static int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, void **pool_prealloc, @@ -2116,6 +2145,15 @@ static void trie_pool_release_reused_objects(struct stack_depot_trie_pool_reques raw_spin_unlock_irqrestore(&pool_lock, flags); } +/* + * Reserve all pool-backed storage for one trie insertion transaction. The + * request may be satisfied by reusable retired fragments or by carving one + * contiguous mark from the current stackdepot pool, possibly after installing + * @prealloc as a new pool. @mark records only the newly carved range so + * rollback can rewind pool_offset; reused fragments are returned to their + * freelists separately on failure. The caller must not publish any returned + * storage until the trie and side-table transaction commits. + */ static int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) { unsigned long flags; @@ -2501,6 +2539,12 @@ __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, if (handle) return handle; no_spin = in_nmi() || !gfpflags_allow_spinning(alloc_flags); + /* + * No-spin callers cannot wait for the workspace lock or allocate side-table + * or pool storage. After the lockless lookup misses, trylock and recheck: a + * concurrent writer may have inserted the stack. Otherwise fail instead of + * spinning or publishing a new leaf. + */ if (no_spin) return trie_save_trylocked(root, entries, nr_entries, workspace, workspace_lock); From 0542d6b2a752845ae941b759c5144653e280ef84 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 22 Jun 2026 10:35:45 +0100 Subject: [PATCH 113/129] KRN-1117: Clarify stackdepot trie namespace checks Update the trie handle namespace comment now that KUnit no longer reaches private stackdepot internals. The remaining guard is defensive for disabled or failed trie initialization, while namespace exhaustion remains a hard trie init failure. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 1bac5277ae9b7..4537761fffb9e 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -618,7 +618,7 @@ static bool stack_depot_trie_namespace_available(void) * pool-index values above stack_max_pools and reinterpret the offset bits as a * dense leaf_id, which the side table maps to a trie leaf. Init treats an * unavailable namespace as a hard trie failure; the checks below are defensive - * because handle helpers can be reached from tests and disabled configurations. + * for disabled and failed-initialization configurations. */ static u32 __stack_depot_trie_max_leaf_id(void) { From c1a9c70be4667ada7ca789cdb66d260b08b78fd7 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 22 Jun 2026 13:13:40 +0100 Subject: [PATCH 114/129] KRN-1117: Avoid signed overflow in arm64 stackdepot compression Compute arm64 stackdepot frame offsets with unsigned arithmetic and validate compression by round-tripping through the signed 32-bit payload. This keeps the signed _text-relative encoding while avoiding assumptions about signed subtraction overflow behavior. Signed-off-by: Caleb Kan --- arch/arm64/include/asm/stackdepot.h | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h index eea824c293caf..6bd63d0f3d5c4 100644 --- a/arch/arm64/include/asm/stackdepot.h +++ b/arch/arm64/include/asm/stackdepot.h @@ -11,19 +11,29 @@ * kernel image. Store a signed 32-bit offset from _text so compression is * independent of 4 GB high-bit boundaries crossed by that window. */ +static inline unsigned long arch_stack_depot_frame_from_low(u32 low) +{ + long offset; + + offset = (s32)low; + if (offset < 0) + return (unsigned long)_text - (unsigned long)(-offset); + return (unsigned long)_text + (unsigned long)offset; +} + static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) { - long offset; + u32 candidate; if (!low) return false; - offset = (long)frame - (long)_text; - if (offset < S32_MIN || offset > S32_MAX) + candidate = (u32)(frame - (unsigned long)_text); + if (arch_stack_depot_frame_from_low(candidate) != frame) return false; - *low = (u32)(s32)offset; + *low = candidate; return true; } @@ -33,7 +43,7 @@ arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) if (!frame) return false; - *frame = (unsigned long)((long)_text + (s32)low); + *frame = arch_stack_depot_frame_from_low(low); return true; } From 272a5551ce36c0e05c2c4f7ad7c434d7d41301c3 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 22 Jun 2026 17:15:55 +0100 Subject: [PATCH 115/129] KRN-1117: Simplify stackdepot trie publication Simplify the trie storage path after review by replacing release acquire ordering with RCU publication helpers, removing the sorted pool side cache, and dropping unused side-table state. Keep tail-only in-place child-array append because it preserves the pool savings that matter for the KASAN workload while avoiding mutation of existing child pointers. Make the stackdepot flag mask name the valid flags directly so future flag additions do not depend on a separate count staying in sync. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 4 +- lib/stackdepot.c | 609 ++++++++++++++----------------------- 2 files changed, 234 insertions(+), 379 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 07dd288ae5b12..7e6bee257566b 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -49,8 +49,8 @@ typedef u32 depot_flags_t; #define STACK_DEPOT_FLAG_GET ((depot_flags_t)0x0002) #define STACK_DEPOT_FLAG_HASH ((depot_flags_t)0x0004) -#define STACK_DEPOT_FLAGS_NUM 3 -#define STACK_DEPOT_FLAGS_MASK ((depot_flags_t)((1 << STACK_DEPOT_FLAGS_NUM) - 1)) +#define STACK_DEPOT_FLAGS_MASK (STACK_DEPOT_FLAG_CAN_ALLOC | \ + STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH) /* * Using stack depot requires its initialization, which can be done in 3 ways: diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 4537761fffb9e..a22fd865dfeb7 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -38,7 +38,6 @@ #include #include #include -#include #include #include #include @@ -362,11 +361,12 @@ static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT); static bool __stack_depot_early_init_passed __initdata; static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); +static DEFINE_MUTEX(stack_depot_init_mutex); static DEFINE_MUTEX(stack_depot_trie_param_lock); static struct stack_depot_trie_root stack_depot_trie_root; -static struct stack_depot_trie_alloc_workspace *stack_depot_trie_workspace; +static struct stack_depot_trie_alloc_workspace __rcu *stack_depot_trie_workspace; static DEFINE_RAW_SPINLOCK(stack_depot_trie_workspace_lock); -static bool stack_depot_trie_ready; +static bool stack_depot_trie_requested; static bool __stack_depot_trie_enabled(void) { @@ -384,6 +384,11 @@ static void __stack_depot_trie_set_enabled(bool enabled) static_branch_disable(&stack_depot_trie_enabled); } +static bool __stack_depot_trie_requested(void) +{ + return READ_ONCE(stack_depot_trie_requested); +} + static int stack_depot_trie_enabled_param_set(const char *val, const struct kernel_param *kp) { @@ -398,15 +403,21 @@ static int stack_depot_trie_enabled_param_set(const char *val, /* Keep runtime toggles serialized outside stack_depot_init_mutex. */ mutex_lock(&stack_depot_trie_param_lock); - /* stack_depot_init() sees this key; save routing still waits for ready. */ - __stack_depot_trie_set_enabled(enabled); - if (enabled && system_state >= SYSTEM_RUNNING) { + WRITE_ONCE(stack_depot_trie_requested, enabled); + if (!enabled) { + __stack_depot_trie_set_enabled(false); + goto out_unlock; + } + + if (system_state >= SYSTEM_RUNNING) { ret = stack_depot_init(); if (ret || !__stack_depot_trie_ready()) { + WRITE_ONCE(stack_depot_trie_requested, false); __stack_depot_trie_set_enabled(false); ret = ret ?: -ENOMEM; } } +out_unlock: mutex_unlock(&stack_depot_trie_param_lock); return ret; } @@ -438,7 +449,6 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage"); #define STACK_BUCKET_NUMBER_ORDER_MAX 20 /* Initial seed for jhash2. */ #define STACK_HASH_SEED 0x9747b28c -#define STACK_POOLS_SORTED_RETRIES 8 /* Compact structure that stores a reference to a stack. */ union handle_parts { @@ -488,10 +498,8 @@ struct stack_depot_trie_node { /* * Children are sorted by first frame and searched with lower_bound(). - * nr_children is the published element count. It can grow in place only when - * capacity has spare room; otherwise writers build a replacement array and - * publish that pointer. Readers use acquire loads for both the array pointer - * and nr_children. + * Writers may append to spare capacity at the sorted tail, but never change + * existing child pointers. Other updates build and publish a replacement array. */ struct stack_depot_trie_child_array { unsigned int nr_children; @@ -530,10 +538,6 @@ static unsigned int stack_hash_mask; /* Array of memory regions that store stack records. */ static void **stack_pools; -/* Stack pools sorted by address for fast membership checks. */ -static void **stack_pools_sorted; -static unsigned int stack_pools_sorted_capacity; -static bool stack_pools_sorted_memblock; /* Newly allocated pool that is not yet added to stack_pools. */ static void *new_pool; /* Whether legacy hash storage may contain normal persistent records. */ @@ -674,7 +678,7 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) } struct stack_depot_trie_side_entry { - const void *leaf; + const void __rcu *leaf; }; /* @@ -682,217 +686,55 @@ struct stack_depot_trie_side_entry { * pointer for lockless fetch/print paths, which can run from diagnostic * contexts where taking trie_side_table_lock would be unsafe. Init installs the * root and first chunk only; additional directories/chunks are preallocated and - * published lazily as leaf IDs grow. Release/acquire pairs publish fully - * initialized dirs, chunks, and leaves to those lockless readers. + * published lazily as leaf IDs grow. RCU pointer publication makes fully + * initialized dirs, chunks, and leaves visible to those lockless readers. */ #define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS 9 #define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE \ (1U << STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS) struct stack_depot_trie_side_dir { - struct stack_depot_trie_side_entry *chunks[STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE]; + struct stack_depot_trie_side_entry __rcu *chunks[STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE]; +}; + +struct stack_depot_trie_side_root { + unsigned int nr_dirs; + struct stack_depot_trie_side_dir __rcu *dirs[]; }; -static struct stack_depot_trie_side_dir **trie_side_table_dirs; +static struct stack_depot_trie_side_root __rcu *trie_side_table_root; static DEFINE_RAW_SPINLOCK(trie_side_table_lock); static DEFINE_RAW_SPINLOCK(trie_alloc_lock); -static unsigned int trie_side_table_high_water; static unsigned int trie_side_table_nr_dirs; static unsigned int trie_side_table_nr_chunks; static unsigned int trie_side_table_root_size; static u32 trie_side_table_max_id; static u32 trie_side_table_next_id; -static bool trie_side_table_initialized; static bool trie_side_table_memblock; /* Lock order: workspace_lock -> trie_alloc_lock -> pool_lock -> trie_side_table_lock. */ -static bool stack_depot_trie_is_ready(void) +static struct stack_depot_trie_alloc_workspace *stack_depot_trie_load_workspace(void) { - /* Pairs with stack_depot_trie_publish_ready(); publishes all trie init. */ - return smp_load_acquire(&stack_depot_trie_ready); + /* Installed once and never freed; acquire the init-time publication. */ + return rcu_dereference_check(stack_depot_trie_workspace, true); } -static bool trie_side_table_is_initialized(void) -{ - return READ_ONCE(trie_side_table_initialized); -} - -static void stack_depot_trie_publish_ready(void) -{ - /* Pairs with stack_depot_trie_is_ready(). */ - smp_store_release(&stack_depot_trie_ready, true); -} - -static void trie_side_table_publish_initialized(void) -{ - WRITE_ONCE(trie_side_table_initialized, true); -} - -static int stack_pool_addr_cmp(const void *a, const void *b) +static struct stack_depot_trie_side_root *trie_side_table_load_root(void) { - unsigned long ap = (unsigned long)*(void * const *)a; - unsigned long bp = (unsigned long)*(void * const *)b; - - return (ap > bp) - (ap < bp); + /* Installed once and never freed; acquire the init-time publication. */ + return rcu_dereference_check(trie_side_table_root, true); } -static unsigned int stack_pools_sorted_round_capacity(unsigned int pools) -{ - unsigned int step = PAGE_SIZE / sizeof(*stack_pools_sorted); - - if (!pools) - pools = 1; - if (pools > stack_max_pools) - return 0; - return min(round_up(pools, step), stack_max_pools); -} - -static unsigned int stack_pools_sorted_lower_bound(unsigned long addr, - unsigned int pools) -{ - unsigned int left = 0; - unsigned int right = pools; - - while (left < right) { - unsigned int mid = left + (right - left) / 2; - unsigned long mid_start = (unsigned long)stack_pools_sorted[mid]; - - if (mid_start < addr) - left = mid + 1; - else - right = mid; - } - - return left; -} - -static int __init stack_depot_trie_init_sorted_pools_memblock(void) -{ - unsigned int capacity; - size_t bytes; - - if (READ_ONCE(stack_pools_sorted)) - return 0; - capacity = stack_pools_sorted_round_capacity(READ_ONCE(pools_num) + 1); - if (!capacity) - return -ENOMEM; - bytes = capacity * sizeof(*stack_pools_sorted); - stack_pools_sorted = memblock_alloc(bytes, PAGE_SIZE); - if (!stack_pools_sorted) - return -ENOMEM; - memset(stack_pools_sorted, 0, bytes); - WRITE_ONCE(stack_pools_sorted_capacity, capacity); - WRITE_ONCE(stack_pools_sorted_memblock, true); - return 0; -} - -static int stack_depot_trie_init_sorted_pools(gfp_t gfp_flags) -{ - unsigned long flags; - unsigned int capacity; - unsigned int pools; - unsigned int retry; - void **sorted; - - if (READ_ONCE(stack_pools_sorted)) - return 0; - capacity = stack_pools_sorted_round_capacity(READ_ONCE(pools_num) + 1); - if (!capacity) - return -ENOMEM; - sorted = kvcalloc(capacity, sizeof(*sorted), gfp_flags); - if (!sorted) - return -ENOMEM; - - for (retry = 0; sorted && retry < STACK_POOLS_SORTED_RETRIES; retry++) { - raw_spin_lock_irqsave(&pool_lock, flags); - if (stack_pools_sorted) { - raw_spin_unlock_irqrestore(&pool_lock, flags); - break; - } - pools = READ_ONCE(pools_num); - if (pools > capacity) { - raw_spin_unlock_irqrestore(&pool_lock, flags); - break; - } - memcpy(sorted, stack_pools, pools * sizeof(*sorted)); - raw_spin_unlock_irqrestore(&pool_lock, flags); - - sort(sorted, pools, sizeof(*sorted), stack_pool_addr_cmp, NULL); - - raw_spin_lock_irqsave(&pool_lock, flags); - if (!stack_pools_sorted && pools == READ_ONCE(pools_num)) { - WRITE_ONCE(stack_pools_sorted_capacity, capacity); - WRITE_ONCE(stack_pools_sorted_memblock, false); - WRITE_ONCE(stack_pools_sorted, sorted); - sorted = NULL; - } - raw_spin_unlock_irqrestore(&pool_lock, flags); - } - - kvfree(sorted); - return 0; -} - -static void stack_pools_sorted_grow(gfp_t gfp_flags) +static bool trie_side_table_is_initialized(void) { - unsigned int old_capacity; - unsigned int capacity; - unsigned long flags; - unsigned int pools; - unsigned int retry; - void **old; - bool old_memblock = false; - void **sorted; - - old_capacity = READ_ONCE(stack_pools_sorted_capacity); - if (old_capacity > READ_ONCE(pools_num)) - return; - capacity = stack_pools_sorted_round_capacity(READ_ONCE(pools_num) + 1); - if (capacity <= old_capacity) - return; - - sorted = kvcalloc(capacity, sizeof(*sorted), gfp_flags); - if (!sorted) - return; - - for (retry = 0; retry < STACK_POOLS_SORTED_RETRIES; retry++) { - raw_spin_lock_irqsave(&pool_lock, flags); - old = stack_pools_sorted; - if (capacity <= stack_pools_sorted_capacity) { - raw_spin_unlock_irqrestore(&pool_lock, flags); - break; - } - pools = READ_ONCE(pools_num); - memcpy(sorted, stack_pools, pools * sizeof(*sorted)); - raw_spin_unlock_irqrestore(&pool_lock, flags); - - sort(sorted, pools, sizeof(*sorted), stack_pool_addr_cmp, NULL); - - raw_spin_lock_irqsave(&pool_lock, flags); - old = stack_pools_sorted; - if (capacity > stack_pools_sorted_capacity && - pools == READ_ONCE(pools_num)) { - old_memblock = stack_pools_sorted_memblock; - WRITE_ONCE(stack_pools_sorted_capacity, capacity); - WRITE_ONCE(stack_pools_sorted_memblock, false); - WRITE_ONCE(stack_pools_sorted, sorted); - sorted = old; - raw_spin_unlock_irqrestore(&pool_lock, flags); - break; - } - raw_spin_unlock_irqrestore(&pool_lock, flags); - } - - if (!old_memblock) - kvfree(sorted); + return !!trie_side_table_load_root(); } static bool __stack_depot_trie_ready(void) { return __stack_depot_trie_enabled() && - stack_depot_trie_is_ready() && - READ_ONCE(stack_depot_trie_workspace) && + stack_depot_trie_load_workspace() && trie_side_table_is_initialized(); } @@ -919,15 +761,27 @@ static unsigned int trie_side_table_slot_index(u32 id) static struct stack_depot_trie_side_dir *trie_side_table_load_dir(unsigned int root) { + struct stack_depot_trie_side_root *root_vec; + + root_vec = trie_side_table_load_root(); + if (!root_vec || root >= root_vec->nr_dirs) + return NULL; /* Pairs with trie_side_table_publish_dir(); lookup is lockless. */ - return smp_load_acquire(&trie_side_table_dirs[root]); + return rcu_dereference_check(root_vec->dirs[root], + lockdep_is_held(&trie_side_table_lock) || + rcu_read_lock_sched_held()); } static void trie_side_table_publish_dir(unsigned int root, struct stack_depot_trie_side_dir *dir) { + struct stack_depot_trie_side_root *root_vec; + + root_vec = trie_side_table_load_root(); + if (!root_vec || root >= root_vec->nr_dirs) + return; /* Publish the zeroed directory before readers can load it locklessly. */ - smp_store_release(&trie_side_table_dirs[root], dir); + rcu_assign_pointer(root_vec->dirs[root], dir); } static struct stack_depot_trie_side_entry * @@ -935,7 +789,9 @@ trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, unsigned int idx) { /* Pairs with trie_side_table_dir_publish_chunk(); lookup is lockless. */ - return smp_load_acquire(&dir->chunks[idx]); + return rcu_dereference_check(dir->chunks[idx], + lockdep_is_held(&trie_side_table_lock) || + rcu_read_lock_sched_held()); } static void @@ -944,7 +800,7 @@ trie_side_table_dir_publish_chunk(struct stack_depot_trie_side_dir *dir, struct stack_depot_trie_side_entry *chunk) { /* Pairs with trie_side_table_dir_load_chunk(). */ - smp_store_release(&dir->chunks[idx], chunk); + rcu_assign_pointer(dir->chunks[idx], chunk); } static u32 @@ -960,6 +816,8 @@ trie_side_table_alloc_id_locked(struct stack_depot_trie_side_prealloc *prealloc) /* Failed or disabled trie init means no leaf IDs can be allocated. */ if (!trie_side_table_is_initialized()) return 0; + if (!prealloc) + return 0; id = trie_side_table_next_id + 1; /* ID zero wraps the 32-bit counter; max_id is handle namespace capacity. */ @@ -973,22 +831,20 @@ trie_side_table_alloc_id_locked(struct stack_depot_trie_side_prealloc *prealloc) dir = trie_side_table_load_dir(root); if (!dir) { - /* Sparse growth needs a preallocated dir before taking this lock. */ - if (!prealloc || !prealloc->dir) + /* Sparse growth preallocation can lose a race to another writer. */ + if (!prealloc->dir) return 0; dir = prealloc->dir; prealloc->dir = NULL; trie_side_table_publish_dir(root, dir); trie_side_table_nr_dirs++; - if (trie_side_table_high_water < root + 1) - trie_side_table_high_water = root + 1; } idx = trie_side_table_dir_index(id); chunk = trie_side_table_dir_load_chunk(dir, idx); if (!chunk) { - /* Sparse growth needs a preallocated chunk before taking this lock. */ - if (!prealloc || !prealloc->chunk) + /* Sparse growth preallocation can lose a race to another writer. */ + if (!prealloc->chunk) return 0; chunk = prealloc->chunk; prealloc->chunk = NULL; @@ -1004,8 +860,8 @@ static size_t trie_side_table_root_bytes(unsigned int root_size) { size_t bytes; - if (check_mul_overflow((size_t)root_size, - sizeof(*trie_side_table_dirs), &bytes)) + bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); + if (bytes == SIZE_MAX) return 0; return PAGE_ALIGN(bytes); } @@ -1044,7 +900,7 @@ static void trie_side_table_free_dir(struct stack_depot_trie_side_dir *dir) } static int -trie_side_table_install(struct stack_depot_trie_side_dir **dirs, +trie_side_table_install(struct stack_depot_trie_side_root *root_vec, unsigned int root_size, u32 max_id, struct stack_depot_trie_side_dir *first_dir, struct stack_depot_trie_side_entry *first_chunk, @@ -1058,27 +914,25 @@ trie_side_table_install(struct stack_depot_trie_side_dir **dirs, */ if (trie_side_table_is_initialized()) return 0; - if (!dirs || !root_size || !max_id) + if (!root_vec || !root_size || !max_id) return -EINVAL; - WRITE_ONCE(trie_side_table_dirs, dirs); WRITE_ONCE(trie_side_table_root_size, root_size); - WRITE_ONCE(trie_side_table_high_water, 0); + root_vec->nr_dirs = root_size; WRITE_ONCE(trie_side_table_nr_dirs, 0); WRITE_ONCE(trie_side_table_nr_chunks, 0); WRITE_ONCE(trie_side_table_max_id, max_id); WRITE_ONCE(trie_side_table_next_id, 0); WRITE_ONCE(trie_side_table_memblock, memblock); if (first_dir) { - trie_side_table_publish_dir(0, first_dir); - WRITE_ONCE(trie_side_table_high_water, 1); + RCU_INIT_POINTER(root_vec->dirs[0], first_dir); WRITE_ONCE(trie_side_table_nr_dirs, 1); } if (first_dir && first_chunk) { - trie_side_table_dir_publish_chunk(first_dir, 0, first_chunk); + RCU_INIT_POINTER(first_dir->chunks[0], first_chunk); WRITE_ONCE(trie_side_table_nr_chunks, 1); } - trie_side_table_publish_initialized(); + rcu_assign_pointer(trie_side_table_root, root_vec); return 0; } @@ -1092,7 +946,7 @@ static unsigned int trie_side_table_root_size_for_max_id(u32 max_leaf_id) static int __init __stack_depot_trie_side_table_init_memblock(void) { - struct stack_depot_trie_side_dir **dirs; + struct stack_depot_trie_side_root *root_vec; struct stack_depot_trie_side_dir *first_dir; struct stack_depot_trie_side_entry *first_chunk; size_t dir_bytes; @@ -1114,42 +968,37 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) if (!root_bytes || !dir_bytes || !chunk_bytes) return -ENOMEM; - dirs = memblock_alloc(root_bytes, PAGE_SIZE); - if (!dirs) + root_vec = memblock_alloc(root_bytes, PAGE_SIZE); + if (!root_vec) return -ENOMEM; - memset(dirs, 0, root_bytes); + memset(root_vec, 0, root_bytes); first_dir = memblock_alloc(dir_bytes, PAGE_SIZE); if (!first_dir) { - memblock_free(dirs, root_bytes); + memblock_free(root_vec, root_bytes); return -ENOMEM; } memset(first_dir, 0, dir_bytes); first_chunk = memblock_alloc(chunk_bytes, PAGE_SIZE); if (!first_chunk) { memblock_free(first_dir, dir_bytes); - memblock_free(dirs, root_bytes); + memblock_free(root_vec, root_bytes); return -ENOMEM; } memset(first_chunk, 0, chunk_bytes); - return trie_side_table_install(dirs, root_size, max_leaf_id, first_dir, + return trie_side_table_install(root_vec, root_size, max_leaf_id, first_dir, first_chunk, true); } -static size_t stack_depot_trie_workspace_size(void) -{ - return sizeof(*stack_depot_trie_workspace); -} - static int stack_depot_trie_install_workspace(struct stack_depot_trie_alloc_workspace *workspace) { - if (READ_ONCE(stack_depot_trie_workspace)) + if (stack_depot_trie_load_workspace()) return 0; if (!workspace) return -EINVAL; - WRITE_ONCE(stack_depot_trie_workspace, workspace); + rcu_assign_pointer(stack_depot_trie_workspace, workspace); return 0; } @@ -1158,10 +1007,10 @@ static int __init stack_depot_trie_init_workspace_memblock(void) struct stack_depot_trie_alloc_workspace *workspace; size_t size; - if (READ_ONCE(stack_depot_trie_workspace)) + if (stack_depot_trie_load_workspace()) return 0; - size = stack_depot_trie_workspace_size(); + size = sizeof(*stack_depot_trie_workspace); workspace = memblock_alloc(size, __alignof__(*workspace)); if (!workspace) return -ENOMEM; @@ -1174,10 +1023,10 @@ static int stack_depot_trie_init_workspace(gfp_t gfp_flags) { struct stack_depot_trie_alloc_workspace *workspace; - if (READ_ONCE(stack_depot_trie_workspace)) + if (stack_depot_trie_load_workspace()) return 0; - workspace = kvzalloc(stack_depot_trie_workspace_size(), gfp_flags); + workspace = kvzalloc(sizeof(*stack_depot_trie_workspace), gfp_flags); if (!workspace) return -ENOMEM; @@ -1188,21 +1037,17 @@ static int __init stack_depot_trie_init_memblock(void) { int ret; - if (!__stack_depot_trie_enabled()) + if (!__stack_depot_trie_requested()) return 0; ret = stack_depot_trie_init_workspace_memblock(); - if (ret) - return ret; - /* Memblock allocations are permanent; keep successful pieces reusable. */ - ret = stack_depot_trie_init_sorted_pools_memblock(); if (ret) return ret; ret = __stack_depot_trie_side_table_init_memblock(); if (ret) return ret; - stack_depot_trie_publish_ready(); + __stack_depot_trie_set_enabled(true); return 0; } @@ -1210,13 +1055,10 @@ static int stack_depot_trie_init(gfp_t gfp_flags) { int ret; - if (!__stack_depot_trie_enabled()) + if (!__stack_depot_trie_requested()) return 0; ret = stack_depot_trie_init_workspace(gfp_flags); - if (ret) - return ret; - ret = stack_depot_trie_init_sorted_pools(gfp_flags); if (ret) return ret; ret = __stack_depot_trie_side_table_init(gfp_flags); @@ -1227,7 +1069,7 @@ static int stack_depot_trie_init(gfp_t gfp_flags) if (system_state >= SYSTEM_RUNNING) WRITE_ONCE(stack_depot_persistent_hash_record_seen, true); - stack_depot_trie_publish_ready(); + __stack_depot_trie_set_enabled(true); return 0; } @@ -1236,7 +1078,9 @@ trie_side_table_load_leaf(struct stack_depot_trie_side_entry *chunk, unsigned int slot) { /* Pairs with trie_side_table_store_leaf(). */ - return smp_load_acquire(&chunk[slot].leaf); + return rcu_dereference_check(chunk[slot].leaf, + lockdep_is_held(&trie_side_table_lock) || + rcu_read_lock_sched_held()); } static void @@ -1244,7 +1088,7 @@ trie_side_table_store_leaf(struct stack_depot_trie_side_entry *chunk, unsigned int slot, const void *leaf) { /* Pairs with trie_side_table_load_leaf(). */ - smp_store_release(&chunk[slot].leaf, leaf); + rcu_assign_pointer(chunk[slot].leaf, leaf); } static void @@ -1256,7 +1100,7 @@ trie_side_table_clear_entry(struct stack_depot_trie_side_entry *chunk, static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) { - struct stack_depot_trie_side_dir **dirs; + struct stack_depot_trie_side_root *root_vec; unsigned int root_size; size_t root_bytes; u32 max_leaf_id; @@ -1272,11 +1116,11 @@ static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) root_bytes = trie_side_table_root_bytes(root_size); if (!root_bytes) return -ENOMEM; - dirs = kvcalloc(root_size, sizeof(*dirs), gfp_flags); - if (!dirs) + root_vec = kvzalloc(root_bytes, gfp_flags); + if (!root_vec) return -ENOMEM; - return trie_side_table_install(dirs, root_size, max_leaf_id, NULL, NULL, + return trie_side_table_install(root_vec, root_size, max_leaf_id, NULL, NULL, false); } @@ -1652,8 +1496,6 @@ static void depot_record_pool_locked(void *pool) { unsigned long start = (unsigned long)pool; unsigned long end; - unsigned int pools = READ_ONCE(pools_num); - unsigned int pos; lockdep_assert_held(&pool_lock); if (!pool || check_add_overflow(start, DEPOT_POOL_SIZE, &end)) @@ -1663,30 +1505,6 @@ static void depot_record_pool_locked(void *pool) pools_min_addr = start; if (end > pools_max_addr) pools_max_addr = end; - - if (!stack_pools_sorted || READ_ONCE(stack_pools_sorted_capacity) <= pools) - return; - pos = stack_pools_sorted_lower_bound(start, pools); - memmove(&stack_pools_sorted[pos + 1], &stack_pools_sorted[pos], - (pools - pos) * sizeof(*stack_pools_sorted)); - stack_pools_sorted[pos] = pool; -} - -static void depot_forget_pool_locked(void *pool) -{ - unsigned int pools = READ_ONCE(pools_num); - unsigned int i; - - lockdep_assert_held(&pool_lock); - if (!pool || !stack_pools_sorted || READ_ONCE(stack_pools_sorted_capacity) < pools) - return; - - i = stack_pools_sorted_lower_bound((unsigned long)pool, pools); - if (i >= pools || stack_pools_sorted[i] != pool) - return; - memmove(&stack_pools_sorted[i], &stack_pools_sorted[i + 1], - (pools - i - 1) * sizeof(*stack_pools_sorted)); - stack_pools_sorted[pools - 1] = NULL; } static bool trie_pool_range_contains_locked(const void *ptr, size_t size) @@ -1694,40 +1512,28 @@ static bool trie_pool_range_contains_locked(const void *ptr, size_t size) unsigned long start = (unsigned long)ptr; unsigned long end; unsigned int pools = READ_ONCE(pools_num); - unsigned int left = 0; - unsigned int right = pools; - unsigned int pos; unsigned long pool_start; + unsigned int i; lockdep_assert_held(&pool_lock); /* Reject stale/non-pool storage before putting COW-retired bytes on freelists. */ if (!ptr || !size || check_add_overflow(start, size, &end)) return false; - if (!stack_pools_sorted || READ_ONCE(stack_pools_sorted_capacity) < pools) { - unsigned int i; - - if (!stack_pools) - return false; - for (i = 0; i < pools; i++) { - if (!stack_pools[i]) - continue; - pool_start = (unsigned long)stack_pools[i]; - if (start >= pool_start && end <= pool_start + DEPOT_POOL_SIZE) - return true; - } + if (!stack_pools) return false; - } if (pools_min_addr && (start < pools_min_addr || end > pools_max_addr)) return false; - left = stack_pools_sorted_lower_bound(start + 1, right); - if (!left) - return false; + for (i = 0; i < pools; i++) { + if (!stack_pools[i]) + continue; + pool_start = (unsigned long)stack_pools[i]; + if (start >= pool_start && end <= pool_start + DEPOT_POOL_SIZE) + return true; + } - pos = left - 1; - pool_start = (unsigned long)stack_pools_sorted[pos]; - return end <= pool_start + DEPOT_POOL_SIZE; + return false; } static bool trie_pool_contains_locked(const void *ptr) @@ -1970,7 +1776,7 @@ static void *trie_pop_free_object(size_t size) return trie_object_payload(free); } -void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags) +static void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags) { struct page *page; @@ -2011,8 +1817,6 @@ __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && gfpflags_allow_spinning(alloc_flags); - if (can_alloc) - stack_pools_sorted_grow(alloc_flags); needs_side_prealloc = __stack_depot_trie_side_table_prealloc_needed(); if (can_alloc && !READ_ONCE(new_pool)) *pool_prealloc = __stack_depot_trie_pool_prealloc(alloc_flags); @@ -2050,7 +1854,6 @@ static bool stack_depot_trie_pool_rollback_locked(const struct stack_depot_trie_ return false; if (new_pool && new_pool != STACK_DEPOT_POISON) return false; - depot_forget_pool_locked(mark->pool); stack_pools[mark->pool_index] = NULL; WRITE_ONCE(pools_num, mark->pool_index); pool_offset = mark->prev_offset; @@ -2831,8 +2634,9 @@ int __init stack_depot_early_init(void) stack_depot_disabled = true; return -ENOMEM; } - if (__stack_depot_trie_enabled() && stack_depot_trie_init_memblock()) { + if (__stack_depot_trie_requested() && stack_depot_trie_init_memblock()) { pr_warn("trie storage initialization failed, disabling trie storage\n"); + WRITE_ONCE(stack_depot_trie_requested, false); __stack_depot_trie_set_enabled(false); } @@ -2842,7 +2646,6 @@ int __init stack_depot_early_init(void) /* Allocates a hash table via kvcalloc. Can be used after boot. */ int stack_depot_init(void) { - static DEFINE_MUTEX(stack_depot_init_mutex); unsigned long entries; int ret = 0; @@ -2900,10 +2703,11 @@ int stack_depot_init(void) goto out_unlock; } init_trie: - if (!ret && __stack_depot_trie_enabled()) { + if (!ret && __stack_depot_trie_requested()) { ret = stack_depot_trie_init(GFP_KERNEL); if (ret) { pr_warn("trie storage initialization failed, disabling trie storage\n"); + WRITE_ONCE(stack_depot_trie_requested, false); __stack_depot_trie_set_enabled(false); ret = 0; } @@ -3280,7 +3084,7 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, { return __stack_depot_trie_save_locked(&stack_depot_trie_root, entries, nr_entries, alloc_flags, depot_flags, - stack_depot_trie_workspace, + stack_depot_trie_load_workspace(), &stack_depot_trie_workspace_lock); } @@ -3583,11 +3387,7 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, } new = old - (int)count; - } while (!atomic_try_cmpxchg_release(&stack->count.refs, &old, new)); - - /* Non-zero results are diagnostic counts; callers consume no ordered data. */ - if (!new) - smp_acquire__after_ctrl_dep(); + } while (!atomic_try_cmpxchg(&stack->count.refs, &old, new)); return !new; } @@ -4090,16 +3890,22 @@ trie_child_array_subtree_overlaps(const struct stack_depot_trie_child_array *arr static const struct stack_depot_trie_node * trie_load_parent(const struct stack_depot_trie_node *node) { - /* Pairs with trie_publish_parent(). */ - return smp_load_acquire(&node->parent); + const struct stack_depot_trie_node __rcu * const *slot; + + slot = (const struct stack_depot_trie_node __rcu * const *)&node->parent; + return rcu_dereference_check(*slot, + lockdep_is_held(&stack_depot_trie_workspace_lock) || + rcu_read_lock_sched_held()); } static void trie_publish_parent(struct stack_depot_trie_node *child, const struct stack_depot_trie_node *parent) { - /* Pairs with trie_load_parent(). */ - smp_store_release(&child->parent, parent); + const struct stack_depot_trie_node __rcu **slot; + + slot = (const struct stack_depot_trie_node __rcu **)&child->parent; + rcu_assign_pointer(*slot, parent); } static bool @@ -4175,11 +3981,60 @@ trie_publish_slot(struct stack_depot_trie_root *root, return &parent->children; } +static const struct stack_depot_trie_child_array * +trie_load_children_slot(const struct stack_depot_trie_child_array * const *slot) +{ + const struct stack_depot_trie_child_array __rcu * const *rcu_slot; + + rcu_slot = (const struct stack_depot_trie_child_array __rcu * const *)slot; + return rcu_dereference_check(*rcu_slot, + lockdep_is_held(&stack_depot_trie_workspace_lock) || + rcu_read_lock_sched_held()); +} + +static const struct stack_depot_trie_node * +trie_child_array_load_child(const struct stack_depot_trie_child_array *array, + unsigned int pos) +{ + const struct stack_depot_trie_node __rcu * const *slot; + + slot = (const struct stack_depot_trie_node __rcu * const *)&array->children[pos]; + return rcu_dereference_check(*slot, + lockdep_is_held(&stack_depot_trie_workspace_lock) || + rcu_read_lock_sched_held()); +} + +static void +trie_child_array_publish_child(struct stack_depot_trie_child_array *array, + unsigned int pos, + const struct stack_depot_trie_node *child) +{ + const struct stack_depot_trie_node __rcu **slot; + + slot = (const struct stack_depot_trie_node __rcu **)&array->children[pos]; + rcu_assign_pointer(*slot, child); +} + +static void +trie_publish_children_slot(const struct stack_depot_trie_child_array **slot, + const struct stack_depot_trie_child_array *children) +{ + const struct stack_depot_trie_child_array __rcu **rcu_slot; + + rcu_slot = (const struct stack_depot_trie_child_array __rcu **)slot; + rcu_assign_pointer(*rcu_slot, children); +} + static bool trie_child_array_can_append(const struct stack_depot_trie_child_array *array, unsigned int pos) { - return array && pos == array->nr_children && array->nr_children < array->capacity; + unsigned int nr_children; + + if (!array) + return false; + nr_children = READ_ONCE(array->nr_children); + return pos == nr_children && nr_children < array->capacity; } static int trie_insert_append_precheck(struct stack_depot_trie_root *root, @@ -4234,8 +4089,7 @@ static int trie_insert_append_precheck(struct stack_depot_trie_root *root, new_storage_size))) return -EINVAL; - /* Pairs with append publication's smp_store_release(). */ - children = smp_load_acquire(slot); + children = trie_load_children_slot(slot); if (!new_storage) { if (!children) return -EINVAL; @@ -4247,7 +4101,7 @@ static int trie_insert_append_precheck(struct stack_depot_trie_root *root, return 0; } size = __stack_depot_trie_child_array_size(children ? - children->nr_children + 1 : 1); + READ_ONCE(children->nr_children) + 1 : 1); if (!size || new_storage_size < size) return -EINVAL; if (children) { @@ -4305,8 +4159,7 @@ static int trie_insert_descend_precheck(struct stack_depot_trie_root *root, sizeof(*slot))) return -EINVAL; - /* Pairs with append publication's smp_store_release(). */ - children = smp_load_acquire(slot); + children = trie_load_children_slot(slot); if (!children) return -EINVAL; size = trie_child_array_size_for_capacity(children->capacity); @@ -4465,8 +4318,7 @@ trie_promote_precheck(struct stack_depot_trie_root *root, new_storage_size)) return -EINVAL; - /* Pairs with append and promote publication's smp_store_release(). */ - *old_array = smp_load_acquire(publish_slot); + *old_array = trie_load_children_slot(publish_slot); if (!*old_array) return -EINVAL; array = *old_array; @@ -4525,7 +4377,7 @@ trie_promote_child(struct stack_depot_trie_root *root, publish_slot = trie_publish_slot(root, parent); /* Publish the fully initialized replacement array last. */ - smp_store_release(publish_slot, new_storage); + trie_publish_children_slot(publish_slot, new_storage); if (prepare && prepare->retire_locked) trie_retire_object_node_locked(old_array, child, child_size); else @@ -4690,10 +4542,8 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array *old_array; const struct stack_depot_trie_node *head = head_ptr; const struct stack_depot_trie_child_array **slot; - struct stack_depot_trie_child_array *append_array = NULL; struct stack_depot_trie_node *parent = parent_ptr; struct stack_depot_trie_child_array *new_array = new_storage; - unsigned int append_pos = 0; size_t storage_size = new_storage_size; size_t new_size; size_t old_size; @@ -4718,11 +4568,10 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, slot = &parent->children; } - /* Pairs with append publication's smp_store_release(). */ - old_array = smp_load_acquire(slot); + old_array = trie_load_children_slot(slot); old_size = old_array ? trie_child_array_size_for_capacity(old_array->capacity) : 0; - new_size = old_array ? old_array->nr_children + 1 : 1; + new_size = old_array ? READ_ONCE(old_array->nr_children) + 1 : 1; new_size = __stack_depot_trie_child_array_size(new_size); if (!new_size) return -EINVAL; @@ -4739,19 +4588,35 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, return -EINVAL; if (found || !trie_child_array_can_append(old_array, pos)) return -EINVAL; - append_array = (struct stack_depot_trie_child_array *)old_array; - append_pos = pos; + if (prepare) { + struct stack_depot_trie_leaf_update update = { + .leaf_id = leaf_id, + .leaf = leaf, + }; + + if (!prepare->fn) + return -EINVAL; + if (!leaf_id || !leaf) + return -EINVAL; + ret = prepare->fn(&update, 1, prepare->ctx); + if (ret) + return ret; + } + trie_child_array_publish_child((struct stack_depot_trie_child_array *)old_array, + pos, head); + WRITE_ONCE(((struct stack_depot_trie_child_array *)old_array)->nr_children, + pos + 1); + return 0; } - if (new_array && storage_size < new_size) + if (storage_size < new_size) return -EINVAL; - if (new_array && old_array && + if (old_array && stack_depot_ranges_overlap(old_array, old_size, new_array, storage_size)) return -EINVAL; - if (new_array && trie_chain_overlaps(head, new_array, storage_size)) + if (trie_chain_overlaps(head, new_array, storage_size)) return -EINVAL; - if (new_array && - __stack_depot_trie_child_array_insert(old_array, head, new_array, storage_size)) + if (__stack_depot_trie_child_array_insert(old_array, head, new_array, storage_size)) return -EINVAL; if (prepare) { struct stack_depot_trie_leaf_update update = { @@ -4767,15 +4632,8 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, if (ret) return ret; } - if (!new_array) { - append_array->children[append_pos] = head; - /* Pairs with child lookup's smp_load_acquire(). */ - smp_store_release(&append_array->nr_children, append_pos + 1); - return 0; - } - /* Publish the fully initialized replacement array last. */ - smp_store_release(slot, new_array); + trie_publish_children_slot(slot, new_array); if (prepare && prepare->retire_locked) trie_retire_object_node_locked(old_array, NULL, 0); else @@ -4803,13 +4661,10 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, if ((root && parent) || (!root && !parent)) return -EINVAL; - if (root) { - /* Pairs with append publication's smp_store_release(). */ - children = smp_load_acquire(&root->children); - } else { - /* Pairs with append publication's smp_store_release(). */ - children = smp_load_acquire(&parent->children); - } + if (root) + children = trie_load_children_slot(&root->children); + else + children = trie_load_children_slot(&parent->children); tmp.status = STACK_DEPOT_TRIE_LOOKUP_APPEND; tmp.parent = parent; @@ -5245,6 +5100,7 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, const struct stack_depot_trie_node *parent = parent_ptr; const struct stack_depot_trie_child_array *children; const struct stack_depot_trie_node *child; + unsigned int nr_children; unsigned int matched; unsigned int pos; bool found; @@ -5258,13 +5114,10 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, for (;;) { if (parent && trie_node_chain_depth_invalid(parent)) return -EINVAL; - if (root) { - /* Pairs with append, promote, and split publication. */ - children = smp_load_acquire(&root->children); - } else { - /* Pairs with append, promote, and split publication. */ - children = smp_load_acquire(&parent->children); - } + if (root) + children = trie_load_children_slot(&root->children); + else + children = trie_load_children_slot(&parent->children); if (!children) { if (trie_plan_append_chain(parent ? parent->stack_len : 0, entries, nr_entries, node_slots, @@ -5289,8 +5142,8 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, *new_storage_size = 0; return 0; } - *new_storage_size = - __stack_depot_trie_child_array_size(children->nr_children + 1); + nr_children = READ_ONCE(children->nr_children); + *new_storage_size = __stack_depot_trie_child_array_size(nr_children + 1); return *new_storage_size ? 0 : -EINVAL; } @@ -5432,7 +5285,7 @@ static unsigned int trie_handle_leaf(depot_stack_handle_t handle, if (!leaf_id) return 0; *leaf = __stack_depot_trie_side_table_lookup(leaf_id); - if (WARN(!*leaf, "corrupt trie handle %08x\n", handle)) + if (WARN_ONCE(!*leaf, "corrupt trie handle %08x\n", handle)) return 0; return trie_validate_leaf(*leaf, NULL); } @@ -5447,7 +5300,7 @@ static void trie_print_frames(const void *leaf, unsigned int nr_entries, if (trie_frame_at(leaf, i, &frame)) return; - pr_info("%*c%pS\n", 1 + spaces, ' ', (void *)frame); + stack_trace_print(&frame, 1, spaces); } } @@ -5564,7 +5417,7 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, rcu_read_lock_sched_notrace(); leaf = __stack_depot_trie_side_table_lookup(leaf_id); - if (WARN(!leaf, "corrupt trie handle %08x\n", handle)) { + if (WARN_ONCE(!leaf, "corrupt trie handle %08x\n", handle)) { rcu_read_unlock_sched_notrace(); return 0; } @@ -5592,13 +5445,10 @@ static unsigned int trie_child_array_storage_capacity(size_t storage_size) static size_t trie_child_array_size_for_capacity(unsigned int capacity) { size_t size; - size_t bytes; - if (check_mul_overflow((size_t)capacity, - sizeof(struct stack_depot_trie_node *), &bytes)) - return 0; - size = sizeof(struct stack_depot_trie_child_array); - if (check_add_overflow(size, bytes, &size)) + size = struct_size_t(struct stack_depot_trie_child_array, children, + capacity); + if (size == SIZE_MAX) return 0; return ALIGN(size, sizeof(unsigned long)); @@ -5829,7 +5679,7 @@ __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, return -EINVAL; /* Pairs with append, promote, and future split publication. */ - children = smp_load_acquire(slot); + children = trie_load_children_slot(slot); if (!children) return -EINVAL; size = trie_child_array_size_for_capacity(children->capacity); @@ -6108,8 +5958,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, if (!publish_slot) return -EINVAL; - /* Pairs with append, promote, and split publication. */ - old_array = smp_load_acquire(publish_slot); + old_array = trie_load_children_slot(publish_slot); ret = trie_child_array_replace_precheck(old_array, child, storage, storage_size, &pos); if (ret) return ret; @@ -6128,7 +5977,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, trie_child_array_replace_at(old_array, prefix, new_storage, new_storage_size, pos); /* Publish the fully initialized replacement array last. */ - smp_store_release(publish_slot, new_storage); + trie_publish_children_slot(publish_slot, new_storage); if (prepare && prepare->retire_locked) trie_retire_object_node_locked(old_array, child, child_size); else @@ -6149,13 +5998,19 @@ stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *ar *pos = 0; *found = false; - /* Pairs with in-place append publication's smp_store_release(). */ - right = smp_load_acquire(&array->nr_children); + right = READ_ONCE(array->nr_children); while (left < right) { unsigned int mid = left + (right - left) / 2; + const struct stack_depot_trie_node *node; unsigned long mid_frame; - if (stack_depot_trie_node_first_frame(array->children[mid], &mid_frame)) + node = trie_child_array_load_child(array, mid); + if (!node) { + /* A tail append may publish nr_children before the child is visible. */ + right = mid; + continue; + } + if (stack_depot_trie_node_first_frame(node, &mid_frame)) return -EINVAL; if (mid_frame < frame) { left = mid + 1; From d98d67e7651e226094ea0d1fdd7faa344d2792bb Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Mon, 22 Jun 2026 20:20:38 +0100 Subject: [PATCH 116/129] KRN-1117: Reserve stackdepot trie IDs before pool storage Reserve the trie side-table ID before carving stackdepot pool storage so an ID reservation failure cannot leave a carved pool range that later rollback cannot rewind. This keeps the trie insertion transaction ordered around the operation that can still fail due to sparse side-table growth. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index a22fd865dfeb7..711a060b4b3d4 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2142,13 +2142,12 @@ static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_re pool_req.prealloc = req->pool_prealloc; pool_req.mark = &req->txn->pool; - ret = __stack_depot_trie_pool_carve(&pool_req); + ret = __stack_depot_trie_alloc_txn_id(req->txn, req->side_prealloc); if (ret) return ret; - ret = __stack_depot_trie_alloc_txn_id(req->txn, req->side_prealloc); + ret = __stack_depot_trie_pool_carve(&pool_req); if (ret) { - trie_alloc_request_release_reused_objects(req); __stack_depot_trie_alloc_txn_rollback(req->txn); trie_alloc_request_clear_outputs(req); return ret; From 722792d7cd3b8fbee42104869c179b891412a6fe Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 23 Jun 2026 09:47:54 +0100 Subject: [PATCH 117/129] KRN-1117: Clear unused stackdepot trie child slots Clear spare child-array slots when building replacement arrays so a tail-only in-place append cannot expose stale child pointers if readers observe the extended count before the new child pointer. This preserves the pool-saving append path while keeping transient lookups fail-closed. Remove the now-unused trie fetch context counter while touching the trie walk path. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 711a060b4b3d4..315c916f0b500 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -4234,6 +4234,8 @@ trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array for (i = 0; i < old_array->nr_children; i++) new_array->children[i] = old_array->children[i]; new_array->children[pos] = new_child; + for (i = old_array->nr_children; i < new_array->capacity; i++) + new_array->children[i] = NULL; } static int @@ -5181,11 +5183,6 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, } } -struct stack_depot_trie_fetch_ctx { - unsigned long *entries; - unsigned int nr_entries; -}; - static unsigned int trie_validate_leaf(const void *leaf, const unsigned long *entries) { @@ -5368,17 +5365,15 @@ trie_snprint_handle(depot_stack_handle_t handle, char *buf, size_t size, static void trie_fetch_frame(unsigned int index, unsigned long frame, void *data) { - struct stack_depot_trie_fetch_ctx *ctx = data; + unsigned long *entries = data; - ctx->entries[index] = frame; - ctx->nr_entries++; + entries[index] = frame; } static unsigned int __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, unsigned int max_entries) { - struct stack_depot_trie_fetch_ctx ctx; unsigned int total; if (!entries) @@ -5389,9 +5384,7 @@ __stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, if (max_entries < total) return 0; - ctx.entries = entries; - ctx.nr_entries = 0; - if (trie_walk_frames(leaf, total, trie_fetch_frame, &ctx) != total) + if (trie_walk_frames(leaf, total, trie_fetch_frame, entries) != total) return 0; kmsan_unpoison_memory(entries, total * sizeof(*entries)); @@ -5503,6 +5496,8 @@ __stack_depot_trie_child_array_init(void *storage, size_t storage_size, array->capacity = capacity; for (i = 0; i < nr_children; i++) array->children[i] = nodes[i]; + for (i = nr_children; i < capacity; i++) + array->children[i] = NULL; return 0; } @@ -6089,6 +6084,8 @@ __stack_depot_trie_child_array_insert(const void *old_storage, const void *child for (i = pos; i < nr_old; i++) new_array->children[i + 1] = old_array->children[i]; } + for (i = nr_old + 1; i < new_array->capacity; i++) + new_array->children[i] = NULL; return 0; } From 3b2ea44f091e39b1d4292a18b11df324f2d3a7fa Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 23 Jun 2026 14:03:03 +0100 Subject: [PATCH 118/129] KRN-1117: Simplify stackdepot trie helpers Remove helper layers that no longer carry distinct behavior after the trie publication cleanup. Use compile-time handle masks directly, fold the workspace planning wrapper into its only caller, pass side-table prepare state directly, and let pool carving use the existing allocation request instead of a mirrored request object. Reject trie handles in the count helpers so counted-stack operations fail closed at the backend boundary. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 238 +++++++++++++++++------------------------------ 1 file changed, 87 insertions(+), 151 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 315c916f0b500..e0bb29978db42 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -108,15 +108,6 @@ struct stack_depot_trie_leaf_update { const void *leaf; }; -struct stack_depot_trie_publish_prepare { - int (*fn)(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, void *ctx); - /* Caller-owned state passed to fn. */ - void *ctx; - /* Use when the publish path already holds pool_lock for retirement. */ - bool retire_locked; -}; - /* A split can repoint the old leaf and publish one new leaf. */ #define STACK_DEPOT_TRIE_MAX_LEAF_UPDATES 2 #define STACK_DEPOT_TRIE_MAX_NODE_SLOTS (CONFIG_STACKDEPOT_MAX_FRAMES + 1) @@ -149,19 +140,6 @@ struct stack_depot_trie_pool_mark { bool added_pool; }; -struct stack_depot_trie_pool_request { - struct stack_depot_trie_node_slot *node_slots; - struct stack_depot_trie_child_array_slot *child_slots; - /* Optional opaque object storage reserved with the node/child slots. */ - void **storage; - /* Optional fresh stackdepot pool page, preallocated outside pool_lock. */ - void **prealloc; - struct stack_depot_trie_pool_mark *mark; - size_t storage_size; - unsigned int nr_node_slots; - unsigned int nr_child_slots; -}; - struct stack_depot_trie_alloc_txn { struct stack_depot_trie_side_prepare side; struct stack_depot_trie_pool_mark pool; @@ -219,7 +197,7 @@ static int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc); -static int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req); +static int __stack_depot_trie_pool_carve(struct stack_depot_trie_alloc_request *req); static void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); static int __stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, @@ -260,7 +238,8 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, static void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); static int __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, void *ctx); + unsigned int nr_updates, + struct stack_depot_trie_side_prepare *state); static void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *state); static int __stack_depot_frame_run_init(const unsigned long *entries, unsigned int nr_entries, @@ -306,7 +285,7 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, - const struct stack_depot_trie_publish_prepare *prepare, + struct stack_depot_trie_side_prepare *side, const void **tail, unsigned int *nr_used); static int __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, @@ -449,6 +428,8 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage"); #define STACK_BUCKET_NUMBER_ORDER_MAX 20 /* Initial seed for jhash2. */ #define STACK_HASH_SEED 0x9747b28c +#define DEPOT_POOL_INDEX_MASK ((1U << DEPOT_POOL_INDEX_BITS) - 1) +#define DEPOT_OFFSET_MASK ((1U << DEPOT_OFFSET_BITS) - 1) /* Compact structure that stores a reference to a stack. */ union handle_parts { @@ -601,20 +582,10 @@ static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); static bool depot_init_pool(void **prealloc); static void depot_try_keep_new_pool(void **prealloc); -static u32 stack_depot_pool_index_mask(void) -{ - return (1U << DEPOT_POOL_INDEX_BITS) - 1; -} - -static u32 stack_depot_offset_mask(void) -{ - return (1U << DEPOT_OFFSET_BITS) - 1; -} - static bool stack_depot_trie_namespace_available(void) { /* Reserve the all-ones pool index as an invalid trie namespace sentinel. */ - return stack_max_pools < stack_depot_pool_index_mask() - 1; + return stack_max_pools < DEPOT_POOL_INDEX_MASK - 1; } /* @@ -629,7 +600,7 @@ static u32 __stack_depot_trie_max_leaf_id(void) if (!stack_depot_trie_namespace_available()) return 0; - return (stack_depot_pool_index_mask() - stack_max_pools - 1) << + return (DEPOT_POOL_INDEX_MASK - stack_max_pools - 1) << DEPOT_OFFSET_BITS; } @@ -648,11 +619,11 @@ static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) index = leaf_id - 1; pool_delta = index >> DEPOT_OFFSET_BITS; pool_index_plus_1 = (u64)stack_max_pools + 1 + pool_delta; - if (pool_index_plus_1 >= stack_depot_pool_index_mask()) + if (pool_index_plus_1 >= DEPOT_POOL_INDEX_MASK) return 0; parts.pool_index_plus_1 = pool_index_plus_1; - parts.offset = index & stack_depot_offset_mask(); + parts.offset = index & DEPOT_OFFSET_MASK; return parts.handle; } @@ -670,7 +641,7 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) return 0; pool_delta = parts.pool_index_plus_1 - stack_max_pools - 1; - if ((u64)pool_delta + stack_max_pools + 1 >= stack_depot_pool_index_mask()) + if ((u64)pool_delta + stack_max_pools + 1 >= DEPOT_POOL_INDEX_MASK) return 0; leaf_id = ((u64)pool_delta << DEPOT_OFFSET_BITS) + parts.offset + 1; @@ -1903,20 +1874,22 @@ static int trie_pool_add_size(size_t size, size_t *total) return *total <= DEPOT_POOL_SIZE ? 0 : -EINVAL; } -static void trie_pool_release_reused_objects_locked(struct stack_depot_trie_pool_request *req) +static void trie_pool_release_reused_objects_locked(struct stack_depot_trie_alloc_request *req) { + struct stack_depot_trie_pool_mark *mark; unsigned long completed; unsigned int i; lockdep_assert_held(&pool_lock); - if (!req) + if (!req || !req->txn) return; + mark = &req->txn->pool; completed = get_completed_synchronize_rcu(); for (i = 0; req->node_slots && i < req->nr_node_slots; i++) { void *node = req->node_slots[i].node; - if (node && !trie_pool_mark_contains(req->mark, node)) { + if (node && !trie_pool_mark_contains(mark, node)) { trie_add_free_node_locked(node, req->node_slots[i].size); req->node_slots[i].node = NULL; } @@ -1925,21 +1898,21 @@ static void trie_pool_release_reused_objects_locked(struct stack_depot_trie_pool void *array = req->child_slots[i].array; if (array && - !trie_pool_mark_contains(req->mark, + !trie_pool_mark_contains(mark, trie_object_header(array))) { trie_free_object_locked(array, completed); req->child_slots[i].array = NULL; } } if (req->storage && *req->storage && - !trie_pool_mark_contains(req->mark, + !trie_pool_mark_contains(mark, trie_object_header(*req->storage))) { trie_free_object_locked(*req->storage, completed); *req->storage = NULL; } } -static void trie_pool_release_reused_objects(struct stack_depot_trie_pool_request *req) +static void trie_pool_release_reused_objects(struct stack_depot_trie_alloc_request *req) { unsigned long flags; @@ -1957,8 +1930,9 @@ static void trie_pool_release_reused_objects(struct stack_depot_trie_pool_reques * freelists separately on failure. The caller must not publish any returned * storage until the trie and side-table transaction commits. */ -static int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *req) +static int __stack_depot_trie_pool_carve(struct stack_depot_trie_alloc_request *req) { + struct stack_depot_trie_pool_mark *mark; unsigned long flags; unsigned int i; size_t offset; @@ -1966,9 +1940,10 @@ static int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *r void *pool; int ret = -EINVAL; - if (!req || !req->mark) + if (!req || !req->txn) return -EINVAL; - memset(req->mark, 0, sizeof(*req->mark)); + mark = &req->txn->pool; + memset(mark, 0, sizeof(*mark)); if (!req->storage || *req->storage || (!req->node_slots && req->nr_node_slots) || (!req->child_slots && req->nr_child_slots)) @@ -2015,32 +1990,32 @@ static int __stack_depot_trie_pool_carve(struct stack_depot_trie_pool_request *r } if (pools_num < 1) { - req->mark->prev_offset = pool_offset; - if (!depot_init_pool(req->prealloc)) { + mark->prev_offset = pool_offset; + if (!depot_init_pool(req->pool_prealloc)) { ret = -ENOSPC; goto out_release_reused; } - req->mark->added_pool = true; + mark->added_pool = true; } if (WARN_ON_ONCE(pool_offset > DEPOT_POOL_SIZE)) goto out_release_reused; if (total > DEPOT_POOL_SIZE - pool_offset) { - req->mark->prev_offset = pool_offset; - if (!depot_init_pool(req->prealloc)) { + mark->prev_offset = pool_offset; + if (!depot_init_pool(req->pool_prealloc)) { ret = -ENOSPC; goto out_release_reused; } - req->mark->added_pool = true; + mark->added_pool = true; } - req->mark->pool_index = pools_num - 1; - pool = stack_pools[req->mark->pool_index]; + mark->pool_index = pools_num - 1; + pool = stack_pools[mark->pool_index]; if (WARN_ON_ONCE(!pool)) goto out_release_reused; - req->mark->offset = pool_offset; - req->mark->pool = pool; - req->mark->size = total; + mark->offset = pool_offset; + mark->pool = pool; + mark->size = total; offset = pool_offset; for (i = 0; i < req->nr_node_slots; i++) { if (req->node_slots[i].node) @@ -2110,22 +2085,13 @@ static void trie_alloc_request_clear_outputs(struct stack_depot_trie_alloc_reque static void trie_alloc_request_release_reused_objects(struct stack_depot_trie_alloc_request *req) { - struct stack_depot_trie_pool_request pool_req = {}; - if (!req || !req->txn) return; - pool_req.node_slots = req->node_slots; - pool_req.nr_node_slots = req->nr_node_slots; - pool_req.child_slots = req->child_slots; - pool_req.nr_child_slots = req->nr_child_slots; - pool_req.storage = req->storage; - pool_req.mark = &req->txn->pool; - trie_pool_release_reused_objects(&pool_req); + trie_pool_release_reused_objects(req); } static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req) { - struct stack_depot_trie_pool_request pool_req = {}; int ret; if (!req || !req->txn) @@ -2133,20 +2099,11 @@ static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_re if (req->txn->leaf_id || req->txn->pool.size || req->txn->side.nr_updates) return -EINVAL; - pool_req.node_slots = req->node_slots; - pool_req.nr_node_slots = req->nr_node_slots; - pool_req.child_slots = req->child_slots; - pool_req.nr_child_slots = req->nr_child_slots; - pool_req.storage = req->storage; - pool_req.storage_size = req->storage_size; - pool_req.prealloc = req->pool_prealloc; - pool_req.mark = &req->txn->pool; - ret = __stack_depot_trie_alloc_txn_id(req->txn, req->side_prealloc); if (ret) return ret; - ret = __stack_depot_trie_pool_carve(&pool_req); + ret = __stack_depot_trie_pool_carve(req); if (ret) { __stack_depot_trie_alloc_txn_rollback(req->txn); trie_alloc_request_clear_outputs(req); @@ -2202,24 +2159,6 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, return 0; } -static int -trie_ws_plan(const struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - struct stack_depot_trie_alloc_workspace *workspace) -{ - if (!workspace) - return -EINVAL; - - memset(workspace, 0, sizeof(*workspace)); - return __stack_depot_trie_alloc_txn_plan(root, entries, nr_entries, - workspace->node_slots, ARRAY_SIZE(workspace->node_slots), - workspace->child_slots, ARRAY_SIZE(workspace->child_slots), - &workspace->txn, &workspace->storage, pool_prealloc, - side_prealloc, &workspace->req); -} - static int __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, const unsigned long *entries, @@ -2228,16 +2167,30 @@ __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, struct stack_depot_trie_alloc_workspace *workspace, const void **tail, u32 *leaf_id) { - void **pool = pool_prealloc; + struct stack_depot_trie_child_array_slot *child_slots; + struct stack_depot_trie_node_slot *node_slots; int ret; - ret = trie_ws_plan(root, entries, nr_entries, pool, side_prealloc, workspace); + if (!workspace) + return -EINVAL; + + memset(workspace, 0, sizeof(*workspace)); + node_slots = workspace->node_slots; + child_slots = workspace->child_slots; + ret = __stack_depot_trie_alloc_txn_plan(root, + entries, nr_entries, node_slots, + ARRAY_SIZE(workspace->node_slots), child_slots, + ARRAY_SIZE(workspace->child_slots), + &workspace->txn, &workspace->storage, + pool_prealloc, side_prealloc, + &workspace->req); if (ret) return ret; return __stack_depot_trie_alloc_txn_insert(root, &workspace->req, entries, - nr_entries, workspace->scratch, ARRAY_SIZE(workspace->scratch), - tail, leaf_id); + nr_entries, workspace->scratch, + ARRAY_SIZE(workspace->scratch), + tail, leaf_id); } static depot_stack_handle_t @@ -2385,7 +2338,6 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, unsigned int nr_scratch, const void **tail, u32 *leaf_id) { - struct stack_depot_trie_publish_prepare prepare; struct stack_depot_trie_alloc_txn *txn; unsigned long flags; u32 id; @@ -2406,16 +2358,13 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, goto out_unlock; storage = req->storage ? *req->storage : NULL; - prepare.fn = __stack_depot_trie_side_prepare; - prepare.ctx = &txn->side; - prepare.retire_locked = false; id = txn->leaf_id; ret = __stack_depot_trie_insert_append_prepare(root, NULL, id, entries, nr_entries, req->node_slots, req->nr_node_slots, req->child_slots, req->nr_child_slots, scratch, nr_scratch, storage, req->storage_size, - &prepare, tail, &nr_used); + &txn->side, tail, &nr_used); if (ret) goto rollback; @@ -2500,13 +2449,14 @@ trie_side_prepare_locked(const struct stack_depot_trie_leaf_update *updates, static int __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, void *ctx) + unsigned int nr_updates, + struct stack_depot_trie_side_prepare *state) { unsigned long flags; int ret; raw_spin_lock_irqsave(&trie_side_table_lock, flags); - ret = trie_side_prepare_locked(updates, nr_updates, ctx); + ret = trie_side_prepare_locked(updates, nr_updates, state); raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); return ret; } @@ -3270,7 +3220,7 @@ bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count) struct stack_record *stack; unsigned int raw; - if (!handle || !count) + if (!handle || !count || __stack_depot_trie_leaf_id(handle)) return false; stack = depot_fetch_stack(handle); @@ -3292,7 +3242,8 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count) struct stack_record *stack; /* Reject values outside positive refcount space. */ - if (!handle || !count || count > (unsigned int)INT_MAX) + if (!handle || !count || count > (unsigned int)INT_MAX || + __stack_depot_trie_leaf_id(handle)) return; stack = depot_fetch_stack(handle); @@ -3313,7 +3264,8 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, if (new_count) *new_count = false; - if (!handle || !count || count > (unsigned int)INT_MAX - 1) + if (!handle || !count || count > (unsigned int)INT_MAX - 1 || + __stack_depot_trie_leaf_id(handle)) return false; stack = depot_fetch_stack(handle); @@ -3353,7 +3305,8 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, int new; int old; - if (!handle || !count || count > (unsigned int)INT_MAX) + if (!handle || !count || count > (unsigned int)INT_MAX || + __stack_depot_trie_leaf_id(handle)) return false; stack = depot_fetch_stack(handle); @@ -4342,7 +4295,7 @@ trie_promote_child(struct stack_depot_trie_root *root, const struct stack_depot_trie_node *child, u32 leaf_id, const struct stack_depot_trie_node_slot *slot, void *new_storage, size_t new_storage_size, - const struct stack_depot_trie_publish_prepare *prepare) + struct stack_depot_trie_side_prepare *side) { const struct stack_depot_trie_child_array **publish_slot; const struct stack_depot_trie_child_array *old_array; @@ -4363,12 +4316,10 @@ trie_promote_child(struct stack_depot_trie_root *root, child_size = __stack_depot_trie_node_size(&child->run); if (!child_size) return -EINVAL; - if (prepare) { - if (!prepare->fn) - return -EINVAL; + if (side) { update.leaf_id = leaf_id; update.leaf = slot->node; - ret = prepare->fn(&update, 1, prepare->ctx); + ret = __stack_depot_trie_side_prepare(&update, 1, side); if (ret) return ret; } @@ -4379,10 +4330,7 @@ trie_promote_child(struct stack_depot_trie_root *root, publish_slot = trie_publish_slot(root, parent); /* Publish the fully initialized replacement array last. */ trie_publish_children_slot(publish_slot, new_storage); - if (prepare && prepare->retire_locked) - trie_retire_object_node_locked(old_array, child, child_size); - else - trie_retire_object_node(old_array, child, child_size); + trie_retire_object_node(old_array, child, child_size); return 0; } @@ -4536,7 +4484,7 @@ __stack_depot_trie_append_chain(const void *parent_ptr, u32 leaf_id, static int trie_publish_append_prepare(struct stack_depot_trie_root *root, void *parent_ptr, const void *head_ptr, void *new_storage, size_t new_storage_size, - const struct stack_depot_trie_publish_prepare *prepare, + struct stack_depot_trie_side_prepare *side, u32 leaf_id, const void *leaf) { @@ -4589,17 +4537,15 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, return -EINVAL; if (found || !trie_child_array_can_append(old_array, pos)) return -EINVAL; - if (prepare) { + if (side) { struct stack_depot_trie_leaf_update update = { .leaf_id = leaf_id, .leaf = leaf, }; - if (!prepare->fn) - return -EINVAL; if (!leaf_id || !leaf) return -EINVAL; - ret = prepare->fn(&update, 1, prepare->ctx); + ret = __stack_depot_trie_side_prepare(&update, 1, side); if (ret) return ret; } @@ -4619,26 +4565,21 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, return -EINVAL; if (__stack_depot_trie_child_array_insert(old_array, head, new_array, storage_size)) return -EINVAL; - if (prepare) { + if (side) { struct stack_depot_trie_leaf_update update = { .leaf_id = leaf_id, .leaf = leaf, }; - if (!prepare->fn) - return -EINVAL; if (!leaf_id || !leaf) return -EINVAL; - ret = prepare->fn(&update, 1, prepare->ctx); + ret = __stack_depot_trie_side_prepare(&update, 1, side); if (ret) return ret; } /* Publish the fully initialized replacement array last. */ trie_publish_children_slot(slot, new_array); - if (prepare && prepare->retire_locked) - trie_retire_object_node_locked(old_array, NULL, 0); - else - trie_retire_object(old_array); + trie_retire_object(old_array); return 0; } @@ -4778,7 +4719,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, - const struct stack_depot_trie_publish_prepare *prepare, + struct stack_depot_trie_side_prepare *side, const void **tail, unsigned int *nr_used); @@ -4794,7 +4735,7 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, - const struct stack_depot_trie_publish_prepare *prepare, + struct stack_depot_trie_side_prepare *side, const void **tail, unsigned int *nr_used) { @@ -4834,14 +4775,14 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, leaf_id, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, scratch, nr_scratch, new_storage, - new_storage_size, prepare, tail, + new_storage_size, side, tail, nr_used); if (lookup.status == STACK_DEPOT_TRIE_LOOKUP_PROMOTE) { if (!node_slots || !nr_node_slots) return -EINVAL; ret = trie_promote_child(root, parent, lookup.node, leaf_id, &node_slots[0], new_storage, new_storage_size, - prepare); + side); if (ret) return ret; *tail = node_slots[0].node; @@ -4864,7 +4805,7 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, if (ret) return ret; ret = trie_publish_append_prepare(root, parent, head, new_storage, - new_storage_size, prepare, leaf_id, last); + new_storage_size, side, leaf_id, last); if (ret) return ret; @@ -5821,7 +5762,7 @@ static int trie_split_subtree_prepare(const void *child_ptr, unsigned int matche const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, - const struct stack_depot_trie_publish_prepare *prepare, + struct stack_depot_trie_side_prepare *side, const void **prefix, const void **tail, unsigned int *nr_used) { @@ -5892,10 +5833,8 @@ static int trie_split_subtree_prepare(const void *child_ptr, unsigned int matche updates[nr_updates].leaf_id = leaf_id; updates[nr_updates].leaf = has_new_tail ? new_tail : pref; nr_updates++; - if (prepare) { - if (!prepare->fn) - return -EINVAL; - ret = prepare->fn(updates, nr_updates, prepare->ctx); + if (side) { + ret = __stack_depot_trie_side_prepare(updates, nr_updates, side); if (ret) { memset(split_array, 0, split_array_size); return ret; @@ -5923,7 +5862,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, - const struct stack_depot_trie_publish_prepare *prepare, + struct stack_depot_trie_side_prepare *side, const void **tail, unsigned int *nr_used) { @@ -5963,7 +5902,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, ret = trie_split_subtree_prepare(child, matched, leaf_id, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, scratch, - nr_scratch, prepare, &prefix, + nr_scratch, side, &prefix, tail, &used); if (ret) return ret; @@ -5972,10 +5911,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, new_storage_size, pos); /* Publish the fully initialized replacement array last. */ trie_publish_children_slot(publish_slot, new_storage); - if (prepare && prepare->retire_locked) - trie_retire_object_node_locked(old_array, child, child_size); - else - trie_retire_object_node(old_array, child, child_size); + trie_retire_object_node(old_array, child, child_size); *nr_used = used; return 0; } From 61a7987f22f4e9df8c5c9314e2ffbcba4c97051d Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 23 Jun 2026 21:24:17 +0100 Subject: [PATCH 119/129] KRN-1117: Simplify stackdepot trie state Remove redundant trie metadata and runtime lifecycle state that made the implementation harder to review. Trie enablement is now fixed at boot, so the runtime requested/enabled split and mixed hash/trie transition marker are no longer needed. Also derive frame-run byte sizes and trie stack lengths from existing state, type trie node pointers where possible, and fold small side-table helper layers that only obscured the mutation path. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 720 ++++++++++++++++++++--------------------------- 1 file changed, 309 insertions(+), 411 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index e0bb29978db42..314754cb50fbb 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -68,44 +68,41 @@ enum stack_depot_trie_lookup_status { * decoding mode. */ struct stack_depot_frame_run { - u16 bytes; u16 nr_entries; u8 mode; }; static_assert(CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(unsigned long) <= U16_MAX); -/* Opaque trie node storage; node layout stays private to stackdepot.c. */ +struct stack_depot_trie_node; +struct stack_depot_trie_child_array; +struct stack_depot_trie_side_dir; +struct stack_depot_trie_side_entry; + struct stack_depot_trie_node_slot { - void *node; + struct stack_depot_trie_node *node; size_t size; }; -/* Opaque child-array storage; child array layout stays private. */ struct stack_depot_trie_child_array_slot { - void *array; + struct stack_depot_trie_child_array *array; size_t size; }; -struct stack_depot_trie_child_array; - struct stack_depot_trie_root { const struct stack_depot_trie_child_array *children; }; struct stack_depot_trie_lookup { - /* Opaque parent trie node for the current lookup step. */ - const void *parent; - /* Opaque trie node matched at this step, if any. */ - const void *node; + const struct stack_depot_trie_node *parent; + const struct stack_depot_trie_node *node; enum stack_depot_trie_lookup_status status; unsigned int matched; }; struct stack_depot_trie_leaf_update { u32 leaf_id; - /* Opaque trie leaf that should become visible for leaf_id. */ - const void *leaf; + const struct stack_depot_trie_node *leaf; }; /* A split can repoint the old leaf and publish one new leaf. */ @@ -115,7 +112,7 @@ struct stack_depot_trie_leaf_update { struct stack_depot_trie_side_checkpoint { u32 leaf_id; - const void *old_leaf; + const struct stack_depot_trie_node *old_leaf; }; struct stack_depot_trie_side_prepare { @@ -125,9 +122,9 @@ struct stack_depot_trie_side_prepare { struct stack_depot_trie_side_prealloc { /* Preallocated side-table directory page for sparse growth. */ - void *dir; + struct stack_depot_trie_side_dir *dir; /* Preallocated side-table leaf chunk for sparse growth. */ - void *chunk; + struct stack_depot_trie_side_entry *chunk; }; struct stack_depot_trie_pool_mark { @@ -174,7 +171,6 @@ struct stack_depot_trie_alloc_workspace { (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) static bool __stack_depot_trie_ready(void); -static void __stack_depot_trie_set_enabled(bool enabled); static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id); static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle); static u32 __stack_depot_trie_max_leaf_id(void); @@ -187,8 +183,10 @@ __stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_preallo static u32 __stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc); static void __stack_depot_trie_side_table_revoke_latest(u32 id); -static void __stack_depot_trie_side_table_restore(u32 id, const void *entry); -static const void *__stack_depot_trie_side_table_lookup(u32 id); +static void +__stack_depot_trie_side_table_restore(u32 id, + const struct stack_depot_trie_node *entry); +static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup(u32 id); static size_t __stack_depot_trie_side_table_bytes(void); static size_t __stack_depot_trie_pool_alloc_size(size_t size); static void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags); @@ -220,7 +218,7 @@ __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_workspace *workspace, - const void **tail, u32 *leaf_id); + u32 *leaf_id); static depot_stack_handle_t __stack_depot_trie_save_locked(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, @@ -234,7 +232,7 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, struct stack_depot_trie_alloc_request *req, const unsigned long *entries, unsigned int nr_entries, u32 *scratch, unsigned int nr_scratch, - const void **tail, u32 *leaf_id); + u32 *leaf_id); static void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); static int __stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, @@ -246,37 +244,47 @@ static int __stack_depot_frame_run_init(const unsigned long *entries, struct stack_depot_frame_run *run); static size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run); static int __stack_depot_trie_node_init(void *storage, size_t storage_size, - const void *parent, u32 leaf_id, + const struct stack_depot_trie_node *parent, + u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, u32 *scratch, unsigned int nr_scratch); static int __stack_depot_trie_node_init_slice(void *storage, size_t storage_size, - const void *parent, u32 leaf_id, - const void *src_node, unsigned int start, + const struct stack_depot_trie_node *parent, + u32 leaf_id, + const struct stack_depot_trie_node *src_node, + unsigned int start, unsigned int nr_entries); -static unsigned int __stack_depot_trie_node_match(const void *node, +static const struct stack_depot_trie_node * +trie_load_parent(const struct stack_depot_trie_node *node); +static unsigned int __stack_depot_trie_node_match(const struct stack_depot_trie_node *node, const unsigned long *entries, unsigned int nr_entries); static int -__stack_depot_trie_append_chain(const void *parent, u32 leaf_id, +__stack_depot_trie_append_chain(const struct stack_depot_trie_node *parent, + u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, const struct stack_depot_trie_node_slot *node_slots, unsigned int nr_node_slots, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, const void **head, - const void **tail, unsigned int *nr_used); + unsigned int nr_scratch, + const struct stack_depot_trie_node **head, + const struct stack_depot_trie_node **tail, + unsigned int *nr_used); static int __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, - const void *parent, const unsigned long *entries, + const struct stack_depot_trie_node *parent, + const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_lookup *lookup); -static const void * +static const struct stack_depot_trie_node * __stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries); static int __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, - void *parent, u32 leaf_id, + struct stack_depot_trie_node *parent, + u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, const struct stack_depot_trie_node_slot *node_slots, unsigned int nr_node_slots, @@ -286,29 +294,32 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, struct stack_depot_trie_side_prepare *side, - const void **tail, unsigned int *nr_used); + const struct stack_depot_trie_node **tail, + unsigned int *nr_used); static int __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, - const void *parent, const unsigned long *entries, + const struct stack_depot_trie_node *parent, + const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_node_slot *node_slots, unsigned int nr_node_slots, struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, size_t *new_storage_size, unsigned int *nr_used, unsigned int *nr_child_used); -static unsigned int __stack_depot_trie_fetch_into(const void *leaf, +static unsigned int __stack_depot_trie_fetch_into(const struct stack_depot_trie_node *leaf, unsigned long *entries, unsigned int max_entries); static unsigned int __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, unsigned long *entries, unsigned int max_entries); -static size_t __stack_depot_trie_child_array_size(unsigned int nr_children); +static inline size_t __stack_depot_trie_child_array_size(unsigned int nr_children); static int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, - const void * const *children, + const struct stack_depot_trie_node * const *children, unsigned int nr_children); static int __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, - const void *old_tail, const void *new_head); + const struct stack_depot_trie_node *old_tail, + const struct stack_depot_trie_node *new_head); static int __stack_depot_trie_split_tail_plan(const unsigned long *entries, unsigned int nr_entries, @@ -318,16 +329,16 @@ __stack_depot_trie_split_tail_plan(const unsigned long *entries, unsigned int nr_child_slots, unsigned int *nr_runs); static int __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, - const void *parent, + struct stack_depot_trie_node *parent, const struct stack_depot_trie_node_slot *node_slots, unsigned int nr_node_slots, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, void *new_storage, size_t new_storage_size); -static int __stack_depot_trie_child_array_insert(const void *old_storage, - const void *child, - void *new_storage, - size_t new_storage_size); +static int +__stack_depot_trie_child_array_insert(const struct stack_depot_trie_child_array *old, + const struct stack_depot_trie_node *child, + void *new_storage, size_t new_storage_size); /* * The pool_index is offset by 1 so the first record does not have a 0 handle. @@ -341,85 +352,26 @@ static bool __stack_depot_early_init_requested __initdata = static bool __stack_depot_early_init_passed __initdata; static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); static DEFINE_MUTEX(stack_depot_init_mutex); -static DEFINE_MUTEX(stack_depot_trie_param_lock); static struct stack_depot_trie_root stack_depot_trie_root; static struct stack_depot_trie_alloc_workspace __rcu *stack_depot_trie_workspace; static DEFINE_RAW_SPINLOCK(stack_depot_trie_workspace_lock); -static bool stack_depot_trie_requested; +static bool stack_depot_trie_enabled_param; static bool __stack_depot_trie_enabled(void) { return static_branch_unlikely(&stack_depot_trie_enabled); } -static void __stack_depot_trie_set_enabled(bool enabled) +static void stack_depot_trie_enable(void) { - if (__stack_depot_trie_enabled() == enabled) + if (__stack_depot_trie_enabled()) return; - if (enabled) - static_branch_enable(&stack_depot_trie_enabled); - else - static_branch_disable(&stack_depot_trie_enabled); -} - -static bool __stack_depot_trie_requested(void) -{ - return READ_ONCE(stack_depot_trie_requested); -} - -static int stack_depot_trie_enabled_param_set(const char *val, - const struct kernel_param *kp) -{ - struct kernel_param tmp = *kp; - bool enabled; - int ret; - - tmp.arg = &enabled; - ret = param_set_bool(val, &tmp); - if (ret) - return ret; - - /* Keep runtime toggles serialized outside stack_depot_init_mutex. */ - mutex_lock(&stack_depot_trie_param_lock); - WRITE_ONCE(stack_depot_trie_requested, enabled); - if (!enabled) { - __stack_depot_trie_set_enabled(false); - goto out_unlock; - } - - if (system_state >= SYSTEM_RUNNING) { - ret = stack_depot_init(); - if (ret || !__stack_depot_trie_ready()) { - WRITE_ONCE(stack_depot_trie_requested, false); - __stack_depot_trie_set_enabled(false); - ret = ret ?: -ENOMEM; - } - } -out_unlock: - mutex_unlock(&stack_depot_trie_param_lock); - return ret; + static_branch_enable(&stack_depot_trie_enabled); } -static int stack_depot_trie_enabled_param_get(char *buffer, - const struct kernel_param *kp) -{ - struct kernel_param tmp = *kp; - bool enabled = __stack_depot_trie_enabled(); - - tmp.arg = &enabled; - return param_get_bool(buffer, &tmp); -} - -static const struct kernel_param_ops stack_depot_trie_enabled_param_ops = { - /* param_set_bool() treats a missing value as true. */ - .flags = KERNEL_PARAM_OPS_FL_NOARG, - .set = stack_depot_trie_enabled_param_set, - .get = stack_depot_trie_enabled_param_get, -}; -module_param_cb(trie_enabled, &stack_depot_trie_enabled_param_ops, - NULL, 0644); -MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage"); +module_param_named(trie_enabled, stack_depot_trie_enabled_param, bool, 0); +MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); /* Use one hash table bucket per 16 KB of memory. */ #define STACK_HASH_TABLE_SCALE 14 @@ -472,7 +424,6 @@ struct stack_depot_trie_node { /* Child arrays are separate RCU/COW generations; nodes stay immutable. */ const struct stack_depot_trie_child_array *children; u32 leaf_id; - u16 stack_len; struct stack_depot_frame_run run; unsigned char data[]; }; @@ -503,7 +454,7 @@ struct stack_depot_trie_free_object { struct list_head list; unsigned long rcu_state; size_t size; - void *pending_node; + struct stack_depot_trie_node *pending_node; size_t pending_node_size; }; @@ -521,8 +472,6 @@ static unsigned int stack_hash_mask; static void **stack_pools; /* Newly allocated pool that is not yet added to stack_pools. */ static void *new_pool; -/* Whether legacy hash storage may contain normal persistent records. */ -static bool stack_depot_persistent_hash_record_seen; /* Number of pools in stack_pools. */ static int pools_num; static unsigned long pools_min_addr; @@ -581,27 +530,23 @@ static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); static bool depot_init_pool(void **prealloc); static void depot_try_keep_new_pool(void **prealloc); - -static bool stack_depot_trie_namespace_available(void) -{ - /* Reserve the all-ones pool index as an invalid trie namespace sentinel. */ - return stack_max_pools < DEPOT_POOL_INDEX_MASK - 1; -} +static u32 trie_side_table_max_id; /* - * Hash handles encode pool_index_plus_1 and offset. Trie handles reserve the - * pool-index values above stack_max_pools and reinterpret the offset bits as a - * dense leaf_id, which the side table maps to a trie leaf. Init treats an - * unavailable namespace as a hard trie failure; the checks below are defensive - * for disabled and failed-initialization configurations. + * Hash handles use pool_index_plus_1 <= stack_max_pools. Trie handles use + * pool_index_plus_1 > stack_max_pools and reinterpret the remaining handle + * bits as a dense leaf_id, which the side table maps to a trie leaf. */ static u32 __stack_depot_trie_max_leaf_id(void) { - if (!stack_depot_trie_namespace_available()) + u64 max_id; + + if (stack_max_pools >= DEPOT_POOL_INDEX_MASK - 1) return 0; - return (DEPOT_POOL_INDEX_MASK - stack_max_pools - 1) << - DEPOT_OFFSET_BITS; + max_id = (u64)(DEPOT_POOL_INDEX_MASK - stack_max_pools - 1) << + DEPOT_OFFSET_BITS; + return min_t(u64, max_id, U32_MAX); } static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) @@ -611,9 +556,7 @@ static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) u32 pool_delta; u32 index; - if (!leaf_id || !stack_depot_trie_namespace_available()) - return 0; - if (leaf_id > __stack_depot_trie_max_leaf_id()) + if (!leaf_id || leaf_id > READ_ONCE(trie_side_table_max_id)) return 0; index = leaf_id - 1; @@ -633,9 +576,6 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) u64 leaf_id; u32 pool_delta; - if (!stack_depot_trie_namespace_available()) - return 0; - parts.extra = 0; if (parts.pool_index_plus_1 <= stack_max_pools) return 0; @@ -645,11 +585,14 @@ static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) return 0; leaf_id = ((u64)pool_delta << DEPOT_OFFSET_BITS) + parts.offset + 1; + if (leaf_id > READ_ONCE(trie_side_table_max_id)) + return 0; + return leaf_id > U32_MAX ? 0 : leaf_id; } struct stack_depot_trie_side_entry { - const void __rcu *leaf; + const struct stack_depot_trie_node __rcu *leaf; }; /* @@ -675,15 +618,13 @@ struct stack_depot_trie_side_root { static struct stack_depot_trie_side_root __rcu *trie_side_table_root; static DEFINE_RAW_SPINLOCK(trie_side_table_lock); -static DEFINE_RAW_SPINLOCK(trie_alloc_lock); static unsigned int trie_side_table_nr_dirs; static unsigned int trie_side_table_nr_chunks; static unsigned int trie_side_table_root_size; -static u32 trie_side_table_max_id; static u32 trie_side_table_next_id; static bool trie_side_table_memblock; -/* Lock order: workspace_lock -> trie_alloc_lock -> pool_lock -> trie_side_table_lock. */ +/* Lock order: workspace_lock -> pool_lock -> trie_side_table_lock. */ static struct stack_depot_trie_alloc_workspace *stack_depot_trie_load_workspace(void) { @@ -709,23 +650,23 @@ static bool __stack_depot_trie_ready(void) trie_side_table_is_initialized(); } -static unsigned int trie_side_table_top_index(u32 id) +static inline unsigned int trie_side_table_top_index(u32 id) { return (id - 1) >> STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS; } -static unsigned int trie_side_table_root_index(u32 id) +static inline unsigned int trie_side_table_root_index(u32 id) { return trie_side_table_top_index(id) >> STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS; } -static unsigned int trie_side_table_dir_index(u32 id) +static inline unsigned int trie_side_table_dir_index(u32 id) { return trie_side_table_top_index(id) & (STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE - 1); } -static unsigned int trie_side_table_slot_index(u32 id) +static inline unsigned int trie_side_table_slot_index(u32 id) { return (id - 1) & (STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE - 1); } @@ -759,52 +700,44 @@ static struct stack_depot_trie_side_entry * trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, unsigned int idx) { - /* Pairs with trie_side_table_dir_publish_chunk(); lookup is lockless. */ + /* Pairs with the chunk rcu_assign_pointer() in leaf ID allocation. */ return rcu_dereference_check(dir->chunks[idx], lockdep_is_held(&trie_side_table_lock) || rcu_read_lock_sched_held()); } -static void -trie_side_table_dir_publish_chunk(struct stack_depot_trie_side_dir *dir, - unsigned int idx, - struct stack_depot_trie_side_entry *chunk) -{ - /* Pairs with trie_side_table_dir_load_chunk(). */ - rcu_assign_pointer(dir->chunks[idx], chunk); -} - static u32 -trie_side_table_alloc_id_locked(struct stack_depot_trie_side_prealloc *prealloc) +__stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc) { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; + unsigned long flags; unsigned int root; unsigned int idx; - u32 id; + u32 id = 0; - lockdep_assert_held(&trie_side_table_lock); + raw_spin_lock_irqsave(&trie_side_table_lock, flags); /* Failed or disabled trie init means no leaf IDs can be allocated. */ if (!trie_side_table_is_initialized()) - return 0; + goto out; if (!prealloc) - return 0; + goto out; id = trie_side_table_next_id + 1; - /* ID zero wraps the 32-bit counter; max_id is handle namespace capacity. */ + /* ID zero wraps the 32-bit counter; max_id is trie handle capacity. */ if (!id || id > trie_side_table_max_id) - return 0; + goto out_clear_id; root = trie_side_table_root_index(id); /* Should be impossible when trie_side_table_max_id/root_size agree. */ if (root >= trie_side_table_root_size) - return 0; + goto out_clear_id; dir = trie_side_table_load_dir(root); if (!dir) { /* Sparse growth preallocation can lose a race to another writer. */ if (!prealloc->dir) - return 0; + goto out_clear_id; dir = prealloc->dir; prealloc->dir = NULL; trie_side_table_publish_dir(root, dir); @@ -816,14 +749,20 @@ trie_side_table_alloc_id_locked(struct stack_depot_trie_side_prealloc *prealloc) if (!chunk) { /* Sparse growth preallocation can lose a race to another writer. */ if (!prealloc->chunk) - return 0; + goto out_clear_id; chunk = prealloc->chunk; prealloc->chunk = NULL; - trie_side_table_dir_publish_chunk(dir, idx, chunk); + rcu_assign_pointer(dir->chunks[idx], chunk); trie_side_table_nr_chunks++; } WRITE_ONCE(trie_side_table_next_id, id); + goto out; + +out_clear_id: + id = 0; +out: + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); return id; } @@ -837,23 +776,23 @@ static size_t trie_side_table_root_bytes(unsigned int root_size) return PAGE_ALIGN(bytes); } -static size_t trie_side_table_dir_bytes(void) +static inline size_t trie_side_table_dir_bytes(void) { return PAGE_ALIGN(sizeof(struct stack_depot_trie_side_dir)); } -static unsigned int trie_side_table_dir_order(void) +static inline unsigned int trie_side_table_dir_order(void) { return get_order(trie_side_table_dir_bytes()); } -static size_t trie_side_table_chunk_bytes(void) +static inline size_t trie_side_table_chunk_bytes(void) { return PAGE_ALIGN(STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE * sizeof(struct stack_depot_trie_side_entry)); } -static unsigned int trie_side_table_chunk_order(void) +static inline unsigned int trie_side_table_chunk_order(void) { return get_order(trie_side_table_chunk_bytes()); } @@ -1008,9 +947,6 @@ static int __init stack_depot_trie_init_memblock(void) { int ret; - if (!__stack_depot_trie_requested()) - return 0; - ret = stack_depot_trie_init_workspace_memblock(); if (ret) return ret; @@ -1018,7 +954,7 @@ static int __init stack_depot_trie_init_memblock(void) if (ret) return ret; - __stack_depot_trie_set_enabled(true); + stack_depot_trie_enable(); return 0; } @@ -1026,9 +962,6 @@ static int stack_depot_trie_init(gfp_t gfp_flags) { int ret; - if (!__stack_depot_trie_requested()) - return 0; - ret = stack_depot_trie_init_workspace(gfp_flags); if (ret) return ret; @@ -1036,15 +969,11 @@ static int stack_depot_trie_init(gfp_t gfp_flags) if (ret) return ret; - /* Runtime enable is not a live migration from hash to trie storage. */ - if (system_state >= SYSTEM_RUNNING) - WRITE_ONCE(stack_depot_persistent_hash_record_seen, true); - - __stack_depot_trie_set_enabled(true); + stack_depot_trie_enable(); return 0; } -static const void * +static const struct stack_depot_trie_node * trie_side_table_load_leaf(struct stack_depot_trie_side_entry *chunk, unsigned int slot) { @@ -1056,7 +985,8 @@ trie_side_table_load_leaf(struct stack_depot_trie_side_entry *chunk, static void trie_side_table_store_leaf(struct stack_depot_trie_side_entry *chunk, - unsigned int slot, const void *leaf) + unsigned int slot, + const struct stack_depot_trie_node *leaf) { /* Pairs with trie_side_table_load_leaf(). */ rcu_assign_pointer(chunk[slot].leaf, leaf); @@ -1201,18 +1131,6 @@ __stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_preallo prealloc->chunk = NULL; } -static u32 -__stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc) -{ - unsigned long flags; - u32 id; - - raw_spin_lock_irqsave(&trie_side_table_lock, flags); - id = trie_side_table_alloc_id_locked(prealloc); - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return id; -} - static void __stack_depot_trie_side_table_revoke_latest(u32 id) { struct stack_depot_trie_side_entry *chunk; @@ -1246,7 +1164,9 @@ static void __stack_depot_trie_side_table_revoke_latest(u32 id) raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); } -static void __stack_depot_trie_side_table_restore(u32 id, const void *entry) +static void +__stack_depot_trie_side_table_restore(u32 id, + const struct stack_depot_trie_node *entry) { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; @@ -1305,7 +1225,7 @@ trie_side_table_chunk_locked(u32 id, unsigned int *slot) return chunk; } -static const void *__stack_depot_trie_side_table_lookup(u32 id) +static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup(u32 id) { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; @@ -1371,7 +1291,7 @@ static size_t __stack_depot_trie_pool_alloc_size(size_t size) return aligned <= DEPOT_POOL_SIZE ? aligned : 0; } -static size_t trie_object_header_size(void) +static inline size_t trie_object_header_size(void) { return ALIGN(sizeof(struct stack_depot_trie_free_object), 1UL << DEPOT_STACK_ALIGN); @@ -1390,12 +1310,12 @@ static size_t trie_object_alloc_size(size_t size) return alloc_size <= DEPOT_POOL_SIZE ? alloc_size : 0; } -static struct stack_depot_trie_free_object *trie_object_header(const void *ptr) +static inline struct stack_depot_trie_free_object *trie_object_header(const void *ptr) { return (void *)ptr - trie_object_header_size(); } -static void *trie_object_payload(struct stack_depot_trie_free_object *free) +static inline void *trie_object_payload(struct stack_depot_trie_free_object *free) { return (void *)free + trie_object_header_size(); } @@ -1676,8 +1596,10 @@ static void trie_split_free_object_locked(struct stack_depot_trie_free_object *f trie_free_object_tail_locked(tail, tail_size); } -static void trie_retire_object_node_locked(const void *ptr, const void *node, - size_t node_size) +static void +trie_retire_object_node_locked(const void *ptr, + const struct stack_depot_trie_node *node, + size_t node_size) { struct stack_depot_trie_free_object *free; size_t size; @@ -1694,7 +1616,7 @@ static void trie_retire_object_node_locked(const void *ptr, const void *node, size = __stack_depot_trie_pool_alloc_size(node_size); if (node && size >= sizeof(struct stack_depot_trie_free_node) && trie_pool_range_contains_locked(node, size)) { - free->pending_node = (void *)node; + free->pending_node = (struct stack_depot_trie_node *)node; free->pending_node_size = size; free_trie_pending_nodes++; } @@ -1705,7 +1627,8 @@ static void trie_retire_object_node_locked(const void *ptr, const void *node, } } -static void trie_retire_object_node(const void *ptr, const void *node, +static void trie_retire_object_node(const void *ptr, + const struct stack_depot_trie_node *node, size_t node_size) { unsigned long flags; @@ -1887,7 +1810,7 @@ static void trie_pool_release_reused_objects_locked(struct stack_depot_trie_allo mark = &req->txn->pool; completed = get_completed_synchronize_rcu(); for (i = 0; req->node_slots && i < req->nr_node_slots; i++) { - void *node = req->node_slots[i].node; + struct stack_depot_trie_node *node = req->node_slots[i].node; if (node && !trie_pool_mark_contains(mark, node)) { trie_add_free_node_locked(node, req->node_slots[i].size); @@ -1895,7 +1818,7 @@ static void trie_pool_release_reused_objects_locked(struct stack_depot_trie_allo } } for (i = 0; req->child_slots && i < req->nr_child_slots; i++) { - void *array = req->child_slots[i].array; + struct stack_depot_trie_child_array *array = req->child_slots[i].array; if (array && !trie_pool_mark_contains(mark, @@ -2083,13 +2006,6 @@ static void trie_alloc_request_clear_outputs(struct stack_depot_trie_alloc_reque req->child_slots[i].array = NULL; } -static void trie_alloc_request_release_reused_objects(struct stack_depot_trie_alloc_request *req) -{ - if (!req || !req->txn) - return; - trie_pool_release_reused_objects(req); -} - static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req) { int ret; @@ -2165,7 +2081,7 @@ __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, unsigned int nr_entries, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_workspace *workspace, - const void **tail, u32 *leaf_id) + u32 *leaf_id) { struct stack_depot_trie_child_array_slot *child_slots; struct stack_depot_trie_node_slot *node_slots; @@ -2189,8 +2105,7 @@ __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, return __stack_depot_trie_alloc_txn_insert(root, &workspace->req, entries, nr_entries, workspace->scratch, - ARRAY_SIZE(workspace->scratch), - tail, leaf_id); + ARRAY_SIZE(workspace->scratch), leaf_id); } static depot_stack_handle_t @@ -2218,7 +2133,6 @@ trie_save_locked_insert(struct stack_depot_trie_root *root, bool can_insert) { depot_stack_handle_t handle; - const void *tail; u32 leaf_id; int ret; @@ -2226,7 +2140,7 @@ trie_save_locked_insert(struct stack_depot_trie_root *root, if (!handle && can_insert) { ret = __stack_depot_trie_workspace_insert(root, entries, nr_entries, pool_prealloc, side_prealloc, - workspace, &tail, &leaf_id); + workspace, &leaf_id); if (!ret) handle = __stack_depot_trie_handle(leaf_id); } @@ -2335,27 +2249,24 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, struct stack_depot_trie_alloc_request *req, const unsigned long *entries, unsigned int nr_entries, u32 *scratch, - unsigned int nr_scratch, const void **tail, - u32 *leaf_id) + unsigned int nr_scratch, u32 *leaf_id) { struct stack_depot_trie_alloc_txn *txn; - unsigned long flags; + const struct stack_depot_trie_node *tail; u32 id; void *storage; unsigned int nr_used; int ret; - if (!root || !req || !req->txn || !tail || !leaf_id) + if (!root || !req || !req->txn || !leaf_id) return -EINVAL; txn = req->txn; - *tail = NULL; *leaf_id = 0; - - raw_spin_lock_irqsave(&trie_alloc_lock, flags); + lockdep_assert_held(&stack_depot_trie_workspace_lock); ret = __stack_depot_trie_alloc_txn_reserve(req); if (ret) - goto out_unlock; + return ret; storage = req->storage ? *req->storage : NULL; id = txn->leaf_id; @@ -2364,21 +2275,17 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, req->nr_node_slots, req->child_slots, req->nr_child_slots, scratch, nr_scratch, storage, req->storage_size, - &txn->side, tail, &nr_used); + &txn->side, &tail, &nr_used); if (ret) goto rollback; *leaf_id = __stack_depot_trie_alloc_txn_commit(txn); - ret = 0; - goto out_unlock; + return 0; rollback: - trie_alloc_request_release_reused_objects(req); + trie_pool_release_reused_objects(req); __stack_depot_trie_alloc_txn_rollback(req->txn); trie_alloc_request_clear_outputs(req); - *tail = NULL; -out_unlock: - raw_spin_unlock_irqrestore(&trie_alloc_lock, flags); return ret; } @@ -2583,10 +2490,9 @@ int __init stack_depot_early_init(void) stack_depot_disabled = true; return -ENOMEM; } - if (__stack_depot_trie_requested() && stack_depot_trie_init_memblock()) { + if (stack_depot_trie_enabled_param && stack_depot_trie_init_memblock()) { pr_warn("trie storage initialization failed, disabling trie storage\n"); - WRITE_ONCE(stack_depot_trie_requested, false); - __stack_depot_trie_set_enabled(false); + stack_depot_trie_enabled_param = false; } return 0; @@ -2652,12 +2558,11 @@ int stack_depot_init(void) goto out_unlock; } init_trie: - if (!ret && __stack_depot_trie_requested()) { + if (!ret && stack_depot_trie_enabled_param) { ret = stack_depot_trie_init(GFP_KERNEL); if (ret) { pr_warn("trie storage initialization failed, disabling trie storage\n"); - WRITE_ONCE(stack_depot_trie_requested, false); - __stack_depot_trie_set_enabled(false); + stack_depot_trie_enabled_param = false; ret = 0; } } @@ -3044,7 +2949,6 @@ struct stack_depot_hash_save { u32 hash; depot_flags_t depot_flags; void **prealloc; - bool normal_persistent; }; static depot_stack_handle_t @@ -3070,9 +2974,6 @@ depot_save_stack_locked(struct stack_depot_hash_save *save) * readers in find_stack(). */ list_add_rcu(&new->hash_list, save->bucket); - if (save->normal_persistent) - WRITE_ONCE(stack_depot_persistent_hash_record_seen, true); - return new->handle.handle; } @@ -3089,7 +2990,6 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, void *prealloc = NULL; bool allow_spin = gfpflags_allow_spinning(alloc_flags); bool can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && allow_spin; - bool normal_persistent; bool trie_candidate; unsigned long flags; u32 hash; @@ -3109,24 +3009,12 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, if (unlikely(nr_entries == 0) || stack_depot_disabled) return 0; - normal_persistent = !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH)) && - nr_entries <= CONFIG_STACKDEPOT_MAX_FRAMES; + if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) + nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; - trie_candidate = normal_persistent && __stack_depot_trie_ready(); + trie_candidate = !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH)) && + __stack_depot_trie_ready(); if (trie_candidate) { - if (READ_ONCE(stack_depot_persistent_hash_record_seen)) { - /* - * Trie storage may be enabled after stack depot has already saved - * hash records. Preserve the same-handle contract by checking hash - * only when such records may exist. - */ - hash = hash_stack(entries, nr_entries); - bucket = &stack_table[hash & stack_hash_mask]; - found = find_stack(bucket, entries, nr_entries, hash, depot_flags); - if (found) - return found->handle.handle; - } - handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, depot_flags); if (handle) @@ -3144,7 +3032,6 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, .hash = hash, .depot_flags = depot_flags, .prealloc = &prealloc, - .normal_persistent = normal_persistent, }; /* Fast path: look the stack trace up without locking. */ @@ -3350,24 +3237,27 @@ static int stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *array, unsigned long frame, unsigned int *pos, bool *found); -static unsigned int trie_child_array_storage_capacity(size_t storage_size); +static inline unsigned int trie_child_array_storage_capacity(size_t storage_size); static size_t trie_child_array_size_for_capacity(unsigned int capacity); static bool trie_parent_chain_matches_prefix(const struct stack_depot_trie_node *node, const unsigned long *entries, unsigned int nr_entries); -static size_t stack_depot_frame_run_entry_bytes(enum stack_depot_frame_mode mode) +static inline size_t stack_depot_frame_run_entry_bytes(enum stack_depot_frame_mode mode) { if (mode == STACK_DEPOT_FRAME_COMPRESSED) return sizeof(u32); return sizeof(unsigned long); } -static int stack_depot_frame_run_validate(const struct stack_depot_frame_run *run) +static inline size_t stack_depot_frame_run_bytes(const struct stack_depot_frame_run *run) { - size_t bytes; + return run->nr_entries * stack_depot_frame_run_entry_bytes(run->mode); +} +static int stack_depot_frame_run_validate(const struct stack_depot_frame_run *run) +{ if (!run || !run->nr_entries || run->nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) return -EINVAL; @@ -3380,10 +3270,6 @@ static int stack_depot_frame_run_validate(const struct stack_depot_frame_run *ru return -EINVAL; } - bytes = run->nr_entries * stack_depot_frame_run_entry_bytes(run->mode); - if (run->bytes != bytes) - return -EINVAL; - return 0; } @@ -3420,7 +3306,6 @@ static int frame_run_init_lows(const unsigned long *entries, /* @i is the first non-matching frame, or @nr_entries if all matched. */ run->mode = compressed ? STACK_DEPOT_FRAME_COMPRESSED : STACK_DEPOT_FRAME_RAW; run->nr_entries = i; - run->bytes = i * stack_depot_frame_run_entry_bytes(run->mode); return 0; } @@ -3479,7 +3364,7 @@ static size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *r if (stack_depot_frame_run_validate(run)) return 0; size = offsetof(struct stack_depot_trie_node, data); - if (check_add_overflow(size, run->bytes, &size)) + if (check_add_overflow(size, stack_depot_frame_run_bytes(run), &size)) return 0; return ALIGN(size, sizeof(unsigned long)); @@ -3496,7 +3381,6 @@ static int stack_depot_frame_run_slice(const struct stack_depot_frame_run *src, *run = *src; run->nr_entries = nr_entries; - run->bytes = nr_entries * stack_depot_frame_run_entry_bytes(run->mode); return 0; } @@ -3530,9 +3414,33 @@ stack_depot_trie_node_first_frame(const struct stack_depot_trie_node *node, return stack_depot_trie_node_frame(node, 0, frame); } +static int +trie_node_stack_len(const struct stack_depot_trie_node *node, + unsigned int *stack_len) +{ + unsigned int depth = 0; + unsigned int total = 0; + + if (!node || !stack_len) + return -EINVAL; + + for (; node; node = trie_load_parent(node)) { + if (depth++ >= CONFIG_STACKDEPOT_MAX_FRAMES) + return -EINVAL; + if (stack_depot_frame_run_validate(&node->run)) + return -EINVAL; + if (total > CONFIG_STACKDEPOT_MAX_FRAMES - node->run.nr_entries) + return -EINVAL; + total += node->run.nr_entries; + } + + *stack_len = total; + return 0; +} + static int __stack_depot_trie_node_init(void *storage, size_t storage_size, - const void *parent, u32 leaf_id, + const struct stack_depot_trie_node *parent, u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, u32 *scratch, unsigned int nr_scratch) @@ -3540,7 +3448,7 @@ __stack_depot_trie_node_init(void *storage, size_t storage_size, const struct stack_depot_trie_node *parent_node = parent; struct stack_depot_trie_node *node = storage; struct stack_depot_frame_run run; - u32 stack_len; + unsigned int parent_len = 0; int ret; if (!node || !entries) @@ -3560,35 +3468,31 @@ __stack_depot_trie_node_init(void *storage, size_t storage_size, (!scratch || nr_scratch < run.nr_entries)) return -EINVAL; if (parent_node) { - if (!parent_node->stack_len || - parent_node->stack_len > U32_MAX - run.nr_entries || - parent_node->stack_len > - CONFIG_STACKDEPOT_MAX_FRAMES - run.nr_entries) + if (trie_node_stack_len(parent_node, &parent_len) || + parent_len > CONFIG_STACKDEPOT_MAX_FRAMES - run.nr_entries) return -EINVAL; - stack_len = parent_node->stack_len + run.nr_entries; - } else { - stack_len = run.nr_entries; } /* Caller-owned storage is not publishable unless the payload write succeeds. */ if (run.mode == STACK_DEPOT_FRAME_COMPRESSED) /* Copy low-bit payloads staged by frame_run_init_lows(). */ - memcpy(node->data, scratch, run.bytes); + memcpy(node->data, scratch, stack_depot_frame_run_bytes(&run)); else - memcpy(node->data, entries, run.bytes); + memcpy(node->data, entries, stack_depot_frame_run_bytes(&run)); node->parent = parent_node; node->children = NULL; node->leaf_id = leaf_id; - node->stack_len = stack_len; node->run = run; return 0; } static int __stack_depot_trie_node_init_slice(void *storage, size_t storage_size, - const void *parent, u32 leaf_id, - const void *src_node, unsigned int start, + const struct stack_depot_trie_node *parent, + u32 leaf_id, + const struct stack_depot_trie_node *src_node, + unsigned int start, unsigned int nr_entries) { const struct stack_depot_trie_node *parent_node = parent; @@ -3597,10 +3501,10 @@ __stack_depot_trie_node_init_slice(void *storage, size_t storage_size, struct stack_depot_frame_run run; size_t entry_bytes; size_t src_size; - u32 stack_len; + unsigned int parent_len = 0; int ret; - if (!node || !src || !src->stack_len) + if (!node || !src) return -EINVAL; if (!IS_ALIGNED((unsigned long)node, __alignof__(*node))) return -EINVAL; @@ -3614,32 +3518,26 @@ __stack_depot_trie_node_init_slice(void *storage, size_t storage_size, if (!src_size || stack_depot_ranges_overlap(node, storage_size, src, src_size)) return -EINVAL; if (parent_node) { - if (!parent_node->stack_len || - parent_node->stack_len > U32_MAX - run.nr_entries || - parent_node->stack_len > - CONFIG_STACKDEPOT_MAX_FRAMES - run.nr_entries) + if (trie_node_stack_len(parent_node, &parent_len) || + parent_len > CONFIG_STACKDEPOT_MAX_FRAMES - run.nr_entries) return -EINVAL; - stack_len = parent_node->stack_len + run.nr_entries; - } else { - stack_len = run.nr_entries; } entry_bytes = stack_depot_frame_run_entry_bytes(src->run.mode); - memcpy(node->data, src->data + start * entry_bytes, run.bytes); + memcpy(node->data, src->data + start * entry_bytes, + stack_depot_frame_run_bytes(&run)); node->parent = parent_node; node->children = NULL; node->leaf_id = leaf_id; - node->stack_len = stack_len; node->run = run; return 0; } static unsigned int -__stack_depot_trie_node_match(const void *node_ptr, +__stack_depot_trie_node_match(const struct stack_depot_trie_node *node, const unsigned long *entries, unsigned int nr_entries) { - const struct stack_depot_trie_node *node = node_ptr; unsigned int limit; unsigned int i; @@ -4201,7 +4099,7 @@ trie_clone_promoted_node(const struct stack_depot_trie_node *old_node, if (!old_node || !leaf_id || !slot || !slot->node) return -EINVAL; - if (old_node->leaf_id || !old_node->stack_len) + if (old_node->leaf_id) return -EINVAL; if (stack_depot_frame_run_validate(&old_node->run)) return -EINVAL; @@ -4346,11 +4244,11 @@ static int trie_append_chain_validate(const struct stack_depot_trie_node *parent unsigned int child_slots_needed; unsigned int pos = 0; unsigned int used = 0; - u32 stack_len = parent ? parent->stack_len : 0; + unsigned int stack_len = 0; if (!entries || !nr_entries || !node_slots || !nr_runs) return -EINVAL; - if (parent && !parent->stack_len) + if (parent && trie_node_stack_len(parent, &stack_len)) return -EINVAL; while (pos < nr_entries) { @@ -4369,8 +4267,7 @@ static int trie_append_chain_validate(const struct stack_depot_trie_node *parent if (run.mode == STACK_DEPOT_FRAME_COMPRESSED && (!scratch || nr_scratch < run.nr_entries)) return -EINVAL; - if (stack_len > U32_MAX - run.nr_entries || - stack_len > CONFIG_STACKDEPOT_MAX_FRAMES - run.nr_entries) + if (stack_len > CONFIG_STACKDEPOT_MAX_FRAMES - run.nr_entries) return -EINVAL; size = __stack_depot_trie_node_size(&run); @@ -4423,17 +4320,19 @@ static int trie_append_chain_validate(const struct stack_depot_trie_node *parent } static int -__stack_depot_trie_append_chain(const void *parent_ptr, u32 leaf_id, +__stack_depot_trie_append_chain(const struct stack_depot_trie_node *parent, + u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, const struct stack_depot_trie_node_slot *node_slots, unsigned int nr_node_slots, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, const void **head, - const void **tail, unsigned int *nr_used) + unsigned int nr_scratch, + const struct stack_depot_trie_node **head, + const struct stack_depot_trie_node **tail, + unsigned int *nr_used) { - const struct stack_depot_trie_node *parent = parent_ptr; const struct stack_depot_trie_node *prev = parent; unsigned int pos = 0; unsigned int used; @@ -4465,9 +4364,9 @@ __stack_depot_trie_append_chain(const void *parent_ptr, u32 leaf_id, } for (i = 0; i + 1 < used; i++) { - const void *next = node_slots[i + 1].node; + const struct stack_depot_trie_node *next = node_slots[i + 1].node; struct stack_depot_trie_node *node = node_slots[i].node; - void *array = child_slots[i].array; + struct stack_depot_trie_child_array *array = child_slots[i].array; size_t size = child_slots[i].size; if (__stack_depot_trie_child_array_insert(NULL, next, array, size)) @@ -4482,16 +4381,15 @@ __stack_depot_trie_append_chain(const void *parent_ptr, u32 leaf_id, } static int trie_publish_append_prepare(struct stack_depot_trie_root *root, - void *parent_ptr, const void *head_ptr, + struct stack_depot_trie_node *parent, + const struct stack_depot_trie_node *head, void *new_storage, size_t new_storage_size, struct stack_depot_trie_side_prepare *side, u32 leaf_id, - const void *leaf) + const struct stack_depot_trie_node *leaf) { const struct stack_depot_trie_child_array *old_array; - const struct stack_depot_trie_node *head = head_ptr; const struct stack_depot_trie_child_array **slot; - struct stack_depot_trie_node *parent = parent_ptr; struct stack_depot_trie_child_array *new_array = new_storage; size_t storage_size = new_storage_size; size_t new_size; @@ -4585,12 +4483,12 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, static int __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, - const void *parent_ptr, const unsigned long *entries, + const struct stack_depot_trie_node *parent, + const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_lookup *lookup) { const struct stack_depot_trie_child_array *children; - const struct stack_depot_trie_node *parent = parent_ptr; const struct stack_depot_trie_node *node; struct stack_depot_trie_lookup tmp; unsigned int matched; @@ -4653,7 +4551,7 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, return 0; } -static const void * +static const struct stack_depot_trie_node * __stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries) { @@ -4675,9 +4573,11 @@ __stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, node = lookup.node; if (node) { const struct stack_depot_trie_node *node_parent; + unsigned int node_len; node_parent = trie_load_parent(node); - if (node->stack_len != pos + lookup.matched) + if (trie_node_stack_len(node, &node_len) || + node_len != pos + lookup.matched) return NULL; if (node_parent != parent && !trie_parent_chain_matches_prefix(node_parent, entries, @@ -4720,12 +4620,13 @@ static int trie_split_child(struct stack_depot_trie_root *root, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, struct stack_depot_trie_side_prepare *side, - const void **tail, + const struct stack_depot_trie_node **tail, unsigned int *nr_used); static int __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, - void *parent_ptr, u32 leaf_id, + struct stack_depot_trie_node *parent, + u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, const struct stack_depot_trie_node_slot *node_slots, @@ -4736,13 +4637,12 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, struct stack_depot_trie_side_prepare *side, - const void **tail, + const struct stack_depot_trie_node **tail, unsigned int *nr_used) { - const void *head; - const void *last; + const struct stack_depot_trie_node *head; + const struct stack_depot_trie_node *last; unsigned int used; - struct stack_depot_trie_node *parent = parent_ptr; struct stack_depot_trie_lookup lookup; int ret; @@ -4874,22 +4774,19 @@ static bool trie_node_depth_invalid(const struct stack_depot_trie_node *parent, const struct stack_depot_trie_node *node) { const struct stack_depot_trie_node *node_parent; - u32 base = parent ? parent->stack_len : 0; + unsigned int base = 0; - if (!node || !node->stack_len) + if (!node) return true; node_parent = trie_load_parent(node); if (node_parent != parent) return true; - if (parent && !parent->stack_len) - return true; if (stack_depot_frame_run_validate(&node->run)) return true; - if (node->run.nr_entries > U32_MAX - base || - base > CONFIG_STACKDEPOT_MAX_FRAMES - node->run.nr_entries) + if (parent && trie_node_stack_len(parent, &base)) return true; - return node->stack_len != base + node->run.nr_entries; + return base > CONFIG_STACKDEPOT_MAX_FRAMES - node->run.nr_entries; } static bool trie_node_chain_depth_invalid(const struct stack_depot_trie_node *node) @@ -4918,48 +4815,36 @@ trie_parent_chain_matches_prefix(const struct stack_depot_trie_node *node, { const struct stack_depot_trie_node *cur; unsigned int depth = 0; + unsigned int pos; unsigned int i; if (!node) return nr_entries == 0; - if (!entries || node->stack_len != nr_entries) + if (!entries || trie_node_stack_len(node, &pos) || pos != nr_entries) return false; cur = node; while (cur) { - const struct stack_depot_trie_node *parent; - unsigned int start; - if (depth++ >= CONFIG_STACKDEPOT_MAX_FRAMES) return false; - parent = trie_load_parent(cur); if (stack_depot_frame_run_validate(&cur->run)) return false; - if (!cur->stack_len || cur->run.nr_entries > cur->stack_len) + if (cur->run.nr_entries > pos) return false; - if (parent) { - if (!parent->stack_len || - parent->stack_len > U32_MAX - cur->run.nr_entries) - return false; - if (cur->stack_len != parent->stack_len + cur->run.nr_entries) - return false; - } else if (cur->stack_len != cur->run.nr_entries) { - return false; - } + pos -= cur->run.nr_entries; - start = cur->stack_len - cur->run.nr_entries; for (i = 0; i < cur->run.nr_entries; i++) { unsigned long frame; if (stack_depot_trie_node_frame(cur, i, &frame) || - frame != entries[start + i]) + frame != entries[pos + i]) return false; } - cur = parent; + cur = trie_load_parent(cur); } - return true; + return !pos; } static int trie_plan_split(const struct stack_depot_trie_child_array *children, @@ -4997,7 +4882,12 @@ static int trie_plan_split(const struct stack_depot_trie_child_array *children, return -EINVAL; has_new_tail = matched < nr_entries; - prefix_stack_len = child->parent ? child->parent->stack_len : 0; + if (child->parent) { + if (trie_node_stack_len(child->parent, &prefix_stack_len)) + return -EINVAL; + } else { + prefix_stack_len = 0; + } if (prefix_stack_len > CONFIG_STACKDEPOT_MAX_FRAMES - matched) return -EINVAL; prefix_stack_len += matched; @@ -5031,7 +4921,8 @@ static int trie_plan_split(const struct stack_depot_trie_child_array *children, static int __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, - const void *parent_ptr, const unsigned long *entries, + const struct stack_depot_trie_node *parent, + const unsigned long *entries, unsigned int nr_entries, struct stack_depot_trie_node_slot *node_slots, unsigned int nr_node_slots, @@ -5039,10 +4930,10 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, unsigned int nr_child_slots, size_t *new_storage_size, unsigned int *nr_used, unsigned int *nr_child_used) { - const struct stack_depot_trie_node *parent = parent_ptr; const struct stack_depot_trie_child_array *children; const struct stack_depot_trie_node *child; unsigned int nr_children; + unsigned int parent_len; unsigned int matched; unsigned int pos; bool found; @@ -5056,12 +4947,14 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, for (;;) { if (parent && trie_node_chain_depth_invalid(parent)) return -EINVAL; + if (parent && trie_node_stack_len(parent, &parent_len)) + return -EINVAL; if (root) children = trie_load_children_slot(&root->children); else children = trie_load_children_slot(&parent->children); if (!children) { - if (trie_plan_append_chain(parent ? parent->stack_len : 0, + if (trie_plan_append_chain(parent ? parent_len : 0, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, nr_used, @@ -5074,7 +4967,7 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, &found)) return -EINVAL; if (!found) { - if (trie_plan_append_chain(parent ? parent->stack_len : 0, + if (trie_plan_append_chain(parent ? parent_len : 0, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, nr_used, @@ -5124,7 +5017,7 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, } } -static unsigned int trie_validate_leaf(const void *leaf, +static unsigned int trie_validate_leaf(const struct stack_depot_trie_node *leaf, const unsigned long *entries) { const struct stack_depot_trie_node *node = leaf; @@ -5132,23 +5025,26 @@ static unsigned int trie_validate_leaf(const void *leaf, unsigned int pos; unsigned int total; - if (!node || !node->stack_len || !node->leaf_id) + if (!node || !node->leaf_id) + return 0; + if (trie_node_stack_len(node, &total)) return 0; - total = node->stack_len; entries_size = total * sizeof(*entries); pos = total; for (node = leaf; node; node = trie_load_parent(node)) { + size_t node_size; bool overlap; if (frame_run_validate_payload(&node->run, node->data)) return 0; + node_size = stack_depot_frame_run_bytes(&node->run); overlap = entries && stack_depot_ranges_overlap(entries, entries_size, node->data, - node->run.bytes); + node_size); if (overlap) return 0; - if (node->stack_len != pos || node->run.nr_entries > pos) + if (node->run.nr_entries > pos) return 0; pos -= node->run.nr_entries; } @@ -5156,13 +5052,15 @@ static unsigned int trie_validate_leaf(const void *leaf, return pos ? 0 : total; } -static unsigned int trie_walk_frames(const void *leaf, unsigned int total, +static unsigned int trie_walk_frames(const struct stack_depot_trie_node *leaf, + unsigned int total, void (*fn)(unsigned int index, unsigned long frame, void *data), void *data) { const struct stack_depot_trie_node *node; + unsigned int pos = total; unsigned int seen = 0; unsigned int i; @@ -5170,48 +5068,48 @@ static unsigned int trie_walk_frames(const void *leaf, unsigned int total, return 0; for (node = leaf; node; node = trie_load_parent(node)) { - unsigned int start; - - if (node->run.nr_entries > node->stack_len) + if (node->run.nr_entries > pos) return 0; - start = node->stack_len - node->run.nr_entries; + pos -= node->run.nr_entries; for (i = 0; i < node->run.nr_entries; i++) { unsigned long frame; if (stack_depot_trie_node_frame(node, i, &frame)) return 0; - fn(start + i, frame, data); + fn(pos + i, frame, data); seen++; } } - return seen == total ? total : 0; + return seen == total && !pos ? total : 0; } -static int trie_frame_at(const void *leaf, unsigned int index, +static int trie_frame_at(const struct stack_depot_trie_node *leaf, + unsigned int index, unsigned long *frame) { const struct stack_depot_trie_node *node; + unsigned int pos; if (!leaf || !frame) return -EINVAL; + if (trie_node_stack_len(leaf, &pos)) + return -EINVAL; for (node = leaf; node; node = trie_load_parent(node)) { - unsigned int start; - - if (node->run.nr_entries > node->stack_len) + if (node->run.nr_entries > pos) return -EINVAL; - start = node->stack_len - node->run.nr_entries; - if (index < start || index >= node->stack_len) + pos -= node->run.nr_entries; + if (index < pos || index >= pos + node->run.nr_entries) continue; - return stack_depot_trie_node_frame(node, index - start, frame); + return stack_depot_trie_node_frame(node, index - pos, frame); } return -EINVAL; } static unsigned int trie_handle_leaf(depot_stack_handle_t handle, - const void **leaf) + const struct stack_depot_trie_node **leaf) { u32 leaf_id; @@ -5227,7 +5125,8 @@ static unsigned int trie_handle_leaf(depot_stack_handle_t handle, return trie_validate_leaf(*leaf, NULL); } -static void trie_print_frames(const void *leaf, unsigned int nr_entries, +static void trie_print_frames(const struct stack_depot_trie_node *leaf, + unsigned int nr_entries, int spaces) { unsigned int i; @@ -5244,7 +5143,7 @@ static void trie_print_frames(const void *leaf, unsigned int nr_entries, static unsigned int trie_print_handle(depot_stack_handle_t handle, int spaces) { unsigned int nr_entries; - const void *leaf; + const struct stack_depot_trie_node *leaf; rcu_read_lock_sched_notrace(); nr_entries = trie_handle_leaf(handle, &leaf); @@ -5256,7 +5155,8 @@ static unsigned int trie_print_handle(depot_stack_handle_t handle, int spaces) } static int -trie_snprint_frames(char *buf, size_t size, const void *leaf, +trie_snprint_frames(char *buf, size_t size, + const struct stack_depot_trie_node *leaf, unsigned int nr_entries, int spaces) { int generated; @@ -5292,7 +5192,7 @@ trie_snprint_handle(depot_stack_handle_t handle, char *buf, size_t size, int spaces) { unsigned int nr_entries; - const void *leaf; + const struct stack_depot_trie_node *leaf; int ret = 0; rcu_read_lock_sched_notrace(); @@ -5312,7 +5212,8 @@ static void trie_fetch_frame(unsigned int index, unsigned long frame, void *data } static unsigned int -__stack_depot_trie_fetch_into(const void *leaf, unsigned long *entries, +__stack_depot_trie_fetch_into(const struct stack_depot_trie_node *leaf, + unsigned long *entries, unsigned int max_entries) { unsigned int total; @@ -5337,7 +5238,7 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, unsigned long *entries, unsigned int max_entries) { - const void *leaf; + const struct stack_depot_trie_node *leaf; u32 leaf_id; unsigned int nr_entries; @@ -5360,14 +5261,14 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, return nr_entries; } -static unsigned int trie_child_array_capacity(unsigned int nr_children) +static inline unsigned int trie_child_array_capacity(unsigned int nr_children) { if (!nr_children) return 0; return roundup_pow_of_two(nr_children); } -static unsigned int trie_child_array_storage_capacity(size_t storage_size) +static inline unsigned int trie_child_array_storage_capacity(size_t storage_size) { if (storage_size < sizeof(struct stack_depot_trie_child_array)) return 0; @@ -5387,7 +5288,7 @@ static size_t trie_child_array_size_for_capacity(unsigned int capacity) return ALIGN(size, sizeof(unsigned long)); } -static size_t __stack_depot_trie_child_array_size(unsigned int nr_children) +static inline size_t __stack_depot_trie_child_array_size(unsigned int nr_children) { unsigned int capacity = trie_child_array_capacity(nr_children); @@ -5396,12 +5297,10 @@ static size_t __stack_depot_trie_child_array_size(unsigned int nr_children) static int __stack_depot_trie_child_array_init(void *storage, size_t storage_size, - const void * const *children, + const struct stack_depot_trie_node * const *nodes, unsigned int nr_children) { struct stack_depot_trie_child_array *array = storage; - const struct stack_depot_trie_node * const *nodes = - (const struct stack_depot_trie_node * const *)children; unsigned int capacity; unsigned long last = 0; unsigned int i; @@ -5445,10 +5344,10 @@ __stack_depot_trie_child_array_init(void *storage, size_t storage_size, static int __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, - const void *old_tail, - const void *new_head) + const struct stack_depot_trie_node *old_tail, + const struct stack_depot_trie_node *new_head) { - const void *children[2]; + const struct stack_depot_trie_node *children[2]; unsigned long new_frame; unsigned long old_frame; @@ -5537,14 +5436,13 @@ __stack_depot_trie_split_tail_plan(const unsigned long *entries, static int __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, - const void *parent_ptr, + struct stack_depot_trie_node *parent, const struct stack_depot_trie_node_slot *node_slots, unsigned int nr_node_slots, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, void *new_storage, size_t new_storage_size) { - struct stack_depot_trie_node *parent = (void *)parent_ptr; const struct stack_depot_trie_child_array **slot; const struct stack_depot_trie_child_array *children; unsigned int i; @@ -5575,7 +5473,7 @@ __stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, for (i = 0; i < nr_child_slots; i++) { const struct stack_depot_trie_child_array_slot *child_slot = &child_slots[i]; - void *array = child_slot->array; + struct stack_depot_trie_child_array *array = child_slot->array; size_t slot_size = child_slot->size; if (!array || !slot_size) @@ -5754,7 +5652,8 @@ trie_split_subtree_precheck(const struct stack_depot_trie_node *child, return 0; } -static int trie_split_subtree_prepare(const void *child_ptr, unsigned int matched, +static int trie_split_subtree_prepare(const struct stack_depot_trie_node *child, + unsigned int matched, u32 leaf_id, const unsigned long *entries, unsigned int nr_entries, const struct stack_depot_trie_node_slot *node_slots, @@ -5763,13 +5662,13 @@ static int trie_split_subtree_prepare(const void *child_ptr, unsigned int matche unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, struct stack_depot_trie_side_prepare *side, - const void **prefix, - const void **tail, unsigned int *nr_used) + const struct stack_depot_trie_node **prefix, + const struct stack_depot_trie_node **tail, + unsigned int *nr_used) { - const struct stack_depot_trie_node *child = child_ptr; const unsigned long *tail_entries; - const void *new_head = NULL; - const void *new_tail = NULL; + const struct stack_depot_trie_node *new_head = NULL; + const struct stack_depot_trie_node *new_tail = NULL; struct stack_depot_trie_leaf_update updates[2]; struct stack_depot_trie_node *old_tail; struct stack_depot_trie_node *pref; @@ -5863,12 +5762,12 @@ static int trie_split_child(struct stack_depot_trie_root *root, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, struct stack_depot_trie_side_prepare *side, - const void **tail, + const struct stack_depot_trie_node **tail, unsigned int *nr_used) { const struct stack_depot_trie_child_array **publish_slot; const struct stack_depot_trie_child_array *old_array; - const void *prefix; + const struct stack_depot_trie_node *prefix; void *storage = new_storage; size_t child_size; size_t storage_size = new_storage_size; @@ -5958,12 +5857,11 @@ stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *ar } static int -__stack_depot_trie_child_array_insert(const void *old_storage, const void *child, +__stack_depot_trie_child_array_insert(const struct stack_depot_trie_child_array *old, + const struct stack_depot_trie_node *node, void *new_storage, size_t new_storage_size) { - const struct stack_depot_trie_child_array *old_array = old_storage; struct stack_depot_trie_child_array *new_array = new_storage; - const struct stack_depot_trie_node *node = child; unsigned int nr_old; unsigned int pos; unsigned int i; @@ -5984,22 +5882,22 @@ __stack_depot_trie_child_array_insert(const void *old_storage, const void *child return -EINVAL; if (!frame) return -EINVAL; - if (old_array == new_array) + if (old == new_array) return -EINVAL; - if (old_array && !IS_ALIGNED((unsigned long)old_array, __alignof__(*old_array))) + if (old && !IS_ALIGNED((unsigned long)old, __alignof__(*old))) return -EINVAL; - nr_old = old_array ? old_array->nr_children : 0; + nr_old = old ? old->nr_children : 0; if (new_storage_size < __stack_depot_trie_child_array_size(nr_old + 1)) return -EINVAL; - old_size = old_array ? trie_child_array_size_for_capacity(old_array->capacity) : 0; - overlaps = old_array && stack_depot_ranges_overlap(old_array, old_size, - new_array, new_storage_size); + old_size = old ? trie_child_array_size_for_capacity(old->capacity) : 0; + overlaps = old && stack_depot_ranges_overlap(old, old_size, new_array, + new_storage_size); if (overlaps) return -EINVAL; - if (old_array) { - if (stack_depot_trie_child_lower_bound(old_array, frame, &pos, + if (old) { + if (stack_depot_trie_child_lower_bound(old, frame, &pos, &found)) return -EINVAL; if (found) @@ -6011,14 +5909,14 @@ __stack_depot_trie_child_array_insert(const void *old_storage, const void *child new_array->nr_children = nr_old + 1; new_array->capacity = trie_child_array_storage_capacity(new_storage_size); - if (old_array) { + if (old) { for (i = 0; i < pos; i++) - new_array->children[i] = old_array->children[i]; + new_array->children[i] = old->children[i]; } new_array->children[pos] = node; - if (old_array) { + if (old) { for (i = pos; i < nr_old; i++) - new_array->children[i + 1] = old_array->children[i]; + new_array->children[i + 1] = old->children[i]; } for (i = nr_old + 1; i < new_array->capacity; i++) new_array->children[i] = NULL; From 891506d310c288454919b409a779a63c4a053b66 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Tue, 23 Jun 2026 21:25:17 +0100 Subject: [PATCH 120/129] KRN-1117: Test stackdepot overlong stack truncation Cover the overlong stack path after truncating saves before backend routing. Verify that an overlong stack fetches as CONFIG_STACKDEPOT_MAX_FRAMES frames and that saving the already-truncated prefix returns the same handle. Signed-off-by: Caleb Kan --- lib/tests/stackdepot_kunit.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index f2614acead548..8a51142783553 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -137,22 +137,29 @@ static void stackdepot_save_flags_public(struct kunit *test) unsigned long noalloc_entries[] = { 0x701000UL, 0x702000UL }; unsigned long fetched[ARRAY_SIZE(entries)] = {}; depot_stack_handle_t noalloc_handle; + depot_stack_handle_t truncated_handle; depot_stack_handle_t overlong_handle; depot_stack_handle_t hash_handle; depot_stack_handle_t get_handle; depot_stack_handle_t again; depot_stack_handle_t extra; depot_flags_t flags; + unsigned long *overlong_fetched; unsigned long *overlong_entries; unsigned int noalloc_nr = ARRAY_SIZE(noalloc_entries); unsigned int overlong_nr = CONFIG_STACKDEPOT_MAX_FRAMES + 1; + unsigned int truncated_nr = CONFIG_STACKDEPOT_MAX_FRAMES; unsigned int nr_entries; + size_t overlong_size; unsigned int i; KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); overlong_entries = kunit_kcalloc(test, overlong_nr, sizeof(*overlong_entries), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, overlong_entries); + overlong_fetched = kunit_kcalloc(test, CONFIG_STACKDEPOT_MAX_FRAMES, + sizeof(*overlong_fetched), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, overlong_fetched); for (i = 0; i < overlong_nr; i++) overlong_entries[i] = 0x800000UL + i * 0x1000UL; @@ -193,6 +200,13 @@ static void stackdepot_save_flags_public(struct kunit *test) overlong_handle = stack_depot_save(overlong_entries, overlong_nr, GFP_KERNEL); KUNIT_ASSERT_NE(test, overlong_handle, (depot_stack_handle_t)0); + nr_entries = stack_depot_fetch_into(overlong_handle, overlong_fetched, + CONFIG_STACKDEPOT_MAX_FRAMES); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)CONFIG_STACKDEPOT_MAX_FRAMES); + overlong_size = CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(*overlong_entries); + KUNIT_EXPECT_MEMEQ(test, overlong_fetched, overlong_entries, overlong_size); + truncated_handle = stack_depot_save(overlong_entries, truncated_nr, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, truncated_handle, overlong_handle); extra = stack_depot_set_extra_bits(hash_handle, 7); KUNIT_ASSERT_NE(test, extra, (depot_stack_handle_t)0); From 2858c848304cdfe82bb11637305dfa1e2b5e7022 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 24 Jun 2026 14:03:18 +0100 Subject: [PATCH 121/129] KRN-1117: Tighten stackdepot trie save semantics Make trie saves and count-helper use fail at clearer backend boundaries. The trie path now prepares side-table IDs without committing them until publication succeeds, clamps overlong stacks like the hash backend, and skips locked insertion work when insertion resources are unavailable. Keep hash records for page_owner accounting distinct from refcounted records so count helpers cannot mutate tag-KASAN references, and make internal count-helper misuse warn instead of being silently ignored. Fold helper layers that no longer carry distinct behavior after these changes, and use the page_owner list lock instead of release/acquire publication. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 35 +-- lib/stackdepot.c | 563 ++++++++++++----------------------- lib/tests/stackdepot_kunit.c | 15 +- mm/page_owner.c | 22 +- 4 files changed, 228 insertions(+), 407 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 7e6bee257566b..444498c5a805b 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -47,10 +47,10 @@ typedef u32 depot_flags_t; */ #define STACK_DEPOT_FLAG_CAN_ALLOC ((depot_flags_t)0x0001) #define STACK_DEPOT_FLAG_GET ((depot_flags_t)0x0002) -#define STACK_DEPOT_FLAG_HASH ((depot_flags_t)0x0004) +#define STACK_DEPOT_FLAG_COUNTABLE ((depot_flags_t)0x0004) #define STACK_DEPOT_FLAGS_MASK (STACK_DEPOT_FLAG_CAN_ALLOC | \ - STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH) + STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE) /* * Using stack depot requires its initialization, which can be done in 3 ways: @@ -108,10 +108,10 @@ static inline int stack_depot_early_init(void) { return 0; } * Users of this flag must also call stack_depot_put() when keeping the stack * trace is no longer required to avoid overflowing the refcount. * - * If STACK_DEPOT_FLAG_HASH is set in @depot_flags, stack depot stores the stack - * trace in legacy hash storage even when trie storage is enabled. This is for - * internal callers that depend on stackdepot count helpers. This flag does not - * imply %STACK_DEPOT_FLAG_CAN_ALLOC. + * If STACK_DEPOT_FLAG_COUNTABLE is set in @depot_flags, stack depot stores the + * stack in a distinct hash-backed record mode that supports the internal count + * helpers. This flag does not imply %STACK_DEPOT_FLAG_CAN_ALLOC and is mutually + * exclusive with %STACK_DEPOT_FLAG_GET. * * When trie storage is enabled, persistent non-refcounted saves use trie * storage. Constrained contexts remain best effort and can return 0 if a @@ -158,9 +158,9 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, * * This function is only for internal purposes. * The returned count is an unsynchronized snapshot for diagnostics. + * @handle must be hash-backed and @count must be valid. * - * Return: true on success, false if @handle is invalid, @count is NULL, or the - * stack record is not in counted mode. + * Return: true on success, false if the stack record is not in counted mode. */ bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count); @@ -171,8 +171,8 @@ bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count); * @count: Count to set * * This function is only for internal purposes. - * If @handle is invalid, @count is 0, or @count is greater than %INT_MAX, - * this function is a no-op. + * @handle must be hash-backed, and @count must be greater than 0 and less than + * or equal to %INT_MAX. * Callers that use this to switch a saturated record to counted mode must * separately make the record discoverable by their own tracking structure. * Callers must have exclusive access to the stack record count. @@ -188,8 +188,8 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); * counted increment * * This function is only for internal purposes. - * If @count is 0, this function is a no-op. Otherwise @count must be less - * than or equal to %INT_MAX - 1 so the saturated-to-counted transition can + * @handle must be hash-backed. @count must be greater than 0 and less than or + * equal to %INT_MAX - 1 so the saturated-to-counted transition can * store the stack_list marker plus @count without overflowing. * * Persistent stack records start with refcount set to %REFCOUNT_SATURATED. If @@ -214,13 +214,14 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, * @count: Count to subtract * * This function is only for internal purposes. - * @count must be greater than 0 and less than or equal to %INT_MAX. + * @handle must be hash-backed. @count must be greater than 0 and less than or + * equal to %INT_MAX. * * Return: true if the resulting count is 0, false if the resulting count is - * non-zero, @handle is invalid, the stack record is not in counted mode, or - * @count is greater than the current count. Saturated persistent records are - * not in counted mode and fail closed without changing the record. Underflow - * attempts warn and leave the count unchanged. + * non-zero, the stack record is not in counted mode, or @count is greater than + * the current count. Saturated persistent records are not in counted mode and + * fail closed without changing the record. Underflow attempts warn and leave the + * count unchanged. */ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, unsigned int count); diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 314754cb50fbb..877af6b995ec3 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -3,7 +3,7 @@ * Stack depot - a stack trace storage that avoids duplication. * * Internally, stack depot has two storage backends. Refcounted entries and - * callers that request STACK_DEPOT_FLAG_HASH use the legacy hash table with + * callers that request STACK_DEPOT_FLAG_COUNTABLE use the legacy hash table with * contiguous stack records in stack pools. Persistent non-refcounted entries * can use trie storage when enabled; trie nodes share common frame prefixes and * are published through RCU/COW child arrays. @@ -110,16 +110,6 @@ struct stack_depot_trie_leaf_update { #define STACK_DEPOT_TRIE_MAX_NODE_SLOTS (CONFIG_STACKDEPOT_MAX_FRAMES + 1) #define STACK_DEPOT_TRIE_MAX_CHILD_SLOTS CONFIG_STACKDEPOT_MAX_FRAMES -struct stack_depot_trie_side_checkpoint { - u32 leaf_id; - const struct stack_depot_trie_node *old_leaf; -}; - -struct stack_depot_trie_side_prepare { - struct stack_depot_trie_side_checkpoint updates[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; - unsigned int nr_updates; -}; - struct stack_depot_trie_side_prealloc { /* Preallocated side-table directory page for sparse growth. */ struct stack_depot_trie_side_dir *dir; @@ -138,7 +128,6 @@ struct stack_depot_trie_pool_mark { }; struct stack_depot_trie_alloc_txn { - struct stack_depot_trie_side_prepare side; struct stack_depot_trie_pool_mark pool; u32 leaf_id; }; @@ -181,16 +170,12 @@ static int __stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, static void __stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_prealloc *prealloc); static u32 -__stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc); -static void __stack_depot_trie_side_table_revoke_latest(u32 id); -static void -__stack_depot_trie_side_table_restore(u32 id, - const struct stack_depot_trie_node *entry); +__stack_depot_trie_side_table_prepare_id(struct stack_depot_trie_side_prealloc *prealloc); +static void __stack_depot_trie_side_table_commit_id(u32 id); static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup(u32 id); static size_t __stack_depot_trie_side_table_bytes(void); static size_t __stack_depot_trie_pool_alloc_size(size_t size); static void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags); -static void __stack_depot_trie_pool_free_prealloc(void *prealloc); static int __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, void **pool_prealloc, @@ -198,9 +183,6 @@ __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, static int __stack_depot_trie_pool_carve(struct stack_depot_trie_alloc_request *req); static void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); static int -__stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, - struct stack_depot_trie_side_prealloc *prealloc); -static int __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, @@ -219,14 +201,7 @@ __stack_depot_trie_workspace_insert(struct stack_depot_trie_root *root, struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_workspace *workspace, u32 *leaf_id); -static depot_stack_handle_t -__stack_depot_trie_save_locked(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - gfp_t alloc_flags, depot_flags_t depot_flags, - struct stack_depot_trie_alloc_workspace *workspace, - raw_spinlock_t *workspace_lock); static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req); -static u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn); static int __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, struct stack_depot_trie_alloc_request *req, @@ -234,11 +209,8 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, u32 *scratch, unsigned int nr_scratch, u32 *leaf_id); static void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); -static int -__stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, - struct stack_depot_trie_side_prepare *state); -static void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *state); +static int trie_side_publish(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, u32 fresh_leaf_id); static int __stack_depot_frame_run_init(const unsigned long *entries, unsigned int nr_entries, struct stack_depot_frame_run *run); @@ -293,7 +265,7 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, - struct stack_depot_trie_side_prepare *side, + u32 fresh_leaf_id, const struct stack_depot_trie_node **tail, unsigned int *nr_used); static int @@ -383,6 +355,9 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); #define DEPOT_POOL_INDEX_MASK ((1U << DEPOT_POOL_INDEX_BITS) - 1) #define DEPOT_OFFSET_MASK ((1U << DEPOT_OFFSET_BITS) - 1) +#define STACK_RECORD_FLAG_REFCOUNTED BIT(0) +#define STACK_RECORD_FLAG_COUNTABLE BIT(1) + /* Compact structure that stores a reference to a stack. */ union handle_parts { depot_stack_handle_t handle; @@ -396,7 +371,8 @@ union handle_parts { struct stack_record { struct list_head hash_list; /* Links in the hash table */ u32 hash; /* Hash in hash table */ - u32 size; /* Number of stored frames */ + u16 size; /* Number of stored frames */ + u16 flags; union handle_parts handle; /* Constant after initialization */ refcount_t count; union { @@ -421,7 +397,7 @@ struct stack_record { struct stack_depot_trie_node { /* Parent links let fetch rebuild a full stack from a leaf to the root. */ const struct stack_depot_trie_node *parent; - /* Child arrays are separate RCU/COW generations; nodes stay immutable. */ + /* Child arrays are separate RCU/COW generations. */ const struct stack_depot_trie_child_array *children; u32 leaf_id; struct stack_depot_frame_run run; @@ -700,14 +676,14 @@ static struct stack_depot_trie_side_entry * trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, unsigned int idx) { - /* Pairs with the chunk rcu_assign_pointer() in leaf ID allocation. */ + /* Pairs with the chunk rcu_assign_pointer() in leaf ID preparation. */ return rcu_dereference_check(dir->chunks[idx], lockdep_is_held(&trie_side_table_lock) || rcu_read_lock_sched_held()); } static u32 -__stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *prealloc) +__stack_depot_trie_side_table_prepare_id(struct stack_depot_trie_side_prealloc *prealloc) { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; @@ -717,6 +693,7 @@ __stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *pr u32 id = 0; raw_spin_lock_irqsave(&trie_side_table_lock, flags); + /* Prepare the next slot without making the ID visible for reuse yet. */ /* Failed or disabled trie init means no leaf IDs can be allocated. */ if (!trie_side_table_is_initialized()) goto out; @@ -756,7 +733,6 @@ __stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *pr trie_side_table_nr_chunks++; } - WRITE_ONCE(trie_side_table_next_id, id); goto out; out_clear_id: @@ -766,6 +742,25 @@ __stack_depot_trie_side_table_alloc_id(struct stack_depot_trie_side_prealloc *pr return id; } +static void __stack_depot_trie_side_table_commit_id(u32 id) +{ + unsigned long flags; + u32 next; + + if (!trie_side_table_is_initialized() || !id) + return; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + next = trie_side_table_next_id; + if (WARN_ON_ONCE(id != next + 1)) { + if (id > next) + WRITE_ONCE(trie_side_table_next_id, id); + } else { + WRITE_ONCE(trie_side_table_next_id, id); + } + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); +} + static size_t trie_side_table_root_bytes(unsigned int root_size) { size_t bytes; @@ -797,12 +792,6 @@ static inline unsigned int trie_side_table_chunk_order(void) return get_order(trie_side_table_chunk_bytes()); } -static void trie_side_table_free_chunk(struct stack_depot_trie_side_entry *chunk) -{ - if (chunk) - free_pages((unsigned long)chunk, trie_side_table_chunk_order()); -} - static void trie_side_table_free_dir(struct stack_depot_trie_side_dir *dir) { if (dir) @@ -977,28 +966,12 @@ static const struct stack_depot_trie_node * trie_side_table_load_leaf(struct stack_depot_trie_side_entry *chunk, unsigned int slot) { - /* Pairs with trie_side_table_store_leaf(). */ + /* Pairs with side-table leaf rcu_assign_pointer(). */ return rcu_dereference_check(chunk[slot].leaf, lockdep_is_held(&trie_side_table_lock) || rcu_read_lock_sched_held()); } -static void -trie_side_table_store_leaf(struct stack_depot_trie_side_entry *chunk, - unsigned int slot, - const struct stack_depot_trie_node *leaf) -{ - /* Pairs with trie_side_table_load_leaf(). */ - rcu_assign_pointer(chunk[slot].leaf, leaf); -} - -static void -trie_side_table_clear_entry(struct stack_depot_trie_side_entry *chunk, - unsigned int slot) -{ - trie_side_table_store_leaf(chunk, slot, NULL); -} - static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) { struct stack_depot_trie_side_root *root_vec; @@ -1126,80 +1099,15 @@ __stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_preallo if (!prealloc) return; trie_side_table_free_dir(prealloc->dir); - trie_side_table_free_chunk(prealloc->chunk); + if (prealloc->chunk) + free_pages((unsigned long)prealloc->chunk, + trie_side_table_chunk_order()); prealloc->dir = NULL; prealloc->chunk = NULL; } -static void __stack_depot_trie_side_table_revoke_latest(u32 id) -{ - struct stack_depot_trie_side_entry *chunk; - struct stack_depot_trie_side_dir *dir; - unsigned long flags; - unsigned int slot; - unsigned int root; - - if (!trie_side_table_is_initialized() || !id || - id != READ_ONCE(trie_side_table_next_id)) - return; - - raw_spin_lock_irqsave(&trie_side_table_lock, flags); - if (id != trie_side_table_next_id) - goto out; - root = trie_side_table_root_index(id); - if (root >= trie_side_table_root_size) - goto out; - - dir = trie_side_table_load_dir(root); - if (!dir) - goto out; - chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); - if (!chunk) - goto out; - - slot = trie_side_table_slot_index(id); - trie_side_table_clear_entry(chunk, slot); - WRITE_ONCE(trie_side_table_next_id, id - 1); -out: - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); -} - -static void -__stack_depot_trie_side_table_restore(u32 id, - const struct stack_depot_trie_node *entry) -{ - struct stack_depot_trie_side_entry *chunk; - struct stack_depot_trie_side_dir *dir; - unsigned long flags; - unsigned int root; - - if (!trie_side_table_is_initialized() || !id) - return; - - raw_spin_lock_irqsave(&trie_side_table_lock, flags); - if (id > trie_side_table_next_id) - goto out; - root = trie_side_table_root_index(id); - if (root >= trie_side_table_root_size) - goto out; - - dir = trie_side_table_load_dir(root); - if (!dir) - goto out; - chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); - if (!chunk) - goto out; - - if (entry) - trie_side_table_store_leaf(chunk, trie_side_table_slot_index(id), entry); - else - trie_side_table_clear_entry(chunk, trie_side_table_slot_index(id)); -out: - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); -} - static struct stack_depot_trie_side_entry * -trie_side_table_chunk_locked(u32 id, unsigned int *slot) +trie_side_table_chunk_locked(u32 id, u32 fresh_leaf_id, unsigned int *slot) { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; @@ -1207,7 +1115,9 @@ trie_side_table_chunk_locked(u32 id, unsigned int *slot) lockdep_assert_held(&trie_side_table_lock); - if (!trie_side_table_is_initialized() || !id || id > trie_side_table_next_id) + if (!trie_side_table_is_initialized() || !id) + return NULL; + if (id > trie_side_table_next_id && id != fresh_leaf_id) return NULL; root = trie_side_table_root_index(id); @@ -1638,11 +1548,6 @@ static void trie_retire_object_node(const void *ptr, raw_spin_unlock_irqrestore(&pool_lock, flags); } -static void trie_retire_object(const void *ptr) -{ - trie_retire_object_node(ptr, NULL, 0); -} - static void *trie_pop_free_object(size_t size) { struct stack_depot_trie_free_object *free; @@ -1681,12 +1586,6 @@ static void *__stack_depot_trie_pool_prealloc(gfp_t gfp_flags) return page ? page_address(page) : NULL; } -static void __stack_depot_trie_pool_free_prealloc(void *prealloc) -{ - if (prealloc) - free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); -} - /* * Preallocate resources that cannot be allocated while trie writers hold raw * spinlocks. Side-table growth is mandatory before a new leaf ID can be @@ -1974,23 +1873,6 @@ static void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn memset(txn, 0, sizeof(*txn)); } -static int -__stack_depot_trie_alloc_txn_id(struct stack_depot_trie_alloc_txn *txn, - struct stack_depot_trie_side_prealloc *prealloc) -{ - u32 leaf_id; - - if (!txn || txn->leaf_id) - return -EINVAL; - - leaf_id = __stack_depot_trie_side_table_alloc_id(prealloc); - if (!leaf_id) - return -ENOSPC; - - txn->leaf_id = leaf_id; - return 0; -} - static void trie_alloc_request_clear_outputs(struct stack_depot_trie_alloc_request *req) { unsigned int i; @@ -2008,17 +1890,14 @@ static void trie_alloc_request_clear_outputs(struct stack_depot_trie_alloc_reque static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req) { + u32 leaf_id; int ret; if (!req || !req->txn) return -EINVAL; - if (req->txn->leaf_id || req->txn->pool.size || req->txn->side.nr_updates) + if (req->txn->leaf_id || req->txn->pool.size) return -EINVAL; - ret = __stack_depot_trie_alloc_txn_id(req->txn, req->side_prealloc); - if (ret) - return ret; - ret = __stack_depot_trie_pool_carve(req); if (ret) { __stack_depot_trie_alloc_txn_rollback(req->txn); @@ -2026,6 +1905,15 @@ static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_re return ret; } + leaf_id = __stack_depot_trie_side_table_prepare_id(req->side_prealloc); + if (!leaf_id) { + trie_pool_release_reused_objects(req); + __stack_depot_trie_alloc_txn_rollback(req->txn); + trie_alloc_request_clear_outputs(req); + return -ENOSPC; + } + req->txn->leaf_id = leaf_id; + return 0; } @@ -2125,52 +2013,8 @@ trie_find_handle(const struct stack_depot_trie_root *root, } static depot_stack_handle_t -trie_save_locked_insert(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_alloc_workspace *workspace, - void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - bool can_insert) -{ - depot_stack_handle_t handle; - u32 leaf_id; - int ret; - - handle = trie_find_handle(root, entries, nr_entries); - if (!handle && can_insert) { - ret = __stack_depot_trie_workspace_insert(root, entries, nr_entries, - pool_prealloc, side_prealloc, - workspace, &leaf_id); - if (!ret) - handle = __stack_depot_trie_handle(leaf_id); - } - - return handle; -} - -static depot_stack_handle_t -trie_save_spinlocked(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_alloc_workspace *workspace, - raw_spinlock_t *workspace_lock, void **pool_prealloc, - struct stack_depot_trie_side_prealloc *side_prealloc, - bool can_insert) -{ - depot_stack_handle_t handle; - unsigned long flags; - - raw_spin_lock_irqsave(workspace_lock, flags); - handle = trie_save_locked_insert(root, entries, nr_entries, workspace, - pool_prealloc, side_prealloc, - can_insert); - raw_spin_unlock_irqrestore(workspace_lock, flags); - return handle; -} - -static depot_stack_handle_t -trie_save_trylocked(struct stack_depot_trie_root *root, +trie_find_trylocked(struct stack_depot_trie_root *root, const unsigned long *entries, unsigned int nr_entries, - struct stack_depot_trie_alloc_workspace *workspace, raw_spinlock_t *workspace_lock) { depot_stack_handle_t handle = 0; @@ -2183,67 +2027,6 @@ trie_save_trylocked(struct stack_depot_trie_root *root, return handle; } -static depot_stack_handle_t -__stack_depot_trie_save_locked(struct stack_depot_trie_root *root, - const unsigned long *entries, unsigned int nr_entries, - gfp_t alloc_flags, depot_flags_t depot_flags, - struct stack_depot_trie_alloc_workspace *workspace, - raw_spinlock_t *workspace_lock) -{ - depot_stack_handle_t handle = 0; - void *pool_prealloc = NULL; - struct stack_depot_trie_side_prealloc side_prealloc = {}; - bool can_insert; - bool no_spin; - int ret; - - if (!root || !entries || !nr_entries || !workspace || !workspace_lock) - return 0; - if (depot_flags & STACK_DEPOT_FLAG_GET) - return 0; - if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) - return 0; - - handle = trie_find_handle(root, entries, nr_entries); - if (handle) - return handle; - no_spin = in_nmi() || !gfpflags_allow_spinning(alloc_flags); - /* - * No-spin callers cannot wait for the workspace lock or allocate side-table - * or pool storage. After the lockless lookup misses, trylock and recheck: a - * concurrent writer may have inserted the stack. Otherwise fail instead of - * spinning or publishing a new leaf. - */ - if (no_spin) - return trie_save_trylocked(root, entries, nr_entries, workspace, - workspace_lock); - - ret = __stack_depot_trie_alloc_prealloc(alloc_flags, depot_flags, - &pool_prealloc, - &side_prealloc); - can_insert = !ret; - handle = trie_save_spinlocked(root, entries, nr_entries, workspace, - workspace_lock, &pool_prealloc, - &side_prealloc, can_insert); - - depot_try_keep_new_pool(&pool_prealloc); - __stack_depot_trie_pool_free_prealloc(pool_prealloc); - __stack_depot_trie_side_table_free_prealloc(&side_prealloc); - return handle; -} - -static u32 __stack_depot_trie_alloc_txn_commit(struct stack_depot_trie_alloc_txn *txn) -{ - u32 leaf_id; - - if (!txn) - return 0; - - leaf_id = txn->leaf_id; - __stack_depot_trie_alloc_txn_init(txn); - return leaf_id; -} - static int __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, struct stack_depot_trie_alloc_request *req, @@ -2275,11 +2058,13 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, req->nr_node_slots, req->child_slots, req->nr_child_slots, scratch, nr_scratch, storage, req->storage_size, - &txn->side, &tail, &nr_used); + id, &tail, &nr_used); if (ret) goto rollback; - *leaf_id = __stack_depot_trie_alloc_txn_commit(txn); + __stack_depot_trie_side_table_commit_id(id); + __stack_depot_trie_alloc_txn_init(txn); + *leaf_id = id; return 0; rollback: @@ -2294,43 +2079,20 @@ static void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_ if (!txn) return; - __stack_depot_trie_side_rollback(&txn->side); - if (txn->leaf_id) { - __stack_depot_trie_side_table_revoke_latest(txn->leaf_id); - txn->leaf_id = 0; - } stack_depot_trie_pool_rollback(&txn->pool); + txn->leaf_id = 0; memset(&txn->pool, 0, sizeof(txn->pool)); } -static void __stack_depot_trie_side_rollback(struct stack_depot_trie_side_prepare *state) -{ - if (!state) - return; - - while (state->nr_updates) { - struct stack_depot_trie_side_checkpoint *update; - - state->nr_updates--; - update = &state->updates[state->nr_updates]; - __stack_depot_trie_side_table_restore(update->leaf_id, update->old_leaf); - } -} - -static int -trie_side_prepare_locked(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, - struct stack_depot_trie_side_prepare *state) +static int trie_side_publish_locked(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, u32 fresh_leaf_id) { struct stack_depot_trie_side_entry *chunks[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; unsigned int slots[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; unsigned int i; lockdep_assert_held(&trie_side_table_lock); - if (!state || (!updates && nr_updates)) - return -EINVAL; - if (nr_updates > ARRAY_SIZE(chunks) || - state->nr_updates > ARRAY_SIZE(state->updates) - nr_updates) + if ((!updates && nr_updates) || nr_updates > ARRAY_SIZE(chunks)) return -EINVAL; for (i = 0; i < nr_updates; i++) { @@ -2338,32 +2100,29 @@ trie_side_prepare_locked(const struct stack_depot_trie_leaf_update *updates, if (!updates[i].leaf) return -EINVAL; - chunks[i] = trie_side_table_chunk_locked(leaf_id, &slots[i]); + chunks[i] = trie_side_table_chunk_locked(leaf_id, fresh_leaf_id, &slots[i]); if (!chunks[i]) return -EINVAL; + if (leaf_id == fresh_leaf_id && + trie_side_table_load_leaf(chunks[i], slots[i])) + return -EINVAL; } - for (i = 0; i < nr_updates; i++) { - state->updates[state->nr_updates].leaf_id = updates[i].leaf_id; - state->updates[state->nr_updates].old_leaf = - trie_side_table_load_leaf(chunks[i], slots[i]); - state->nr_updates++; - trie_side_table_store_leaf(chunks[i], slots[i], updates[i].leaf); - } + /* Pairs with trie_side_table_load_leaf(). */ + for (i = 0; i < nr_updates; i++) + rcu_assign_pointer(chunks[i][slots[i]].leaf, updates[i].leaf); return 0; } -static int -__stack_depot_trie_side_prepare(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, - struct stack_depot_trie_side_prepare *state) +static int trie_side_publish(const struct stack_depot_trie_leaf_update *updates, + unsigned int nr_updates, u32 fresh_leaf_id) { unsigned long flags; int ret; raw_spin_lock_irqsave(&trie_side_table_lock, flags); - ret = trie_side_prepare_locked(updates, nr_updates, state); + ret = trie_side_publish_locked(updates, nr_updates, fresh_leaf_id); raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); return ret; } @@ -2727,6 +2486,15 @@ static inline size_t depot_stack_record_size(struct stack_record *s, unsigned in return ALIGN(sizeof(struct stack_record) - unused, 1 << DEPOT_STACK_ALIGN); } +static u16 stack_record_flags(depot_flags_t flags) +{ + if (flags & STACK_DEPOT_FLAG_GET) + return STACK_RECORD_FLAG_REFCOUNTED; + if (flags & STACK_DEPOT_FLAG_COUNTABLE) + return STACK_RECORD_FLAG_COUNTABLE; + return 0; +} + /* Allocates a new stack in a stack depot pool. */ static struct stack_record * depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, @@ -2765,6 +2533,7 @@ depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, /* Save the stack trace. */ stack->hash = hash; stack->size = nr_entries; + stack->flags = stack_record_flags(flags); /* stack->handle is already filled in by depot_pop_free_pool(). */ memcpy(stack->entries, entries, flex_array_size(stack, entries, nr_entries)); @@ -2901,7 +2670,8 @@ static inline struct stack_record *find_stack(struct list_head *bucket, rcu_read_lock_sched_notrace(); list_for_each_entry_rcu(stack, bucket, hash_list) { - if (stack->hash != hash || stack->size != size) + if (stack->hash != hash || stack->size != size || + stack->flags != stack_record_flags(flags)) continue; /* @@ -2936,10 +2706,61 @@ static depot_stack_handle_t stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags, depot_flags_t depot_flags) { - return __stack_depot_trie_save_locked(&stack_depot_trie_root, entries, - nr_entries, alloc_flags, depot_flags, - stack_depot_trie_load_workspace(), - &stack_depot_trie_workspace_lock); + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_side_prealloc side_prealloc = {}; + void *pool_prealloc = NULL; + depot_stack_handle_t handle; + unsigned long flags; + u32 leaf_id; + int ret; + + workspace = stack_depot_trie_load_workspace(); + if (!entries || !nr_entries || !workspace) + return 0; + if (depot_flags & STACK_DEPOT_FLAG_GET) + return 0; + if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) + nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; + + handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); + if (handle) + return handle; + /* + * No-spin callers cannot wait for the workspace lock or allocate side-table + * or pool storage. After the lockless lookup misses, trylock and recheck: a + * concurrent writer may have inserted the stack. Otherwise fail instead of + * spinning or publishing a new leaf. + */ + if (in_nmi() || !gfpflags_allow_spinning(alloc_flags)) + return trie_find_trylocked(&stack_depot_trie_root, entries, + nr_entries, + &stack_depot_trie_workspace_lock); + + ret = __stack_depot_trie_alloc_prealloc(alloc_flags, depot_flags, + &pool_prealloc, + &side_prealloc); + if (ret) + goto out_free; + + raw_spin_lock_irqsave(&stack_depot_trie_workspace_lock, flags); + handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); + if (!handle) { + ret = __stack_depot_trie_workspace_insert(&stack_depot_trie_root, + entries, nr_entries, + &pool_prealloc, + &side_prealloc, workspace, + &leaf_id); + if (!ret) + handle = __stack_depot_trie_handle(leaf_id); + } + raw_spin_unlock_irqrestore(&stack_depot_trie_workspace_lock, flags); + +out_free: + depot_try_keep_new_pool(&pool_prealloc); + if (pool_prealloc) + free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); + __stack_depot_trie_side_table_free_prealloc(&side_prealloc); + return handle; } struct stack_depot_hash_save { @@ -2996,6 +2817,9 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, if (WARN_ON(depot_flags & ~STACK_DEPOT_FLAGS_MASK)) return 0; + if (WARN_ON_ONCE((depot_flags & STACK_DEPOT_FLAG_GET) && + (depot_flags & STACK_DEPOT_FLAG_COUNTABLE))) + return 0; /* * If this stack trace is from an interrupt, including anything before @@ -3012,7 +2836,7 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; - trie_candidate = !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_HASH)) && + trie_candidate = !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE)) && __stack_depot_trie_ready(); if (trie_candidate) { handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, @@ -3105,22 +2929,24 @@ EXPORT_SYMBOL_GPL(stack_depot_save); bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count) { struct stack_record *stack; - unsigned int raw; + int raw; - if (!handle || !count || __stack_depot_trie_leaf_id(handle)) + if (WARN_ON_ONCE(!handle || !count || __stack_depot_trie_leaf_id(handle))) return false; stack = depot_fetch_stack(handle); if (!stack) return false; + if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_COUNTABLE))) + return false; - /* Negative saturated counts wrap above INT_MAX when converted to unsigned. */ - raw = (unsigned int)refcount_read(&stack->count); - /* Saturated and zero records are not in counted mode. */ - if (!raw || raw > INT_MAX) + raw = refcount_read(&stack->count); + if (raw == REFCOUNT_SATURATED) + return false; + if (WARN_ON_ONCE(raw <= 0)) return false; - *count = raw; + *count = (unsigned int)raw; return true; } @@ -3128,14 +2954,15 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count) { struct stack_record *stack; - /* Reject values outside positive refcount space. */ - if (!handle || !count || count > (unsigned int)INT_MAX || - __stack_depot_trie_leaf_id(handle)) + if (WARN_ON_ONCE(!handle || !count || count > (unsigned int)INT_MAX || + __stack_depot_trie_leaf_id(handle))) return; stack = depot_fetch_stack(handle); if (!stack) return; + if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_COUNTABLE))) + return; refcount_set(&stack->count, (int)count); } @@ -3151,13 +2978,15 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, if (new_count) *new_count = false; - if (!handle || !count || count > (unsigned int)INT_MAX - 1 || - __stack_depot_trie_leaf_id(handle)) + if (WARN_ON_ONCE(!handle || !count || count > (unsigned int)INT_MAX - 1 || + __stack_depot_trie_leaf_id(handle))) return false; stack = depot_fetch_stack(handle); if (!stack) return false; + if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_COUNTABLE))) + return false; new = 1 + (int)count; /* @@ -3172,7 +3001,9 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, } else { /* cmpxchg reloads @old before each retry check. */ do { - if (old <= 0) + if (old == REFCOUNT_SATURATED) + return false; + if (WARN_ON_ONCE(old <= 0)) return false; if (count > (unsigned int)INT_MAX - (unsigned int)old) return false; @@ -3192,13 +3023,15 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, int new; int old; - if (!handle || !count || count > (unsigned int)INT_MAX || - __stack_depot_trie_leaf_id(handle)) + if (WARN_ON_ONCE(!handle || !count || count > (unsigned int)INT_MAX || + __stack_depot_trie_leaf_id(handle))) return false; stack = depot_fetch_stack(handle); if (!stack) return false; + if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_COUNTABLE))) + return false; /* * Intentional refcount_t internals use: refcount_sub_and_test() would @@ -3215,8 +3048,9 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, * linearization point. A racing increment not observed here is * ordered after this decrement. */ - /* Saturated counts are negative and intentionally fail closed here. */ - if (old <= 0) + if (old == REFCOUNT_SATURATED) + return false; + if (WARN_ON_ONCE(old <= 0)) return false; underflow = count > (unsigned int)old; @@ -4193,7 +4027,7 @@ trie_promote_child(struct stack_depot_trie_root *root, const struct stack_depot_trie_node *child, u32 leaf_id, const struct stack_depot_trie_node_slot *slot, void *new_storage, size_t new_storage_size, - struct stack_depot_trie_side_prepare *side) + u32 fresh_leaf_id) { const struct stack_depot_trie_child_array **publish_slot; const struct stack_depot_trie_child_array *old_array; @@ -4214,10 +4048,10 @@ trie_promote_child(struct stack_depot_trie_root *root, child_size = __stack_depot_trie_node_size(&child->run); if (!child_size) return -EINVAL; - if (side) { + if (fresh_leaf_id) { update.leaf_id = leaf_id; update.leaf = slot->node; - ret = __stack_depot_trie_side_prepare(&update, 1, side); + ret = trie_side_publish(&update, 1, fresh_leaf_id); if (ret) return ret; } @@ -4384,7 +4218,7 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, const struct stack_depot_trie_node *head, void *new_storage, size_t new_storage_size, - struct stack_depot_trie_side_prepare *side, + u32 fresh_leaf_id, u32 leaf_id, const struct stack_depot_trie_node *leaf) { @@ -4435,7 +4269,7 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, return -EINVAL; if (found || !trie_child_array_can_append(old_array, pos)) return -EINVAL; - if (side) { + if (fresh_leaf_id) { struct stack_depot_trie_leaf_update update = { .leaf_id = leaf_id, .leaf = leaf, @@ -4443,7 +4277,7 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, if (!leaf_id || !leaf) return -EINVAL; - ret = __stack_depot_trie_side_prepare(&update, 1, side); + ret = trie_side_publish(&update, 1, fresh_leaf_id); if (ret) return ret; } @@ -4463,7 +4297,7 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, return -EINVAL; if (__stack_depot_trie_child_array_insert(old_array, head, new_array, storage_size)) return -EINVAL; - if (side) { + if (fresh_leaf_id) { struct stack_depot_trie_leaf_update update = { .leaf_id = leaf_id, .leaf = leaf, @@ -4471,13 +4305,13 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, if (!leaf_id || !leaf) return -EINVAL; - ret = __stack_depot_trie_side_prepare(&update, 1, side); + ret = trie_side_publish(&update, 1, fresh_leaf_id); if (ret) return ret; } /* Publish the fully initialized replacement array last. */ trie_publish_children_slot(slot, new_array); - trie_retire_object(old_array); + trie_retire_object_node(old_array, NULL, 0); return 0; } @@ -4527,9 +4361,9 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, if (!node) return -EINVAL; /* - * Do not validate node->parent here. COW splits may reparent descendants - * to an equivalent replacement prefix before the structural publish; the - * multi-step finder has enough prefix context to validate that equivalence. + * Do not require direct parent identity here. COW updates may publish a + * child whose parent chain is an equivalent replacement prefix; the finder + * has enough input prefix context to validate that equivalence. */ matched = __stack_depot_trie_node_match(node, entries, nr_entries); @@ -4619,7 +4453,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, - struct stack_depot_trie_side_prepare *side, + u32 fresh_leaf_id, const struct stack_depot_trie_node **tail, unsigned int *nr_used); @@ -4636,7 +4470,7 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, - struct stack_depot_trie_side_prepare *side, + u32 fresh_leaf_id, const struct stack_depot_trie_node **tail, unsigned int *nr_used) { @@ -4675,14 +4509,14 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, leaf_id, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, scratch, nr_scratch, new_storage, - new_storage_size, side, tail, + new_storage_size, fresh_leaf_id, tail, nr_used); if (lookup.status == STACK_DEPOT_TRIE_LOOKUP_PROMOTE) { if (!node_slots || !nr_node_slots) return -EINVAL; ret = trie_promote_child(root, parent, lookup.node, leaf_id, &node_slots[0], new_storage, new_storage_size, - side); + fresh_leaf_id); if (ret) return ret; *tail = node_slots[0].node; @@ -4705,7 +4539,7 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, if (ret) return ret; ret = trie_publish_append_prepare(root, parent, head, new_storage, - new_storage_size, side, leaf_id, last); + new_storage_size, fresh_leaf_id, leaf_id, last); if (ret) return ret; @@ -5261,13 +5095,6 @@ __stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, return nr_entries; } -static inline unsigned int trie_child_array_capacity(unsigned int nr_children) -{ - if (!nr_children) - return 0; - return roundup_pow_of_two(nr_children); -} - static inline unsigned int trie_child_array_storage_capacity(size_t storage_size) { if (storage_size < sizeof(struct stack_depot_trie_child_array)) @@ -5290,7 +5117,7 @@ static size_t trie_child_array_size_for_capacity(unsigned int capacity) static inline size_t __stack_depot_trie_child_array_size(unsigned int nr_children) { - unsigned int capacity = trie_child_array_capacity(nr_children); + unsigned int capacity = nr_children ? roundup_pow_of_two(nr_children) : 0; return trie_child_array_size_for_capacity(capacity); } @@ -5661,7 +5488,7 @@ static int trie_split_subtree_prepare(const struct stack_depot_trie_node *child, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, - struct stack_depot_trie_side_prepare *side, + u32 fresh_leaf_id, const struct stack_depot_trie_node **prefix, const struct stack_depot_trie_node **tail, unsigned int *nr_used) @@ -5732,8 +5559,8 @@ static int trie_split_subtree_prepare(const struct stack_depot_trie_node *child, updates[nr_updates].leaf_id = leaf_id; updates[nr_updates].leaf = has_new_tail ? new_tail : pref; nr_updates++; - if (side) { - ret = __stack_depot_trie_side_prepare(updates, nr_updates, side); + if (fresh_leaf_id) { + ret = trie_side_publish(updates, nr_updates, fresh_leaf_id); if (ret) { memset(split_array, 0, split_array_size); return ret; @@ -5761,7 +5588,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, size_t new_storage_size, - struct stack_depot_trie_side_prepare *side, + u32 fresh_leaf_id, const struct stack_depot_trie_node **tail, unsigned int *nr_used) { @@ -5801,7 +5628,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, ret = trie_split_subtree_prepare(child, matched, leaf_id, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, scratch, - nr_scratch, side, &prefix, + nr_scratch, fresh_leaf_id, &prefix, tail, &used); if (ret) return ret; @@ -5994,7 +5821,7 @@ void stack_depot_put(depot_stack_handle_t handle) if (!handle || stack_depot_disabled) return; - if (__stack_depot_trie_leaf_id(handle)) + if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) return; stack = depot_fetch_stack(handle); @@ -6004,6 +5831,8 @@ void stack_depot_put(depot_stack_handle_t handle) */ if (WARN(!stack, "corrupt handle or unbalanced %s()", __func__)) return; + if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_REFCOUNTED))) + return; if (refcount_dec_and_test(&stack->count)) depot_free_stack(stack); diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 8a51142783553..1c9243981b2df 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -97,7 +97,7 @@ static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) static depot_stack_handle_t save_hash(unsigned long *entries, unsigned int nr) { - depot_flags_t flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_HASH; + depot_flags_t flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE; return stack_depot_save_flags(entries, nr, GFP_KERNEL, flags); } @@ -191,11 +191,16 @@ static void stackdepot_save_flags_public(struct kunit *test) GFP_KERNEL, flags); KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); stack_depot_put(get_handle); + get_handle = stack_depot_save_flags(entries, ARRAY_SIZE(entries), GFP_KERNEL, + flags); + KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); - flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_HASH; + flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE; hash_handle = stack_depot_save_flags(entries, ARRAY_SIZE(entries), GFP_KERNEL, flags); KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); + KUNIT_EXPECT_NE(test, hash_handle, get_handle); + stack_depot_put(get_handle); overlong_handle = stack_depot_save(overlong_entries, overlong_nr, GFP_KERNEL); @@ -256,12 +261,6 @@ static void stackdepot_count_helpers(struct kunit *test) KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(0, &count)); - KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(0, NULL)); - __stack_depot_set_count(0, 1); - KUNIT_EXPECT_FALSE(test, __stack_depot_inc_count(0, 1, &new_count)); - KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(0, 1)); - handle = save_hash(entries, ARRAY_SIZE(entries)); KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); diff --git a/mm/page_owner.c b/mm/page_owner.c index 199c580a53cdc..3fd46599676ab 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -100,7 +100,7 @@ static __always_inline depot_stack_handle_t create_dummy_stack(void) nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); return stack_depot_save_flags(entries, nr_entries, GFP_KERNEL, - STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_HASH); + STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE); } static noinline void register_dummy_stack(void) @@ -165,7 +165,7 @@ static noinline depot_stack_handle_t save_stack(gfp_t flags) set_current_in_page_owner(); nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); handle = stack_depot_save_flags(entries, nr_entries, flags, - STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_HASH); + STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE); if (!handle) handle = failure_handle; unset_current_in_page_owner(); @@ -207,13 +207,7 @@ static void add_stack_record_to_list(depot_stack_handle_t handle, spin_lock_irqsave(&stack_list_lock, flags); stack->next = stack_list; - /* - * This pairs with smp_load_acquire() from function - * stack_start(). This guarantees that stack_start() - * will see an updated stack_list before starting to - * traverse the list. - */ - smp_store_release(&stack_list, stack); + stack_list = stack; spin_unlock_irqrestore(&stack_list_lock, flags); } @@ -906,18 +900,16 @@ static const struct file_operations proc_page_owner_operations = { static void *stack_start(struct seq_file *m, loff_t *ppos) { struct page_owner_stack_seq *priv = m->private; + unsigned long flags; struct stack *stack; if (*ppos == -1UL) return NULL; if (!*ppos) { - /* - * This pairs with smp_store_release() from function - * add_stack_record_to_list(), so we get a consistent - * value of stack_list. - */ - stack = smp_load_acquire(&stack_list); + spin_lock_irqsave(&stack_list_lock, flags); + stack = stack_list; + spin_unlock_irqrestore(&stack_list_lock, flags); } else { stack = priv->stack; } From a4af172e383498bdb10a91b152266c689f2546d2 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Wed, 24 Jun 2026 17:17:28 +0100 Subject: [PATCH 122/129] KRN-1117: Tighten stackdepot trie reservation state Prepare trie side-table IDs before carving pool storage so an expected side-table allocation failure cannot require unwinding pool bytes. Keep the remaining rollback path limited to private pool reservations that fail before publication. Make countable hash records use an explicit atomic page_owner count in the existing count storage instead of reaching through refcount_t internals. Document that countable records intentionally do not deduplicate with non-countable records, and drop unused side-table init state while touching the path. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 3 +- lib/stackdepot.c | 58 ++++++++++++++++++-------------------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 444498c5a805b..5569f7aacb0cd 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -111,7 +111,8 @@ static inline int stack_depot_early_init(void) { return 0; } * If STACK_DEPOT_FLAG_COUNTABLE is set in @depot_flags, stack depot stores the * stack in a distinct hash-backed record mode that supports the internal count * helpers. This flag does not imply %STACK_DEPOT_FLAG_CAN_ALLOC and is mutually - * exclusive with %STACK_DEPOT_FLAG_GET. + * exclusive with %STACK_DEPOT_FLAG_GET. Countable records do not deduplicate + * with non-countable records that have the same frames. * * When trie storage is enabled, persistent non-refcounted saves use trie * storage. Constrained contexts remain best effort and can return 0 if a diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 877af6b995ec3..24c033a8cc62d 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -16,6 +16,7 @@ #define pr_fmt(fmt) "stackdepot: " fmt +#include #include #include #include @@ -374,7 +375,10 @@ struct stack_record { u16 size; /* Number of stored frames */ u16 flags; union handle_parts handle; /* Constant after initialization */ - refcount_t count; + union { + refcount_t count; + atomic_t page_count; + }; union { unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; /* Frames */ struct { @@ -598,7 +602,6 @@ static unsigned int trie_side_table_nr_dirs; static unsigned int trie_side_table_nr_chunks; static unsigned int trie_side_table_root_size; static u32 trie_side_table_next_id; -static bool trie_side_table_memblock; /* Lock order: workspace_lock -> pool_lock -> trie_side_table_lock. */ @@ -802,8 +805,7 @@ static int trie_side_table_install(struct stack_depot_trie_side_root *root_vec, unsigned int root_size, u32 max_id, struct stack_depot_trie_side_dir *first_dir, - struct stack_depot_trie_side_entry *first_chunk, - bool memblock) + struct stack_depot_trie_side_entry *first_chunk) { /* * Early init installs the first directory and chunk so early leaf ID @@ -822,7 +824,6 @@ trie_side_table_install(struct stack_depot_trie_side_root *root_vec, WRITE_ONCE(trie_side_table_nr_chunks, 0); WRITE_ONCE(trie_side_table_max_id, max_id); WRITE_ONCE(trie_side_table_next_id, 0); - WRITE_ONCE(trie_side_table_memblock, memblock); if (first_dir) { RCU_INIT_POINTER(root_vec->dirs[0], first_dir); WRITE_ONCE(trie_side_table_nr_dirs, 1); @@ -886,7 +887,7 @@ static int __init __stack_depot_trie_side_table_init_memblock(void) memset(first_chunk, 0, chunk_bytes); return trie_side_table_install(root_vec, root_size, max_leaf_id, first_dir, - first_chunk, true); + first_chunk); } static int @@ -994,8 +995,7 @@ static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) if (!root_vec) return -ENOMEM; - return trie_side_table_install(root_vec, root_size, max_leaf_id, NULL, NULL, - false); + return trie_side_table_install(root_vec, root_size, max_leaf_id, NULL, NULL); } static bool __stack_depot_trie_side_table_prealloc_needed(void) @@ -1898,22 +1898,19 @@ static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_re if (req->txn->leaf_id || req->txn->pool.size) return -EINVAL; + leaf_id = __stack_depot_trie_side_table_prepare_id(req->side_prealloc); + if (!leaf_id) + return -ENOSPC; + req->txn->leaf_id = leaf_id; + ret = __stack_depot_trie_pool_carve(req); if (ret) { + req->txn->leaf_id = 0; __stack_depot_trie_alloc_txn_rollback(req->txn); trie_alloc_request_clear_outputs(req); return ret; } - leaf_id = __stack_depot_trie_side_table_prepare_id(req->side_prealloc); - if (!leaf_id) { - trie_pool_release_reused_objects(req); - __stack_depot_trie_alloc_txn_rollback(req->txn); - trie_alloc_request_clear_outputs(req); - return -ENOSPC; - } - req->txn->leaf_id = leaf_id; - return 0; } @@ -2541,6 +2538,10 @@ depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, refcount_set(&stack->count, 1); counters[DEPOT_COUNTER_REFD_ALLOCS]++; counters[DEPOT_COUNTER_REFD_INUSE]++; + } else if (flags & STACK_DEPOT_FLAG_COUNTABLE) { + atomic_set(&stack->page_count, REFCOUNT_SATURATED); + counters[DEPOT_COUNTER_PERSIST_COUNT]++; + counters[DEPOT_COUNTER_PERSIST_BYTES] += record_size; } else { /* Warn on attempts to switch to refcounting this entry. */ refcount_set(&stack->count, REFCOUNT_SATURATED); @@ -2940,7 +2941,7 @@ bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count) if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_COUNTABLE))) return false; - raw = refcount_read(&stack->count); + raw = atomic_read(&stack->page_count); if (raw == REFCOUNT_SATURATED) return false; if (WARN_ON_ONCE(raw <= 0)) @@ -2964,7 +2965,7 @@ void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count) if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_COUNTABLE))) return; - refcount_set(&stack->count, (int)count); + atomic_set(&stack->page_count, (int)count); } bool __stack_depot_inc_count(depot_stack_handle_t handle, @@ -2990,13 +2991,11 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, new = 1 + (int)count; /* - * Intentional refcount_t internals use: no helper conditionally - * converts the persistent REFCOUNT_SATURATED sentinel to a positive - * page_owner count. The first cmpxchg only performs that one-way - * transition; normal counted records continue through a checked cmpxchg - * loop so overflow cannot recreate the saturated sentinel. + * The first cmpxchg only performs the one-way transition from the persistent + * sentinel to a page_owner count. Normal counted records continue through a + * checked cmpxchg loop so overflow cannot recreate the saturated sentinel. */ - if (atomic_try_cmpxchg(&stack->count.refs, &old, new)) { + if (atomic_try_cmpxchg(&stack->page_count, &old, new)) { was_saturated = true; } else { /* cmpxchg reloads @old before each retry check. */ @@ -3008,7 +3007,7 @@ bool __stack_depot_inc_count(depot_stack_handle_t handle, if (count > (unsigned int)INT_MAX - (unsigned int)old) return false; new = old + (int)count; - } while (!atomic_try_cmpxchg(&stack->count.refs, &old, new)); + } while (!atomic_try_cmpxchg(&stack->page_count, &old, new)); } if (new_count) @@ -3034,9 +3033,8 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, return false; /* - * Intentional refcount_t internals use: refcount_sub_and_test() would - * saturate on underflow, but page_owner accounting must warn and leave - * the existing count unchanged. + * refcount_sub_and_test() would saturate on underflow, but page_owner + * accounting must warn and leave the existing count unchanged. */ /* The first cmpxchg failure reloads @old before retry checks. */ old = INT_MAX; @@ -3060,7 +3058,7 @@ bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, } new = old - (int)count; - } while (!atomic_try_cmpxchg(&stack->count.refs, &old, new)); + } while (!atomic_try_cmpxchg(&stack->page_count, &old, new)); return !new; } From 98b5919bbbb8459ad11aa30b7b0d79f8fbe26587 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 25 Jun 2026 14:02:48 +0100 Subject: [PATCH 123/129] KRN-1117: Simplify stackdepot trie and count handling Remove the remaining helper layers and defensive state that obscured the trie backend without enforcing real invariants. Keep page_owner on hash-backed countable records, make hash lookup separate plain, refcounted, and countable lifetimes, and let page_owner use struct stack_record counts directly. Fold single-use trie walkers, remove duplicated validation and derived side-table state, use RCU accessors consistently for published child arrays, and keep arch frame decompression hooks to the production contract. Add KUnit coverage for countable records not aliasing plain or refcounted hash records. Signed-off-by: Caleb Kan --- arch/arm64/include/asm/stackdepot.h | 6 - arch/x86/include/asm/stackdepot.h | 6 - include/linux/stackdepot.h | 125 +- lib/Kconfig.debug | 2 +- lib/stackdepot.c | 1944 ++++----------------------- lib/tests/stackdepot_kunit.c | 106 +- mm/kmsan/report.c | 5 +- mm/page_owner.c | 187 +-- 8 files changed, 452 insertions(+), 1929 deletions(-) diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h index 6bd63d0f3d5c4..70bd2d60cd07b 100644 --- a/arch/arm64/include/asm/stackdepot.h +++ b/arch/arm64/include/asm/stackdepot.h @@ -26,9 +26,6 @@ arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) { u32 candidate; - if (!low) - return false; - candidate = (u32)(frame - (unsigned long)_text); if (arch_stack_depot_frame_from_low(candidate) != frame) return false; @@ -40,9 +37,6 @@ arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) static inline bool arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) { - if (!frame) - return false; - *frame = arch_stack_depot_frame_from_low(low); return true; } diff --git a/arch/x86/include/asm/stackdepot.h b/arch/x86/include/asm/stackdepot.h index 46e0cb0e7eb4c..5bd7c58a167e6 100644 --- a/arch/x86/include/asm/stackdepot.h +++ b/arch/x86/include/asm/stackdepot.h @@ -19,9 +19,6 @@ static_assert(STACK_DEPOT_X86_64_FRAME_PREFIX != 0); static inline bool arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) { - if (!low) - return false; - if ((frame & ~STACK_DEPOT_X86_64_FRAME_LOW_MASK) != STACK_DEPOT_X86_64_FRAME_PREFIX) return false; @@ -33,9 +30,6 @@ arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) static inline bool arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) { - if (!frame) - return false; - *frame = STACK_DEPOT_X86_64_FRAME_PREFIX | low; return true; } diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 5569f7aacb0cd..d21ddfaba3966 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -21,6 +21,8 @@ #define _LINUX_STACKDEPOT_H #include +#include +#include typedef u32 depot_stack_handle_t; @@ -39,6 +41,44 @@ typedef u32 depot_stack_handle_t; #define DEPOT_POOL_INDEX_BITS (DEPOT_HANDLE_BITS - DEPOT_OFFSET_BITS - \ STACK_DEPOT_EXTRA_BITS) +#ifdef CONFIG_STACKDEPOT +/* Compact structure that stores a reference to a stack. */ +union handle_parts { + depot_stack_handle_t handle; + struct { + u32 pool_index_plus_1 : DEPOT_POOL_INDEX_BITS; + u32 offset : DEPOT_OFFSET_BITS; + u32 extra : STACK_DEPOT_EXTRA_BITS; + }; +}; + +struct stack_record { + struct list_head hash_list; /* Links in the hash table */ + u32 hash; /* Hash in hash table */ + u16 size; /* Number of stored frames */ + u16 flags; + union handle_parts handle; /* Constant after initialization */ + refcount_t count; + union { + unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; /* Frames */ + struct { + /* + * An important invariant of the implementation is to + * only place a stack record onto the freelist iff its + * refcount is zero. Because stack records with a zero + * refcount are never considered as valid, it is safe to + * union @entries and freelist management state below. + * Conversely, as soon as an entry is off the freelist + * and its refcount becomes non-zero, the below must not + * be accessed until being placed back on the freelist. + */ + struct list_head free_list; /* Links in the freelist */ + unsigned long rcu_state; /* RCU cookie */ + }; + }; +}; +#endif + typedef u32 depot_flags_t; /* @@ -49,8 +89,8 @@ typedef u32 depot_flags_t; #define STACK_DEPOT_FLAG_GET ((depot_flags_t)0x0002) #define STACK_DEPOT_FLAG_COUNTABLE ((depot_flags_t)0x0004) -#define STACK_DEPOT_FLAGS_MASK (STACK_DEPOT_FLAG_CAN_ALLOC | \ - STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE) +#define STACK_DEPOT_FLAGS_NUM 3 +#define STACK_DEPOT_FLAGS_MASK ((depot_flags_t)((1 << STACK_DEPOT_FLAGS_NUM) - 1)) /* * Using stack depot requires its initialization, which can be done in 3 ways: @@ -109,10 +149,9 @@ static inline int stack_depot_early_init(void) { return 0; } * trace is no longer required to avoid overflowing the refcount. * * If STACK_DEPOT_FLAG_COUNTABLE is set in @depot_flags, stack depot stores the - * stack in a distinct hash-backed record mode that supports the internal count - * helpers. This flag does not imply %STACK_DEPOT_FLAG_CAN_ALLOC and is mutually - * exclusive with %STACK_DEPOT_FLAG_GET. Countable records do not deduplicate - * with non-countable records that have the same frames. + * stack in hash-backed storage for callers that need direct stack_record count + * access. This flag does not imply %STACK_DEPOT_FLAG_CAN_ALLOC and is mutually + * exclusive with %STACK_DEPOT_FLAG_GET. * * When trie storage is enabled, persistent non-refcounted saves use trie * storage. Constrained contexts remain best effort and can return 0 if a @@ -152,80 +191,16 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags); /** - * __stack_depot_get_count - Get a counted stack record count - * - * @handle: Stack depot handle - * @count: Pointer to store the count - * - * This function is only for internal purposes. - * The returned count is an unsynchronized snapshot for diagnostics. - * @handle must be hash-backed and @count must be valid. - * - * Return: true on success, false if the stack record is not in counted mode. - */ -bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count); - -/** - * __stack_depot_set_count - Set a stack record count - * - * @handle: Stack depot handle - * @count: Count to set - * - * This function is only for internal purposes. - * @handle must be hash-backed, and @count must be greater than 0 and less than - * or equal to %INT_MAX. - * Callers that use this to switch a saturated record to counted mode must - * separately make the record discoverable by their own tracking structure. - * Callers must have exclusive access to the stack record count. - */ -void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count); - -/** - * __stack_depot_inc_count - Increment a stack record count - * - * @handle: Stack depot handle - * @count: Count to add - * @new_count: Optional storage for whether this call performed the first - * counted increment - * - * This function is only for internal purposes. - * @handle must be hash-backed. @count must be greater than 0 and less than or - * equal to %INT_MAX - 1 so the saturated-to-counted transition can - * store the stack_list marker plus @count without overflowing. - * - * Persistent stack records start with refcount set to %REFCOUNT_SATURATED. If - * this helper switches a saturated record to counted mode, it stores @count + 1. - * For records already in counted mode, cumulative overflow is rejected and - * leaves the count unchanged. If @new_count is non-NULL, it is set to true when - * this call switches the record from saturated to counted and false otherwise. - * If a racing decrement brings an already-counted diagnostic record to zero, - * this helper does not resurrect it. - * Callers must ensure @handle remains valid for the duration of this call. - * - * Return: true if @count was applied, false otherwise. - */ -bool __stack_depot_inc_count(depot_stack_handle_t handle, - unsigned int count, - bool *new_count); - -/** - * __stack_depot_dec_count_and_test - Decrement a stack record count + * __stack_depot_get_stack_record - Get a hash-backed stack record * * @handle: Stack depot handle - * @count: Count to subtract * - * This function is only for internal purposes. - * @handle must be hash-backed. @count must be greater than 0 and less than or - * equal to %INT_MAX. + * This function is only for internal purposes. @handle must have been saved + * with %STACK_DEPOT_FLAG_COUNTABLE. * - * Return: true if the resulting count is 0, false if the resulting count is - * non-zero, the stack record is not in counted mode, or @count is greater than - * the current count. Saturated persistent records are not in counted mode and - * fail closed without changing the record. Underflow attempts warn and leave the - * count unchanged. + * Return: Returns a pointer to a stack_record struct. */ -bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, - unsigned int count); +struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle); /** * stack_depot_fetch - Fetch a stack trace from stack depot diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 5ac3d34edb84c..3a1e7c9e6c1bd 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2713,7 +2713,7 @@ config STACKDEPOT_KUNIT_TEST help Enable this option to test stack depot API behavior at boot. This test is built in because it exercises internal, non-exported - stack depot helpers. + stack depot helpers, so KUNIT must also be built in. KUnit tests run during boot and output the results to the debug log in TAP format (https://testanything.org/). Only useful for kernel diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 24c033a8cc62d..40299526beb1e 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -16,7 +16,6 @@ #define pr_fmt(fmt) "stackdepot: " fmt -#include #include #include #include @@ -73,7 +72,7 @@ struct stack_depot_frame_run { u8 mode; }; -static_assert(CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(unsigned long) <= U16_MAX); +static_assert(CONFIG_STACKDEPOT_MAX_FRAMES <= U16_MAX); struct stack_depot_trie_node; struct stack_depot_trie_child_array; @@ -95,10 +94,11 @@ struct stack_depot_trie_root { }; struct stack_depot_trie_lookup { - const struct stack_depot_trie_node *parent; + const struct stack_depot_trie_child_array *children; const struct stack_depot_trie_node *node; enum stack_depot_trie_lookup_status status; unsigned int matched; + unsigned int pos; }; struct stack_depot_trie_leaf_update { @@ -165,7 +165,6 @@ static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id); static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle); static u32 __stack_depot_trie_max_leaf_id(void); static int __stack_depot_trie_side_table_init(gfp_t gfp_flags); -static bool __stack_depot_trie_side_table_prealloc_needed(void); static int __stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, struct stack_depot_trie_side_prealloc *prealloc); static void @@ -211,7 +210,7 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, u32 *leaf_id); static void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn); static int trie_side_publish(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, u32 fresh_leaf_id); + unsigned int nr_updates); static int __stack_depot_frame_run_init(const unsigned long *entries, unsigned int nr_entries, struct stack_depot_frame_run *run); @@ -243,8 +242,7 @@ __stack_depot_trie_append_chain(const struct stack_depot_trie_node *parent, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, const struct stack_depot_trie_node **head, - const struct stack_depot_trie_node **tail, - unsigned int *nr_used); + const struct stack_depot_trie_node **tail); static int __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, const struct stack_depot_trie_node *parent, @@ -265,10 +263,7 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, *child_slots, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, - size_t new_storage_size, - u32 fresh_leaf_id, - const struct stack_depot_trie_node **tail, - unsigned int *nr_used); + size_t new_storage_size); static int __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, const struct stack_depot_trie_node *parent, @@ -294,21 +289,6 @@ __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, const struct stack_depot_trie_node *old_tail, const struct stack_depot_trie_node *new_head); static int -__stack_depot_trie_split_tail_plan(const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, unsigned int *nr_runs); -static int -__stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, - struct stack_depot_trie_node *parent, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, - void *new_storage, size_t new_storage_size); -static int __stack_depot_trie_child_array_insert(const struct stack_depot_trie_child_array *old, const struct stack_depot_trie_node *child, void *new_storage, size_t new_storage_size); @@ -316,6 +296,7 @@ __stack_depot_trie_child_array_insert(const struct stack_depot_trie_child_array /* * The pool_index is offset by 1 so the first record does not have a 0 handle. */ +/* Parsed before mm_core_init(); trie handle decoding assumes this is then fixed. */ static unsigned int stack_max_pools __read_mostly = MIN((1LL << DEPOT_POOL_INDEX_BITS) - 1, 8192); @@ -356,48 +337,6 @@ MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); #define DEPOT_POOL_INDEX_MASK ((1U << DEPOT_POOL_INDEX_BITS) - 1) #define DEPOT_OFFSET_MASK ((1U << DEPOT_OFFSET_BITS) - 1) -#define STACK_RECORD_FLAG_REFCOUNTED BIT(0) -#define STACK_RECORD_FLAG_COUNTABLE BIT(1) - -/* Compact structure that stores a reference to a stack. */ -union handle_parts { - depot_stack_handle_t handle; - struct { - u32 pool_index_plus_1 : DEPOT_POOL_INDEX_BITS; - u32 offset : DEPOT_OFFSET_BITS; - u32 extra : STACK_DEPOT_EXTRA_BITS; - }; -}; - -struct stack_record { - struct list_head hash_list; /* Links in the hash table */ - u32 hash; /* Hash in hash table */ - u16 size; /* Number of stored frames */ - u16 flags; - union handle_parts handle; /* Constant after initialization */ - union { - refcount_t count; - atomic_t page_count; - }; - union { - unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; /* Frames */ - struct { - /* - * An important invariant of the implementation is to - * only place a stack record onto the freelist iff its - * refcount is zero. Because stack records with a zero - * refcount are never considered as valid, it is safe to - * union @entries and freelist management state below. - * Conversely, as soon as an entry is off the freelist - * and its refcount becomes non-zero, the below must not - * be accessed until being placed back on the freelist. - */ - struct list_head free_list; /* Links in the freelist */ - unsigned long rcu_state; /* RCU cookie */ - }; - }; -}; - struct stack_depot_trie_node { /* Parent links let fetch rebuild a full stack from a leaf to the root. */ const struct stack_depot_trie_node *parent; @@ -454,8 +393,6 @@ static void **stack_pools; static void *new_pool; /* Number of pools in stack_pools. */ static int pools_num; -static unsigned long pools_min_addr; -static unsigned long pools_max_addr; /* Offset to the unused space in the currently used pool. */ static size_t pool_offset = DEPOT_POOL_SIZE; /* Freelist of stack records within stack_pools. */ @@ -480,7 +417,6 @@ static struct list_head free_trie_nodes[STACK_DEPOT_TRIE_FREE_CLASSES]; static DECLARE_BITMAP(free_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); static DECLARE_BITMAP(pending_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); static DECLARE_BITMAP(free_trie_node_map, STACK_DEPOT_TRIE_FREE_CLASSES); -static unsigned int free_trie_pending_nodes; static bool free_trie_objects_initialized; /* The lock must be held when performing pool or freelist modifications. */ static DEFINE_RAW_SPINLOCK(pool_lock); @@ -592,7 +528,7 @@ struct stack_depot_trie_side_dir { }; struct stack_depot_trie_side_root { - unsigned int nr_dirs; + unsigned int dir_capacity; struct stack_depot_trie_side_dir __rcu *dirs[]; }; @@ -600,7 +536,6 @@ static struct stack_depot_trie_side_root __rcu *trie_side_table_root; static DEFINE_RAW_SPINLOCK(trie_side_table_lock); static unsigned int trie_side_table_nr_dirs; static unsigned int trie_side_table_nr_chunks; -static unsigned int trie_side_table_root_size; static u32 trie_side_table_next_id; /* Lock order: workspace_lock -> pool_lock -> trie_side_table_lock. */ @@ -655,7 +590,7 @@ static struct stack_depot_trie_side_dir *trie_side_table_load_dir(unsigned int r struct stack_depot_trie_side_root *root_vec; root_vec = trie_side_table_load_root(); - if (!root_vec || root >= root_vec->nr_dirs) + if (!root_vec || root >= root_vec->dir_capacity) return NULL; /* Pairs with trie_side_table_publish_dir(); lookup is lockless. */ return rcu_dereference_check(root_vec->dirs[root], @@ -669,7 +604,7 @@ static void trie_side_table_publish_dir(unsigned int root, struct stack_depot_trie_side_root *root_vec; root_vec = trie_side_table_load_root(); - if (!root_vec || root >= root_vec->nr_dirs) + if (!root_vec || root >= root_vec->dir_capacity) return; /* Publish the zeroed directory before readers can load it locklessly. */ rcu_assign_pointer(root_vec->dirs[root], dir); @@ -690,6 +625,7 @@ __stack_depot_trie_side_table_prepare_id(struct stack_depot_trie_side_prealloc * { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; + struct stack_depot_trie_side_root *root_vec; unsigned long flags; unsigned int root; unsigned int idx; @@ -697,8 +633,9 @@ __stack_depot_trie_side_table_prepare_id(struct stack_depot_trie_side_prealloc * raw_spin_lock_irqsave(&trie_side_table_lock, flags); /* Prepare the next slot without making the ID visible for reuse yet. */ + root_vec = trie_side_table_load_root(); /* Failed or disabled trie init means no leaf IDs can be allocated. */ - if (!trie_side_table_is_initialized()) + if (!root_vec) goto out; if (!prealloc) goto out; @@ -709,8 +646,8 @@ __stack_depot_trie_side_table_prepare_id(struct stack_depot_trie_side_prealloc * goto out_clear_id; root = trie_side_table_root_index(id); - /* Should be impossible when trie_side_table_max_id/root_size agree. */ - if (root >= trie_side_table_root_size) + /* Should be impossible when trie_side_table_max_id and capacity agree. */ + if (root >= root_vec->dir_capacity) goto out_clear_id; dir = trie_side_table_load_dir(root); @@ -818,8 +755,7 @@ trie_side_table_install(struct stack_depot_trie_side_root *root_vec, if (!root_vec || !root_size || !max_id) return -EINVAL; - WRITE_ONCE(trie_side_table_root_size, root_size); - root_vec->nr_dirs = root_size; + root_vec->dir_capacity = root_size; WRITE_ONCE(trie_side_table_nr_dirs, 0); WRITE_ONCE(trie_side_table_nr_chunks, 0); WRITE_ONCE(trie_side_table_max_id, max_id); @@ -998,38 +934,6 @@ static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) return trie_side_table_install(root_vec, root_size, max_leaf_id, NULL, NULL); } -static bool __stack_depot_trie_side_table_prealloc_needed(void) -{ - struct stack_depot_trie_side_dir *dir; - unsigned long flags; - bool needed; - u32 id; - unsigned int root; - - if (!trie_side_table_is_initialized()) - return false; - - raw_spin_lock_irqsave(&trie_side_table_lock, flags); - id = READ_ONCE(trie_side_table_next_id) + 1; - if (!id || id > READ_ONCE(trie_side_table_max_id)) { - needed = false; - goto out; - } - - root = trie_side_table_root_index(id); - if (root >= trie_side_table_root_size) { - needed = false; - goto out; - } - - dir = trie_side_table_load_dir(root); - needed = !dir || !trie_side_table_dir_load_chunk(dir, - trie_side_table_dir_index(id)); -out: - raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); - return needed; -} - static void *trie_side_table_alloc_page(gfp_t gfp_flags, unsigned int order) { struct page *page; @@ -1044,6 +948,7 @@ __stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, struct stack_depot_trie_side_prealloc *prealloc) { struct stack_depot_trie_side_dir *dir; + struct stack_depot_trie_side_root *root_vec; unsigned long flags; bool need_chunk; bool need_dir; @@ -1056,13 +961,18 @@ __stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, return 0; raw_spin_lock_irqsave(&trie_side_table_lock, flags); + root_vec = trie_side_table_load_root(); + if (!root_vec) { + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return 0; + } id = READ_ONCE(trie_side_table_next_id) + 1; if (!id || id > READ_ONCE(trie_side_table_max_id)) { raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); return 0; } root = trie_side_table_root_index(id); - if (root >= trie_side_table_root_size) { + if (root >= root_vec->dir_capacity) { raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); return 0; } @@ -1107,21 +1017,25 @@ __stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_preallo } static struct stack_depot_trie_side_entry * -trie_side_table_chunk_locked(u32 id, u32 fresh_leaf_id, unsigned int *slot) +trie_side_table_chunk_locked(u32 id, unsigned int *slot) { struct stack_depot_trie_side_entry *chunk; struct stack_depot_trie_side_dir *dir; + struct stack_depot_trie_side_root *root_vec; + u32 next_id; unsigned int root; lockdep_assert_held(&trie_side_table_lock); - if (!trie_side_table_is_initialized() || !id) + root_vec = trie_side_table_load_root(); + if (!root_vec || !id) return NULL; - if (id > trie_side_table_next_id && id != fresh_leaf_id) + next_id = trie_side_table_next_id; + if (id > next_id + 1) return NULL; root = trie_side_table_root_index(id); - if (root >= trie_side_table_root_size) + if (root >= root_vec->dir_capacity) return NULL; dir = trie_side_table_load_dir(root); @@ -1141,13 +1055,10 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( struct stack_depot_trie_side_dir *dir; unsigned int root; - if (!trie_side_table_is_initialized() || !id) + if (!id) return NULL; root = trie_side_table_root_index(id); - if (root >= trie_side_table_root_size) - return NULL; - dir = trie_side_table_load_dir(root); if (!dir) return NULL; @@ -1160,15 +1071,17 @@ static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup( static size_t __stack_depot_trie_side_table_bytes(void) { + struct stack_depot_trie_side_root *root_vec; unsigned int nr_dirs; unsigned int nr_chunks; size_t dir_bytes; size_t bytes; size_t root_bytes; - if (!trie_side_table_is_initialized()) + root_vec = trie_side_table_load_root(); + if (!root_vec) return 0; - root_bytes = trie_side_table_root_bytes(trie_side_table_root_size); + root_bytes = trie_side_table_root_bytes(root_vec->dir_capacity); if (!root_bytes) return SIZE_MAX; @@ -1293,55 +1206,6 @@ static void *trie_object_init_fresh(void *ptr, size_t size) return trie_object_payload(free); } -static void depot_record_pool_locked(void *pool) -{ - unsigned long start = (unsigned long)pool; - unsigned long end; - - lockdep_assert_held(&pool_lock); - if (!pool || check_add_overflow(start, DEPOT_POOL_SIZE, &end)) - return; - - if (!pools_min_addr || start < pools_min_addr) - pools_min_addr = start; - if (end > pools_max_addr) - pools_max_addr = end; -} - -static bool trie_pool_range_contains_locked(const void *ptr, size_t size) -{ - unsigned long start = (unsigned long)ptr; - unsigned long end; - unsigned int pools = READ_ONCE(pools_num); - unsigned long pool_start; - unsigned int i; - - lockdep_assert_held(&pool_lock); - - /* Reject stale/non-pool storage before putting COW-retired bytes on freelists. */ - if (!ptr || !size || check_add_overflow(start, size, &end)) - return false; - if (!stack_pools) - return false; - if (pools_min_addr && (start < pools_min_addr || end > pools_max_addr)) - return false; - - for (i = 0; i < pools; i++) { - if (!stack_pools[i]) - continue; - pool_start = (unsigned long)stack_pools[i]; - if (start >= pool_start && end <= pool_start + DEPOT_POOL_SIZE) - return true; - } - - return false; -} - -static bool trie_pool_contains_locked(const void *ptr) -{ - return trie_pool_range_contains_locked(ptr, 1); -} - static bool trie_pool_mark_contains(const struct stack_depot_trie_pool_mark *mark, const void *ptr) { @@ -1366,8 +1230,6 @@ static void trie_free_object_locked(const void *ptr, unsigned long rcu_state) return; trie_free_object_buckets_init_locked(); free = trie_object_header(ptr); - if (!trie_pool_contains_locked(free)) - return; free->rcu_state = rcu_state; class = trie_free_class(free->size); INIT_LIST_HEAD(&free->list); @@ -1406,7 +1268,6 @@ static void trie_drain_free_object_node_locked(struct stack_depot_trie_free_obje trie_add_free_node_locked(free->pending_node, free->pending_node_size); free->pending_node = NULL; free->pending_node_size = 0; - free_trie_pending_nodes--; } static void trie_drain_pending_objects_locked(void) @@ -1417,8 +1278,7 @@ static void trie_drain_pending_objects_locked(void) lockdep_assert_held(&pool_lock); - if (!free_trie_pending_nodes && - bitmap_empty(pending_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES)) + if (bitmap_empty(pending_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES)) return; for_each_set_bit(class, pending_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES) { @@ -1520,21 +1380,17 @@ trie_retire_object_node_locked(const void *ptr, trie_free_object_buckets_init_locked(); free = trie_object_header(ptr); - if (trie_pool_contains_locked(free)) { - free->pending_node = NULL; - free->pending_node_size = 0; - size = __stack_depot_trie_pool_alloc_size(node_size); - if (node && size >= sizeof(struct stack_depot_trie_free_node) && - trie_pool_range_contains_locked(node, size)) { - free->pending_node = (struct stack_depot_trie_node *)node; - free->pending_node_size = size; - free_trie_pending_nodes++; - } - free->rcu_state = get_state_synchronize_rcu(); - trie_free_list_add(&free->list, pending_trie_objects, - pending_trie_object_map, - trie_free_class(free->size), true); + free->pending_node = NULL; + free->pending_node_size = 0; + size = __stack_depot_trie_pool_alloc_size(node_size); + if (node && size >= sizeof(struct stack_depot_trie_free_node)) { + free->pending_node = (struct stack_depot_trie_node *)node; + free->pending_node_size = size; } + free->rcu_state = get_state_synchronize_rcu(); + trie_free_list_add(&free->list, pending_trie_objects, + pending_trie_object_map, + trie_free_class(free->size), true); } static void trie_retire_object_node(const void *ptr, @@ -1600,7 +1456,6 @@ __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc) { - bool needs_side_prealloc; bool can_alloc; int ret = 0; @@ -1610,15 +1465,12 @@ __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && gfpflags_allow_spinning(alloc_flags); - needs_side_prealloc = __stack_depot_trie_side_table_prealloc_needed(); if (can_alloc && !READ_ONCE(new_pool)) *pool_prealloc = __stack_depot_trie_pool_prealloc(alloc_flags); - if (needs_side_prealloc && !can_alloc) - return -ENOSPC; - if (can_alloc && needs_side_prealloc) + if (can_alloc) ret = __stack_depot_trie_side_table_prealloc(alloc_flags, side_prealloc); - if (needs_side_prealloc && ret) + if (ret) return -ENOSPC; return 0; } @@ -1660,18 +1512,6 @@ static bool stack_depot_trie_pool_rollback_locked(const struct stack_depot_trie_ return ret; } -static bool stack_depot_trie_pool_rollback(const struct stack_depot_trie_pool_mark *mark) -{ - unsigned long flags; - bool ret; - - raw_spin_lock_irqsave(&pool_lock, flags); - ret = stack_depot_trie_pool_rollback_locked(mark); - raw_spin_unlock_irqrestore(&pool_lock, flags); - - return ret; -} - static int trie_pool_add_object_size(size_t size, size_t *total) { size_t alloc_size; @@ -1873,21 +1713,6 @@ static void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn memset(txn, 0, sizeof(*txn)); } -static void trie_alloc_request_clear_outputs(struct stack_depot_trie_alloc_request *req) -{ - unsigned int i; - - if (!req) - return; - - if (req->storage) - *req->storage = NULL; - for (i = 0; req->node_slots && i < req->nr_node_slots; i++) - req->node_slots[i].node = NULL; - for (i = 0; req->child_slots && i < req->nr_child_slots; i++) - req->child_slots[i].array = NULL; -} - static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req) { u32 leaf_id; @@ -1907,7 +1732,6 @@ static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_re if (ret) { req->txn->leaf_id = 0; __stack_depot_trie_alloc_txn_rollback(req->txn); - trie_alloc_request_clear_outputs(req); return ret; } @@ -2032,10 +1856,8 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, unsigned int nr_scratch, u32 *leaf_id) { struct stack_depot_trie_alloc_txn *txn; - const struct stack_depot_trie_node *tail; u32 id; void *storage; - unsigned int nr_used; int ret; if (!root || !req || !req->txn || !leaf_id) @@ -2054,8 +1876,7 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, nr_entries, req->node_slots, req->nr_node_slots, req->child_slots, req->nr_child_slots, scratch, - nr_scratch, storage, req->storage_size, - id, &tail, &nr_used); + nr_scratch, storage, req->storage_size); if (ret) goto rollback; @@ -2067,25 +1888,29 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, rollback: trie_pool_release_reused_objects(req); __stack_depot_trie_alloc_txn_rollback(req->txn); - trie_alloc_request_clear_outputs(req); return ret; } static void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn) { + unsigned long flags; + if (!txn) return; - stack_depot_trie_pool_rollback(&txn->pool); + raw_spin_lock_irqsave(&pool_lock, flags); + stack_depot_trie_pool_rollback_locked(&txn->pool); + raw_spin_unlock_irqrestore(&pool_lock, flags); txn->leaf_id = 0; memset(&txn->pool, 0, sizeof(txn->pool)); } static int trie_side_publish_locked(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, u32 fresh_leaf_id) + unsigned int nr_updates) { struct stack_depot_trie_side_entry *chunks[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; unsigned int slots[STACK_DEPOT_TRIE_MAX_LEAF_UPDATES]; + u32 next_leaf_id = trie_side_table_next_id + 1; unsigned int i; lockdep_assert_held(&trie_side_table_lock); @@ -2097,10 +1922,10 @@ static int trie_side_publish_locked(const struct stack_depot_trie_leaf_update *u if (!updates[i].leaf) return -EINVAL; - chunks[i] = trie_side_table_chunk_locked(leaf_id, fresh_leaf_id, &slots[i]); + chunks[i] = trie_side_table_chunk_locked(leaf_id, &slots[i]); if (!chunks[i]) return -EINVAL; - if (leaf_id == fresh_leaf_id && + if (leaf_id == next_leaf_id && trie_side_table_load_leaf(chunks[i], slots[i])) return -EINVAL; } @@ -2113,13 +1938,13 @@ static int trie_side_publish_locked(const struct stack_depot_trie_leaf_update *u } static int trie_side_publish(const struct stack_depot_trie_leaf_update *updates, - unsigned int nr_updates, u32 fresh_leaf_id) + unsigned int nr_updates) { unsigned long flags; int ret; raw_spin_lock_irqsave(&trie_side_table_lock, flags); - ret = trie_side_publish_locked(updates, nr_updates, fresh_leaf_id); + ret = trie_side_publish_locked(updates, nr_updates); raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); return ret; } @@ -2357,7 +2182,6 @@ static bool depot_init_pool(void **prealloc) /* Save reference to the pool to be used by depot_fetch_stack(). */ stack_pools[pools_num] = new_pool; - depot_record_pool_locked(new_pool); /* * Stack depot tries to keep an extra pool allocated even before it runs @@ -2483,15 +2307,6 @@ static inline size_t depot_stack_record_size(struct stack_record *s, unsigned in return ALIGN(sizeof(struct stack_record) - unused, 1 << DEPOT_STACK_ALIGN); } -static u16 stack_record_flags(depot_flags_t flags) -{ - if (flags & STACK_DEPOT_FLAG_GET) - return STACK_RECORD_FLAG_REFCOUNTED; - if (flags & STACK_DEPOT_FLAG_COUNTABLE) - return STACK_RECORD_FLAG_COUNTABLE; - return 0; -} - /* Allocates a new stack in a stack depot pool. */ static struct stack_record * depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, @@ -2530,7 +2345,7 @@ depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, /* Save the stack trace. */ stack->hash = hash; stack->size = nr_entries; - stack->flags = stack_record_flags(flags); + stack->flags = flags & STACK_DEPOT_FLAGS_MASK; /* stack->handle is already filled in by depot_pop_free_pool(). */ memcpy(stack->entries, entries, flex_array_size(stack, entries, nr_entries)); @@ -2538,10 +2353,6 @@ depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, refcount_set(&stack->count, 1); counters[DEPOT_COUNTER_REFD_ALLOCS]++; counters[DEPOT_COUNTER_REFD_INUSE]++; - } else if (flags & STACK_DEPOT_FLAG_COUNTABLE) { - atomic_set(&stack->page_count, REFCOUNT_SATURATED); - counters[DEPOT_COUNTER_PERSIST_COUNT]++; - counters[DEPOT_COUNTER_PERSIST_BYTES] += record_size; } else { /* Warn on attempts to switch to refcounting this entry. */ refcount_set(&stack->count, REFCOUNT_SATURATED); @@ -2657,6 +2468,7 @@ static inline struct stack_record *find_stack(struct list_head *bucket, unsigned long *entries, int size, u32 hash, depot_flags_t flags) { + depot_flags_t mode = STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE; struct stack_record *stack, *ret = NULL; /* @@ -2671,8 +2483,10 @@ static inline struct stack_record *find_stack(struct list_head *bucket, rcu_read_lock_sched_notrace(); list_for_each_entry_rcu(stack, bucket, hash_list) { - if (stack->hash != hash || stack->size != size || - stack->flags != stack_record_flags(flags)) + if (stack->hash != hash || stack->size != size) + continue; + /* Plain, refcounted, and countable records have distinct lifetimes. */ + if ((stack->flags & mode) != (flags & mode)) continue; /* @@ -2927,144 +2741,24 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, } EXPORT_SYMBOL_GPL(stack_depot_save); -bool __stack_depot_get_count(depot_stack_handle_t handle, unsigned int *count) -{ - struct stack_record *stack; - int raw; - - if (WARN_ON_ONCE(!handle || !count || __stack_depot_trie_leaf_id(handle))) - return false; - - stack = depot_fetch_stack(handle); - if (!stack) - return false; - if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_COUNTABLE))) - return false; - - raw = atomic_read(&stack->page_count); - if (raw == REFCOUNT_SATURATED) - return false; - if (WARN_ON_ONCE(raw <= 0)) - return false; - - *count = (unsigned int)raw; - return true; -} - -void __stack_depot_set_count(depot_stack_handle_t handle, unsigned int count) -{ - struct stack_record *stack; - - if (WARN_ON_ONCE(!handle || !count || count > (unsigned int)INT_MAX || - __stack_depot_trie_leaf_id(handle))) - return; - - stack = depot_fetch_stack(handle); - if (!stack) - return; - if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_COUNTABLE))) - return; - - atomic_set(&stack->page_count, (int)count); -} - -bool __stack_depot_inc_count(depot_stack_handle_t handle, - unsigned int count, - bool *new_count) -{ - struct stack_record *stack; - int new; - int old = REFCOUNT_SATURATED; - bool was_saturated = false; - - if (new_count) - *new_count = false; - if (WARN_ON_ONCE(!handle || !count || count > (unsigned int)INT_MAX - 1 || - __stack_depot_trie_leaf_id(handle))) - return false; - - stack = depot_fetch_stack(handle); - if (!stack) - return false; - if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_COUNTABLE))) - return false; - - new = 1 + (int)count; - /* - * The first cmpxchg only performs the one-way transition from the persistent - * sentinel to a page_owner count. Normal counted records continue through a - * checked cmpxchg loop so overflow cannot recreate the saturated sentinel. - */ - if (atomic_try_cmpxchg(&stack->page_count, &old, new)) { - was_saturated = true; - } else { - /* cmpxchg reloads @old before each retry check. */ - do { - if (old == REFCOUNT_SATURATED) - return false; - if (WARN_ON_ONCE(old <= 0)) - return false; - if (count > (unsigned int)INT_MAX - (unsigned int)old) - return false; - new = old + (int)count; - } while (!atomic_try_cmpxchg(&stack->page_count, &old, new)); - } - - if (new_count) - *new_count = was_saturated; - return true; -} - -bool __stack_depot_dec_count_and_test(depot_stack_handle_t handle, - unsigned int count) +struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle) { struct stack_record *stack; - int new; - int old; - if (WARN_ON_ONCE(!handle || !count || count > (unsigned int)INT_MAX || - __stack_depot_trie_leaf_id(handle))) - return false; + if (!handle) + return NULL; + if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) + return NULL; stack = depot_fetch_stack(handle); if (!stack) - return false; - if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_COUNTABLE))) - return false; - - /* - * refcount_sub_and_test() would saturate on underflow, but page_owner - * accounting must warn and leave the existing count unchanged. - */ - /* The first cmpxchg failure reloads @old before retry checks. */ - old = INT_MAX; - do { - bool underflow; - - /* - * Retry checks use the observed count as this operation's - * linearization point. A racing increment not observed here is - * ordered after this decrement. - */ - if (old == REFCOUNT_SATURATED) - return false; - if (WARN_ON_ONCE(old <= 0)) - return false; - - underflow = count > (unsigned int)old; - if (underflow) { - WARN_RATELIMIT(underflow, "stack depot count underflow\n"); - return false; - } - - new = old - (int)count; - } while (!atomic_try_cmpxchg(&stack->page_count, &old, new)); + return NULL; + if (WARN_ON_ONCE(!(stack->flags & STACK_DEPOT_FLAG_COUNTABLE))) + return NULL; - return !new; + return stack; } -static bool stack_depot_ranges_overlap(const void *a, size_t a_size, - const void *b, size_t b_size); static int stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *array, unsigned long frame, unsigned int *pos, @@ -3150,45 +2844,6 @@ __stack_depot_frame_run_init(const unsigned long *entries, return frame_run_init_lows(entries, nr_entries, run, NULL, 0); } -static int frame_run_validate_payload(const struct stack_depot_frame_run *run, - const void *src) -{ - unsigned long frame; - unsigned int i; - - if (!src || stack_depot_frame_run_validate(run)) - return -EINVAL; - if (run->mode == STACK_DEPOT_FRAME_RAW) - return 0; - - for (i = 0; i < run->nr_entries; i++) { - u32 low; - - memcpy(&low, (const char *)src + i * sizeof(low), sizeof(low)); - if (!arch_stack_depot_frame_decompress(low, &frame)) - return -EINVAL; - } - - return 0; -} - -static bool stack_depot_ranges_overlap(const void *a, size_t a_size, - const void *b, size_t b_size) -{ - unsigned long a_start = (unsigned long)a; - unsigned long b_start = (unsigned long)b; - unsigned long a_end; - unsigned long b_end; - - if (!a_size || !b_size) - return false; - if (check_add_overflow(a_start, a_size, &a_end) || - check_add_overflow(b_start, b_size, &b_end)) - return true; - - return a_start < b_end && b_start < a_end; -} - static size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) { size_t size; @@ -3222,8 +2877,7 @@ stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, { u32 low; - if (!node || !frame || stack_depot_frame_run_validate(&node->run) || - index >= node->run.nr_entries) + if (!node || !frame || index >= node->run.nr_entries) return -EINVAL; if (node->run.mode == STACK_DEPOT_FRAME_RAW) { @@ -3259,8 +2913,6 @@ trie_node_stack_len(const struct stack_depot_trie_node *node, for (; node; node = trie_load_parent(node)) { if (depth++ >= CONFIG_STACKDEPOT_MAX_FRAMES) return -EINVAL; - if (stack_depot_frame_run_validate(&node->run)) - return -EINVAL; if (total > CONFIG_STACKDEPOT_MAX_FRAMES - node->run.nr_entries) return -EINVAL; total += node->run.nr_entries; @@ -3347,7 +2999,7 @@ __stack_depot_trie_node_init_slice(void *storage, size_t storage_size, if (storage_size < __stack_depot_trie_node_size(&run)) return -EINVAL; src_size = __stack_depot_trie_node_size(&src->run); - if (!src_size || stack_depot_ranges_overlap(node, storage_size, src, src_size)) + if (!src_size) return -EINVAL; if (parent_node) { if (trie_node_stack_len(parent_node, &parent_len) || @@ -3373,8 +3025,7 @@ __stack_depot_trie_node_match(const struct stack_depot_trie_node *node, unsigned int limit; unsigned int i; - if (!node || !entries || !nr_entries || - stack_depot_frame_run_validate(&node->run)) + if (!node || !entries || !nr_entries) return 0; limit = min(node->run.nr_entries, nr_entries); @@ -3402,266 +3053,37 @@ __stack_depot_trie_node_match(const struct stack_depot_trie_node *node, return i; } -static bool trie_ancestor_overlaps(const struct stack_depot_trie_node *node, - const void *ptr, size_t size) +static const struct stack_depot_trie_node * +trie_load_parent(const struct stack_depot_trie_node *node) { - unsigned int depth = 0; - - for (; node; node = node->parent, depth++) { - size_t child_size; - size_t node_size; - - if (depth >= CONFIG_STACKDEPOT_MAX_FRAMES) - return true; - if (stack_depot_frame_run_validate(&node->run)) - return true; - - node_size = __stack_depot_trie_node_size(&node->run); - if (!node_size) - return true; - if (stack_depot_ranges_overlap(ptr, size, node, node_size)) - return true; - - if (!node->children) - continue; - child_size = trie_child_array_size_for_capacity(node->children->capacity); - if (!child_size) - return true; - if (stack_depot_ranges_overlap(ptr, size, node->children, - child_size)) - return true; - } + const struct stack_depot_trie_node __rcu * const *slot; - return false; + slot = (const struct stack_depot_trie_node __rcu * const *)&node->parent; + return rcu_dereference_check(*slot, + lockdep_is_held(&stack_depot_trie_workspace_lock) || + rcu_read_lock_sched_held()); } -static bool trie_chain_overlaps(const struct stack_depot_trie_node *node, - const void *ptr, size_t size) +static void +trie_publish_parent(struct stack_depot_trie_node *child, + const struct stack_depot_trie_node *parent) { - unsigned int depth = 0; - - for (; node; depth++) { - const struct stack_depot_trie_child_array *children; - size_t child_size; - size_t node_size; - - if (depth >= CONFIG_STACKDEPOT_MAX_FRAMES) - return true; - if (stack_depot_frame_run_validate(&node->run)) - return true; - - node_size = __stack_depot_trie_node_size(&node->run); - if (!node_size) - return true; - if (stack_depot_ranges_overlap(ptr, size, node, node_size)) - return true; - - children = node->children; - if (!children) - break; - if (children->nr_children != 1) - return true; - child_size = trie_child_array_size_for_capacity(children->capacity); - if (!child_size) - return true; - if (stack_depot_ranges_overlap(ptr, size, children, child_size)) - return true; - if (!children->children[0]) - return true; - if (children->children[0]->parent != node) - return true; - node = children->children[0]; - } + const struct stack_depot_trie_node __rcu **slot; - return false; + slot = (const struct stack_depot_trie_node __rcu **)&child->parent; + rcu_assign_pointer(*slot, parent); } -static bool -trie_child_array_subtree_overlaps(const struct stack_depot_trie_child_array *array, - const struct stack_depot_trie_node *parent, - const void *ptr, size_t size) +static const struct stack_depot_trie_child_array ** +trie_publish_slot(struct stack_depot_trie_root *root, + struct stack_depot_trie_node *parent) { - const struct stack_depot_trie_node *node; - size_t array_size; - unsigned int depth = 0; - - if (!array) - return false; - array_size = trie_child_array_size_for_capacity(array->capacity); - if (!array_size) - return true; - if (stack_depot_ranges_overlap(ptr, size, array, array_size)) - return true; - if (!array->nr_children) - return false; - - node = array->children[0]; - if (!node || node->parent != parent) - return true; - - for (;;) { - const struct stack_depot_trie_child_array *children; - const struct stack_depot_trie_child_array *siblings; - const struct stack_depot_trie_node *child; - const struct stack_depot_trie_node *node_parent; - size_t child_size; - size_t node_size; - unsigned int i; - - if (depth >= CONFIG_STACKDEPOT_MAX_FRAMES) - return true; - if (stack_depot_frame_run_validate(&node->run)) - return true; - - node_size = __stack_depot_trie_node_size(&node->run); - if (!node_size) - return true; - if (stack_depot_ranges_overlap(ptr, size, node, node_size)) - return true; - - children = node->children; - if (children) { - child_size = trie_child_array_size_for_capacity(children->capacity); - if (!child_size) - return true; - if (stack_depot_ranges_overlap(ptr, size, children, - child_size)) - return true; - if (children->nr_children) { - child = children->children[0]; - if (!child || child->parent != node) - return true; - node = child; - depth++; - continue; - } - } - - for (;;) { - node_parent = node->parent; - if (node_parent == parent) { - siblings = array; - } else { - if (!node_parent || !node_parent->children) - return true; - siblings = node_parent->children; - } - - for (i = 0; i < siblings->nr_children; i++) { - if (siblings->children[i] == node) - break; - } - if (i == siblings->nr_children) - return true; - if (i + 1 < siblings->nr_children) { - node = siblings->children[i + 1]; - if (!node || node->parent != node_parent) - return true; - break; - } - if (node_parent == parent) - return false; - if (!depth) - return true; - node = node_parent; - depth--; - } - } -} - -static const struct stack_depot_trie_node * -trie_load_parent(const struct stack_depot_trie_node *node) -{ - const struct stack_depot_trie_node __rcu * const *slot; - - slot = (const struct stack_depot_trie_node __rcu * const *)&node->parent; - return rcu_dereference_check(*slot, - lockdep_is_held(&stack_depot_trie_workspace_lock) || - rcu_read_lock_sched_held()); -} - -static void -trie_publish_parent(struct stack_depot_trie_node *child, - const struct stack_depot_trie_node *parent) -{ - const struct stack_depot_trie_node __rcu **slot; - - slot = (const struct stack_depot_trie_node __rcu **)&child->parent; - rcu_assign_pointer(*slot, parent); -} - -static bool -trie_node_slots_subtree_overlap(const struct stack_depot_trie_child_array *array, - const struct stack_depot_trie_node *parent, - const struct stack_depot_trie_node_slot *slots, - unsigned int nr_slots) -{ - unsigned int i; - - for (i = 0; i < nr_slots; i++) { - if (trie_child_array_subtree_overlaps(array, parent, slots[i].node, slots[i].size)) - return true; - } - - return false; -} - -static bool -trie_child_slots_subtree_overlap(const struct stack_depot_trie_child_array *array, - const struct stack_depot_trie_node *parent, - const struct stack_depot_trie_child_array_slot *slots, - unsigned int nr_slots) -{ - unsigned int i; - - for (i = 0; i < nr_slots; i++) { - if (trie_child_array_subtree_overlaps(array, parent, slots[i].array, slots[i].size)) - return true; - } - - return false; -} - -static bool -trie_node_slot_overlaps(const struct stack_depot_trie_node_slot *slots, - unsigned int used, const void *ptr, size_t size) -{ - unsigned int i; - - for (i = 0; i < used; i++) { - if (stack_depot_ranges_overlap(ptr, size, slots[i].node, - slots[i].size)) - return true; - } - - return false; -} - -static bool -trie_child_slot_overlaps(const struct stack_depot_trie_child_array_slot *slots, - unsigned int used, const void *ptr, size_t size) -{ - unsigned int i; - - for (i = 0; i < used; i++) { - if (stack_depot_ranges_overlap(ptr, size, slots[i].array, - slots[i].size)) - return true; - } - - return false; -} - -static const struct stack_depot_trie_child_array ** -trie_publish_slot(struct stack_depot_trie_root *root, - struct stack_depot_trie_node *parent) -{ - if ((root && parent) || (!root && !parent)) - return NULL; - if (root) - return &root->children; - return &parent->children; -} + if ((root && parent) || (!root && !parent)) + return NULL; + if (root) + return &root->children; + return &parent->children; +} static const struct stack_depot_trie_child_array * trie_load_children_slot(const struct stack_depot_trie_child_array * const *slot) @@ -3719,190 +3141,6 @@ trie_child_array_can_append(const struct stack_depot_trie_child_array *array, return pos == nr_children && nr_children < array->capacity; } -static int trie_insert_append_precheck(struct stack_depot_trie_root *root, - struct stack_depot_trie_node *parent, - const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, void *new_storage, - size_t new_storage_size) -{ - const struct stack_depot_trie_child_array **slot; - const struct stack_depot_trie_child_array *children; - size_t size; - unsigned int pos; - bool found; - - if (!entries || !nr_entries) - return -EINVAL; - if (!entries[0]) - return -EINVAL; - if ((nr_node_slots && !node_slots) || (nr_child_slots && !child_slots)) - return -EINVAL; - if (new_storage && - !IS_ALIGNED((unsigned long)new_storage, - __alignof__(struct stack_depot_trie_child_array))) - return -EINVAL; - - slot = trie_publish_slot(root, parent); - if (!slot) - return -EINVAL; - if (new_storage && - stack_depot_ranges_overlap(new_storage, new_storage_size, - slot, sizeof(*slot))) - return -EINVAL; - if (root) { - if (trie_node_slot_overlaps(node_slots, nr_node_slots, slot, - sizeof(*slot))) - return -EINVAL; - if (trie_child_slot_overlaps(child_slots, nr_child_slots, slot, - sizeof(*slot))) - return -EINVAL; - } - if (new_storage && parent && - trie_ancestor_overlaps(parent, new_storage, new_storage_size)) - return -EINVAL; - if (new_storage && - (trie_node_slot_overlaps(node_slots, nr_node_slots, new_storage, - new_storage_size) || - trie_child_slot_overlaps(child_slots, nr_child_slots, new_storage, - new_storage_size))) - return -EINVAL; - - children = trie_load_children_slot(slot); - if (!new_storage) { - if (!children) - return -EINVAL; - if (stack_depot_trie_child_lower_bound(children, entries[0], &pos, - &found)) - return -EINVAL; - if (found || !trie_child_array_can_append(children, pos)) - return -EINVAL; - return 0; - } - size = __stack_depot_trie_child_array_size(children ? - READ_ONCE(children->nr_children) + 1 : 1); - if (!size || new_storage_size < size) - return -EINVAL; - if (children) { - size = trie_child_array_size_for_capacity(children->capacity); - if (!size) - return -EINVAL; - if (stack_depot_ranges_overlap(children, size, new_storage, - new_storage_size)) - return -EINVAL; - if (trie_node_slot_overlaps(node_slots, nr_node_slots, children, - size) || - trie_child_slot_overlaps(child_slots, nr_child_slots, children, - size)) - return -EINVAL; - if (stack_depot_trie_child_lower_bound(children, entries[0], &pos, - &found)) - return -EINVAL; - if (found) - return -EINVAL; - } - - return 0; -} - -static int trie_insert_descend_precheck(struct stack_depot_trie_root *root, - struct stack_depot_trie_node *parent, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, - void *new_storage, size_t new_storage_size) -{ - const struct stack_depot_trie_child_array **slot; - const struct stack_depot_trie_child_array *children; - bool overlap; - size_t size; - - slot = trie_publish_slot(root, parent); - if (!slot) - return -EINVAL; - if ((nr_node_slots && !node_slots) || (nr_child_slots && !child_slots)) - return -EINVAL; - if (new_storage && - !IS_ALIGNED((unsigned long)new_storage, - __alignof__(struct stack_depot_trie_child_array))) - return -EINVAL; - if (new_storage && - stack_depot_ranges_overlap(new_storage, new_storage_size, - slot, sizeof(*slot))) - return -EINVAL; - if (trie_node_slot_overlaps(node_slots, nr_node_slots, slot, - sizeof(*slot))) - return -EINVAL; - if (trie_child_slot_overlaps(child_slots, nr_child_slots, slot, - sizeof(*slot))) - return -EINVAL; - - children = trie_load_children_slot(slot); - if (!children) - return -EINVAL; - size = trie_child_array_size_for_capacity(children->capacity); - if (!size) - return -EINVAL; - if (new_storage && - stack_depot_ranges_overlap(children, size, new_storage, - new_storage_size)) - return -EINVAL; - if (trie_node_slot_overlaps(node_slots, nr_node_slots, children, size)) - return -EINVAL; - if (trie_child_slot_overlaps(child_slots, nr_child_slots, children, - size)) - return -EINVAL; - - if (trie_node_slots_subtree_overlap(children, parent, node_slots, nr_node_slots)) - return -EINVAL; - if (trie_child_slots_subtree_overlap(children, parent, child_slots, nr_child_slots)) - return -EINVAL; - overlap = new_storage && - trie_child_array_subtree_overlaps(children, parent, new_storage, - new_storage_size); - if (overlap) - return -EINVAL; - - return 0; -} - -static int -trie_child_array_replace_precheck(const struct stack_depot_trie_child_array *old_array, - const struct stack_depot_trie_node *old_child, - void *new_storage, size_t new_storage_size, - unsigned int *pos) -{ - struct stack_depot_trie_child_array *new_array = new_storage; - unsigned long frame; - bool found; - size_t size; - - if (!old_array || !old_child || !new_array || !pos) - return -EINVAL; - if (!IS_ALIGNED((unsigned long)new_array, - __alignof__(struct stack_depot_trie_child_array))) - return -EINVAL; - if (old_array == new_array) - return -EINVAL; - - size = trie_child_array_size_for_capacity(old_array->capacity); - if (!size || new_storage_size < size) - return -EINVAL; - if (stack_depot_ranges_overlap(old_array, size, new_array, new_storage_size)) - return -EINVAL; - if (stack_depot_trie_node_first_frame(old_child, &frame)) - return -EINVAL; - if (stack_depot_trie_child_lower_bound(old_array, frame, pos, &found) || - !found || old_array->children[*pos] != old_child) - return -EINVAL; - - return 0; -} - static void trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array, const struct stack_depot_trie_node *new_child, @@ -3915,7 +3153,7 @@ trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array new_array->nr_children = old_array->nr_children; new_array->capacity = trie_child_array_storage_capacity(new_storage_size); for (i = 0; i < old_array->nr_children; i++) - new_array->children[i] = old_array->children[i]; + new_array->children[i] = trie_child_array_load_child(old_array, i); new_array->children[pos] = new_child; for (i = old_array->nr_children; i < new_array->capacity; i++) new_array->children[i] = NULL; @@ -3933,8 +3171,6 @@ trie_clone_promoted_node(const struct stack_depot_trie_node *old_node, return -EINVAL; if (old_node->leaf_id) return -EINVAL; - if (stack_depot_frame_run_validate(&old_node->run)) - return -EINVAL; if (!IS_ALIGNED((unsigned long)slot->node, __alignof__(struct stack_depot_trie_node))) return -EINVAL; @@ -3942,8 +3178,6 @@ trie_clone_promoted_node(const struct stack_depot_trie_node *old_node, size = __stack_depot_trie_node_size(&old_node->run); if (!size || slot->size < size) return -EINVAL; - if (stack_depot_ranges_overlap(slot->node, slot->size, old_node, size)) - return -EINVAL; new_node = slot->node; memcpy(new_node, old_node, size); @@ -3962,97 +3196,45 @@ static void trie_reparent_children(struct stack_depot_trie_node *parent) struct stack_depot_trie_node *child; /* Child arrays are const for readers; writers serialize reparenting. */ - child = (struct stack_depot_trie_node *)children->children[i]; + child = (struct stack_depot_trie_node *)trie_child_array_load_child(children, i); trie_publish_parent(child, parent); } } -static int -trie_promote_precheck(struct stack_depot_trie_root *root, - struct stack_depot_trie_node *parent, - const struct stack_depot_trie_node *child, - const struct stack_depot_trie_node_slot *slot, - void *new_storage, size_t new_storage_size, - const struct stack_depot_trie_child_array **old_array, - unsigned int *pos) -{ - const struct stack_depot_trie_child_array **publish_slot; - const struct stack_depot_trie_child_array *array; - const struct stack_depot_trie_node *new_child; - size_t new_child_size; - - if (!child || !slot || !slot->node || !new_storage || !old_array || !pos) - return -EINVAL; - if (child->parent != parent || child->leaf_id) - return -EINVAL; - - publish_slot = trie_publish_slot(root, parent); - if (!publish_slot) - return -EINVAL; - if (stack_depot_ranges_overlap(slot->node, slot->size, publish_slot, - sizeof(*publish_slot))) - return -EINVAL; - if (stack_depot_ranges_overlap(new_storage, new_storage_size, publish_slot, - sizeof(*publish_slot))) - return -EINVAL; - if (parent && (trie_ancestor_overlaps(parent, slot->node, slot->size) || - trie_ancestor_overlaps(parent, new_storage, new_storage_size))) - return -EINVAL; - if (stack_depot_ranges_overlap(slot->node, slot->size, new_storage, - new_storage_size)) - return -EINVAL; - - *old_array = trie_load_children_slot(publish_slot); - if (!*old_array) - return -EINVAL; - array = *old_array; - new_child = slot->node; - new_child_size = __stack_depot_trie_node_size(&child->run); - if (!new_child_size || slot->size < new_child_size) - return -EINVAL; - if (trie_child_array_subtree_overlaps(array, parent, new_child, new_child_size)) - return -EINVAL; - if (trie_child_array_subtree_overlaps(array, parent, new_storage, new_storage_size)) - return -EINVAL; - - return trie_child_array_replace_precheck(array, child, new_storage, - new_storage_size, pos); -} - static int trie_promote_child(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, + const struct stack_depot_trie_child_array *old_array, + unsigned int pos, const struct stack_depot_trie_node *child, u32 leaf_id, const struct stack_depot_trie_node_slot *slot, - void *new_storage, size_t new_storage_size, - u32 fresh_leaf_id) + void *new_storage, size_t new_storage_size) { const struct stack_depot_trie_child_array **publish_slot; - const struct stack_depot_trie_child_array *old_array; struct stack_depot_trie_leaf_update update; size_t child_size; - unsigned int pos; int ret; if (!leaf_id) return -EINVAL; - ret = trie_promote_precheck(root, parent, child, slot, new_storage, - new_storage_size, &old_array, &pos); + if (!old_array || pos >= READ_ONCE(old_array->nr_children) || + trie_child_array_load_child(old_array, pos) != child) + return -EINVAL; + if (!child || !slot || !slot->node || !new_storage) + return -EINVAL; + if (child->parent != parent || child->leaf_id) + return -EINVAL; + child_size = __stack_depot_trie_node_size(&child->run); + if (!child_size || slot->size < child_size) + return -EINVAL; + ret = trie_clone_promoted_node(child, leaf_id, slot); if (ret) return ret; - ret = trie_clone_promoted_node(child, leaf_id, slot); + update.leaf_id = leaf_id; + update.leaf = slot->node; + ret = trie_side_publish(&update, 1); if (ret) return ret; - child_size = __stack_depot_trie_node_size(&child->run); - if (!child_size) - return -EINVAL; - if (fresh_leaf_id) { - update.leaf_id = leaf_id; - update.leaf = slot->node; - ret = trie_side_publish(&update, 1, fresh_leaf_id); - if (ret) - return ret; - } trie_child_array_replace_at(old_array, slot->node, new_storage, new_storage_size, pos); trie_reparent_children(slot->node); @@ -4064,64 +3246,71 @@ trie_promote_child(struct stack_depot_trie_root *root, return 0; } -static int trie_append_chain_validate(const struct stack_depot_trie_node *parent, - const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, unsigned int *nr_runs) +static int +__stack_depot_trie_append_chain(const struct stack_depot_trie_node *parent, + u32 leaf_id, + const unsigned long *entries, + unsigned int nr_entries, + const struct stack_depot_trie_node_slot *node_slots, + unsigned int nr_node_slots, + const struct stack_depot_trie_child_array_slot *child_slots, + unsigned int nr_child_slots, u32 *scratch, + unsigned int nr_scratch, + const struct stack_depot_trie_node **head, + const struct stack_depot_trie_node **tail) { + const struct stack_depot_trie_node *prev = parent; unsigned int child_slots_needed; + unsigned int stack_len = 0; unsigned int pos = 0; unsigned int used = 0; - unsigned int stack_len = 0; + unsigned int i; - if (!entries || !nr_entries || !node_slots || !nr_runs) + if (!leaf_id || !entries || !nr_entries || !node_slots || !head || !tail) return -EINVAL; if (parent && trie_node_stack_len(parent, &stack_len)) return -EINVAL; while (pos < nr_entries) { - const struct stack_depot_trie_node_slot *slot; struct stack_depot_frame_run run; + struct stack_depot_trie_node *node; size_t size; + u32 id; - if (__stack_depot_frame_run_init(&entries[pos], nr_entries - pos, - &run)) - return -EINVAL; if (used >= nr_node_slots) return -EINVAL; - slot = &node_slots[used]; - if (!slot->node) + node = node_slots[used].node; + if (!node) + return -EINVAL; + if (__stack_depot_frame_run_init(&entries[pos], nr_entries - pos, + &run)) return -EINVAL; if (run.mode == STACK_DEPOT_FRAME_COMPRESSED && (!scratch || nr_scratch < run.nr_entries)) return -EINVAL; if (stack_len > CONFIG_STACKDEPOT_MAX_FRAMES - run.nr_entries) return -EINVAL; - size = __stack_depot_trie_node_size(&run); - if (slot->size < size) + if (!size || node_slots[used].size < size) return -EINVAL; - if (!IS_ALIGNED((unsigned long)slot->node, + if (!IS_ALIGNED((unsigned long)node, __alignof__(struct stack_depot_trie_node))) return -EINVAL; - if (trie_ancestor_overlaps(parent, slot->node, slot->size)) - return -EINVAL; - if (trie_node_slot_overlaps(node_slots, used, slot->node, slot->size)) + + id = pos + run.nr_entries == nr_entries ? leaf_id : 0; + if (__stack_depot_trie_node_init(node, node_slots[used].size, prev, + id, &entries[pos], run.nr_entries, + scratch, nr_scratch)) return -EINVAL; stack_len += run.nr_entries; + prev = node; pos += run.nr_entries; used++; } child_slots_needed = used > 1 ? used - 1 : 0; if (child_slots_needed) { - unsigned int i; - if (!child_slots || nr_child_slots < child_slots_needed) return -EINVAL; for (i = 0; i < child_slots_needed; i++) { @@ -4133,68 +3322,9 @@ static int trie_append_chain_validate(const struct stack_depot_trie_node *parent if (!IS_ALIGNED(addr, __alignof__(struct stack_depot_trie_child_array))) return -EINVAL; - if (trie_ancestor_overlaps(parent, child_slots[i].array, - child_slots[i].size)) - return -EINVAL; - if (trie_node_slot_overlaps(node_slots, used, - child_slots[i].array, - child_slots[i].size)) - return -EINVAL; - if (trie_child_slot_overlaps(child_slots, i, - child_slots[i].array, - child_slots[i].size)) - return -EINVAL; } } - *nr_runs = used; - return 0; -} - -static int -__stack_depot_trie_append_chain(const struct stack_depot_trie_node *parent, - u32 leaf_id, - const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, - const struct stack_depot_trie_node **head, - const struct stack_depot_trie_node **tail, - unsigned int *nr_used) -{ - const struct stack_depot_trie_node *prev = parent; - unsigned int pos = 0; - unsigned int used; - unsigned int i; - - if (!leaf_id || !head || !tail || !nr_used) - return -EINVAL; - if (trie_append_chain_validate(parent, entries, nr_entries, node_slots, - nr_node_slots, child_slots, nr_child_slots, - scratch, nr_scratch, &used)) - return -EINVAL; - - for (i = 0; i < used; i++) { - struct stack_depot_frame_run run; - struct stack_depot_trie_node *node = node_slots[i].node; - u32 id; - - if (__stack_depot_frame_run_init(&entries[pos], nr_entries - pos, - &run)) - return -EINVAL; - id = pos + run.nr_entries == nr_entries ? leaf_id : 0; - if (__stack_depot_trie_node_init(node, node_slots[i].size, prev, - id, &entries[pos], run.nr_entries, - scratch, nr_scratch)) - return -EINVAL; - - prev = node; - pos += run.nr_entries; - } - for (i = 0; i + 1 < used; i++) { const struct stack_depot_trie_node *next = node_slots[i + 1].node; struct stack_depot_trie_node *node = node_slots[i].node; @@ -4208,7 +3338,6 @@ __stack_depot_trie_append_chain(const struct stack_depot_trie_node *parent, *head = node_slots[0].node; *tail = node_slots[used - 1].node; - *nr_used = used; return 0; } @@ -4216,7 +3345,6 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, const struct stack_depot_trie_node *head, void *new_storage, size_t new_storage_size, - u32 fresh_leaf_id, u32 leaf_id, const struct stack_depot_trie_node *leaf) { @@ -4225,7 +3353,6 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, struct stack_depot_trie_child_array *new_array = new_storage; size_t storage_size = new_storage_size; size_t new_size; - size_t old_size; int ret; if ((root && parent) || (!root && !parent) || !head) @@ -4233,23 +3360,12 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, if (head->parent != parent) return -EINVAL; - if (root) { - if (new_array && - stack_depot_ranges_overlap(new_array, storage_size, - &root->children, - sizeof(root->children))) - return -EINVAL; + if (root) slot = &root->children; - } else { - if (new_array && - trie_ancestor_overlaps(parent, new_array, storage_size)) - return -EINVAL; + else slot = &parent->children; - } old_array = trie_load_children_slot(slot); - old_size = old_array ? - trie_child_array_size_for_capacity(old_array->capacity) : 0; new_size = old_array ? READ_ONCE(old_array->nr_children) + 1 : 1; new_size = __stack_depot_trie_child_array_size(new_size); if (!new_size) @@ -4267,7 +3383,7 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, return -EINVAL; if (found || !trie_child_array_can_append(old_array, pos)) return -EINVAL; - if (fresh_leaf_id) { + { struct stack_depot_trie_leaf_update update = { .leaf_id = leaf_id, .leaf = leaf, @@ -4275,12 +3391,13 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, if (!leaf_id || !leaf) return -EINVAL; - ret = trie_side_publish(&update, 1, fresh_leaf_id); + ret = trie_side_publish(&update, 1); if (ret) return ret; } trie_child_array_publish_child((struct stack_depot_trie_child_array *)old_array, pos, head); + /* Only tail append mutates a live array; COW arrays are unpublished. */ WRITE_ONCE(((struct stack_depot_trie_child_array *)old_array)->nr_children, pos + 1); return 0; @@ -4288,14 +3405,11 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, if (storage_size < new_size) return -EINVAL; if (old_array && - stack_depot_ranges_overlap(old_array, old_size, new_array, - storage_size)) - return -EINVAL; - if (trie_chain_overlaps(head, new_array, storage_size)) + new_array == old_array) return -EINVAL; if (__stack_depot_trie_child_array_insert(old_array, head, new_array, storage_size)) return -EINVAL; - if (fresh_leaf_id) { + { struct stack_depot_trie_leaf_update update = { .leaf_id = leaf_id, .leaf = leaf, @@ -4303,7 +3417,7 @@ static int trie_publish_append_prepare(struct stack_depot_trie_root *root, if (!leaf_id || !leaf) return -EINVAL; - ret = trie_side_publish(&update, 1, fresh_leaf_id); + ret = trie_side_publish(&update, 1); if (ret) return ret; } @@ -4339,9 +3453,10 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, children = trie_load_children_slot(&parent->children); tmp.status = STACK_DEPOT_TRIE_LOOKUP_APPEND; - tmp.parent = parent; + tmp.children = children; tmp.node = NULL; tmp.matched = 0; + tmp.pos = 0; if (!children) { *lookup = tmp; return 0; @@ -4351,11 +3466,12 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, if (stack_depot_trie_child_lower_bound(children, key, &pos, &found)) return -EINVAL; if (!found) { + tmp.pos = pos; *lookup = tmp; return 0; } - node = children->children[pos]; + node = trie_child_array_load_child(children, pos); if (!node) return -EINVAL; /* @@ -4370,6 +3486,7 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, tmp.node = node; tmp.matched = matched; + tmp.pos = pos; if (matched < node->run.nr_entries) tmp.status = STACK_DEPOT_TRIE_LOOKUP_SPLIT; else if (matched < nr_entries) @@ -4441,6 +3558,8 @@ __stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, static int trie_split_child(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, + const struct stack_depot_trie_child_array *old_array, + unsigned int pos, const struct stack_depot_trie_node *child, unsigned int matched, u32 leaf_id, const unsigned long *entries, @@ -4450,10 +3569,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, - size_t new_storage_size, - u32 fresh_leaf_id, - const struct stack_depot_trie_node **tail, - unsigned int *nr_used); + size_t new_storage_size); static int __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, @@ -4467,18 +3583,14 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, *child_slots, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, - size_t new_storage_size, - u32 fresh_leaf_id, - const struct stack_depot_trie_node **tail, - unsigned int *nr_used) + size_t new_storage_size) { const struct stack_depot_trie_node *head; const struct stack_depot_trie_node *last; - unsigned int used; struct stack_depot_trie_lookup lookup; int ret; - if (!leaf_id || !tail || !nr_used) + if (!leaf_id) return -EINVAL; for (;;) { @@ -4487,12 +3599,6 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, return ret; if (lookup.status != STACK_DEPOT_TRIE_LOOKUP_DESCEND) break; - ret = trie_insert_descend_precheck(root, parent, node_slots, - nr_node_slots, child_slots, - nr_child_slots, new_storage, - new_storage_size); - if (ret) - return ret; /* Insert callers serialize writers and may publish below this node. */ parent = (struct stack_depot_trie_node *)lookup.node; root = NULL; @@ -4503,46 +3609,35 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, if (!entries || !nr_entries) return -EINVAL; if (lookup.status == STACK_DEPOT_TRIE_LOOKUP_SPLIT) - return trie_split_child(root, parent, lookup.node, lookup.matched, - leaf_id, entries, nr_entries, node_slots, - nr_node_slots, child_slots, nr_child_slots, - scratch, nr_scratch, new_storage, - new_storage_size, fresh_leaf_id, tail, - nr_used); + return trie_split_child(root, parent, lookup.children, lookup.pos, + lookup.node, lookup.matched, leaf_id, entries, + nr_entries, node_slots, nr_node_slots, child_slots, + nr_child_slots, scratch, nr_scratch, new_storage, + new_storage_size); if (lookup.status == STACK_DEPOT_TRIE_LOOKUP_PROMOTE) { if (!node_slots || !nr_node_slots) return -EINVAL; - ret = trie_promote_child(root, parent, lookup.node, leaf_id, - &node_slots[0], new_storage, new_storage_size, - fresh_leaf_id); + ret = trie_promote_child(root, parent, lookup.children, lookup.pos, + lookup.node, leaf_id, &node_slots[0], + new_storage, new_storage_size); if (ret) return ret; - *tail = node_slots[0].node; - *nr_used = 1; return 0; } if (lookup.status != STACK_DEPOT_TRIE_LOOKUP_APPEND) return -EINVAL; - ret = trie_insert_append_precheck(root, parent, entries, nr_entries, - node_slots, nr_node_slots, child_slots, - nr_child_slots, new_storage, - new_storage_size); - if (ret) - return ret; ret = __stack_depot_trie_append_chain(parent, leaf_id, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, scratch, nr_scratch, - &head, &last, &used); + &head, &last); if (ret) return ret; ret = trie_publish_append_prepare(root, parent, head, new_storage, - new_storage_size, fresh_leaf_id, leaf_id, last); + new_storage_size, leaf_id, last); if (ret) return ret; - *tail = last; - *nr_used = used; return 0; } @@ -4602,44 +3697,6 @@ static int trie_plan_append_chain(unsigned int base_stack_len, return 0; } -static bool trie_node_depth_invalid(const struct stack_depot_trie_node *parent, - const struct stack_depot_trie_node *node) -{ - const struct stack_depot_trie_node *node_parent; - unsigned int base = 0; - - if (!node) - return true; - node_parent = trie_load_parent(node); - if (node_parent != parent) - return true; - if (stack_depot_frame_run_validate(&node->run)) - return true; - if (parent && trie_node_stack_len(parent, &base)) - return true; - - return base > CONFIG_STACKDEPOT_MAX_FRAMES - node->run.nr_entries; -} - -static bool trie_node_chain_depth_invalid(const struct stack_depot_trie_node *node) -{ - unsigned int depth = 0; - - while (node) { - const struct stack_depot_trie_node *parent; - - if (depth >= CONFIG_STACKDEPOT_MAX_FRAMES) - return true; - parent = trie_load_parent(node); - if (trie_node_depth_invalid(parent, node)) - return true; - node = parent; - depth++; - } - - return false; -} - static bool trie_parent_chain_matches_prefix(const struct stack_depot_trie_node *node, const unsigned long *entries, @@ -4659,8 +3716,6 @@ trie_parent_chain_matches_prefix(const struct stack_depot_trie_node *node, while (cur) { if (depth++ >= CONFIG_STACKDEPOT_MAX_FRAMES) return false; - if (stack_depot_frame_run_validate(&cur->run)) - return false; if (cur->run.nr_entries > pos) return false; pos -= cur->run.nr_entries; @@ -4689,6 +3744,7 @@ static int trie_plan_split(const struct stack_depot_trie_child_array *children, unsigned int nr_child_slots, size_t *new_storage_size, unsigned int *nr_used, unsigned int *nr_child_used) { + const struct stack_depot_trie_node *parent; struct stack_depot_frame_run old_tail_run; struct stack_depot_frame_run prefix_run; unsigned int new_child_used = 0; @@ -4701,9 +3757,7 @@ static int trie_plan_split(const struct stack_depot_trie_child_array *children, return -EINVAL; if (!matched || matched >= child->run.nr_entries || matched > nr_entries) return -EINVAL; - if (trie_node_depth_invalid(child->parent, child)) - return -EINVAL; - if (!child->leaf_id && !child->children) + if (!child->leaf_id && !trie_load_children_slot(&child->children)) return -EINVAL; if (nr_node_slots < 2 || nr_child_slots < 1) return -EINVAL; @@ -4714,8 +3768,9 @@ static int trie_plan_split(const struct stack_depot_trie_child_array *children, return -EINVAL; has_new_tail = matched < nr_entries; - if (child->parent) { - if (trie_node_stack_len(child->parent, &prefix_stack_len)) + parent = trie_load_parent(child); + if (parent) { + if (trie_node_stack_len(parent, &prefix_stack_len)) return -EINVAL; } else { prefix_stack_len = 0; @@ -4765,7 +3820,7 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array *children; const struct stack_depot_trie_node *child; unsigned int nr_children; - unsigned int parent_len; + unsigned int parent_len = 0; unsigned int matched; unsigned int pos; bool found; @@ -4777,8 +3832,6 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, return -EINVAL; for (;;) { - if (parent && trie_node_chain_depth_invalid(parent)) - return -EINVAL; if (parent && trie_node_stack_len(parent, &parent_len)) return -EINVAL; if (root) @@ -4814,8 +3867,10 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, return *new_storage_size ? 0 : -EINVAL; } - child = children->children[pos]; - if (trie_node_depth_invalid(parent, child)) + child = trie_child_array_load_child(children, pos); + if (!child || trie_load_parent(child) != parent) + return -EINVAL; + if (parent_len > CONFIG_STACKDEPOT_MAX_FRAMES - child->run.nr_entries) return -EINVAL; matched = __stack_depot_trie_node_match(child, entries, nr_entries); if (!matched || (matched == nr_entries && @@ -4835,212 +3890,58 @@ __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, node_slots[0].size = __stack_depot_trie_node_size(&child->run); *new_storage_size = trie_child_array_size_for_capacity(children->capacity); - if (!node_slots[0].size || !*new_storage_size) - return -EINVAL; - *nr_used = 1; - *nr_child_used = 0; - return 0; - } - - root = NULL; - parent = child; - entries += matched; - nr_entries -= matched; - } -} - -static unsigned int trie_validate_leaf(const struct stack_depot_trie_node *leaf, - const unsigned long *entries) -{ - const struct stack_depot_trie_node *node = leaf; - size_t entries_size; - unsigned int pos; - unsigned int total; - - if (!node || !node->leaf_id) - return 0; - if (trie_node_stack_len(node, &total)) - return 0; - - entries_size = total * sizeof(*entries); - pos = total; - for (node = leaf; node; node = trie_load_parent(node)) { - size_t node_size; - bool overlap; - - if (frame_run_validate_payload(&node->run, node->data)) - return 0; - node_size = stack_depot_frame_run_bytes(&node->run); - overlap = entries && stack_depot_ranges_overlap(entries, - entries_size, node->data, - node_size); - if (overlap) - return 0; - if (node->run.nr_entries > pos) - return 0; - pos -= node->run.nr_entries; - } - - return pos ? 0 : total; -} - -static unsigned int trie_walk_frames(const struct stack_depot_trie_node *leaf, - unsigned int total, - void (*fn)(unsigned int index, - unsigned long frame, - void *data), - void *data) -{ - const struct stack_depot_trie_node *node; - unsigned int pos = total; - unsigned int seen = 0; - unsigned int i; - - if (!fn) - return 0; - - for (node = leaf; node; node = trie_load_parent(node)) { - if (node->run.nr_entries > pos) - return 0; - pos -= node->run.nr_entries; - for (i = 0; i < node->run.nr_entries; i++) { - unsigned long frame; - - if (stack_depot_trie_node_frame(node, i, &frame)) - return 0; - fn(pos + i, frame, data); - seen++; - } - } - - return seen == total && !pos ? total : 0; -} - -static int trie_frame_at(const struct stack_depot_trie_node *leaf, - unsigned int index, - unsigned long *frame) -{ - const struct stack_depot_trie_node *node; - unsigned int pos; - - if (!leaf || !frame) - return -EINVAL; - if (trie_node_stack_len(leaf, &pos)) - return -EINVAL; + if (!node_slots[0].size || !*new_storage_size) + return -EINVAL; + *nr_used = 1; + *nr_child_used = 0; + return 0; + } - for (node = leaf; node; node = trie_load_parent(node)) { - if (node->run.nr_entries > pos) - return -EINVAL; - pos -= node->run.nr_entries; - if (index < pos || index >= pos + node->run.nr_entries) - continue; - return stack_depot_trie_node_frame(node, index - pos, frame); + root = NULL; + parent = child; + entries += matched; + nr_entries -= matched; } - - return -EINVAL; } -static unsigned int trie_handle_leaf(depot_stack_handle_t handle, - const struct stack_depot_trie_node **leaf) +static unsigned int trie_leaf_stack_len(const struct stack_depot_trie_node *leaf) { - u32 leaf_id; + unsigned int total; - if (!leaf) - return 0; - *leaf = NULL; - leaf_id = __stack_depot_trie_leaf_id(handle); - if (!leaf_id) + if (!leaf || !leaf->leaf_id) return 0; - *leaf = __stack_depot_trie_side_table_lookup(leaf_id); - if (WARN_ONCE(!*leaf, "corrupt trie handle %08x\n", handle)) + if (trie_node_stack_len(leaf, &total)) return 0; - return trie_validate_leaf(*leaf, NULL); -} - -static void trie_print_frames(const struct stack_depot_trie_node *leaf, - unsigned int nr_entries, - int spaces) -{ - unsigned int i; - - for (i = 0; i < nr_entries; i++) { - unsigned long frame; - if (trie_frame_at(leaf, i, &frame)) - return; - stack_trace_print(&frame, 1, spaces); - } + return total; } static unsigned int trie_print_handle(depot_stack_handle_t handle, int spaces) { + unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; + unsigned int max_entries = ARRAY_SIZE(entries); unsigned int nr_entries; - const struct stack_depot_trie_node *leaf; - rcu_read_lock_sched_notrace(); - nr_entries = trie_handle_leaf(handle, &leaf); + nr_entries = __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); if (nr_entries) - trie_print_frames(leaf, nr_entries, spaces); - rcu_read_unlock_sched_notrace(); + stack_trace_print(entries, nr_entries, spaces); return nr_entries; } -static int -trie_snprint_frames(char *buf, size_t size, - const struct stack_depot_trie_node *leaf, - unsigned int nr_entries, int spaces) -{ - int generated; - int total = 0; - unsigned int i; - - for (i = 0; i < nr_entries && size; i++) { - unsigned long frame; - - if (trie_frame_at(leaf, i, &frame)) - break; - generated = snprintf(buf, size, "%*c%pS\n", 1 + spaces, ' ', - (void *)frame); - if (generated < 0) - break; - if (generated > INT_MAX - total) - return INT_MAX; - total += generated; - if (generated >= size) { - buf += size; - size = 0; - } else { - buf += generated; - size -= generated; - } - } - - return total; -} - static int trie_snprint_handle(depot_stack_handle_t handle, char *buf, size_t size, int spaces) { + unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES]; + unsigned int max_entries = ARRAY_SIZE(entries); unsigned int nr_entries; - const struct stack_depot_trie_node *leaf; - int ret = 0; - - rcu_read_lock_sched_notrace(); - nr_entries = trie_handle_leaf(handle, &leaf); - if (nr_entries) - ret = trie_snprint_frames(buf, size, leaf, nr_entries, spaces); - rcu_read_unlock_sched_notrace(); - - return ret; -} -static void trie_fetch_frame(unsigned int index, unsigned long frame, void *data) -{ - unsigned long *entries = data; + nr_entries = __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); + if (!nr_entries) + return 0; - entries[index] = frame; + return stack_trace_snprint(buf, size, entries, nr_entries, spaces); } static unsigned int @@ -5048,17 +3949,32 @@ __stack_depot_trie_fetch_into(const struct stack_depot_trie_node *leaf, unsigned long *entries, unsigned int max_entries) { + const struct stack_depot_trie_node *node; unsigned int total; + unsigned int seen = 0; + unsigned int pos; + unsigned int i; if (!entries) return 0; - total = trie_validate_leaf(leaf, entries); + total = trie_leaf_stack_len(leaf); if (!total) return 0; if (max_entries < total) return 0; - if (trie_walk_frames(leaf, total, trie_fetch_frame, entries) != total) + pos = total; + for (node = leaf; node; node = trie_load_parent(node)) { + if (node->run.nr_entries > pos) + return 0; + pos -= node->run.nr_entries; + for (i = 0; i < node->run.nr_entries; i++) { + if (stack_depot_trie_node_frame(node, i, &entries[pos + i])) + return 0; + seen++; + } + } + if (seen != total || pos) return 0; kmsan_unpoison_memory(entries, total * sizeof(*entries)); @@ -5142,14 +4058,10 @@ __stack_depot_trie_child_array_init(void *storage, size_t storage_size, for (i = 0; i < nr_children; i++) { unsigned long frame; - size_t size; if (stack_depot_trie_node_first_frame(nodes[i], &frame)) return -EINVAL; - size = __stack_depot_trie_node_size(&nodes[i]->run); - if (!size) - return -EINVAL; - if (stack_depot_ranges_overlap(array, storage_size, nodes[i], size)) + if (!__stack_depot_trie_node_size(&nodes[i]->run)) return -EINVAL; /* Child key zero is reserved so NULL lookup remains unambiguous. */ if (!frame || (i && frame <= last)) @@ -5199,284 +4111,6 @@ __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, return __stack_depot_trie_child_array_init(storage, storage_size, children, 2); } -static int -__stack_depot_trie_split_tail_plan(const unsigned long *entries, - unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, unsigned int *nr_runs) -{ - unsigned int pos = 0; - unsigned int runs = 0; - unsigned int child_slots_needed; - unsigned int i; - - if (!entries || !nr_entries || nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES || - !node_slots || !nr_runs) - return -EINVAL; - - while (pos < nr_entries) { - const struct stack_depot_trie_node_slot *slot; - struct stack_depot_frame_run run; - size_t size; - - if (__stack_depot_frame_run_init(&entries[pos], nr_entries - pos, - &run)) - return -EINVAL; - if (runs >= nr_node_slots) - return -EINVAL; - slot = &node_slots[runs]; - if (!slot->node) - return -EINVAL; - size = __stack_depot_trie_node_size(&run); - if (!size || slot->size < size) - return -EINVAL; - if (!IS_ALIGNED((unsigned long)slot->node, - __alignof__(struct stack_depot_trie_node))) - return -EINVAL; - - pos += run.nr_entries; - runs++; - } - - child_slots_needed = runs > 1 ? runs - 1 : 0; - if (child_slots_needed) { - if (!child_slots || nr_child_slots < child_slots_needed) - return -EINVAL; - for (i = 0; i < child_slots_needed; i++) { - size_t size = __stack_depot_trie_child_array_size(1); - - if (!child_slots[i].array || child_slots[i].size < size) - return -EINVAL; - if (!IS_ALIGNED((unsigned long)child_slots[i].array, - __alignof__(struct stack_depot_trie_child_array))) - return -EINVAL; - } - } - - *nr_runs = runs; - return 0; -} - -static int -__stack_depot_trie_split_precheck(struct stack_depot_trie_root *root, - struct stack_depot_trie_node *parent, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, - void *new_storage, size_t new_storage_size) -{ - const struct stack_depot_trie_child_array **slot; - const struct stack_depot_trie_child_array *children; - unsigned int i; - size_t size; - - if (!new_storage || !new_storage_size || - (nr_node_slots && !node_slots) || (nr_child_slots && !child_slots)) - return -EINVAL; - if (!IS_ALIGNED((unsigned long)new_storage, - __alignof__(struct stack_depot_trie_child_array))) - return -EINVAL; - - for (i = 0; i < nr_node_slots; i++) { - const struct stack_depot_trie_node_slot *node_slot = &node_slots[i]; - - if (!node_slot->node || !node_slot->size) - return -EINVAL; - if (!IS_ALIGNED((unsigned long)node_slot->node, - __alignof__(struct stack_depot_trie_node))) - return -EINVAL; - if (parent && - trie_ancestor_overlaps(parent, node_slot->node, node_slot->size)) - return -EINVAL; - if (trie_node_slot_overlaps(node_slots, i, node_slot->node, node_slot->size)) - return -EINVAL; - } - - for (i = 0; i < nr_child_slots; i++) { - const struct stack_depot_trie_child_array_slot *child_slot = - &child_slots[i]; - struct stack_depot_trie_child_array *array = child_slot->array; - size_t slot_size = child_slot->size; - - if (!array || !slot_size) - return -EINVAL; - if (!IS_ALIGNED((unsigned long)array, - __alignof__(struct stack_depot_trie_child_array))) - return -EINVAL; - if (parent && - trie_ancestor_overlaps(parent, array, slot_size)) - return -EINVAL; - if (trie_child_slot_overlaps(child_slots, i, array, slot_size)) - return -EINVAL; - } - - for (i = 0; i < nr_node_slots; i++) { - if (trie_child_slot_overlaps(child_slots, nr_child_slots, - node_slots[i].node, node_slots[i].size)) - return -EINVAL; - } - if (trie_node_slot_overlaps(node_slots, nr_node_slots, new_storage, - new_storage_size) || - trie_child_slot_overlaps(child_slots, nr_child_slots, new_storage, - new_storage_size)) - return -EINVAL; - - slot = trie_publish_slot(root, parent); - if (!slot) - return -EINVAL; - if (stack_depot_ranges_overlap(new_storage, new_storage_size, slot, - sizeof(*slot))) - return -EINVAL; - if (trie_node_slot_overlaps(node_slots, nr_node_slots, slot, sizeof(*slot)) || - trie_child_slot_overlaps(child_slots, nr_child_slots, slot, sizeof(*slot))) - return -EINVAL; - if (parent && trie_ancestor_overlaps(parent, new_storage, new_storage_size)) - return -EINVAL; - - /* Pairs with append, promote, and future split publication. */ - children = trie_load_children_slot(slot); - if (!children) - return -EINVAL; - size = trie_child_array_size_for_capacity(children->capacity); - if (!size || new_storage_size < size) - return -EINVAL; - if (stack_depot_ranges_overlap(children, size, new_storage, - new_storage_size)) - return -EINVAL; - if (trie_node_slot_overlaps(node_slots, nr_node_slots, children, size) || - trie_child_slot_overlaps(child_slots, nr_child_slots, children, size)) - return -EINVAL; - if (trie_node_slots_subtree_overlap(children, parent, node_slots, nr_node_slots)) - return -EINVAL; - if (trie_child_slots_subtree_overlap(children, parent, child_slots, nr_child_slots)) - return -EINVAL; - if (trie_child_array_subtree_overlaps(children, parent, new_storage, - new_storage_size)) - return -EINVAL; - - return 0; -} - -static bool -trie_split_subtree_overlaps(const struct stack_depot_trie_node *child, - const void *ptr, size_t size) -{ - if (trie_ancestor_overlaps(child, ptr, size)) - return true; - return trie_child_array_subtree_overlaps(child->children, child, ptr, - size); -} - -static bool -trie_split_slots_overlap(const struct stack_depot_trie_node *child, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots) -{ - unsigned int i; - - for (i = 0; i < nr_node_slots; i++) { - const struct stack_depot_trie_node_slot *slot = &node_slots[i]; - - if (trie_split_subtree_overlaps(child, slot->node, slot->size)) - return true; - if (trie_node_slot_overlaps(node_slots, i, slot->node, - slot->size)) - return true; - if (trie_child_slot_overlaps(child_slots, nr_child_slots, - slot->node, slot->size)) - return true; - } - - for (i = 0; i < nr_child_slots; i++) { - const struct stack_depot_trie_child_array_slot *slot = - &child_slots[i]; - - if (trie_split_subtree_overlaps(child, slot->array, slot->size)) - return true; - if (trie_child_slot_overlaps(child_slots, i, slot->array, - slot->size)) - return true; - } - - return false; -} - -static int -trie_split_subtree_precheck(const struct stack_depot_trie_node *child, - unsigned int matched, u32 leaf_id, - const unsigned long *entries, unsigned int nr_entries, - const struct stack_depot_trie_node_slot *node_slots, - unsigned int nr_node_slots, - const struct stack_depot_trie_child_array_slot *child_slots, - unsigned int nr_child_slots, unsigned int *new_runs) -{ - struct stack_depot_frame_run old_tail_run; - struct stack_depot_frame_run prefix_run; - const unsigned long *tail_entries; - unsigned int child_slots_needed; - unsigned int slots_needed; - unsigned int tail_child_slots; - unsigned int tail_node_slots; - unsigned int tail_len; - bool has_new_tail; - size_t size; - int ret; - - if (!child || !leaf_id || !entries || !nr_entries || !node_slots || - !child_slots || !new_runs) - return -EINVAL; - if (!matched || matched >= child->run.nr_entries || - matched > nr_entries) - return -EINVAL; - if (!child->leaf_id && !child->children) - return -EINVAL; - if (stack_depot_frame_run_slice(&child->run, 0, matched, &prefix_run)) - return -EINVAL; - tail_len = child->run.nr_entries - matched; - if (stack_depot_frame_run_slice(&child->run, matched, tail_len, &old_tail_run)) - return -EINVAL; - - has_new_tail = matched < nr_entries; - *new_runs = 0; - if (has_new_tail) { - tail_entries = &entries[matched]; - tail_len = nr_entries - matched; - tail_node_slots = nr_node_slots > 2 ? nr_node_slots - 2 : 0; - tail_child_slots = nr_child_slots > 1 ? nr_child_slots - 1 : 0; - ret = __stack_depot_trie_split_tail_plan(tail_entries, tail_len, - &node_slots[2], tail_node_slots, - tail_child_slots ? &child_slots[1] : NULL, - tail_child_slots, new_runs); - if (ret) - return ret; - } - - slots_needed = 2 + *new_runs; - child_slots_needed = 1 + (*new_runs ? *new_runs - 1 : 0); - if (nr_node_slots < slots_needed || nr_child_slots < child_slots_needed) - return -EINVAL; - - size = __stack_depot_trie_node_size(&prefix_run); - if (!node_slots[0].node || node_slots[0].size < size) - return -EINVAL; - size = __stack_depot_trie_node_size(&old_tail_run); - if (!node_slots[1].node || node_slots[1].size < size) - return -EINVAL; - size = __stack_depot_trie_child_array_size(has_new_tail ? 2 : 1); - if (!child_slots[0].array || child_slots[0].size < size) - return -EINVAL; - if (trie_split_slots_overlap(child, node_slots, slots_needed, - child_slots, child_slots_needed)) - return -EINVAL; - - return 0; -} - static int trie_split_subtree_prepare(const struct stack_depot_trie_node *child, unsigned int matched, u32 leaf_id, const unsigned long *entries, @@ -5486,42 +4120,49 @@ static int trie_split_subtree_prepare(const struct stack_depot_trie_node *child, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, - u32 fresh_leaf_id, - const struct stack_depot_trie_node **prefix, - const struct stack_depot_trie_node **tail, - unsigned int *nr_used) + const struct stack_depot_trie_node **prefix) { + const struct stack_depot_trie_child_array *child_children; + const struct stack_depot_trie_node *child_parent; const unsigned long *tail_entries; const struct stack_depot_trie_node *new_head = NULL; const struct stack_depot_trie_node *new_tail = NULL; struct stack_depot_trie_leaf_update updates[2]; struct stack_depot_trie_node *old_tail; struct stack_depot_trie_node *pref; - unsigned int chain_used = 0; + unsigned int child_slots_needed; unsigned int nr_updates = 0; u32 prefix_leaf_id; void *split_array; size_t split_array_size; - unsigned int new_runs; unsigned int tail_len; bool has_new_tail; int ret; - if (!prefix || !tail || !nr_used) + if (!prefix || !child || !leaf_id || !entries || + !nr_entries || !node_slots || !child_slots) + return -EINVAL; + if (!matched || matched >= child->run.nr_entries || matched > nr_entries) + return -EINVAL; + child_children = trie_load_children_slot(&child->children); + if (!child->leaf_id && !child_children) + return -EINVAL; + child_parent = trie_load_parent(child); + + has_new_tail = matched < nr_entries; + if (nr_node_slots < 2 || nr_child_slots < 1) + return -EINVAL; + if (has_new_tail && nr_node_slots < 3) + return -EINVAL; + child_slots_needed = 1 + (nr_node_slots > 3 ? nr_node_slots - 3 : 0); + if (nr_child_slots != child_slots_needed) return -EINVAL; - ret = trie_split_subtree_precheck(child, matched, leaf_id, entries, - nr_entries, node_slots, nr_node_slots, - child_slots, nr_child_slots, - &new_runs); - if (ret) - return ret; pref = node_slots[0].node; old_tail = node_slots[1].node; - has_new_tail = matched < nr_entries; prefix_leaf_id = has_new_tail ? 0 : leaf_id; ret = __stack_depot_trie_node_init_slice(pref, node_slots[0].size, - child->parent, prefix_leaf_id, + child_parent, prefix_leaf_id, child, 0, matched); if (ret) return ret; @@ -5538,8 +4179,7 @@ static int trie_split_subtree_prepare(const struct stack_depot_trie_node *child, ret = __stack_depot_trie_append_chain(pref, leaf_id, tail_entries, tail_len, &node_slots[2], nr_node_slots - 2, &child_slots[1], nr_child_slots - 1, - scratch, nr_scratch, &new_head, &new_tail, - &chain_used); + scratch, nr_scratch, &new_head, &new_tail); if (ret) return ret; } @@ -5557,25 +4197,23 @@ static int trie_split_subtree_prepare(const struct stack_depot_trie_node *child, updates[nr_updates].leaf_id = leaf_id; updates[nr_updates].leaf = has_new_tail ? new_tail : pref; nr_updates++; - if (fresh_leaf_id) { - ret = trie_side_publish(updates, nr_updates, fresh_leaf_id); - if (ret) { - memset(split_array, 0, split_array_size); - return ret; - } + ret = trie_side_publish(updates, nr_updates); + if (ret) { + memset(split_array, 0, split_array_size); + return ret; } - old_tail->children = child->children; + old_tail->children = child_children; pref->children = child_slots[0].array; trie_reparent_children(old_tail); *prefix = pref; - *tail = has_new_tail ? new_tail : pref; - *nr_used = 2 + chain_used; return 0; } static int trie_split_child(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, + const struct stack_depot_trie_child_array *old_array, + unsigned int pos, const struct stack_depot_trie_node *child, unsigned int matched, u32 leaf_id, const unsigned long *entries, @@ -5585,40 +4223,25 @@ static int trie_split_child(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, unsigned int nr_scratch, void *new_storage, - size_t new_storage_size, - u32 fresh_leaf_id, - const struct stack_depot_trie_node **tail, - unsigned int *nr_used) + size_t new_storage_size) { const struct stack_depot_trie_child_array **publish_slot; - const struct stack_depot_trie_child_array *old_array; const struct stack_depot_trie_node *prefix; - void *storage = new_storage; size_t child_size; - size_t storage_size = new_storage_size; - unsigned int pos; - unsigned int used; int ret; - if (!child || !tail || !nr_used) + if (!child) return -EINVAL; if (child->parent != parent) return -EINVAL; + if (!old_array || pos >= READ_ONCE(old_array->nr_children) || + old_array->children[pos] != child) + return -EINVAL; - ret = __stack_depot_trie_split_precheck(root, parent, node_slots, - nr_node_slots, child_slots, - nr_child_slots, new_storage, - new_storage_size); - if (ret) - return ret; publish_slot = trie_publish_slot(root, parent); if (!publish_slot) return -EINVAL; - old_array = trie_load_children_slot(publish_slot); - ret = trie_child_array_replace_precheck(old_array, child, storage, storage_size, &pos); - if (ret) - return ret; child_size = __stack_depot_trie_node_size(&child->run); if (!child_size) return -EINVAL; @@ -5626,8 +4249,7 @@ static int trie_split_child(struct stack_depot_trie_root *root, ret = trie_split_subtree_prepare(child, matched, leaf_id, entries, nr_entries, node_slots, nr_node_slots, child_slots, nr_child_slots, scratch, - nr_scratch, fresh_leaf_id, &prefix, - tail, &used); + nr_scratch, &prefix); if (ret) return ret; @@ -5636,7 +4258,6 @@ static int trie_split_child(struct stack_depot_trie_root *root, /* Publish the fully initialized replacement array last. */ trie_publish_children_slot(publish_slot, new_storage); trie_retire_object_node(old_array, child, child_size); - *nr_used = used; return 0; } @@ -5692,8 +4313,6 @@ __stack_depot_trie_child_array_insert(const struct stack_depot_trie_child_array unsigned int i; unsigned long frame; size_t node_size; - size_t old_size; - bool overlaps; bool found; if (!node || !new_array || stack_depot_trie_node_first_frame(node, &frame)) @@ -5701,8 +4320,6 @@ __stack_depot_trie_child_array_insert(const struct stack_depot_trie_child_array node_size = __stack_depot_trie_node_size(&node->run); if (!node_size) return -EINVAL; - if (stack_depot_ranges_overlap(new_array, new_storage_size, node, node_size)) - return -EINVAL; if (!IS_ALIGNED((unsigned long)new_array, __alignof__(*new_array))) return -EINVAL; if (!frame) @@ -5715,11 +4332,6 @@ __stack_depot_trie_child_array_insert(const struct stack_depot_trie_child_array nr_old = old ? old->nr_children : 0; if (new_storage_size < __stack_depot_trie_child_array_size(nr_old + 1)) return -EINVAL; - old_size = old ? trie_child_array_size_for_capacity(old->capacity) : 0; - overlaps = old && stack_depot_ranges_overlap(old, old_size, new_array, - new_storage_size); - if (overlaps) - return -EINVAL; if (old) { if (stack_depot_trie_child_lower_bound(old, frame, &pos, @@ -5736,12 +4348,12 @@ __stack_depot_trie_child_array_insert(const struct stack_depot_trie_child_array new_array->capacity = trie_child_array_storage_capacity(new_storage_size); if (old) { for (i = 0; i < pos; i++) - new_array->children[i] = old->children[i]; + new_array->children[i] = trie_child_array_load_child(old, i); } new_array->children[pos] = node; if (old) { for (i = pos; i < nr_old; i++) - new_array->children[i + 1] = old->children[i]; + new_array->children[i + 1] = trie_child_array_load_child(old, i); } for (i = nr_old + 1; i < new_array->capacity; i++) new_array->children[i] = NULL; @@ -5794,20 +4406,14 @@ unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, return __stack_depot_trie_fetch_handle_into(handle, entries, max_entries); - rcu_read_lock_sched_notrace(); stack = depot_fetch_stack(handle); - if (!stack) { - rcu_read_unlock_sched_notrace(); + if (!stack) return 0; - } nr_entries = stack->size; - if (!nr_entries || nr_entries > max_entries) { - rcu_read_unlock_sched_notrace(); + if (!nr_entries || nr_entries > max_entries) return 0; - } memcpy(entries, stack->entries, nr_entries * sizeof(*entries)); - rcu_read_unlock_sched_notrace(); kmsan_unpoison_memory(entries, nr_entries * sizeof(*entries)); return nr_entries; } @@ -5829,9 +4435,8 @@ void stack_depot_put(depot_stack_handle_t handle) */ if (WARN(!stack, "corrupt handle or unbalanced %s()", __func__)) return; - if (WARN_ON_ONCE(!(stack->flags & STACK_RECORD_FLAG_REFCOUNTED))) + if (WARN_ON_ONCE(!(stack->flags & STACK_DEPOT_FLAG_GET))) return; - if (refcount_dec_and_test(&stack->count)) depot_free_stack(stack); } @@ -5874,8 +4479,7 @@ depot_stack_handle_t __must_check stack_depot_set_extra_bits(depot_stack_handle_ union handle_parts parts = { .handle = handle }; /* Do not set extra bits on empty handles. */ - parts.extra = 0; - if (!parts.handle) + if (!handle) return 0; parts.extra = extra_bits; diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 1c9243981b2df..e9034907bc909 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -134,11 +134,13 @@ static void stackdepot_save_flags_public(struct kunit *test) { unsigned long entries[] = { 0x501000UL, 0x502000UL, 0x503000UL }; unsigned long get_entries[] = { 0x601000UL, 0x602000UL }; + unsigned long count_entries[] = { 0x603000UL, 0x604000UL }; unsigned long noalloc_entries[] = { 0x701000UL, 0x702000UL }; unsigned long fetched[ARRAY_SIZE(entries)] = {}; depot_stack_handle_t noalloc_handle; depot_stack_handle_t truncated_handle; depot_stack_handle_t overlong_handle; + depot_stack_handle_t count_handle; depot_stack_handle_t hash_handle; depot_stack_handle_t get_handle; depot_stack_handle_t again; @@ -191,16 +193,12 @@ static void stackdepot_save_flags_public(struct kunit *test) GFP_KERNEL, flags); KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); stack_depot_put(get_handle); - get_handle = stack_depot_save_flags(entries, ARRAY_SIZE(entries), GFP_KERNEL, - flags); - KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE; - hash_handle = stack_depot_save_flags(entries, ARRAY_SIZE(entries), - GFP_KERNEL, flags); - KUNIT_ASSERT_NE(test, hash_handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_NE(test, hash_handle, get_handle); - stack_depot_put(get_handle); + count_handle = stack_depot_save_flags(count_entries, + ARRAY_SIZE(count_entries), + GFP_KERNEL, flags); + KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); overlong_handle = stack_depot_save(overlong_entries, overlong_nr, GFP_KERNEL); @@ -242,56 +240,67 @@ static void stackdepot_snprint_public(struct kunit *test) KUNIT_EXPECT_STREQ(test, actual, expected); } -static void stackdepot_count_helpers(struct kunit *test) +static void stackdepot_get_stack_record(struct kunit *test) { unsigned long entries[] = { 0x1234567800310000UL, 0x1234567800320000UL, 0x1234567800330000UL, }; - unsigned long second_entries[] = { + struct stack_record *record; + depot_stack_handle_t handle; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = save_hash(entries, ARRAY_SIZE(entries)); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + record = __stack_depot_get_stack_record(handle); + KUNIT_ASSERT_NOT_NULL(test, record); + KUNIT_EXPECT_EQ(test, record->size, (u16)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, record->entries, entries, sizeof(entries)); +} + +static void stackdepot_countable_does_not_alias_other_modes(struct kunit *test) +{ + unsigned long plain_entries[] = { 0x1234567800410000UL, 0x1234567800420000UL, 0x1234567800430000UL, }; - depot_stack_handle_t second_handle; - depot_stack_handle_t handle; - unsigned int count; - bool new_count; + unsigned long get_entries[] = { + 0x1234567800510000UL, + 0x1234567800520000UL, + 0x1234567800530000UL, + }; + depot_flags_t get = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; + struct stack_record *record; + depot_stack_handle_t count_handle; + depot_stack_handle_t plain_handle; + depot_stack_handle_t get_handle; + unsigned int get_nr = ARRAY_SIZE(get_entries); + unsigned int plain_nr = ARRAY_SIZE(plain_entries); KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); - handle = save_hash(entries, ARRAY_SIZE(entries)); - KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); - KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); - - new_count = false; - KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 2, &new_count)); - KUNIT_EXPECT_TRUE(test, new_count); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 3U); - - new_count = true; - KUNIT_EXPECT_TRUE(test, __stack_depot_inc_count(handle, 4, &new_count)); - KUNIT_EXPECT_FALSE(test, new_count); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 7U); - - KUNIT_EXPECT_FALSE(test, __stack_depot_dec_count_and_test(handle, 5)); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(handle, &count)); - KUNIT_EXPECT_EQ(test, count, 2U); - KUNIT_EXPECT_TRUE(test, __stack_depot_dec_count_and_test(handle, 2)); - KUNIT_EXPECT_FALSE(test, __stack_depot_get_count(handle, &count)); - - second_handle = save_hash(second_entries, ARRAY_SIZE(second_entries)); - KUNIT_ASSERT_NE(test, second_handle, (depot_stack_handle_t)0); - __stack_depot_set_count(second_handle, INT_MAX - 1); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(second_handle, &count)); - KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX - 1); - KUNIT_EXPECT_FALSE(test, - __stack_depot_inc_count(second_handle, 2, &new_count)); - KUNIT_ASSERT_TRUE(test, __stack_depot_get_count(second_handle, &count)); - KUNIT_EXPECT_EQ(test, count, (unsigned int)INT_MAX - 1); + plain_handle = stack_depot_save(plain_entries, plain_nr, GFP_KERNEL); + KUNIT_ASSERT_NE(test, plain_handle, (depot_stack_handle_t)0); + count_handle = save_hash(plain_entries, plain_nr); + KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); + record = __stack_depot_get_stack_record(count_handle); + KUNIT_ASSERT_NOT_NULL(test, record); + KUNIT_EXPECT_MEMEQ(test, record->entries, plain_entries, + sizeof(plain_entries)); + + get_handle = stack_depot_save_flags(get_entries, get_nr, GFP_KERNEL, get); + KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); + count_handle = save_hash(get_entries, get_nr); + KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); + record = __stack_depot_get_stack_record(count_handle); + KUNIT_ASSERT_NOT_NULL(test, record); + KUNIT_EXPECT_MEMEQ(test, record->entries, get_entries, sizeof(get_entries)); + + stack_depot_put(get_handle); } static void stackdepot_frame_raw_fallback(struct kunit *test) @@ -299,7 +308,6 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) unsigned long frame = 0xffff888000001000UL; unsigned long out = 0x12345678UL; bool compressed; - bool decoded; u32 low = 0xfeedbeef; #ifdef CONFIG_ARM64 @@ -312,9 +320,6 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) compressed = arch_stack_depot_frame_try_compress(frame, &low); KUNIT_EXPECT_FALSE(test, compressed); KUNIT_EXPECT_EQ(test, low, (u32)0xfeedbeef); - - decoded = arch_stack_depot_frame_decompress(0x81234567, NULL); - KUNIT_EXPECT_FALSE(test, decoded); KUNIT_EXPECT_EQ(test, out, 0x12345678UL); } @@ -496,7 +501,8 @@ static struct kunit_case stackdepot_test_cases[] = { KUNIT_CASE(stackdepot_hash_flag_roundtrip), KUNIT_CASE(stackdepot_save_flags_public), KUNIT_CASE(stackdepot_snprint_public), - KUNIT_CASE(stackdepot_count_helpers), + KUNIT_CASE(stackdepot_get_stack_record), + KUNIT_CASE(stackdepot_countable_does_not_alias_other_modes), KUNIT_CASE(stackdepot_frame_raw_fallback), #ifdef CONFIG_X86_64 KUNIT_CASE(stackdepot_frame_x86_64), diff --git a/mm/kmsan/report.c b/mm/kmsan/report.c index 8e62e98916a5f..88d318a819ddd 100644 --- a/mm/kmsan/report.c +++ b/mm/kmsan/report.c @@ -88,7 +88,6 @@ void kmsan_print_origin(depot_stack_handle_t origin) unsigned long entries[KMSAN_STACK_DEPTH]; const unsigned int max_entries = ARRAY_SIZE(entries); unsigned int nr_entries, chained_nr_entries, skipnr; - size_t chained_size; void *pc1 = NULL, *pc2 = NULL; depot_stack_handle_t head; unsigned long magic; @@ -124,11 +123,9 @@ void kmsan_print_origin(depot_stack_handle_t origin) head = entries[1]; origin = entries[2]; pr_err("Uninit was stored to memory at:\n"); - /* Save head/origin locally before reusing entries below. */ + /* Reuse entries after saving head and origin above. */ chained_nr_entries = stack_depot_fetch_into(head, entries, max_entries); - chained_size = chained_nr_entries * sizeof(*entries); - kmsan_internal_unpoison_memory(entries, chained_size, false); if (chained_nr_entries) { skipnr = get_stack_skipnr(entries, chained_nr_entries); stack_trace_print(entries + skipnr, diff --git a/mm/page_owner.c b/mm/page_owner.c index 3fd46599676ab..3fdfc922e0920 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -38,15 +38,10 @@ struct page_owner { }; struct stack { - depot_stack_handle_t handle; + struct stack_record *stack_record; struct stack *next; }; -struct page_owner_stack_seq { - struct stack *stack; - unsigned long entries[PAGE_OWNER_STACK_DEPTH]; -}; - static struct stack dummy_stack; static struct stack failure_stack; static struct stack *stack_list; @@ -128,14 +123,12 @@ static __init void init_page_owner(void) register_early_stack(); init_early_allocated_pages(); /* Initialize dummy and failure stacks and link them to stack_list. */ - dummy_stack.handle = dummy_handle; - failure_stack.handle = failure_handle; - /* These counts are the stack_list membership markers. */ - /* No page_owner count updates can race before page_owner_inited flips. */ - if (dummy_handle) - __stack_depot_set_count(dummy_handle, 1); - if (failure_handle) - __stack_depot_set_count(failure_handle, 1); + dummy_stack.stack_record = __stack_depot_get_stack_record(dummy_handle); + failure_stack.stack_record = __stack_depot_get_stack_record(failure_handle); + if (dummy_stack.stack_record) + refcount_set(&dummy_stack.stack_record->count, 1); + if (failure_stack.stack_record) + refcount_set(&failure_stack.stack_record->count, 1); dummy_stack.next = &failure_stack; stack_list = &dummy_stack; static_branch_enable(&page_owner_inited); @@ -173,89 +166,72 @@ static noinline depot_stack_handle_t save_stack(gfp_t flags) return handle; } -static struct stack *alloc_stack_record(gfp_t gfp_mask) +static void add_stack_record_to_list(struct stack_record *stack_record, + gfp_t gfp_mask) { + unsigned long flags; struct stack *stack; if (!gfpflags_allow_spinning(gfp_mask)) - return NULL; + return; set_current_in_page_owner(); stack = kmalloc(sizeof(*stack), gfp_nested_mask(gfp_mask)); - unset_current_in_page_owner(); - - return stack; -} - -static void free_stack_record(struct stack *stack) -{ - set_current_in_page_owner(); - kfree(stack); - unset_current_in_page_owner(); -} - -static void add_stack_record_to_list(depot_stack_handle_t handle, - struct stack *stack) -{ - unsigned long flags; - - if (WARN_ON_ONCE(!stack)) + if (!stack) { + unset_current_in_page_owner(); return; + } + unset_current_in_page_owner(); - stack->handle = handle; + stack->stack_record = stack_record; stack->next = NULL; spin_lock_irqsave(&stack_list_lock, flags); stack->next = stack_list; - stack_list = stack; + /* + * This pairs with smp_load_acquire() from function + * stack_start(). This guarantees that stack_start() + * will see an updated stack_list before starting to + * traverse the list. + */ + smp_store_release(&stack_list, stack); spin_unlock_irqrestore(&stack_list_lock, flags); } -static bool inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, - unsigned int nr_base_pages) +static void inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, + int nr_base_pages) { - struct stack *stack = NULL; - bool new_count = false; - unsigned int count; + struct stack_record *stack_record = __stack_depot_get_stack_record(handle); - if (!handle || !nr_base_pages) - return false; + if (!stack_record) + return; /* - * Snapshot only avoids allocation when the stack is already counted. If this - * races a final decrement to zero, inc_count() fails safely. + * New stack_record's that do not use STACK_DEPOT_FLAG_GET start + * with REFCOUNT_SATURATED to catch spurious increments of their + * refcount. + * Since we do not use STACK_DEPOT_FLAG_GET API, let us + * set a refcount of 1 ourselves. */ - if (!__stack_depot_get_count(handle, &count)) - stack = alloc_stack_record(gfp_mask); + if (refcount_read(&stack_record->count) == REFCOUNT_SATURATED) { + int old = REFCOUNT_SATURATED; - /* - * Only one caller can win the saturated-to-counted cmpxchg transition. - * Racing transition losers free their unused list node below. - */ - if (!__stack_depot_inc_count(handle, nr_base_pages, &new_count)) { - if (stack) - free_stack_record(stack); - return false; + if (atomic_try_cmpxchg_relaxed(&stack_record->count.refs, &old, 1)) + /* Add the new stack_record to our list. */ + add_stack_record_to_list(stack_record, gfp_mask); } - /* - * new_count includes the list marker. If list allocation failed, keep - * the count and handle anyway; show_stacks remains best effort. - */ - if (new_count) { - if (stack) - add_stack_record_to_list(handle, stack); - } else if (stack) { - free_stack_record(stack); - } - - return true; + refcount_add(nr_base_pages, &stack_record->count); } static void dec_stack_record_count(depot_stack_handle_t handle, - unsigned int nr_base_pages) + int nr_base_pages) { - /* Counted handles keep a marker unit; zero means it was decremented. */ - if (__stack_depot_dec_count_and_test(handle, nr_base_pages)) + struct stack_record *stack_record = __stack_depot_get_stack_record(handle); + + if (!stack_record) + return; + + if (refcount_sub_and_test(nr_base_pages, &stack_record->count)) pr_warn("%s: refcount went to 0 for %u handle\n", __func__, handle); } @@ -357,24 +333,12 @@ noinline void __set_page_owner(struct page *page, unsigned short order, { u64 ts_nsec = local_clock(); depot_stack_handle_t handle; - bool counted; - /* Any previous allocation handle for this page was decremented at free. */ handle = save_stack(gfp_mask); - counted = inc_stack_record_count(handle, gfp_mask, 1 << order); - if (!counted && handle != failure_handle) { - /* Store failure_handle only if the matching count was applied. */ - handle = failure_handle; - counted = inc_stack_record_count(handle, gfp_mask, 1 << order); - } - /* Avoid storing a handle that would later decrement an unapplied count. */ - if (!counted) { - pr_warn_ratelimited("failed to count page owner stack\n"); - handle = 0; - } __update_page_owner_handle(page, handle, order, gfp_mask, -1, ts_nsec, current->pid, current->tgid, current->comm); + inc_stack_record_count(handle, gfp_mask, 1 << order); } void __folio_set_owner_migrate_reason(struct folio *folio, int reason) @@ -899,33 +863,34 @@ static const struct file_operations proc_page_owner_operations = { static void *stack_start(struct seq_file *m, loff_t *ppos) { - struct page_owner_stack_seq *priv = m->private; - unsigned long flags; struct stack *stack; + /* m->private is only the current list cursor, not seq_open_private() data. */ if (*ppos == -1UL) return NULL; if (!*ppos) { - spin_lock_irqsave(&stack_list_lock, flags); - stack = stack_list; - spin_unlock_irqrestore(&stack_list_lock, flags); + /* + * This pairs with smp_store_release() from function + * add_stack_record_to_list(), so we get a consistent + * value of stack_list. + */ + stack = smp_load_acquire(&stack_list); } else { - stack = priv->stack; + stack = m->private; } - priv->stack = stack; + m->private = stack; return stack; } static void *stack_next(struct seq_file *m, void *v, loff_t *ppos) { - struct page_owner_stack_seq *priv = m->private; struct stack *stack = v; stack = stack->next; *ppos = stack ? *ppos + 1 : -1UL; - priv->stack = stack; + m->private = stack; return stack; } @@ -934,36 +899,25 @@ static unsigned int page_owner_pages_threshold; static int stack_print(struct seq_file *m, void *v) { - struct page_owner_stack_seq *priv = m->private; + int i, nr_base_pages; struct stack *stack = v; - depot_stack_handle_t handle = stack->handle; - unsigned int nr_base_pages = 0; - unsigned int i, nr_entries; + unsigned long *entries; + unsigned long nr_entries; + struct stack_record *stack_record = stack->stack_record; - if (!handle) + if (!stack->stack_record) return 0; - /* Counts can race with page_owner updates; seq_file output is best effort. */ - /* Treat count 1 as marker-only for best-effort reporting. */ - if (!__stack_depot_get_count(handle, &nr_base_pages) || nr_base_pages <= 1) - return 0; - /* The <= 1 guard above makes removing the list marker safe. */ - nr_base_pages--; - - /* Drop the list marker before applying the page-count threshold. */ - if (nr_base_pages < READ_ONCE(page_owner_pages_threshold)) - return 0; + nr_entries = stack_record->size; + entries = stack_record->entries; + nr_base_pages = refcount_read(&stack_record->count) - 1; - /* Keep show_stacks independent of stackdepot's internal storage layout. */ - nr_entries = stack_depot_fetch_into(handle, priv->entries, - ARRAY_SIZE(priv->entries)); - /* Buffer matches save_stack()'s cap, so exact-or-nothing fetch should fit. */ - if (!nr_entries) + if (nr_base_pages < 1 || nr_base_pages < READ_ONCE(page_owner_pages_threshold)) return 0; for (i = 0; i < nr_entries; i++) - seq_printf(m, " %pS\n", (void *)priv->entries[i]); - seq_printf(m, "nr_base_pages: %u\n\n", nr_base_pages); + seq_printf(m, " %pS\n", (void *)entries[i]); + seq_printf(m, "nr_base_pages: %d\n\n", nr_base_pages); return 0; } @@ -981,15 +935,14 @@ static const struct seq_operations page_owner_stack_op = { static int page_owner_stack_open(struct inode *inode, struct file *file) { - return seq_open_private(file, &page_owner_stack_op, - sizeof(struct page_owner_stack_seq)); + return seq_open(file, &page_owner_stack_op); } static const struct file_operations page_owner_stack_operations = { .open = page_owner_stack_open, .read = seq_read, .llseek = seq_lseek, - .release = seq_release_private, + .release = seq_release, }; static int page_owner_threshold_get(void *data, u64 *val) From 7029a5268163f01a138dfa4648b76da34c8e9041 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 25 Jun 2026 14:47:52 +0100 Subject: [PATCH 124/129] KRN-1117: Clarify stackdepot trie failure semantics Document that trie-backed saves do not fall back to hash storage when trie insertion cannot proceed. This keeps trie_enabled pool pressure visible instead of mixing backends in the same run. Also fix a indentation issue in trie lookup. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 3 ++- lib/stackdepot.c | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index d21ddfaba3966..83ddf0e100475 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -155,7 +155,8 @@ static inline int stack_depot_early_init(void) { return 0; } * * When trie storage is enabled, persistent non-refcounted saves use trie * storage. Constrained contexts remain best effort and can return 0 if a - * required trylock or reserved resource is unavailable. + * required trylock or reserved resource is unavailable; trie failures do not + * fall back to hash storage. * * If the provided stack trace comes from the interrupt context, only the part * up to the interrupt entry is saved. diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 40299526beb1e..95ebe49529c73 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -3471,7 +3471,7 @@ __stack_depot_trie_lookup_step(const struct stack_depot_trie_root *root, return 0; } - node = trie_child_array_load_child(children, pos); + node = trie_child_array_load_child(children, pos); if (!node) return -EINVAL; /* @@ -4281,7 +4281,7 @@ stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *ar node = trie_child_array_load_child(array, mid); if (!node) { - /* A tail append may publish nr_children before the child is visible. */ + /* Tail append may produce a transient lockless lookup miss. */ right = mid; continue; } From 6bbefcbbbd22f758bbda035b0e0e2e0470e354c5 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 25 Jun 2026 15:03:30 +0100 Subject: [PATCH 125/129] KRN-1117: Clarify stackdepot fetch handle constraints Document that stack_depot_save() may return trie-backed handles when trie storage is enabled. Callers that need backend-independent stack contents must use stack_depot_fetch_into(), stack_depot_print(), or stack_depot_snprint(). Also tighten the stack_depot_fetch() kernel-doc to state that it only accepts hash-backed handles, matching the legacy pointer-returning API. Signed-off-by: Caleb Kan --- include/linux/stackdepot.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 83ddf0e100475..8970537852547 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -183,6 +183,10 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, * Does not increment the refcount on the saved stack trace; see * stack_depot_save_flags() for more details. * + * When trie storage is enabled, this can return trie-backed handles. Use + * stack_depot_fetch_into(), stack_depot_print(), or stack_depot_snprint() for + * backend-independent access to the stack contents. + * * Context: Contexts where allocations via alloc_pages() are allowed; * see stack_depot_save_flags() for more details. * @@ -206,7 +210,7 @@ struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle) /** * stack_depot_fetch - Fetch a stack trace from stack depot * - * @handle: Stack depot handle returned from stack_depot_save() + * @handle: Hash-backed stack depot handle * @entries: Pointer to store the address of the stack trace * * This helper returns a pointer to stackdepot-owned contiguous storage for From 20bb97206fa3b5c45eaf78a9f436e481082f7b32 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 25 Jun 2026 15:48:26 +0100 Subject: [PATCH 126/129] KRN-1117: Clarify stackdepot hash-only diagnostics Make debugfs persistent counters explicitly hash-backed now that normal persistent saves can use trie storage. This avoids reading hash-only record counts as total stackdepot usage during trie validation. Also make the gdb stackdepot helper reject trie-backed handles instead of decoding them as hash pool offsets, and fix its pool-index diagnostic. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 4 ++-- scripts/gdb/linux/stackdepot.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 95ebe49529c73..aeda0b1d898ef 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -438,8 +438,8 @@ static const char *const counter_names[] = { [DEPOT_COUNTER_REFD_FREES] = "refcounted_frees", [DEPOT_COUNTER_REFD_INUSE] = "refcounted_in_use", [DEPOT_COUNTER_FREELIST_SIZE] = "freelist_size", - [DEPOT_COUNTER_PERSIST_COUNT] = "persistent_count", - [DEPOT_COUNTER_PERSIST_BYTES] = "persistent_bytes", + [DEPOT_COUNTER_PERSIST_COUNT] = "hash_persistent_count", + [DEPOT_COUNTER_PERSIST_BYTES] = "hash_persistent_bytes", }; static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); diff --git a/scripts/gdb/linux/stackdepot.py b/scripts/gdb/linux/stackdepot.py index 37313a5a51a0d..c661a5ae5331d 100644 --- a/scripts/gdb/linux/stackdepot.py +++ b/scripts/gdb/linux/stackdepot.py @@ -33,13 +33,17 @@ def stack_depot_fetch(handle): parts = handle.cast(handle_parts_t) offset = parts['offset'] << DEPOT_STACK_ALIGN pools_num = gdb.parse_and_eval('pools_num') + stack_max_pools = gdb.parse_and_eval('stack_max_pools') if handle == 0: raise gdb.GdbError("handle is 0\n") + if parts['pool_index_plus_1'] > stack_max_pools: + raise gdb.GdbError("trie-backed stackdepot handles are not supported\n") + pool_index = parts['pool_index_plus_1'] - 1 if pool_index >= pools_num: - gdb.write("pool index %d out of bounds (%d) for stack id 0x%08x\n" % (parts['pool_index'], pools_num, handle)) + gdb.write("pool index %d out of bounds (%d) for stack id 0x%08x\n" % (pool_index, pools_num, handle)) return gdb.Value(0), 0 stack_pools = gdb.parse_and_eval('stack_pools') From 202dfc9db5ce1fbb1df73fc124fb7ea710b9bfc0 Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Thu, 25 Jun 2026 17:41:55 +0100 Subject: [PATCH 127/129] KRN-1117: Simplify stackdepot trie init paths Initialize trie free-list buckets once during trie init instead of lazily checking and deriving that state from allocation paths. This removes repeated hot-path checks while keeping trie enablement as the single publication point for initialized trie state. Also remove a single-use transaction init wrapper, make side-table ID commit warn-only on impossible ordering, and drop a useless KUnit assertion that only checked a local initializer. Signed-off-by: Caleb Kan --- lib/stackdepot.c | 62 +++++++++++++----------------------- lib/tests/stackdepot_kunit.c | 2 -- 2 files changed, 22 insertions(+), 42 deletions(-) diff --git a/lib/stackdepot.c b/lib/stackdepot.c index aeda0b1d898ef..37ac6a7aabc60 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -181,7 +181,6 @@ __stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc); static int __stack_depot_trie_pool_carve(struct stack_depot_trie_alloc_request *req); -static void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn); static int __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, const unsigned long *entries, @@ -417,7 +416,6 @@ static struct list_head free_trie_nodes[STACK_DEPOT_TRIE_FREE_CLASSES]; static DECLARE_BITMAP(free_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); static DECLARE_BITMAP(pending_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); static DECLARE_BITMAP(free_trie_node_map, STACK_DEPOT_TRIE_FREE_CLASSES); -static bool free_trie_objects_initialized; /* The lock must be held when performing pool or freelist modifications. */ static DEFINE_RAW_SPINLOCK(pool_lock); @@ -692,12 +690,8 @@ static void __stack_depot_trie_side_table_commit_id(u32 id) raw_spin_lock_irqsave(&trie_side_table_lock, flags); next = trie_side_table_next_id; - if (WARN_ON_ONCE(id != next + 1)) { - if (id > next) - WRITE_ONCE(trie_side_table_next_id, id); - } else { + if (!WARN_ON_ONCE(id != next + 1)) WRITE_ONCE(trie_side_table_next_id, id); - } raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); } @@ -869,10 +863,24 @@ static int stack_depot_trie_init_workspace(gfp_t gfp_flags) return stack_depot_trie_install_workspace(workspace); } +static void trie_free_object_buckets_init(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) { + INIT_LIST_HEAD(&free_trie_objects[i]); + INIT_LIST_HEAD(&pending_trie_objects[i]); + INIT_LIST_HEAD(&free_trie_nodes[i]); + } +} + static int __init stack_depot_trie_init_memblock(void) { int ret; + if (__stack_depot_trie_enabled()) + return 0; + ret = stack_depot_trie_init_workspace_memblock(); if (ret) return ret; @@ -880,6 +888,7 @@ static int __init stack_depot_trie_init_memblock(void) if (ret) return ret; + trie_free_object_buckets_init(); stack_depot_trie_enable(); return 0; } @@ -888,6 +897,9 @@ static int stack_depot_trie_init(gfp_t gfp_flags) { int ret; + if (__stack_depot_trie_enabled()) + return 0; + ret = stack_depot_trie_init_workspace(gfp_flags); if (ret) return ret; @@ -895,6 +907,7 @@ static int stack_depot_trie_init(gfp_t gfp_flags) if (ret) return ret; + trie_free_object_buckets_init(); stack_depot_trie_enable(); return 0; } @@ -1143,22 +1156,6 @@ static inline void *trie_object_payload(struct stack_depot_trie_free_object *fre return (void *)free + trie_object_header_size(); } -static void trie_free_object_buckets_init_locked(void) -{ - unsigned int i; - - lockdep_assert_held(&pool_lock); - - if (free_trie_objects_initialized) - return; - for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) { - INIT_LIST_HEAD(&free_trie_objects[i]); - INIT_LIST_HEAD(&pending_trie_objects[i]); - INIT_LIST_HEAD(&free_trie_nodes[i]); - } - free_trie_objects_initialized = true; -} - static unsigned int trie_free_class(size_t size) { size = __stack_depot_trie_pool_alloc_size(size); @@ -1228,7 +1225,6 @@ static void trie_free_object_locked(const void *ptr, unsigned long rcu_state) if (!ptr) return; - trie_free_object_buckets_init_locked(); free = trie_object_header(ptr); free->rcu_state = rcu_state; class = trie_free_class(free->size); @@ -1378,7 +1374,6 @@ trie_retire_object_node_locked(const void *ptr, if (!ptr) return; - trie_free_object_buckets_init_locked(); free = trie_object_header(ptr); free->pending_node = NULL; free->pending_node_size = 0; @@ -1630,7 +1625,6 @@ static int __stack_depot_trie_pool_carve(struct stack_depot_trie_alloc_request * ret = -ENOSPC; goto out; } - trie_free_object_buckets_init_locked(); trie_drain_pending_objects_locked(); for (i = 0; i < req->nr_node_slots; i++) { req->node_slots[i].node = trie_pop_free_node(req->node_slots[i].size); @@ -1707,12 +1701,6 @@ static int __stack_depot_trie_pool_carve(struct stack_depot_trie_alloc_request * return ret; } -static void __stack_depot_trie_alloc_txn_init(struct stack_depot_trie_alloc_txn *txn) -{ - if (txn) - memset(txn, 0, sizeof(*txn)); -} - static int __stack_depot_trie_alloc_txn_reserve(struct stack_depot_trie_alloc_request *req) { u32 leaf_id; @@ -1768,7 +1756,7 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, if (ret) return ret; - __stack_depot_trie_alloc_txn_init(txn); + memset(txn, 0, sizeof(*txn)); *storage = NULL; *req = (struct stack_depot_trie_alloc_request) { .txn = txn, @@ -1881,7 +1869,7 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, goto rollback; __stack_depot_trie_side_table_commit_id(id); - __stack_depot_trie_alloc_txn_init(txn); + memset(txn, 0, sizeof(*txn)); *leaf_id = id; return 0; @@ -2530,12 +2518,6 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, int ret; workspace = stack_depot_trie_load_workspace(); - if (!entries || !nr_entries || !workspace) - return 0; - if (depot_flags & STACK_DEPOT_FLAG_GET) - return 0; - if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) - nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); if (handle) diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index e9034907bc909..48b3612a634c4 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -306,7 +306,6 @@ static void stackdepot_countable_does_not_alias_other_modes(struct kunit *test) static void stackdepot_frame_raw_fallback(struct kunit *test) { unsigned long frame = 0xffff888000001000UL; - unsigned long out = 0x12345678UL; bool compressed; u32 low = 0xfeedbeef; @@ -320,7 +319,6 @@ static void stackdepot_frame_raw_fallback(struct kunit *test) compressed = arch_stack_depot_frame_try_compress(frame, &low); KUNIT_EXPECT_FALSE(test, compressed); KUNIT_EXPECT_EQ(test, low, (u32)0xfeedbeef); - KUNIT_EXPECT_EQ(test, out, 0x12345678UL); } #ifdef CONFIG_X86_64 From 8cb7e4d31d96b55f5783774618bd84f6b80c9bbc Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 26 Jun 2026 11:11:03 +0100 Subject: [PATCH 128/129] KRN-1117: Tighten stackdepot trie invariants Type replacement child-array storage, warn when trie pool rollback cannot unwind, and avoid a redundant parent-chain length walk during trie lookup. Also keep arch frame decompression infallible to match the compressor contract, retry side-table boundary races once, and drop small page_owner and KUnit dead checks. Signed-off-by: Caleb Kan --- arch/arm64/include/asm/stackdepot.h | 3 +- arch/x86/include/asm/stackdepot.h | 3 +- include/asm-generic/stackdepot.h | 4 +- lib/stackdepot.c | 111 +++++++++++++++------------- lib/tests/stackdepot_kunit.c | 14 +--- mm/page_owner.c | 3 +- 6 files changed, 68 insertions(+), 70 deletions(-) diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h index 70bd2d60cd07b..794afb203a39e 100644 --- a/arch/arm64/include/asm/stackdepot.h +++ b/arch/arm64/include/asm/stackdepot.h @@ -34,11 +34,10 @@ arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) return true; } -static inline bool +static inline void arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) { *frame = arch_stack_depot_frame_from_low(low); - return true; } #endif /* __ASM_STACKDEPOT_H */ diff --git a/arch/x86/include/asm/stackdepot.h b/arch/x86/include/asm/stackdepot.h index 5bd7c58a167e6..14229b731fc32 100644 --- a/arch/x86/include/asm/stackdepot.h +++ b/arch/x86/include/asm/stackdepot.h @@ -27,11 +27,10 @@ arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) return true; } -static inline bool +static inline void arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) { *frame = STACK_DEPOT_X86_64_FRAME_PREFIX | low; - return true; } #else diff --git a/include/asm-generic/stackdepot.h b/include/asm-generic/stackdepot.h index 26c2073ccda03..846975767bdd4 100644 --- a/include/asm-generic/stackdepot.h +++ b/include/asm-generic/stackdepot.h @@ -10,10 +10,10 @@ arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) return false; } -static inline bool +static inline void arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) { - return false; + /* Generic code never compresses frames, so this hook is unreachable. */ } #endif /* __ASM_GENERIC_STACKDEPOT_H */ diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 37ac6a7aabc60..59c8cc5360e3b 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -137,8 +137,8 @@ struct stack_depot_trie_alloc_request { struct stack_depot_trie_alloc_txn *txn; struct stack_depot_trie_node_slot *node_slots; struct stack_depot_trie_child_array_slot *child_slots; - /* Optional opaque replacement child-array storage. */ - void **storage; + /* Optional replacement child-array storage. */ + struct stack_depot_trie_child_array **storage; /* Optional fresh stackdepot pool page, preallocated before insertion. */ void **pool_prealloc; struct stack_depot_trie_side_prealloc *side_prealloc; @@ -153,7 +153,7 @@ struct stack_depot_trie_alloc_workspace { struct stack_depot_trie_node_slot node_slots[STACK_DEPOT_TRIE_MAX_NODE_SLOTS]; struct stack_depot_trie_child_array_slot child_slots[STACK_DEPOT_TRIE_MAX_CHILD_SLOTS]; u32 scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; - void *storage; + struct stack_depot_trie_child_array *storage; }; #define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 @@ -190,7 +190,8 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, struct stack_depot_trie_alloc_txn *txn, - void **storage, void **pool_prealloc, + struct stack_depot_trie_child_array **storage, + void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_request *req); static int @@ -261,7 +262,8 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, void *new_storage, + unsigned int nr_scratch, + struct stack_depot_trie_child_array *new_storage, size_t new_storage_size); static int __stack_depot_trie_insert_plan(const struct stack_depot_trie_root *root, @@ -290,7 +292,8 @@ __stack_depot_trie_split_child_array_init(void *storage, size_t storage_size, static int __stack_depot_trie_child_array_insert(const struct stack_depot_trie_child_array *old, const struct stack_depot_trie_node *child, - void *new_storage, size_t new_storage_size); + struct stack_depot_trie_child_array *new_storage, + size_t new_storage_size); /* * The pool_index is offset by 1 so the first record does not have a 0 handle. @@ -1735,7 +1738,8 @@ __stack_depot_trie_alloc_txn_plan(const struct stack_depot_trie_root *root, struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, struct stack_depot_trie_alloc_txn *txn, - void **storage, void **pool_prealloc, + struct stack_depot_trie_child_array **storage, + void **pool_prealloc, struct stack_depot_trie_side_prealloc *side_prealloc, struct stack_depot_trie_alloc_request *req) { @@ -1845,7 +1849,7 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, { struct stack_depot_trie_alloc_txn *txn; u32 id; - void *storage; + struct stack_depot_trie_child_array *storage; int ret; if (!root || !req || !req->txn || !leaf_id) @@ -1882,13 +1886,17 @@ __stack_depot_trie_alloc_txn_insert(struct stack_depot_trie_root *root, static void __stack_depot_trie_alloc_txn_rollback(struct stack_depot_trie_alloc_txn *txn) { unsigned long flags; + bool rolled_back = true; if (!txn) return; - raw_spin_lock_irqsave(&pool_lock, flags); - stack_depot_trie_pool_rollback_locked(&txn->pool); - raw_spin_unlock_irqrestore(&pool_lock, flags); + if (txn->pool.size) { + raw_spin_lock_irqsave(&pool_lock, flags); + rolled_back = stack_depot_trie_pool_rollback_locked(&txn->pool); + raw_spin_unlock_irqrestore(&pool_lock, flags); + WARN_ON_ONCE(!rolled_back); + } txn->leaf_id = 0; memset(&txn->pool, 0, sizeof(txn->pool)); } @@ -2514,11 +2522,16 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, void *pool_prealloc = NULL; depot_stack_handle_t handle; unsigned long flags; + bool can_alloc; + bool retried = false; u32 leaf_id; int ret; workspace = stack_depot_trie_load_workspace(); + can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && + gfpflags_allow_spinning(alloc_flags); +retry: handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); if (handle) return handle; @@ -2551,6 +2564,16 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, handle = __stack_depot_trie_handle(leaf_id); } raw_spin_unlock_irqrestore(&stack_depot_trie_workspace_lock, flags); + if (!handle && ret == -ENOSPC && can_alloc && !retried) { + retried = true; + depot_try_keep_new_pool(&pool_prealloc); + if (pool_prealloc) { + free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); + pool_prealloc = NULL; + } + __stack_depot_trie_side_table_free_prealloc(&side_prealloc); + goto retry; + } out_free: depot_try_keep_new_pool(&pool_prealloc); @@ -2560,17 +2583,10 @@ stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, return handle; } -struct stack_depot_hash_save { - struct list_head *bucket; - unsigned long *entries; - unsigned int nr_entries; - u32 hash; - depot_flags_t depot_flags; - void **prealloc; -}; - static depot_stack_handle_t -depot_save_stack_locked(struct stack_depot_hash_save *save) +depot_save_stack_locked(struct list_head *bucket, unsigned long *entries, + unsigned int nr_entries, u32 hash, + depot_flags_t depot_flags, void **prealloc) { struct stack_record *found; struct stack_record *new; @@ -2578,12 +2594,10 @@ depot_save_stack_locked(struct stack_depot_hash_save *save) lockdep_assert_held(&pool_lock); /* Try to find again, to avoid concurrently inserting duplicates. */ - found = find_stack(save->bucket, save->entries, save->nr_entries, - save->hash, save->depot_flags); + found = find_stack(bucket, entries, nr_entries, hash, depot_flags); if (found) return found->handle.handle; - new = depot_alloc_stack(save->entries, save->nr_entries, save->hash, - save->depot_flags, save->prealloc); + new = depot_alloc_stack(entries, nr_entries, hash, depot_flags, prealloc); if (!new) return 0; @@ -2591,7 +2605,7 @@ depot_save_stack_locked(struct stack_depot_hash_save *save) * This releases the stack record into the bucket and makes it visible to * readers in find_stack(). */ - list_add_rcu(&new->hash_list, save->bucket); + list_add_rcu(&new->hash_list, bucket); return new->handle.handle; } @@ -2601,7 +2615,6 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, depot_flags_t depot_flags) { struct list_head *bucket; - struct stack_depot_hash_save save; struct stack_record *found = NULL; depot_stack_handle_t handle = 0; struct page *page = NULL; @@ -2646,15 +2659,6 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, hash = hash_stack(entries, nr_entries); bucket = &stack_table[hash & stack_hash_mask]; - save = (struct stack_depot_hash_save) { - .bucket = bucket, - .entries = entries, - .nr_entries = nr_entries, - .hash = hash, - .depot_flags = depot_flags, - .prealloc = &prealloc, - }; - /* Fast path: look the stack trace up without locking. */ found = find_stack(bucket, entries, nr_entries, hash, depot_flags); if (found) @@ -2677,7 +2681,8 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, if (!raw_spin_trylock_irqsave(&pool_lock, flags)) goto out_free; printk_deferred_enter(); - handle = depot_save_stack_locked(&save); + handle = depot_save_stack_locked(bucket, entries, nr_entries, + hash, depot_flags, &prealloc); if (prealloc) { /* * Either stack depot already contains this stack trace, or @@ -2693,7 +2698,8 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, raw_spin_lock_irqsave(&pool_lock, flags); printk_deferred_enter(); - handle = depot_save_stack_locked(&save); + handle = depot_save_stack_locked(bucket, entries, nr_entries, + hash, depot_flags, &prealloc); if (prealloc) { /* * Either stack depot already contains this stack trace, or @@ -2869,8 +2875,7 @@ stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, } memcpy(&low, node->data + index * sizeof(low), sizeof(low)); - if (!arch_stack_depot_frame_decompress(low, frame)) - return -EINVAL; + arch_stack_depot_frame_decompress(low, frame); return 0; } @@ -3126,7 +3131,8 @@ trie_child_array_can_append(const struct stack_depot_trie_child_array *array, static void trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array, const struct stack_depot_trie_node *new_child, - void *new_storage, size_t new_storage_size, + struct stack_depot_trie_child_array *new_storage, + size_t new_storage_size, unsigned int pos) { struct stack_depot_trie_child_array *new_array = new_storage; @@ -3190,7 +3196,8 @@ trie_promote_child(struct stack_depot_trie_root *root, unsigned int pos, const struct stack_depot_trie_node *child, u32 leaf_id, const struct stack_depot_trie_node_slot *slot, - void *new_storage, size_t new_storage_size) + struct stack_depot_trie_child_array *new_storage, + size_t new_storage_size) { const struct stack_depot_trie_child_array **publish_slot; struct stack_depot_trie_leaf_update update; @@ -3326,8 +3333,8 @@ __stack_depot_trie_append_chain(const struct stack_depot_trie_node *parent, static int trie_publish_append_prepare(struct stack_depot_trie_root *root, struct stack_depot_trie_node *parent, const struct stack_depot_trie_node *head, - void *new_storage, size_t new_storage_size, - u32 leaf_id, + struct stack_depot_trie_child_array *new_storage, + size_t new_storage_size, u32 leaf_id, const struct stack_depot_trie_node *leaf) { const struct stack_depot_trie_child_array *old_array; @@ -3504,12 +3511,8 @@ __stack_depot_trie_find_leaf(const struct stack_depot_trie_root *root, node = lookup.node; if (node) { const struct stack_depot_trie_node *node_parent; - unsigned int node_len; node_parent = trie_load_parent(node); - if (trie_node_stack_len(node, &node_len) || - node_len != pos + lookup.matched) - return NULL; if (node_parent != parent && !trie_parent_chain_matches_prefix(node_parent, entries, pos)) @@ -3550,7 +3553,8 @@ static int trie_split_child(struct stack_depot_trie_root *root, unsigned int nr_node_slots, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, void *new_storage, + unsigned int nr_scratch, + struct stack_depot_trie_child_array *new_storage, size_t new_storage_size); static int @@ -3564,7 +3568,8 @@ __stack_depot_trie_insert_append_prepare(struct stack_depot_trie_root *root, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, void *new_storage, + unsigned int nr_scratch, + struct stack_depot_trie_child_array *new_storage, size_t new_storage_size) { const struct stack_depot_trie_node *head; @@ -4204,7 +4209,8 @@ static int trie_split_child(struct stack_depot_trie_root *root, unsigned int nr_node_slots, const struct stack_depot_trie_child_array_slot *child_slots, unsigned int nr_child_slots, u32 *scratch, - unsigned int nr_scratch, void *new_storage, + unsigned int nr_scratch, + struct stack_depot_trie_child_array *new_storage, size_t new_storage_size) { const struct stack_depot_trie_child_array **publish_slot; @@ -4287,7 +4293,8 @@ stack_depot_trie_child_lower_bound(const struct stack_depot_trie_child_array *ar static int __stack_depot_trie_child_array_insert(const struct stack_depot_trie_child_array *old, const struct stack_depot_trie_node *node, - void *new_storage, size_t new_storage_size) + struct stack_depot_trie_child_array *new_storage, + size_t new_storage_size) { struct stack_depot_trie_child_array *new_array = new_storage; unsigned int nr_old; diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c index 48b3612a634c4..167543136adad 100644 --- a/lib/tests/stackdepot_kunit.c +++ b/lib/tests/stackdepot_kunit.c @@ -328,14 +328,12 @@ static void stackdepot_frame_x86_64(struct kunit *test) unsigned long frame = 0xffffffff81234567UL; unsigned long out; bool compressed; - bool decoded; u32 low; compressed = arch_stack_depot_frame_try_compress(frame, &low); KUNIT_EXPECT_TRUE(test, compressed); KUNIT_EXPECT_EQ(test, low, (u32)0x81234567); - decoded = arch_stack_depot_frame_decompress(low, &out); - KUNIT_EXPECT_TRUE(test, decoded); + arch_stack_depot_frame_decompress(low, &out); KUNIT_EXPECT_EQ(test, out, frame); compressed = arch_stack_depot_frame_try_compress(direct_map, &low); @@ -352,30 +350,26 @@ static void stackdepot_frame_arm64(struct kunit *test) unsigned long frame = stackdepot_arm64_frame(offset); unsigned long out; bool compressed; - bool decoded; u32 low; compressed = arch_stack_depot_frame_try_compress(frame, &low); KUNIT_EXPECT_TRUE(test, compressed); KUNIT_EXPECT_EQ(test, low, (u32)(s32)offset); - decoded = arch_stack_depot_frame_decompress(low, &out); - KUNIT_EXPECT_TRUE(test, decoded); + arch_stack_depot_frame_decompress(low, &out); KUNIT_EXPECT_EQ(test, out, frame); frame = stackdepot_arm64_frame(negative_offset); compressed = arch_stack_depot_frame_try_compress(frame, &low); KUNIT_EXPECT_TRUE(test, compressed); KUNIT_EXPECT_EQ(test, low, (u32)(s32)negative_offset); - decoded = arch_stack_depot_frame_decompress(low, &out); - KUNIT_EXPECT_TRUE(test, decoded); + arch_stack_depot_frame_decompress(low, &out); KUNIT_EXPECT_EQ(test, out, frame); frame = stackdepot_arm64_frame(positive_offset); compressed = arch_stack_depot_frame_try_compress(frame, &low); KUNIT_EXPECT_TRUE(test, compressed); KUNIT_EXPECT_EQ(test, low, (u32)(s32)positive_offset); - decoded = arch_stack_depot_frame_decompress(low, &out); - KUNIT_EXPECT_TRUE(test, decoded); + arch_stack_depot_frame_decompress(low, &out); KUNIT_EXPECT_EQ(test, out, frame); } #endif /* CONFIG_ARM64 */ diff --git a/mm/page_owner.c b/mm/page_owner.c index 3fdfc922e0920..e7fbc0525d074 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -316,8 +316,7 @@ void __reset_page_owner(struct page *page, unsigned short order) __update_page_owner_free_handle(page, handle, order, current->pid, current->tgid, free_ts_nsec); - /* A zero handle means no allocation stack count was applied. */ - if (alloc_handle && alloc_handle != early_handle) + if (alloc_handle != early_handle) /* * early_handle is being set as a handle for all those * early allocated pages. See init_pages_in_zone(). From 09c919f799a4adf47da015efb1a10290e0debacf Mon Sep 17 00:00:00 2001 From: Caleb Kan Date: Fri, 26 Jun 2026 11:44:02 +0100 Subject: [PATCH 129/129] KRN-1117: Remove unrelated page_owner changes Keep the page_owner changes in this series limited to routing stack saves through countable stackdepot records. Drop the independent seq_file and threshold fixes, along with formatting-only churn, so those changes do not get buried in the trie series. Signed-off-by: Caleb Kan --- mm/page_owner.c | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/mm/page_owner.c b/mm/page_owner.c index e7fbc0525d074..9f78413fee60a 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -41,7 +40,6 @@ struct stack { struct stack_record *stack_record; struct stack *next; }; - static struct stack dummy_stack; static struct stack failure_stack; static struct stack *stack_list; @@ -122,7 +120,7 @@ static __init void init_page_owner(void) register_failure_stack(); register_early_stack(); init_early_allocated_pages(); - /* Initialize dummy and failure stacks and link them to stack_list. */ + /* Initialize dummy and failure stacks and link them to stack_list */ dummy_stack.stack_record = __stack_depot_get_stack_record(dummy_handle); failure_stack.stack_record = __stack_depot_get_stack_record(failure_handle); if (dummy_stack.stack_record) @@ -217,7 +215,7 @@ static void inc_stack_record_count(depot_stack_handle_t handle, gfp_t gfp_mask, int old = REFCOUNT_SATURATED; if (atomic_try_cmpxchg_relaxed(&stack_record->count.refs, &old, 1)) - /* Add the new stack_record to our list. */ + /* Add the new stack_record to our list */ add_stack_record_to_list(stack_record, gfp_mask); } refcount_add(nr_base_pages, &stack_record->count); @@ -541,8 +539,8 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret, static ssize_t print_page_owner(char __user *buf, size_t count, unsigned long pfn, - struct page *page, struct page_owner *page_owner, - depot_stack_handle_t handle) + struct page *page, struct page_owner *page_owner, + depot_stack_handle_t handle) { int ret, pageblock_mt, page_mt; char *kbuf; @@ -641,13 +639,13 @@ void __dump_page_owner(const struct page *page) pr_alert("page_owner free stack trace missing\n"); } else { pr_alert("page last free pid %d tgid %d stack trace:\n", - page_owner->free_pid, page_owner->free_tgid); + page_owner->free_pid, page_owner->free_tgid); stack_depot_print(handle); } if (page_owner->last_migrate_reason != -1) pr_alert("page has been migrated, last migrate reason: %s\n", - migrate_reason_names[page_owner->last_migrate_reason]); + migrate_reason_names[page_owner->last_migrate_reason]); page_ext_put(page_ext); } @@ -864,7 +862,6 @@ static void *stack_start(struct seq_file *m, loff_t *ppos) { struct stack *stack; - /* m->private is only the current list cursor, not seq_open_private() data. */ if (*ppos == -1UL) return NULL; @@ -875,10 +872,10 @@ static void *stack_start(struct seq_file *m, loff_t *ppos) * value of stack_list. */ stack = smp_load_acquire(&stack_list); + m->private = stack; } else { stack = m->private; } - m->private = stack; return stack; } @@ -894,7 +891,7 @@ static void *stack_next(struct seq_file *m, void *v, loff_t *ppos) return stack; } -static unsigned int page_owner_pages_threshold; +static unsigned long page_owner_pages_threshold; static int stack_print(struct seq_file *m, void *v) { @@ -911,7 +908,7 @@ static int stack_print(struct seq_file *m, void *v) entries = stack_record->entries; nr_base_pages = refcount_read(&stack_record->count) - 1; - if (nr_base_pages < 1 || nr_base_pages < READ_ONCE(page_owner_pages_threshold)) + if (nr_base_pages < 1 || nr_base_pages < page_owner_pages_threshold) return 0; for (i = 0; i < nr_entries; i++) @@ -934,14 +931,14 @@ static const struct seq_operations page_owner_stack_op = { static int page_owner_stack_open(struct inode *inode, struct file *file) { - return seq_open(file, &page_owner_stack_op); + return seq_open_private(file, &page_owner_stack_op, 0); } static const struct file_operations page_owner_stack_operations = { .open = page_owner_stack_open, .read = seq_read, .llseek = seq_lseek, - .release = seq_release, + .release = seq_release_private, }; static int page_owner_threshold_get(void *data, u64 *val) @@ -952,16 +949,14 @@ static int page_owner_threshold_get(void *data, u64 *val) static int page_owner_threshold_set(void *data, u64 val) { - if (val > UINT_MAX) - return -ERANGE; - - WRITE_ONCE(page_owner_pages_threshold, (unsigned int)val); + WRITE_ONCE(page_owner_pages_threshold, val); return 0; } DEFINE_SIMPLE_ATTRIBUTE(proc_page_owner_threshold, &page_owner_threshold_get, &page_owner_threshold_set, "%llu"); + static int __init pageowner_init(void) { struct dentry *dir;