diff --git a/.github/codeql/stackdepot/StackDepot.qll b/.github/codeql/stackdepot/StackDepot.qll new file mode 100644 index 0000000000000..c59edaa253364 --- /dev/null +++ b/.github/codeql/stackdepot/StackDepot.qll @@ -0,0 +1,69 @@ +import cpp + +predicate isStackDepotFile(File f) { + f.getRelativePath() = "lib/stackdepot.c" +} + +predicate isStackDepotFunction(Function f) { + isStackDepotFile(f.getFile()) +} + +predicate isTrieWriterOrchestrator(Function f) { + isStackDepotFunction(f) and + f.getName() = "stack_depot_trie_insert_locked" +} + +predicate trustedStructuralHelperName(string name) { + name = "trie_build_append_chain" or + name = "trie_build_split" or + name = "trie_child_array_insert_at" or + name = "trie_child_array_replace_at" or + name = "trie_node_init_slice" or + name = "trie_publish_tail_append" or + name = "trie_reparent_children" +} + +predicate isTrustedStructuralHelper(Function f) { + isStackDepotFunction(f) and + trustedStructuralHelperName(f.getName()) +} + +predicate isSidePublishFunction(Function f) { + isStackDepotFunction(f) and + (f.getName() = "trie_side_table_publish_new_leaf" or + f.getName() = "trie_side_table_publish_split_leaves") +} + +predicate isBoundaryFunction(Function f) { + isStackDepotFunction(f) and + ( + f.getName().matches("%init%") or + f.getName().matches("%prealloc%") or + f.getName().matches("%lookup%") or + f.getName().matches("%fetch%") or + f.getName().matches("%save%") or + f.getName().matches("%put%") or + f.getName().matches("%print%") or + f.getName().matches("%snprint%") or + f.getName().matches("%handle%") + ) +} + +bindingset[name] +predicate isForbiddenTrieAbstractionName(string name) { + name.matches("%lookup_status%") or + (name.matches("%trie_lookup%") and name.matches("%status%")) or + name.matches("%alloc_txn%") or + name.matches("%alloc_request%") or + name.matches("%pool_mark%") or + name.matches("%rollback%") or + name.matches("%insert_plan%") or + name.matches("%child_array_slot%") or + name.matches("%node_slot%") or + name.matches("%leaf_update%") +} + +predicate returnsEinvalExpr(Expr e) { + // Linux UAPI errno value for EINVAL. + e.getValue().toInt() = -22 +} diff --git a/.github/codeql/stackdepot/StackDepotForbiddenTrieAbstractions.ql b/.github/codeql/stackdepot/StackDepotForbiddenTrieAbstractions.ql new file mode 100644 index 0000000000000..0ccf3a822b3e1 --- /dev/null +++ b/.github/codeql/stackdepot/StackDepotForbiddenTrieAbstractions.ql @@ -0,0 +1,17 @@ +/** + * @name Deleted stackdepot trie abstraction reintroduced + * @description Flags names matching deleted lookup/status/transaction/slot/rollback machinery. + * @kind problem + * @problem.severity recommendation + * @precision high + * @id stackdepot/forbidden-trie-abstraction + */ + +import cpp +import StackDepot + +from Declaration d +where + isStackDepotFile(d.getFile()) and + isForbiddenTrieAbstractionName(d.getName()) +select d, "Stackdepot trie declaration '$@' looks like deleted status, transaction, plan, slot, update, or rollback machinery.", d, d.getName() diff --git a/.github/codeql/stackdepot/StackDepotGenericStateCarrier.ql b/.github/codeql/stackdepot/StackDepotGenericStateCarrier.ql new file mode 100644 index 0000000000000..1613ba91ace20 --- /dev/null +++ b/.github/codeql/stackdepot/StackDepotGenericStateCarrier.ql @@ -0,0 +1,45 @@ +/** + * @name Stackdepot trie generic state carrier + * @description Flags one-field trie wrappers and pointer/size carrier structs that may lack a real invariant. + * @kind problem + * @problem.severity recommendation + * @precision medium + * @id stackdepot/generic-state-carrier + */ + +import cpp +import StackDepot + +predicate isPointerSizeCarrier(Struct s) { + exists(Field ptr, Field sz | + ptr = s.getAField() and + sz = s.getAField() and + ptr != sz and + ptr.getType().toString().matches("%*%") and + (sz.getName().matches("%size%") or + sz.getName().matches("%bytes%") or + sz.getName().matches("%capacity%")) + ) +} + +predicate isKnownTrieStateStruct(Struct s) { + s.getName() = "stack_depot_trie_node" or + s.getName() = "stack_depot_trie_child_array" or + s.getName() = "stack_depot_trie_free_node" or + s.getName() = "stack_depot_trie_free_object" or + s.getName() = "stack_depot_trie_side_dir" or + s.getName() = "stack_depot_trie_side_root" or + s.getName() = "stack_depot_trie_side_prealloc" or + s.getName() = "stack_depot_trie_alloc_workspace" +} + +from Struct s +where + isStackDepotFile(s.getFile()) and + s.getName().matches("%trie%") and + not isKnownTrieStateStruct(s) and + ( + count(Field f | f = s.getAField()) = 1 or + isPointerSizeCarrier(s) + ) +select s, "Trie struct '$@' looks like a one-field wrapper or generic pointer/size carrier; verify it carries a real invariant.", s, s.getName() diff --git a/.github/codeql/stackdepot/StackDepotKunitOnlyProductionHook.ql b/.github/codeql/stackdepot/StackDepotKunitOnlyProductionHook.ql new file mode 100644 index 0000000000000..d5a75a29d3b93 --- /dev/null +++ b/.github/codeql/stackdepot/StackDepotKunitOnlyProductionHook.ql @@ -0,0 +1,35 @@ +/** + * @name Stackdepot production helper used only by KUnit + * @description Flags stackdepot production helpers that appear to exist only for KUnit tests. + * @kind problem + * @problem.severity recommendation + * @precision medium + * @id stackdepot/kunit-only-production-hook + */ + +import cpp +import StackDepot + +predicate calledFromStackdepotKunit(Function f) { + exists(FunctionCall call | + call.getTarget() = f and + call.getFile().getRelativePath() = "lib/tests/stackdepot_kunit.c" + ) +} + +predicate calledFromNonKunit(Function f) { + exists(FunctionCall call | + call.getTarget() = f and + call.getFile().getRelativePath() != "lib/tests/stackdepot_kunit.c" + ) +} + +from Function f +where + isStackDepotFunction(f) and + not f.getName().matches("%kunit%") and + not f.getName().matches("stack_depot_%") and + not f.getName().matches("__stack_depot_%") and + calledFromStackdepotKunit(f) and + not calledFromNonKunit(f) +select f, "Production stackdepot helper '$@' appears KUnit-only; avoid production helper surface just for tests.", f, f.getName() diff --git a/.github/codeql/stackdepot/StackDepotPageOwnerCountableSaves.ql b/.github/codeql/stackdepot/StackDepotPageOwnerCountableSaves.ql new file mode 100644 index 0000000000000..ebf854bc46537 --- /dev/null +++ b/.github/codeql/stackdepot/StackDepotPageOwnerCountableSaves.ql @@ -0,0 +1,20 @@ +/** + * @name page_owner stackdepot save must stay countable + * @description Flags page_owner stackdepot saves that are not explicitly countable/hash-backed. + * @kind problem + * @problem.severity recommendation + * @precision high + * @id stackdepot/page-owner-countable-save + */ + +import cpp + +predicate isPageOwnerFile(File f) { + f.getRelativePath() = "mm/page_owner.c" +} + +from FunctionCall call +where + isPageOwnerFile(call.getFile()) and + call.getTarget().hasName("stack_depot_save") +select call, "page_owner direct stack_depot_save() can route into trie storage; use stack_depot_save_flags() with STACK_DEPOT_FLAG_COUNTABLE." diff --git a/.github/codeql/stackdepot/StackDepotTrieNodeSizeRevalidation.ql b/.github/codeql/stackdepot/StackDepotTrieNodeSizeRevalidation.ql new file mode 100644 index 0000000000000..1be46d0b367fe --- /dev/null +++ b/.github/codeql/stackdepot/StackDepotTrieNodeSizeRevalidation.ql @@ -0,0 +1,23 @@ +/** + * @name Stackdepot trie node-size helper performs defensive checks + * @description Flags defensive checks inside __stack_depot_trie_node_size(); constructed trie runs should make node size a direct calculation. + * @kind problem + * @problem.severity recommendation + * @precision medium + * @id stackdepot/trie-node-size-defensive-check + * @previous-id stackdepot/trie-node-size-revalidation + */ + +import cpp +import StackDepot + +predicate isNodeSizeHelper(Function f) { + isStackDepotFile(f.getFile()) and + f.getName() = "__stack_depot_trie_node_size" +} + +from Function f, IfStmt ifs +where + isNodeSizeHelper(f) and + ifs.getEnclosingFunction() = f +select ifs, "__stack_depot_trie_node_size() contains conditional validation; constructed frame-run metadata should make node size a direct calculation." diff --git a/.github/codeql/stackdepot/StackDepotTrieNullModeSelector.ql b/.github/codeql/stackdepot/StackDepotTrieNullModeSelector.ql new file mode 100644 index 0000000000000..22b8c390151a9 --- /dev/null +++ b/.github/codeql/stackdepot/StackDepotTrieNullModeSelector.ql @@ -0,0 +1,27 @@ +/** + * @name Stackdepot trie helper uses NULL as operation mode + * @description Flags trusted structural helpers branching on pointer parameters, a common hidden operation selector. + * @kind problem + * @problem.severity recommendation + * @precision medium + * @id stackdepot/trie-null-mode-selector + */ + +import cpp +import StackDepot + +predicate conditionMentionsPointerParam(IfStmt ifs, Parameter p) { + p.getType().toString().matches("%*%") and + ( + ifs.getCondition().toString().matches("%" + p.getName() + "%NULL%") or + ifs.getCondition().toString().matches("%!" + p.getName() + "%") + ) +} + +from Function f, Parameter p, IfStmt ifs +where + isTrustedStructuralHelper(f) and + p.getFunction() = f and + ifs.getEnclosingFunction() = f and + conditionMentionsPointerParam(ifs, p) +select ifs, "Trusted trie helper $@ branches on pointer parameter '$@'; verify NULL is not an operation selector.", f, f.getName(), p, p.getName() diff --git a/.github/codeql/stackdepot/StackDepotTrieOncePrimitive.ql b/.github/codeql/stackdepot/StackDepotTrieOncePrimitive.ql new file mode 100644 index 0000000000000..d7c78f455272a --- /dev/null +++ b/.github/codeql/stackdepot/StackDepotTrieOncePrimitive.ql @@ -0,0 +1,21 @@ +/** + * @name Stackdepot trie READ_ONCE or WRITE_ONCE use + * @description Flags trie-specific READ_ONCE/WRITE_ONCE uses so reviewers verify a lockless reader or publication pairing. + * @kind problem + * @problem.severity recommendation + * @precision high + * @id stackdepot/trie-once-primitive + */ + +import cpp +import StackDepot + +from MacroInvocation m +where + isStackDepotFile(m.getFile()) and + (m.getMacroName() = "READ_ONCE" or m.getMacroName() = "WRITE_ONCE") and + ( + m.toString().matches("%nr_children%") or + m.toString().matches("%trie%") + ) +select m, "Trie-specific $@ use; verify a documented lockless reader or publication pairing justifies it.", m, m.getMacroName() diff --git a/.github/codeql/stackdepot/StackDepotTriePublishThenFailure.ql b/.github/codeql/stackdepot/StackDepotTriePublishThenFailure.ql new file mode 100644 index 0000000000000..5d18c090e2f7b --- /dev/null +++ b/.github/codeql/stackdepot/StackDepotTriePublishThenFailure.ql @@ -0,0 +1,43 @@ +/** + * @name Stackdepot trie normal failure after publication + * @description Flags normal error returns after side-table or structural publication starts in the writer path. + * @kind problem + * @problem.severity recommendation + * @precision medium + * @id stackdepot/trie-publish-then-failure + */ + +import cpp +import StackDepot + +predicate publishCall(FunctionCall call) { + call.getTarget().getName().matches("trie_side_table_publish_%") or + call.getTarget().getName().matches("trie_publish_%") +} + +predicate normalFailureReturn(ReturnStmt ret) { + ret.hasExpr() and + ( + ret.getExpr().toString().matches("-%") or + ret.getExpr().toString() = "ret" + ) +} + +predicate retFailureGuard(IfStmt guard) { + guard.getCondition().toString().matches("%ret%") +} + +from Function f, FunctionCall pub, IfStmt guard, ReturnStmt ret +where + isTrieWriterOrchestrator(f) and + pub.getEnclosingFunction() = f and + guard.getEnclosingFunction() = f and + ret.getEnclosingFunction() = f and + publishCall(pub) and + retFailureGuard(guard) and + normalFailureReturn(ret) and + guard.getLocation().getStartLine() > pub.getLocation().getStartLine() and + guard.getLocation().getStartLine() <= pub.getLocation().getStartLine() + 2 and + ret.getLocation().getStartLine() >= guard.getLocation().getStartLine() and + ret.getLocation().getStartLine() <= guard.getLocation().getEndLine() +select ret, "Normal failure guard immediately after trie side-table or structural publication; expected failures should happen before publication begins." diff --git a/.github/codeql/stackdepot/StackDepotTrieSidePublishInTrustedHelper.ql b/.github/codeql/stackdepot/StackDepotTrieSidePublishInTrustedHelper.ql new file mode 100644 index 0000000000000..7f21483762688 --- /dev/null +++ b/.github/codeql/stackdepot/StackDepotTrieSidePublishInTrustedHelper.ql @@ -0,0 +1,23 @@ +/** + * @name Trusted stackdepot trie helper publishes side-table state directly + * @description Flags direct side-table publication calls in trusted trie insertion helpers. + * @kind problem + * @problem.severity recommendation + * @precision high + * @id stackdepot/trie-side-publish-in-trusted-helper + */ + +import cpp +import StackDepot + +predicate sidePublishFunctionName(string name) { + name = "trie_side_table_publish_new_leaf" or + name = "trie_side_table_publish_split_leaves" +} + +from FunctionCall call, Function f +where + f = call.getEnclosingFunction() and + isTrustedStructuralHelper(f) and + sidePublishFunctionName(call.getTarget().getName()) +select call, "Trusted trie insertion helper $@ publishes side-table state directly; verify this belongs in the planned publish operation and has clear failure handling.", f, f.getName() diff --git a/.github/codeql/stackdepot/StackDepotTrieStateRediscovery.ql b/.github/codeql/stackdepot/StackDepotTrieStateRediscovery.ql new file mode 100644 index 0000000000000..193b9b1db85c8 --- /dev/null +++ b/.github/codeql/stackdepot/StackDepotTrieStateRediscovery.ql @@ -0,0 +1,24 @@ +/** + * @name Trusted stackdepot trie helper rediscovers insertion state + * @description Flags calls that rediscover planner/writer state inside trusted trie insertion helpers. + * @kind problem + * @problem.severity recommendation + * @precision high + * @id stackdepot/trie-state-rediscovery + */ + +import cpp +import StackDepot + +predicate rediscoveryFunctionName(string name) { + name = "trie_child_array_find_slot" or + name = "stack_depot_trie_lookup" +} + +from FunctionCall call, Function f, Function target +where + f = call.getEnclosingFunction() and + isTrustedStructuralHelper(f) and + target = call.getTarget() and + rediscoveryFunctionName(target.getName()) +select call, "Trusted trie insertion helper $@ calls $@ here; verify this state was not already established by planning/writer serialization.", f, f.getName(), target, target.getName() diff --git a/.github/codeql/stackdepot/StackDepotTrustedHelperReturnEinval.ql b/.github/codeql/stackdepot/StackDepotTrustedHelperReturnEinval.ql new file mode 100644 index 0000000000000..d0deaba0e6dfe --- /dev/null +++ b/.github/codeql/stackdepot/StackDepotTrustedHelperReturnEinval.ql @@ -0,0 +1,24 @@ +/** + * @name Trusted stackdepot trie helper returns direct -EINVAL + * @description Flags direct -EINVAL returns in trusted trie structural helpers. + * @kind problem + * @problem.severity recommendation + * @precision high + * @id stackdepot/trusted-helper-return-einval + */ + +import cpp +import StackDepot + +predicate returnsEinval(ReturnStmt ret) { + ret.hasExpr() and + // Linux UAPI errno value for EINVAL. + ret.getExpr().getValue().toInt() = -22 +} + +from ReturnStmt ret, Function f +where + f = ret.getEnclosingFunction() and + isTrustedStructuralHelper(f) and + returnsEinval(ret) +select ret, "Trusted trie structural helper $@ returns direct -EINVAL here; verify this is not rediscovering a writer-side invariant.", f, f.getName() diff --git a/.github/codeql/stackdepot/qlpack.yml b/.github/codeql/stackdepot/qlpack.yml new file mode 100644 index 0000000000000..35a4eb60b0e03 --- /dev/null +++ b/.github/codeql/stackdepot/qlpack.yml @@ -0,0 +1,6 @@ +name: stackdepot/codeql +version: 0.0.1 +dependencies: + codeql/cpp-all: "*" +extractor: cpp +warnOnImplicitThis: true diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000000000..d80e6a9487034 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,328 @@ +name: CodeQL + +on: + workflow_dispatch: + push: + branches: + - caleb/KRN-1117-stackdepot-port-codeql + +permissions: + contents: read + +concurrency: + group: codeql-${{ github.ref }} + cancel-in-progress: true + +env: + BASE_REF: linux-6.18.y + +jobs: + codeql: + name: CodeQL (${{ matrix.name }}) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - name: x86 + make_arch: "" + cross_compile: "" + targets: >- + lib/stackdepot.o + lib/tests/stackdepot_kunit.o + mm/page_owner.o + mm/slub.o + mm/kmemleak.o + drivers/gpu/drm/drm_modeset_lock.o + config: >- + -e DEBUG_KERNEL + -e KUNIT + -e STACKDEPOT + -e STACKDEPOT_ALWAYS_INIT + -e STACKDEPOT_KUNIT_TEST + -e PAGE_OWNER + -e DEBUG_KMEMLEAK + -e SLUB_DEBUG + -e SYSFS + -e DEBUG_FS + -e DRM + -e DRM_KMS_HELPER + - name: arm64 + make_arch: arm64 + cross_compile: aarch64-linux-gnu- + targets: >- + lib/stackdepot.o + lib/tests/stackdepot_kunit.o + config: >- + -e DEBUG_KERNEL + -e KUNIT + -e STACKDEPOT + -e STACKDEPOT_ALWAYS_INIT + -e STACKDEPOT_KUNIT_TEST + -e SLUB_DEBUG + -e SYSFS + -e DEBUG_FS + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential bc bison flex libssl-dev libelf-dev dwarves \ + curl zstd python3 ca-certificates gcc-aarch64-linux-gnu + + - name: Locate CodeQL CLI + run: | + set -euo pipefail + if command -v codeql >/dev/null 2>&1; then + codeql_bin="$(command -v codeql)" + else + codeql_bin="$(find /opt/hostedtoolcache/CodeQL -path '*/codeql/codeql' -type f | sort -V | tail -n 1)" + fi + test -x "$codeql_bin" + echo "CODEQL=$codeql_bin" >> "$GITHUB_ENV" + "$codeql_bin" version + + - name: Generate changed-file list + run: | + set -euo pipefail + root="$RUNNER_TEMP/codeql" + mkdir -p "$root/out" + git fetch --depth=1 origin "$BASE_REF:refs/remotes/origin/$BASE_REF" + git diff --name-only --diff-filter=ACMRTUXB \ + "origin/$BASE_REF" HEAD > "$root/changed-files.txt" + cp "$root/changed-files.txt" "$root/out/changed-files.txt" + + - name: Configure kernel + env: + MAKE_ARCH: ${{ matrix.make_arch }} + CROSS_COMPILE: ${{ matrix.cross_compile }} + CONFIG_FLAGS: ${{ matrix.config }} + run: | + set -euo pipefail + root="$RUNNER_TEMP/codeql" + out="$root/build-${{ matrix.name }}" + rm -rf "$out" + mkdir -p "$out" + + make_args=(O="$out") + if [ -n "$MAKE_ARCH" ]; then + make_args+=(ARCH="$MAKE_ARCH") + fi + if [ -n "$CROSS_COMPILE" ]; then + make_args+=(CROSS_COMPILE="$CROSS_COMPILE") + fi + + read -r -a config_args <<< "$CONFIG_FLAGS" + make "${make_args[@]}" defconfig + scripts/config --file "$out/.config" "${config_args[@]}" + make "${make_args[@]}" olddefconfig prepare scripts + cp "$out/.config" "$root/out/config-${{ matrix.name }}" + + - name: Create CodeQL database + env: + MAKE_ARCH: ${{ matrix.make_arch }} + CROSS_COMPILE: ${{ matrix.cross_compile }} + TARGETS: ${{ matrix.targets }} + run: | + set -euo pipefail + root="$RUNNER_TEMP/codeql" + out="$root/build-${{ matrix.name }}" + db="$root/db-${{ matrix.name }}" + rm -rf "$db" + + make_cmd=(make O="$out") + if [ -n "$MAKE_ARCH" ]; then + make_cmd+=(ARCH="$MAKE_ARCH") + fi + if [ -n "$CROSS_COMPILE" ]; then + make_cmd+=(CROSS_COMPILE="$CROSS_COMPILE") + fi + make_cmd+=(-j"$(nproc)") + targets_manifest="$root/out/targets-${{ matrix.name }}.txt" + : > "$targets_manifest" + for target in $TARGETS; do + make_cmd+=("$target") + printf "%s\n" "$target" >> "$targets_manifest" + done + + "$CODEQL" database create "$db" \ + --language=cpp \ + --source-root="$GITHUB_WORKSPACE" \ + --overwrite \ + --command "${make_cmd[*]}" + + - name: Analyze CodeQL database + run: | + set -euo pipefail + root="$RUNNER_TEMP/codeql" + db="$root/db-${{ matrix.name }}" + sarif="$root/out/codeql-${{ matrix.name }}.sarif" + + "$CODEQL" database analyze "$db" \ + 'codeql/cpp-queries:codeql-suites/cpp-security-and-quality.qls' \ + '.github/codeql/stackdepot' \ + --format=sarif-latest \ + --sarif-category="linux-${{ matrix.name }}" \ + --threads=0 \ + --output="$sarif" + + - name: Filter SARIF to changed-file TSV + run: | + set -euo pipefail + root="$RUNNER_TEMP/codeql" + python3 - "$GITHUB_WORKSPACE" "$root/changed-files.txt" \ + "$root/out/codeql-${{ matrix.name }}.sarif" \ + "$root/out/codeql-${{ matrix.name }}.changed.tsv" \ + "${{ matrix.name }}" <<'PY' + import csv + import json + import os + import sys + import urllib.parse + + src, changed_path, sarif_path, out_path, arch = sys.argv[1:] + src = os.path.normpath(src) + + with open(changed_path, encoding="utf-8") as f: + changed = {line.strip() for line in f if line.strip()} + + def normalize_uri(uri): + if not uri: + return None + parsed = urllib.parse.urlparse(uri) + if parsed.scheme == "file": + path = urllib.parse.unquote(parsed.path) + else: + path = urllib.parse.unquote(uri) + path = path.replace("\\", "/") + if os.path.isabs(path): + try: + rel = os.path.relpath(os.path.normpath(path), src) + except ValueError: + return None + else: + rel = os.path.normpath(path) + rel = rel.replace("\\", "/") + if rel == "." or rel == ".." or rel.startswith("../"): + return None + if rel.startswith("./"): + rel = rel[2:] + return rel + + with open(sarif_path, encoding="utf-8") as f: + sarif = json.load(f) + + def physical_locations(result): + for loc in result.get("locations") or []: + phys = loc.get("physicalLocation") or {} + if phys: + yield "primary", phys + for loc in result.get("relatedLocations") or []: + phys = loc.get("physicalLocation") or {} + if phys: + yield "related", phys + for flow in result.get("codeFlows") or []: + for thread in flow.get("threadFlows") or []: + for loc in thread.get("locations") or []: + phys = (loc.get("location") or {}).get("physicalLocation") or {} + if phys: + yield "codeflow", phys + + def loc_info(phys): + artifact = phys.get("artifactLocation") or {} + rel = normalize_uri(artifact.get("uri")) + region = phys.get("region") or {} + return rel, region.get("startLine", ""), region.get("startColumn", "") + + rows = [] + for run in sarif.get("runs", []): + rules = {} + driver = run.get("tool", {}).get("driver", {}) + for rule in driver.get("rules", []): + rules[rule.get("id")] = rule + + for result in run.get("results", []): + locs = [(kind, *loc_info(phys)) for kind, phys in physical_locations(result)] + primary_locs = [loc for loc in locs if loc[0] == "primary"] + changed_locs = [loc for loc in locs if loc[1] in changed] + if not changed_locs: + continue + primary = primary_locs[0] if primary_locs else ("", "", "", "") + matched = changed_locs[0] + + rule_id = result.get("ruleId", "") + rule = rules.get(rule_id, {}) + level = ( + result.get("level") + or rule.get("defaultConfiguration", {}).get("level") + or "" + ) + msg = result.get("message", {}).get("text", "") + msg = " ".join(msg.split()) + rows.append([ + arch, + rule_id, + level, + primary[1], + primary[2], + primary[3], + matched[1], + matched[2], + matched[3], + matched[0], + msg, + ]) + + with open(out_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, delimiter="\t", lineterminator="\n") + writer.writerow([ + "arch", "rule", "level", + "primary_path", "primary_line", "primary_column", + "matched_path", "matched_line", "matched_column", "matched_kind", + "message", + ]) + writer.writerows(rows) + + custom_out_path = out_path.replace(".changed.tsv", ".stackdepot.changed.tsv") + with open(custom_out_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, delimiter="\t", lineterminator="\n") + writer.writerow([ + "arch", "rule", "level", + "primary_path", "primary_line", "primary_column", + "matched_path", "matched_line", "matched_column", "matched_kind", + "message", + ]) + writer.writerows(row for row in rows if row[1].startswith("stackdepot/")) + PY + + - name: Compress SARIF and clean large scratch data + if: always() + run: | + set -euo pipefail + root="$RUNNER_TEMP/codeql" + sarif="$root/out/codeql-${{ matrix.name }}.sarif" + if [ -f "$sarif" ]; then + gzip -f "$sarif" + fi + rm -rf "$root/build-${{ matrix.name }}" "$root/db-${{ matrix.name }}" + du -sh "$root/out" || true + + - name: Upload CodeQL artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: codeql-${{ matrix.name }} + path: | + ${{ runner.temp }}/codeql/out/changed-files.txt + ${{ runner.temp }}/codeql/out/config-${{ matrix.name }} + ${{ runner.temp }}/codeql/out/targets-${{ matrix.name }}.txt + ${{ runner.temp }}/codeql/out/codeql-${{ matrix.name }}.changed.tsv + ${{ runner.temp }}/codeql/out/codeql-${{ matrix.name }}.stackdepot.changed.tsv + ${{ runner.temp }}/codeql/out/codeql-${{ matrix.name }}.sarif.gz + if-no-files-found: warn diff --git a/arch/arm64/include/asm/stackdepot.h b/arch/arm64/include/asm/stackdepot.h new file mode 100644 index 0000000000000..df8959d593366 --- /dev/null +++ b/arch/arm64/include/asm/stackdepot.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_STACKDEPOT_H +#define __ASM_STACKDEPOT_H + +#include +#include + +/* + * Modules are allocated inside a 2 GB relocation window containing the + * kernel image. Store a signed 32-bit offset from _text so compression is + * independent of 4 GB high-bit boundaries crossed by that window. + */ +static inline unsigned long arch_stack_depot_frame_from_payload(u32 payload) +{ + long offset; + + offset = (s32)payload; + if (offset < 0) + return (unsigned long)_text - (unsigned long)(-offset); + return (unsigned long)_text + (unsigned long)offset; +} + +static inline bool +arch_stack_depot_frame_try_compress(unsigned long frame, u32 *payload) +{ + u32 candidate; + + candidate = (u32)(frame - (unsigned long)_text); + if (arch_stack_depot_frame_from_payload(candidate) != frame) + return false; + + *payload = candidate; + return true; +} + +static inline void +arch_stack_depot_frame_decompress(u32 payload, unsigned long *frame) +{ + *frame = arch_stack_depot_frame_from_payload(payload); +} + +#endif /* __ASM_STACKDEPOT_H */ diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index 9be3ee2e37013..731e6832a3829 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -21,6 +21,7 @@ generic-y += preempt.h generic-y += ring_buffer.h generic-y += runtime-const.h generic-y += softirq_stack.h +generic-y += stackdepot.h generic-y += switch_to.h generic-y += topology.h generic-y += trace_clock.h diff --git a/arch/x86/include/asm/stackdepot.h b/arch/x86/include/asm/stackdepot.h new file mode 100644 index 0000000000000..14229b731fc32 --- /dev/null +++ b/arch/x86/include/asm/stackdepot.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_STACKDEPOT_H +#define _ASM_X86_STACKDEPOT_H + +#include +#include + +#ifdef CONFIG_X86_64 +/* + * Compress canonical kernel text/module addresses whose upper 32 bits are all + * ones. Other kernel virtual addresses stay raw, so decompression reconstructs + * the original frame by restoring this prefix. + */ +#define STACK_DEPOT_X86_64_FRAME_PREFIX 0xffffffff00000000UL +#define STACK_DEPOT_X86_64_FRAME_LOW_MASK 0x00000000ffffffffUL + +static_assert(STACK_DEPOT_X86_64_FRAME_PREFIX != 0); + +static inline bool +arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) +{ + if ((frame & ~STACK_DEPOT_X86_64_FRAME_LOW_MASK) != + STACK_DEPOT_X86_64_FRAME_PREFIX) + return false; + + *low = (u32)frame; + return true; +} + +static inline void +arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) +{ + *frame = STACK_DEPOT_X86_64_FRAME_PREFIX | low; +} + +#else +#include +#endif /* CONFIG_X86_64 */ + +#endif /* _ASM_X86_STACKDEPOT_H */ diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c index beb91a13a3124..e43202df77e30 100644 --- a/drivers/gpu/drm/drm_modeset_lock.c +++ b/drivers/gpu/drm/drm_modeset_lock.c @@ -81,9 +81,12 @@ static DEFINE_WW_CLASS(crtc_ww_class); #if IS_ENABLED(CONFIG_DRM_DEBUG_MODESET_LOCK) +/* Modeset-lock diagnostics only need a short caller chain. */ +#define DRM_STACK_DEPOT_MAX_FRAMES 8 + static noinline depot_stack_handle_t __drm_stack_depot_save(void) { - unsigned long entries[8]; + unsigned long entries[DRM_STACK_DEPOT_MAX_FRAMES]; unsigned int n; n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); @@ -94,16 +97,13 @@ static noinline depot_stack_handle_t __drm_stack_depot_save(void) static void __drm_stack_depot_print(depot_stack_handle_t stack_depot) { struct drm_printer p = drm_dbg_printer(NULL, DRM_UT_KMS, "drm_modeset_lock"); - unsigned long *entries; - unsigned int nr_entries; char *buf; buf = kmalloc(PAGE_SIZE, GFP_NOWAIT | __GFP_NOWARN); if (!buf) return; - nr_entries = stack_depot_fetch(stack_depot, &entries); - stack_trace_snprint(buf, PAGE_SIZE, entries, nr_entries, 2); + stack_depot_snprint(stack_depot, buf, PAGE_SIZE, 2); drm_printf(&p, "attempting to lock a contended lock without backoff:\n%s", buf); diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 295c94a3ccc1c..a126f8e237bab 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -53,6 +53,7 @@ mandatory-y += serial.h mandatory-y += shmparam.h mandatory-y += simd.h mandatory-y += softirq_stack.h +mandatory-y += stackdepot.h mandatory-y += switch_to.h mandatory-y += timex.h mandatory-y += tlbflush.h diff --git a/include/asm-generic/stackdepot.h b/include/asm-generic/stackdepot.h new file mode 100644 index 0000000000000..846975767bdd4 --- /dev/null +++ b/include/asm-generic/stackdepot.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_GENERIC_STACKDEPOT_H +#define __ASM_GENERIC_STACKDEPOT_H + +#include + +static inline bool +arch_stack_depot_frame_try_compress(unsigned long frame, u32 *low) +{ + return false; +} + +static inline void +arch_stack_depot_frame_decompress(u32 low, unsigned long *frame) +{ + /* Generic code never compresses frames, so this hook is unreachable. */ +} + +#endif /* __ASM_GENERIC_STACKDEPOT_H */ diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index 2cc21ffcdaf9e..61e19cfcfae66 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -53,7 +53,8 @@ union handle_parts { struct stack_record { struct list_head hash_list; /* Links in the hash table */ u32 hash; /* Hash in hash table */ - u32 size; /* Number of stored frames */ + u16 size; /* Number of stored frames */ + u16 flags; union handle_parts handle; /* Constant after initialization */ refcount_t count; union { @@ -84,8 +85,9 @@ typedef u32 depot_flags_t; */ #define STACK_DEPOT_FLAG_CAN_ALLOC ((depot_flags_t)0x0001) #define STACK_DEPOT_FLAG_GET ((depot_flags_t)0x0002) +#define STACK_DEPOT_FLAG_COUNTABLE ((depot_flags_t)0x0004) -#define STACK_DEPOT_FLAGS_NUM 2 +#define STACK_DEPOT_FLAGS_NUM 3 #define STACK_DEPOT_FLAGS_MASK ((depot_flags_t)((1 << STACK_DEPOT_FLAGS_NUM) - 1)) /* @@ -144,6 +146,16 @@ static inline int stack_depot_early_init(void) { return 0; } * Users of this flag must also call stack_depot_put() when keeping the stack * trace is no longer required to avoid overflowing the refcount. * + * If STACK_DEPOT_FLAG_COUNTABLE is set in @depot_flags, stack depot stores the + * stack in hash-backed storage for callers that need direct stack_record count + * access. This flag does not imply %STACK_DEPOT_FLAG_CAN_ALLOC and is mutually + * exclusive with %STACK_DEPOT_FLAG_GET. + * + * When trie storage is enabled, persistent non-refcounted saves use trie + * storage. Constrained contexts remain best effort and can return 0 if a + * required trylock or reserved resource is unavailable; trie failures do not + * fall back to hash storage. + * * If the provided stack trace comes from the interrupt context, only the part * up to the interrupt entry is saved. * @@ -169,6 +181,10 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, * Does not increment the refcount on the saved stack trace; see * stack_depot_save_flags() for more details. * + * When trie storage is enabled, this can return trie-backed handles. Use + * stack_depot_fetch_into(), stack_depot_print(), or stack_depot_snprint() for + * backend-independent access to the stack contents. + * * Context: Contexts where allocations via alloc_pages() are allowed; * see stack_depot_save_flags() for more details. * @@ -178,27 +194,66 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags); /** - * __stack_depot_get_stack_record - Get a pointer to a stack_record struct + * __stack_depot_get_stack_record - Get a hash-backed stack record * * @handle: Stack depot handle * - * This function is only for internal purposes. + * This function is only for internal purposes. @handle must have been saved + * with %STACK_DEPOT_FLAG_COUNTABLE. * - * Return: Returns a pointer to a stack_record struct + * Return: Returns a pointer to a stack_record struct. */ struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle); /** * stack_depot_fetch - Fetch a stack trace from stack depot * - * @handle: Stack depot handle returned from stack_depot_save() + * @handle: Hash-backed stack depot handle * @entries: Pointer to store the address of the stack trace * + * This helper returns a pointer to stackdepot-owned contiguous storage for + * legacy hash-backed handles. Callers that need backend-independent access to + * stack contents should use stack_depot_fetch_into(), stack_depot_print(), or + * stack_depot_snprint(). Passing a trie-backed handle is invalid and may WARN. + * * Return: Number of frames for the fetched stack */ unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries); +/** + * stack_depot_fetch_into - Fetch a stack trace into caller-owned storage + * + * @handle: Stack depot handle returned from stack_depot_save() + * @entries: Caller-owned buffer to copy the stack trace into + * @max_entries: Number of frames that fit in @entries + * + * Copies the stored frames into caller-owned @entries. If fewer frames are + * stored than @max_entries, only the stored frames are written and their count + * is returned. If more frames are stored than @max_entries, the copy is skipped + * entirely and 0 is returned. + * + * Callers should size @entries to match the save-side stack depth cap (for + * example, %CONFIG_STACKDEPOT_MAX_FRAMES or the local stack_trace_save() limit) + * when losing diagnostics on an undersized buffer would be surprising. + * + * A non-zero invalid or post-put @handle is treated like stack_depot_fetch(): it + * returns 0 and may WARN because such handles indicate a corrupt caller state. + * + * Callers must ensure @handle remains valid for the duration of this call. + * Persistent handles saved without %STACK_DEPOT_FLAG_GET require no extra + * reference; handles saved with %STACK_DEPOT_FLAG_GET require a held reference. + * Callers must not call stack_depot_put() on persistent handles. + * Racing this helper with stack_depot_put() on the same handle is invalid. + * + * Return: Number of frames copied, 0 if @entries is NULL, @max_entries is 0, + * @handle is 0 or invalid, stack depot is disabled, or @max_entries is less + * than the number of stored frames. + */ +unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries); + /** * stack_depot_print - Print a stack trace from stack depot * @@ -224,10 +279,14 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, * * @handle: Stack depot handle returned from stack_depot_save() * - * The stack trace is evicted from stack depot once all references to it have - * been dropped (once the number of stack_depot_evict() calls matches the - * number of stack_depot_save_flags() calls with STACK_DEPOT_FLAG_GET set for - * this stack trace). + * Drop a reference acquired by stack_depot_save_flags() with + * %STACK_DEPOT_FLAG_GET. Calling this for a handle saved without + * %STACK_DEPOT_FLAG_GET is invalid; persistent handles, including trie-backed + * handles, are owned by stack depot for the lifetime of the system. + * + * The stack trace is evicted once the number of stack_depot_put() calls matches + * the number of successful stack_depot_save_flags() calls with + * %STACK_DEPOT_FLAG_GET for this stack trace. */ void stack_depot_put(depot_stack_handle_t handle); diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 21cd68084e468..3a1e7c9e6c1bd 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2706,6 +2706,22 @@ config RESOURCE_KUNIT_TEST If unsure, say N. +config STACKDEPOT_KUNIT_TEST + bool "KUnit test for stack depot" if !KUNIT_ALL_TESTS + depends on KUNIT=y && STACKDEPOT + default KUNIT_ALL_TESTS + help + Enable this option to test stack depot API behavior at boot. + This test is built in because it exercises internal, non-exported + stack depot helpers, so KUNIT must also be built in. + + KUnit tests run during boot and output the results to the debug log + in TAP format (https://testanything.org/). Only useful for kernel + developers running the KUnit test harness, and not intended for + inclusion into a production build. + + If unsure, say N. + config SYSCTL_KUNIT_TEST tristate "KUnit test for sysctl" if !KUNIT_ALL_TESTS depends on KUNIT diff --git a/lib/stackdepot.c b/lib/stackdepot.c index de0b0025af2b9..46e049286e431 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -2,9 +2,11 @@ /* * Stack depot - a stack trace storage that avoids duplication. * - * Internally, stack depot maintains a hash table of unique stacktraces. The - * stack traces themselves are stored contiguously one after another in a set - * of separate page allocations. + * Internally, stack depot has two storage backends. Refcounted entries and + * callers that request STACK_DEPOT_FLAG_COUNTABLE use the legacy hash table with + * contiguous stack records in stack pools. Persistent non-refcounted entries + * can use trie storage when enabled; trie nodes share common frame prefixes and + * are published through RCU/COW child arrays. * * Author: Alexander Potapenko * Copyright (C) 2016 Google, Inc. @@ -14,14 +16,21 @@ #define pr_fmt(fmt) "stackdepot: " fmt +#include +#include #include +#include #include #include +#include #include +#include #include #include #include +#include #include +#include #include #include #include @@ -36,9 +45,12 @@ #include #include +#include + /* * The pool_index is offset by 1 so the first record does not have a 0 handle. */ +/* Parsed before mm_core_init(); trie handle decoding assumes this is then fixed. */ static unsigned int stack_max_pools __read_mostly = MIN((1LL << DEPOT_POOL_INDEX_BITS) - 1, 8192); @@ -95,6 +107,1230 @@ static const char *const counter_names[] = { }; static_assert(ARRAY_SIZE(counter_names) == DEPOT_COUNTER_COUNT); +enum stack_depot_frame_mode { + STACK_DEPOT_FRAME_RAW, + STACK_DEPOT_FRAME_COMPRESSED, +}; + +/* + * A trie node stores one run of frames that all use the same payload format. + * Architectures may compress some frames to 32-bit payloads; mixed raw and + * compressed input is split across multiple trie nodes so each node has one + * decoding mode. + */ +struct stack_depot_frame_run { + u16 nr_entries; + u8 mode; +}; + +static_assert(CONFIG_STACKDEPOT_MAX_FRAMES <= U16_MAX); + +struct stack_depot_trie_child_array; + +struct stack_depot_trie_node { + /* Parent links let fetch rebuild a full stack from a leaf to the root. */ + const struct stack_depot_trie_node *parent; + /* Child arrays are separate RCU/COW generations. */ + const struct stack_depot_trie_child_array *children; + u32 leaf_id; + struct stack_depot_frame_run run; + unsigned char data[]; +}; + +/* + * Children are sorted by first frame and searched by insertion slot. + * Writers may append to spare capacity at the sorted tail, but never change + * existing child pointers. Other updates build and publish a replacement array. + */ +struct stack_depot_trie_child_array { + unsigned int nr_children; + unsigned int capacity; + const struct stack_depot_trie_node *children[]; +}; + +/* Headerless reusable storage for trie nodes. */ +struct stack_depot_trie_free_node { + struct list_head list; +}; + +/* + * Reusable object storage for child arrays and other payloads that need an + * object header. A retired child array can carry the old child node that was + * replaced with it; both become reusable after the array's RCU grace period. + */ +struct stack_depot_trie_free_object { + struct list_head list; + unsigned long rcu_state; + size_t size; + struct stack_depot_trie_node *pending_node; +}; + +static_assert(sizeof(struct stack_depot_trie_node) >= + sizeof(struct stack_depot_trie_free_node)); + +#define STACK_DEPOT_TRIE_MAX_NODES (CONFIG_STACKDEPOT_MAX_FRAMES + 1) +#define STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS CONFIG_STACKDEPOT_MAX_FRAMES + +struct stack_depot_trie_alloc_workspace { + struct stack_depot_trie_node *nodes[STACK_DEPOT_TRIE_MAX_NODES]; + size_t node_sizes[STACK_DEPOT_TRIE_MAX_NODES]; + struct stack_depot_trie_child_array *child_arrays[STACK_DEPOT_TRIE_MAX_CHILD_ARRAYS]; + u32 scratch[CONFIG_STACKDEPOT_MAX_FRAMES]; + struct stack_depot_trie_child_array *split_child_array; + struct stack_depot_trie_child_array *child_array_storage; +}; + +static DEFINE_STATIC_KEY_FALSE(stack_depot_trie_enabled); +static const struct stack_depot_trie_child_array *stack_depot_trie_root; +static struct stack_depot_trie_alloc_workspace __rcu *stack_depot_trie_workspace; +static DEFINE_RAW_SPINLOCK(stack_depot_trie_writer_lock); +static bool stack_depot_trie_requested; + +module_param_named(trie_enabled, stack_depot_trie_requested, bool, 0); +MODULE_PARM_DESC(trie_enabled, "Enable stack depot trie storage at boot"); + +#define DEPOT_POOL_INDEX_MASK ((1U << DEPOT_POOL_INDEX_BITS) - 1) +#define DEPOT_OFFSET_MASK ((1U << DEPOT_OFFSET_BITS) - 1) + +/* Size classes bucket reusable trie storage by aligned allocation size. */ +#define STACK_DEPOT_TRIE_FREE_CLASSES \ + ((DEPOT_POOL_SIZE >> DEPOT_STACK_ALIGN) + 1) + +/* + * Trie storage is suballocated from stackdepot pools, not slab caches, so pool + * pressure stays visible through stack_depot_max_pools and no-spin callers can + * fail without allocator recursion. Trie COW insertion retires child arrays + * and sometimes the node they replaced. Objects carry the RCU cookie for + * child-array payloads; headerless node fragments either live directly on + * free_trie_nodes or are attached to a pending object until that object's grace + * period has elapsed. + */ +static struct list_head free_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; +static struct list_head pending_trie_objects[STACK_DEPOT_TRIE_FREE_CLASSES]; +static struct list_head free_trie_nodes[STACK_DEPOT_TRIE_FREE_CLASSES]; +static DECLARE_BITMAP(free_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); +static DECLARE_BITMAP(pending_trie_object_map, STACK_DEPOT_TRIE_FREE_CLASSES); +static DECLARE_BITMAP(free_trie_node_map, STACK_DEPOT_TRIE_FREE_CLASSES); + +static u32 trie_side_table_max_id; + +/* + * Hash handles use pool_index_plus_1 <= stack_max_pools. Trie handles use + * pool_index_plus_1 > stack_max_pools and reinterpret the remaining handle + * bits as a dense leaf_id, which the side table maps to a trie leaf. + */ +static u32 __stack_depot_trie_max_leaf_id(void) +{ + u64 max_id; + + if (stack_max_pools >= DEPOT_POOL_INDEX_MASK - 1) + return 0; + + max_id = (u64)(DEPOT_POOL_INDEX_MASK - stack_max_pools - 1) << + DEPOT_OFFSET_BITS; + return min_t(u64, max_id, U32_MAX); +} + +static depot_stack_handle_t __stack_depot_trie_handle(u32 leaf_id) +{ + union handle_parts parts = {}; + u64 pool_index_plus_1; + u32 pool_delta; + u32 index; + + if (!leaf_id || leaf_id > trie_side_table_max_id) + return 0; + + index = leaf_id - 1; + pool_delta = index >> DEPOT_OFFSET_BITS; + pool_index_plus_1 = (u64)stack_max_pools + 1 + pool_delta; + if (pool_index_plus_1 >= DEPOT_POOL_INDEX_MASK) + return 0; + + parts.pool_index_plus_1 = pool_index_plus_1; + parts.offset = index & DEPOT_OFFSET_MASK; + return parts.handle; +} + +static u32 __stack_depot_trie_leaf_id(depot_stack_handle_t handle) +{ + union handle_parts parts = { .handle = handle }; + u64 leaf_id; + u32 pool_delta; + + parts.extra = 0; + if (parts.pool_index_plus_1 <= stack_max_pools) + return 0; + + pool_delta = parts.pool_index_plus_1 - stack_max_pools - 1; + if ((u64)pool_delta + stack_max_pools + 1 >= DEPOT_POOL_INDEX_MASK) + return 0; + + leaf_id = ((u64)pool_delta << DEPOT_OFFSET_BITS) + parts.offset + 1; + if (leaf_id > trie_side_table_max_id) + return 0; + + return leaf_id; +} + +/* + * Trie handles encode a dense leaf ID. The side table maps that ID to a leaf + * pointer for lockless fetch/print paths, which can run from diagnostic + * contexts where taking trie_side_table_lock would be unsafe. Init installs the + * root and first chunk only; additional directories/chunks are preallocated and + * published lazily as leaf IDs grow. RCU pointer publication makes fully + * initialized dirs, chunks, and leaves visible to those lockless readers. + */ +#define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS 9 +#define STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE \ + (1U << STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS) +#define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS 9 +#define STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE \ + (1U << STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS) + +struct stack_depot_trie_side_dir { + const struct stack_depot_trie_node __rcu * __rcu * + chunks[STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE]; +}; + +struct stack_depot_trie_side_root { + unsigned int dir_capacity; + struct stack_depot_trie_side_dir __rcu *dirs[]; +}; + +struct stack_depot_trie_side_prealloc { + /* Preallocated side-table directory page for sparse growth. */ + struct stack_depot_trie_side_dir *dir; + /* Preallocated side-table leaf chunk for sparse growth. */ + const struct stack_depot_trie_node __rcu **chunk; +}; + +static struct stack_depot_trie_side_root __rcu *trie_side_table_root; +static DEFINE_RAW_SPINLOCK(trie_side_table_lock); +static u32 trie_side_table_last_leaf_id; + +/* Lock order: writer_lock -> pool_lock -> trie_side_table_lock. */ + +static bool __stack_depot_trie_enabled(void) +{ + return static_branch_unlikely(&stack_depot_trie_enabled); +} + +static void stack_depot_trie_enable(void) +{ + if (__stack_depot_trie_enabled()) + return; + + static_branch_enable(&stack_depot_trie_enabled); +} + +static inline size_t stack_depot_frame_run_entry_bytes(enum stack_depot_frame_mode mode) +{ + if (mode == STACK_DEPOT_FRAME_COMPRESSED) + return sizeof(u32); + return sizeof(unsigned long); +} + +static inline size_t stack_depot_frame_run_bytes(const struct stack_depot_frame_run *run) +{ + return run->nr_entries * stack_depot_frame_run_entry_bytes(run->mode); +} + +static size_t __stack_depot_trie_node_size(const struct stack_depot_frame_run *run) +{ + return ALIGN(offsetof(struct stack_depot_trie_node, data) + + stack_depot_frame_run_bytes(run), sizeof(unsigned long)); +} + +static unsigned int trie_child_array_capacity(unsigned int nr_children) +{ + return nr_children ? roundup_pow_of_two(nr_children) : 0; +} + +static size_t trie_child_array_size_for_capacity(unsigned int capacity) +{ + size_t size; + + size = struct_size_t(struct stack_depot_trie_child_array, children, + capacity); + if (size == SIZE_MAX) + return 0; + + return ALIGN(size, sizeof(unsigned long)); +} + +static inline size_t __stack_depot_trie_child_array_size(unsigned int nr_children) +{ + return trie_child_array_size_for_capacity(trie_child_array_capacity(nr_children)); +} + +static void trie_child_array_init(void *storage, unsigned int capacity, + const struct stack_depot_trie_node * const *nodes, + unsigned int nr_children) +{ + struct stack_depot_trie_child_array *array = storage; + unsigned int i; + + array->nr_children = nr_children; + array->capacity = capacity; + for (i = 0; i < nr_children; i++) + array->children[i] = nodes[i]; + for (i = nr_children; i < capacity; i++) + array->children[i] = NULL; +} + +static struct stack_depot_trie_alloc_workspace *stack_depot_trie_load_workspace(void) +{ + /* Installed once and never freed; acquire the init-time publication. */ + return rcu_dereference_check(stack_depot_trie_workspace, true); +} + +static struct stack_depot_trie_side_root *trie_side_table_load_root(void) +{ + /* Installed once and never freed; acquire the init-time publication. */ + return rcu_dereference_check(trie_side_table_root, true); +} + +static bool trie_side_table_is_initialized(void) +{ + return !!trie_side_table_load_root(); +} + +static bool __stack_depot_trie_ready(void) +{ + return __stack_depot_trie_enabled() && + stack_depot_trie_load_workspace() && + trie_side_table_is_initialized(); +} + +static inline unsigned int trie_side_table_top_index(u32 id) +{ + return (id - 1) >> STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_BITS; +} + +static inline unsigned int trie_side_table_root_index(u32 id) +{ + return trie_side_table_top_index(id) >> STACK_DEPOT_TRIE_SIDE_TABLE_DIR_BITS; +} + +static inline unsigned int trie_side_table_dir_index(u32 id) +{ + return trie_side_table_top_index(id) & + (STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE - 1); +} + +static inline unsigned int trie_side_table_slot_index(u32 id) +{ + return (id - 1) & (STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE - 1); +} + +static struct stack_depot_trie_side_dir *trie_side_table_load_dir(unsigned int root) +{ + struct stack_depot_trie_side_root *root_vec; + + root_vec = trie_side_table_load_root(); + if (!root_vec || root >= root_vec->dir_capacity) + return NULL; + /* Pairs with side-table directory rcu_assign_pointer(). */ + return rcu_dereference_check(root_vec->dirs[root], + lockdep_is_held(&trie_side_table_lock) || + rcu_read_lock_sched_held()); +} + +static const struct stack_depot_trie_node __rcu ** +trie_side_table_dir_load_chunk(struct stack_depot_trie_side_dir *dir, + unsigned int idx) +{ + /* Pairs with the chunk rcu_assign_pointer() in leaf ID preparation. */ + return rcu_dereference_check(dir->chunks[idx], + lockdep_is_held(&trie_side_table_lock) || + rcu_read_lock_sched_held()); +} + +static u32 trie_side_table_next_leaf_id(void) +{ + unsigned long flags; + u32 id = 0; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + id = trie_side_table_last_leaf_id + 1; + /* ID zero wraps the 32-bit counter; max_id is trie handle capacity. */ + if (!id || id > trie_side_table_max_id) + id = 0; + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return id; +} + +static bool +trie_side_table_ensure_leaf_slot(u32 id, + struct stack_depot_trie_side_prealloc *prealloc) +{ + const struct stack_depot_trie_node __rcu **chunk; + struct stack_depot_trie_side_dir *dir; + struct stack_depot_trie_side_root *root_vec; + unsigned long flags; + unsigned int root; + unsigned int idx; + bool ret = false; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + root_vec = trie_side_table_load_root(); + root = trie_side_table_root_index(id); + dir = trie_side_table_load_dir(root); + if (!dir) { + /* Sparse growth preallocation can lose a race to another writer. */ + if (!prealloc->dir) + goto out; + dir = prealloc->dir; + prealloc->dir = NULL; + /* Publish the zeroed directory before readers can load it locklessly. */ + rcu_assign_pointer(root_vec->dirs[root], dir); + } + + idx = trie_side_table_dir_index(id); + chunk = trie_side_table_dir_load_chunk(dir, idx); + if (!chunk) { + /* Sparse growth preallocation can lose a race to another writer. */ + if (!prealloc->chunk) + goto out; + chunk = prealloc->chunk; + prealloc->chunk = NULL; + rcu_assign_pointer(dir->chunks[idx], chunk); + } + + ret = true; +out: + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return ret; +} + +static size_t trie_side_table_root_bytes(unsigned int root_size) +{ + size_t bytes; + + bytes = struct_size_t(struct stack_depot_trie_side_root, dirs, root_size); + if (bytes == SIZE_MAX) + return 0; + return PAGE_ALIGN(bytes); +} + +static inline size_t trie_side_table_dir_bytes(void) +{ + return PAGE_ALIGN(sizeof(struct stack_depot_trie_side_dir)); +} + +static inline unsigned int trie_side_table_dir_order(void) +{ + return get_order(trie_side_table_dir_bytes()); +} + +static inline size_t trie_side_table_chunk_bytes(void) +{ + return PAGE_ALIGN(STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE * + sizeof(struct stack_depot_trie_node *)); +} + +static inline unsigned int trie_side_table_chunk_order(void) +{ + return get_order(trie_side_table_chunk_bytes()); +} + +static void trie_side_table_free_dir(struct stack_depot_trie_side_dir *dir) +{ + if (dir) + free_pages((unsigned long)dir, trie_side_table_dir_order()); +} + +static void trie_side_table_root_init(struct stack_depot_trie_side_root *root_vec, + unsigned int root_size, u32 max_id) +{ + root_vec->dir_capacity = root_size; + trie_side_table_max_id = max_id; + trie_side_table_last_leaf_id = 0; +} + +static unsigned int trie_side_table_root_size_for_max_id(u32 max_leaf_id) +{ + unsigned int top_size; + + top_size = DIV_ROUND_UP(max_leaf_id, STACK_DEPOT_TRIE_SIDE_TABLE_CHUNK_SIZE); + return DIV_ROUND_UP(top_size, STACK_DEPOT_TRIE_SIDE_TABLE_DIR_SIZE); +} + +static int __init __stack_depot_trie_side_table_init_memblock(void) +{ + struct stack_depot_trie_side_root *root_vec; + struct stack_depot_trie_side_dir *first_dir; + const struct stack_depot_trie_node __rcu **first_chunk; + size_t dir_bytes; + size_t chunk_bytes; + size_t root_bytes; + u32 max_leaf_id; + unsigned int root_size; + + if (trie_side_table_is_initialized()) + return 0; + + max_leaf_id = __stack_depot_trie_max_leaf_id(); + if (!max_leaf_id) + return -EINVAL; + root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_bytes = trie_side_table_root_bytes(root_size); + dir_bytes = trie_side_table_dir_bytes(); + chunk_bytes = trie_side_table_chunk_bytes(); + if (!root_bytes || !dir_bytes || !chunk_bytes) + return -ENOMEM; + + root_vec = memblock_alloc(root_bytes, PAGE_SIZE); + if (!root_vec) + return -ENOMEM; + memset(root_vec, 0, root_bytes); + first_dir = memblock_alloc(dir_bytes, PAGE_SIZE); + if (!first_dir) { + memblock_free(root_vec, root_bytes); + return -ENOMEM; + } + memset(first_dir, 0, dir_bytes); + first_chunk = memblock_alloc(chunk_bytes, PAGE_SIZE); + if (!first_chunk) { + memblock_free(first_dir, dir_bytes); + memblock_free(root_vec, root_bytes); + return -ENOMEM; + } + memset(first_chunk, 0, chunk_bytes); + + trie_side_table_root_init(root_vec, root_size, max_leaf_id); + RCU_INIT_POINTER(root_vec->dirs[0], first_dir); + RCU_INIT_POINTER(first_dir->chunks[0], first_chunk); + rcu_assign_pointer(trie_side_table_root, root_vec); + return 0; +} + +static int __stack_depot_trie_side_table_init(gfp_t gfp_flags) +{ + struct stack_depot_trie_side_root *root_vec; + unsigned int root_size; + size_t root_bytes; + u32 max_leaf_id; + + if (trie_side_table_is_initialized()) + return 0; + + max_leaf_id = __stack_depot_trie_max_leaf_id(); + if (!max_leaf_id) + return -EINVAL; + + root_size = trie_side_table_root_size_for_max_id(max_leaf_id); + root_bytes = trie_side_table_root_bytes(root_size); + if (!root_bytes) + return -ENOMEM; + root_vec = kvzalloc(root_bytes, gfp_flags); + if (!root_vec) + return -ENOMEM; + + trie_side_table_root_init(root_vec, root_size, max_leaf_id); + rcu_assign_pointer(trie_side_table_root, root_vec); + return 0; +} + +static void trie_free_object_buckets_init(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(free_trie_objects); i++) { + INIT_LIST_HEAD(&free_trie_objects[i]); + INIT_LIST_HEAD(&pending_trie_objects[i]); + INIT_LIST_HEAD(&free_trie_nodes[i]); + } +} + +static int __init stack_depot_trie_init_memblock(void) +{ + struct stack_depot_trie_alloc_workspace *workspace = NULL; + size_t size; + int ret; + + if (__stack_depot_trie_ready()) + return 0; + + if (!stack_depot_trie_load_workspace()) { + size = sizeof(*stack_depot_trie_workspace); + workspace = memblock_alloc(size, __alignof__(*workspace)); + if (!workspace) + return -ENOMEM; + memset(workspace, 0, size); + } + ret = __stack_depot_trie_side_table_init_memblock(); + if (ret) { + if (workspace) + memblock_free(workspace, size); + return ret; + } + if (workspace) + rcu_assign_pointer(stack_depot_trie_workspace, workspace); + + trie_free_object_buckets_init(); + stack_depot_trie_enable(); + return 0; +} + +static int stack_depot_trie_init(gfp_t gfp_flags) +{ + struct stack_depot_trie_alloc_workspace *workspace = NULL; + int ret; + + if (__stack_depot_trie_ready()) + return 0; + + if (!stack_depot_trie_load_workspace()) { + workspace = kvzalloc(sizeof(*stack_depot_trie_workspace), gfp_flags); + if (!workspace) + return -ENOMEM; + } + ret = __stack_depot_trie_side_table_init(gfp_flags); + if (ret) { + kvfree(workspace); + return ret; + } + if (workspace) + rcu_assign_pointer(stack_depot_trie_workspace, workspace); + + trie_free_object_buckets_init(); + stack_depot_trie_enable(); + return 0; +} + +static const struct stack_depot_trie_node * +trie_side_table_load_leaf(const struct stack_depot_trie_node __rcu **slot) +{ + /* Pairs with side-table leaf rcu_assign_pointer(). */ + return rcu_dereference_check(*slot, + lockdep_is_held(&trie_side_table_lock) || + rcu_read_lock_sched_held()); +} + +static void *trie_side_table_alloc_page(gfp_t gfp_flags, unsigned int order) +{ + struct page *page; + + page = alloc_pages(gfp_nested_mask(gfp_flags) | __GFP_ZERO, + order); + return page ? page_address(page) : NULL; +} + +static int +__stack_depot_trie_side_table_prealloc(gfp_t gfp_flags, + struct stack_depot_trie_side_prealloc *prealloc) +{ + struct stack_depot_trie_side_dir *dir; + unsigned long flags; + bool need_chunk; + bool need_dir; + u32 id; + unsigned int root; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + id = trie_side_table_last_leaf_id + 1; + if (!id || id > trie_side_table_max_id) { + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + return 0; + } + root = trie_side_table_root_index(id); + dir = trie_side_table_load_dir(root); + need_dir = !dir; + need_chunk = need_dir || !trie_side_table_dir_load_chunk(dir, + trie_side_table_dir_index(id)); + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + + if (need_dir) { + unsigned int order = trie_side_table_dir_order(); + + prealloc->dir = trie_side_table_alloc_page(gfp_flags, order); + if (!prealloc->dir) + return -ENOMEM; + } + if (need_chunk) { + unsigned int order = trie_side_table_chunk_order(); + + prealloc->chunk = trie_side_table_alloc_page(gfp_flags, order); + if (!prealloc->chunk) { + trie_side_table_free_dir(prealloc->dir); + prealloc->dir = NULL; + return -ENOMEM; + } + } + + return 0; +} + +static void +__stack_depot_trie_side_table_free_prealloc(struct stack_depot_trie_side_prealloc *prealloc) +{ + trie_side_table_free_dir(prealloc->dir); + if (prealloc->chunk) + free_pages((unsigned long)prealloc->chunk, + trie_side_table_chunk_order()); + prealloc->dir = NULL; + prealloc->chunk = NULL; +} + +static const struct stack_depot_trie_node __rcu **trie_side_table_leaf_slot(u32 id) +{ + const struct stack_depot_trie_node __rcu **chunk; + struct stack_depot_trie_side_dir *dir; + unsigned int root; + + root = trie_side_table_root_index(id); + dir = trie_side_table_load_dir(root); + chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); + + return &chunk[trie_side_table_slot_index(id)]; +} + +static const struct stack_depot_trie_node *__stack_depot_trie_side_table_lookup(u32 id) +{ + const struct stack_depot_trie_node __rcu **chunk; + struct stack_depot_trie_side_dir *dir; + unsigned int root; + + if (!id) + return NULL; + if (!trie_side_table_is_initialized()) + return NULL; + + root = trie_side_table_root_index(id); + dir = trie_side_table_load_dir(root); + if (!dir) + return NULL; + chunk = trie_side_table_dir_load_chunk(dir, trie_side_table_dir_index(id)); + if (!chunk) + return NULL; + + return trie_side_table_load_leaf(&chunk[trie_side_table_slot_index(id)]); +} + +static size_t __stack_depot_trie_pool_alloc_size(size_t size) +{ + size_t align = 1UL << DEPOT_STACK_ALIGN; + size_t aligned; + + if (!size || size > DEPOT_POOL_SIZE) + return 0; + if (check_add_overflow(size, align - 1, &aligned)) + return 0; + aligned = ALIGN(size, align); + return aligned <= DEPOT_POOL_SIZE ? aligned : 0; +} + +static inline size_t trie_object_header_size(void) +{ + return ALIGN(sizeof(struct stack_depot_trie_free_object), + 1UL << DEPOT_STACK_ALIGN); +} + +static size_t trie_object_alloc_size(size_t size) +{ + size_t alloc_size; + + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return 0; + if (check_add_overflow(trie_object_header_size(), size, + &alloc_size)) + return 0; + return alloc_size <= DEPOT_POOL_SIZE ? alloc_size : 0; +} + +static inline struct stack_depot_trie_free_object *trie_object_header(const void *ptr) +{ + return (struct stack_depot_trie_free_object *)((const char *)ptr - + trie_object_header_size()); +} + +static inline void *trie_object_payload(struct stack_depot_trie_free_object *free) +{ + return (char *)free + trie_object_header_size(); +} + +static unsigned int trie_free_class(size_t size) +{ + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return 0; + + return size >> DEPOT_STACK_ALIGN; +} + +static void trie_free_list_add(struct list_head *entry, struct list_head *heads, + unsigned long *map, unsigned int class, bool tail) +{ + lockdep_assert_held(&pool_lock); + + if (tail) + list_add_tail(entry, &heads[class]); + else + list_add(entry, &heads[class]); + __set_bit(class, map); +} + +static void trie_free_list_del(struct list_head *entry, struct list_head *heads, + unsigned long *map, unsigned int class) +{ + lockdep_assert_held(&pool_lock); + + list_del_init(entry); + if (list_empty(&heads[class])) + __clear_bit(class, map); +} + +static bool depot_init_pool(void **prealloc); + +static unsigned int trie_find_next_set_class(unsigned long *map, unsigned int class) +{ + for (; class < STACK_DEPOT_TRIE_FREE_CLASSES; class++) { + if (map[class / BITS_PER_LONG] & BIT(class % BITS_PER_LONG)) + return class; + } + + return STACK_DEPOT_TRIE_FREE_CLASSES; +} + +static void *trie_object_init_fresh(void *ptr, size_t size) +{ + struct stack_depot_trie_free_object *free = ptr; + + free->size = __stack_depot_trie_pool_alloc_size(size); + free->rcu_state = 0; + free->pending_node = NULL; + INIT_LIST_HEAD(&free->list); + return trie_object_payload(free); +} + +static void trie_free_object_locked(const void *ptr, unsigned long rcu_state) +{ + struct stack_depot_trie_free_object *free; + unsigned int class; + + lockdep_assert_held(&pool_lock); + + free = trie_object_header(ptr); + free->rcu_state = rcu_state; + class = trie_free_class(free->size); + INIT_LIST_HEAD(&free->list); + if (poll_state_synchronize_rcu(rcu_state)) + trie_free_list_add(&free->list, free_trie_objects, + free_trie_object_map, class, false); + else + trie_free_list_add(&free->list, pending_trie_objects, + pending_trie_object_map, class, true); +} + +static void trie_add_free_node_locked(void *ptr, size_t size) +{ + struct stack_depot_trie_free_node *free = ptr; + unsigned int class; + + lockdep_assert_held(&pool_lock); + + size = __stack_depot_trie_pool_alloc_size(size); + if (size < sizeof(*free)) + return; + + INIT_LIST_HEAD(&free->list); + class = trie_free_class(size); + trie_free_list_add(&free->list, free_trie_nodes, free_trie_node_map, + class, false); +} + +static void trie_drain_free_object_node_locked(struct stack_depot_trie_free_object *free) +{ + size_t size; + + lockdep_assert_held(&pool_lock); + + if (!free->pending_node) + return; + size = __stack_depot_trie_node_size(&free->pending_node->run); + trie_add_free_node_locked(free->pending_node, size); + free->pending_node = NULL; +} + +static void trie_drain_pending_objects_locked(void) +{ + struct stack_depot_trie_free_object *free; + struct stack_depot_trie_free_object *tmp; + unsigned int class; + + lockdep_assert_held(&pool_lock); + + for (class = trie_find_next_set_class(pending_trie_object_map, 0); + class < STACK_DEPOT_TRIE_FREE_CLASSES; + class = trie_find_next_set_class(pending_trie_object_map, class + 1)) { + list_for_each_entry_safe(free, tmp, &pending_trie_objects[class], list) { + /* Pending lists are FIFO; later entries cannot be ready yet. */ + if (!poll_state_synchronize_rcu(free->rcu_state)) + break; + trie_drain_free_object_node_locked(free); + trie_free_list_del(&free->list, pending_trie_objects, + pending_trie_object_map, class); + free->rcu_state = 0; + trie_free_list_add(&free->list, free_trie_objects, + free_trie_object_map, class, false); + } + } +} + +static void trie_free_object_tail_locked(void *ptr, size_t size) +{ + struct stack_depot_trie_free_object *free = ptr; + size_t header_size = trie_object_header_size(); + size_t tail_size; + + lockdep_assert_held(&pool_lock); + + if (size >= header_size + (1UL << DEPOT_STACK_ALIGN)) { + tail_size = size - header_size; + free->size = tail_size; + free->rcu_state = get_completed_synchronize_rcu(); + free->pending_node = NULL; + INIT_LIST_HEAD(&free->list); + trie_free_list_add(&free->list, free_trie_objects, + free_trie_object_map, trie_free_class(tail_size), + false); + return; + } + + trie_add_free_node_locked(ptr, size); +} + +static void *trie_pop_free_node(size_t size) +{ + struct stack_depot_trie_free_node *free; + unsigned int class; + size_t old_size; + + lockdep_assert_held(&pool_lock); + + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return NULL; + class = trie_free_class(size); + class = trie_find_next_set_class(free_trie_node_map, class); + if (class >= STACK_DEPOT_TRIE_FREE_CLASSES) + return NULL; + free = list_first_entry(&free_trie_nodes[class], typeof(*free), list); + old_size = (size_t)class << DEPOT_STACK_ALIGN; + trie_free_list_del(&free->list, free_trie_nodes, free_trie_node_map, + class); + if (old_size > size) + trie_free_object_tail_locked((char *)free + size, old_size - size); + return free; +} + +static void trie_retire_child_array_locked(const void *ptr) +{ + struct stack_depot_trie_free_object *free; + + lockdep_assert_held(&pool_lock); + free = trie_object_header(ptr); + free->pending_node = NULL; + free->rcu_state = get_state_synchronize_rcu(); + trie_free_list_add(&free->list, pending_trie_objects, + pending_trie_object_map, + trie_free_class(free->size), true); +} + +static void +trie_retire_child_array_with_node(const void *ptr, + const struct stack_depot_trie_node *node) +{ + struct stack_depot_trie_free_object *free; + unsigned long flags; + + raw_spin_lock_irqsave(&pool_lock, flags); + trie_retire_child_array_locked(ptr); + free = trie_object_header(ptr); + free->pending_node = (struct stack_depot_trie_node *)node; + raw_spin_unlock_irqrestore(&pool_lock, flags); +} + +static void *trie_pop_free_object(size_t size) +{ + struct stack_depot_trie_free_object *free; + void *tail; + size_t old_size; + size_t tail_size; + unsigned int class; + + lockdep_assert_held(&pool_lock); + + size = __stack_depot_trie_pool_alloc_size(size); + if (!size) + return NULL; + class = trie_free_class(size); + class = trie_find_next_set_class(free_trie_object_map, class); + if (class >= STACK_DEPOT_TRIE_FREE_CLASSES) + return NULL; + + free = list_first_entry(&free_trie_objects[class], typeof(*free), list); + trie_drain_free_object_node_locked(free); + trie_free_list_del(&free->list, free_trie_objects, + free_trie_object_map, class); + free->rcu_state = 0; + old_size = free->size; + if (old_size > size) { + free->size = size; + tail = trie_object_payload(free) + size; + tail_size = old_size - size; + trie_free_object_tail_locked(tail, tail_size); + } + return trie_object_payload(free); +} + +/* + * Preallocate resources that cannot be allocated while trie writers hold raw + * spinlocks. Side-table growth is mandatory before a new leaf ID can be + * used, so side-table preallocation failure disables insertion for this + * save. Pool preallocation is opportunistic: reusable trie storage or active + * pool space may still satisfy the insertion, and pool_carve() reports + * -ENOSPC if they do not. Callers without spinning allocation context skip + * insertion and perform only best-effort lookup. + */ +static int +__stack_depot_trie_alloc_prealloc(gfp_t alloc_flags, depot_flags_t depot_flags, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc) +{ + unsigned long flags; + bool can_alloc; + bool need_pool; + int ret = 0; + + can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && + gfpflags_allow_spinning(alloc_flags); + if (can_alloc) { + raw_spin_lock_irqsave(&pool_lock, flags); + need_pool = !new_pool; + raw_spin_unlock_irqrestore(&pool_lock, flags); + if (need_pool) { + struct page *page; + + page = alloc_pages(gfp_nested_mask(alloc_flags), DEPOT_POOL_ORDER); + if (page) + *pool_prealloc = page_address(page); + } + ret = __stack_depot_trie_side_table_prealloc(alloc_flags, side_prealloc); + } + + if (ret) { + if (*pool_prealloc) { + free_pages((unsigned long)*pool_prealloc, DEPOT_POOL_ORDER); + *pool_prealloc = NULL; + } + return -ENOSPC; + } + return 0; +} + +static int trie_pool_add_object_size(size_t size, size_t *total) +{ + size_t alloc_size; + + alloc_size = trie_object_alloc_size(size); + if (!alloc_size) + return -ENOSPC; + if (check_add_overflow(*total, alloc_size, total)) + return -ENOSPC; + return *total <= DEPOT_POOL_SIZE ? 0 : -ENOSPC; +} + +/* + * Allocate pool-backed storage for one trie insertion. Reusable retired + * fragments are preferred; any missing storage is carved as one contiguous range + * from the current stackdepot pool, possibly after installing @prealloc as a new + * pool. If allocation fails, any free-list pops are returned locally and + * pool_offset is not advanced. + * The caller must not publish any returned storage before side-table and trie + * publication succeeds. + */ +static int trie_pool_carve(struct stack_depot_trie_alloc_workspace *workspace, + void **pool_prealloc, unsigned int nr_nodes, + unsigned int nr_child_arrays, + size_t split_child_array_size, + size_t child_array_size) +{ + unsigned long flags; + unsigned long completed; + size_t one_child_size; + unsigned int i; + size_t alloc_size; + size_t offset; + size_t total = 0; + void *pool; + int ret = -ENOSPC; + + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + trie_drain_pending_objects_locked(); + one_child_size = __stack_depot_trie_child_array_size(1); + for (i = 0; i < nr_nodes; i++) { + workspace->nodes[i] = trie_pop_free_node(workspace->node_sizes[i]); + if (!workspace->nodes[i]) { + alloc_size = __stack_depot_trie_pool_alloc_size(workspace->node_sizes[i]); + if (!alloc_size || check_add_overflow(total, alloc_size, &total) || + total > DEPOT_POOL_SIZE) + goto out_discard; + } + } + for (i = 0; i < nr_child_arrays; i++) { + workspace->child_arrays[i] = trie_pop_free_object(one_child_size); + if (!workspace->child_arrays[i] && + trie_pool_add_object_size(one_child_size, &total)) + goto out_discard; + } + if (split_child_array_size) { + workspace->split_child_array = + trie_pop_free_object(split_child_array_size); + if (!workspace->split_child_array && + trie_pool_add_object_size(split_child_array_size, &total)) + goto out_discard; + } + if (child_array_size) { + workspace->child_array_storage = trie_pop_free_object(child_array_size); + if (!workspace->child_array_storage && + trie_pool_add_object_size(child_array_size, &total)) + goto out_discard; + } + + if (pools_num < 1) { + if (!depot_init_pool(pool_prealloc)) { + ret = -ENOSPC; + goto out_discard; + } + } + if (total > DEPOT_POOL_SIZE - pool_offset) { + if (!depot_init_pool(pool_prealloc)) { + ret = -ENOSPC; + goto out_discard; + } + } + + pool = stack_pools[pools_num - 1]; + + offset = pool_offset; + for (i = 0; i < nr_nodes; i++) { + if (workspace->nodes[i]) + continue; + workspace->nodes[i] = pool + offset; + offset += __stack_depot_trie_pool_alloc_size(workspace->node_sizes[i]); + } + for (i = 0; i < nr_child_arrays; i++) { + if (workspace->child_arrays[i]) + continue; + workspace->child_arrays[i] = + trie_object_init_fresh(pool + offset, one_child_size); + offset += trie_object_alloc_size(one_child_size); + } + if (split_child_array_size && !workspace->split_child_array) { + workspace->split_child_array = + trie_object_init_fresh(pool + offset, split_child_array_size); + offset += trie_object_alloc_size(split_child_array_size); + } + if (child_array_size && !workspace->child_array_storage) + workspace->child_array_storage = + trie_object_init_fresh(pool + offset, child_array_size); + pool_offset += total; + ret = 0; + goto out; +out_discard: + completed = get_completed_synchronize_rcu(); + for (i = 0; i < nr_nodes; i++) { + struct stack_depot_trie_node *node = workspace->nodes[i]; + + if (node) { + trie_add_free_node_locked(node, workspace->node_sizes[i]); + workspace->nodes[i] = NULL; + } + } + for (i = 0; i < nr_child_arrays; i++) { + struct stack_depot_trie_child_array *array = workspace->child_arrays[i]; + + if (!array) + continue; + trie_free_object_locked(array, completed); + workspace->child_arrays[i] = NULL; + } + if (split_child_array_size && workspace->split_child_array) { + trie_free_object_locked(workspace->split_child_array, completed); + workspace->split_child_array = NULL; + } + if (child_array_size && workspace->child_array_storage) { + trie_free_object_locked(workspace->child_array_storage, completed); + workspace->child_array_storage = NULL; + } +out: + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + return ret; +} + +static const struct stack_depot_trie_node * +stack_depot_trie_lookup(const struct stack_depot_trie_child_array * const *root_slot, + const unsigned long *entries, unsigned int nr_entries); + +static depot_stack_handle_t +trie_find_handle(const struct stack_depot_trie_child_array * const *root_slot, + const unsigned long *entries, unsigned int nr_entries) +{ + depot_stack_handle_t handle = 0; + const struct stack_depot_trie_node *leaf; + + rcu_read_lock_sched_notrace(); + leaf = stack_depot_trie_lookup(root_slot, entries, nr_entries); + if (leaf) + handle = __stack_depot_trie_handle(leaf->leaf_id); + rcu_read_unlock_sched_notrace(); + + return handle; +} + +static void trie_side_table_publish_new_leaf(u32 leaf_id, + const struct stack_depot_trie_node *leaf) +{ + const struct stack_depot_trie_node __rcu **slot; + unsigned long flags; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + slot = trie_side_table_leaf_slot(leaf_id); + /* Pairs with trie_side_table_load_leaf(). */ + rcu_assign_pointer(*slot, leaf); + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); +} + +static void trie_side_table_publish_split_leaves(u32 old_leaf_id, + const struct stack_depot_trie_node *old_leaf, + u32 new_leaf_id, + const struct stack_depot_trie_node *new_leaf) +{ + const struct stack_depot_trie_node __rcu **new_slot; + const struct stack_depot_trie_node __rcu **old_slot = NULL; + unsigned long flags; + + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + if (old_leaf_id) + old_slot = trie_side_table_leaf_slot(old_leaf_id); + + new_slot = trie_side_table_leaf_slot(new_leaf_id); + if (old_slot) { + /* Pairs with trie_side_table_load_leaf(). */ + rcu_assign_pointer(*old_slot, old_leaf); + } + + /* Pairs with trie_side_table_load_leaf(). */ + rcu_assign_pointer(*new_slot, new_leaf); + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); +} + static int __init disable_stack_depot(char *str) { return kstrtobool(str, &stack_depot_disabled); @@ -220,6 +1456,10 @@ int __init stack_depot_early_init(void) stack_depot_disabled = true; return -ENOMEM; } + if (stack_depot_trie_requested && stack_depot_trie_init_memblock()) { + pr_warn("trie storage initialization failed, disabling trie storage\n"); + stack_depot_trie_requested = false; + } return 0; } @@ -233,8 +1473,10 @@ int stack_depot_init(void) mutex_lock(&stack_depot_init_mutex); - if (stack_depot_disabled || stack_table) + if (stack_depot_disabled) goto out_unlock; + if (stack_table) + goto init_trie; /* * Similarly to stack_depot_early_init, use stack_bucket_number_order @@ -278,6 +1520,16 @@ int stack_depot_init(void) kvfree(stack_table); stack_depot_disabled = true; ret = -ENOMEM; + goto out_unlock; + } +init_trie: + if (!ret && stack_depot_trie_requested) { + ret = stack_depot_trie_init(GFP_KERNEL); + if (ret) { + pr_warn("trie storage initialization failed, disabling trie storage\n"); + stack_depot_trie_requested = false; + ret = 0; + } } out_unlock: @@ -352,6 +1604,19 @@ static void depot_keep_new_pool(void **prealloc) *prealloc = NULL; } +static void depot_try_keep_new_pool(void **prealloc) +{ + unsigned long flags; + + if (!prealloc || !*prealloc) + return; + + if (!raw_spin_trylock_irqsave(&pool_lock, flags)) + return; + depot_keep_new_pool(prealloc); + raw_spin_unlock_irqrestore(&pool_lock, flags); +} + /* * Try to initialize a new stack record from the current pool, a cached pool, or * the current pre-allocation. @@ -427,7 +1692,8 @@ static inline size_t depot_stack_record_size(struct stack_record *s, unsigned in /* Allocates a new stack in a stack depot pool. */ static struct stack_record * -depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, depot_flags_t flags, void **prealloc) +depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, + depot_flags_t flags, void **prealloc) { struct stack_record *stack = NULL; size_t record_size; @@ -462,6 +1728,7 @@ depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, dep /* Save the stack trace. */ stack->hash = hash; stack->size = nr_entries; + stack->flags = flags & STACK_DEPOT_FLAGS_MASK; /* stack->handle is already filled in by depot_pop_free_pool(). */ memcpy(stack->entries, entries, flex_array_size(stack, entries, nr_entries)); @@ -582,9 +1849,10 @@ int stackdepot_memcmp(const unsigned long *u1, const unsigned long *u2, /* Finds a stack in a bucket of the hash table. */ static inline struct stack_record *find_stack(struct list_head *bucket, - unsigned long *entries, int size, + unsigned long *entries, unsigned int size, u32 hash, depot_flags_t flags) { + depot_flags_t mode = STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE; struct stack_record *stack, *ret = NULL; /* @@ -601,6 +1869,9 @@ static inline struct stack_record *find_stack(struct list_head *bucket, list_for_each_entry_rcu(stack, bucket, hash_list) { if (stack->hash != hash || stack->size != size) continue; + /* Plain, refcounted, and countable records have distinct lifetimes. */ + if ((stack->flags & mode) != (flags & mode)) + continue; /* * This may race with depot_free_stack() accessing the freelist @@ -630,6 +1901,112 @@ static inline struct stack_record *find_stack(struct list_head *bucket, return ret; } +static int +stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_slot, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace, + u32 *leaf_id, + u32 *scratch); + +static depot_stack_handle_t +stack_depot_trie_save(unsigned long *entries, unsigned int nr_entries, + gfp_t alloc_flags, depot_flags_t depot_flags) +{ + struct stack_depot_trie_alloc_workspace *workspace; + struct stack_depot_trie_side_prealloc side_prealloc = {}; + void *pool_prealloc = NULL; + depot_stack_handle_t handle; + unsigned long flags; + bool can_alloc; + bool retried = false; + u32 leaf_id; + int ret; + + workspace = stack_depot_trie_load_workspace(); + can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && + gfpflags_allow_spinning(alloc_flags); + +retry: + handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); + if (handle) + return handle; + /* + * No-spin callers cannot wait for the workspace lock or allocate side-table + * or pool storage. After the lockless lookup misses, trylock and recheck: a + * concurrent writer may have inserted the stack. Otherwise fail instead of + * spinning or publishing a new leaf. + */ + if (in_nmi() || !gfpflags_allow_spinning(alloc_flags)) { + if (!raw_spin_trylock_irqsave(&stack_depot_trie_writer_lock, flags)) + return 0; + handle = trie_find_handle(&stack_depot_trie_root, entries, nr_entries); + raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); + return handle; + } + + ret = __stack_depot_trie_alloc_prealloc(alloc_flags, depot_flags, + &pool_prealloc, + &side_prealloc); + if (ret) + goto out_free; + + raw_spin_lock_irqsave(&stack_depot_trie_writer_lock, flags); + memset(workspace, 0, sizeof(*workspace)); + ret = stack_depot_trie_insert_locked(&stack_depot_trie_root, + entries, nr_entries, + &pool_prealloc, &side_prealloc, + workspace, &leaf_id, + workspace->scratch); + if (!ret) + handle = __stack_depot_trie_handle(leaf_id); + raw_spin_unlock_irqrestore(&stack_depot_trie_writer_lock, flags); + if (!handle && ret == -ENOSPC && can_alloc && !retried) { + retried = true; + depot_try_keep_new_pool(&pool_prealloc); + if (pool_prealloc) { + free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); + pool_prealloc = NULL; + } + __stack_depot_trie_side_table_free_prealloc(&side_prealloc); + goto retry; + } + +out_free: + depot_try_keep_new_pool(&pool_prealloc); + if (pool_prealloc) + free_pages((unsigned long)pool_prealloc, DEPOT_POOL_ORDER); + __stack_depot_trie_side_table_free_prealloc(&side_prealloc); + return handle; +} + +static depot_stack_handle_t +depot_save_stack_locked(struct list_head *bucket, unsigned long *entries, + unsigned int nr_entries, u32 hash, + depot_flags_t depot_flags, void **prealloc) +{ + struct stack_record *found; + struct stack_record *new; + + lockdep_assert_held(&pool_lock); + + /* Try to find again, to avoid concurrently inserting duplicates. */ + found = find_stack(bucket, entries, nr_entries, hash, depot_flags); + if (found) + return found->handle.handle; + new = depot_alloc_stack(entries, nr_entries, hash, depot_flags, prealloc); + if (!new) + return 0; + + /* + * This releases the stack record into the bucket and makes it visible to + * readers in find_stack(). + */ + list_add_rcu(&new->hash_list, bucket); + return new->handle.handle; +} + depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags, @@ -642,11 +2019,15 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, void *prealloc = NULL; bool allow_spin = gfpflags_allow_spinning(alloc_flags); bool can_alloc = (depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC) && allow_spin; + bool trie_candidate; unsigned long flags; u32 hash; if (WARN_ON(depot_flags & ~STACK_DEPOT_FLAGS_MASK)) return 0; + if (WARN_ON_ONCE((depot_flags & STACK_DEPOT_FLAG_GET) && + (depot_flags & STACK_DEPOT_FLAG_COUNTABLE))) + return 0; /* * If this stack trace is from an interrupt, including anything before @@ -660,15 +2041,26 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, if (unlikely(nr_entries == 0) || stack_depot_disabled) return 0; + if (nr_entries > CONFIG_STACKDEPOT_MAX_FRAMES) + nr_entries = CONFIG_STACKDEPOT_MAX_FRAMES; + + trie_candidate = !(depot_flags & (STACK_DEPOT_FLAG_GET | STACK_DEPOT_FLAG_COUNTABLE)) && + __stack_depot_trie_ready(); + if (trie_candidate) { + handle = stack_depot_trie_save(entries, nr_entries, alloc_flags, + depot_flags); + if (handle) + return handle; + /* Keep trie failures visible; hash fallback hides trie pool pressure. */ + return 0; + } hash = hash_stack(entries, nr_entries); bucket = &stack_table[hash & stack_hash_mask]; - /* Fast path: look the stack trace up without locking. */ found = find_stack(bucket, entries, nr_entries, hash, depot_flags); if (found) - goto exit; - + return found->handle.handle; /* * Allocate memory for a new pool if required now: * we won't be able to do that under the lock. @@ -685,28 +2077,27 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, WARN_ON_ONCE(can_alloc); /* Best effort; bail if we fail to take the lock. */ if (!raw_spin_trylock_irqsave(&pool_lock, flags)) - goto exit; - } else { - raw_spin_lock_irqsave(&pool_lock, flags); - } - printk_deferred_enter(); - - /* Try to find again, to avoid concurrently inserting duplicates. */ - found = find_stack(bucket, entries, nr_entries, hash, depot_flags); - if (!found) { - struct stack_record *new = - depot_alloc_stack(entries, nr_entries, hash, depot_flags, &prealloc); - - if (new) { + goto out_free; + printk_deferred_enter(); + handle = depot_save_stack_locked(bucket, entries, nr_entries, + hash, depot_flags, &prealloc); + if (prealloc) { /* - * This releases the stack record into the bucket and - * makes it visible to readers in find_stack(). + * Either stack depot already contains this stack trace, or + * depot_alloc_stack() did not consume the preallocated memory. + * Try to keep the preallocated memory for future. */ - list_add_rcu(&new->hash_list, bucket); - found = new; + depot_keep_new_pool(&prealloc); } + printk_deferred_exit(); + raw_spin_unlock_irqrestore(&pool_lock, flags); + goto out_free; } + raw_spin_lock_irqsave(&pool_lock, flags); + printk_deferred_enter(); + handle = depot_save_stack_locked(bucket, entries, nr_entries, + hash, depot_flags, &prealloc); if (prealloc) { /* * Either stack depot already contains this stack trace, or @@ -715,10 +2106,10 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, */ depot_keep_new_pool(&prealloc); } - printk_deferred_exit(); raw_spin_unlock_irqrestore(&pool_lock, flags); -exit: + +out_free: if (prealloc) { /* Stack depot didn't use this memory, free it. */ if (!allow_spin) @@ -726,8 +2117,6 @@ depot_stack_handle_t stack_depot_save_flags(unsigned long *entries, else free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER); } - if (found) - handle = found->handle.handle; return handle; } EXPORT_SYMBOL_GPL(stack_depot_save_flags); @@ -743,10 +2132,760 @@ EXPORT_SYMBOL_GPL(stack_depot_save); struct stack_record *__stack_depot_get_stack_record(depot_stack_handle_t handle) { + struct stack_record *stack; + if (!handle) return NULL; + if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) + return NULL; + + stack = depot_fetch_stack(handle); + if (!stack) + return NULL; + if (WARN_ON_ONCE(!(stack->flags & STACK_DEPOT_FLAG_COUNTABLE))) + return NULL; + + return stack; +} + +static void frame_run_init(const unsigned long *entries, + unsigned int nr_entries, + struct stack_depot_frame_run *run) +{ + u32 payload; + unsigned int i; + bool compressed; + + compressed = arch_stack_depot_frame_try_compress(entries[0], &payload); + for (i = 1; i < nr_entries; i++) { + bool next; + + next = arch_stack_depot_frame_try_compress(entries[i], &payload); + if (next != compressed) + break; + } + + /* @i is the first non-matching frame, or @nr_entries if all matched. */ + run->mode = compressed ? STACK_DEPOT_FRAME_COMPRESSED : STACK_DEPOT_FRAME_RAW; + run->nr_entries = i; +} + +static void +stack_depot_trie_node_frame(const struct stack_depot_trie_node *node, + unsigned int index, unsigned long *frame) +{ + u32 payload; + + if (node->run.mode == STACK_DEPOT_FRAME_RAW) { + memcpy(frame, node->data + index * sizeof(*frame), + sizeof(*frame)); + return; + } + + memcpy(&payload, node->data + index * sizeof(payload), sizeof(payload)); + arch_stack_depot_frame_decompress(payload, frame); +} + +static void trie_node_init(void *storage, + const struct stack_depot_trie_node *parent, u32 leaf_id, + const unsigned long *entries, unsigned int nr_entries, + u32 *scratch) +{ + struct stack_depot_trie_node *node = storage; + struct stack_depot_frame_run run; + + frame_run_init(entries, nr_entries, &run); + + /* Caller-owned storage is not publishable unless the payload write succeeds. */ + if (run.mode == STACK_DEPOT_FRAME_COMPRESSED) { + unsigned int i; + + for (i = 0; i < run.nr_entries; i++) + arch_stack_depot_frame_try_compress(entries[i], &scratch[i]); + memcpy(node->data, scratch, stack_depot_frame_run_bytes(&run)); + } else { + memcpy(node->data, entries, stack_depot_frame_run_bytes(&run)); + } + + node->parent = parent; + node->children = NULL; + node->leaf_id = leaf_id; + node->run = run; +} + +static void trie_node_init_slice(void *storage, + const struct stack_depot_trie_node *parent, u32 leaf_id, + const struct stack_depot_trie_node *src_node, + unsigned int start, unsigned int nr_entries) +{ + struct stack_depot_trie_node *node = storage; + struct stack_depot_frame_run run; + size_t entry_bytes; + + run = src_node->run; + run.nr_entries = nr_entries; + + entry_bytes = stack_depot_frame_run_entry_bytes(src_node->run.mode); + memcpy(node->data, src_node->data + start * entry_bytes, + stack_depot_frame_run_bytes(&run)); + node->parent = parent; + node->children = NULL; + node->leaf_id = leaf_id; + node->run = run; +} + +static unsigned int +__stack_depot_trie_node_match(const struct stack_depot_trie_node *node, + const unsigned long *entries, + unsigned int nr_entries) +{ + unsigned int limit; + unsigned int i; + + limit = min(node->run.nr_entries, nr_entries); + if (node->run.mode == STACK_DEPOT_FRAME_RAW) { + for (i = 0; i < limit; i++) { + unsigned long frame; + + memcpy(&frame, node->data + i * sizeof(frame), sizeof(frame)); + if (frame != entries[i]) + break; + } + + return i; + } + + for (i = 0; i < limit; i++) { + unsigned long frame; + + stack_depot_trie_node_frame(node, i, &frame); + if (frame != entries[i]) + break; + } + + return i; +} + +static const struct stack_depot_trie_node * +trie_load_parent(const struct stack_depot_trie_node *node) +{ + const struct stack_depot_trie_node __rcu * const *slot; + + slot = (const struct stack_depot_trie_node __rcu * const *)&node->parent; + return rcu_dereference_check(*slot, + lockdep_is_held(&stack_depot_trie_writer_lock) || + rcu_read_lock_sched_held()); +} + +static const struct stack_depot_trie_child_array * +trie_load_children_slot(const struct stack_depot_trie_child_array * const *slot) +{ + const struct stack_depot_trie_child_array __rcu * const *rcu_slot; + + rcu_slot = (const struct stack_depot_trie_child_array __rcu * const *)slot; + return rcu_dereference_check(*rcu_slot, + lockdep_is_held(&stack_depot_trie_writer_lock) || + rcu_read_lock_sched_held()); +} + +static const struct stack_depot_trie_node * +trie_child_array_load_child(const struct stack_depot_trie_child_array *array, + unsigned int pos) +{ + const struct stack_depot_trie_node __rcu * const *slot; + + slot = (const struct stack_depot_trie_node __rcu * const *)&array->children[pos]; + return rcu_dereference_check(*slot, + lockdep_is_held(&stack_depot_trie_writer_lock) || + rcu_read_lock_sched_held()); +} + +static void +trie_child_array_find_slot(const struct stack_depot_trie_child_array *array, + unsigned long frame, unsigned int *pos, bool *found) +{ + unsigned int left = 0; + unsigned int right; + + *pos = 0; + *found = false; + + right = READ_ONCE(array->nr_children); + while (left < right) { + unsigned int mid = left + (right - left) / 2; + const struct stack_depot_trie_node *node; + unsigned long mid_frame; + + node = trie_child_array_load_child(array, mid); + if (!node) { + /* Tail append may produce a transient lockless lookup miss. */ + right = mid; + continue; + } + stack_depot_trie_node_frame(node, 0, &mid_frame); + if (mid_frame < frame) { + left = mid + 1; + } else if (mid_frame > frame) { + right = mid; + } else { + *pos = mid; + *found = true; + return; + } + } + + *pos = left; +} - return depot_fetch_stack(handle); +static void +trie_child_array_insert_at(const struct stack_depot_trie_child_array *old, + unsigned int pos, + const struct stack_depot_trie_node *node, + struct stack_depot_trie_child_array *new_array, + unsigned int new_capacity) +{ + unsigned int nr_old; + unsigned int i; + + nr_old = old->nr_children; + + new_array->nr_children = nr_old + 1; + new_array->capacity = new_capacity; + for (i = 0; i < pos; i++) + new_array->children[i] = trie_child_array_load_child(old, i); + new_array->children[pos] = node; + for (i = pos; i < nr_old; i++) + new_array->children[i + 1] = trie_child_array_load_child(old, i); + for (i = nr_old + 1; i < new_array->capacity; i++) + new_array->children[i] = NULL; +} + +static void +trie_publish_children_slot(const struct stack_depot_trie_child_array **slot, + const struct stack_depot_trie_child_array *children) +{ + const struct stack_depot_trie_child_array __rcu **rcu_slot; + + rcu_slot = (const struct stack_depot_trie_child_array __rcu **)slot; + rcu_assign_pointer(*rcu_slot, children); +} + +static void +trie_child_array_replace_at(const struct stack_depot_trie_child_array *old_array, + const struct stack_depot_trie_node *new_child, + struct stack_depot_trie_child_array *new_array, + unsigned int pos) +{ + unsigned int i; + + new_array->nr_children = old_array->nr_children; + new_array->capacity = old_array->capacity; + for (i = 0; i < old_array->nr_children; i++) + new_array->children[i] = trie_child_array_load_child(old_array, i); + new_array->children[pos] = new_child; + for (i = old_array->nr_children; i < new_array->capacity; i++) + new_array->children[i] = NULL; +} + +static void trie_reparent_children(struct stack_depot_trie_node *parent) +{ + const struct stack_depot_trie_child_array *children = parent->children; + unsigned int i; + + if (!children) + return; + /* + * COW updates reuse unchanged descendant subtrees. Repoint their parent + * links before retiring the old parent so fetch never follows a freed node. + * Lockless lookups that still observe the old child array may see the new + * parent early and miss. A writer-lock recheck observes the new array before + * inserting, so the miss cannot create a permanent duplicate. + */ + for (i = 0; i < children->nr_children; i++) { + struct stack_depot_trie_node *child; + const struct stack_depot_trie_node __rcu **slot; + + child = (struct stack_depot_trie_node *)trie_child_array_load_child(children, i); + slot = (const struct stack_depot_trie_node __rcu **)&child->parent; + rcu_assign_pointer(*slot, parent); + } +} + +static void +trie_build_append_chain(const struct stack_depot_trie_node *parent, u32 leaf_id, + const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_trie_node * const *nodes, + struct stack_depot_trie_child_array * const *child_arrays, + u32 *scratch, + const struct stack_depot_trie_node **head, + const struct stack_depot_trie_node **tail) +{ + const struct stack_depot_trie_node *prev = parent; + unsigned int pos = 0; + unsigned int used = 0; + unsigned int i; + + while (pos < nr_entries) { + struct stack_depot_frame_run run; + struct stack_depot_trie_node *node; + u32 id; + + node = nodes[used]; + frame_run_init(&entries[pos], nr_entries - pos, &run); + + id = pos + run.nr_entries == nr_entries ? leaf_id : 0; + trie_node_init(node, prev, id, &entries[pos], run.nr_entries, + scratch); + + prev = node; + pos += run.nr_entries; + used++; + } + + for (i = 0; i + 1 < used; i++) { + const struct stack_depot_trie_node *next = nodes[i + 1]; + struct stack_depot_trie_node *node = nodes[i]; + struct stack_depot_trie_child_array *array = child_arrays[i]; + + trie_child_array_init(array, 1, &next, 1); + node->children = array; + } + + *head = nodes[0]; + *tail = nodes[used - 1]; +} + +static void trie_publish_tail_append(struct stack_depot_trie_child_array *array, + unsigned int pos, + const struct stack_depot_trie_node *head) +{ + const struct stack_depot_trie_node __rcu **slot; + + /* + * Writers hold stack_depot_trie_writer_lock. Existing children are + * immutable, so tail append publishes the new child before increasing the + * visible count. Lockless readers that see the old count miss; readers that + * see the new count either load the initialized child or treat a transient + * NULL as a miss. The writer-lock recheck prevents permanent duplicates. + */ + slot = (const struct stack_depot_trie_node __rcu **)&array->children[pos]; + rcu_assign_pointer(*slot, head); + WRITE_ONCE(array->nr_children, pos + 1); +} + +static const struct stack_depot_trie_node * +stack_depot_trie_lookup(const struct stack_depot_trie_child_array * const *root_slot, + const unsigned long *entries, unsigned int nr_entries) +{ + const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_node *parent = NULL; + unsigned int pos = 0; + + children = trie_load_children_slot(root_slot); + + while (pos < nr_entries) { + const struct stack_depot_trie_node *node; + unsigned int remaining = nr_entries - pos; + unsigned int matched; + unsigned int slot; + bool found; + + if (!children) + return NULL; + trie_child_array_find_slot(children, entries[pos], &slot, &found); + if (!found) + return NULL; + + node = trie_child_array_load_child(children, slot); + if (!node) + return NULL; + if (trie_load_parent(node) != parent) + return NULL; + + matched = __stack_depot_trie_node_match(node, &entries[pos], remaining); + if (!matched || matched < node->run.nr_entries) + return NULL; + pos += matched; + if (pos == nr_entries) + return node->leaf_id ? node : NULL; + + parent = node; + children = trie_load_children_slot(&node->children); + } + + return NULL; +} + +static void trie_build_split(const struct stack_depot_trie_node *child, + unsigned int matched, u32 leaf_id, + const unsigned long *entries, unsigned int nr_entries, + struct stack_depot_trie_node * const *nodes, + struct stack_depot_trie_child_array * const *child_arrays, + struct stack_depot_trie_child_array *split_child_array, + u32 *scratch, + const struct stack_depot_trie_node **prefix, + const struct stack_depot_trie_node **old_tail, + const struct stack_depot_trie_node **new_leaf) +{ + const struct stack_depot_trie_child_array *child_children; + const struct stack_depot_trie_node *child_parent; + const unsigned long *tail_entries; + const struct stack_depot_trie_node *split_children[2]; + const struct stack_depot_trie_node *new_head = NULL; + const struct stack_depot_trie_node *new_tail = NULL; + struct stack_depot_trie_node *old_tail_node; + struct stack_depot_trie_node *pref; + unsigned long new_frame; + unsigned long old_frame; + u32 prefix_leaf_id; + unsigned int tail_len; + bool has_new_tail; + + child_children = trie_load_children_slot(&child->children); + child_parent = trie_load_parent(child); + + has_new_tail = matched < nr_entries; + pref = nodes[0]; + old_tail_node = nodes[1]; + prefix_leaf_id = has_new_tail ? 0 : leaf_id; + trie_node_init_slice(pref, child_parent, prefix_leaf_id, child, 0, + matched); + tail_len = child->run.nr_entries - matched; + trie_node_init_slice(old_tail_node, pref, child->leaf_id, child, matched, + tail_len); + + if (has_new_tail) { + tail_entries = &entries[matched]; + tail_len = nr_entries - matched; + trie_build_append_chain(pref, leaf_id, tail_entries, tail_len, + &nodes[2], child_arrays, scratch, + &new_head, &new_tail); + stack_depot_trie_node_frame(old_tail_node, 0, &old_frame); + stack_depot_trie_node_frame(new_head, 0, &new_frame); + if (old_frame < new_frame) { + split_children[0] = old_tail_node; + split_children[1] = new_head; + } else { + split_children[0] = new_head; + split_children[1] = old_tail_node; + } + trie_child_array_init(split_child_array, ARRAY_SIZE(split_children), + split_children, ARRAY_SIZE(split_children)); + } else { + split_children[0] = old_tail_node; + trie_child_array_init(split_child_array, 1, split_children, 1); + } + old_tail_node->children = child_children; + pref->children = split_child_array; + *prefix = pref; + *old_tail = old_tail_node; + *new_leaf = has_new_tail ? new_tail : pref; +} + +static void trie_size_append_chain(const unsigned long *entries, + unsigned int nr_entries, + size_t *node_sizes, + unsigned int *nr_nodes, + unsigned int *nr_child_arrays) +{ + unsigned int pos = 0; + unsigned int used = 0; + + while (pos < nr_entries) { + struct stack_depot_frame_run run; + + frame_run_init(&entries[pos], nr_entries - pos, &run); + node_sizes[used] = __stack_depot_trie_node_size(&run); + pos += run.nr_entries; + used++; + } + + *nr_nodes = used; + *nr_child_arrays = used > 1 ? used - 1 : 0; +} + +static int +trie_pool_alloc_append_chain(struct stack_depot_trie_alloc_workspace *workspace, + void **pool_prealloc, + const unsigned long *entries, + unsigned int nr_entries, + size_t child_array_size) +{ + unsigned int nr_child_arrays; + unsigned int nr_nodes; + + trie_size_append_chain(entries, nr_entries, workspace->node_sizes, + &nr_nodes, &nr_child_arrays); + return trie_pool_carve(workspace, pool_prealloc, nr_nodes, nr_child_arrays, + 0, child_array_size); +} + +static int +trie_pool_alloc_split(struct stack_depot_trie_alloc_workspace *workspace, + void **pool_prealloc, + const struct stack_depot_trie_child_array *children, + const struct stack_depot_trie_node *child, + unsigned int matched, const unsigned long *entries, + unsigned int nr_entries) +{ + struct stack_depot_frame_run old_tail_run; + struct stack_depot_frame_run prefix_run; + unsigned int new_child_arrays = 0; + unsigned int new_used = 0; + unsigned int nr_child_arrays; + unsigned int nr_nodes; + bool has_new_tail; + size_t child_array_size; + size_t split_child_array_size; + + prefix_run = child->run; + prefix_run.nr_entries = matched; + old_tail_run = child->run; + old_tail_run.nr_entries = child->run.nr_entries - matched; + has_new_tail = matched < nr_entries; + + workspace->node_sizes[0] = __stack_depot_trie_node_size(&prefix_run); + workspace->node_sizes[1] = __stack_depot_trie_node_size(&old_tail_run); + if (has_new_tail) + trie_size_append_chain(&entries[matched], nr_entries - matched, + &workspace->node_sizes[2], &new_used, + &new_child_arrays); + + child_array_size = trie_child_array_size_for_capacity(children->capacity); + split_child_array_size = + __stack_depot_trie_child_array_size(has_new_tail ? 2 : 1); + nr_nodes = 2 + new_used; + nr_child_arrays = new_child_arrays; + return trie_pool_carve(workspace, pool_prealloc, nr_nodes, nr_child_arrays, + split_child_array_size, child_array_size); +} + +static int +stack_depot_trie_insert_locked(const struct stack_depot_trie_child_array **root_slot, + const unsigned long *entries, unsigned int nr_entries, + void **pool_prealloc, + struct stack_depot_trie_side_prealloc *side_prealloc, + struct stack_depot_trie_alloc_workspace *workspace, + u32 *leaf_id, + u32 *scratch) +{ + struct stack_depot_trie_child_array *new_array; + struct stack_depot_trie_child_array *split_child_array; + const struct stack_depot_trie_child_array *children; + const struct stack_depot_trie_child_array **slot = root_slot; + const struct stack_depot_trie_node *child; + const struct stack_depot_trie_node *head; + const struct stack_depot_trie_node *last; + const struct stack_depot_trie_node *old_tail; + const struct stack_depot_trie_node *new_leaf; + struct stack_depot_trie_node *new_node; + struct stack_depot_trie_node *parent = NULL; + unsigned int matched; + unsigned int pos; + unsigned long flags; + u32 new_leaf_id; + size_t child_array_size; + bool found; + int ret; + + for (;;) { + children = trie_load_children_slot(slot); + if (!children) { + new_leaf_id = trie_side_table_next_leaf_id(); + if (!new_leaf_id) + return -ENOSPC; + if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) + return -ENOSPC; + child_array_size = __stack_depot_trie_child_array_size(1); + ret = trie_pool_alloc_append_chain(workspace, pool_prealloc, entries, + nr_entries, child_array_size); + if (ret) + return ret; + new_array = workspace->child_array_storage; + trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, + workspace->nodes, workspace->child_arrays, + scratch, &head, &last); + trie_side_table_publish_new_leaf(new_leaf_id, last); + new_array->nr_children = 1; + new_array->capacity = 1; + new_array->children[0] = head; + /* Publish the fully initialized replacement array last. */ + trie_publish_children_slot(slot, new_array); + goto out_success; + } + trie_child_array_find_slot(children, entries[0], &pos, &found); + if (!found) { + struct stack_depot_trie_child_array *tail_array; + bool tail_append; + + tail_append = pos == children->nr_children && + children->nr_children < children->capacity; + + new_leaf_id = trie_side_table_next_leaf_id(); + if (!new_leaf_id) + return -ENOSPC; + if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) + return -ENOSPC; + child_array_size = tail_append ? 0 : + __stack_depot_trie_child_array_size(children->nr_children + 1); + ret = trie_pool_alloc_append_chain(workspace, pool_prealloc, entries, + nr_entries, child_array_size); + if (ret) + return ret; + new_array = workspace->child_array_storage; + trie_build_append_chain(parent, new_leaf_id, entries, nr_entries, + workspace->nodes, workspace->child_arrays, + scratch, &head, &last); + trie_side_table_publish_new_leaf(new_leaf_id, last); + if (tail_append) { + tail_array = (struct stack_depot_trie_child_array *)children; + trie_publish_tail_append(tail_array, pos, head); + } else { + unsigned int capacity; + + capacity = trie_child_array_capacity(children->nr_children + 1); + trie_child_array_insert_at(children, pos, head, + new_array, capacity); + /* Publish the fully initialized replacement array last. */ + trie_publish_children_slot(slot, new_array); + raw_spin_lock_irqsave(&pool_lock, flags); + trie_retire_child_array_locked(children); + raw_spin_unlock_irqrestore(&pool_lock, flags); + } + goto out_success; + } + + child = trie_child_array_load_child(children, pos); + matched = __stack_depot_trie_node_match(child, entries, nr_entries); + if (matched < child->run.nr_entries) { + new_leaf_id = trie_side_table_next_leaf_id(); + if (!new_leaf_id) + return -ENOSPC; + if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) + return -ENOSPC; + ret = trie_pool_alloc_split(workspace, pool_prealloc, children, child, + matched, entries, nr_entries); + if (ret) + return ret; + new_array = workspace->child_array_storage; + split_child_array = workspace->split_child_array; + trie_build_split(child, matched, new_leaf_id, entries, nr_entries, + workspace->nodes, workspace->child_arrays, + split_child_array, scratch, &head, &old_tail, + &new_leaf); + trie_side_table_publish_split_leaves(child->leaf_id, old_tail, + new_leaf_id, new_leaf); + trie_child_array_replace_at(children, head, new_array, pos); + trie_reparent_children((struct stack_depot_trie_node *)old_tail); + /* Publish the fully initialized replacement array last. */ + trie_publish_children_slot(slot, new_array); + trie_retire_child_array_with_node(children, child); + goto out_success; + } + if (matched == nr_entries) { + if (child->leaf_id) { + *leaf_id = child->leaf_id; + return 0; + } + new_leaf_id = trie_side_table_next_leaf_id(); + if (!new_leaf_id) + return -ENOSPC; + if (!trie_side_table_ensure_leaf_slot(new_leaf_id, side_prealloc)) + return -ENOSPC; + workspace->node_sizes[0] = __stack_depot_trie_node_size(&child->run); + child_array_size = trie_child_array_size_for_capacity(children->capacity); + ret = trie_pool_carve(workspace, pool_prealloc, 1, 0, 0, + child_array_size); + if (ret) + return ret; + new_node = workspace->nodes[0]; + new_array = workspace->child_array_storage; + memcpy(new_node, child, __stack_depot_trie_node_size(&child->run)); + new_node->leaf_id = new_leaf_id; + trie_side_table_publish_new_leaf(new_leaf_id, new_node); + trie_child_array_replace_at(children, new_node, new_array, pos); + trie_reparent_children(new_node); + /* Publish the fully initialized replacement array last. */ + trie_publish_children_slot(slot, new_array); + trie_retire_child_array_with_node(children, child); + goto out_success; + } + + parent = (struct stack_depot_trie_node *)child; + slot = &parent->children; + entries += matched; + nr_entries -= matched; + } + +out_success: + raw_spin_lock_irqsave(&trie_side_table_lock, flags); + trie_side_table_last_leaf_id = new_leaf_id; + raw_spin_unlock_irqrestore(&trie_side_table_lock, flags); + *leaf_id = new_leaf_id; + return 0; +} + +static unsigned int +__stack_depot_trie_fetch_into(const struct stack_depot_trie_node *leaf, + unsigned long *entries, + unsigned int max_entries) +{ + const struct stack_depot_trie_node *node; + unsigned int total; + unsigned int seen = 0; + unsigned int pos; + unsigned int i; + + total = 0; + for (node = leaf; node; node = trie_load_parent(node)) + total += node->run.nr_entries; + if (max_entries < total) + return 0; + + pos = total; + for (node = leaf; node; node = trie_load_parent(node)) { + if (node->run.nr_entries > pos) + return 0; + pos -= node->run.nr_entries; + for (i = 0; i < node->run.nr_entries; i++) { + stack_depot_trie_node_frame(node, i, &entries[pos + i]); + seen++; + } + } + if (seen != total || pos) + return 0; + + return total; +} + +static unsigned int +__stack_depot_trie_fetch_handle_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries) +{ + const struct stack_depot_trie_node *leaf; + u32 leaf_id; + unsigned int nr_entries; + + if (!handle || !entries || !max_entries) + return 0; + + leaf_id = __stack_depot_trie_leaf_id(handle); + if (!leaf_id) + return 0; + + rcu_read_lock_sched_notrace(); + leaf = __stack_depot_trie_side_table_lookup(leaf_id); + if (WARN_ONCE(!leaf, "corrupt trie handle %08x\n", handle)) { + rcu_read_unlock_sched_notrace(); + return 0; + } + nr_entries = __stack_depot_trie_fetch_into(leaf, entries, max_entries); + rcu_read_unlock_sched_notrace(); + if (nr_entries) + kmsan_unpoison_memory(entries, nr_entries * sizeof(*entries)); + + return nr_entries; } unsigned int stack_depot_fetch(depot_stack_handle_t handle, @@ -763,6 +2902,8 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, if (!handle || stack_depot_disabled) return 0; + if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) + return 0; stack = depot_fetch_stack(handle); /* @@ -777,12 +2918,42 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle, } EXPORT_SYMBOL_GPL(stack_depot_fetch); +unsigned int stack_depot_fetch_into(depot_stack_handle_t handle, + unsigned long *entries, + unsigned int max_entries) +{ + struct stack_record *stack; + unsigned int nr_entries; + + if (!handle || !entries || !max_entries) + return 0; + if (stack_depot_disabled) + return 0; + if (__stack_depot_trie_leaf_id(handle)) + return __stack_depot_trie_fetch_handle_into(handle, entries, + max_entries); + + stack = depot_fetch_stack(handle); + if (!stack) + return 0; + nr_entries = stack->size; + if (!nr_entries || nr_entries > max_entries) + return 0; + + memcpy(entries, stack->entries, nr_entries * sizeof(*entries)); + kmsan_unpoison_memory(entries, nr_entries * sizeof(*entries)); + return nr_entries; +} +EXPORT_SYMBOL_GPL(stack_depot_fetch_into); + void stack_depot_put(depot_stack_handle_t handle) { struct stack_record *stack; if (!handle || stack_depot_disabled) return; + if (WARN_ON_ONCE(__stack_depot_trie_leaf_id(handle))) + return; stack = depot_fetch_stack(handle); /* @@ -791,7 +2962,8 @@ void stack_depot_put(depot_stack_handle_t handle) */ if (WARN(!stack, "corrupt handle or unbalanced stack_depot_put()")) return; - + if (WARN_ON_ONCE(!(stack->flags & STACK_DEPOT_FLAG_GET))) + return; if (refcount_dec_and_test(&stack->count)) depot_free_stack(stack); } @@ -802,6 +2974,16 @@ void stack_depot_print(depot_stack_handle_t stack) unsigned long *entries; unsigned int nr_entries; + if (__stack_depot_trie_leaf_id(stack)) { + unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; + const unsigned int max_entries = ARRAY_SIZE(trie_entries); + + nr_entries = __stack_depot_trie_fetch_handle_into(stack, trie_entries, max_entries); + if (nr_entries) + stack_trace_print(trie_entries, nr_entries, 0); + return; + } + nr_entries = stack_depot_fetch(stack, &entries); if (nr_entries > 0) stack_trace_print(entries, nr_entries, 0); @@ -814,6 +2996,16 @@ int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, unsigned long *entries; unsigned int nr_entries; + if (__stack_depot_trie_leaf_id(handle)) { + unsigned long trie_entries[CONFIG_STACKDEPOT_MAX_FRAMES]; + const unsigned int max_entries = ARRAY_SIZE(trie_entries); + + nr_entries = __stack_depot_trie_fetch_handle_into(handle, + trie_entries, max_entries); + return nr_entries ? stack_trace_snprint(buf, size, trie_entries, + nr_entries, spaces) : 0; + } + nr_entries = stack_depot_fetch(handle, &entries); return nr_entries ? stack_trace_snprint(buf, size, entries, nr_entries, spaces) : 0; diff --git a/lib/tests/Makefile b/lib/tests/Makefile index f7460831cfdd4..1d0954575ad5d 100644 --- a/lib/tests/Makefile +++ b/lib/tests/Makefile @@ -40,6 +40,7 @@ obj-$(CONFIG_SCANF_KUNIT_TEST) += scanf_kunit.o obj-$(CONFIG_SEQ_BUF_KUNIT_TEST) += seq_buf_kunit.o obj-$(CONFIG_SIPHASH_KUNIT_TEST) += siphash_kunit.o obj-$(CONFIG_SLUB_KUNIT_TEST) += slub_kunit.o +obj-$(CONFIG_STACKDEPOT_KUNIT_TEST) += stackdepot_kunit.o obj-$(CONFIG_TEST_SORT) += test_sort.o CFLAGS_stackinit_kunit.o += $(call cc-disable-warning, switch-unreachable) obj-$(CONFIG_STACKINIT_KUNIT_TEST) += stackinit_kunit.o diff --git a/lib/tests/stackdepot_kunit.c b/lib/tests/stackdepot_kunit.c new file mode 100644 index 0000000000000..f9aaf2d72f25a --- /dev/null +++ b/lib/tests/stackdepot_kunit.c @@ -0,0 +1,401 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef CONFIG_ARM64 +#include + +static unsigned long stackdepot_arm64_frame(long offset) +{ + return (unsigned long)((long)_text + offset); +} +#endif + +static void stackdepot_fetch_into_roundtrip(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800010000UL, + 0x1234567800020000UL, + 0x1234567800030000UL, + }; + unsigned long exact[ARRAY_SIZE(entries)] = {}; + unsigned long fetched[ARRAY_SIZE(entries) + 1] = { + [ARRAY_SIZE(entries)] = 0xa5a5a5a5a5a5a5a5UL, + }; + unsigned long expected_tail = fetched[ARRAY_SIZE(entries)]; + depot_stack_handle_t handle; + unsigned int nr_entries; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + nr_entries = stack_depot_fetch_into(handle, exact, ARRAY_SIZE(exact)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, exact, entries, sizeof(entries)); + + nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); + KUNIT_EXPECT_EQ(test, fetched[ARRAY_SIZE(entries)], expected_tail); +} + +static void stackdepot_fetch_into_rejects_bad_inputs(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800110000UL, + 0x1234567800120000UL, + 0x1234567800130000UL, + }; + unsigned long fetched[ARRAY_SIZE(entries)] = { + 0xa1a1a1a1a1a1a1a1UL, + 0xb2b2b2b2b2b2b2b2UL, + 0xc3c3c3c3c3c3c3c3UL, + }; + unsigned long expected[ARRAY_SIZE(fetched)]; + depot_stack_handle_t handle; + unsigned int nr_entries; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + memcpy(expected, fetched, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(0, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(0, NULL, 0); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + + nr_entries = stack_depot_fetch_into(handle, NULL, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(handle, fetched, 0); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); + + nr_entries = stack_depot_fetch_into(handle, fetched, + ARRAY_SIZE(fetched) - 1); + KUNIT_EXPECT_EQ(test, nr_entries, 0U); + KUNIT_EXPECT_MEMEQ(test, fetched, expected, sizeof(expected)); +} + +static depot_stack_handle_t save_countable(unsigned long *entries, unsigned int nr) +{ + depot_flags_t flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE; + + return stack_depot_save_flags(entries, nr, GFP_KERNEL, flags); +} + +static void stackdepot_countable_flag_roundtrip(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800210000UL, + 0x1234567800220000UL, + 0x1234567800230000UL, + }; + unsigned long fetched[ARRAY_SIZE(entries)] = {}; + depot_stack_handle_t handle; + unsigned int nr_entries; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = save_countable(entries, ARRAY_SIZE(entries)); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + nr_entries = stack_depot_fetch_into(handle, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); +} + +static depot_stack_handle_t save_noalloc(unsigned long *entries, unsigned int nr) +{ + gfp_t no_spin = GFP_NOWAIT & ~__GFP_RECLAIM; + + return stack_depot_save_flags(entries, nr, no_spin, 0); +} + +static void stackdepot_save_flags_public(struct kunit *test) +{ + unsigned long entries[] = { 0x501000UL, 0x502000UL, 0x503000UL }; + unsigned long get_entries[] = { 0x601000UL, 0x602000UL }; + unsigned long count_entries[] = { 0x603000UL, 0x604000UL }; + unsigned long noalloc_entries[] = { 0x701000UL, 0x702000UL }; + unsigned long fetched[ARRAY_SIZE(entries)] = {}; + depot_stack_handle_t noalloc_handle; + depot_stack_handle_t truncated_handle; + depot_stack_handle_t overlong_handle; + depot_stack_handle_t count_handle; + depot_stack_handle_t plain_handle; + depot_stack_handle_t get_handle; + depot_stack_handle_t again; + depot_stack_handle_t extra; + depot_flags_t flags; + unsigned long *overlong_fetched; + unsigned long *overlong_entries; + unsigned int noalloc_nr = ARRAY_SIZE(noalloc_entries); + unsigned int overlong_nr = CONFIG_STACKDEPOT_MAX_FRAMES + 1; + unsigned int truncated_nr = CONFIG_STACKDEPOT_MAX_FRAMES; + unsigned int nr_entries; + size_t overlong_size; + unsigned int i; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + overlong_entries = kunit_kcalloc(test, overlong_nr, + sizeof(*overlong_entries), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, overlong_entries); + overlong_fetched = kunit_kcalloc(test, CONFIG_STACKDEPOT_MAX_FRAMES, + sizeof(*overlong_fetched), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, overlong_fetched); + for (i = 0; i < overlong_nr; i++) + overlong_entries[i] = 0x800000UL + i * 0x1000UL; + + plain_handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, plain_handle, (depot_stack_handle_t)0); + again = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_EXPECT_EQ(test, again, plain_handle); + + nr_entries = stack_depot_fetch_into(plain_handle, fetched, + ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); + + noalloc_handle = save_noalloc(entries, ARRAY_SIZE(entries)); + KUNIT_EXPECT_EQ(test, noalloc_handle, plain_handle); + noalloc_handle = save_noalloc(noalloc_entries, noalloc_nr); + if (noalloc_handle) { + unsigned long noalloc_fetched[ARRAY_SIZE(noalloc_entries)] = {}; + + nr_entries = stack_depot_fetch_into(noalloc_handle, noalloc_fetched, + ARRAY_SIZE(noalloc_fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, noalloc_nr); + KUNIT_EXPECT_MEMEQ(test, noalloc_fetched, noalloc_entries, + sizeof(noalloc_entries)); + } + + flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; + get_handle = stack_depot_save_flags(get_entries, ARRAY_SIZE(get_entries), + GFP_KERNEL, flags); + KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); + stack_depot_put(get_handle); + + flags = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE; + count_handle = stack_depot_save_flags(count_entries, + ARRAY_SIZE(count_entries), + GFP_KERNEL, flags); + KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); + + overlong_handle = stack_depot_save(overlong_entries, overlong_nr, + GFP_KERNEL); + KUNIT_ASSERT_NE(test, overlong_handle, (depot_stack_handle_t)0); + nr_entries = stack_depot_fetch_into(overlong_handle, overlong_fetched, + CONFIG_STACKDEPOT_MAX_FRAMES); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)CONFIG_STACKDEPOT_MAX_FRAMES); + overlong_size = CONFIG_STACKDEPOT_MAX_FRAMES * sizeof(*overlong_entries); + KUNIT_EXPECT_MEMEQ(test, overlong_fetched, overlong_entries, overlong_size); + truncated_handle = stack_depot_save(overlong_entries, truncated_nr, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, truncated_handle, overlong_handle); + + extra = stack_depot_set_extra_bits(plain_handle, 7); + KUNIT_ASSERT_NE(test, extra, (depot_stack_handle_t)0); + KUNIT_EXPECT_EQ(test, stack_depot_get_extra_bits(extra), 7U); + memset(fetched, 0, sizeof(fetched)); + nr_entries = stack_depot_fetch_into(extra, fetched, ARRAY_SIZE(fetched)); + KUNIT_EXPECT_EQ(test, nr_entries, (unsigned int)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, fetched, entries, sizeof(entries)); +} + +static void stackdepot_snprint_public(struct kunit *test) +{ + unsigned long entries[] = { 0x1000UL, 0x2000UL, 0x3000UL }; + char expected[256]; + char actual[256]; + depot_stack_handle_t handle; + unsigned int expected_len; + int actual_len; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_KERNEL); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + expected_len = stack_trace_snprint(expected, sizeof(expected), entries, + ARRAY_SIZE(entries), 2); + actual_len = stack_depot_snprint(handle, actual, sizeof(actual), 2); + KUNIT_EXPECT_EQ(test, actual_len, (int)expected_len); + KUNIT_EXPECT_STREQ(test, actual, expected); +} + +static void stackdepot_get_stack_record(struct kunit *test) +{ + unsigned long entries[] = { + 0x1234567800310000UL, + 0x1234567800320000UL, + 0x1234567800330000UL, + }; + struct stack_record *record; + depot_stack_handle_t handle; + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + handle = save_countable(entries, ARRAY_SIZE(entries)); + KUNIT_ASSERT_NE(test, handle, (depot_stack_handle_t)0); + + record = __stack_depot_get_stack_record(handle); + KUNIT_ASSERT_NOT_NULL(test, record); + KUNIT_EXPECT_EQ(test, record->size, (u16)ARRAY_SIZE(entries)); + KUNIT_EXPECT_MEMEQ(test, record->entries, entries, sizeof(entries)); +} + +static void stackdepot_countable_does_not_alias_other_modes(struct kunit *test) +{ + unsigned long plain_entries[] = { + 0x1234567800410000UL, + 0x1234567800420000UL, + 0x1234567800430000UL, + }; + unsigned long get_entries[] = { + 0x1234567800510000UL, + 0x1234567800520000UL, + 0x1234567800530000UL, + }; + depot_flags_t get = STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET; + struct stack_record *record; + depot_stack_handle_t count_handle; + depot_stack_handle_t plain_handle; + depot_stack_handle_t get_handle; + unsigned int get_nr = ARRAY_SIZE(get_entries); + unsigned int plain_nr = ARRAY_SIZE(plain_entries); + + KUNIT_ASSERT_EQ(test, stack_depot_init(), 0); + + plain_handle = stack_depot_save(plain_entries, plain_nr, GFP_KERNEL); + KUNIT_ASSERT_NE(test, plain_handle, (depot_stack_handle_t)0); + count_handle = save_countable(plain_entries, plain_nr); + KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); + record = __stack_depot_get_stack_record(count_handle); + KUNIT_ASSERT_NOT_NULL(test, record); + KUNIT_EXPECT_MEMEQ(test, record->entries, plain_entries, + sizeof(plain_entries)); + + get_handle = stack_depot_save_flags(get_entries, get_nr, GFP_KERNEL, get); + KUNIT_ASSERT_NE(test, get_handle, (depot_stack_handle_t)0); + count_handle = save_countable(get_entries, get_nr); + KUNIT_ASSERT_NE(test, count_handle, (depot_stack_handle_t)0); + record = __stack_depot_get_stack_record(count_handle); + KUNIT_ASSERT_NOT_NULL(test, record); + KUNIT_EXPECT_MEMEQ(test, record->entries, get_entries, sizeof(get_entries)); + + stack_depot_put(get_handle); +} + +static void stackdepot_frame_raw_fallback(struct kunit *test) +{ + unsigned long frame = 0xffff888000001000UL; + bool compressed; + u32 payload = 0xfeedbeef; + +#ifdef CONFIG_ARM64 + if ((unsigned long)_text <= ULONG_MAX - ((unsigned long)S32_MAX + 1UL)) + frame = (unsigned long)_text + (unsigned long)S32_MAX + 1UL; + else + frame = stackdepot_arm64_frame((long)S32_MIN - 1L); +#endif + + compressed = arch_stack_depot_frame_try_compress(frame, &payload); + KUNIT_EXPECT_FALSE(test, compressed); + KUNIT_EXPECT_EQ(test, payload, (u32)0xfeedbeef); +} + +#ifdef CONFIG_X86_64 +static void stackdepot_frame_x86_64(struct kunit *test) +{ + unsigned long direct_map = 0xffff888000001000UL; + unsigned long frame = 0xffffffff81234567UL; + unsigned long out; + bool compressed; + u32 low; + + compressed = arch_stack_depot_frame_try_compress(frame, &low); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, low, (u32)0x81234567); + arch_stack_depot_frame_decompress(low, &out); + KUNIT_EXPECT_EQ(test, out, frame); + + compressed = arch_stack_depot_frame_try_compress(direct_map, &low); + KUNIT_EXPECT_FALSE(test, compressed); +} +#endif /* CONFIG_X86_64 */ + +#ifdef CONFIG_ARM64 +static void stackdepot_frame_arm64(struct kunit *test) +{ + long negative_offset = S32_MIN; + long positive_offset = S32_MAX; + long offset = 0x123456; + unsigned long frame = stackdepot_arm64_frame(offset); + unsigned long out; + bool compressed; + u32 payload; + + compressed = arch_stack_depot_frame_try_compress(frame, &payload); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, payload, (u32)(s32)offset); + arch_stack_depot_frame_decompress(payload, &out); + KUNIT_EXPECT_EQ(test, out, frame); + + frame = stackdepot_arm64_frame(negative_offset); + compressed = arch_stack_depot_frame_try_compress(frame, &payload); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, payload, (u32)(s32)negative_offset); + arch_stack_depot_frame_decompress(payload, &out); + KUNIT_EXPECT_EQ(test, out, frame); + + frame = stackdepot_arm64_frame(positive_offset); + compressed = arch_stack_depot_frame_try_compress(frame, &payload); + KUNIT_EXPECT_TRUE(test, compressed); + KUNIT_EXPECT_EQ(test, payload, (u32)(s32)positive_offset); + arch_stack_depot_frame_decompress(payload, &out); + KUNIT_EXPECT_EQ(test, out, frame); +} +#endif /* CONFIG_ARM64 */ + +static struct kunit_case stackdepot_test_cases[] = { + KUNIT_CASE(stackdepot_fetch_into_roundtrip), + KUNIT_CASE(stackdepot_fetch_into_rejects_bad_inputs), + KUNIT_CASE(stackdepot_countable_flag_roundtrip), + KUNIT_CASE(stackdepot_save_flags_public), + KUNIT_CASE(stackdepot_snprint_public), + KUNIT_CASE(stackdepot_get_stack_record), + KUNIT_CASE(stackdepot_countable_does_not_alias_other_modes), + KUNIT_CASE(stackdepot_frame_raw_fallback), +#ifdef CONFIG_X86_64 + KUNIT_CASE(stackdepot_frame_x86_64), +#endif +#ifdef CONFIG_ARM64 + KUNIT_CASE(stackdepot_frame_arm64), +#endif + {} +}; + +static struct kunit_suite stackdepot_test_suite = { + .name = "stackdepot", + .test_cases = stackdepot_test_cases, +}; + +kunit_test_suite(stackdepot_test_suite); + +MODULE_DESCRIPTION("KUnit tests for stack depot"); +MODULE_AUTHOR("Caleb Kan "); +MODULE_LICENSE("GPL"); diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 1ac56ceb29b6b..d7fbcc9f121c9 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -364,10 +364,10 @@ static void print_unreferenced(struct seq_file *seq, struct kmemleak_object *object) { int i; - unsigned long *entries; + unsigned long entries[MAX_TRACE]; unsigned int nr_entries; - nr_entries = stack_depot_fetch(object->trace_handle, &entries); + nr_entries = stack_depot_fetch_into(object->trace_handle, entries, ARRAY_SIZE(entries)); warn_or_seq_printf(seq, "unreferenced object%s 0x%08lx (size %zu):\n", __object_type_str(object), object->pointer, object->size); diff --git a/mm/kmsan/kmsan_test.c b/mm/kmsan/kmsan_test.c index 902ec48b1e3e6..123bcc9565721 100644 --- a/mm/kmsan/kmsan_test.c +++ b/mm/kmsan/kmsan_test.c @@ -610,7 +610,7 @@ static void test_long_origin_chain(struct kunit *test) */ static void test_stackdepot_roundtrip(struct kunit *test) { - unsigned long src_entries[16], *dst_entries; + unsigned long src_entries[16], dst_entries[16]; unsigned int src_nentries, dst_nentries; EXPECTATION_NO_REPORT(expect); depot_stack_handle_t handle; @@ -621,11 +621,10 @@ static void test_stackdepot_roundtrip(struct kunit *test) stack_trace_save(src_entries, ARRAY_SIZE(src_entries), 1); handle = stack_depot_save(src_entries, src_nentries, GFP_KERNEL); stack_depot_print(handle); - dst_nentries = stack_depot_fetch(handle, &dst_entries); + dst_nentries = stack_depot_fetch_into(handle, dst_entries, ARRAY_SIZE(dst_entries)); KUNIT_EXPECT_TRUE(test, src_nentries == dst_nentries); - kmsan_check_memory((void *)dst_entries, - sizeof(*dst_entries) * dst_nentries); + kmsan_check_memory(dst_entries, sizeof(*dst_entries) * dst_nentries); KUNIT_EXPECT_TRUE(test, report_matches(&expect)); } diff --git a/mm/kmsan/report.c b/mm/kmsan/report.c index d6853ce089541..8d7b86b38d84c 100644 --- a/mm/kmsan/report.c +++ b/mm/kmsan/report.c @@ -85,7 +85,8 @@ static char *pretty_descr(char *descr) void kmsan_print_origin(depot_stack_handle_t origin) { - unsigned long *entries = NULL, *chained_entries = NULL; + unsigned long entries[KMSAN_STACK_DEPTH]; + const unsigned int max_entries = ARRAY_SIZE(entries); unsigned int nr_entries, chained_nr_entries, skipnr; void *pc1 = NULL, *pc2 = NULL; depot_stack_handle_t head; @@ -97,7 +98,7 @@ void kmsan_print_origin(depot_stack_handle_t origin) return; while (true) { - nr_entries = stack_depot_fetch(origin, &entries); + nr_entries = stack_depot_fetch_into(origin, entries, max_entries); depth = kmsan_depth_from_eb(stack_depot_get_extra_bits(origin)); magic = nr_entries ? entries[0] : 0; if ((nr_entries == 4) && (magic == KMSAN_ALLOCA_MAGIC_ORIGIN)) { @@ -122,16 +123,14 @@ void kmsan_print_origin(depot_stack_handle_t origin) head = entries[1]; origin = entries[2]; pr_err("Uninit was stored to memory at:\n"); + /* Reuse entries after saving head and origin above. */ chained_nr_entries = - stack_depot_fetch(head, &chained_entries); - kmsan_internal_unpoison_memory( - chained_entries, - chained_nr_entries * sizeof(*chained_entries), - /*checked*/ false); - skipnr = get_stack_skipnr(chained_entries, - chained_nr_entries); - stack_trace_print(chained_entries + skipnr, - chained_nr_entries - skipnr, 0); + stack_depot_fetch_into(head, entries, max_entries); + if (chained_nr_entries) { + skipnr = get_stack_skipnr(entries, chained_nr_entries); + stack_trace_print(entries + skipnr, + chained_nr_entries - skipnr, 0); + } pr_err("\n"); continue; } diff --git a/mm/page_owner.c b/mm/page_owner.c index bc26764142ba5..9f78413fee60a 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -92,7 +92,8 @@ static __always_inline depot_stack_handle_t create_dummy_stack(void) unsigned int nr_entries; nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); - return stack_depot_save(entries, nr_entries, GFP_KERNEL); + return stack_depot_save_flags(entries, nr_entries, GFP_KERNEL, + STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE); } static noinline void register_dummy_stack(void) @@ -154,7 +155,8 @@ static noinline depot_stack_handle_t save_stack(gfp_t flags) set_current_in_page_owner(); nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); - handle = stack_depot_save(entries, nr_entries, flags); + handle = stack_depot_save_flags(entries, nr_entries, flags, + STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_COUNTABLE); if (!handle) handle = failure_handle; unset_current_in_page_owner(); diff --git a/mm/slub.c b/mm/slub.c index 5fdec3b837060..79d5dac7617a1 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -8148,12 +8148,12 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) #ifdef CONFIG_STACKDEPOT { depot_stack_handle_t handle; - unsigned long *entries; + unsigned long entries[TRACK_ADDRS_COUNT]; unsigned int nr_entries; handle = READ_ONCE(trackp->handle); if (handle) { - nr_entries = stack_depot_fetch(handle, &entries); + nr_entries = stack_depot_fetch_into(handle, entries, ARRAY_SIZE(entries)); for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) kpp->kp_stack[i] = (void *)entries[i]; } @@ -8161,7 +8161,7 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) trackp = get_track(s, objp, TRACK_FREE); handle = READ_ONCE(trackp->handle); if (handle) { - nr_entries = stack_depot_fetch(handle, &entries); + nr_entries = stack_depot_fetch_into(handle, entries, ARRAY_SIZE(entries)); for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) kpp->kp_free_stack[i] = (void *)entries[i]; } @@ -9931,12 +9931,14 @@ static int slab_debugfs_show(struct seq_file *seq, void *v) #ifdef CONFIG_STACKDEPOT { depot_stack_handle_t handle; - unsigned long *entries; + unsigned long entries[TRACK_ADDRS_COUNT]; unsigned int nr_entries, j; handle = READ_ONCE(l->handle); if (handle) { - nr_entries = stack_depot_fetch(handle, &entries); + nr_entries = + stack_depot_fetch_into(handle, entries, + ARRAY_SIZE(entries)); seq_puts(seq, "\n"); for (j = 0; j < nr_entries; j++) seq_printf(seq, " %pS\n", (void *)entries[j]);