From 203c27652d3093c5723209f56ca3e3bdf6c793e9 Mon Sep 17 00:00:00 2001 From: majianpeng Date: Mon, 14 Sep 2026 15:10:44 +0800 Subject: [PATCH] build: pick the BTF kernel by capability, not by directory name GEN-BTF decided which kernel to pull ceph.ko's BTF from with: kver=$(find /lib/modules -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \ | sort -V | tail -1) i.e. "the highest-sorting directory directly under /lib/modules is the newest kernel". That assumes every such directory is named after a kernel release. Distributions are not obliged to comply. Kylin Linux Advanced Server V11 (Swan25) ships /lib/modules/ksaf: a kernel-release-independent directory holding Kylin's own LSM modules (ksaf_main.ko from ksaf-main-module, kysec_exectl.ko from kysec2-exectl-module). It is deliberately kept outside /lib/modules// so that the dracut module 98ksaf_main can copy one fixed path into the initramfs for every kernel, and the security modules get loaded as early as possible. It contains no ceph.ko. GNU `sort -V` ranks a run of letters ABOVE a run of digits, so any name that starts with a letter sorts above every real version number: $ find /lib/modules -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort -V 5.4.241-24.0017.26 6.6.0-32.12.v2505.ky11.x86_64 6.6.0-32.21.v2505.ky11.x86_64 6.6.0-32.22.v2505.ky11.x86_64 6.6.0-32.24.v2505.ky11.x86_64 ksaf Hence `tail -1` returns ksaf, the recipe looks for ceph.ko under /lib/modules/ksaf, finds nothing, and aborts the build: GEN-BTF .../src/ceph_btf_local.h No ceph.ko* found under /lib/modules/ksaf make: *** [Makefile:164: .../src/ceph_btf_local.h] Error 1 ceph_btf_local.h is a prerequisite of the .bpf.o targets, so the build dies before a single object is compiled, which makes the error look unrelated to kernel discovery. The failure is also conditional: when the generated header is already present, make skips GEN-BTF and the bug stays hidden until a `make clean`, `git clean`, or a fresh checkout removes it. Fix the selection to match on capability instead of on the directory name: a candidate must actually contain a ceph.ko*, which excludes any non-kernel directory no matter what it is called. Prefer the running kernel when it is installed and ships ceph.ko*, because - tracing attaches to the live kernel, and - the base BTF fallback /sys/kernel/btf/vmlinux is only valid for the running kernel, so falling back to a different (e.g. newer, not yet rebooted) kernel would otherwise fail later with "No usable vmlinux/base BTF found". Verified on Kylin V11, kernel 6.6.0-32.24.v2505.ky11.x86_64: Using installed kernel: 6.6.0-32.24.v2505.ky11.x86_64 Found ceph kernel module: /lib/modules/6.6.0-32.24.v2505.ky11.x86_64/kernel/fs/ceph/ceph.ko.xz Using base BTF source: /sys/kernel/btf/vmlinux --- Makefile | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b6c0ac9..480938a 100755 --- a/Makefile +++ b/Makefile @@ -159,12 +159,58 @@ $(BPFTOOL): | $(BPFTOOL_OUTPUT) # Generate Ceph BTF header +# +# Which kernel do we take ceph.ko's BTF from? +# +# "The newest kernel under /lib/modules" is not a safe answer, because it +# assumes every directory directly under /lib/modules is named after a kernel +# release. Distributions are not obliged to comply. Kylin Linux Advanced Server +# V11 ships /lib/modules/ksaf: a kernel-release-independent directory holding +# Kylin's own LSM modules (ksaf_main.ko from ksaf-main-module, kysec_exectl.ko +# from kysec2-exectl-module). It is deliberately kept out of +# /lib/modules// so that the dracut module 98ksaf_main can copy one +# fixed path into the initramfs for every kernel, and the security modules can +# be loaded as early as possible. There is no ceph.ko in it. +# +# GNU `sort -V` ranks a run of letters ABOVE a run of digits, so a name that +# starts with a letter sorts above every real version number: +# +# 5.4.241-24.0017.26 +# 6.6.0-32.12.v2505.ky11.x86_64 +# 6.6.0-32.21.v2505.ky11.x86_64 +# 6.6.0-32.22.v2505.ky11.x86_64 +# 6.6.0-32.24.v2505.ky11.x86_64 <- the running kernel +# ksaf <- ... so `tail -1` returns this one +# +# This recipe then searched /lib/modules/ksaf for ceph.ko and aborted with +# "No ceph.ko* found under /lib/modules/ksaf". Because ceph_btf_local.h is a +# prerequisite of the .bpf.o targets, that killed the build before a single +# object was compiled -- which makes the failure look unrelated to kernel +# discovery. It is only observable when the header must be regenerated, so it +# stays hidden until a `make clean`, `git clean`, or fresh checkout removes it. +# +# Select by CAPABILITY instead: a candidate must actually contain a ceph.ko*, +# which excludes any non-kernel directory whatever it happens to be called. +# Prefer the running kernel when it qualifies, because tracing attaches to the +# live kernel, and the base BTF fallback /sys/kernel/btf/vmlinux below is only +# valid for the running kernel. $(OSDTRACE_SRC)/ceph_btf_local.h: | $(OUTPUT) $(BPFTOOL) @$(call msg,GEN-BTF,$@) @set -eu; \ src=""; tmp=""; kver=""; kdir=""; base_btf=""; \ - kver=$$(find /lib/modules -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | sort -V | tail -1); \ - [ -n "$$kver" ] || { echo "No installed kernels found under /lib/modules" >&2; exit 1; }; \ + running_kver=$$(uname -r); \ + if [ -d "/lib/modules/$$running_kver" ] && \ + [ -n "$$(find "/lib/modules/$$running_kver" -type f -name 'ceph.ko*' -print -quit 2>/dev/null)" ]; then \ + kver="$$running_kver"; \ + fi; \ + if [ -z "$$kver" ]; then \ + kver=$$(for d in /lib/modules/*/; do \ + [ -d "$$d" ] || continue; \ + [ -n "$$(find "$$d" -type f -name 'ceph.ko*' -print -quit 2>/dev/null)" ] || continue; \ + basename "$$d"; \ + done | sort -V | tail -1); \ + fi; \ + [ -n "$$kver" ] || { echo "No installed kernel under /lib/modules provides a ceph.ko* module" >&2; exit 1; }; \ kdir="/lib/modules/$$kver"; \ CEPH_KO=$$(find "$$kdir" -type f -name 'ceph.ko*' 2>/dev/null | head -1); \ [ -n "$$CEPH_KO" ] || { echo "No ceph.ko* found under $$kdir" >&2; exit 1; }; \