diff --git a/.github/workflows/kpm-matrix.yml b/.github/workflows/kpm-matrix.yml new file mode 100644 index 00000000..40c68f9e --- /dev/null +++ b/.github/workflows/kpm-matrix.yml @@ -0,0 +1,137 @@ +name: NoMount KPM — compile matrix + +on: + push: + branches: [dev] + paths: + - 'kernel/kpm/**' + - 'kernel/src/**' + - '.github/workflows/kpm-matrix.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: kpm-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: ${{ matrix.ver }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - { ver: '4.9', kref: 'deprecated/android-4.9-q', tc: gcc } + - { ver: '4.14', kref: 'deprecated/android-4.14-stable', tc: gcc } + - { ver: '4.19', kref: 'deprecated/android-4.19-stable', tc: gcc } + - { ver: '5.4', kref: 'android11-5.4', tc: gcc } + - { ver: '5.10', kref: 'android12-5.10', tc: clang } + - { ver: '5.15', kref: 'android13-5.15', tc: clang } + - { ver: '6.1', kref: 'android14-6.1', tc: clang } + - { ver: '6.6', kref: 'android15-6.6', tc: clang } + steps: + - uses: actions/checkout@v4 + with: { path: repo } + + - name: Toolchain + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq gcc-aarch64-linux-gnu bc flex bison libssl-dev libelf-dev \ + clang lld llvm + echo "clang: $(clang --version | head -1)" + + - name: KernelPatch headers + run: git clone --depth 1 https://github.com/741afb7/KPatch-Next-EXP kp + + - name: Cache kernel clone + id: kcache + uses: actions/cache@v4 + with: + path: ksrc + key: kpm-ksrc-${{ matrix.ver }}-${{ matrix.kref }}-v1 + + - name: Clone kernel ${{ matrix.kref }} + if: steps.kcache.outputs.cache-hit != 'true' + run: | + git clone --depth 1 -b "${{ matrix.kref }}" \ + https://android.googlesource.com/kernel/common ksrc + head -3 ksrc/Makefile | tr '\n' ' ' + + - name: Configure kernel headers + run: | + set -e -o pipefail + cd ksrc + if [ "${{ matrix.tc }}" = "clang" ]; then + M="ARCH=arm64 LLVM=1 LLVM_IAS=1 CROSS_COMPILE=aarch64-linux-gnu- KCFLAGS=-Wno-error" + else + M="ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- KCFLAGS=-Wno-error" + fi + if [ -f arch/arm64/configs/gki_defconfig ]; then + make $M gki_defconfig 2>&1 | tail -40 + else + make $M defconfig 2>&1 | tail -40 + fi + ./scripts/config --disable KASAN --disable FORTIFY_SOURCE \ + --disable DEBUG_LIST --disable PAGE_PINNER \ + --disable LOCKDEP --disable PROVE_LOCKING \ + --disable DEBUG_LOCK_ALLOC --disable TRACE_IRQFLAGS \ + --disable DEBUG_ATOMIC_SLEEP || true + make $M olddefconfig 2>&1 | tail -20 + make $M modules_prepare 2>&1 | tail -30 + test -f include/generated/autoconf.h || { echo "::error::prepare produced no autoconf.h"; exit 1; } + + - name: Build the engine object + run: | + set -o pipefail + cd repo/kernel/kpm + if [ "${{ matrix.tc }}" = "clang" ]; then + CCX="clang --target=aarch64-linux-gnu" + else + CCX="aarch64-linux-gnu-gcc" + fi + make CC="$CCX" TARGET_COMPILE=aarch64-linux-gnu- \ + KP_DIR="$GITHUB_WORKSPACE/kp" \ + KERNEL_DIR="$GITHUB_WORKSPACE/ksrc" \ + nm_engine.o 2>&1 | tee /tmp/kpm.log + ls -l nm_engine.o && echo "ENGINE_OK ${{ matrix.ver }}" + + - name: Undefined symbols + run: | + cd repo/kernel/kpm + U=$(aarch64-linux-gnu-nm -u nm_engine.o | awk '{print $2}' | sort -u | grep -v '^nm_kpm_sym$' || true) + N=$(printf '%s' "$U" | grep -c . || true) + echo "::notice::${{ matrix.ver }} undefined=$N" + [ "$N" = "0" ] && exit 0 + printf '%s\n' "$U" + if [ "${{ matrix.tc }}" = "clang" ]; then CCX="clang --target=aarch64-linux-gnu"; else CCX="aarch64-linux-gnu-gcc"; fi + make CC="$CCX" TARGET_COMPILE=aarch64-linux-gnu- \ + KP_DIR="$GITHUB_WORKSPACE/kp" KERNEL_DIR="$GITHUB_WORKSPACE/ksrc" \ + COMMON_EXTRA=-ffunction-sections clean >/dev/null 2>&1 || true + $CCX -O2 -fno-stack-protector -fno-pic -ffreestanding -w -ffunction-sections \ + -mgeneral-regs-only -nostdinc -isystem $($CCX -print-file-name=include) \ + -I$GITHUB_WORKSPACE/ksrc/include -I$GITHUB_WORKSPACE/ksrc/include/uapi \ + -I$GITHUB_WORKSPACE/ksrc/include/generated -I$GITHUB_WORKSPACE/ksrc/include/generated/uapi \ + -I$GITHUB_WORKSPACE/ksrc/arch/arm64/include -I$GITHUB_WORKSPACE/ksrc/arch/arm64/include/uapi \ + -I$GITHUB_WORKSPACE/ksrc/arch/arm64/include/generated \ + -I$GITHUB_WORKSPACE/ksrc/arch/arm64/include/generated/uapi \ + -include $GITHUB_WORKSPACE/ksrc/include/linux/kconfig.h \ + -D__KERNEL__ -DMODULE -DKBUILD_MODNAME='"nomount"' \ + -c -o /tmp/attr.o nm_engine.c 2>/dev/null || true + if [ -f /tmp/attr.o ]; then + for sym in $U; do + callers=$(aarch64-linux-gnu-objdump -r /tmp/attr.o 2>/dev/null \ + | awk -v s="$sym" '/^RELOCATION RECORDS FOR/{sec=$0} $0 ~ ("[ \t]"s"$"){print sec}' \ + | sed 's/.*\[\.text\.//; s/\]://' | sort -u | tr '\n' ' ') + if [ -n "$callers" ]; then + echo " $sym <- $callers" + else + echo " $sym <- (no .text ref) raw records:" + aarch64-linux-gnu-objdump -r /tmp/attr.o 2>/dev/null \ + | grep -B40 "[ \t]$sym$" | grep "RELOCATION RECORDS" | tail -3 | sed 's/^/ /' + fi + done + fi + echo "::warning::${{ matrix.ver }} has $N unresolved symbol(s)" diff --git a/kernel/kpm/Makefile b/kernel/kpm/Makefile new file mode 100644 index 00000000..a88dd154 --- /dev/null +++ b/kernel/kpm/Makefile @@ -0,0 +1,47 @@ +ifndef TARGET_COMPILE + $(error TARGET_COMPILE not set, e.g. aarch64-linux-android- or aarch64-linux-gnu-) +endif +ifndef KP_DIR + $(error KP_DIR not set: checkout of KernelPatch / KPatch-Next-EXP) +endif +ifndef KERNEL_DIR + $(error KERNEL_DIR not set: configured kernel source for the TARGET KMI (<= 6.6)) +endif + +CC ?= $(TARGET_COMPILE)gcc +LD ?= $(TARGET_COMPILE)ld + +KP_INCLUDE_DIRS := . include patch/include linux/include linux/arch/arm64/include \ + linux/tools/arch/arm64/include +KP_FLAGS := $(foreach d,$(KP_INCLUDE_DIRS),-I$(KP_DIR)/kernel/$(d)) + +CC_INCLUDE := $(shell $(CC) -print-file-name=include) + +K_FLAGS := -nostdinc -isystem $(CC_INCLUDE) -I$(KERNEL_DIR)/include -I$(KERNEL_DIR)/include/uapi \ + -I$(KERNEL_DIR)/include/generated -I$(KERNEL_DIR)/include/generated/uapi \ + -I$(KERNEL_DIR)/arch/arm64/include -I$(KERNEL_DIR)/arch/arm64/include/uapi \ + -I$(KERNEL_DIR)/arch/arm64/include/generated \ + -I$(KERNEL_DIR)/arch/arm64/include/generated/uapi \ + -include $(KERNEL_DIR)/include/linux/kconfig.h \ + -D__KERNEL__ -DMODULE -DKBUILD_MODNAME='"nomount"' + +COMMON := -O2 -fno-stack-protector -fno-pic -ffreestanding -Wall + +all: nomount.kpm + +nomount.kpm: nm_kpm_entry.o nm_engine.o + $(CC) -r -o $@ $^ + +nm_kpm_entry.o: nm_kpm_entry.c nm_kpm_syms.h + $(CC) $(COMMON) $(KP_FLAGS) -c -o $@ $< + +nm_engine.o: nm_engine.c nm_kpm_shim.h nm_kpm_syms.h ../src/nomount.c ../src/nomount.h + $(CC) $(COMMON) $(K_FLAGS) -c -o $@ $< + +.PHONY: undefined +undefined: nm_engine.o + @$(TARGET_COMPILE)nm -u $< | awk '{print $$2}' | sort -u + +.PHONY: clean +clean: + rm -f *.o *.kpm diff --git a/kernel/kpm/nm_engine.c b/kernel/kpm/nm_engine.c new file mode 100644 index 00000000..459bf9df --- /dev/null +++ b/kernel/kpm/nm_engine.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +#undef CONFIG_ARM64_LSE_ATOMICS + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nm_kpm_shim.h" + +#define NM_KPM_BUILD 1 +#include "../src/nomount.c" + +#undef memcpy +#undef memset +#undef memmove + +void *memcpy(void *d, const void *s, __kernel_size_t n) +{ + char *dp = d; const char *sp = s; + while (n--) *dp++ = *sp++; + return d; +} +void *memset(void *d, int c, __kernel_size_t n) +{ + char *dp = d; + while (n--) *dp++ = (char)c; + return d; +} +void *memmove(void *d, const void *s, __kernel_size_t n) +{ + char *dp = d; const char *sp = s; + if (dp <= sp || dp >= sp + n) return memcpy(d, s, n); + dp += n; sp += n; + while (n--) *--dp = *--sp; + return d; +} + +void *__memcpy(void *d, const void *s, __kernel_size_t n) { return memcpy(d, s, n); } +void *__memset(void *d, int c, __kernel_size_t n) { return memset(d, c, n); } +void *__memmove(void *d, const void *s, __kernel_size_t n) { return memmove(d, s, n); } + +#if defined(CONFIG_DEBUG_LIST) || defined(CONFIG_LIST_HARDENED) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 2, 0) +bool __list_add_valid_or_report(struct list_head *new, struct list_head *prev, + struct list_head *next) +#else +bool __list_add_valid(struct list_head *new, struct list_head *prev, + struct list_head *next) +#endif +{ + if (!nm_kpm_sym[NMS___list_add_valid_or_report]) + return true; + return ((bool (*)(struct list_head *, struct list_head *, struct list_head *)) + nm_kpm_sym[NMS___list_add_valid_or_report])(new, prev, next); +} +#endif + +void alt_cb_patch_nops(struct alt_instr *alt, __le32 *origptr, + __le32 *updptr, int nr_inst) +{ + if (!nm_kpm_sym[NMS_alt_cb_patch_nops]) + return; + ((void (*)(struct alt_instr *, __le32 *, __le32 *, int)) + nm_kpm_sym[NMS_alt_cb_patch_nops])(alt, origptr, updptr, nr_inst); +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 17, 0) +void __folio_put(struct folio *folio) +{ + ((void (*)(struct folio *))nm_kpm_sym[NMS___folio_put])(folio); +} +#else +void __put_page(struct page *page) +{ + ((void (*)(struct page *))nm_kpm_sym[NMS___folio_put])(page); +} +#endif + +int nm_kpm_engine_init(void) +{ + return nomount_init(); +} + +void nm_kpm_engine_exit(void) +{ +#ifdef NOMOUNT_HAS_EXIT + nomount_exit(); +#endif +} + +long nm_kpm_engine_ctl(const char *args) +{ + (void)args; + return 0; +} diff --git a/kernel/kpm/nm_kpm_entry.c b/kernel/kpm/nm_kpm_entry.c new file mode 100644 index 00000000..31ed0532 --- /dev/null +++ b/kernel/kpm/nm_kpm_entry.c @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +#include "nm_kpm_syms.h" + +KPM_NAME("nomount"); +KPM_VERSION("1.0.0"); +KPM_LICENSE("GPL v2"); +KPM_AUTHOR("NoMount"); +KPM_DESCRIPTION("hookless VFS path redirection"); + +void *nm_kpm_sym[NM_KPM_SYM_COUNT]; + +struct nm_sym_ent { + int idx; + const char *name; + const char *alt; + int optional; +}; + +static const struct nm_sym_ent nm_syms[] = { + { NMS_d_add, "d_add", 0, 0 }, + { NMS_d_drop, "d_drop", 0, 0 }, + { NMS_d_lookup, "d_lookup", 0, 0 }, + { NMS_d_splice_alias, "d_splice_alias", 0, 0 }, + { NMS_d_parent_ino, "d_parent_ino", 0, 1 }, + { NMS_dput, "dput", 0, 0 }, + { NMS_path_get, "path_get", 0, 0 }, + { NMS_path_put, "path_put", 0, 0 }, + { NMS_kern_path, "kern_path", 0, 0 }, + { NMS_full_name_hash, "full_name_hash", 0, 0 }, + { NMS_shrink_dcache_sb, "shrink_dcache_sb", 0, 1 }, + + { NMS_new_inode, "new_inode", 0, 0 }, + { NMS_iput, "iput", 0, 0 }, + { NMS_igrab, "igrab", 0, 1 }, + { NMS_clear_inode, "clear_inode", 0, 0 }, + { NMS_notify_change, "notify_change", 0, 1 }, + { NMS_truncate_inode_pages_final, "truncate_inode_pages_final", 0, 1 }, + { NMS_generic_fillattr, "generic_fillattr", 0, 0 }, + { NMS_vfs_getattr_nosec, "vfs_getattr_nosec", 0, 1 }, + + { NMS_dentry_open, "dentry_open", 0, 0 }, + { NMS_fput, "fput", 0, 0 }, + { NMS_generic_read_dir, "generic_read_dir", 0, 0 }, + { NMS_vfs_llseek, "vfs_llseek", 0, 1 }, + { NMS_vfs_copy_file_range, "vfs_copy_file_range", 0, 1 }, + { NMS_shmem_file_setup, "shmem_file_setup", 0, 1 }, + { NMS_generic_file_mmap, "generic_file_mmap", 0, 0 }, + + { NMS___vfs_getxattr, "__vfs_getxattr", 0, 1 }, + { NMS___vfs_setxattr, "__vfs_setxattr", 0, 0 }, + { NMS_xattr_full_name, "xattr_full_name", 0, 1 }, + + { NMS___kmalloc, "__kmalloc", "__kmalloc_noprof", 0 }, + { NMS_kfree, "kfree", 0, 0 }, + { NMS_kmem_cache_create, "kmem_cache_create", "__kmem_cache_create_args", 0 }, + { NMS_kmem_cache_alloc, "kmem_cache_alloc", "kmem_cache_alloc_noprof", 0 }, + { NMS_kmem_cache_free, "kmem_cache_free", 0, 0 }, + { NMS_kmem_cache_destroy, "kmem_cache_destroy", 0, 0 }, + + { NMS_idr_alloc, "idr_alloc", 0, 0 }, + { NMS_idr_find, "idr_find", 0, 0 }, + { NMS_idr_remove, "idr_remove", 0, 0 }, + { NMS_idr_get_next, "idr_get_next", 0, 0 }, + { NMS_idr_destroy, "idr_destroy", 0, 0 }, + { NMS_idr_init, "idr_init", 0, 1 }, + { NMS_rb_insert_color, "rb_insert_color", 0, 1 }, + { NMS_rb_erase, "rb_erase", 0, 0 }, + { NMS_rb_next, "rb_next", 0, 0 }, + { NMS_radix_tree_tagged, "radix_tree_tagged", 0, 0 }, + + { NMS_call_rcu, "call_rcu", 0, 0 }, + { NMS_synchronize_rcu, "synchronize_rcu", 0, 0 }, + { NMS_rcu_barrier, "rcu_barrier", 0, 1 }, + { NMS_kvfree_call_rcu, "kvfree_call_rcu", "kfree_call_rcu", 1 }, + { NMS__raw_spin_lock, "_raw_spin_lock", 0, 0 }, + { NMS___rcu_read_lock, "__rcu_read_lock", 0, 1 }, + { NMS___rcu_read_unlock, "__rcu_read_unlock", 0, 1 }, + { NMS___srcu_read_lock, "__srcu_read_lock", 0, 0 }, + { NMS___srcu_read_unlock, "__srcu_read_unlock", 0, 0 }, + { NMS_synchronize_srcu, "synchronize_srcu", 0, 0 }, + { NMS_init_srcu_struct, "init_srcu_struct", 0, 0 }, + { NMS_cleanup_srcu_struct, "cleanup_srcu_struct", 0, 1 }, + { NMS__raw_spin_unlock, "_raw_spin_unlock", 0, 0 }, + { NMS_down_read, "down_read", 0, 0 }, + { NMS_up_read, "up_read", 0, 0 }, + { NMS_down_write, "down_write", 0, 0 }, + { NMS_up_write, "up_write", 0, 0 }, + + { NMS_register_key_type, "register_key_type", 0, 0 }, + { NMS_unregister_key_type, "unregister_key_type", 0, 0 }, + + { NMS__printk, "_printk", "printk", 0 }, + { NMS_capable, "capable", 0, 0 }, + { NMS_strlen, "strlen", 0, 1 }, + { NMS_memcmp, "memcmp", 0, 1 }, + { NMS_get_user_pages_fast, "get_user_pages_fast", 0, 1 }, + { NMS___folio_put, "__folio_put", "__put_page", 0 }, + { NMS_static_key_enable, "static_key_enable", 0, 1 }, + { NMS_static_key_disable, "static_key_disable", 0, 1 }, + { NMS___list_add_valid_or_report, "__list_add_valid_or_report", "__list_add_valid", 1 }, + { NMS_alt_cb_patch_nops, "alt_cb_patch_nops", 0, 1 }, +}; + +static long nm_resolve_all(void) +{ + unsigned long i; + long missing = 0; + + for (i = 0; i < sizeof(nm_syms) / sizeof(nm_syms[0]); i++) { + const struct nm_sym_ent *e = &nm_syms[i]; + void *a = (void *)kallsyms_lookup_name(e->name); + + if (!a && e->alt) + a = (void *)kallsyms_lookup_name(e->alt); + + nm_kpm_sym[e->idx] = a; + + if (!a && !e->optional) { + logke("nomount: cannot resolve %s\n", e->name); + missing++; + } + } + return missing; +} + +static long nm_kpm_init(const char *args, const char *event, void *__user reserved) +{ + long missing; + + (void)args; (void)event; (void)reserved; + + missing = nm_resolve_all(); + if (missing) { + logke("nomount: %ld required symbol(s) unresolved, not starting\n", missing); + return -2; + } + return nm_kpm_engine_init(); +} + +static long nm_kpm_ctl0(const char *args, char *__user out_msg, int outlen) +{ + (void)out_msg; (void)outlen; + return nm_kpm_engine_ctl(args); +} + +static long nm_kpm_exit(void *__user reserved) +{ + (void)reserved; + nm_kpm_engine_exit(); + return 0; +} + +KPM_INIT(nm_kpm_init); +KPM_CTL0(nm_kpm_ctl0); +KPM_EXIT(nm_kpm_exit); diff --git a/kernel/kpm/nm_kpm_shim.h b/kernel/kpm/nm_kpm_shim.h new file mode 100644 index 00000000..7d7355c1 --- /dev/null +++ b/kernel/kpm/nm_kpm_shim.h @@ -0,0 +1,279 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _NM_KPM_SHIM_H +#define _NM_KPM_SHIM_H + +#include "nm_kpm_syms.h" + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0) +#error "KPM build targets 6.6 and older: KernelPatch does not come up on 6.12+ (boot hangs in its pagetable bring-up), so a module built for it could never load." +#endif + +#define NM_SYM(i, type) ((type)nm_kpm_sym[(i)]) + +#define d_add(d, i) NM_SYM(NMS_d_add, void (*)(struct dentry *, struct inode *))((d), (i)) +#define d_drop(d) NM_SYM(NMS_d_drop, void (*)(struct dentry *))((d)) +#define d_lookup(p, n) NM_SYM(NMS_d_lookup, struct dentry *(*)(const struct dentry *, const struct qstr *))((p), (n)) +#define d_splice_alias(i, d) NM_SYM(NMS_d_splice_alias, struct dentry *(*)(struct inode *, struct dentry *))((i), (d)) +#define d_parent_ino(d) NM_SYM(NMS_d_parent_ino, ino_t (*)(struct dentry *))((d)) +#define dput(d) NM_SYM(NMS_dput, void (*)(struct dentry *))((d)) +#define path_get(p) NM_SYM(NMS_path_get, void (*)(const struct path *))((p)) +#define path_put(p) NM_SYM(NMS_path_put, void (*)(const struct path *))((p)) +#define kern_path(n, f, p) NM_SYM(NMS_kern_path, int (*)(const char *, unsigned int, struct path *))((n), (f), (p)) +#define full_name_hash(s, n, l) NM_SYM(NMS_full_name_hash, unsigned int (*)(const void *, const char *, unsigned int))((s), (n), (l)) +#define shrink_dcache_sb(sb) NM_SYM(NMS_shrink_dcache_sb, void (*)(struct super_block *))((sb)) + +#define new_inode(sb) NM_SYM(NMS_new_inode, struct inode *(*)(struct super_block *))((sb)) +#define iput(i) NM_SYM(NMS_iput, void (*)(struct inode *))((i)) +#define igrab(i) NM_SYM(NMS_igrab, struct inode *(*)(struct inode *))((i)) +#define clear_inode(i) NM_SYM(NMS_clear_inode, void (*)(struct inode *))((i)) +#define truncate_inode_pages_final(m) \ + NM_SYM(NMS_truncate_inode_pages_final, void (*)(struct address_space *))((m)) + +#define dentry_open(p, f, c) NM_SYM(NMS_dentry_open, struct file *(*)(const struct path *, int, const struct cred *))((p), (f), (c)) +#define fput(f) NM_SYM(NMS_fput, void (*)(struct file *))((f)) + +#define xattr_full_name(h, n) NM_SYM(NMS_xattr_full_name, const char *(*)(const struct xattr_handler *, const char *))((h), (n)) + +#define kfree(p) NM_SYM(NMS_kfree, void (*)(const void *))((p)) +#define kmem_cache_create(...) ((struct kmem_cache *(*)())nm_kpm_sym[NMS_kmem_cache_create])(__VA_ARGS__) +#define kmem_cache_alloc(c, f) NM_SYM(NMS_kmem_cache_alloc, void *(*)(struct kmem_cache *, gfp_t))((c), (f)) +#define kmem_cache_free(c, p) NM_SYM(NMS_kmem_cache_free, void (*)(struct kmem_cache *, void *))((c), (p)) +#define kmem_cache_destroy(c) NM_SYM(NMS_kmem_cache_destroy, void (*)(struct kmem_cache *))((c)) + +#define idr_alloc(r, p, s, e, g) \ + NM_SYM(NMS_idr_alloc, int (*)(struct idr *, void *, int, int, gfp_t))((r), (p), (s), (e), (g)) +#define idr_find(r, id) NM_SYM(NMS_idr_find, void *(*)(const struct idr *, unsigned long))((r), (id)) +#define idr_remove(r, id) NM_SYM(NMS_idr_remove, void *(*)(struct idr *, unsigned long))((r), (id)) +#define idr_get_next(r, id) NM_SYM(NMS_idr_get_next, void *(*)(struct idr *, int *))((r), (id)) +#define idr_destroy(r) NM_SYM(NMS_idr_destroy, void (*)(struct idr *))((r)) +#define idr_init(r) NM_SYM(NMS_idr_init, void (*)(struct idr *))((r)) + +#define call_rcu(h, f) NM_SYM(NMS_call_rcu, void (*)(struct rcu_head *, rcu_callback_t))((h), (f)) +#define synchronize_rcu() NM_SYM(NMS_synchronize_rcu, void (*)(void))() +#define rcu_barrier() NM_SYM(NMS_rcu_barrier, void (*)(void))() + +#define register_key_type(t) NM_SYM(NMS_register_key_type, int (*)(struct key_type *))((t)) +#define unregister_key_type(t) NM_SYM(NMS_unregister_key_type, void (*)(struct key_type *))((t)) + +#define capable(c) NM_SYM(NMS_capable, bool (*)(int))((c)) + +#define generic_fillattr(...) nm_kpm_generic_fillattr(__VA_ARGS__) +#define notify_change(...) nm_kpm_notify_change(__VA_ARGS__) + +#define vfs_getattr_nosec(...) ((int (*)())nm_kpm_sym[NMS_vfs_getattr_nosec])(__VA_ARGS__) +#define vfs_llseek(f, o, w) NM_SYM(NMS_vfs_llseek, loff_t (*)(struct file *, loff_t, int))((f), (o), (w)) +#define vfs_copy_file_range(a, b, c, d, e, g) \ + NM_SYM(NMS_vfs_copy_file_range, ssize_t (*)(struct file *, loff_t, struct file *, loff_t, size_t, unsigned int))((a), (b), (c), (d), (e), (g)) +#define shmem_file_setup(n, sz, f) \ + NM_SYM(NMS_shmem_file_setup, struct file *(*)(const char *, loff_t, unsigned long))((n), (sz), (f)) +#define generic_file_mmap(f, v) NM_SYM(NMS_generic_file_mmap, int (*)(struct file *, struct vm_area_struct *))((f), (v)) +#define __vfs_getxattr(d, i, n, v, sz) \ + NM_SYM(NMS___vfs_getxattr, int (*)(struct dentry *, struct inode *, const char *, void *, size_t))((d), (i), (n), (v), (sz)) +#define get_user_pages_fast(a, n, g, p) \ + NM_SYM(NMS_get_user_pages_fast, int (*)(unsigned long, int, unsigned int, struct page **))((a), (n), (g), (p)) + +#define _printk(...) NM_SYM(NMS__printk, int (*)(const char *, ...))(__VA_ARGS__) +#define printk(...) NM_SYM(NMS__printk, int (*)(const char *, ...))(__VA_ARGS__) +#define strlen(x) NM_SYM(NMS_strlen, __kernel_size_t (*)(const char *))((x)) +#define memcmp(a, b, n) NM_SYM(NMS_memcmp, int (*)(const void *, const void *, __kernel_size_t))((a), (b), (n)) +#define rb_insert_color(n, r) NM_SYM(NMS_rb_insert_color, void (*)(struct rb_node *, struct rb_root *))((n), (r)) +#define rb_erase(n, r) NM_SYM(NMS_rb_erase, void (*)(struct rb_node *, struct rb_root *))((n), (r)) +#define rb_next(n) NM_SYM(NMS_rb_next, struct rb_node *(*)(const struct rb_node *))((n)) +#define radix_tree_tagged(r, t) NM_SYM(NMS_radix_tree_tagged, int (*)(const struct radix_tree_root *, unsigned int))((r), (t)) + +#define nm_kpm_generic_fillattr(...) \ + ((void (*)())nm_kpm_sym[NMS_generic_fillattr])(__VA_ARGS__) +#define nm_kpm_notify_change(...) \ + ((int (*)())nm_kpm_sym[NMS_notify_change])(__VA_ARGS__) + +static inline void nm_kpm_spin_lock(spinlock_t *l) +{ NM_SYM(NMS__raw_spin_lock, void (*)(raw_spinlock_t *))(&l->rlock); } +static inline void nm_kpm_spin_unlock(spinlock_t *l) +{ NM_SYM(NMS__raw_spin_unlock, void (*)(raw_spinlock_t *))(&l->rlock); } +#undef spin_lock +#define spin_lock(l) nm_kpm_spin_lock(l) +#undef spin_unlock +#define spin_unlock(l) nm_kpm_spin_unlock(l) + +#define down_read(s) NM_SYM(NMS_down_read, void (*)(struct rw_semaphore *))((s)) +#define up_read(s) NM_SYM(NMS_up_read, void (*)(struct rw_semaphore *))((s)) +#define down_write(s) NM_SYM(NMS_down_write, void (*)(struct rw_semaphore *))((s)) +#define up_write(s) NM_SYM(NMS_up_write, void (*)(struct rw_semaphore *))((s)) + +static inline void nm_kpm_rcu_read_lock(void) +{ if (nm_kpm_sym[NMS___rcu_read_lock]) NM_SYM(NMS___rcu_read_lock, void (*)(void))(); } +static inline void nm_kpm_rcu_read_unlock(void) +{ if (nm_kpm_sym[NMS___rcu_read_unlock]) NM_SYM(NMS___rcu_read_unlock, void (*)(void))(); } +#undef rcu_read_lock +#define rcu_read_lock() nm_kpm_rcu_read_lock() +#undef rcu_read_unlock +#define rcu_read_unlock() nm_kpm_rcu_read_unlock() + +static inline int nm_kpm_srcu_read_lock(struct srcu_struct *ssp) +{ return NM_SYM(NMS___srcu_read_lock, int (*)(struct srcu_struct *))(ssp); } +static inline void nm_kpm_srcu_read_unlock(struct srcu_struct *ssp, int idx) +{ NM_SYM(NMS___srcu_read_unlock, void (*)(struct srcu_struct *, int))(ssp, idx); } +static inline void nm_kpm_synchronize_srcu(struct srcu_struct *ssp) +{ NM_SYM(NMS_synchronize_srcu, void (*)(struct srcu_struct *))(ssp); } +static inline int nm_kpm_init_srcu_struct(struct srcu_struct *ssp) +{ return NM_SYM(NMS_init_srcu_struct, int (*)(struct srcu_struct *))(ssp); } +static inline void nm_kpm_cleanup_srcu_struct(struct srcu_struct *ssp) +{ if (nm_kpm_sym[NMS_cleanup_srcu_struct]) NM_SYM(NMS_cleanup_srcu_struct, void (*)(struct srcu_struct *))(ssp); } +#undef srcu_read_lock +#define srcu_read_lock(s) nm_kpm_srcu_read_lock(s) +#undef srcu_read_unlock +#define srcu_read_unlock(s, i) nm_kpm_srcu_read_unlock((s), (i)) +#undef synchronize_srcu +#define synchronize_srcu(s) nm_kpm_synchronize_srcu(s) +#undef init_srcu_struct +#define init_srcu_struct(s) nm_kpm_init_srcu_struct(s) +#undef cleanup_srcu_struct +#define cleanup_srcu_struct(s) nm_kpm_cleanup_srcu_struct(s) +/* Real DEFINE_STATIC_SRCU() bakes a compile-time struct initializer that, + * on some kernel versions (e.g. 6.6's Tree SRCU), embeds a raw function + * pointer to `delayed_work_timer_fn` in .data — an unrelocatable address + * in a KPM .o. Declare it zeroed instead and bring it up at runtime via + * init_srcu_struct(), which lets the running kernel fill in its own + * addresses. */ +#undef DEFINE_STATIC_SRCU +#define DEFINE_STATIC_SRCU(name) static struct srcu_struct name + +static inline void nm_kpm_rb_erase_cached(struct rb_node *node, struct rb_root_cached *root) +{ + if (root->rb_leftmost == node) + root->rb_leftmost = NM_SYM(NMS_rb_next, struct rb_node *(*)(const struct rb_node *))(node); + NM_SYM(NMS_rb_erase, void (*)(struct rb_node *, struct rb_root *))(node, &root->rb_root); +} +#undef rb_erase_cached +#define rb_erase_cached(n, r) nm_kpm_rb_erase_cached((n), (r)) + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) +static inline bool nm_kpm_idr_is_empty(const struct idr *idr) +{ + return radix_tree_empty(&idr->idr_rt) && + !NM_SYM(NMS_radix_tree_tagged, int (*)(const struct radix_tree_root *, unsigned int)) + (&idr->idr_rt, IDR_FREE); +} +#undef idr_is_empty +#define idr_is_empty(i) nm_kpm_idr_is_empty(i) +#else +#define idr_is_empty(i) NM_SYM(NMS_radix_tree_tagged, int (*)(const struct idr *))((i)) +#endif + +#define __vfs_setxattr(...) ((int (*)())nm_kpm_sym[NMS___vfs_setxattr])(__VA_ARGS__) + +static ssize_t nm_kpm_generic_read_dir(struct file *f, char __user *b, size_t n, loff_t *o) +{ return NM_SYM(NMS_generic_read_dir, ssize_t (*)(struct file *, char __user *, size_t, loff_t *))(f, b, n, o); } +#undef generic_read_dir +#define generic_read_dir nm_kpm_generic_read_dir + +#undef DEFINE_STATIC_KEY_FALSE +#define DEFINE_STATIC_KEY_FALSE(name) bool name +#undef static_branch_unlikely +#define static_branch_unlikely(k) (*(k)) +#undef static_branch_likely +#define static_branch_likely(k) (*(k)) +#undef static_branch_enable +#define static_branch_enable(k) (*(k) = true) +#undef static_branch_disable +#define static_branch_disable(k) (*(k) = false) + +static inline void nm_kpm_inode_lock(struct inode *i) +{ NM_SYM(NMS_down_write, void (*)(struct rw_semaphore *))(&i->i_rwsem); } +static inline void nm_kpm_inode_unlock(struct inode *i) +{ NM_SYM(NMS_up_write, void (*)(struct rw_semaphore *))(&i->i_rwsem); } +#undef inode_lock +#define inode_lock(i) nm_kpm_inode_lock(i) +#undef inode_unlock +#define inode_unlock(i) nm_kpm_inode_unlock(i) + +static inline void nm_kpm_rb_insert_color_cached(struct rb_node *node, + struct rb_root_cached *root, + bool leftmost) +{ + if (leftmost) + root->rb_leftmost = node; + NM_SYM(NMS_rb_insert_color, void (*)(struct rb_node *, struct rb_root *))(node, &root->rb_root); +} +#undef rb_insert_color_cached +#define rb_insert_color_cached(n, r, l) nm_kpm_rb_insert_color_cached((n), (r), (l)) + +#define kvfree_call_rcu(...) ((void (*)())nm_kpm_sym[NMS_kvfree_call_rcu])(__VA_ARGS__) + +#undef kfree_rcu +#define kfree_rcu(ptr, rhf) ((void (*)(struct rcu_head *, void *)) \ + nm_kpm_sym[NMS_kvfree_call_rcu])(&((ptr)->rhf), (void *)(ptr)) +#define d_parent_ino(d) NM_SYM(NMS_d_parent_ino, ino_t (*)(struct dentry *))((d)) + +static inline bool nm_kpm_dir_emit_dotdot(struct file *file, struct dir_context *ctx) +{ + return dir_emit(ctx, "..", 2, + NM_SYM(NMS_d_parent_ino, ino_t (*)(struct dentry *))(file->f_path.dentry), + DT_DIR); +} +#undef dir_emit_dotdot +#define dir_emit_dotdot(f, c) nm_kpm_dir_emit_dotdot((f), (c)) + +static inline bool nm_kpm_dir_emit_dots(struct file *file, struct dir_context *ctx) +{ + if (ctx->pos == 0) { + if (!dir_emit(ctx, ".", 1, file->f_path.dentry->d_inode->i_ino, DT_DIR)) + return false; + ctx->pos = 1; + } + if (ctx->pos == 1) { + if (!nm_kpm_dir_emit_dotdot(file, ctx)) + return false; + ctx->pos = 2; + } + return true; +} +#undef dir_emit_dots +#define dir_emit_dots(f, c) nm_kpm_dir_emit_dots((f), (c)) + +#undef THIS_MODULE +#define THIS_MODULE ((struct module *)0) + +static inline void *nm_kpm_kmalloc(size_t sz, gfp_t flags) +{ + return NM_SYM(NMS___kmalloc, void *(*)(size_t, gfp_t))(sz, flags); +} +static inline void *nm_kpm_kzalloc(size_t sz, gfp_t flags) +{ + void *p = nm_kpm_kmalloc(sz, flags); + if (p) + __builtin_memset(p, 0, sz); + return p; +} +#undef kmalloc +#define kmalloc(sz, f) nm_kpm_kmalloc((sz), (f)) +#undef kzalloc +#define kzalloc(sz, f) nm_kpm_kzalloc((sz), (f)) +#undef kcalloc +#define kcalloc(n, sz, f) nm_kpm_kzalloc((n) * (sz), (f)) +#undef kmem_cache_zalloc +#define kmem_cache_zalloc(c, f) nm_kpm_kmem_cache_zalloc((c), (f)) +static inline void *nm_kpm_kmem_cache_zalloc(struct kmem_cache *c, gfp_t f) +{ + void *p = NM_SYM(NMS_kmem_cache_alloc, void *(*)(struct kmem_cache *, gfp_t))(c, f); + return p; +} + +#undef fs_initcall +#define fs_initcall(fn) +#undef module_init +#define module_init(fn) +#undef module_exit +#define module_exit(fn) +#undef MODULE_LICENSE +#define MODULE_LICENSE(s) +#undef MODULE_AUTHOR +#define MODULE_AUTHOR(s) +#undef MODULE_DESCRIPTION +#define MODULE_DESCRIPTION(s) +#undef MODULE_VERSION +#define MODULE_VERSION(s) +#undef MODULE_IMPORT_NS +#define MODULE_IMPORT_NS(ns) + +#endif diff --git a/kernel/kpm/nm_kpm_syms.h b/kernel/kpm/nm_kpm_syms.h new file mode 100644 index 00000000..30b42e4a --- /dev/null +++ b/kernel/kpm/nm_kpm_syms.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _NM_KPM_SYMS_H +#define _NM_KPM_SYMS_H + +enum nm_kpm_sym { + + NMS_d_add, NMS_d_drop, NMS_d_lookup, NMS_d_splice_alias, NMS_d_parent_ino, + NMS_dput, NMS_path_get, NMS_path_put, NMS_kern_path, NMS_full_name_hash, + NMS_shrink_dcache_sb, + + NMS_new_inode, NMS_iput, NMS_igrab, NMS_clear_inode, NMS_notify_change, + NMS_truncate_inode_pages_final, NMS_generic_fillattr, NMS_vfs_getattr_nosec, + + NMS_dentry_open, NMS_fput, NMS_generic_read_dir, NMS_vfs_llseek, + NMS_vfs_copy_file_range, NMS_shmem_file_setup, NMS_generic_file_mmap, + + NMS___vfs_getxattr, NMS___vfs_setxattr, NMS_xattr_full_name, + + NMS___kmalloc, NMS_kfree, NMS_kmem_cache_create, NMS_kmem_cache_alloc, + NMS_kmem_cache_free, NMS_kmem_cache_destroy, + + NMS_idr_alloc, NMS_idr_find, NMS_idr_remove, NMS_idr_get_next, + NMS_idr_destroy, NMS_idr_init, NMS_rb_insert_color, NMS_rb_erase, NMS_rb_next, + NMS_radix_tree_tagged, + + NMS_call_rcu, NMS_synchronize_rcu, NMS_rcu_barrier, NMS_kvfree_call_rcu, + NMS__raw_spin_lock, NMS__raw_spin_unlock, + NMS___rcu_read_lock, NMS___rcu_read_unlock, + NMS___srcu_read_lock, NMS___srcu_read_unlock, NMS_synchronize_srcu, + NMS_init_srcu_struct, NMS_cleanup_srcu_struct, + NMS_down_read, NMS_up_read, NMS_down_write, NMS_up_write, + + NMS_register_key_type, NMS_unregister_key_type, + + NMS__printk, NMS_capable, NMS_strlen, NMS_memcmp, + NMS_get_user_pages_fast, NMS___folio_put, + NMS_static_key_enable, NMS_static_key_disable, + + NMS___list_add_valid_or_report, NMS_alt_cb_patch_nops, + + NM_KPM_SYM_COUNT +}; + +extern void *nm_kpm_sym[NM_KPM_SYM_COUNT]; + +int nm_kpm_engine_init(void); +void nm_kpm_engine_exit(void); +long nm_kpm_engine_ctl(const char *args); + +#endif diff --git a/kernel/src/nomount.c b/kernel/src/nomount.c index af02d2cd..f4464bce 100644 --- a/kernel/src/nomount.c +++ b/kernel/src/nomount.c @@ -1455,6 +1455,15 @@ static struct key_type nm_key_type = { static int __init nomount_init(void) { +#ifdef NM_KPM_BUILD + /* KPM builds declare nomount_srcu zeroed (see DEFINE_STATIC_SRCU in + * nm_kpm_shim.h) and must bring it up at runtime instead. */ + if (init_srcu_struct(&nomount_srcu)) { + nm_err("Failed to initialize SRCU\n"); + return -ENOMEM; + } +#endif + nm_dir_cachep = KMEM_CACHE(nomount_dir_node, SLAB_HWCACHE_ALIGN); nm_inode_cachep = KMEM_CACHE(nm_inode_info, SLAB_HWCACHE_ALIGN); nm_iop_cachep = KMEM_CACHE(nm_iop, SLAB_HWCACHE_ALIGN); @@ -1466,6 +1475,9 @@ static int __init nomount_init(void) if (nm_inode_cachep) kmem_cache_destroy(nm_inode_cachep); if (nm_iop_cachep) kmem_cache_destroy(nm_iop_cachep); if (nm_fop_cachep) kmem_cache_destroy(nm_fop_cachep); +#ifdef NM_KPM_BUILD + cleanup_srcu_struct(&nomount_srcu); +#endif return -ENOMEM; } @@ -1476,6 +1488,9 @@ static int __init nomount_init(void) kmem_cache_destroy(nm_inode_cachep); kmem_cache_destroy(nm_iop_cachep); kmem_cache_destroy(nm_fop_cachep); +#ifdef NM_KPM_BUILD + cleanup_srcu_struct(&nomount_srcu); +#endif return ret; } @@ -1495,6 +1510,9 @@ static void __exit nomount_exit(void) kmem_cache_destroy(nm_inode_cachep); kmem_cache_destroy(nm_iop_cachep); kmem_cache_destroy(nm_fop_cachep); +#ifdef NM_KPM_BUILD + cleanup_srcu_struct(&nomount_srcu); +#endif nm_info("Unloaded successfully\n"); }