From f796338245bc867ed249ff699492a25145b6df04 Mon Sep 17 00:00:00 2001 From: Pragyansh Chaturvedi Date: Fri, 7 Aug 2026 05:37:05 +0530 Subject: [PATCH 1/3] Tests: Silence deprecation warnings in test-cov and test-verifier c2de7da added -W ignore::DeprecationWarning to the 'test' target only, so 'make test-cov' and 'make test-verifier' still drown in warnings from the generated vmlinux.py. Also correct the test-verifier note: the suite does not run pytest under sudo, tests/framework/verifier.py shells out to 'sudo bpftool' itself. Co-Authored-By: Claude Opus 5 (1M context) --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7874b44e..5e3be67b 100644 --- a/Makefile +++ b/Makefile @@ -10,12 +10,13 @@ test: pytest tests/ -W ignore::DeprecationWarning -v --tb=short -m "not verifier" test-cov: - pytest tests/ -v --tb=short -m "not verifier" \ + pytest tests/ -W ignore::DeprecationWarning -v --tb=short -m "not verifier" \ --cov=pythonbpf --cov-report=term-missing --cov-report=html test-verifier: - @echo "NOTE: verifier tests require sudo and bpftool. Uses sudo .venv/bin/python3." - pytest tests/test_verifier.py -v --tb=short -m verifier + @echo "NOTE: verifier tests shell out to 'sudo bpftool'; run 'sudo -v' first so" + @echo " the timestamp does not lapse mid-run. bpftool must be installed." + pytest tests/test_verifier.py -W ignore::DeprecationWarning -v --tb=short -m verifier all: clean install From edefe626d097f48c1e6af6c3aac349cc3d6c48da Mon Sep 17 00:00:00 2001 From: Pragyansh Chaturvedi Date: Fri, 7 Aug 2026 05:37:05 +0530 Subject: [PATCH 2/3] Tests: Use a real program section in return, var_rval and if These three declared @section("sometag1"), which is not a section name libbpf can map to a program type: libbpf: failed to guess program type from ELF section 'sometag1' so they were rejected before the kernel verifier ever saw them. They generate IR and compile fine; only the section was wrong. Switch to tracepoint/syscalls/sys_enter_execve, matching the other tracepoint tests. Co-Authored-By: Claude Opus 5 (1M context) --- tests/failing_tests/if.py | 2 +- tests/passing_tests/return.py | 2 +- tests/passing_tests/var_rval.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/failing_tests/if.py b/tests/failing_tests/if.py index 638c2ce3..8949d253 100644 --- a/tests/failing_tests/if.py +++ b/tests/failing_tests/if.py @@ -3,7 +3,7 @@ @bpf -@section("sometag1") +@section("tracepoint/syscalls/sys_enter_execve") def sometag(ctx: c_void_p) -> c_int64: if 3 + 2 == 5: return c_int64(5) diff --git a/tests/passing_tests/return.py b/tests/passing_tests/return.py index 9bd048b6..67a0bb05 100644 --- a/tests/passing_tests/return.py +++ b/tests/passing_tests/return.py @@ -3,7 +3,7 @@ @bpf -@section("sometag1") +@section("tracepoint/syscalls/sys_enter_execve") def sometag(ctx: c_void_p) -> c_int64: return c_int64(1 - 1) diff --git a/tests/passing_tests/var_rval.py b/tests/passing_tests/var_rval.py index ee1735e7..0742c040 100644 --- a/tests/passing_tests/var_rval.py +++ b/tests/passing_tests/var_rval.py @@ -5,7 +5,7 @@ @bpf -@section("sometag1") +@section("tracepoint/syscalls/sys_enter_execve") def sometag(ctx: c_void_p) -> c_int64: a = 1 - 1 return c_int64(a) From a7e2bc39efc9656e7989cd7061ae3f0cf0842e80 Mon Sep 17 00:00:00 2001 From: Pragyansh Chaturvedi Date: Fri, 7 Aug 2026 05:39:49 +0530 Subject: [PATCH 3/3] Tests: Add a verifier level to the expected-failure tiers test_config.toml could only declare a failure at the "ir" or "llc" level, and test_verifier.py dropped every declared-xfail case from the level-3 run outright. A program that generates IR and compiles cleanly but that the kernel verifier rejects therefore had no way to be declared: it was silently treated as must-pass at all three levels. Levels now form an ordered pipeline (ir < llc < verifier) and a declared level marks that level and every later one xfail, which is what the old ir-implies-llc special case was expressing. Level 3 runs every test file and reports declared failures as expected ones rather than skipping them. Co-Authored-By: Claude Opus 5 (1M context) --- tests/README.md | 12 ++++++++++-- tests/conftest.py | 22 ++++++++++++++-------- tests/framework/bpf_test_case.py | 16 +++++++++++++++- tests/test_verifier.py | 17 +++++++++++------ 4 files changed, 50 insertions(+), 17 deletions(-) diff --git a/tests/README.md b/tests/README.md index 2861f4f3..6b63fd45 100644 --- a/tests/README.md +++ b/tests/README.md @@ -69,8 +69,16 @@ Known-broken tests are declared in `tests/test_config.toml`: "failing_tests/my_test.py" = {reason = "...", level = "ir"} ``` -- `level = "ir"` — fails during IR generation; both IR and LLC tests are marked xfail. -- `level = "llc"` — IR generates fine but `llc` rejects it; only the LLC test is marked xfail. +- `level = "ir"` — fails during IR generation. +- `level = "llc"` — IR generates fine but `llc` rejects it. +- `level = "verifier"` — IR and `llc` both succeed, but the kernel verifier rejects the object. + +A failure at one level implies failure at every later one, so the declared level marks +that level **and all later ones** xfail. An `"ir"` entry is xfail at all three levels; a +`"verifier"` entry is xfail at level 3 only and must still pass levels 1 and 2. + +Every test file runs at every level, including the ones declared here — level 3 does not +skip declared failures, it reports them as expected ones. All xfails use `strict = True`: if a test starts **passing** it shows up as **XPASS** and is treated as a test failure. This is intentional — it means the bug was fixed and the test should be promoted to `passing_tests/`. diff --git a/tests/conftest.py b/tests/conftest.py index ce92d1dd..42ab30ed 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,6 +19,7 @@ import pytest +from tests.framework.bpf_test_case import level_index from tests.framework.collector import collect_all_test_files # ── vmlinux availability ──────────────────────────────────────────────────── @@ -70,14 +71,19 @@ def pytest_collection_modifyitems(items): # xfail (strict: XPASS counts as a test failure, alerting us to fixed bugs) if case.is_expected_fail: - # Level "ir" → fails at IR generation: xfail both IR and LLC tests - # Level "llc" → IR succeeds but LLC fails: only xfail the LLC test - is_llc_test = item.nodeid.startswith("tests/test_llc_compilation.py") - - apply_xfail = (case.xfail_level == "ir") or ( - case.xfail_level == "llc" and is_llc_test - ) - if apply_xfail: + # A failure at one level implies failure at every later one, so mark + # this item xfail whenever the declared level is at or before it: + # "ir" → IR, LLC and verifier + # "llc" → LLC and verifier (IR is expected to succeed) + # "verifier" → verifier only + if item.nodeid.startswith("tests/test_verifier.py"): + item_level = "verifier" + elif item.nodeid.startswith("tests/test_llc_compilation.py"): + item_level = "llc" + else: + item_level = "ir" + + if level_index(case.xfail_level) <= level_index(item_level): item.add_marker( pytest.mark.xfail( reason=case.xfail_reason, diff --git a/tests/framework/bpf_test_case.py b/tests/framework/bpf_test_case.py index d80a7134..a993166e 100644 --- a/tests/framework/bpf_test_case.py +++ b/tests/framework/bpf_test_case.py @@ -1,6 +1,20 @@ from dataclasses import dataclass from pathlib import Path +# The three test levels, in pipeline order. A test declared as failing at one +# level is also expected to fail at every later level: a program that cannot +# generate IR cannot reach llc, and one that llc rejects never reaches the +# kernel. Used by conftest to decide which items to mark xfail. +LEVELS = ("ir", "llc", "verifier") + + +def level_index(level: str) -> int: + """Position of a level in the pipeline. Unknown levels sort first ("ir").""" + try: + return LEVELS.index(level) + except ValueError: + return 0 + @dataclass class BpfTestCase: @@ -8,7 +22,7 @@ class BpfTestCase: rel_path: str is_expected_fail: bool = False xfail_reason: str = "" - xfail_level: str = "ir" # "ir" or "llc" + xfail_level: str = "ir" # one of LEVELS needs_vmlinux: bool = False skip_reason: str = "" diff --git a/tests/test_verifier.py b/tests/test_verifier.py index 413ef453..3966e3f6 100644 --- a/tests/test_verifier.py +++ b/tests/test_verifier.py @@ -24,12 +24,12 @@ from tests.framework.verifier import verify_object -def _passing_test_files(): - return [c.path for c in collect_all_test_files() if not c.is_expected_fail] +def _verifier_test_files(): + return [c.path for c in collect_all_test_files()] -def _passing_test_ids(): - return [c.rel_path for c in collect_all_test_files() if not c.is_expected_fail] +def _verifier_test_ids(): + return [c.rel_path for c in collect_all_test_files()] def _get_rejection_reason(verifier_test_file: Path, output) -> str: @@ -43,11 +43,16 @@ def _get_rejection_reason(verifier_test_file: Path, output) -> str: return errstr +# Every test file runs at this level, including the ones declared in +# test_config.toml. conftest marks those xfail, so an "ir"- or "llc"-level +# failure is still reported as an expected failure rather than being silently +# dropped from the level-3 run — and a "verifier"-level entry becomes possible +# at all. @pytest.mark.verifier @pytest.mark.parametrize( "verifier_test_file", - _passing_test_files(), - ids=_passing_test_ids(), + _verifier_test_files(), + ids=_verifier_test_ids(), ) def test_kernel_verifier(verifier_test_file: Path, tmp_path, caplog): """Compile the BPF test and verify it passes the kernel verifier."""