Tests: Add a verifier level to the expected-failure tiers

test_config.toml could only declare a failure at the "ir" or "llc" level, and
test_verifier.py dropped every declared-xfail case from the level-3 run
outright. A program that generates IR and compiles cleanly but that the kernel
verifier rejects therefore had no way to be declared: it was silently treated
as must-pass at all three levels.

Levels now form an ordered pipeline (ir < llc < verifier) and a declared level
marks that level and every later one xfail, which is what the old
ir-implies-llc special case was expressing. Level 3 runs every test file and
reports declared failures as expected ones rather than skipping them.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: r41k0u

tests/README.md

@@ -69,8 +69,16 @@ Known-broken tests are declared in `tests/test_config.toml`:
 "failing_tests/my_test.py" = {reason = "...", level = "ir"}
 ```
 
-- `level = "ir"` — fails during IR generation; both IR and LLC tests are marked xfail.
-- `level = "llc"` — IR generates fine but `llc` rejects it; only the LLC test is marked xfail.
+- `level = "ir"` — fails during IR generation.
+- `level = "llc"` — IR generates fine but `llc` rejects it.
+- `level = "verifier"` — IR and `llc` both succeed, but the kernel verifier rejects the object.
+
+A failure at one level implies failure at every later one, so the declared level marks
+that level **and all later ones** xfail. An `"ir"` entry is xfail at all three levels; a
+`"verifier"` entry is xfail at level 3 only and must still pass levels 1 and 2.
+
+Every test file runs at every level, including the ones declared here — level 3 does not
+skip declared failures, it reports them as expected ones.
 
 All xfails use `strict = True`: if a test starts **passing** it shows up as **XPASS** and is treated as a test failure. This is intentional — it means the bug was fixed and the test should be promoted to `passing_tests/`.
 

tests/conftest.py

@@ -19,6 +19,7 @@
 
 import pytest
 
+from tests.framework.bpf_test_case import level_index
 from tests.framework.collector import collect_all_test_files
 
 # ── vmlinux availability ────────────────────────────────────────────────────
@@ -70,14 +71,19 @@ def pytest_collection_modifyitems(items):
 
         # xfail (strict: XPASS counts as a test failure, alerting us to fixed bugs)
         if case.is_expected_fail:
-            # Level "ir" → fails at IR generation: xfail both IR and LLC tests
-            # Level "llc" → IR succeeds but LLC fails: only xfail the LLC test
-            is_llc_test = item.nodeid.startswith("tests/test_llc_compilation.py")
-
-            apply_xfail = (case.xfail_level == "ir") or (
-                case.xfail_level == "llc" and is_llc_test
-            )
-            if apply_xfail:
+            # A failure at one level implies failure at every later one, so mark
+            # this item xfail whenever the declared level is at or before it:
+            #   "ir"       → IR, LLC and verifier
+            #   "llc"      → LLC and verifier (IR is expected to succeed)
+            #   "verifier" → verifier only
+            if item.nodeid.startswith("tests/test_verifier.py"):
+                item_level = "verifier"
+            elif item.nodeid.startswith("tests/test_llc_compilation.py"):
+                item_level = "llc"
+            else:
+                item_level = "ir"
+
+            if level_index(case.xfail_level) <= level_index(item_level):
                 item.add_marker(
                     pytest.mark.xfail(
                         reason=case.xfail_reason,

tests/framework/bpf_test_case.py

@@ -1,14 +1,28 @@
 from dataclasses import dataclass
 from pathlib import Path
 
+# The three test levels, in pipeline order. A test declared as failing at one
+# level is also expected to fail at every later level: a program that cannot
+# generate IR cannot reach llc, and one that llc rejects never reaches the
+# kernel. Used by conftest to decide which items to mark xfail.
+LEVELS = ("ir", "llc", "verifier")
+
+
+def level_index(level: str) -> int:
+    """Position of a level in the pipeline. Unknown levels sort first ("ir")."""
+    try:
+        return LEVELS.index(level)
+    except ValueError:
+        return 0
+
 
 @dataclass
 class BpfTestCase:
     path: Path
     rel_path: str
     is_expected_fail: bool = False
     xfail_reason: str = ""
-    xfail_level: str = "ir"  # "ir" or "llc"
+    xfail_level: str = "ir"  # one of LEVELS
     needs_vmlinux: bool = False
     skip_reason: str = ""
 

tests/test_verifier.py

@@ -24,12 +24,12 @@
 from tests.framework.verifier import verify_object
 
 
-def _passing_test_files():
-    return [c.path for c in collect_all_test_files() if not c.is_expected_fail]
+def _verifier_test_files():
+    return [c.path for c in collect_all_test_files()]
 
 
-def _passing_test_ids():
-    return [c.rel_path for c in collect_all_test_files() if not c.is_expected_fail]
+def _verifier_test_ids():
+    return [c.rel_path for c in collect_all_test_files()]
 
 
 def _get_rejection_reason(verifier_test_file: Path, output) -> str:
@@ -43,11 +43,16 @@ def _get_rejection_reason(verifier_test_file: Path, output) -> str:
     return errstr
 
 
+# Every test file runs at this level, including the ones declared in
+# test_config.toml. conftest marks those xfail, so an "ir"- or "llc"-level
+# failure is still reported as an expected failure rather than being silently
+# dropped from the level-3 run — and a "verifier"-level entry becomes possible
+# at all.
 @pytest.mark.verifier
 @pytest.mark.parametrize(
     "verifier_test_file",
-    _passing_test_files(),
-    ids=_passing_test_ids(),
+    _verifier_test_files(),
+    ids=_verifier_test_ids(),
 )
 def test_kernel_verifier(verifier_test_file: Path, tmp_path, caplog):
     """Compile the BPF test and verify it passes the kernel verifier."""