Fix GDB JIT unwind info and harden the tests

The GDB CFI hand-rolled in jit_unwind.c couldn't correctly describe the
compiled shim's prologue on AArch64 and relied on hardcoded offsets
that would silently invalidate under any compiler/flag change. Tools/jit
now compiles shim.c with -fasynchronous-unwind-tables, extracts its
.eh_frame at build time, and ships the CIE/FDE CFI bytes as a blob in
jit_stencils.h; jit_unwind.c splices those bytes into the synthetic
EH frame at runtime, so whatever prologue clang emits is described
accurately. Executor regions keep the hand-written steady-state rule,
which is our pinned-frame-pointer invariant (enforced by
Tools/jit/_optimizers.py _validate()), not a guess at compiler output.
Also: "Backtrace stopped: frame did not save the PC" is now a hard
AssertionError instead of a silent skip (it always indicates a real
unwind bug), and the three JIT tests share a get_stack_trace override
that opts out of the generic "?? ()" skip so unrelated libc-without-
debug-info frames don't mask a passing test.

Author: pablogsal

Include/internal/pycore_jit_unwind.h

@@ -5,9 +5,32 @@
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#if defined(PY_HAVE_PERF_TRAMPOLINE) || (defined(__linux__) && defined(__ELF__))
-
 #include <stddef.h>
+#include <stdint.h>
+
+/*
+ * Compiler-emitted CFI for the shim region (GDB path only).
+ *
+ * Captured at build time by Tools/jit from the shim's compiled .eh_frame
+ * so the runtime CIE/FDE can describe whatever prologue the compiler
+ * chose, without hand-rolling DWARF. Executors pass NULL and fall back
+ * to the invariant-based steady-state rule that the CIE emits by hand.
+ *
+ * The struct is defined unconditionally so jit_record_code() in Python/jit.c
+ * has a valid pointer type on every platform — callers on non-(Linux+ELF)
+ * always pass NULL, matching jit_record_code()'s internal #ifdef.
+ */
+typedef struct {
+    const uint8_t *cie_init_cfi;
+    size_t         cie_init_cfi_size;
+    const uint8_t *fde_cfi;
+    size_t         fde_cfi_size;
+    uint32_t       code_align;
+    int32_t        data_align;
+    uint32_t       ra_column;
+} _PyJitUnwind_ShimCfi;
+
+#if defined(PY_HAVE_PERF_TRAMPOLINE) || (defined(__linux__) && defined(__ELF__))
 
 /* DWARF exception-handling pointer encodings shared by JIT unwind users. */
 enum {
@@ -35,22 +58,30 @@ enum {
 };
 
 /* Return the size of the generated .eh_frame data for the given encoding. */
-size_t _PyJitUnwind_EhFrameSize(int absolute_addr);
+size_t _PyJitUnwind_EhFrameSize(int absolute_addr,
+                                const _PyJitUnwind_ShimCfi *shim_cfi);
 
 /*
  * Build DWARF .eh_frame data for JIT code; returns size written or 0 on error.
  * absolute_addr selects the FDE address encoding:
  * - 0: PC-relative offsets (perf jitdump synthesized DSO).
  * - nonzero: absolute addresses (GDB JIT in-memory ELF).
+ *
+ * shim_cfi selects which JIT region the CFI describes (GDB path only):
+ * - NULL: executor trace; steady-state rule in the CIE applies at every PC.
+ * - non-NULL: compile_shim() output; the captured CIE/FDE CFI bytes are
+ *            spliced in so unwinding is valid at every PC in the shim.
  */
 size_t _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size,
                                  const void *code_addr, size_t code_size,
-                                 int absolute_addr);
+                                 int absolute_addr,
+                                 const _PyJitUnwind_ShimCfi *shim_cfi);
 
 void *_PyJitUnwind_GdbRegisterCode(const void *code_addr,
                                   size_t code_size,
                                   const char *entry,
-                                  const char *filename);
+                                  const char *filename,
+                                  const _PyJitUnwind_ShimCfi *shim_cfi);
 
 void _PyJitUnwind_GdbUnregisterCode(void *handle);
 

Lib/test/test_gdb/test_jit.py

@@ -85,6 +85,13 @@ def setUpModule():
 @unittest.skipUnless(hasattr(sys, "_jit") and sys._jit.is_enabled(),
                      "requires a JIT-enabled build with JIT execution active")
 class JitBacktraceTests(DebuggerTests):
+    def get_stack_trace(self, **kwargs):
+        # These tests validate the JIT-relevant part of the backtrace via
+        # _assert_jit_backtrace_shape, so an unrelated "?? ()" frame below
+        # the JIT/eval segment (e.g. libc without debug info) is tolerable.
+        kwargs.setdefault("skip_on_truncation", False)
+        return super().get_stack_trace(**kwargs)
+
     def _extract_backtrace_frames(self, gdb_output):
         frames = BACKTRACE_FRAME_RE.findall(gdb_output)
         self.assertGreater(
@@ -135,6 +142,20 @@ def _assert_jit_backtrace_shape(self, gdb_output, *, anchor_at_top):
             f"expected an eval frame after the JIT frame\n"
             f"backtrace:\n{backtrace}",
         )
+        relevant_end = max(
+            i
+            for i, frame in enumerate(frames)
+            if "py::jit_entry:<jit>" in frame or re.search(EVAL_FRAME_RE, frame)
+        )
+        truncated_frames = [
+            frame for frame in frames[: relevant_end + 1]
+            if " ?? ()" in frame
+        ]
+        self.assertFalse(
+            truncated_frames,
+            "unexpected truncated frame before the validated JIT/eval segment\n"
+            f"backtrace:\n{backtrace}",
+        )
         if anchor_at_top:
             self.assertRegex(
                 frames[0],

Lib/test/test_gdb/util.py

@@ -20,6 +20,27 @@
 
 PYTHONHASHSEED = '123'
 
+# gh-91960, bpo-40019: gdb reports these when the optimizer has dropped
+# python-frame debug info; the test can't read what's not there.
+_OPTIMIZED_OUT_PATTERNS = (
+    '(frame information optimized out)',
+    'Unable to read information on python frame',
+    '(unable to read python frame information)',
+)
+# gdb prints this when the unwinder genuinely failed to walk a frame —
+# i.e. the CFI (ours or a library's) is wrong. Treat as a hard failure,
+# not a skip, so regressions in our own unwind info don't hide.
+_UNWIND_FAILURE_PATTERNS = (
+    'Backtrace stopped: frame did not save the PC',
+)
+# gh-104736: " ?? ()" in the bt usually means the unwinder bailed early,
+# but can also be unrelated frames without debug info (e.g. libc). Tests
+# that validate the JIT-relevant part of the backtrace themselves can
+# opt out via skip_on_truncation=False.
+_TRUNCATED_BACKTRACE_PATTERNS = (
+    ' ?? ()',
+)
+
 
 def clean_environment():
     # Remove PYTHON* environment variables such as PYTHONHOME
@@ -161,6 +182,7 @@ def get_stack_trace(self, source=None, script=None,
                         cmds_after_breakpoint=None,
                         import_site=False,
                         ignore_stderr=False,
+                        skip_on_truncation=True,
                         **env_vars):
         '''
         Run 'python -c SOURCE' under gdb with a breakpoint.
@@ -256,26 +278,20 @@ def get_stack_trace(self, source=None, script=None,
                                     " because the Program Counter is"
                                     " not present")
 
+        for pattern in _UNWIND_FAILURE_PATTERNS:
+            if pattern in out:
+                raise AssertionError(
+                    f"gdb unwinder failed ({pattern!r}) — CFI bug in our "
+                    f"generated code or in a linked library.\n"
+                    f"Full gdb output:\n{out}"
+                )
+
         # bpo-40019: Skip the test if gdb failed to read debug information
         # because the Python binary is optimized.
-        for pattern in (
-            '(frame information optimized out)',
-            'Unable to read information on python frame',
-
-            # gh-91960: On Python built with "clang -Og", gdb gets
-            # "frame=<optimized out>" for _PyEval_EvalFrameDefault() parameter
-            '(unable to read python frame information)',
-
-            # gh-104736: On Python built with "clang -Og" on ppc64le,
-            # "py-bt" displays a truncated or not traceback, but "where"
-            # logs this error message:
-            'Backtrace stopped: frame did not save the PC',
-
-            # gh-104736: When "bt" command displays something like:
-            # "#1  0x0000000000000000 in ?? ()", the traceback is likely
-            # truncated or wrong.
-            ' ?? ()',
-        ):
+        patterns = _OPTIMIZED_OUT_PATTERNS
+        if skip_on_truncation:
+            patterns = patterns + _TRUNCATED_BACKTRACE_PATTERNS
+        for pattern in patterns:
             if pattern in out:
                 raise unittest.SkipTest(f"{pattern!r} found in gdb output")
 

Python/jit.c

@@ -63,7 +63,8 @@ jit_error(const char *message)
 
 static void *
 jit_record_code(const void *code_addr, size_t code_size,
-                const char *entry, const char *filename)
+                const char *entry, const char *filename,
+                const _PyJitUnwind_ShimCfi *shim_cfi)
 {
 #ifdef PY_HAVE_PERF_TRAMPOLINE
     _PyPerf_Callbacks callbacks;
@@ -77,12 +78,13 @@ jit_record_code(const void *code_addr, size_t code_size,
 
 #if defined(__linux__) && defined(__ELF__)
     return _PyJitUnwind_GdbRegisterCode(
-        code_addr, code_size, entry, filename);
+        code_addr, code_size, entry, filename, shim_cfi);
 #else
     (void)code_addr;
     (void)code_size;
     (void)entry;
     (void)filename;
+    (void)shim_cfi;
     return NULL;
 #endif
 }
@@ -762,7 +764,8 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz
     executor->jit_gdb_handle = jit_record_code(memory,
                     code_size + state.trampolines.size,
                     "jit_entry",
-                    "<jit>");
+                    "<jit>",
+                    /*shim_cfi=*/NULL);
     return 0;
 }
 
@@ -813,10 +816,29 @@ compile_shim(void)
         return NULL;
     }
     _Py_jit_shim_size = total_size;
+    /* GDB JIT unwind info (and the captured-.eh_frame blob that feeds it)
+     * is only wired up on Linux+ELF — see jit_record_code() below. Even
+     * there, user-provided JIT CFLAGS can suppress the shim's .eh_frame,
+     * so jit_stencils.h advertises whether the captured CFI blob exists. */
+#if defined(__linux__) && defined(__ELF__) && _Py_JIT_HAS_SHIM_CFI
+    static const _PyJitUnwind_ShimCfi shim_cfi = {
+        .cie_init_cfi      = _Py_jit_shim_cie_init_cfi,
+        .cie_init_cfi_size = sizeof(_Py_jit_shim_cie_init_cfi),
+        .fde_cfi           = _Py_jit_shim_fde_cfi,
+        .fde_cfi_size      = sizeof(_Py_jit_shim_fde_cfi),
+        .code_align        = _Py_jit_shim_code_align,
+        .data_align        = _Py_jit_shim_data_align,
+        .ra_column         = _Py_jit_shim_ra_column,
+    };
+    const _PyJitUnwind_ShimCfi *shim_cfi_ptr = &shim_cfi;
+#else
+    const _PyJitUnwind_ShimCfi *shim_cfi_ptr = NULL;
+#endif
     _Py_jit_shim_gdb_handle = jit_record_code(memory,
                                               code_size + state.trampolines.size,
                                               "jit_entry",
-                                              "<jit>");
+                                              "<jit>",
+                                              shim_cfi_ptr);
     return (_PyJitEntryFuncPtr)memory;
 }