diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..95cd6c389d --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,4 @@ +RELEASE_TYPE: patch + +This release fixes (:issue:`4681`), which would cause line numbers to frequently be wrong in tracebacks for +errors found by Hypothesis. diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index beb28c539a..3cec37923f 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -1619,6 +1619,12 @@ def shortlex_key(error): return result +# Exists solely to insert a line in the traceback where we need to. +def _reraise_exception_group(the_error_hypothesis_found): + __tracebackhide__ = False + raise the_error_hypothesis_found + + def _raise_to_user( errors_to_report, settings, target_lines, trailer="", *, unsound_backend=None ): @@ -2248,19 +2254,16 @@ def wrapped_test(*arguments, **kwargs): f"{pytest_extra_msg}." ) report(msg) - # The dance here is to avoid showing users long tracebacks - # full of Hypothesis internals they don't care about. - # We have to do this inline, to avoid adding another - # internal stack frame just when we've removed the rest. - # - # Using a variable for our trimmed error ensures that the line - # which will actually appear in tracebacks is as clear as - # possible - "raise the_error_hypothesis_found". + # Trim the traceback to remove hypothesis internals the_error_hypothesis_found = e.with_traceback( None if isinstance(e, BaseExceptionGroup) else get_trimmed_traceback() ) + if isinstance(e, BaseExceptionGroup): + # Insert a frame here as otherwise all base frames are + # trimmed which causes pytest problems + _reraise_exception_group(the_error_hypothesis_found) raise the_error_hypothesis_found if not (ran_explicit_examples or state.ever_executed): diff --git a/hypothesis-python/src/hypothesis/extra/_patching.py b/hypothesis-python/src/hypothesis/extra/_patching.py index 3dba3f7b5c..c60173c766 100644 --- a/hypothesis-python/src/hypothesis/extra/_patching.py +++ b/hypothesis-python/src/hypothesis/extra/_patching.py @@ -183,7 +183,7 @@ def get_patch_for( if patch is None: return None - (before, after) = patch + before, after = patch return (str(fname), before, after) @@ -196,8 +196,11 @@ def _get_patch_for( strip_via: tuple[str, ...] = (), namespace: dict[str, Any], ) -> tuple[str, str] | None: + # Follow __wrapped_target to the original function, since the wrapper's + # co_filename may not point to the real source file (see #4681). + source_func = getattr(func, "__wrapped_target", func) try: - before = inspect.getsource(func) + before = inspect.getsource(source_func) except Exception: # pragma: no cover return None diff --git a/hypothesis-python/src/hypothesis/internal/escalation.py b/hypothesis-python/src/hypothesis/internal/escalation.py index 031a99004c..f9d330965c 100644 --- a/hypothesis-python/src/hypothesis/internal/escalation.py +++ b/hypothesis-python/src/hypothesis/internal/escalation.py @@ -80,14 +80,27 @@ def get_trimmed_traceback( ) ): return tb - while tb.tb_next is not None and ( - # If the frame is from one of our files, it's been added by Hypothesis. - is_hypothesis_file(getsourcefile(tb.tb_frame) or getfile(tb.tb_frame)) - # But our `@proxies` decorator overrides the source location, - # so we check for an attribute it injects into the frame too. - or tb.tb_frame.f_globals.get("__hypothesistracebackhide__") is True - ): + + def _is_hypothesis_frame(frame): + return ( + is_hypothesis_file(getsourcefile(frame) or getfile(frame)) + or frame.f_globals.get("__hypothesistracebackhide__") is True + ) + + # Strip leading hypothesis frames + while tb.tb_next is not None and _is_hypothesis_frame(tb.tb_frame): tb = tb.tb_next + + # Strip any remaining hypothesis frames from the middle of the traceback + prev = tb + current = tb.tb_next + while current is not None: + if current.tb_next is not None and _is_hypothesis_frame(current.tb_frame): + prev.tb_next = current.tb_next + else: + prev = current + current = prev.tb_next + return tb diff --git a/hypothesis-python/src/hypothesis/internal/reflection.py b/hypothesis-python/src/hypothesis/internal/reflection.py index f462b10fcf..8b7a2edc8e 100644 --- a/hypothesis-python/src/hypothesis/internal/reflection.py +++ b/hypothesis-python/src/hypothesis/internal/reflection.py @@ -357,6 +357,7 @@ def source_exec_as_module(source: str) -> ModuleType: def accept({funcname}): def {name}{signature}: + __tracebackhide__ = True return {funcname}({invocation}) return {name} """.lstrip() @@ -468,18 +469,11 @@ def impersonate(target): """ def accept(f): - # Lie shamelessly about where this code comes from, to hide the hypothesis - # internals from pytest, ipython, and other runtime introspection. - f.__code__ = f.__code__.replace( - co_filename=target.__code__.co_filename, - co_firstlineno=target.__code__.co_firstlineno, - ) f.__name__ = target.__name__ f.__module__ = target.__module__ f.__doc__ = target.__doc__ f.__globals__["__hypothesistracebackhide__"] = True - # But leave an breadcrumb for _describe_lambda to follow, it's - # just confused by the lies above + # Leave a breadcrumb for _describe_lambda and _get_patch_for to follow f.__wrapped_target = target return f diff --git a/hypothesis-python/tests/cover/test_traceback_elision.py b/hypothesis-python/tests/cover/test_traceback_elision.py index 2deb02cd01..f38396e847 100644 --- a/hypothesis-python/tests/cover/test_traceback_elision.py +++ b/hypothesis-python/tests/cover/test_traceback_elision.py @@ -30,7 +30,7 @@ def simplest_failure(x): except ValueError as e: tb = traceback.extract_tb(e.__traceback__) # Unless in debug mode, Hypothesis adds 1 frame - the least possible! - # (4 frames: this one, simplest_failure, internal frame, assert False) + # (4 frames: this one, simplest_failure, internal frame, raise ValueError) if verbosity < Verbosity.debug and not env_value: assert len(tb) == 4 else: diff --git a/hypothesis-python/tests/pytest/test_capture.py b/hypothesis-python/tests/pytest/test_capture.py index ac750c81ec..8141094aa3 100644 --- a/hypothesis-python/tests/pytest/test_capture.py +++ b/hypothesis-python/tests/pytest/test_capture.py @@ -10,7 +10,6 @@ import pytest -from hypothesis._settings import _CI_VARS from hypothesis.internal.compat import WINDOWS, escape_unicode_characters pytest_plugins = "pytester" @@ -71,44 +70,6 @@ def test_output_emitting_unicode(testdir, monkeypatch): assert result.ret == 0 -def get_line_num(token, lines, skip_n=0): - skipped = 0 - for i, line in enumerate(lines): - if token in line: - if skip_n == skipped: - return i - else: - skipped += 1 - raise AssertionError( - f"Token {token!r} not found (skipped {skipped} of planned {skip_n} skips)" - ) - - -TRACEBACKHIDE_HEALTHCHECK = """ -from hypothesis import given, settings -from hypothesis.strategies import integers -import time -@given(integers().map(lambda x: time.sleep(0.2))) -def test_healthcheck_traceback_is_hidden(x): - pass -""" - - -def test_healthcheck_traceback_is_hidden(testdir, monkeypatch): - for key in _CI_VARS: - monkeypatch.delenv(key, raising=False) - - script = testdir.makepyfile(TRACEBACKHIDE_HEALTHCHECK) - lines = testdir.runpytest(script, "--verbose").stdout.lines - def_token = "__ test_healthcheck_traceback_is_hidden __" - timeout_token = ": FailedHealthCheck" - def_line = get_line_num(def_token, lines) - timeout_line = get_line_num(timeout_token, lines) - # 16 on pytest 8.4.0 combined with py3{9, 10} or 3.13 free-threading (but - # not with 3.13 normal??) - assert timeout_line - def_line in {15, 16} - - COMPOSITE_IS_NOT_A_TEST = """ from hypothesis.strategies import composite, none @composite diff --git a/hypothesis-python/tests/snapshots/__snapshots__/test_impersonation_investigation.ambr b/hypothesis-python/tests/snapshots/__snapshots__/test_impersonation_investigation.ambr new file mode 100644 index 0000000000..1158a9e12b --- /dev/null +++ b/hypothesis-python/tests/snapshots/__snapshots__/test_impersonation_investigation.ambr @@ -0,0 +1,227 @@ +# serializer version: 1 +# name: test_deep_traceback + ''' + ============================= test session starts ============================== + collected 1 item + + test_deep_traceback.py F [100%] + + =================================== FAILURES =================================== + _____________________________ test_deep_traceback ______________________________ + + x = 0 + + @settings(database=None, derandomize=True) + @given(st.integers()) + def test_deep_traceback(x): + > helper(x) + + test_deep_traceback.py: # helper(x) + _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + x = 0 + + def helper(x): + > raise ValueError("boom") + E ValueError: boom + E Falsifying example: test_deep_traceback( + E x=0, # or any other generated value + E ) + + test_deep_traceback.py: # raise ValueError("boom") ValueError + ============================== 1 failed in T =============================== + ''' +# --- +# name: test_healthcheck_traceback + ''' + ============================= test session starts ============================== + collected 1 item + + test_healthcheck_traceback.py F [100%] + + =================================== FAILURES =================================== + _____________________ test_healthcheck_traceback_is_hidden _____________________ + + E hypothesis.errors.FailedHealthCheck: Input generation is slow: Hypothesis only generated N valid inputs after T seconds. + + count | fraction | slowest draws (seconds) + x | + + This could be for a few reasons: + 1. This strategy could be generating too much data per input. Try decreasing the amount of data generated, for example by decreasing the minimum size of collection strategies like st.lists(). + 2. Some other expensive computation could be running during input generation. For example, if @st.composite or st.data() is interspersed with an expensive computation, HealthCheck.too_slow is likely to trigger. If this computation is unrelated to input generation, move it elsewhere. Otherwise, try making it more efficient, or disable this health check if that is not possible. + + If you expect input generation to take this long, you can disable this health check with @settings(suppress_health_check=[HealthCheck.too_slow]). See https://hypothesis.readthedocs.io/en/latest/reference/api.html#hypothesis.HealthCheck for details. + All traceback entries are hidden. Pass `--full-trace` to see hidden and internal frames. + ---------------------------------- Hypothesis ---------------------------------- + You can reproduce this failure by adding @seed(0) to this test, or by running pytest with --hypothesis-seed=0. + ============================== 1 failed in T =============================== + ''' +# --- +# name: test_issue_basic_traceback + ''' + ============================= test session starts ============================== + collected 1 item + + test_issue_basic_traceback.py F [100%] + + =================================== FAILURES =================================== + _____________________________ test_basic_traceback _____________________________ + + _ = None + + @settings(database=None, derandomize=True) + @given(st.none()) + def test_basic_traceback(_): + > 1/0 + E ZeroDivisionError: division by zero + E Falsifying example: test_basic_traceback( + E _=None, + E ) + + test_issue_basic_traceback.py: # 1/0 ZeroDivisionError + ============================== 1 failed in T =============================== + ''' +# --- +# name: test_issue_lambda_traceback + ''' + ============================= test session starts ============================== + collected 1 item + + test_issue_lambda_traceback.py F [100%] + + =================================== FAILURES =================================== + ____________________________ test_lambda_traceback _____________________________ + + _ = None + + @settings(database=None, derandomize=True) + @given(st.none()) + def test_lambda_traceback(_): + f = lambda: """Hi!""" + > 1/0 + E ZeroDivisionError: division by zero + E Falsifying example: test_lambda_traceback( + E _=None, + E ) + + test_issue_lambda_traceback.py: # 1/0 ZeroDivisionError + ============================== 1 failed in T =============================== + ''' +# --- +# name: test_issue_wrong_function_traceback + ''' + ============================= test session starts ============================== + collected 1 item + + test_issue_wrong_function_traceback.py F [100%] + + =================================== FAILURES =================================== + _____________________________ test_wrong_function ______________________________ + + arguments = (), kwargs = {} + + @impersonate(test) + def wrapped_test(*arguments, **kwargs): # pragma: no cover # coverage limitation + > raise exc(message) + E hypothesis.errors.InvalidArgument: Too many positional arguments for test_wrong_function() were passed to @given - expected at most 0 arguments, but got 1 (none(),) + + .../hypothesis/core.py:NN: InvalidArgument + ============================== 1 failed in T =============================== + ''' +# --- +# name: test_map_failure_traceback + ''' + ============================= test session starts ============================== + collected 1 item + + test_map_failure_traceback.py F [100%] + + =================================== FAILURES =================================== + _______________________________ test_map_failure _______________________________ + + x = 0 + + > @given(st.integers().map(lambda x: 1/0)) + ^^^ + E ZeroDivisionError: division by zero + E while generating 'x' from integers().map(lambda x: 1 / 0) + + test_map_failure_traceback.py: # @given(st.integers().map(lambda x: 1/0)) ZeroDivisionError + ============================== 1 failed in T =============================== + ''' +# --- +# name: test_pr1582_multiple_failures + ''' + ============================= test session starts ============================== + collected 1 item + + test_pr1582_multiple_failures.py F [100%] + + =================================== FAILURES =================================== + ____________________________ test_multiple_failures ____________________________ + + Exception Group Traceback (most recent call last): + .../hypothesis/core.py", line NN, in _reraise_exception_group + | raise the_error_hypothesis_found + | ExceptionGroup: Hypothesis found 2 distinct failures. (2 sub-exceptions) + +-+---------------- 1 ---------------- + | Traceback (most recent call last): + | File "test_pr1582_multiple_failures.py", line NN, in test_multiple_failures + | raise RuntimeError("This number is too small!") + | RuntimeError: This number is too small! + | Falsifying example: test_multiple_failures( + | x=-101, + | ) + | Explanation: + | These lines were always and only run by failing examples: + | test_pr1582_multiple_failures.py:10 + +---------------- 2 ---------------- + | Traceback (most recent call last): + | File "test_pr1582_multiple_failures.py", line NN, in test_multiple_failures + | raise ValueError("This number is too big!") + | ValueError: This number is too big! + | Falsifying example: test_multiple_failures( + | x=101, + | ) + | Explanation: + | These lines were always and only run by failing examples: + | test_pr1582_multiple_failures.py:8 + +------------------------------------ + ============================== 1 failed in T =============================== + ''' +# --- +# name: test_pr1582_simple + ''' + ============================= test session starts ============================== + collected 1 item + + test_pr1582_simple.py F [100%] + + =================================== FAILURES =================================== + _________________________________ test_simple __________________________________ + + x = 0 + + @settings(database=None, derandomize=True) + @given(st.integers()) + def test_simple(x): + > assert x + E assert 0 + E Falsifying example: test_simple( + E x=0, + E ) + + test_pr1582_simple.py: # assert x AssertionError + ============================== 1 failed in T =============================== + ''' +# --- +# name: test_simple_passing + ''' + ============================= test session starts ============================== + collected 1 item + + test_simple_passing.py . [100%] + + ============================== 1 passed in T =============================== + ''' +# --- diff --git a/hypothesis-python/tests/snapshots/test_impersonation_investigation.py b/hypothesis-python/tests/snapshots/test_impersonation_investigation.py new file mode 100644 index 0000000000..6f24496da6 --- /dev/null +++ b/hypothesis-python/tests/snapshots/test_impersonation_investigation.py @@ -0,0 +1,269 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +"""Snapshot tests investigating impersonation traceback issues. + +These tests capture the full pytest output for various scenarios where +hypothesis's function impersonation (co_filename/co_firstlineno replacement) +affects tracebacks. See https://github.com/HypothesisWorks/hypothesis/issues/4681 +""" + +import re +import textwrap + +from hypothesis._settings import _CI_VARS + +pytest_plugins = "pytester" + + +def normalize_output(output, test_filename, test_source_lines): + """Normalize unstable parts of pytest output for snapshot comparison. + + Line numbers from the test file are replaced with the actual source line + content, making the snapshots self-documenting about what the traceback is + pointing at. Line numbers from other files are replaced with NN. + """ + + def replace_lineno(m): + filename = m.group(1) + lineno_str = m.group(2) + trailing = m.group(3) or "" + if filename.endswith(test_filename): + lineno = int(lineno_str) + if 1 <= lineno <= len(test_source_lines): + line_content = test_source_lines[lineno - 1].strip() + else: + line_content = f"" + return f"{filename}: # {line_content}{trailing}" + return f"{filename}:NN:{trailing}" + + output = re.sub( + r"([\w./\\-]+\.py):(\d+):(\s.*)?$", + replace_lineno, + output, + flags=re.MULTILINE, + ) + output = re.sub(r", line \d+,", ", line NN,", output) + # Normalize hypothesis source paths to a stable prefix + output = re.sub( + r'(?:File ")?.*/src/hypothesis/', + ".../hypothesis/", + output, + ) + # Normalize seeds + output = re.sub(r"@seed\(\d+\)", "@seed(0)", output) + output = re.sub(r"--hypothesis-seed=\d+", "--hypothesis-seed=0", output) + # Normalize patch file paths + output = re.sub( + r"`git apply .hypothesis/patches/[^`]+`", + "`git apply .hypothesis/patches/PATCH`", + output, + ) + # Normalize timing in healthcheck output + output = re.sub( + r"only generated \d+ valid inputs after [\d.]+ seconds", + "only generated N valid inputs after T seconds", + output, + ) + output = re.sub( + r"(count \| fraction \| slowest draws \(seconds\))\n.*", + r"\1\n x | ", + output, + ) + # Normalize test session timing + return re.sub(r"in [\d.]+s =", "in T =", output) + + +def get_failure_output(testdir, test_code, *extra_args): + """Run a test file via pytester and return normalized failure output.""" + source = textwrap.dedent(test_code).strip() + source_lines = source.splitlines() + script = testdir.makepyfile(source) + result = testdir.runpytest(script, "--tb=long", "--no-header", "-rN", *extra_args) + raw = "\n".join(result.stdout.lines) + # Replace the full absolute path of the test file with just the basename, + # so exception group tracebacks and explain sections are stable. + raw = raw.replace(str(script), script.basename) + return normalize_output(raw, script.basename, source_lines) + + +# ---- Issue #4681 Example 1: Basic ZeroDivisionError ---- +# The impersonation causes an intermediate frame to appear pointing at the +# user's function definition line rather than actual hypothesis internals. + +ISSUE_EXAMPLE_BASIC = """ +from hypothesis import given, settings +import hypothesis.strategies as st + +@settings(database=None, derandomize=True) +@given(st.none()) +def test_basic_traceback(_): + 1/0 +""" + + +def test_issue_basic_traceback(testdir, snapshot): + assert get_failure_output(testdir, ISSUE_EXAMPLE_BASIC) == snapshot + + +# ---- Issue #4681 Example 2: Lambda false positive ---- +# When an impersonated function contains a lambda, the carets in the traceback +# erroneously point to content inside the lambda. + +ISSUE_EXAMPLE_LAMBDA = """ +from hypothesis import given, settings +import hypothesis.strategies as st + +@settings(database=None, derandomize=True) +@given(st.none()) +def test_lambda_traceback(_): + f = lambda: \"\"\"Hi!\"\"\" + 1/0 +""" + + +def test_issue_lambda_traceback(testdir, snapshot): + assert get_failure_output(testdir, ISSUE_EXAMPLE_LAMBDA) == snapshot + + +# ---- Issue #4681 Example 3: Wrong function name ---- +# When a user forgets a parameter in their test signature, the traceback claims +# code was executed inside `wrapped_test` at a line in the user's file. + +ISSUE_EXAMPLE_WRONG_FUNCTION = """ +from hypothesis import given, settings +import hypothesis.strategies as st + +@settings(database=None, derandomize=True) +@given(st.none()) +def test_wrong_function(): + 1/0 +""" + + +def test_issue_wrong_function_traceback(testdir, snapshot): + assert get_failure_output(testdir, ISSUE_EXAMPLE_WRONG_FUNCTION) == snapshot + + +# ---- Healthcheck traceback hiding ---- +# This is the test from test_capture.py that verifies hypothesis internals +# are hidden from the traceback when a HealthCheck fails. + +HEALTHCHECK_TRACEBACK = """ +from hypothesis import given, settings +from hypothesis.strategies import integers +import time +@given(integers().map(lambda x: time.sleep(0.2))) +def test_healthcheck_traceback_is_hidden(x): + pass +""" + + +def test_healthcheck_traceback(testdir, monkeypatch, snapshot): + for key in _CI_VARS: + monkeypatch.delenv(key, raising=False) + assert get_failure_output(testdir, HEALTHCHECK_TRACEBACK) == snapshot + + +# ---- Successful test (baseline) ---- +# A simple passing test to confirm no extraneous traceback output. + +SIMPLE_PASSING = """ +from hypothesis import given, settings +import hypothesis.strategies as st + +@settings(database=None, derandomize=True) +@given(st.none()) +def test_simple_passing(_): + pass +""" + + +def test_simple_passing(testdir, snapshot): + assert get_failure_output(testdir, SIMPLE_PASSING) == snapshot + + +# ---- Multiple frames in traceback ---- +# Test that exercises a deeper call stack to see how impersonation +# affects the intermediate frames. + +DEEP_TRACEBACK = """ +from hypothesis import given, settings +import hypothesis.strategies as st + +def helper(x): + raise ValueError("boom") + +@settings(database=None, derandomize=True) +@given(st.integers()) +def test_deep_traceback(x): + helper(x) +""" + + +def test_deep_traceback(testdir, snapshot): + assert get_failure_output(testdir, DEEP_TRACEBACK) == snapshot + + +# ---- Map with traceback ---- +# Test that exercises a .map() call that fails, to see how the +# impersonation affects the traceback through map internals. + +MAP_FAILURE = """ +from hypothesis import given, settings +import hypothesis.strategies as st + +@settings(database=None, derandomize=True) +@given(st.integers().map(lambda x: 1/0)) +def test_map_failure(x): + pass +""" + + +def test_map_failure_traceback(testdir, snapshot): + assert get_failure_output(testdir, MAP_FAILURE) == snapshot + + +# ---- PR #1582 motivating examples ---- +# These are the examples from the PR that introduced traceback trimming. +# test_simple: a basic assertion failure (the simplest possible case) +# test_multiple_failures: different exception types from branches + +PR1582_SIMPLE = """ +from hypothesis import given, settings +import hypothesis.strategies as st + +@settings(database=None, derandomize=True) +@given(st.integers()) +def test_simple(x): + assert x +""" + + +def test_pr1582_simple(testdir, snapshot): + assert get_failure_output(testdir, PR1582_SIMPLE) == snapshot + + +PR1582_MULTIPLE_FAILURES = """ +from hypothesis import given, settings +import hypothesis.strategies as st + +@settings(database=None, derandomize=True) +@given(st.integers()) +def test_multiple_failures(x): + if x > 100: + raise ValueError("This number is too big!") + elif x < -100: + raise RuntimeError("This number is too small!") +""" + + +def test_pr1582_multiple_failures(testdir, snapshot): + assert get_failure_output(testdir, PR1582_MULTIPLE_FAILURES) == snapshot