From 387c75ebf421b7628122da0973598cc049d1895a Mon Sep 17 00:00:00 2001 From: Louis Deconinck Date: Thu, 17 Sep 2026 21:29:30 +0200 Subject: [PATCH 1/3] Add --rerun-warning flag to emit a warning on each scheduled rerun Emits a PytestWarning per rerun so CI tooling such as pytest-github-actions-annotate-failures can surface flaky tests as annotations. Opt-in so filterwarnings=error users are unaffected. Closes #318. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- README.rst | 15 +++++++++++++++ changes/318.feature.rst | 3 +++ src/pytest_rerunfailures.py | 16 ++++++++++++++++ tests/test_pytest_rerunfailures.py | 23 +++++++++++++++++++++++ 4 files changed, 57 insertions(+) create mode 100644 changes/318.feature.rst diff --git a/README.rst b/README.rst index 747072e..e07c60d 100644 --- a/README.rst +++ b/README.rst @@ -320,6 +320,21 @@ Each retried attempt's traceback is appended to the ``rerun test summary info`` section. The section is emitted automatically when the flag is set, so ``-rR`` is not required. +Warn when a test is rerun +------------------------- + +To make reruns more visible on CI, pass ``--rerun-warning``. Each time a +test is scheduled for rerun a ``PytestWarning`` is emitted, which tools +like `pytest-github-actions-annotate-failures +`_ can +turn into annotations: + +.. code-block:: bash + + $ pytest --reruns 2 --rerun-warning + +Note that ``filterwarnings = error`` turns these warnings into errors. + Output ------ diff --git a/changes/318.feature.rst b/changes/318.feature.rst new file mode 100644 index 0000000..2fe3228 --- /dev/null +++ b/changes/318.feature.rst @@ -0,0 +1,3 @@ +Add ``--rerun-warning`` flag that emits a ``PytestWarning`` each time a test +is scheduled for rerun, to surface flaky tests in CI (e.g. via +pytest-github-actions-annotate-failures annotations). diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 616e658..b8396e8 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -178,6 +178,15 @@ def pytest_addoption(parser): "'rerun test summary info' section, which is emitted automatically " "when this flag is set.", ) + group._addoption( + "--rerun-warning", + action="store_true", + dest="rerun_warning", + help="Emit a PytestWarning each time a test is scheduled for rerun. " + "Useful to surface flaky tests in CI, e.g. via annotations from " + "pytest-github-actions-annotate-failures. Note that " + "filterwarnings=error turns these warnings into errors.", + ) group.addoption( "--max-suite-reruns", action="store", @@ -1298,6 +1307,13 @@ def pytest_runtest_protocol(item, nextitem): continue report.outcome = "rerun" + if item.session.config.option.rerun_warning: + item.warn( + pytest.PytestWarning( + f"{item.nodeid} failed on attempt " + f"{item.execution_count} and will be rerun" + ) + ) time.sleep(delay * delay_backoff_factor ** (item.execution_count - 1)) if not parallel or works_with_current_xdist(): diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index a4208c5..54739c4 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -693,6 +693,29 @@ def test_skips(bad_teardown): ] +def test_rerun_warning_emitted(testdir): + testdir.makepyfile( + f""" + def test_pass(): + {temporary_failure()}""" + ) + result = testdir.runpytest("--reruns", "1", "--rerun-warning") + assert_outcomes(result, passed=1, rerun=1) + result.stdout.fnmatch_lines_random([ + "*PytestWarning: *test_pass* failed on attempt 1 and will be rerun*" + ]) + + +def test_no_rerun_warning_by_default(testdir): + testdir.makepyfile( + f""" + def test_pass(): + {temporary_failure()}""" + ) + result = testdir.runpytest("--reruns", "1") + assert "will be rerun" not in result.stdout.str() + + def test_rerun_show_tracebacks_for_eventual_pass(testdir): testdir.makepyfile( f""" From e56702144290df06b5006ac9d1edb28ee6afaa45 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Wed, 7 Oct 2026 08:50:15 +0200 Subject: [PATCH 2/3] Keep --rerun-warning from aborting the run under -W error An escalated warning is recorded through pytest's warning hook instead, as already done for the --pdb warning. --- README.rst | 3 ++- src/pytest_rerunfailures.py | 42 +++++++++++++++++++++++++----- tests/test_pytest_rerunfailures.py | 18 +++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 5e584cb..f2cf4a8 100644 --- a/README.rst +++ b/README.rst @@ -347,7 +347,8 @@ turn into annotations: $ pytest --reruns 2 --rerun-warning -Note that ``filterwarnings = error`` turns these warnings into errors. +``filterwarnings = error`` does not turn these warnings into errors, so they +never fail a test or abort the run. Output ------ diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 5c6edb5..590d04e 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -184,8 +184,8 @@ def pytest_addoption(parser): dest="rerun_warning", help="Emit a PytestWarning each time a test is scheduled for rerun. " "Useful to surface flaky tests in CI, e.g. via annotations from " - "pytest-github-actions-annotate-failures. Note that " - "filterwarnings=error turns these warnings into errors.", + "pytest-github-actions-annotate-failures. filterwarnings=error " + "does not turn these warnings into errors.", ) group.addoption( "--max-suite-reruns", @@ -289,6 +289,37 @@ def _warn_pdb_disables_reruns(config, item=None): ) +def _warn_rerun(config, nodeid, attempt, item=None): + """Warn that a test failed and will be rerun (``--rerun-warning``). + + As in ``_warn_pdb_disables_reruns``, ``-W error`` must not escalate the + warning into an INTERNALERROR, so an escalated warning is recorded through + pytest's warning hook instead. Without an item (a crashed test rescheduled + by the xdist controller) the warning is always recorded that way. + """ + warning = pytest.PytestWarning( + f"{nodeid} failed on attempt {attempt} and will be rerun" + ) + if item is not None: + try: + item.warn(warning) + return + except Warning: + filename, lineno = str(item.path), (item.location[1] or 0) + 1 + else: + filename, lineno = nodeid.split("::")[0], 0 + config.hook.pytest_warning_recorded.call_historic( + kwargs=dict( + warning_message=warnings.WarningMessage( + warning, type(warning), filename, lineno + ), + when="runtest", + nodeid=nodeid, + location=None, + ) + ) + + def get_reruns_count(item): reruns = item.session.config.getoption("force_reruns") if reruns is not None: @@ -1347,11 +1378,8 @@ def pytest_runtest_protocol(item, nextitem): report.outcome = "rerun" if item.session.config.option.rerun_warning: - item.warn( - pytest.PytestWarning( - f"{item.nodeid} failed on attempt " - f"{item.execution_count} and will be rerun" - ) + _warn_rerun( + item.config, item.nodeid, item.execution_count, item=item ) time.sleep(delay * delay_backoff_factor ** (item.execution_count - 1)) diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 921a57c..b0032e9 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -811,6 +811,24 @@ def test_pass(): ]) +def test_rerun_warning_with_warnings_as_errors(testdir): + """`-W error` must not escalate the warning into an INTERNALERROR.""" + testdir.makepyfile( + f""" + def test_pass(): + {temporary_failure()} + + def test_other(): + pass""" + ) + result = testdir.runpytest("-W", "error", "--reruns", "1", "--rerun-warning") + assert_outcomes(result, passed=2, rerun=1) + result.stdout.no_fnmatch_line("INTERNALERROR*") + result.stdout.fnmatch_lines_random([ + "*PytestWarning: *test_pass* failed on attempt 1 and will be rerun*" + ]) + + def test_no_rerun_warning_by_default(testdir): testdir.makepyfile( f""" From 8316eac313b790e18e5edd4c710d0b4860d846c3 Mon Sep 17 00:00:00 2001 From: Michael Howitz Date: Wed, 7 Oct 2026 08:51:30 +0200 Subject: [PATCH 3/3] Emit --rerun-warning for crashed tests rescheduled by xdist --- src/pytest_rerunfailures.py | 4 ++++ tests/test_pytest_rerunfailures.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 590d04e..60c4b08 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -861,6 +861,10 @@ def pytest_handlecrashitem(self, crashitem, report, sched): try: sched.mark_test_pending(crashitem) report.outcome = "rerun" + if sched.config.option.rerun_warning: + _warn_rerun( + sched.config, crashitem, db.get_test_failures(crashitem) + 1 + ) except NotImplementedError: if reserved_suite_rerun: db.decrement_suite_reruns() diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index b0032e9..8791c67 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -829,6 +829,25 @@ def test_other(): ]) +@pytest.mark.skipif(not has_xdist, reason="requires xdist with crashitem") +def test_rerun_warning_emitted_for_temporary_test_crash(testdir): + testdir.makepyfile( + f""" + def test_crash(): + {temporary_crash()} + + def test_pass(): + pass""" + ) + result = testdir.runpytest( + "-p", "xdist", "-n", "1", "--reruns", "1", "--rerun-warning" + ) + assert_outcomes(result, passed=2, rerun=1) + result.stdout.fnmatch_lines_random([ + "*PytestWarning: *test_crash failed on attempt 1 and will be rerun*" + ]) + + def test_no_rerun_warning_by_default(testdir): testdir.makepyfile( f"""