diff --git a/README.rst b/README.rst index 2ff8820..f2cf4a8 100644 --- a/README.rst +++ b/README.rst @@ -334,6 +334,22 @@ Each retried attempt's traceback is appended to the ``rerun test summary info`` section. The section is emitted automatically when the flag is set, so ``-rR`` is not required. +Warn when a test is rerun +------------------------- + +To make reruns more visible on CI, pass ``--rerun-warning``. Each time a +test is scheduled for rerun a ``PytestWarning`` is emitted, which tools +like `pytest-github-actions-annotate-failures +`_ can +turn into annotations: + +.. code-block:: bash + + $ pytest --reruns 2 --rerun-warning + +``filterwarnings = error`` does not turn these warnings into errors, so they +never fail a test or abort the run. + Output ------ diff --git a/changes/318.feature.rst b/changes/318.feature.rst new file mode 100644 index 0000000..2fe3228 --- /dev/null +++ b/changes/318.feature.rst @@ -0,0 +1,3 @@ +Add ``--rerun-warning`` flag that emits a ``PytestWarning`` each time a test +is scheduled for rerun, to surface flaky tests in CI (e.g. via +pytest-github-actions-annotate-failures annotations). diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index db3974a..60c4b08 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -178,6 +178,15 @@ def pytest_addoption(parser): "'rerun test summary info' section, which is emitted automatically " "when this flag is set.", ) + group._addoption( + "--rerun-warning", + action="store_true", + dest="rerun_warning", + help="Emit a PytestWarning each time a test is scheduled for rerun. " + "Useful to surface flaky tests in CI, e.g. via annotations from " + "pytest-github-actions-annotate-failures. filterwarnings=error " + "does not turn these warnings into errors.", + ) group.addoption( "--max-suite-reruns", action="store", @@ -280,6 +289,37 @@ def _warn_pdb_disables_reruns(config, item=None): ) +def _warn_rerun(config, nodeid, attempt, item=None): + """Warn that a test failed and will be rerun (``--rerun-warning``). + + As in ``_warn_pdb_disables_reruns``, ``-W error`` must not escalate the + warning into an INTERNALERROR, so an escalated warning is recorded through + pytest's warning hook instead. Without an item (a crashed test rescheduled + by the xdist controller) the warning is always recorded that way. + """ + warning = pytest.PytestWarning( + f"{nodeid} failed on attempt {attempt} and will be rerun" + ) + if item is not None: + try: + item.warn(warning) + return + except Warning: + filename, lineno = str(item.path), (item.location[1] or 0) + 1 + else: + filename, lineno = nodeid.split("::")[0], 0 + config.hook.pytest_warning_recorded.call_historic( + kwargs=dict( + warning_message=warnings.WarningMessage( + warning, type(warning), filename, lineno + ), + when="runtest", + nodeid=nodeid, + location=None, + ) + ) + + def get_reruns_count(item): reruns = item.session.config.getoption("force_reruns") if reruns is not None: @@ -821,6 +861,10 @@ def pytest_handlecrashitem(self, crashitem, report, sched): try: sched.mark_test_pending(crashitem) report.outcome = "rerun" + if sched.config.option.rerun_warning: + _warn_rerun( + sched.config, crashitem, db.get_test_failures(crashitem) + 1 + ) except NotImplementedError: if reserved_suite_rerun: db.decrement_suite_reruns() @@ -1337,6 +1381,10 @@ def pytest_runtest_protocol(item, nextitem): continue report.outcome = "rerun" + if item.session.config.option.rerun_warning: + _warn_rerun( + item.config, item.nodeid, item.execution_count, item=item + ) time.sleep(delay * delay_backoff_factor ** (item.execution_count - 1)) if not parallel or works_with_current_xdist(): diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 2094448..8791c67 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -798,6 +798,66 @@ def test_skips(bad_teardown): ] +def test_rerun_warning_emitted(testdir): + testdir.makepyfile( + f""" + def test_pass(): + {temporary_failure()}""" + ) + result = testdir.runpytest("--reruns", "1", "--rerun-warning") + assert_outcomes(result, passed=1, rerun=1) + result.stdout.fnmatch_lines_random([ + "*PytestWarning: *test_pass* failed on attempt 1 and will be rerun*" + ]) + + +def test_rerun_warning_with_warnings_as_errors(testdir): + """`-W error` must not escalate the warning into an INTERNALERROR.""" + testdir.makepyfile( + f""" + def test_pass(): + {temporary_failure()} + + def test_other(): + pass""" + ) + result = testdir.runpytest("-W", "error", "--reruns", "1", "--rerun-warning") + assert_outcomes(result, passed=2, rerun=1) + result.stdout.no_fnmatch_line("INTERNALERROR*") + result.stdout.fnmatch_lines_random([ + "*PytestWarning: *test_pass* failed on attempt 1 and will be rerun*" + ]) + + +@pytest.mark.skipif(not has_xdist, reason="requires xdist with crashitem") +def test_rerun_warning_emitted_for_temporary_test_crash(testdir): + testdir.makepyfile( + f""" + def test_crash(): + {temporary_crash()} + + def test_pass(): + pass""" + ) + result = testdir.runpytest( + "-p", "xdist", "-n", "1", "--reruns", "1", "--rerun-warning" + ) + assert_outcomes(result, passed=2, rerun=1) + result.stdout.fnmatch_lines_random([ + "*PytestWarning: *test_crash failed on attempt 1 and will be rerun*" + ]) + + +def test_no_rerun_warning_by_default(testdir): + testdir.makepyfile( + f""" + def test_pass(): + {temporary_failure()}""" + ) + result = testdir.runpytest("--reruns", "1") + assert "will be rerun" not in result.stdout.str() + + @pytest.mark.parametrize("report_flag", ["-ra", "-rA"]) def test_extra_test_summary_for_reruns_with_reportchars_all(testdir, report_flag): testdir.makepyfile(