From b3a7261922fffda9085f86ae602046aa5386f7f8 Mon Sep 17 00:00:00 2001 From: Louis Deconinck Date: Thu, 17 Sep 2026 21:41:54 +0200 Subject: [PATCH 1/2] Record retried failures as flakyFailure elements in JUnit XML pytest's junitxml plugin drops reports with outcome 'rerun', so failures from attempts that were retried left no trace in the XML. Stash rerun reports and append flakyFailure/flakyError elements (the Maven Surefire convention) to the test's final testcase so flaky failures are visible to XML consumers. Closes #170. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- README.rst | 4 ++ changes/170.feature.rst | 3 ++ src/pytest_rerunfailures.py | 79 ++++++++++++++++++++++++++++++ tests/test_pytest_rerunfailures.py | 34 +++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 changes/170.feature.rst diff --git a/README.rst b/README.rst index 747072e..399c5b5 100644 --- a/README.rst +++ b/README.rst @@ -352,6 +352,10 @@ be marked as failed. Compatibility ------------- +* When using ``--junitxml``, failed attempts that are re-run are recorded as + ```` (or ```` for setup/teardown failures) + elements in the test's final ````, following the convention used + by Maven Surefire. * This plugin is *not* compatible with pytest-xdist's --looponfail flag. * This plugin is *not* compatible with the core --pdb flag. * This plugin is *not* compatible with the plugin diff --git a/changes/170.feature.rst b/changes/170.feature.rst new file mode 100644 index 0000000..16ec8d4 --- /dev/null +++ b/changes/170.feature.rst @@ -0,0 +1,3 @@ +Record failed attempts that are re-run as ```` (and +```` for setup/teardown failures) elements in the test's final +```` in JUnit XML output, instead of dropping them. diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 616e658..fdccb4b 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -14,6 +14,7 @@ from contextlib import suppress from itertools import chain from typing import Any +from xml.etree import ElementTree import pytest from _pytest.outcomes import Exit, fail @@ -46,6 +47,11 @@ failed_subtests_key = _failed_subtests_key SubtestReport = _SubtestReport +try: + from _pytest.junitxml import bin_xml_escape +except ImportError: + bin_xml_escape = None + try: from xdist.newhooks import pytest_handlecrashitem @@ -729,6 +735,79 @@ def pytest_configure(config): else: config.failures_db = StatusDB() # no-op db + config.pluginmanager.register(JunitXmlRerunReporter(config)) + + +def _get_junitxml_plugin(config): + """Return the registered LogXML plugin, or None if --junitxml is off.""" + for plugin in config.pluginmanager.get_plugins(): + if ( + type(plugin).__module__ == "_pytest.junitxml" + and type(plugin).__name__ == "LogXML" + ): + return plugin + return None + + +class JunitXmlRerunReporter: + """Record failed attempts as ``flakyFailure`` elements in JUnit XML. + + pytest's junitxml plugin ignores reports with outcome "rerun" and + finalizes the ```` element after every attempt's teardown + report, so the failure information of retried attempts is lost. Stash the + rerun reports and append ```` elements (the convention used + by Maven Surefire and the ``flaky`` plugin; ``flakyError`` for setup and + teardown failures) when the final ``call`` report arrives, so they land + in the last attempt's ````. + """ + + def __init__(self, config): + self.config = config + self.pending = {} + + @staticmethod + def _key(report): + return ( + report.nodeid, + getattr(report, "worker_id", None), + getattr(report, "item_index", None), + getattr(report, "node", None), + ) + + @pytest.hookimpl(tryfirst=True) + def pytest_runtest_logreport(self, report): + if bin_xml_escape is None: + return + key = self._key(report) + if report.outcome == "rerun": + self.pending.setdefault(key, []).append(report) + return + if report.when != "call": + return + reruns = self.pending.pop(key, None) + if not reruns: + return + xml = _get_junitxml_plugin(self.config) + if xml is None: + return + reporter = xml.node_reporter(report) + reporter.record_testreport(report) + for rerun_report in reruns: + if rerun_report.longrepr is None: + continue + tag = "flakyFailure" if rerun_report.when == "call" else "flakyError" + reprcrash = getattr(rerun_report.longrepr, "reprcrash", None) + message = ( + reprcrash.message + if reprcrash is not None + else str(rerun_report.longrepr) + ) + node = ElementTree.Element( + tag, type="failure", message=bin_xml_escape(message) + ) + node.text = bin_xml_escape(str(rerun_report.longrepr)) + reporter.append(node) + class XDistHooks: def pytest_sessionstart(self, session): diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index a4208c5..965fbc3 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -548,6 +548,40 @@ def test_pass(): assert_outcomes(result, passed=1, rerun=2) +def _junitxml_testcases(testdir, filename="result.xml"): + import xml.etree.ElementTree as ET + + tree = ET.parse(str(testdir.tmpdir.join(filename))) # noqa: S314 + return tree.getroot().findall(".//testcase") + + +def test_junitxml_records_rerun_failures_as_flaky_failure(testdir): + testdir.makepyfile( + f""" + def test_pass(): + {temporary_failure(2)}""" + ) + result = testdir.runpytest("--reruns", "2", "--junitxml=result.xml") + assert_outcomes(result, passed=1, rerun=2) + + testcases = _junitxml_testcases(testdir) + flaky_failures = testcases[-1].findall("flakyFailure") + assert len(flaky_failures) == 2 + assert testcases[-1].findall("failure") == [] + assert "Failure: 1" in flaky_failures[0].get("message") + assert "Failure: 1" in flaky_failures[0].text + + +def test_junitxml_rerun_failures_before_final_failure(testdir): + testdir.makepyfile("def test_fail(): assert False") + result = testdir.runpytest("--reruns", "1", "--junitxml=result.xml") + assert_outcomes(result, passed=0, failed=1, rerun=1) + + testcases = _junitxml_testcases(testdir) + assert len(testcases[-1].findall("flakyFailure")) == 1 + assert len(testcases[-1].findall("failure")) == 1 + + def test_no_extra_test_summary_for_reruns_by_default(testdir): testdir.makepyfile( f""" From 873476201f0433a78972dd6ef81cbfbd6db7bd39 Mon Sep 17 00:00:00 2001 From: Louis Deconinck Date: Fri, 18 Sep 2026 08:19:10 +0200 Subject: [PATCH 2/2] Flush pending rerun reports on terminal setup/teardown failures pending was only flushed when a call report arrived, so rerun failures from attempts whose final execution failed in setup were dropped from the JUnit XML. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- src/pytest_rerunfailures.py | 4 +++- tests/test_pytest_rerunfailures.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index fdccb4b..9acada3 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -782,8 +782,10 @@ def pytest_runtest_logreport(self, report): if report.outcome == "rerun": self.pending.setdefault(key, []).append(report) return - if report.when != "call": + if report.when != "call" and not report.failed: return + # a failed setup or teardown report that is not a rerun means the + # attempts are exhausted in that phase and no call report follows reruns = self.pending.pop(key, None) if not reruns: return diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 965fbc3..ffc8490 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -582,6 +582,26 @@ def test_junitxml_rerun_failures_before_final_failure(testdir): assert len(testcases[-1].findall("failure")) == 1 +def test_junitxml_rerun_errors_before_final_setup_failure(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture + def broken_fixture(): + raise ValueError("setup always fails") + + def test_fail(broken_fixture): pass + """ + ) + result = testdir.runpytest("--reruns", "1", "--junitxml=result.xml") + assert_outcomes(result, passed=0, failed=0, error=1, rerun=1) + + testcases = _junitxml_testcases(testdir) + assert len(testcases[-1].findall("flakyError")) == 1 + assert len(testcases[-1].findall("error")) == 1 + + def test_no_extra_test_summary_for_reruns_by_default(testdir): testdir.makepyfile( f"""