From ffdfb5700e3add6e685ae4cbe2ae180f92789033 Mon Sep 17 00:00:00 2001
From: Dmitrii Gridnev <dgridnev@qase.io>
Date: Mon, 31 Aug 2026 11:08:18 +0300
Subject: [PATCH 1/2] fix: derive failure status from test phase instead of
 exception type

All four framework reporters guessed failed vs invalid ("broken") by
matching the failure's exception type/message against a keyword list
instead of asking the framework which phase actually failed. That made
any non-AssertionError call-phase failure (custom exceptions,
pytest.fail(), timeouts) report as invalid, and let a broken
setup/teardown silently overwrite or hide a real failure -- including a
passing test whose teardown raised being reported as passed.

Every framework already exposes the phase natively (call.when for
pytest/tavern, hook_failed for behave, setup/teardown results for Robot
Framework), so each reporter now uses that instead of message-sniffing,
with a guard so a broken setup/teardown can never downgrade or overwrite
an already-failed test-body result. Fixes #511.

Breaking change: failure statuses change for real for some users, so all
four packages get a major version bump with a changelog table describing
the old vs new behavior.
---
 qase-behave/changelog.md                      |  19 +++
 qase-behave/pyproject.toml                    |   2 +-
 qase-behave/src/qase/behave/formatter.py      |  48 ++++--
 qase-behave/src/qase/behave/utils.py          |  11 +-
 qase-behave/tests/test_formatter.py           | 159 ++++++++++++++++++
 qase-pytest/changelog.md                      |  16 ++
 qase-pytest/pyproject.toml                    |   2 +-
 qase-pytest/src/qase/pytest/plugin.py         |  30 +++-
 .../tests/tests_qase_pytest/test_plugin.py    | 118 +++++++++++++
 qase-robotframework/changelog.md              |  16 ++
 qase-robotframework/pyproject.toml            |   2 +-
 .../src/qase/robotframework/listener.py       |  31 ++--
 .../test_listener.py                          |  51 ++++++
 qase-tavern/changelog.md                      |  16 ++
 qase-tavern/pyproject.toml                    |   2 +-
 qase-tavern/src/qase/tavern/plugin.py         |  20 ++-
 qase-tavern/tests/test_plugin.py              |  82 +++++++++
 17 files changed, 583 insertions(+), 42 deletions(-)

diff --git a/qase-behave/changelog.md b/qase-behave/changelog.md
index 092e5abf..b4ba4fba 100644
--- a/qase-behave/changelog.md
+++ b/qase-behave/changelog.md
@@ -1,3 +1,22 @@
+# qase-behave 4.0.0
+
+## Breaking changes
+
+- Failure status is now derived from behave's own `hook_failed` signal instead of guessed by matching keywords (`assert`, `should`, `equal`, ...) against the error text. That heuristic was broken for the common case: behave's own failure message is `"Assertion Failed: ..."` (capital A), which never matched the lowercase `'assert'` keyword, so a literal `assert` in a step body was misclassified as `invalid`. Two further gaps are fixed alongside it:
+  - If a `before_scenario`/`before_step` hook raised, no steps ran at all and the scenario was silently reported as `passed` (the parser's optimistic default was never corrected, since `result()` is never called when a scenario's steps don't run). It's now reported as `invalid`, unless the scenario had already failed for real (a later `after_scenario` hook failure never downgrades a real step failure).
+  - On behave >=1.3, the newer `Status.error`/`Status.hook_error` names weren't in the status-mapping table and silently collapsed to `skipped`; they now route correctly.
+  ([#511](https://github.com/qase-tms/qase-python/issues/511))
+
+  | Failure phase | Old status (guessed from exception text) | New status (from behave's hook_failed) |
+  |---|---|---|
+  | Step body fails with what looks like an assertion (message contains `assert`/`should`/`equal`/...) | `failed` (in practice, almost never — see above) | `failed` |
+  | Step body fails with anything else (custom exception, timeout, behave's own `"Assertion Failed: ..."` text) | `invalid` — wrong | `failed` |
+  | `before_scenario`/`before_step` hook fails | `passed` (no steps ran) or `invalid`, inconsistently | `invalid` |
+  | `after_scenario`/`after_step` hook fails, scenario already passed | `passed` — wrong, hides the failure | `invalid` |
+  | `after_scenario`/`after_step` hook fails, scenario already failed | status could be overwritten by the hook's own message | scenario's status and diagnostics are kept untouched — the real failure always wins |
+
+  If your suite relies on the old behaviour, review any downstream logic (dashboards, defect linking, status filters) that branches on `invalid` vs `failed`.
+
 # qase-behave 3.2.0
 
 ## What's new
diff --git a/qase-behave/pyproject.toml b/qase-behave/pyproject.toml
index 55c14329..abd5888c 100644
--- a/qase-behave/pyproject.toml
+++ b/qase-behave/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "qase-behave"
-version = "3.2.2"
+version = "4.0.0"
 description = "Qase Behave Plugin for Qase TestOps and Qase Report"
 readme = "README.md"
 keywords = ["qase", "behave", "plugin", "testops", "report", "qase reporting", "test observability"]
diff --git a/qase-behave/src/qase/behave/formatter.py b/qase-behave/src/qase/behave/formatter.py
index 201a5f0b..38bbe286 100644
--- a/qase-behave/src/qase/behave/formatter.py
+++ b/qase-behave/src/qase/behave/formatter.py
@@ -26,6 +26,7 @@ def __init__(self, stream_opener=None, config=None):
         self.__already_started = False
         self.__case_ids = []
         self.__current_scenario = None
+        self.__current_raw_scenario = None
         self.__current_step_start = None
 
         if not self._behavex_mode:
@@ -73,15 +74,39 @@ def feature(self, feature: Feature):
             self.__case_ids, feature.scenarios)
 
     def scenario(self, scenario: Scenario):
-        if self.__current_scenario and self.__current_scenario.ignore == False:
-            self.__current_scenario.execution.complete()
-            self.reporter.add_result(self.__current_scenario)
-            self.__current_scenario = None
+        self.__finalize_current_scenario()
         self.__current_scenario = parse_scenario(scenario)
+        self.__current_raw_scenario = scenario
         qase._set_current_scenario(self.__current_scenario)
         qase._set_current_step(None)
         self.__current_step_start = None
 
+    def __finalize_current_scenario(self):
+        """Complete and send the in-flight scenario, if any.
+
+        A ``before_scenario``/``after_scenario`` hook failure is invisible
+        to ``result()`` -- if ``before_scenario`` fails, behave skips every
+        step so ``result()`` is never called at all, and the parsed
+        scenario keeps the optimistic 'passed' default set by
+        ``parse_scenario``. Check the raw behave ``Scenario``'s
+        ``hook_failed`` (present since behave 1.2.6) here instead, and
+        never let it downgrade a real step failure.
+        """
+        if self.__current_scenario is None or self.__current_scenario.ignore:
+            self.__current_scenario = None
+            self.__current_raw_scenario = None
+            return
+
+        if (self.__current_raw_scenario is not None and
+                getattr(self.__current_raw_scenario, 'hook_failed', False) and
+                self.__current_scenario.execution.status != 'failed'):
+            self.__current_scenario.execution.set_status('invalid')
+
+        self.__current_scenario.execution.complete()
+        self.reporter.add_result(self.__current_scenario)
+        self.__current_scenario = None
+        self.__current_raw_scenario = None
+
     def step(self, step):
         """Capture the real wall-clock start of the upcoming step.
 
@@ -100,13 +125,11 @@ def result(self, result: Step):
         qase._set_current_step(step)
 
         if step.execution.status != 'passed':
-            is_assertion_error = False
-            if result.error_message:
-                assertion_keywords = ['assert', 'AssertionError', 'expect', 'should', 'must']
-                is_assertion_error = any(keyword in result.error_message for keyword in assertion_keywords)
-
             if step.execution.status == 'failed':
-                status = 'failed' if is_assertion_error else 'invalid'
+                # A before_step/after_step hook failure is a broken step,
+                # like a setup/teardown error; any other step-body failure
+                # (assertion or otherwise) is a real test failure.
+                status = 'invalid' if getattr(result, 'hook_failed', False) else 'failed'
                 step.execution.set_status(status)
                 self.__current_scenario.execution.set_status(status)
             else:
@@ -118,10 +141,7 @@ def result(self, result: Step):
         qase._set_current_step(None)
 
     def eof(self):
-        if self.__current_scenario and self.__current_scenario.ignore == False:
-            self.__current_scenario.execution.complete()
-            self.reporter.add_result(self.__current_scenario)
-            self.__current_scenario = None
+        self.__finalize_current_scenario()
 
     def close(self):
         if self._is_behavex_worker:
diff --git a/qase-behave/src/qase/behave/utils.py b/qase-behave/src/qase/behave/utils.py
index 19000c32..1cfdc387 100644
--- a/qase-behave/src/qase/behave/utils.py
+++ b/qase-behave/src/qase/behave/utils.py
@@ -290,10 +290,17 @@ def parse_step(step: Step, start_time: float = None) -> QaseStep:
         data=StepGherkinData(keyword=step.keyword, name=step.name, line=step.line)
     )
 
-    # Map behave status to qase status
+    # Map behave status to qase status. The failed-vs-invalid decision is
+    # made in the formatter, from ``step.hook_failed`` (present since behave
+    # 1.2.6) rather than from any of these names, so 'error'/'hook_error'/
+    # 'cleanup_error' (behave >=1.3's richer Status enum; absent on 1.2.6)
+    # only need to route into the same "failed" branch here.
     status_mapping = {
         'passed': 'passed',
-        'failed': 'failed',  # This will be updated in formatter based on error type
+        'failed': 'failed',
+        'error': 'failed',
+        'hook_error': 'failed',
+        'cleanup_error': 'failed',
         'skipped': 'skipped',
         'undefined': 'skipped',
         'pending': 'skipped'
diff --git a/qase-behave/tests/test_formatter.py b/qase-behave/tests/test_formatter.py
index b8c24a46..a7b4be5f 100644
--- a/qase-behave/tests/test_formatter.py
+++ b/qase-behave/tests/test_formatter.py
@@ -397,6 +397,165 @@ def test_scenario_resets_step_start(self):
         assert formatter._QaseFormatter__current_step_start is None
 
 
+class TestStepFailureStatusFromHookFailed:
+    """A step's failed-vs-invalid status is derived from ``hook_failed``, not
+    from matching keywords against the error text (#511)."""
+
+    def _make_formatter(self):
+        formatter = QaseFormatter()
+        formatter._behavex_mode = False
+        formatter.reporter = MagicMock()
+        formatter._QaseFormatter__current_scenario = MagicMock()
+        formatter._QaseFormatter__current_scenario.ignore = False
+        formatter._QaseFormatter__current_scenario.steps = []
+        return formatter
+
+    @staticmethod
+    def _make_step(status="failed", hook_failed=False, error_message="boom"):
+        step = MagicMock()
+        step.keyword = "Then"
+        step.name = "a step"
+        step.line = 1
+        step.status.name = status
+        step.duration = 0.1
+        step.hook_failed = hook_failed
+        step.error_message = error_message
+        return step
+
+    def test_assertion_failure_in_step_body_is_failed(self):
+        """behave's own error text is 'Assertion Failed: ...' (capital A) --
+        the old lowercase keyword match never matched this."""
+        formatter = self._make_formatter()
+        step = self._make_step(
+            status="failed", hook_failed=False,
+            error_message="Assertion Failed: expected X")
+
+        formatter.result(step)
+
+        formatter._QaseFormatter__current_scenario.execution.set_status.assert_called_with('failed')
+
+    def test_custom_exception_in_step_body_is_failed(self):
+        formatter = self._make_formatter()
+        step = self._make_step(
+            status="failed", hook_failed=False,
+            error_message="cluster never became Ready within 900s")
+
+        formatter.result(step)
+
+        formatter._QaseFormatter__current_scenario.execution.set_status.assert_called_with('failed')
+
+    def test_before_or_after_step_hook_failure_is_invalid(self):
+        formatter = self._make_formatter()
+        step = self._make_step(
+            status="failed", hook_failed=True,
+            error_message="before_step hook exploded")
+
+        formatter.result(step)
+
+        formatter._QaseFormatter__current_scenario.execution.set_status.assert_called_with('invalid')
+
+    def test_undefined_step_status_is_untouched(self):
+        """Non-failed statuses (skipped/undefined/pending) pass through as before."""
+        formatter = self._make_formatter()
+        step = self._make_step(status="undefined", hook_failed=False, error_message=None)
+
+        formatter.result(step)
+
+        formatter._QaseFormatter__current_scenario.execution.set_status.assert_called_with('skipped')
+
+
+class TestScenarioHookFailure:
+    """before_scenario/after_scenario hook failures must not report a
+    scenario as 'passed' just because result() was never called (#511)."""
+
+    def _make_formatter(self):
+        formatter = QaseFormatter()
+        formatter._behavex_mode = False
+        formatter.reporter = MagicMock()
+        return formatter
+
+    @staticmethod
+    def _make_result(status="passed", ignore=False):
+        from qase.commons.models import Result
+        result = Result(title="s", signature="s")
+        result.execution.set_status(status)
+        result.ignore = ignore
+        return result
+
+    def test_before_scenario_hook_failure_is_invalid_not_passed(self):
+        """No steps ran (before_scenario hook exploded), so the parsed
+        scenario is still stuck on parse_scenario's optimistic 'passed'
+        default -- it must not survive."""
+        formatter = self._make_formatter()
+        result = self._make_result(status="passed")
+        raw_scenario = MagicMock(hook_failed=True)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter._QaseFormatter__finalize_current_scenario()
+
+        assert result.execution.status == "invalid"
+        formatter.reporter.add_result.assert_called_once_with(result)
+
+    def test_after_scenario_hook_failure_after_pass_is_invalid(self):
+        formatter = self._make_formatter()
+        result = self._make_result(status="passed")
+        raw_scenario = MagicMock(hook_failed=True)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter._QaseFormatter__finalize_current_scenario()
+
+        assert result.execution.status == "invalid"
+
+    def test_after_scenario_hook_failure_does_not_downgrade_a_real_failure(self):
+        """A step already failed the scenario -- a later after_scenario hook
+        failure must not overwrite that real failure with 'invalid'."""
+        formatter = self._make_formatter()
+        result = self._make_result(status="failed")
+        raw_scenario = MagicMock(hook_failed=True)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter._QaseFormatter__finalize_current_scenario()
+
+        assert result.execution.status == "failed"
+
+    def test_no_hook_failure_leaves_status_untouched(self):
+        formatter = self._make_formatter()
+        result = self._make_result(status="passed")
+        raw_scenario = MagicMock(hook_failed=False)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter._QaseFormatter__finalize_current_scenario()
+
+        assert result.execution.status == "passed"
+
+    def test_ignored_scenario_is_never_sent(self):
+        formatter = self._make_formatter()
+        result = self._make_result(status="passed", ignore=True)
+        raw_scenario = MagicMock(hook_failed=True)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter._QaseFormatter__finalize_current_scenario()
+
+        formatter.reporter.add_result.assert_not_called()
+
+    def test_eof_finalizes_a_hook_failed_scenario_too(self):
+        formatter = self._make_formatter()
+        result = self._make_result(status="passed")
+        raw_scenario = MagicMock(hook_failed=True)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter.eof()
+
+        assert result.execution.status == "invalid"
+        formatter.reporter.add_result.assert_called_once_with(result)
+
+
 class TestBehaveXWorkerMode:
     """Test QaseFormatter in BehaveX worker mode (lock file coordination)."""
 
diff --git a/qase-pytest/changelog.md b/qase-pytest/changelog.md
index 0748449f..34961bfa 100644
--- a/qase-pytest/changelog.md
+++ b/qase-pytest/changelog.md
@@ -1,3 +1,19 @@
+# qase-pytest 9.0.0
+
+## Breaking changes
+
+- Failure status is now derived from the pytest phase (`call.when`) instead of guessed from the exception type. Previously any failure other than `AssertionError` (a custom exception, `pytest.fail()`, a timeout) was reported as `invalid` even when it happened in the test body, and a test that passed but whose `teardown` fixture raised was reported as `passed` — the teardown phase wasn't processed at all. A failing `setup`/`teardown` can no longer overwrite an already-failed test's status or diagnostics. ([#511](https://github.com/qase-tms/qase-python/issues/511))
+
+  | Failure phase | Old status (guessed from exception text) | New status (from pytest phase) |
+  |---|---|---|
+  | Test body (`call`) fails with what looks like an assertion (message contains `assert`/`should`/`equal`/...) | `failed` | `failed` |
+  | Test body (`call`) fails with anything else (custom exception, timeout, `pytest.fail()`, framework error text) | `invalid` — wrong | `failed` |
+  | Setup / before-hook fails | `invalid`, unless the exception's message happened to match an assertion keyword, in which case `failed` — inconsistent | `invalid` |
+  | Teardown / after-hook fails, test body already passed | `passed` — wrong, hides the failure | `invalid` |
+  | Teardown / after-hook fails, test body already failed | body's status kept, but the teardown message could silently overwrite the real failure's message/stacktrace | body's status and diagnostics are kept untouched — the real failure always wins |
+
+  If your suite relies on the old behaviour, review any downstream logic (dashboards, defect linking, status filters) that branches on `invalid` vs `failed`.
+
 # qase-pytest 8.3.1
 
 ## What's fixed
diff --git a/qase-pytest/pyproject.toml b/qase-pytest/pyproject.toml
index 0ca53a98..724fbe27 100644
--- a/qase-pytest/pyproject.toml
+++ b/qase-pytest/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "qase-pytest"
-version = "8.3.1"
+version = "9.0.0"
 description = "Qase Pytest Plugin for Qase TestOps and Qase Report"
 readme = "README.md"
 keywords = ["qase", "pytest", "plugin", "testops", "report", "qase reporting", "test observability"]
diff --git a/qase-pytest/src/qase/pytest/plugin.py b/qase-pytest/src/qase/pytest/plugin.py
index eb12e743..37fa26b1 100644
--- a/qase-pytest/src/qase/pytest/plugin.py
+++ b/qase-pytest/src/qase/pytest/plugin.py
@@ -157,8 +157,12 @@ def pytest_runtest_makereport(self, item, call):
 
         report = (yield).get_result()
 
-        # Skip processing if not in the relevant phase
-        if call.when not in ["setup", "call"]:
+        # Skip processing if not in the relevant phase. A failed teardown is
+        # let through too, so a broken teardown can't leave a green result
+        # behind; a passing teardown is still skipped.
+        if call.when not in ["setup", "call"] and not (
+            call.when == "teardown" and report.failed
+        ):
             return
 
         # Process test result and update status
@@ -187,6 +191,13 @@ def pytest_runtest_makereport(self, item, call):
 
     def _process_test_result(self, report, call, item):
         """Process test results and set appropriate status."""
+        # A broken setup/teardown must never downgrade or overwrite an
+        # already-failed call-phase result: skip it before touching the
+        # stacktrace, status, or message.
+        if (report.failed and call.when != "call" and
+                self.runtime.result.execution.status == PYTEST_TO_QASE_STATUS['FAILED']):
+            return
+
         # Add stacktrace if available
         if report.longrepr:
             self.runtime.result.execution.stacktrace = report.longreprtext
@@ -202,15 +213,18 @@ def _process_test_result(self, report, call, item):
             self._handle_passed_test(report, call)
 
     def _handle_failed_test(self, call):
-        """Handle failed test case and set appropriate status."""
-        is_assertion_error = False
+        """Handle failed test case and set appropriate status.
+
+        A call-phase failure is a test failure, whatever the exception type;
+        a failure in setup or teardown is a broken test. This is the
+        distinction pytest itself makes via ``call.when``.
+        """
         error_message = "Test failed"
-        
+
         if call.excinfo is not None:
-            is_assertion_error = call.excinfo.typename == "AssertionError"
             error_message = call.excinfo.exconly()
-        
-        status = PYTEST_TO_QASE_STATUS['FAILED'] if is_assertion_error else PYTEST_TO_QASE_STATUS['BROKEN']
+
+        status = PYTEST_TO_QASE_STATUS['FAILED'] if call.when == "call" else PYTEST_TO_QASE_STATUS['BROKEN']
         self._set_result_status(status)
         self.runtime.result.add_message(error_message)
 
diff --git a/qase-pytest/tests/tests_qase_pytest/test_plugin.py b/qase-pytest/tests/tests_qase_pytest/test_plugin.py
index 9dd338cf..958bf8b5 100644
--- a/qase-pytest/tests/tests_qase_pytest/test_plugin.py
+++ b/qase-pytest/tests/tests_qase_pytest/test_plugin.py
@@ -348,6 +348,124 @@ def test_logs_not_attached_when_capture_logs_disabled(self):
             mock_attach.assert_not_called()
 
 
+class TestFailureStatusFromPhase:
+    """Failure status is derived from call.when, not the exception type (#511)."""
+
+    def test_call_phase_assertion_error_is_failed(self):
+        plugin = make_plugin_with_capture_logs(capture_logs=False)
+        excinfo = MagicMock()
+        excinfo.typename = "AssertionError"
+        excinfo.exconly.return_value = "AssertionError: expected X"
+        report = make_report(failed=True)
+        call_obj = make_call(when="call", excinfo=excinfo)
+        item = make_mock_item()
+
+        run_makereport(plugin, item, call_obj, report)
+
+        assert plugin.runtime.result.execution.status == 'failed'
+
+    def test_call_phase_custom_exception_is_failed(self):
+        """A non-AssertionError raised in the call phase is still a real test failure."""
+        plugin = make_plugin_with_capture_logs(capture_logs=False)
+        excinfo = MagicMock()
+        excinfo.typename = "MyTimeoutError"
+        excinfo.exconly.return_value = "MyTimeoutError: cluster never became Ready"
+        report = make_report(failed=True)
+        call_obj = make_call(when="call", excinfo=excinfo)
+        item = make_mock_item()
+
+        run_makereport(plugin, item, call_obj, report)
+
+        assert plugin.runtime.result.execution.status == 'failed'
+
+    def test_call_phase_pytest_fail_is_failed(self):
+        """pytest.fail() raises Failed, not AssertionError, but is still a call-phase failure."""
+        plugin = make_plugin_with_capture_logs(capture_logs=False)
+        excinfo = MagicMock()
+        excinfo.typename = "Failed"
+        excinfo.exconly.return_value = "Failed: explicit pytest.fail"
+        report = make_report(failed=True)
+        call_obj = make_call(when="call", excinfo=excinfo)
+        item = make_mock_item()
+
+        run_makereport(plugin, item, call_obj, report)
+
+        assert plugin.runtime.result.execution.status == 'failed'
+
+    def test_setup_phase_assertion_error_is_invalid(self):
+        """An assert inside a fixture is a broken setup, not a test failure."""
+        plugin = make_plugin_with_capture_logs(capture_logs=False)
+        excinfo = MagicMock()
+        excinfo.typename = "AssertionError"
+        excinfo.exconly.return_value = "AssertionError: bad fixture"
+        report = make_report(failed=True)
+        call_obj = make_call(when="setup", excinfo=excinfo)
+        item = make_mock_item()
+
+        run_makereport(plugin, item, call_obj, report)
+
+        assert plugin.runtime.result.execution.status == 'invalid'
+
+    def test_setup_phase_custom_exception_is_invalid(self):
+        plugin = make_plugin_with_capture_logs(capture_logs=False)
+        excinfo = MagicMock()
+        excinfo.typename = "RuntimeError"
+        excinfo.exconly.return_value = "RuntimeError: fixture broke"
+        report = make_report(failed=True)
+        call_obj = make_call(when="setup", excinfo=excinfo)
+        item = make_mock_item()
+
+        run_makereport(plugin, item, call_obj, report)
+
+        assert plugin.runtime.result.execution.status == 'invalid'
+
+    def test_teardown_failure_after_pass_is_invalid(self):
+        """A test that passes but whose teardown raises must not be reported as passed."""
+        plugin = make_plugin_with_capture_logs(capture_logs=False)
+        plugin.runtime.result.execution.status = 'passed'
+        excinfo = MagicMock()
+        excinfo.typename = "RuntimeError"
+        excinfo.exconly.return_value = "RuntimeError: cleanup never finished"
+        report = make_report(failed=True)
+        call_obj = make_call(when="teardown", excinfo=excinfo)
+        item = make_mock_item()
+
+        run_makereport(plugin, item, call_obj, report)
+
+        assert plugin.runtime.result.execution.status == 'invalid'
+
+    def test_teardown_success_does_not_touch_existing_status(self):
+        """A passing teardown must not touch a status already set by the call phase."""
+        plugin = make_plugin_with_capture_logs(capture_logs=False)
+        plugin.runtime.result.execution.status = 'passed'
+        report = make_report(failed=False)
+        call_obj = make_call(when="teardown")
+        item = make_mock_item()
+
+        run_makereport(plugin, item, call_obj, report)
+
+        assert plugin.runtime.result.execution.status == 'passed'
+
+    def test_teardown_failure_after_call_failure_keeps_original_status_and_message(self):
+        """A broken teardown must never downgrade or overwrite an existing call-phase failure."""
+        plugin = make_plugin_with_capture_logs(capture_logs=False)
+        plugin.runtime.result.execution.status = 'failed'
+        plugin.runtime.result.execution.stacktrace = 'original call-phase stacktrace'
+        item = make_mock_item()
+
+        excinfo = MagicMock()
+        excinfo.exconly.return_value = "RuntimeError: cleanup never finished"
+        report = make_report(
+            failed=True, longrepr=MagicMock(), longreprtext="teardown stacktrace")
+        call_obj = make_call(when="teardown", excinfo=excinfo)
+
+        run_makereport(plugin, item, call_obj, report)
+
+        assert plugin.runtime.result.execution.status == 'failed'
+        assert plugin.runtime.result.execution.stacktrace == 'original call-phase stacktrace'
+        plugin.runtime.result.add_message.assert_not_called()
+
+
 class TestXdistRunIdRoundtrip:
     """xdist controller -> worker handoff of run_id via lock file.
 
diff --git a/qase-robotframework/changelog.md b/qase-robotframework/changelog.md
index 39ebfb74..cbcb9877 100644
--- a/qase-robotframework/changelog.md
+++ b/qase-robotframework/changelog.md
@@ -1,3 +1,19 @@
+# qase-robotframework 7.0.0
+
+## Breaking changes
+
+- Failure status is now derived from `[Setup]`/`[Teardown]` keyword results instead of guessed by matching keywords (`assert`, `should`, `equal`, ...) against the error message. That heuristic missed even the idiomatic case: `Should Be Equal` fails with a message like `"1 != 2"`, which matches none of the old keywords, so a textbook assertion failure was reported as `invalid`. The reporter now checks `result.setup.failed` / `result.teardown.failed` directly; a teardown failure never overwrites an already-failed test body's status. ([#511](https://github.com/qase-tms/qase-python/issues/511))
+
+  | Failure phase | Old status (guessed from error message) | New status (from `[Setup]`/`[Teardown]` result) |
+  |---|---|---|
+  | Test body fails with what looks like an assertion (message contains `assert`/`should`/`equal`/...) | `failed` (in practice, rarely — see above) | `failed` |
+  | Test body fails with anything else (custom keyword failure, `Fail`, framework error text) | `invalid` — wrong | `failed` |
+  | `[Setup]` fails | `invalid`, unless the message happened to match a keyword, in which case `failed` — inconsistent | `invalid` |
+  | `[Teardown]` fails, test body already passed | `invalid` only if the message matched a keyword — inconsistent | `invalid` |
+  | `[Teardown]` fails, test body already failed | status decided purely by whichever message happened to match | test body's status wins — the real failure is never downgraded |
+
+  If your suite relies on the old behaviour, review any downstream logic (dashboards, defect linking, status filters) that branches on `invalid` vs `failed`.
+
 # qase-robotframework 6.0.0
 
 ## Breaking changes
diff --git a/qase-robotframework/pyproject.toml b/qase-robotframework/pyproject.toml
index 4328dc52..9a15a09d 100644
--- a/qase-robotframework/pyproject.toml
+++ b/qase-robotframework/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "qase-robotframework"
-version = "6.0.2"
+version = "7.0.0"
 description = "Qase Robot Framework Plugin"
 readme = "README.md"
 authors = [{name = "Qase Team", email = "support@qase.io"}]
diff --git a/qase-robotframework/src/qase/robotframework/listener.py b/qase-robotframework/src/qase/robotframework/listener.py
index a84558fe..9561379c 100644
--- a/qase-robotframework/src/qase/robotframework/listener.py
+++ b/qase-robotframework/src/qase/robotframework/listener.py
@@ -122,16 +122,7 @@ def end_test(self, test, result):
 
         self.runtime.result.execution.complete()
 
-        # Determine if it's an assertion error or other error
-        status = STATUSES[result.status]
-        if status == "failed" and hasattr(result, 'message'):
-            # Check if the error message contains assertion-related keywords
-            assertion_keywords = ['assert', 'AssertionError',
-                                  'expect', 'should', 'must', 'equal', 'not equal']
-            is_assertion_error = any(
-                keyword in result.message for keyword in assertion_keywords)
-            status = "failed" if is_assertion_error else "invalid"
-
+        status = self._resolve_failure_status(STATUSES[result.status], result)
         self.runtime.result.execution.set_status(status)
         if hasattr(result, 'message'):
             self.runtime.result.execution.stacktrace = result.message
@@ -176,6 +167,26 @@ def end_test(self, test, result):
             f"Finished case result: {result.status}, error: {hasattr(result, 'message') and result.message or None}"
         )
 
+    @staticmethod
+    def _resolve_failure_status(status, result):
+        """Attribute a test failure to the right phase.
+
+        A failure in ``[Setup]`` or ``[Teardown]`` is a broken test, not a
+        test failure -- but a teardown failure must never downgrade a real
+        failure already present in the test body.
+        """
+        if status != "failed":
+            return status
+
+        if getattr(result, 'has_setup', False) and result.setup.failed:
+            return "invalid"
+
+        if (getattr(result, 'has_teardown', False) and result.teardown.failed and
+                not any(kw.failed for kw in result.body)):
+            return "invalid"
+
+        return status
+
     def close(self):
         if self.last_level_flag is not None:
             if int(self.last_level_flag) == 1:
diff --git a/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py b/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py
index e7263c04..dc0bf1a0 100644
--- a/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py
+++ b/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py
@@ -352,3 +352,54 @@ def test_else_if_chain_each_branch_keeps_timing(self):
         assert [s.execution.duration for s in steps] == [0, 76, 0]
         assert all(s.execution.start_time is not None for s in steps)
         assert all(s.execution.end_time is not None for s in steps)
+
+
+class TestResolveFailureStatus:
+    """A test-body failure is `failed`; a [Setup]/[Teardown] failure is
+    `invalid` (#511). Replaces the old keyword-matching heuristic, which
+    misclassified idiomatic RF failures like ``Should Be Equal`` (message
+    ``"1 != 2"``, matching none of the old keywords) as `invalid`.
+    """
+
+    @staticmethod
+    def _make_result(has_setup=False, setup_failed=False,
+                      has_teardown=False, teardown_failed=False,
+                      body_failed_flags=()):
+        result = MagicMock(spec=["has_setup", "setup", "has_teardown", "teardown", "body"])
+        result.has_setup = has_setup
+        result.setup = MagicMock(spec=["failed"], failed=setup_failed)
+        result.has_teardown = has_teardown
+        result.teardown = MagicMock(spec=["failed"], failed=teardown_failed)
+        result.body = [MagicMock(spec=["failed"], failed=f) for f in body_failed_flags]
+        return result
+
+    def test_non_failed_status_is_returned_unchanged(self):
+        result = self._make_result()
+        assert Listener._resolve_failure_status("passed", result) == "passed"
+        assert Listener._resolve_failure_status("skipped", result) == "skipped"
+
+    def test_body_failure_with_no_setup_or_teardown_stays_failed(self):
+        """The idiomatic case: `Should Be Equal` fails in the test body."""
+        result = self._make_result(body_failed_flags=[True])
+        assert Listener._resolve_failure_status("failed", result) == "failed"
+
+    def test_setup_failure_is_invalid(self):
+        result = self._make_result(has_setup=True, setup_failed=True)
+        assert Listener._resolve_failure_status("failed", result) == "invalid"
+
+    def test_setup_ran_fine_stays_failed(self):
+        result = self._make_result(has_setup=True, setup_failed=False,
+                                    body_failed_flags=[True])
+        assert Listener._resolve_failure_status("failed", result) == "failed"
+
+    def test_teardown_failure_after_passing_body_is_invalid(self):
+        result = self._make_result(has_teardown=True, teardown_failed=True,
+                                    body_failed_flags=[False])
+        assert Listener._resolve_failure_status("failed", result) == "invalid"
+
+    def test_teardown_failure_does_not_downgrade_a_failing_body(self):
+        """The test body itself already failed -- a teardown failure on top
+        must not overwrite that real failure with 'invalid'."""
+        result = self._make_result(has_teardown=True, teardown_failed=True,
+                                    body_failed_flags=[True])
+        assert Listener._resolve_failure_status("failed", result) == "failed"
diff --git a/qase-tavern/changelog.md b/qase-tavern/changelog.md
index cc9b164f..5d2d404f 100644
--- a/qase-tavern/changelog.md
+++ b/qase-tavern/changelog.md
@@ -1,3 +1,19 @@
+# qase-tavern 4.0.0
+
+## Breaking changes
+
+- Failure status is now derived from the pytest phase (`call.when`) instead of guessed from the exception type. pytest-tavern raises its own `TestFailError` for a failing stage, never `AssertionError`, so under the old logic **every** call-phase failure was reported as `invalid` — real product/test failures never showed up as `failed`. `setup`/`teardown` failures (e.g. a broken pytest fixture used by a Tavern test) weren't processed at all and shipped with `status: null`; they're now reported as `invalid`, without ever overwriting an already-failed call-phase result. ([#511](https://github.com/qase-tms/qase-python/issues/511))
+
+  | Failure phase | Old status (guessed from exception text) | New status (from pytest phase) |
+  |---|---|---|
+  | Test body (`call`) fails with what looks like an assertion (message contains `assert`/`should`/`equal`/...) | `failed` | `failed` |
+  | Test body (`call`) fails with anything else (custom exception, timeout, `pytest.fail()`, framework error text) | `invalid` — wrong | `failed` |
+  | Setup / before-hook fails | `invalid`, unless the exception's message happened to match an assertion keyword, in which case `failed` — inconsistent | `invalid` |
+  | Teardown / after-hook fails, test body already passed | `passed` — wrong, hides the failure | `invalid` |
+  | Teardown / after-hook fails, test body already failed | body's status kept, but the teardown message could silently overwrite the real failure's message/stacktrace | body's status and diagnostics are kept untouched — the real failure always wins |
+
+  If your suite relies on the old behaviour, review any downstream logic (dashboards, defect linking, status filters) that branches on `invalid` vs `failed`.
+
 # qase-tavern 3.1.0
 
 ## What's new
diff --git a/qase-tavern/pyproject.toml b/qase-tavern/pyproject.toml
index 84e33da7..5ce1bb51 100644
--- a/qase-tavern/pyproject.toml
+++ b/qase-tavern/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "qase-tavern"
-version = "3.1.1"
+version = "4.0.0"
 description = "Qase Tavern Plugin for Qase TestOps and Qase Report"
 readme = "README.md"
 keywords = ["qase", "tavern", "plugin", "testops", "report", "qase reporting", "test observability"]
diff --git a/qase-tavern/src/qase/tavern/plugin.py b/qase-tavern/src/qase/tavern/plugin.py
index 9820d068..72797b08 100644
--- a/qase-tavern/src/qase/tavern/plugin.py
+++ b/qase-tavern/src/qase/tavern/plugin.py
@@ -65,12 +65,13 @@ def pytest_tavern_beta_after_every_response(self, expected, response):
     def pytest_runtest_makereport(self, item, call):
         if self.runtime.result is None:
             return
+
         if call.when == "call":
             if call.excinfo:
-                # Determine if it's an assertion error or other error
-                is_assertion_error = call.excinfo.typename == "AssertionError"
-                status = "failed" if is_assertion_error else "invalid"
-                self.runtime.result.execution.status = status
+                # A call-phase failure is a real test failure, whatever the
+                # exception type -- pytest-tavern raises its own TestFailError,
+                # never AssertionError.
+                self.runtime.result.execution.status = "failed"
                 if hasattr(call.excinfo, "value"):
                     self.runtime.result.execution.stacktrace = '\n'.join(str(a) for a in call.excinfo.value.args)
                     if hasattr(call.excinfo.value, "failures"):
@@ -101,6 +102,17 @@ def pytest_runtest_makereport(self, item, call):
             self.runtime.result.execution.status = "passed"
             for key, step in self.runtime.steps.items():
                 self._ensure_step_closed(step, "passed")
+            return
+
+        # A setup/teardown failure (e.g. a broken pytest fixture used by the
+        # Tavern test) is a broken test, not a call-phase failure -- but it
+        # must never downgrade or overwrite an already-failed call-phase
+        # result.
+        if call.when in ("setup", "teardown") and call.excinfo:
+            if self.runtime.result.execution.status == "failed":
+                return
+            self.runtime.result.execution.status = "invalid"
+            self.runtime.result.execution.stacktrace = call.excinfo.exconly()
 
     @staticmethod
     def _ensure_step_closed(step, status):
diff --git a/qase-tavern/tests/test_plugin.py b/qase-tavern/tests/test_plugin.py
index 07af8df5..c0ce1ff2 100644
--- a/qase-tavern/tests/test_plugin.py
+++ b/qase-tavern/tests/test_plugin.py
@@ -353,3 +353,85 @@ def test_ensure_step_closed_marks_unrun_skipped_as_zero_duration(self):
         assert step.execution.start_time == step.execution.end_time
         assert step.execution.duration == 0
         assert step.execution.status == "skipped"
+
+
+# ---------------------------------------------------------------------------
+# pytest_runtest_makereport -- failure status is derived from call.when,
+# not the exception type (#511)
+# ---------------------------------------------------------------------------
+
+
+def _make_call(when="call", excinfo=None):
+    """Create a mock pytest CallInfo."""
+    call_obj = MagicMock()
+    call_obj.when = when
+    call_obj.excinfo = excinfo
+    return call_obj
+
+
+def _make_excinfo(exconly="TestFailError: stage failed", value=None):
+    excinfo = MagicMock()
+    excinfo.exconly.return_value = exconly
+    excinfo.value = value if value is not None else Exception()
+    return excinfo
+
+
+class TestFailureStatusFromPhase:
+    """pytest-tavern raises its own TestFailError, never AssertionError --
+    the old exception-type check made every call-phase failure ``invalid``.
+    """
+
+    def test_call_phase_failure_is_failed(self):
+        """A Tavern stage failure (TestFailError) must be `failed`, not `invalid`."""
+        plugin = _make_plugin(with_result=True)
+        excinfo = _make_excinfo(value=Exception("stage failed"))
+
+        plugin.pytest_runtest_makereport(item=MagicMock(), call=_make_call(when="call", excinfo=excinfo))
+
+        assert plugin.runtime.result.execution.status == "failed"
+
+    def test_call_phase_pass_is_passed(self):
+        plugin = _make_plugin(with_result=True)
+
+        plugin.pytest_runtest_makereport(item=MagicMock(), call=_make_call(when="call", excinfo=None))
+
+        assert plugin.runtime.result.execution.status == "passed"
+
+    def test_setup_phase_fixture_failure_is_invalid(self):
+        """A broken pytest fixture used by a Tavern test is a broken test."""
+        plugin = _make_plugin(with_result=True)
+        excinfo = _make_excinfo(exconly="RuntimeError: fixture broke")
+
+        plugin.pytest_runtest_makereport(item=MagicMock(), call=_make_call(when="setup", excinfo=excinfo))
+
+        assert plugin.runtime.result.execution.status == "invalid"
+        assert plugin.runtime.result.execution.stacktrace == "RuntimeError: fixture broke"
+
+    def test_teardown_failure_after_pass_is_invalid(self):
+        plugin = _make_plugin(with_result=True)
+        plugin.runtime.result.execution.status = "passed"
+        excinfo = _make_excinfo(exconly="RuntimeError: cleanup never finished")
+
+        plugin.pytest_runtest_makereport(item=MagicMock(), call=_make_call(when="teardown", excinfo=excinfo))
+
+        assert plugin.runtime.result.execution.status == "invalid"
+
+    def test_teardown_failure_after_call_failure_keeps_failed_status(self):
+        """A broken teardown must never downgrade an existing call-phase failure."""
+        plugin = _make_plugin(with_result=True)
+        plugin.runtime.result.execution.status = "failed"
+        plugin.runtime.result.execution.stacktrace = "original call-phase stacktrace"
+        excinfo = _make_excinfo(exconly="RuntimeError: cleanup never finished")
+
+        plugin.pytest_runtest_makereport(item=MagicMock(), call=_make_call(when="teardown", excinfo=excinfo))
+
+        assert plugin.runtime.result.execution.status == "failed"
+        assert plugin.runtime.result.execution.stacktrace == "original call-phase stacktrace"
+
+    def test_teardown_success_does_not_touch_existing_status(self):
+        plugin = _make_plugin(with_result=True)
+        plugin.runtime.result.execution.status = "passed"
+
+        plugin.pytest_runtest_makereport(item=MagicMock(), call=_make_call(when="teardown", excinfo=None))
+
+        assert plugin.runtime.result.execution.status == "passed"

From 17c3791ba94ca0ee80ea92fd4a2e929f0395ddf4 Mon Sep 17 00:00:00 2001
From: Dmitrii Gridnev <dgridnev@qase.io>
Date: Mon, 31 Aug 2026 11:21:53 +0300
Subject: [PATCH 2/2] fix: update integration expected fixtures for phase-based
 status

The golden files under integration/*/expected/ pinned the exception-type
bug: a plain `assert False` in a behave step or a Robot Framework body
keyword was expected as `invalid`, and a Tavern TestFailError was
expected as `invalid` even though its own step was already recorded as
`failed` (an inconsistency the old logic produced). None of these
failures happen in setup/teardown/hooks, so under the phase-based fix
they're `failed`. qase-pytest's fixture set only exercises plain
call-phase asserts, so it needed no changes.

Verified locally against reporters-validator with isolated per-framework
venvs mirroring the CI job (only the relevant qase-* package installed
per venv, matching .github/workflows/pythonpackage.yml exactly) -- all
four integration suites pass.
---
 integration/behave/expected/behave-examples.yaml | 12 ++++++------
 integration/robot/expected/robot-examples.yaml   |  6 +++---
 integration/tavern/expected/tavern-examples.yaml |  6 +++---
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/integration/behave/expected/behave-examples.yaml b/integration/behave/expected/behave-examples.yaml
index 8da9440b..349599ff 100644
--- a/integration/behave/expected/behave-examples.yaml
+++ b/integration/behave/expected/behave-examples.yaml
@@ -1,8 +1,8 @@
 run:
   stats:
     blocked: 0
-    failed: 0
-    invalid: 2
+    failed: 2
+    invalid: 0
     muted: 0
     passed: 18
     skipped: 1
@@ -73,7 +73,7 @@ results:
   signature: 402::features::steps_feature.feature::test_with_failing_step
   testops_ids:
   - 402
-  status: invalid
+  status: failed
   relations:
     suite:
       data:
@@ -87,7 +87,7 @@ results:
   - data:
       action: When the user triggers a failure
     execution:
-      status: invalid
+      status: failed
 - title: Test in deeply nested suite
   signature: 303::authentication::oauth::google::test_in_deeply_nested_suite
   testops_ids:
@@ -188,7 +188,7 @@ results:
   signature: 102::features::basic.feature::simple_failing_test
   testops_ids:
   - 102
-  status: invalid
+  status: failed
   relations:
     suite:
       data:
@@ -198,7 +198,7 @@ results:
   - data:
       action: Given a failing condition
     execution:
-      status: invalid
+      status: failed
 - title: Test with file attachment
   signature: 601::features::attachments.feature::test_with_file_attachment
   testops_ids:
diff --git a/integration/robot/expected/robot-examples.yaml b/integration/robot/expected/robot-examples.yaml
index 53acd320..3f77a76c 100644
--- a/integration/robot/expected/robot-examples.yaml
+++ b/integration/robot/expected/robot-examples.yaml
@@ -1,8 +1,8 @@
 run:
   stats:
     blocked: 0
-    failed: 0
-    invalid: 1
+    failed: 1
+    invalid: 0
     muted: 0
     passed: 3
     skipped: 0
@@ -12,7 +12,7 @@ results:
   signature: 302::tests::steps::test_with_failing_step
   testops_ids:
   - 302
-  status: invalid
+  status: failed
   fields:
     description: ''
   relations:
diff --git a/integration/tavern/expected/tavern-examples.yaml b/integration/tavern/expected/tavern-examples.yaml
index ead6508b..d56a782e 100644
--- a/integration/tavern/expected/tavern-examples.yaml
+++ b/integration/tavern/expected/tavern-examples.yaml
@@ -1,8 +1,8 @@
 run:
   stats:
     blocked: 0
-    failed: 0
-    invalid: 1
+    failed: 1
+    invalid: 0
     muted: 0
     passed: 7
     skipped: 0
@@ -59,7 +59,7 @@ results:
   signature: 104::test_api.tavern.yaml::qaseid=104_intentionally_failing_test
   testops_ids:
   - 104
-  status: invalid
+  status: failed
   relations:
     suite:
       data: