fix: derive failure status from test phase instead of exception type

integration/behave/expected/behave-examples.yaml

@@ -1,8 +1,8 @@
 run:
   stats:
     blocked: 0
-    failed: 0
-    invalid: 2
+    failed: 2
+    invalid: 0
     muted: 0
     passed: 18
     skipped: 1
@@ -73,7 +73,7 @@ results:
   signature: 402::features::steps_feature.feature::test_with_failing_step
   testops_ids:
   - 402
-  status: invalid
+  status: failed
   relations:
     suite:
       data:
@@ -87,7 +87,7 @@ results:
   - data:
       action: When the user triggers a failure
     execution:
-      status: invalid
+      status: failed
 - title: Test in deeply nested suite
   signature: 303::authentication::oauth::google::test_in_deeply_nested_suite
   testops_ids:
@@ -188,7 +188,7 @@ results:
   signature: 102::features::basic.feature::simple_failing_test
   testops_ids:
   - 102
-  status: invalid
+  status: failed
   relations:
     suite:
       data:
@@ -198,7 +198,7 @@ results:
   - data:
       action: Given a failing condition
     execution:
-      status: invalid
+      status: failed
 - title: Test with file attachment
   signature: 601::features::attachments.feature::test_with_file_attachment
   testops_ids:

integration/robot/expected/robot-examples.yaml

@@ -1,8 +1,8 @@
 run:
   stats:
     blocked: 0
-    failed: 0
-    invalid: 1
+    failed: 1
+    invalid: 0
     muted: 0
     passed: 3
     skipped: 0
@@ -12,7 +12,7 @@ results:
   signature: 302::tests::steps::test_with_failing_step
   testops_ids:
   - 302
-  status: invalid
+  status: failed
   fields:
     description: ''
   relations:

integration/tavern/expected/tavern-examples.yaml

@@ -1,8 +1,8 @@
 run:
   stats:
     blocked: 0
-    failed: 0
-    invalid: 1
+    failed: 1
+    invalid: 0
     muted: 0
     passed: 7
     skipped: 0
@@ -59,7 +59,7 @@ results:
   signature: 104::test_api.tavern.yaml::qaseid=104_intentionally_failing_test
   testops_ids:
   - 104
-  status: invalid
+  status: failed
   relations:
     suite:
       data:

qase-behave/changelog.md

@@ -1,3 +1,22 @@
+# qase-behave 4.0.0
+
+## Breaking changes
+
+- Failure status is now derived from behave's own `hook_failed` signal instead of guessed by matching keywords (`assert`, `should`, `equal`, ...) against the error text. That heuristic was broken for the common case: behave's own failure message is `"Assertion Failed: ..."` (capital A), which never matched the lowercase `'assert'` keyword, so a literal `assert` in a step body was misclassified as `invalid`. Two further gaps are fixed alongside it:
+  - If a `before_scenario`/`before_step` hook raised, no steps ran at all and the scenario was silently reported as `passed` (the parser's optimistic default was never corrected, since `result()` is never called when a scenario's steps don't run). It's now reported as `invalid`, unless the scenario had already failed for real (a later `after_scenario` hook failure never downgrades a real step failure).
+  - On behave >=1.3, the newer `Status.error`/`Status.hook_error` names weren't in the status-mapping table and silently collapsed to `skipped`; they now route correctly.
+  ([#511](https://github.com/qase-tms/qase-python/issues/511))
+
+  | Failure phase | Old status (guessed from exception text) | New status (from behave's hook_failed) |
+  |---|---|---|
+  | Step body fails with what looks like an assertion (message contains `assert`/`should`/`equal`/...) | `failed` (in practice, almost never — see above) | `failed` |
+  | Step body fails with anything else (custom exception, timeout, behave's own `"Assertion Failed: ..."` text) | `invalid` — wrong | `failed` |
+  | `before_scenario`/`before_step` hook fails | `passed` (no steps ran) or `invalid`, inconsistently | `invalid` |
+  | `after_scenario`/`after_step` hook fails, scenario already passed | `passed` — wrong, hides the failure | `invalid` |
+  | `after_scenario`/`after_step` hook fails, scenario already failed | status could be overwritten by the hook's own message | scenario's status and diagnostics are kept untouched — the real failure always wins |
+
+  If your suite relies on the old behaviour, review any downstream logic (dashboards, defect linking, status filters) that branches on `invalid` vs `failed`.
+
 # qase-behave 3.2.0
 
 ## What's new

qase-behave/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "qase-behave"
-version = "3.2.2"
+version = "4.0.0"
 description = "Qase Behave Plugin for Qase TestOps and Qase Report"
 readme = "README.md"
 keywords = ["qase", "behave", "plugin", "testops", "report", "qase reporting", "test observability"]

qase-behave/src/qase/behave/formatter.py

@@ -26,6 +26,7 @@ def __init__(self, stream_opener=None, config=None):
         self.__already_started = False
         self.__case_ids = []
         self.__current_scenario = None
+        self.__current_raw_scenario = None
         self.__current_step_start = None
 
         if not self._behavex_mode:
@@ -73,15 +74,39 @@ def feature(self, feature: Feature):
             self.__case_ids, feature.scenarios)
 
     def scenario(self, scenario: Scenario):
-        if self.__current_scenario and self.__current_scenario.ignore == False:
-            self.__current_scenario.execution.complete()
-            self.reporter.add_result(self.__current_scenario)
-            self.__current_scenario = None
+        self.__finalize_current_scenario()
         self.__current_scenario = parse_scenario(scenario)
+        self.__current_raw_scenario = scenario
         qase._set_current_scenario(self.__current_scenario)
         qase._set_current_step(None)
         self.__current_step_start = None
 
+    def __finalize_current_scenario(self):
+        """Complete and send the in-flight scenario, if any.
+
+        A ``before_scenario``/``after_scenario`` hook failure is invisible
+        to ``result()`` -- if ``before_scenario`` fails, behave skips every
+        step so ``result()`` is never called at all, and the parsed
+        scenario keeps the optimistic 'passed' default set by
+        ``parse_scenario``. Check the raw behave ``Scenario``'s
+        ``hook_failed`` (present since behave 1.2.6) here instead, and
+        never let it downgrade a real step failure.
+        """
+        if self.__current_scenario is None or self.__current_scenario.ignore:
+            self.__current_scenario = None
+            self.__current_raw_scenario = None
+            return
+
+        if (self.__current_raw_scenario is not None and
+                getattr(self.__current_raw_scenario, 'hook_failed', False) and
+                self.__current_scenario.execution.status != 'failed'):
+            self.__current_scenario.execution.set_status('invalid')
+
+        self.__current_scenario.execution.complete()
+        self.reporter.add_result(self.__current_scenario)
+        self.__current_scenario = None
+        self.__current_raw_scenario = None
+
     def step(self, step):
         """Capture the real wall-clock start of the upcoming step.
 
@@ -100,13 +125,11 @@ def result(self, result: Step):
         qase._set_current_step(step)
 
         if step.execution.status != 'passed':
-            is_assertion_error = False
-            if result.error_message:
-                assertion_keywords = ['assert', 'AssertionError', 'expect', 'should', 'must']
-                is_assertion_error = any(keyword in result.error_message for keyword in assertion_keywords)
-
             if step.execution.status == 'failed':
-                status = 'failed' if is_assertion_error else 'invalid'
+                # A before_step/after_step hook failure is a broken step,
+                # like a setup/teardown error; any other step-body failure
+                # (assertion or otherwise) is a real test failure.
+                status = 'invalid' if getattr(result, 'hook_failed', False) else 'failed'
                 step.execution.set_status(status)
                 self.__current_scenario.execution.set_status(status)
             else:
@@ -118,10 +141,7 @@ def result(self, result: Step):
         qase._set_current_step(None)
 
     def eof(self):
-        if self.__current_scenario and self.__current_scenario.ignore == False:
-            self.__current_scenario.execution.complete()
-            self.reporter.add_result(self.__current_scenario)
-            self.__current_scenario = None
+        self.__finalize_current_scenario()
 
     def close(self):
         if self._is_behavex_worker:

qase-behave/src/qase/behave/utils.py

@@ -290,10 +290,17 @@ def parse_step(step: Step, start_time: float = None) -> QaseStep:
         data=StepGherkinData(keyword=step.keyword, name=step.name, line=step.line)
     )
 
-    # Map behave status to qase status
+    # Map behave status to qase status. The failed-vs-invalid decision is
+    # made in the formatter, from ``step.hook_failed`` (present since behave
+    # 1.2.6) rather than from any of these names, so 'error'/'hook_error'/
+    # 'cleanup_error' (behave >=1.3's richer Status enum; absent on 1.2.6)
+    # only need to route into the same "failed" branch here.
     status_mapping = {
         'passed': 'passed',
-        'failed': 'failed',  # This will be updated in formatter based on error type
+        'failed': 'failed',
+        'error': 'failed',
+        'hook_error': 'failed',
+        'cleanup_error': 'failed',
         'skipped': 'skipped',
         'undefined': 'skipped',
         'pending': 'skipped'

qase-behave/tests/test_formatter.py

@@ -397,6 +397,165 @@ def test_scenario_resets_step_start(self):
         assert formatter._QaseFormatter__current_step_start is None
 
 
+class TestStepFailureStatusFromHookFailed:
+    """A step's failed-vs-invalid status is derived from ``hook_failed``, not
+    from matching keywords against the error text (#511)."""
+
+    def _make_formatter(self):
+        formatter = QaseFormatter()
+        formatter._behavex_mode = False
+        formatter.reporter = MagicMock()
+        formatter._QaseFormatter__current_scenario = MagicMock()
+        formatter._QaseFormatter__current_scenario.ignore = False
+        formatter._QaseFormatter__current_scenario.steps = []
+        return formatter
+
+    @staticmethod
+    def _make_step(status="failed", hook_failed=False, error_message="boom"):
+        step = MagicMock()
+        step.keyword = "Then"
+        step.name = "a step"
+        step.line = 1
+        step.status.name = status
+        step.duration = 0.1
+        step.hook_failed = hook_failed
+        step.error_message = error_message
+        return step
+
+    def test_assertion_failure_in_step_body_is_failed(self):
+        """behave's own error text is 'Assertion Failed: ...' (capital A) --
+        the old lowercase keyword match never matched this."""
+        formatter = self._make_formatter()
+        step = self._make_step(
+            status="failed", hook_failed=False,
+            error_message="Assertion Failed: expected X")
+
+        formatter.result(step)
+
+        formatter._QaseFormatter__current_scenario.execution.set_status.assert_called_with('failed')
+
+    def test_custom_exception_in_step_body_is_failed(self):
+        formatter = self._make_formatter()
+        step = self._make_step(
+            status="failed", hook_failed=False,
+            error_message="cluster never became Ready within 900s")
+
+        formatter.result(step)
+
+        formatter._QaseFormatter__current_scenario.execution.set_status.assert_called_with('failed')
+
+    def test_before_or_after_step_hook_failure_is_invalid(self):
+        formatter = self._make_formatter()
+        step = self._make_step(
+            status="failed", hook_failed=True,
+            error_message="before_step hook exploded")
+
+        formatter.result(step)
+
+        formatter._QaseFormatter__current_scenario.execution.set_status.assert_called_with('invalid')
+
+    def test_undefined_step_status_is_untouched(self):
+        """Non-failed statuses (skipped/undefined/pending) pass through as before."""
+        formatter = self._make_formatter()
+        step = self._make_step(status="undefined", hook_failed=False, error_message=None)
+
+        formatter.result(step)
+
+        formatter._QaseFormatter__current_scenario.execution.set_status.assert_called_with('skipped')
+
+
+class TestScenarioHookFailure:
+    """before_scenario/after_scenario hook failures must not report a
+    scenario as 'passed' just because result() was never called (#511)."""
+
+    def _make_formatter(self):
+        formatter = QaseFormatter()
+        formatter._behavex_mode = False
+        formatter.reporter = MagicMock()
+        return formatter
+
+    @staticmethod
+    def _make_result(status="passed", ignore=False):
+        from qase.commons.models import Result
+        result = Result(title="s", signature="s")
+        result.execution.set_status(status)
+        result.ignore = ignore
+        return result
+
+    def test_before_scenario_hook_failure_is_invalid_not_passed(self):
+        """No steps ran (before_scenario hook exploded), so the parsed
+        scenario is still stuck on parse_scenario's optimistic 'passed'
+        default -- it must not survive."""
+        formatter = self._make_formatter()
+        result = self._make_result(status="passed")
+        raw_scenario = MagicMock(hook_failed=True)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter._QaseFormatter__finalize_current_scenario()
+
+        assert result.execution.status == "invalid"
+        formatter.reporter.add_result.assert_called_once_with(result)
+
+    def test_after_scenario_hook_failure_after_pass_is_invalid(self):
+        formatter = self._make_formatter()
+        result = self._make_result(status="passed")
+        raw_scenario = MagicMock(hook_failed=True)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter._QaseFormatter__finalize_current_scenario()
+
+        assert result.execution.status == "invalid"
+
+    def test_after_scenario_hook_failure_does_not_downgrade_a_real_failure(self):
+        """A step already failed the scenario -- a later after_scenario hook
+        failure must not overwrite that real failure with 'invalid'."""
+        formatter = self._make_formatter()
+        result = self._make_result(status="failed")
+        raw_scenario = MagicMock(hook_failed=True)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter._QaseFormatter__finalize_current_scenario()
+
+        assert result.execution.status == "failed"
+
+    def test_no_hook_failure_leaves_status_untouched(self):
+        formatter = self._make_formatter()
+        result = self._make_result(status="passed")
+        raw_scenario = MagicMock(hook_failed=False)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter._QaseFormatter__finalize_current_scenario()
+
+        assert result.execution.status == "passed"
+
+    def test_ignored_scenario_is_never_sent(self):
+        formatter = self._make_formatter()
+        result = self._make_result(status="passed", ignore=True)
+        raw_scenario = MagicMock(hook_failed=True)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter._QaseFormatter__finalize_current_scenario()
+
+        formatter.reporter.add_result.assert_not_called()
+
+    def test_eof_finalizes_a_hook_failed_scenario_too(self):
+        formatter = self._make_formatter()
+        result = self._make_result(status="passed")
+        raw_scenario = MagicMock(hook_failed=True)
+        formatter._QaseFormatter__current_scenario = result
+        formatter._QaseFormatter__current_raw_scenario = raw_scenario
+
+        formatter.eof()
+
+        assert result.execution.status == "invalid"
+        formatter.reporter.add_result.assert_called_once_with(result)
+
+
 class TestBehaveXWorkerMode:
     """Test QaseFormatter in BehaveX worker mode (lock file coordination)."""