From abdf44299967d5eb409f72fb60b775a8fd869516 Mon Sep 17 00:00:00 2001 From: Balaji Venkatesan Date: Wed, 28 Oct 2026 16:23:56 +0530 Subject: [PATCH 01/10] Report abort results reliably and let tasks stop cooperatively --- README.md | 13 + digitalai/release/integration/base_task.py | 30 +- digitalai/release/integration/wrapper.py | 208 +++++++++---- docs/classes/base_task.md | 14 + docs/examples.md | 4 +- tests/release/integration/cooperative_task.py | 19 ++ .../release/integration/test_wrapper_abort.py | 277 ++++++++++++++++-- 7 files changed, 486 insertions(+), 79 deletions(-) create mode 100644 tests/release/integration/cooperative_task.py diff --git a/README.md b/README.md index e6eeb3b..ea07d3a 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,19 @@ class ShowVersion(ApiBaseTask): ## Changelog +### Unreleased + +#### 🚀 Features + +- Tasks can be aborted: the SDK runs `execute()` on a worker thread and handles `SIGABRT`/`SIGTERM` by calling the task's `abort()` hook. Long-running tasks can poll the new `is_abort_requested()` to stop cooperatively. +- After an abort is requested, the SDK waits up to `ABORT_GRACE_PERIOD_SECONDS` (default 3) for `execute()` to return, so the outputs and reporting records a task sets on its way out are reported. + +#### 🐛 Fixes + +- A signal received while the task result is being reported no longer cuts off the report; repeat signals during an abort are ignored. +- An aborted result is never reported with exit code 0, and a task calling `sys.exit(n)` is reported with exit code `n`. +- Tasks whose `__init__` does not call `super().__init__()` work again. + ### Version 26.3.1 #### 🚀 Features diff --git a/digitalai/release/integration/base_task.py b/digitalai/release/integration/base_task.py index 07efa82..0441e93 100644 --- a/digitalai/release/integration/base_task.py +++ b/digitalai/release/integration/base_task.py @@ -58,6 +58,12 @@ def abort(self) -> None: immediately. When called from the SDK's signal handler (outside of execute_task()), the caller is responsible for catching this exception and performing its own cleanup/exit. + + The SDK's signal handler runs abort() on the main thread while execute() + may still be running on the worker thread, so an override must be + thread-safe. Output properties and reporting records set in an override + are sent with the aborted result; set them before calling super().abort(), + since it raises. """ self.set_exit_code(1) self.set_error_message("Abort requested") @@ -69,26 +75,38 @@ def is_abort_requested(self) -> bool: (e.g. the process received SIGABRT/SIGTERM). Tasks may poll this from within execute() to cooperatively stop long-running work promptly. """ - return self._abort_event.is_set() + abort_event = getattr(self, "_abort_event", None) + return abort_event is not None and abort_event.is_set() def _request_abort(self) -> None: """ Internal SDK hook: signals is_abort_requested() to start returning True. Not intended to be called by task authors. """ + if getattr(self, "_abort_event", None) is None: + self._abort_event = threading.Event() self._abort_event.set() + def _ensure_output_context(self) -> None: + """ + Creates the output context if it is missing, for subclasses whose + __init__ does not call super().__init__(). + """ + if getattr(self, "output_context", None) is None: + self.output_context = OutputContext(0, "", {}, []) + def get_output_context(self) -> OutputContext: """ Returns the OutputContext object associated with the task. """ + self._ensure_output_context() return self.output_context def get_output_properties(self) -> Dict[str, Any]: """ Returns the output properties dictionary of the task's OutputContext object. """ - return self.output_context.output_properties + return self.get_output_context().output_properties def get_input_properties(self) -> Dict[str, Any]: """ @@ -112,19 +130,19 @@ def set_output_property(self, name: str, value: Any) -> None: f"Invalid data type for value '{value}' in name '{name}' in set_output_property. Accepted data types " f"are: str, int, list, dict, bool") - self.output_context.output_properties[name] = value + self.get_output_context().output_properties[name] = value def set_exit_code(self, value) -> None: """ Sets the exit code of the task's OutputContext object. """ - self.output_context.exit_code = value + self.get_output_context().exit_code = value def set_error_message(self, value) -> None: """ Sets the error message of the task's OutputContext object. """ - self.output_context.job_error_message = value + self.get_output_context().job_error_message = value def add_comment(self, comment: str) -> None: """ @@ -142,7 +160,7 @@ def add_reporting_record(self, reporting_record: Any) -> None: """ Adds a reporting record to the OutputContext """ - self.output_context.reporting_records.append(reporting_record) + self.get_output_context().reporting_records.append(reporting_record) def get_release_server_url(self) -> str: """ diff --git a/digitalai/release/integration/wrapper.py b/digitalai/release/integration/wrapper.py index d5348ee..96062b5 100644 --- a/digitalai/release/integration/wrapper.py +++ b/digitalai/release/integration/wrapper.py @@ -51,6 +51,12 @@ # execute_task()) via join(), so it stays responsive to SIGABRT/SIGTERM. WORKER_POLL_INTERVAL_SECONDS = 0.5 +# How long the abort handler waits for the task's execute() to stop after an +# abort is requested, before it reports the aborted result and exits. Keep it +# below the task type's abortTimeout (10s when unset), after which the runner +# force-kills the container. +ABORT_GRACE_PERIOD_SECONDS = float(os.getenv('ABORT_GRACE_PERIOD_SECONDS', '3')) + # A single Session reused across all callback requests so the underlying urllib3 # connection pool is shared (instead of opening a fresh connection per call). _http_session: requests.Session = requests.Session() @@ -69,83 +75,167 @@ def get_encryptor(): # Initialize the global task object dai_task_object: Optional[BaseTask] = None -# Single-writer guard so the output context is flushed exactly once, whether -# the task finishes normally (worker thread's completion path) or is aborted -# (main thread's signal handler). Reset at the start of each execute_task() -# call so daemon-mode iterations each get a fresh guard. +# Report state for the current task execution. The output context is reported +# exactly once, either by execute_task() after the worker finishes or by +# abort_handler(). Both run on the main thread (Python runs signal handlers +# there), so a signal can arrive while a report is in progress; the state lets +# the handler leave that report to finish instead of exiting mid-push. +# Reset at the start of each execute_task() call so daemon-mode iterations +# each get a fresh state. +_REPORT_PENDING = "pending" +_REPORT_IN_PROGRESS = "in_progress" +_REPORT_DONE = "done" + _report_lock = threading.RLock() -_result_reported = False +_report_state = _REPORT_PENDING + +# True while abort_handler() is handling a signal; further signals are ignored. +_abort_in_progress = False + +# Set when a signal arrives during a report, so the process exits once the +# report finishes. +_exit_after_report = False + +# The worker thread running the current task, which abort_handler() waits on +# for up to ABORT_GRACE_PERIOD_SECONDS. +_worker: Optional[threading.Thread] = None def _reset_report_guard() -> None: - """Clears the single-writer flush guard ahead of a new task execution.""" - global _result_reported + """Resets the report and abort state ahead of a new task execution.""" + global _report_state, _abort_in_progress, _exit_after_report, _worker with _report_lock: - _result_reported = False + _report_state = _REPORT_PENDING + _abort_in_progress = False + _exit_after_report = False + _worker = None -def _report_once() -> bool: +def _snapshot_output_context(task: Optional[BaseTask], no_task_message: str) -> OutputContext: """ - Flushes the current dai_task_object's output context exactly once. The - first caller (either the worker thread's normal completion path or the - main thread's abort handler) wins; subsequent calls are no-ops. + Returns a copy of the task's output context, so the result is serialized + from a copy the worker thread cannot change mid-serialization. + """ + if task is None: + dai_logger.error(no_task_message) + return OutputContext(1, no_task_message, {}, []) + context = task.get_output_context() + return OutputContext( + context.exit_code, + context.job_error_message, + dict(context.output_properties or {}), + list(context.reporting_records or []), + ) - Returns True if this call performed the flush, False if it was already - reported by another caller. + +def _flush_std_streams() -> None: + """Flushes stdout/stderr before os._exit(), which skips Python's own flush.""" + for stream in (sys.stdout, sys.stderr): + try: + stream.flush() + except Exception: + pass + + +def _exit_process(code: int) -> None: + _flush_std_streams() + os._exit(code) + + +def _report_once(aborted: bool = False, + no_task_message: str = "Task produced no output context") -> bool: """ - global _result_reported + Reports the current dai_task_object's output context exactly once. The + first caller (execute_task() after the worker finishes, or abort_handler()) + wins; later calls are no-ops. + + When aborted is True, the reported result never has exit code 0, even if + the task's abort() override did not set one. + + Returns True if this call reported the result, False if it was already + reported or is being reported. + """ + global _report_state with _report_lock: - if _result_reported: + if _report_state != _REPORT_PENDING: return False - _result_reported = True + _report_state = _REPORT_IN_PROGRESS - if dai_task_object is not None and dai_task_object.get_output_context() is not None: - update_output_context(dai_task_object.get_output_context()) - else: - dai_logger.error("No output context available to report task result") - update_output_context(OutputContext(1, "Task produced no output context", {}, [])) + try: + context = _snapshot_output_context(dai_task_object, no_task_message) + if aborted and not context.exit_code: + context.exit_code = 1 + context.job_error_message = context.job_error_message or "Abort requested" + update_output_context(context) + finally: + with _report_lock: + _report_state = _REPORT_DONE + if _exit_after_report: + dai_logger.info("Exiting after reporting the task result, as an abort was requested") + _exit_process(1) return True def abort_handler(signum, frame): """ - Handles SIGTERM/SIGABRT by requesting a cooperative abort and running the - task's abort() cleanup hook (if a task object exists), then flushing the - (aborted) output context exactly once via _report_once(), and finally - terminating the process immediately with os._exit(1). + Handles SIGTERM/SIGABRT: + + 1. signals is_abort_requested() and runs the task's abort() hook; + 2. waits up to ABORT_GRACE_PERIOD_SECONDS for execute() to stop, so a task + that polls is_abort_requested() can finish and have its outputs reported; + 3. reports the aborted result exactly once and exits with os._exit(1). + + If the signal arrives while a result is being reported, the report is left + to finish and the process exits right after it. Signals received while an + abort is already being handled are ignored. os._exit() (rather than sys.exit()) is used deliberately: it bypasses all Python finally/atexit machinery, which matters in daemon mode where run()'s finally block would otherwise restart a blocking input-context watcher and prevent the process from actually exiting. - - The worker thread that may still be running the task's execute() is a - daemon thread and is not waited on; process exit here is sufficient to - end it. """ + global _abort_in_progress, _exit_after_report signal_name = signal.Signals(signum).name + + with _report_lock: + if _abort_in_progress: + dai_logger.info(f"Received {signal_name}, ignored: abort already in progress") + return + if _report_state == _REPORT_IN_PROGRESS: + dai_logger.info(f"Received {signal_name} while reporting the task result; exiting once it is reported") + _exit_after_report = True + return + _abort_in_progress = True + dai_logger.info(f"Received {signal_name} to gracefully stop the process") - global dai_task_object - if dai_task_object: - dai_task_object._request_abort() + task = dai_task_object + if task is not None: + task._request_abort() try: - dai_task_object.abort() + task.abort() except AbortException: # Expected: the default abort() raises this after marking the # output context as aborted. pass except SystemExit: # A task's overridden abort() might still call sys.exit()/exit(); - # ignore it so we can still flush and exit deliberately below. + # ignore it so we can still report and exit deliberately below. pass except Exception: dai_logger.error("Unexpected error occurred in abort()", exc_info=True) + + worker = _worker + if worker is not None and worker.is_alive() and ABORT_GRACE_PERIOD_SECONDS > 0: + dai_logger.info(f"Waiting up to {ABORT_GRACE_PERIOD_SECONDS}s for the task to stop") + worker.join(timeout=ABORT_GRACE_PERIOD_SECONDS) + if worker.is_alive(): + dai_logger.info("Task did not stop within the grace period; reporting the aborted result") else: dai_logger.info("Abort requested") - _report_once() - os._exit(1) + _report_once(aborted=True, no_task_message="Abort requested before the task started") + _exit_process(1) # Register abort handler @@ -292,6 +382,16 @@ def should_retry_callback_request(encrypted_data) -> bool: return len(encrypted_data) >= size_of_1Mb and not input_context_file +def _exit_code_of(system_exit: SystemExit) -> int: + """Returns the process exit status that sys.exit(code) produces.""" + code = system_exit.code + if code is None: + return 0 + if isinstance(code, int): + return code + return 1 + + def execute_task(task_object: BaseTask): """ Execute the given task object on a dedicated daemon worker thread, while the @@ -300,18 +400,21 @@ def execute_task(task_object: BaseTask): an uninterruptible call. The worker thread is a daemon thread: if the process exits (e.g. via the - signal handler's os._exit(1) during an abort), it is not waited on. + signal handler's os._exit(1) during an abort), it is not waited on beyond + the abort grace period. After the worker finishes on its own (the non-aborted path), the output - context is flushed exactly once via _report_once() -- a no-op if the + context is reported exactly once via _report_once() -- a no-op if the abort handler already reported it first. If the worker raised SystemExit (e.g. BaseTask.execute_task()'s sys.exit(1) after an AbortException, or a user task calling sys.exit() directly), it is captured on the worker thread and re-raised here on the main thread after - reporting, so the process exits with the intended status. + reporting, so the process exits with the intended status. A non-zero + sys.exit() code is also reported as the task's exit code when the task + did not set one itself. """ - global dai_task_object + global dai_task_object, _worker _reset_report_guard() dai_task_object = task_object dai_logger.info("Starting task execution") @@ -325,29 +428,28 @@ def _run_task(): # SystemExit (a BaseException, not Exception) only terminates this # worker thread and is otherwise swallowed by the interpreter. # Capture it so the main thread can propagate the intended process - # exit status after the result is reported. This covers both - # BaseTask.execute_task()'s sys.exit(1) after an AbortException and - # a user task calling sys.exit() directly. + # exit status after the result is reported. worker_outcome["exit"] = e except Exception: dai_logger.error("Unexpected error occurred.", exc_info=True) worker = threading.Thread(target=_run_task, name="dai-task-worker", daemon=True) + _worker = worker worker.start() while worker.is_alive(): worker.join(timeout=WORKER_POLL_INTERVAL_SECONDS) + if "exit" in worker_outcome: + exit_code = _exit_code_of(worker_outcome["exit"]) + context = task_object.get_output_context() + if exit_code and not context.exit_code: + context.exit_code = exit_code + if not context.job_error_message: + code = worker_outcome["exit"].code + context.job_error_message = code if isinstance(code, str) else f"Task exited with code {exit_code}" + _report_once() - # Re-raise the worker's SystemExit on the main thread (after reporting) so - # the process exits with the status the task intended, matching the - # pre-worker-thread (main-branch) behavior. The abort-handler path - # (os._exit) is separate and unaffected. - # - # TODO: Should the worker's SystemExit code also be reflected in the - # reported OutputContext.exit_code? Currently (matching master) the context - # is reported as-is, so a task calling sys.exit(2) without set_exit_code() - # reports a success context (exit_code 0) while the process exits 2. if "exit" in worker_outcome: raise worker_outcome["exit"] diff --git a/docs/classes/base_task.md b/docs/classes/base_task.md index cba29e9..9b84d8b 100644 --- a/docs/classes/base_task.md +++ b/docs/classes/base_task.md @@ -115,6 +115,13 @@ For long-running tasks, prefer polling [`is_abort_requested`](#is_abort_requeste from within `execute` to stop cooperatively and promptly, rather than relying solely on `abort` being called. +When the SDK calls `abort` on a signal, it runs on the main thread while `execute` +may still be running on the worker thread, so an override must be thread-safe. +Output properties and reporting records set in an override are sent with the +aborted result; set them before calling `super().abort()`, since it raises. The +aborted result is always reported with a non-zero exit code, even if an override +does not set one. + _No parameters._ **Returns:** _None._ @@ -137,6 +144,13 @@ class MyTask(BaseTask): do_one_unit_of_work() ``` +After an abort is requested, the SDK waits up to `ABORT_GRACE_PERIOD_SECONDS` +(environment variable, default `3`) for `execute` to return, then reports the +aborted result and exits. Output properties and reporting records the task sets +before it returns within that time are included in the result. Keep the grace +period below the task type's `abortTimeout` (10 seconds when unset), after which +the runner force-kills the container. + _No parameters._ **Returns:** `bool` — `True` if an abort has been requested, `False` otherwise. diff --git a/docs/examples.md b/docs/examples.md index 700eb61..977a5ec 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -131,7 +131,9 @@ For long-running tasks, the SDK also handles `SIGABRT`/`SIGTERM` (e.g. when a Release job is aborted) by running `abort` on your task automatically — you don't need to install a signal handler yourself. If your `execute` performs work in a loop or in chunks, poll `is_abort_requested` to stop promptly instead -of waiting for the whole task to finish: +of waiting for the whole task to finish. The SDK waits up to +`ABORT_GRACE_PERIOD_SECONDS` (default `3`) for `execute` to return, so anything +the task records on its way out is reported with the aborted result: ```python class LongRunningTask(BaseTask): diff --git a/tests/release/integration/cooperative_task.py b/tests/release/integration/cooperative_task.py new file mode 100644 index 0000000..1f3ede4 --- /dev/null +++ b/tests/release/integration/cooperative_task.py @@ -0,0 +1,19 @@ +import time + +from digitalai.release.integration import BaseTask + + +class CooperativeTask(BaseTask): + """ + Fixture task used by test_wrapper_abort.py to verify that a task polling + is_abort_requested() can stop within the abort grace period and have the + outputs it sets on the way out reported. + """ + + def execute(self) -> None: + processed = 0 + while not self.is_abort_requested(): + processed += 1 + time.sleep(0.05) + self.set_output_property("stoppedCooperatively", True) + self.add_reporting_record({"processed": processed}) diff --git a/tests/release/integration/test_wrapper_abort.py b/tests/release/integration/test_wrapper_abort.py index c883498..487fcad 100644 --- a/tests/release/integration/test_wrapper_abort.py +++ b/tests/release/integration/test_wrapper_abort.py @@ -227,7 +227,7 @@ def execute(self) -> None: self.assertEqual(reported_ctx.exit_code, 1) self.assertEqual(reported_ctx.job_error_message, "Abort requested") - def test_direct_sys_exit_propagates_status_and_reports_context_as_is(self): + def test_direct_sys_exit_propagates_status_and_reports_its_code(self): class _ExitingTask(BaseTask): def execute(self) -> None: sys.exit(2) @@ -240,9 +240,34 @@ def execute(self) -> None: # The exact SystemExit code must be preserved. self.assertEqual(ctx.exception.code, 2) - # Strict master parity: the output context is reported as-is (default - # success), not reconciled to the SystemExit code. + # ...and reported, so Release does not see a success. mock_update.assert_called_once() + (reported_ctx,), _ = mock_update.call_args + self.assertEqual(reported_ctx.exit_code, 2) + self.assertEqual(reported_ctx.job_error_message, "Task exited with code 2") + + def test_sys_exit_with_message_reports_exit_code_1_and_the_message(self): + class _ExitingTask(BaseTask): + def execute(self) -> None: + sys.exit("Configuration missing") + + with mock.patch.object(wrapper, "update_output_context") as mock_update: + with self.assertRaises(SystemExit): + wrapper.execute_task(_ExitingTask()) + + (reported_ctx,), _ = mock_update.call_args + self.assertEqual(reported_ctx.exit_code, 1) + self.assertEqual(reported_ctx.job_error_message, "Configuration missing") + + def test_sys_exit_zero_reports_success(self): + class _ExitingTask(BaseTask): + def execute(self) -> None: + sys.exit(0) + + with mock.patch.object(wrapper, "update_output_context") as mock_update: + with self.assertRaises(SystemExit): + wrapper.execute_task(_ExitingTask()) + (reported_ctx,), _ = mock_update.call_args self.assertEqual(reported_ctx.exit_code, 0) @@ -260,17 +285,216 @@ def execute(self) -> None: mock_update.assert_called_once() +class WrapperAbortGracePeriodTests(unittest.TestCase): + """ + abort_handler() waits for the worker for up to ABORT_GRACE_PERIOD_SECONDS + before reporting, guards an in-progress report, and ignores repeat signals. + """ + + def setUp(self): + wrapper._reset_report_guard() + wrapper.dai_task_object = None + + def tearDown(self): + wrapper._reset_report_guard() + wrapper.dai_task_object = None + + def _start_worker(self, task): + """Starts task.execute_task() on a worker thread, as execute_task() does.""" + wrapper.dai_task_object = task + worker = threading.Thread(target=task.execute_task, daemon=True) + wrapper._worker = worker + worker.start() + return worker + + def test_cooperative_stop_within_grace_period_is_reported(self): + class _PollingTask(BaseTask): + def execute(self) -> None: + while not self.is_abort_requested(): + time.sleep(0.01) + self.set_output_property("processed", 3) + self.add_reporting_record({"status": "ABORTED"}) + + task = _PollingTask() + worker = self._start_worker(task) + + with mock.patch.object(wrapper, "update_output_context") as mock_update, \ + mock.patch.object(wrapper.os, "_exit") as mock_exit: + wrapper.abort_handler(signal.SIGABRT, None) + + self.assertFalse(worker.is_alive()) + mock_update.assert_called_once() + (reported_ctx,), _ = mock_update.call_args + self.assertEqual(reported_ctx.exit_code, 1) + self.assertEqual(reported_ctx.job_error_message, "Abort requested") + self.assertEqual(reported_ctx.output_properties, {"processed": 3}) + self.assertEqual(reported_ctx.reporting_records, [{"status": "ABORTED"}]) + mock_exit.assert_called_once_with(1) + + def test_task_that_does_not_stop_is_reported_after_grace_period(self): + release = threading.Event() + + class _StuckTask(BaseTask): + def execute(self) -> None: + release.wait(10) + + task = _StuckTask() + self._start_worker(task) + + try: + with mock.patch.object(wrapper, "ABORT_GRACE_PERIOD_SECONDS", 0.2), \ + mock.patch.object(wrapper, "update_output_context") as mock_update, \ + mock.patch.object(wrapper.os, "_exit") as mock_exit: + start = time.monotonic() + wrapper.abort_handler(signal.SIGABRT, None) + elapsed = time.monotonic() - start + finally: + release.set() + + self.assertGreaterEqual(elapsed, 0.2) + self.assertLess(elapsed, 2.0) + (reported_ctx,), _ = mock_update.call_args + self.assertEqual(reported_ctx.exit_code, 1) + mock_exit.assert_called_once_with(1) + + def test_reported_result_is_a_copy_of_the_output_context(self): + task = _RecordingAbortTask() + task.set_output_property("a", 1) + wrapper.dai_task_object = task + + with mock.patch.object(wrapper, "update_output_context") as mock_update, \ + mock.patch.object(wrapper.os, "_exit"): + wrapper.abort_handler(signal.SIGABRT, None) + + (reported_ctx,), _ = mock_update.call_args + self.assertIsNot(reported_ctx, task.get_output_context()) + self.assertIsNot(reported_ctx.output_properties, task.get_output_properties()) + self.assertEqual(reported_ctx.output_properties, {"a": 1}) + + def test_signal_during_report_lets_the_report_finish_then_exits(self): + class _FastTask(BaseTask): + def execute(self) -> None: + self.set_output_property("greeting", "Hello") + + pushed = [] + + def _push_interrupted_by_signal(context): + # The signal handler runs on the main thread in the middle of the push. + wrapper.abort_handler(signal.SIGABRT, None) + pushed.append(context) + + with mock.patch.object(wrapper, "update_output_context", side_effect=_push_interrupted_by_signal), \ + mock.patch.object(wrapper.os, "_exit") as mock_exit: + wrapper.execute_task(_FastTask()) + + # The push completed with the task's own result, and only then did the process exit. + self.assertEqual(len(pushed), 1) + self.assertEqual(pushed[0].exit_code, 0) + self.assertEqual(pushed[0].output_properties, {"greeting": "Hello"}) + mock_exit.assert_called_once_with(1) + + def test_second_signal_during_abort_is_ignored(self): + class _ReentrantAbortTask(BaseTask): + def execute(self) -> None: # pragma: no cover - not used directly + pass + + def abort(self) -> None: + # A second signal arrives while the first is being handled. + wrapper.abort_handler(signal.SIGTERM, None) + super().abort() + + wrapper.dai_task_object = _ReentrantAbortTask() + + with mock.patch.object(wrapper, "update_output_context") as mock_update, \ + mock.patch.object(wrapper.os, "_exit") as mock_exit: + wrapper.abort_handler(signal.SIGABRT, None) + + mock_update.assert_called_once() + mock_exit.assert_called_once_with(1) + + def test_aborted_result_never_reports_exit_code_0(self): + class _QuietAbortTask(BaseTask): + def execute(self) -> None: # pragma: no cover - not used directly + pass + + def abort(self) -> None: + # Overrides abort() without calling super() or setting an exit code. + self.set_output_property("cleaned_up", True) + + wrapper.dai_task_object = _QuietAbortTask() + + with mock.patch.object(wrapper, "update_output_context") as mock_update, \ + mock.patch.object(wrapper.os, "_exit"): + wrapper.abort_handler(signal.SIGABRT, None) + + (reported_ctx,), _ = mock_update.call_args + self.assertEqual(reported_ctx.exit_code, 1) + self.assertEqual(reported_ctx.job_error_message, "Abort requested") + self.assertEqual(reported_ctx.output_properties, {"cleaned_up": True}) + + def test_signal_before_task_exists_reports_a_clear_message(self): + with mock.patch.object(wrapper, "update_output_context") as mock_update, \ + mock.patch.object(wrapper.os, "_exit"): + wrapper.abort_handler(signal.SIGTERM, None) + + (reported_ctx,), _ = mock_update.call_args + self.assertEqual(reported_ctx.exit_code, 1) + self.assertEqual(reported_ctx.job_error_message, "Abort requested before the task started") + + +class _NoSuperInitTask(BaseTask): + """Task whose __init__ does not call super().__init__().""" + + def __init__(self): + self.calls = 0 + + def execute(self) -> None: + self.set_output_property("ok", True) + + +class WrapperTaskWithoutSuperInitTests(unittest.TestCase): + + def setUp(self): + wrapper._reset_report_guard() + wrapper.dai_task_object = None + + def tearDown(self): + wrapper._reset_report_guard() + wrapper.dai_task_object = None + + def test_runs_and_reports_outputs(self): + with mock.patch.object(wrapper, "update_output_context") as mock_update: + wrapper.execute_task(_NoSuperInitTask()) + + (reported_ctx,), _ = mock_update.call_args + self.assertEqual(reported_ctx.exit_code, 0) + self.assertEqual(reported_ctx.output_properties, {"ok": True}) + + def test_can_be_aborted(self): + task = _NoSuperInitTask() + self.assertFalse(task.is_abort_requested()) + wrapper.dai_task_object = task + + with mock.patch.object(wrapper, "update_output_context") as mock_update, \ + mock.patch.object(wrapper.os, "_exit"): + wrapper.abort_handler(signal.SIGABRT, None) + + self.assertTrue(task.is_abort_requested()) + (reported_ctx,), _ = mock_update.call_args + self.assertEqual(reported_ctx.exit_code, 1) + + +@unittest.skipIf(sys.platform == "win32", "Windows cannot deliver SIGABRT to a child process") class WrapperRealSigabrtEndToEndTest(unittest.TestCase): """ - Full end-to-end test: runs the real wrapper module as a subprocess against a - task that blocks in execute() without polling is_abort_requested(), sends it - a genuine SIGABRT partway through, and asserts it reports an aborted result - promptly instead of waiting for the 30s sleep to finish. + Full end-to-end tests: run the real wrapper module as a subprocess, send it + a genuine SIGABRT partway through (as the release-runner does), and check + the reported result. """ def setUp(self): - self.input_path = os.path.join(THIS_DIR, "blocking_input.json") - self.output_path = os.path.join(THIS_DIR, "blocking_output.json") + self.input_path = os.path.join(THIS_DIR, "abort_input.json") + self.output_path = os.path.join(THIS_DIR, "abort_output.json") if os.path.exists(self.output_path): os.remove(self.output_path) @@ -279,7 +503,8 @@ def tearDown(self): if os.path.exists(path): os.remove(path) - def test_real_sigabrt_interrupts_blocking_task(self): + def _run_and_abort(self, task_type: str): + """Runs the wrapper on task_type, sends SIGABRT after 1s, and returns (seconds to exit, output).""" input_context = { "release": { "id": "Applications/Folder1f65c7220b394afbb941154342fd9fc6/Release31de09e95c8e4ebb95aaed29a8082d0b", @@ -287,7 +512,7 @@ def test_real_sigabrt_interrupts_blocking_task(self): }, "task": { "id": "Applications/Folder1f65c7220b394afbb941154342fd9fc6/Release31de09e95c8e4ebb95aaed29a8082d0b/Phase723a601c78804f7dbcaa8b05b83708f5/Task3a35b67b42b6428b854857fba470b39a", - "type": "containerExamples.BlockingTask", + "type": task_type, "properties": [ {"name": "capabilities", "value": ["remote"], "kind": "SET_OF_STRING", "category": "input", "password": False}, ], @@ -297,8 +522,8 @@ def test_real_sigabrt_interrupts_blocking_task(self): json.dump(input_context, f) env = dict(os.environ) - env["INPUT_LOCATION"] = "blocking_input.json" - env["OUTPUT_LOCATION"] = "blocking_output.json" + env["INPUT_LOCATION"] = os.path.basename(self.input_path) + env["OUTPUT_LOCATION"] = os.path.basename(self.output_path) env["RELEASE_URL"] = "http://localhost:5516" proc = subprocess.Popen( @@ -310,7 +535,7 @@ def test_real_sigabrt_interrupts_blocking_task(self): text=True, ) try: - # Give the task time to enter its blocking sleep(30) call. + # Give the task time to start running execute(). time.sleep(1.0) start = time.monotonic() proc.send_signal(signal.SIGABRT) @@ -327,16 +552,30 @@ def test_real_sigabrt_interrupts_blocking_task(self): proc.returncode, 1, msg=f"wrapper exited with {proc.returncode}\nstdout:\n{stdout}\nstderr:\n{stderr}", ) - # Must be well under the task's 30s sleep -- proves the abort was - # timely rather than relying on the runner's Phase-2 force-kill. + self.assertTrue(os.path.exists(self.output_path), "wrapper did not produce an output file") + with open(self.output_path) as f: + return elapsed, json.load(f) + + def test_real_sigabrt_interrupts_blocking_task(self): + elapsed, actual_output = self._run_and_abort("containerExamples.BlockingTask") + + # The task ignores the abort, so the wrapper reports after the grace + # period -- well under the task's 30s sleep and the runner's force-kill. + self.assertGreaterEqual(elapsed, wrapper.ABORT_GRACE_PERIOD_SECONDS) self.assertLess(elapsed, 10.0) + self.assertEqual(actual_output["exitCode"], 1) + self.assertEqual(actual_output["jobErrorMessage"], "Abort requested") - self.assertTrue(os.path.exists(self.output_path), "wrapper did not produce blocking_output.json") - with open(self.output_path) as f: - actual_output = json.load(f) + def test_real_sigabrt_reports_outputs_of_cooperative_task(self): + elapsed, actual_output = self._run_and_abort("containerExamples.CooperativeTask") + # The task stops as soon as it sees the abort, well within the grace period. + self.assertLess(elapsed, wrapper.ABORT_GRACE_PERIOD_SECONDS) self.assertEqual(actual_output["exitCode"], 1) self.assertEqual(actual_output["jobErrorMessage"], "Abort requested") + self.assertEqual(actual_output["outputProperties"], {"stoppedCooperatively": True}) + self.assertEqual(len(actual_output["reportingRecords"]), 1) + self.assertGreater(actual_output["reportingRecords"][0]["processed"], 0) if __name__ == '__main__': From b19022cd30366a08677f5d522fbfebced81b07ee Mon Sep 17 00:00:00 2001 From: Balaji Venkatesan Date: Wed, 28 Oct 2026 21:51:27 +0530 Subject: [PATCH 02/10] Document the abort fixes and correct the default abortTimeout --- digitalai/release/integration/wrapper.py | 4 +- docs/abort-changes.md | 369 +++++++++++++++++++++++ docs/classes/base_task.md | 6 +- 3 files changed, 375 insertions(+), 4 deletions(-) create mode 100644 docs/abort-changes.md diff --git a/digitalai/release/integration/wrapper.py b/digitalai/release/integration/wrapper.py index 96062b5..52e112e 100644 --- a/digitalai/release/integration/wrapper.py +++ b/digitalai/release/integration/wrapper.py @@ -53,8 +53,8 @@ # How long the abort handler waits for the task's execute() to stop after an # abort is requested, before it reports the aborted result and exits. Keep it -# below the task type's abortTimeout (10s when unset), after which the runner -# force-kills the container. +# below the task type's abortTimeout (30s by default on xlrelease.ContainerTask; +# the runner uses 10s if it is 0), after which the runner force-kills the container. ABORT_GRACE_PERIOD_SECONDS = float(os.getenv('ABORT_GRACE_PERIOD_SECONDS', '3')) # A single Session reused across all callback requests so the underlying urllib3 diff --git a/docs/abort-changes.md b/docs/abort-changes.md new file mode 100644 index 0000000..f624bf2 --- /dev/null +++ b/docs/abort-changes.md @@ -0,0 +1,369 @@ +# Abort handling: issues on `main` and the fixes on `ReportRecords` + +This document records the problems found in the abort support that was merged to `main` in +`75990ee` (D-43840, "Support abort signal in container python plugin"), and how the +`ReportRecords` branch fixes them. It covers every file the branch changes. + +- **Baseline:** `main` at `75990ee`. +- **Branch:** `ReportRecords`, commit `abdf442` ("Report abort results reliably and let tasks + stop cooperatively"), plus a wording fix for the default `abortTimeout` in `wrapper.py` and + `docs/classes/base_task.md`. + +## Contents + +- [How an abort reaches the SDK](#how-an-abort-reaches-the-sdk) +- [Abort handling on `main`](#abort-handling-on-main) +- [Issues on `main`](#issues-on-main) +- [How the issues are resolved](#how-the-issues-are-resolved) +- [Changes by file](#changes-by-file) +- [Behaviour changes for task authors](#behaviour-changes-for-task-authors) +- [Verification](#verification) +- [Out of scope](#out-of-scope) + +## How an abort reaches the SDK + +When a user aborts a running container task in Release, the release-runner aborts it in two +phases: + +1. **Phase 1 — signal.** The runner sends `SIGABRT` to process 1 of the task container: + - Docker: `ContainerKill(…, Signal: "SIGABRT")` (`docker/container.go`). + - Kubernetes: `exec /bin/kill -6 1` in the task container (`k8s/pod.go`). +2. **Phase 2 — force kill.** The runner waits for the task type's `abortTimeout`, a hidden + property of `xlrelease.ContainerTask` that defaults to 30 seconds; the runner uses 10 seconds + if it is `0`. If the task is still running after that, the runner kills the container + (Docker) or deletes the pod (Kubernetes, which sends `SIGTERM` and then `SIGKILL`). + +The result Release shows (exit code, error message, output properties, reporting records) is +whatever the SDK reports before the process exits. A result that is not reported before +Phase 2 is lost. + +## Abort handling on `main` + +`75990ee` introduced the threading model that the branch keeps: + +- `wrapper.execute_task()` runs the task's `execute_task()` on a daemon worker thread. The main + thread polls `worker.join(0.5)` so that it stays free to run signal handlers. +- `SIGABRT` and `SIGTERM` are handled by `wrapper.abort_handler()`. Python always runs signal + handlers on the main thread. +- `BaseTask.abort()` sets exit code `1` and "Abort requested", then raises `AbortException`. +- `BaseTask.is_abort_requested()` lets a task poll for an abort. +- `_report_once()` makes sure the result is reported only once, whichever path reports first. + +On `main`, `abort_handler()` does this, in order: + +```python +dai_task_object._request_abort() # is_abort_requested() now returns True +dai_task_object.abort() # marks exit code 1, "Abort requested" +_report_once() # reports the result +os._exit(1) # ends the process, including the worker thread +``` + +## Issues on `main` + +### 1. `is_abort_requested()` has no effect: no grace period + +**Where:** `wrapper.py`, `abort_handler()` (lines 131–148 on `main`). + +The handler sets the abort flag and then reports and calls `os._exit(1)` within microseconds, +without waiting for the worker thread. A task that polls `is_abort_requested()` never gets to +act on it: the process is gone before the next poll. + +**Example:** + +```python +class LongRunningTask(BaseTask): + def execute(self) -> None: + for item in items: + if self.is_abort_requested(): + self.set_output_property("processed", count) # never reported + self.add_comment("Stopping early") # never printed + return + process(item) +``` + +Anything the task sets while stopping is lost. The cooperative-stop API exists, but it cannot +change the reported result. + +### 2. A signal during a report destroys the result + +**Where:** `wrapper.py`, `_report_once()` (line 98) and `abort_handler()`. + +Both the normal completion path (`execute_task()` → `_report_once()`) and the abort handler run +on the main thread. A signal handler runs between bytecodes of whatever the main thread is +doing, so it can interrupt a report that is in progress, for example while +`update_output_context()` is pushing to the callback URL or is in +`retry_push_result_infinitely()`. + +`_report_lock` is an `RLock`, so the handler re-enters it on the same thread. It sees +`_result_reported = True`, returns without reporting, and calls `os._exit(1)` partway through +the push. **Release receives no result at all.** + +A second signal while the handler's own report is being sent has the same effect, for example a +`SIGTERM` from a pod deletion after a `SIGABRT`. + +### 3. The result is serialized while the worker thread may change it + +**Where:** `wrapper.py`, `update_output_context()` (line 231) called from the handler. + +The handler serializes `dai_task_object.get_output_context()` (the task's own object) with +`json.dumps(output_context.to_dict())` while `execute()` may still be adding output properties +or reporting records on the worker thread. This can fail with +`RuntimeError: dictionary changed size during iteration`. + +`json.dumps` is outside the `try` in `update_output_context()`, so the exception escapes the +signal handler, `os._exit(1)` is never reached, and `run()`'s `except` reports a generic failure +with the exception message. The task's outputs and records are lost. + +### 4. Regression: tasks whose `__init__` skips `super().__init__()` crash + +**Where:** `base_task.py`, `__init__` (line 24) and `execute_task()`. + +Before `75990ee`, `execute_task()` created the output context. `75990ee` moved that into +`BaseTask.__init__()`, and added `_abort_event` there too. A subclass that defines its own +`__init__` without calling `super().__init__()` now fails: + +- `set_output_property()`, `set_exit_code()`, `add_reporting_record()` raise `AttributeError` + (`output_context` does not exist). `execute_task()`'s `except` then calls `set_exit_code()`, + which raises the same error again. +- `is_abort_requested()` raises `AttributeError` (`_abort_event` does not exist). + +Such tasks worked on the previous SDK release. + +### 5. An aborted or exited task can be reported as succeeded + +**Where:** `wrapper.py`, `abort_handler()` and the `TODO` in `execute_task()` (line 347). + +- If a task overrides `abort()` without calling `super().abort()` and without setting an exit + code, the handler reports exit code `0`. Release then records an aborted task as successful. +- If a task calls `sys.exit(2)` (or `sys.exit("message")`) without calling `set_exit_code()`, + the process exits with that status, but the reported result has exit code `0`. The `TODO` on + `main` notes this and leaves it unresolved. + +### 6. Smaller issues + +| Issue | Where | Effect | +|-------|-------|--------| +| No flush before `os._exit()` | `abort_handler()` | `os._exit()` skips Python's stream flushing, so the last lines on stdout/stderr (including `##[start: comment]` markers Release reads) can be lost. | +| Misleading message when no task exists | `_report_once()` | A signal that arrives before the task object is created is reported as "Task produced no output context". | +| Thread safety of `abort()` overrides undocumented | `base_task.py`, docs | The SDK calls `abort()` on the main thread while `execute()` runs on the worker thread. Nothing told task authors that an override must be thread-safe. | +| Changelog | `README.md` | The abort feature was not in the changelog. | + +## How the issues are resolved + +### Grace period (issue 1) + +`abort_handler()` now waits for the worker thread before reporting: + +```python +task._request_abort() +task.abort() # AbortException / SystemExit / errors caught +worker.join(timeout=ABORT_GRACE_PERIOD_SECONDS) # new: let execute() stop +_report_once(aborted=True, …) +_exit_process(1) +``` + +- `ABORT_GRACE_PERIOD_SECONDS` is read from the environment and defaults to `3`. It must stay + below `abortTimeout` (30 seconds by default). +- The worker thread is stored in a new module global `_worker` by `execute_task()`. +- The wait is skipped when there is no worker or it has already finished, so a task that is + already done or never started is reported immediately. +- A task that never checks `is_abort_requested()` is reported after the grace period, as before + apart from the wait. + +### Report state instead of a flag (issue 2) + +`_result_reported: bool` is replaced by `_report_state`, which is one of `pending`, +`in_progress` or `done`, plus two flags: + +- `_abort_in_progress` — set while the handler handles a signal. +- `_exit_after_report` — set when a signal arrives during a report. + +`abort_handler()` now starts with: + +```python +with _report_lock: + if _abort_in_progress: # repeat signal + return # ignored + if _report_state == _REPORT_IN_PROGRESS: # interrupted a report on the main thread + _exit_after_report = True # exit once the report finishes + return # let the push continue + _abort_in_progress = True +``` + +`_report_once()` sets `in_progress` before reporting and `done` in a `finally`. If +`_exit_after_report` was set meanwhile, it exits after the report. The interrupted push +therefore always completes, and the process still exits. + +### Report a copy of the output context (issue 3) + +`_report_once()` now reports a copy made by `_snapshot_output_context()`: a new `OutputContext` +with `dict(output_properties)` and `list(reporting_records)`. Under CPython's GIL these copies +are taken in a single C call, so the worker thread cannot change the dictionary while it is +being serialized. With the grace period, the copy is normally taken after the worker has +already stopped. + +### Create the output context and abort flag on first use (issue 4) + +`BaseTask` gains `_ensure_output_context()`. `get_output_context()` calls it, and all setters +and getters go through `get_output_context()` instead of `self.output_context`: +`get_output_properties()`, `set_output_property()`, `set_exit_code()`, `set_error_message()`, +`add_reporting_record()`. + +`is_abort_requested()` and `_request_abort()` read `_abort_event` with `getattr` and create it +if it is missing. Subclasses that skip `super().__init__()` work again. + +### Never report an aborted or exited task as succeeded (issue 5) + +- `_report_once(aborted=True)`: if the copy's exit code is `0`, it is set to `1`, and the error + message to "Abort requested" if it is empty. The task's own message is kept when it set one. +- `execute_task()`: if the worker raised `SystemExit` with a non-zero status and the task did not + set an exit code, that status is reported. The new helper `_exit_code_of()` maps + `SystemExit.code` the way Python does: `None` → `0`, an `int` → itself, anything else → `1`. + The error message is the string passed to `sys.exit()`, or "Task exited with code N". + `sys.exit(0)` still reports success. The `SystemExit` is still re-raised after reporting, so + the process exit status is unchanged. + +### Smaller issues (issue 6) + +- `_exit_process()` flushes `sys.stdout` and `sys.stderr` before `os._exit()`, both in the + handler and in the exit-after-report path. +- A signal before the task object exists is reported as "Abort requested before the task + started". +- The `abort()` docstring and `docs/classes/base_task.md` state that an override must be + thread-safe, that outputs set in an override are reported, and that they must be set before + `super().abort()` because it raises. +- `README.md` has an "Unreleased" changelog entry. + +## Changes by file + +### `digitalai/release/integration/wrapper.py` + +| Change | Details | +|--------|---------| +| `ABORT_GRACE_PERIOD_SECONDS` | New setting, from the environment, default `3`. The comment explains the relation to `abortTimeout`. | +| Report state | `_result_reported` replaced by `_report_state` (`_REPORT_PENDING`, `_REPORT_IN_PROGRESS`, `_REPORT_DONE`), plus `_abort_in_progress`, `_exit_after_report` and `_worker`. The comment above them explains why a signal can arrive during a report. | +| `_reset_report_guard()` | Resets all four values, so each daemon-mode execution starts clean. | +| `_snapshot_output_context()` | New. Returns a copy of the task's output context, or a failure context with the given message when there is no task. | +| `_flush_std_streams()`, `_exit_process()` | New. Flush stdout/stderr, then `os._exit()`. | +| `_report_once()` | New parameters `aborted` and `no_task_message`. Moves `pending` → `in_progress` → `done`, reports the copy, forces a non-zero exit code when `aborted`, and exits afterwards if `_exit_after_report` is set. | +| `abort_handler()` | Ignores repeat signals, defers to an in-progress report, waits up to the grace period for the worker, reports with `aborted=True`, exits through `_exit_process()`. The docstring describes the three steps. The `global dai_task_object` statement is removed; the handler reads it into a local. | +| `_exit_code_of()` | New. Maps `SystemExit.code` to the process exit status. | +| `execute_task()` | Stores the worker in `_worker`; copies a non-zero `sys.exit()` status into the output context when the task set none. The `TODO` is removed because it is resolved. The docstring is updated. | + +### `digitalai/release/integration/base_task.py` + +| Change | Details | +|--------|---------| +| `abort()` docstring | Adds the thread-safety note and when to set outputs in an override. | +| `is_abort_requested()` | Reads `_abort_event` with `getattr`; returns `False` if it does not exist. | +| `_request_abort()` | Creates `_abort_event` if it does not exist. | +| `_ensure_output_context()` | New. Creates `OutputContext(0, "", {}, [])` if `output_context` is missing or `None`. | +| `get_output_context()` | Calls `_ensure_output_context()` first. | +| Getters and setters | `get_output_properties()`, `set_output_property()`, `set_exit_code()`, `set_error_message()` and `add_reporting_record()` use `get_output_context()`. | + +`BaseTask.__init__()` still creates the output context and the abort event; the new code only +covers subclasses that skip it. + +### `tests/release/integration/test_wrapper_abort.py` + +Existing tests on `main` are kept, except one that asserted the old behaviour: + +- `test_direct_sys_exit_propagates_status_and_reports_context_as_is` (expected exit code `0` + reported for `sys.exit(2)`) becomes `test_direct_sys_exit_propagates_status_and_reports_its_code` + (expects `2` and "Task exited with code 2"). + +New tests: + +| Test class | Test | What it checks | +|------------|------|----------------| +| `WrapperExecuteTaskThreadingTests` | `test_sys_exit_with_message_reports_exit_code_1_and_the_message` | `sys.exit("Configuration missing")` reports exit code `1` and the message. | +| | `test_sys_exit_zero_reports_success` | `sys.exit(0)` still reports exit code `0`. | +| `WrapperAbortGracePeriodTests` | `test_cooperative_stop_within_grace_period_is_reported` | A task polling `is_abort_requested()` stops; its output property and reporting record are in the reported result, with exit code `1`. | +| | `test_task_that_does_not_stop_is_reported_after_grace_period` | A blocked task is reported after the (patched, 0.2 s) grace period, not before and not much later. | +| | `test_reported_result_is_a_copy_of_the_output_context` | The reported context and its properties dictionary are copies, not the task's own objects. | +| | `test_signal_during_report_lets_the_report_finish_then_exits` | A signal raised inside `update_output_context()` lets the push complete with the task's own result, then exits once. | +| | `test_second_signal_during_abort_is_ignored` | A signal arriving inside `abort()` is ignored; one report, one exit. | +| | `test_aborted_result_never_reports_exit_code_0` | An `abort()` override that sets no exit code is still reported with `1` and "Abort requested", keeping its outputs. | +| | `test_signal_before_task_exists_reports_a_clear_message` | Reports "Abort requested before the task started". | +| `WrapperTaskWithoutSuperInitTests` | `test_runs_and_reports_outputs` | A task without `super().__init__()` runs and reports its outputs. | +| | `test_can_be_aborted` | Such a task can be aborted, and `is_abort_requested()` works. | + +`WrapperRealSigabrtEndToEndTest` runs the real wrapper in a subprocess and sends a real +`SIGABRT`: + +- The shared logic moves into `_run_and_abort(task_type)`; input and output files are renamed + to `abort_input.json` / `abort_output.json`. +- `test_real_sigabrt_interrupts_blocking_task` now also asserts that the wrapper waits at least + the grace period before exiting. +- New `test_real_sigabrt_reports_outputs_of_cooperative_task` checks that a cooperative task + exits within the grace period and that its output property and reporting record are reported. +- The class is skipped on Windows, which cannot deliver `SIGABRT` to a child process. + +### `tests/release/integration/cooperative_task.py` + +New fixture task for the end-to-end test. `CooperativeTask` loops until `is_abort_requested()`, +then sets `stoppedCooperatively` and adds a reporting record with the number of loop passes. + +### `docs/classes/base_task.md` + +- `abort`: adds the thread-safety note, how outputs set in an override are reported, and that + an aborted result is always reported with a non-zero exit code. +- `is_abort_requested`: describes `ABORT_GRACE_PERIOD_SECONDS`, what is included in the result, + and the relation to `abortTimeout` (30 seconds by default; 10 seconds if set to `0`). + +### `docs/examples.md` + +The `LongRunningTask` example now mentions the grace period and that what the task records while +stopping is reported. + +### `README.md` + +New "Unreleased" changelog section: the abort feature and grace period under Features; the +report-during-signal, exit-code and `super().__init__()` fixes under Fixes. + +## Behaviour changes for task authors + +| Situation | `main` | `ReportRecords` | +|-----------|--------|-----------------| +| Task polls `is_abort_requested()` and returns | Process killed immediately; outputs set while stopping are lost | Outputs and records set within the grace period are reported | +| Task ignores the abort | Reported immediately | Reported after up to `ABORT_GRACE_PERIOD_SECONDS` (default 3 s) | +| Signal while the result is being sent | Send cut off; no result | Send completes, then the process exits | +| Second signal during an abort | Can cut off the first report | Ignored | +| `abort()` override sets no exit code | Reported as success (exit code `0`) | Reported with exit code `1`, "Abort requested" | +| `sys.exit(2)` in `execute()` | Reported exit code `0` | Reported exit code `2` | +| Subclass without `super().__init__()` | `AttributeError` | Works | + +The only timing change is the wait for a task that does not stop: it exits up to 3 seconds +later. Set `ABORT_GRACE_PERIOD_SECONDS=0` to disable the wait. + +## Verification + +| Level | Result | +|-------|--------| +| Unit tests, Windows | 126 passed, 2 skipped (the real-`SIGABRT` tests). | +| Unit and end-to-end tests, Linux (`python:3.12-slim` container) | All pass, including both real-`SIGABRT` tests. The other skipped tests need a live Release server. | +| Real wrapper, real `SIGABRT`, Linux container | Template example tasks: cooperative stop, checkpoint `abort()`, `abort()` override cleanup, and a task that ignores the abort all report exit code `1` with their outputs. The ignoring task exits 3.0 s after the signal. | +| Live Release, Docker runner | All six template abort examples behave as above. Release records the outputs and the task's comments; results arrive 0.1–3.1 s after the abort. | +| Live Release, Kubernetes runner (Docker Desktop) | Same results. The runner log shows `SIGABRT` delivered via `exec /bin/kill -6 1` for every aborted pod, with no fallback to pod deletion. Results arrive 0.3–3.2 s after the abort. | + +Release marks an aborted container task as FAILED, with the comments "Script aborted by user" +and "Execution failed. Cause: Abort requested". That is Release's own behaviour. + +## Out of scope + +These are outside the SDK or unchanged by the branch: + +- **Images without `/bin/kill`.** On Kubernetes the runner runs `/bin/kill -6 1` in the task + container. Images such as `python:3.12-slim` have no `kill` binary, so the exec fails and the + runner deletes the pod immediately. The SDK still handles the resulting `SIGTERM`, but this has + not been tested end to end. Alpine-based images have `kill` (busybox). +- **Shell-form entrypoints.** If the image starts the wrapper through a shell (`CMD python -m …` + without brackets, or a script without `exec`), process 1 is the shell, which does not pass the + signal on. +- **Daemon mode.** An abort still ends the whole executor process with `os._exit()`, as on + `main`. The runner starts a new executor for the next task. +- **Slow Kubernetes API calls.** Writing the result secret uses the HTTP timeouts (10 s connect, + 60 s read). A slow enough API server can still let Phase 2 delete the pod first. +- **Executor reuse in the runner.** With `evictionTime > 0`, sequential tasks with the same image + do not always share a pod: the runner marks an executor idle only after draining its logs, so + the next task can arrive first and get a new pod. This is runner behaviour. diff --git a/docs/classes/base_task.md b/docs/classes/base_task.md index 9b84d8b..5ac1583 100644 --- a/docs/classes/base_task.md +++ b/docs/classes/base_task.md @@ -148,8 +148,10 @@ After an abort is requested, the SDK waits up to `ABORT_GRACE_PERIOD_SECONDS` (environment variable, default `3`) for `execute` to return, then reports the aborted result and exits. Output properties and reporting records the task sets before it returns within that time are included in the result. Keep the grace -period below the task type's `abortTimeout` (10 seconds when unset), after which -the runner force-kills the container. +period below the task type's `abortTimeout`, after which the runner force-kills +the container. `abortTimeout` is a hidden property of `xlrelease.ContainerTask` +that defaults to 30 seconds; the runner falls back to 10 seconds if it is set +to `0`. _No parameters._ From cb649aef6d0e24dd305dcda6d42ce828b7504c79 Mon Sep 17 00:00:00 2001 From: Balaji Venkatesan Date: Thu, 29 Oct 2026 09:38:03 +0530 Subject: [PATCH 03/10] abort doc update --- docs/abort-changes.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/abort-changes.md b/docs/abort-changes.md index f624bf2..ccd03e6 100644 --- a/docs/abort-changes.md +++ b/docs/abort-changes.md @@ -8,9 +8,12 @@ This document records the problems found in the abort support that was merged to - **Branch:** `ReportRecords`, commit `abdf442` ("Report abort results reliably and let tasks stop cooperatively"), plus a wording fix for the default `abortTimeout` in `wrapper.py` and `docs/classes/base_task.md`. +- **Status:** all issues below are fixed on `ReportRecords` and verified on the Docker and + Kubernetes runners. `main` still has them, and no SDK release contains the fixes yet. ## Contents +- [Summary of issues and fixes](#summary-of-issues-and-fixes) - [How an abort reaches the SDK](#how-an-abort-reaches-the-sdk) - [Abort handling on `main`](#abort-handling-on-main) - [Issues on `main`](#issues-on-main) @@ -20,6 +23,26 @@ This document records the problems found in the abort support that was merged to - [Verification](#verification) - [Out of scope](#out-of-scope) +## Summary of issues and fixes + +Line numbers refer to `main` at `75990ee` (column "On `main`") and to `ReportRecords` +(column "Fixed on `ReportRecords`"). Each issue is described in detail under +[Issues on `main`](#issues-on-main), and each fix under +[How the issues are resolved](#how-the-issues-are-resolved). + +| # | Issue | On `main`: where and effect | Fixed on `ReportRecords`: where | How it is fixed | Tests | +|---|-------|-----------------------------|---------------------------------|-----------------|-------| +| 1 | `is_abort_requested()` has no effect: no grace period | `wrapper.py` `abort_handler()`, lines 131–148: sets the abort flag, reports and calls `os._exit(1)` immediately, without waiting for `execute()`.
**Effect:** outputs, reporting records and comments a task sets while stopping are lost. | `wrapper.py`:
• [`ABORT_GRACE_PERIOD_SECONDS`, line 58](../digitalai/release/integration/wrapper.py#L58)
• [`_worker`, line 101](../digitalai/release/integration/wrapper.py#L101), set in [`execute_task()`, line 437](../digitalai/release/integration/wrapper.py#L437)
• [`abort_handler()` wait, line 231](../digitalai/release/integration/wrapper.py#L231) | After calling `abort()`, the handler waits up to `ABORT_GRACE_PERIOD_SECONDS` (environment variable, default `3`) for the worker thread with `worker.join(timeout=…)`, then reports. The wait is skipped if there is no worker or it has finished. A task that never stops is reported when the wait ends. | `test_cooperative_stop_within_grace_period_is_reported`
`test_task_that_does_not_stop_is_reported_after_grace_period`
`test_real_sigabrt_reports_outputs_of_cooperative_task` (real `SIGABRT`)
`test_real_sigabrt_interrupts_blocking_task` (real `SIGABRT`) | +| 2 | A signal during a report destroys the result | `wrapper.py` `_report_once()`, line 98 (`_result_reported` flag) and `abort_handler()`: the handler runs on the main thread, can interrupt the main thread's own report, re-enters the `RLock`, sees the flag set and calls `os._exit(1)` mid-send. A second signal during an abort does the same.
**Effect:** Release receives no result. | `wrapper.py`:
• [report state, lines 85–97](../digitalai/release/integration/wrapper.py#L85-L97)
• [`_report_once()` state changes, lines 162 and 172–174](../digitalai/release/integration/wrapper.py#L162-L174)
• [`abort_handler()` guards, lines 201–208](../digitalai/release/integration/wrapper.py#L201-L208)
• [`_reset_report_guard()`, line 104](../digitalai/release/integration/wrapper.py#L104) | The boolean flag is replaced by `_report_state` (`pending` → `in_progress` → `done`) plus `_abort_in_progress` and `_exit_after_report`. If a signal arrives while a report is `in_progress`, the handler sets `_exit_after_report` and returns, so the send completes; `_report_once()` then exits. A signal while an abort is already being handled is ignored. `_reset_report_guard()` resets all of this for each daemon-mode execution. | `test_signal_during_report_lets_the_report_finish_then_exits`
`test_second_signal_during_abort_is_ignored`
`test_execute_task_resets_guard_across_daemon_iterations` | +| 3 | The result is serialized while the task may still change it | `wrapper.py` `update_output_context()`, line 231: `json.dumps()` of the task's own output context, called from the handler while `execute()` may still add outputs on the worker thread. `json.dumps` is outside the `try`.
**Effect:** possible `RuntimeError: dictionary changed size during iteration`; the handler never reaches `os._exit`, and the task is reported as a generic failure without its outputs. | `wrapper.py`:
• [`_snapshot_output_context()`, lines 114–128](../digitalai/release/integration/wrapper.py#L114-L128), called from [`_report_once()`](../digitalai/release/integration/wrapper.py#L145) | `_report_once()` reports a new `OutputContext` with `dict(output_properties)` and `list(reporting_records)`. Each copy is a single C-level call under the GIL, so the worker cannot change it mid-copy. With the grace period, the copy is normally taken after the worker has stopped. | `test_reported_result_is_a_copy_of_the_output_context` | +| 4 | Regression: tasks whose `__init__` skips `super().__init__()` crash | `base_task.py` `__init__()`, line 24: the output context and `_abort_event` are created only here; `execute_task()` no longer creates the output context.
**Effect:** `AttributeError` in `set_output_property()`, `set_exit_code()`, `add_reporting_record()` (raised again from `execute_task()`'s error handler) and in `is_abort_requested()`. | `base_task.py`:
• [`_ensure_output_context()`, line 90](../digitalai/release/integration/base_task.py#L90), called by [`get_output_context()`, line 98](../digitalai/release/integration/base_task.py#L98)
• getters/setters use `get_output_context()`: [lines 109](../digitalai/release/integration/base_task.py#L109), [133](../digitalai/release/integration/base_task.py#L133), [139](../digitalai/release/integration/base_task.py#L139), [145](../digitalai/release/integration/base_task.py#L145), [163](../digitalai/release/integration/base_task.py#L163)
• [`is_abort_requested()`, line 78](../digitalai/release/integration/base_task.py#L78) and [`_request_abort()`, line 86](../digitalai/release/integration/base_task.py#L86) | The output context is created on first use if it is missing. `get_output_properties()`, `set_output_property()`, `set_exit_code()`, `set_error_message()` and `add_reporting_record()` go through `get_output_context()`. `is_abort_requested()` returns `False` and `_request_abort()` creates the event when `_abort_event` is missing. `__init__()` is unchanged. | `test_runs_and_reports_outputs`
`test_can_be_aborted` | +| 5a | An `abort()` override that sets no exit code is reported as succeeded | `wrapper.py` `abort_handler()`: reports the task's context as it is.
**Effect:** an override that does not call `super().abort()` reports exit code `0`, and Release records the aborted task as successful. | `wrapper.py`:
• [`_report_once(aborted=True)`, lines 166–168](../digitalai/release/integration/wrapper.py#L166-L168), called from [`abort_handler()`, line 237](../digitalai/release/integration/wrapper.py#L237) | On the abort path, an exit code of `0` in the reported copy becomes `1`, and an empty error message becomes "Abort requested". A message the task set itself is kept. | `test_aborted_result_never_reports_exit_code_0` | +| 5b | `sys.exit(n)` is reported as exit code `0` | `wrapper.py` `execute_task()`, `TODO` at line 347: the `SystemExit` is re-raised so the process exits with `n`, but the reported context keeps exit code `0`.
**Effect:** Release records the task as successful. | `wrapper.py`:
• [`_exit_code_of()`, line 385](../digitalai/release/integration/wrapper.py#L385)
• [`execute_task()`, lines 443–449](../digitalai/release/integration/wrapper.py#L443-L449) | If the worker raised `SystemExit` with a non-zero status and the task set no exit code, that status is reported. `_exit_code_of()` maps `SystemExit.code` like Python does: `None` → `0`, an `int` → itself, anything else → `1`. The error message is the string passed to `sys.exit()`, or "Task exited with code N". The `SystemExit` is still re-raised. The `TODO` is removed. | `test_direct_sys_exit_propagates_status_and_reports_its_code`
`test_sys_exit_with_message_reports_exit_code_1_and_the_message`
`test_sys_exit_zero_reports_success` | +| 6a | Output not flushed before exit | `wrapper.py` `abort_handler()`: `os._exit(1)` skips Python's stream flushing.
**Effect:** the last stdout/stderr lines, including the `##[start: comment]` markers Release reads, can be lost. | `wrapper.py`:
• [`_flush_std_streams()`, line 131](../digitalai/release/integration/wrapper.py#L131) and [`_exit_process()`, line 140](../digitalai/release/integration/wrapper.py#L140), used by `abort_handler()` and `_report_once()` | `sys.stdout` and `sys.stderr` are flushed before every `os._exit()`. | Covered by the real-`SIGABRT` end-to-end tests; live runs show the task's comments in Release. | +| 6b | Misleading message when a signal arrives before the task exists | `wrapper.py` `_report_once()`: reports "Task produced no output context". | `wrapper.py`:
• [`_snapshot_output_context()`, line 114](../digitalai/release/integration/wrapper.py#L114) with the message passed from [`abort_handler()`, line 237](../digitalai/release/integration/wrapper.py#L237) | The handler reports "Abort requested before the task started". The normal path keeps the old message. | `test_signal_before_task_exists_reports_a_clear_message` | +| 6c | Thread safety of `abort()` overrides undocumented | `base_task.py` `abort()` docstring and `docs/classes/base_task.md`: no mention that `abort()` runs on the main thread while `execute()` runs on the worker thread. | • [`base_task.py` `abort()` docstring, lines 62–67](../digitalai/release/integration/base_task.py#L62-L67)
• [`docs/classes/base_task.md`, `abort`](classes/base_task.md#abort) and [`is_abort_requested`](classes/base_task.md#is_abort_requested)
• [`docs/examples.md`](examples.md) | The docs state that an override must be thread-safe, that outputs set in it are reported, and that they must be set before `super().abort()` because it raises. They also describe the grace period and its relation to `abortTimeout`. | Documentation only. | +| 6d | Abort feature missing from the changelog | `README.md`. | [`README.md`, "Unreleased"](../README.md#unreleased) | New "Unreleased" section listing the abort feature, the grace period and the fixes. | Documentation only. | + ## How an abort reaches the SDK When a user aborts a running container task in Release, the release-runner aborts it in two From f4eba2e6ac0e96e2c377bffc039640c80f91f806 Mon Sep 17 00:00:00 2001 From: Balaji Venkatesan Date: Thu, 29 Oct 2026 10:19:52 +0530 Subject: [PATCH 04/10] Comprehensive Test Suite Review and Revamp --- DEVELOPMENT.md | 32 +- pyproject.toml | 7 + tests/__init__.py | 0 tests/release/__init__.py | 0 tests/release/integration/__init__.py | 0 tests/release/integration/blocking_task.py | 14 - tests/release/integration/conftest.py | 104 ++ .../release/integration/expected_output.json | 8 - tests/release/integration/hello.py | 24 - tests/release/integration/input.json | 1 - tests/release/integration/output.json | 1 - tests/release/integration/src/hello.py | 24 - tests/release/integration/src/sample/hello.py | 27 - .../integration/tasks/blocking_task.py | 16 + .../release/integration/tasks/context_echo.py | 12 + .../{ => tasks}/cooperative_task.py | 7 +- .../release/integration/tasks/failing_task.py | 6 + tests/release/integration/tasks/hello.py | 8 + .../release/integration/tasks/sample/hello.py | 11 + .../release/integration/tasks/secret_task.py | 12 + .../release/integration/test_api_base_task.py | 729 ++++++------ .../integration/test_api_base_task_live.py | 1002 ++++++----------- tests/release/integration/test_base_task.py | 433 +++---- tests/release/integration/test_ids.py | 48 + .../release/integration/test_input_context.py | 101 ++ .../integration/test_job_data_encryptor.py | 54 + tests/release/integration/test_k8s.py | 90 ++ tests/release/integration/test_masked_io.py | 73 ++ .../integration/test_reporting_records.py | 80 ++ tests/release/integration/test_watcher.py | 119 ++ tests/release/integration/test_wrapper.py | 104 -- .../release/integration/test_wrapper_abort.py | 769 ++++--------- tests/release/integration/test_wrapper_e2e.py | 149 +++ .../integration/test_wrapper_execute_task.py | 156 +++ tests/release/integration/test_wrapper_io.py | 388 +++++++ tests/release/integration/test_wrapper_k8s.py | 283 ----- .../integration/test_wrapper_k8s_live.py | 309 +++++ tests/release/integration/test_wrapper_run.py | 135 +++ tests/release/test_release_api_client.py | 46 +- tests/support.py | 45 + 40 files changed, 3137 insertions(+), 2290 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/release/__init__.py create mode 100644 tests/release/integration/__init__.py delete mode 100644 tests/release/integration/blocking_task.py create mode 100644 tests/release/integration/conftest.py delete mode 100644 tests/release/integration/expected_output.json delete mode 100644 tests/release/integration/hello.py delete mode 100644 tests/release/integration/input.json delete mode 100644 tests/release/integration/output.json delete mode 100644 tests/release/integration/src/hello.py delete mode 100644 tests/release/integration/src/sample/hello.py create mode 100644 tests/release/integration/tasks/blocking_task.py create mode 100644 tests/release/integration/tasks/context_echo.py rename tests/release/integration/{ => tasks}/cooperative_task.py (64%) create mode 100644 tests/release/integration/tasks/failing_task.py create mode 100644 tests/release/integration/tasks/hello.py create mode 100644 tests/release/integration/tasks/sample/hello.py create mode 100644 tests/release/integration/tasks/secret_task.py create mode 100644 tests/release/integration/test_ids.py create mode 100644 tests/release/integration/test_input_context.py create mode 100644 tests/release/integration/test_job_data_encryptor.py create mode 100644 tests/release/integration/test_k8s.py create mode 100644 tests/release/integration/test_masked_io.py create mode 100644 tests/release/integration/test_reporting_records.py create mode 100644 tests/release/integration/test_watcher.py delete mode 100644 tests/release/integration/test_wrapper.py create mode 100644 tests/release/integration/test_wrapper_e2e.py create mode 100644 tests/release/integration/test_wrapper_execute_task.py create mode 100644 tests/release/integration/test_wrapper_io.py delete mode 100644 tests/release/integration/test_wrapper_k8s.py create mode 100644 tests/release/integration/test_wrapper_k8s_live.py create mode 100644 tests/release/integration/test_wrapper_run.py create mode 100644 tests/support.py diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 60b7022..ecbdcca 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -37,11 +37,37 @@ uv build --sdist ## Run tests ```sh -# Run the tests -cd tests/release/integration -uv run python -m unittest test_wrapper.py +# Run the whole suite +uv run python -m pytest + +# Skip the slower subprocess tests, or the tests that need a Release server +uv run python -m pytest -m "not e2e" +uv run python -m pytest -m "not live" ``` +Tests are grouped by module under `tests/release/integration/` (`test_.py`), with the +wrapper split into `test_wrapper_io.py`, `test_wrapper_run.py`, `test_wrapper_execute_task.py`, +`test_wrapper_abort.py` and the subprocess runs in `test_wrapper_e2e.py`. Shared ids and input +contexts are in `tests/support.py`; fixtures are in `tests/release/integration/conftest.py`, and +the task classes the wrapper loads in tests are in `tests/release/integration/tasks/`. + +Markers: + +- `e2e`: runs the wrapper with `python -m` in a subprocess. The two real-`SIGABRT` tests are + skipped on Windows, which cannot deliver `SIGABRT` to a child process. +- `live`: needs a real server, and is skipped when none is reachable. + - `test_api_base_task_live.py` creates and deletes a folder, template, release and variables + on a Release server, by default `http://localhost:5516` as `admin`/`admin`; set + `RELEASE_LIVE_URL`, `RELEASE_LIVE_USER` and `RELEASE_LIVE_PASSWORD` to use another one. + - `test_wrapper_k8s_live.py` runs the wrapper's Kubernetes path against a cluster: input and + result secrets, the callback, the input-context watcher, and a full wrapper run. It uses only + the kubeconfig context named by `RELEASE_LIVE_K8S_CONTEXT` (default `docker-desktop`, e.g. + Docker Desktop with Kubernetes enabled), whatever the current context is, and works in a + temporary `sdk-live-` namespace that it deletes afterwards. + +Known bugs are pinned by tests marked `xfail`. `xfail_strict` is on, so when a fix makes such a +test pass, the run fails until the marker is removed. + ## Publish to PyPI ```sh diff --git a/pyproject.toml b/pyproject.toml index 01d5b3c..f1367f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,3 +71,10 @@ include = [ [tool.pytest.ini_options] testpaths = ["tests"] +addopts = ["-ra", "--strict-markers", "--strict-config"] +# A known-bug test marked xfail fails the run once the bug is fixed, so the marker gets removed. +xfail_strict = true +markers = [ + "e2e: runs the wrapper in a subprocess (a few seconds each)", + "live: needs a Digital.ai Release server or Kubernetes cluster; skipped when none is reachable", +] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/release/__init__.py b/tests/release/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/release/integration/__init__.py b/tests/release/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/release/integration/blocking_task.py b/tests/release/integration/blocking_task.py deleted file mode 100644 index de10ce6..0000000 --- a/tests/release/integration/blocking_task.py +++ /dev/null @@ -1,14 +0,0 @@ -import time - -from digitalai.release.integration import BaseTask - - -class BlockingTask(BaseTask): - """ - Fixture task used by test_wrapper_abort.py to verify that SIGABRT interrupts - a task promptly even while it is blocked in an uninterruptible call (i.e. - without polling is_abort_requested()). - """ - - def execute(self) -> None: - time.sleep(30) diff --git a/tests/release/integration/conftest.py b/tests/release/integration/conftest.py new file mode 100644 index 0000000..f9d6f09 --- /dev/null +++ b/tests/release/integration/conftest.py @@ -0,0 +1,104 @@ +""" +Fixtures shared by the integration tests. + +Importing ``wrapper`` has process-wide side effects: it replaces sys.stdout and +sys.stderr with MaskedIO streams, installs SIGTERM/SIGABRT handlers, and reads +its settings from environment variables once. Tests that use the wrapper +in-process therefore go through ``isolated_wrapper``, which pins those +settings so the developer's environment cannot change a test's outcome, and +resets the per-execution state the wrapper keeps in module globals. +""" +import shutil +import sys +from pathlib import Path + +import pytest + +import digitalai.release.integration.wrapper as wrapper +from digitalai.release.integration.logger import dai_logger + +TASKS_DIR = Path(__file__).parent / "tasks" + +# Module globals the wrapper initialises from environment variables at import. +_WRAPPER_SETTINGS = ( + "input_context_file", + "output_context_file", + "base64_session_key", + "release_server_url", + "callback_url", + "input_context_secret", + "result_secret_key", + "runner_namespace", + "execution_mode", +) + + +@pytest.fixture +def isolated_wrapper(monkeypatch): + """The wrapper module with blank settings and fresh per-execution state.""" + for name in _WRAPPER_SETTINGS: + monkeypatch.setattr(wrapper, name, "") + monkeypatch.setattr(wrapper, "input_context", None) + monkeypatch.setattr(wrapper, "dai_task_object", None) + monkeypatch.setattr(wrapper.masked_std_out, "secrets", []) + monkeypatch.setattr(wrapper.masked_std_err, "secrets", []) + wrapper._reset_report_guard() + yield wrapper + wrapper._reset_report_guard() + + +@pytest.fixture +def reported(isolated_wrapper, monkeypatch): + """Every OutputContext the wrapper reports, captured instead of sent.""" + contexts = [] + monkeypatch.setattr(wrapper, "update_output_context", contexts.append) + return contexts + + +@pytest.fixture +def exit_codes(isolated_wrapper, monkeypatch): + """Codes passed to os._exit(), which is replaced so the test runner survives.""" + codes = [] + monkeypatch.setattr(wrapper.os, "_exit", codes.append) + return codes + + +@pytest.fixture +def dai_log(caplog): + """caplog wired to dai_logger, which does not propagate to the root logger.""" + dai_logger.addHandler(caplog.handler) + yield caplog + dai_logger.removeHandler(caplog.handler) + + +def make_task_workspace(workspace: Path) -> Path: + """ + Fills ``workspace`` as a runner working directory with the fixture tasks + under ``src/``, laid out the way a task image is: ``scriptLocation`` paths + resolve against ``src/``. + """ + shutil.copytree(TASKS_DIR, workspace / "src", ignore=shutil.ignore_patterns("__pycache__")) + return workspace + + +@pytest.fixture +def task_workspace(tmp_path): + return make_task_workspace(tmp_path / "workspace") + + +@pytest.fixture +def importable_workspace(task_workspace, monkeypatch): + """ + task_workspace as the current directory and on sys.path, so wrapper.run() + can import fixture tasks in-process. The ``src`` modules it imports are + dropped afterwards so the next test's workspace is imported afresh. + """ + def drop_src_modules(): + for name in [m for m in sys.modules if m == "src" or m.startswith("src.")]: + del sys.modules[name] + + drop_src_modules() + monkeypatch.chdir(task_workspace) + monkeypatch.syspath_prepend(str(task_workspace)) + yield task_workspace + drop_src_modules() diff --git a/tests/release/integration/expected_output.json b/tests/release/integration/expected_output.json deleted file mode 100644 index abd4920..0000000 --- a/tests/release/integration/expected_output.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "exitCode": 0, - "jobErrorMessage": "", - "outputProperties": { - "greeting": "Hello World" - }, - "reportingRecords": [] -} \ No newline at end of file diff --git a/tests/release/integration/hello.py b/tests/release/integration/hello.py deleted file mode 100644 index 83bb992..0000000 --- a/tests/release/integration/hello.py +++ /dev/null @@ -1,24 +0,0 @@ -from digitalai.release.integration import BaseTask -from digitalai.release.integration import dai_logger - -class Hello(BaseTask): - """ - The purpose of this task is to greet by the given name. - """ - def execute(self) -> None: - name = self.input_properties['yourName'] - if not name: - raise ValueError("The 'name' field cannot be empty") - - greeting = f"Hello {name}" - - dai_logger.info(f"get_release_server_url() : {self.get_release_server_url()}") - dai_logger.info(f"get_task_user() : {self.get_task_user()}") - dai_logger.info(f"get_release_id() : {self.get_release_id()}") - dai_logger.info(f"get_task_id() : {self.get_task_id()}") - - # Add to the comment section of the task in the UI - self.add_comment(greeting) - - self.set_output_property('greeting', greeting) - diff --git a/tests/release/integration/input.json b/tests/release/integration/input.json deleted file mode 100644 index 16bb5f4..0000000 --- a/tests/release/integration/input.json +++ /dev/null @@ -1 +0,0 @@ -{"release": {"id": "Applications/Folder1f65c7220b394afbb941154342fd9fc6/Release31de09e95c8e4ebb95aaed29a8082d0b", "automatedTaskAsUser": {"username": "admin", "password": "admin"}}, "task": {"id": "Applications/Folder1f65c7220b394afbb941154342fd9fc6/Release31de09e95c8e4ebb95aaed29a8082d0b/Phase723a601c78804f7dbcaa8b05b83708f5/Task3a35b67b42b6428b854857fba470b39a", "type": "containerExamples.Hello1", "properties": [{"name": "capabilities", "value": ["remote"], "kind": "SET_OF_STRING", "category": "input", "password": false}, {"name": "yourName", "value": "World", "kind": "STRING", "category": "input", "password": false}, {"name": "greeting", "value": null, "kind": "STRING", "category": "output", "password": false}, {"name": "scriptLocation", "value": "sample/hello.py", "kind": "STRING", "category": "input", "password": false}]}} \ No newline at end of file diff --git a/tests/release/integration/output.json b/tests/release/integration/output.json deleted file mode 100644 index 8f89495..0000000 --- a/tests/release/integration/output.json +++ /dev/null @@ -1 +0,0 @@ -{"exitCode": 0, "jobErrorMessage": "", "outputProperties": {"greeting": "Hello World"}, "reportingRecords": []} \ No newline at end of file diff --git a/tests/release/integration/src/hello.py b/tests/release/integration/src/hello.py deleted file mode 100644 index 83bb992..0000000 --- a/tests/release/integration/src/hello.py +++ /dev/null @@ -1,24 +0,0 @@ -from digitalai.release.integration import BaseTask -from digitalai.release.integration import dai_logger - -class Hello(BaseTask): - """ - The purpose of this task is to greet by the given name. - """ - def execute(self) -> None: - name = self.input_properties['yourName'] - if not name: - raise ValueError("The 'name' field cannot be empty") - - greeting = f"Hello {name}" - - dai_logger.info(f"get_release_server_url() : {self.get_release_server_url()}") - dai_logger.info(f"get_task_user() : {self.get_task_user()}") - dai_logger.info(f"get_release_id() : {self.get_release_id()}") - dai_logger.info(f"get_task_id() : {self.get_task_id()}") - - # Add to the comment section of the task in the UI - self.add_comment(greeting) - - self.set_output_property('greeting', greeting) - diff --git a/tests/release/integration/src/sample/hello.py b/tests/release/integration/src/sample/hello.py deleted file mode 100644 index a4c94ba..0000000 --- a/tests/release/integration/src/sample/hello.py +++ /dev/null @@ -1,27 +0,0 @@ -from digitalai.release.integration import BaseTask -from digitalai.release.integration import dai_logger - -class Hello1(BaseTask): - """ - The purpose of this task is to greet by the given name. - """ - def execute(self) -> None: - - print("Inside Sample Folder - Hello Task") - - name = self.input_properties['yourName'] - if not name: - raise ValueError("The 'name' field cannot be empty") - - greeting = f"Hello {name}" - - dai_logger.info(f"get_release_server_url() : {self.get_release_server_url()}") - dai_logger.info(f"get_task_user() : {self.get_task_user()}") - dai_logger.info(f"get_release_id() : {self.get_release_id()}") - dai_logger.info(f"get_task_id() : {self.get_task_id()}") - - # Add to the comment section of the task in the UI - self.add_comment(greeting) - - self.set_output_property('greeting', greeting) - diff --git a/tests/release/integration/tasks/blocking_task.py b/tests/release/integration/tasks/blocking_task.py new file mode 100644 index 0000000..efeb721 --- /dev/null +++ b/tests/release/integration/tasks/blocking_task.py @@ -0,0 +1,16 @@ +import time +from pathlib import Path + +from digitalai.release.integration import BaseTask + + +class BlockingTask(BaseTask): + """ + Blocks without polling is_abort_requested(), so an abort can only end it by + reporting after the grace period. Creates the ``task_started`` file once + execute() runs, so a test knows when it is safe to signal. + """ + + def execute(self) -> None: + Path("task_started").touch() + time.sleep(30) diff --git a/tests/release/integration/tasks/context_echo.py b/tests/release/integration/tasks/context_echo.py new file mode 100644 index 0000000..c0608f3 --- /dev/null +++ b/tests/release/integration/tasks/context_echo.py @@ -0,0 +1,12 @@ +from digitalai.release.integration import BaseTask + + +class ContextEcho(BaseTask): + """Reports the task context the wrapper wired into the task as output properties.""" + + def execute(self) -> None: + self.set_output_property("serverUrl", self.get_release_server_url()) + self.set_output_property("releaseId", self.get_release_id()) + self.set_output_property("taskId", self.get_task_id()) + self.set_output_property("username", self.get_task_user().username) + self.set_output_property("inputs", self.get_input_properties()) diff --git a/tests/release/integration/cooperative_task.py b/tests/release/integration/tasks/cooperative_task.py similarity index 64% rename from tests/release/integration/cooperative_task.py rename to tests/release/integration/tasks/cooperative_task.py index 1f3ede4..5742260 100644 --- a/tests/release/integration/cooperative_task.py +++ b/tests/release/integration/tasks/cooperative_task.py @@ -1,16 +1,17 @@ import time +from pathlib import Path from digitalai.release.integration import BaseTask class CooperativeTask(BaseTask): """ - Fixture task used by test_wrapper_abort.py to verify that a task polling - is_abort_requested() can stop within the abort grace period and have the - outputs it sets on the way out reported. + Polls is_abort_requested() and records outputs on the way out, which the + wrapper must report. Creates the ``task_started`` file once execute() runs. """ def execute(self) -> None: + Path("task_started").touch() processed = 0 while not self.is_abort_requested(): processed += 1 diff --git a/tests/release/integration/tasks/failing_task.py b/tests/release/integration/tasks/failing_task.py new file mode 100644 index 0000000..406f5d0 --- /dev/null +++ b/tests/release/integration/tasks/failing_task.py @@ -0,0 +1,6 @@ +from digitalai.release.integration import BaseTask + + +class FailingTask(BaseTask): + def execute(self) -> None: + raise ValueError("Deployment target unreachable") diff --git a/tests/release/integration/tasks/hello.py b/tests/release/integration/tasks/hello.py new file mode 100644 index 0000000..617404d --- /dev/null +++ b/tests/release/integration/tasks/hello.py @@ -0,0 +1,8 @@ +from digitalai.release.integration import BaseTask + + +class Hello(BaseTask): + """Greets the given name. Found by class name when no scriptLocation is set.""" + + def execute(self) -> None: + self.set_output_property("greeting", f"Hello {self.input_properties['yourName']}") diff --git a/tests/release/integration/tasks/sample/hello.py b/tests/release/integration/tasks/sample/hello.py new file mode 100644 index 0000000..5be523c --- /dev/null +++ b/tests/release/integration/tasks/sample/hello.py @@ -0,0 +1,11 @@ +from digitalai.release.integration import BaseTask + + +class Hello(BaseTask): + """ + Same class name as ../hello.py with a different greeting, so a test can + tell that the wrapper loaded the class from the scriptLocation path. + """ + + def execute(self) -> None: + self.set_output_property("greeting", f"Hello {self.input_properties['yourName']} from sample") diff --git a/tests/release/integration/tasks/secret_task.py b/tests/release/integration/tasks/secret_task.py new file mode 100644 index 0000000..a8b31b3 --- /dev/null +++ b/tests/release/integration/tasks/secret_task.py @@ -0,0 +1,12 @@ +from digitalai.release.integration import BaseTask, dai_logger + + +class SecretTask(BaseTask): + """Writes its password input and the run-as-user password through every output channel.""" + + def execute(self) -> None: + token = self.input_properties["token"] + run_as_password = self.get_task_user().password + print(f"print: {token} {run_as_password}") + dai_logger.info(f"logger: {token} {run_as_password}") + self.add_comment(f"comment: {token} {run_as_password}") diff --git a/tests/release/integration/test_api_base_task.py b/tests/release/integration/test_api_base_task.py index b707ddc..aa309ed 100644 --- a/tests/release/integration/test_api_base_task.py +++ b/tests/release/integration/test_api_base_task.py @@ -1,16 +1,10 @@ -import socket -import unittest -from datetime import date, datetime, timezone +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone from types import SimpleNamespace -from unittest.mock import MagicMock +from typing import Any, Dict, Tuple +from unittest import mock -from digitalai.release.integration.input_context import ( - AutomatedTaskAsUserContext, - ReleaseContext, -) -from digitalai.release.integration.api_base_task import ApiBaseTask - -from digitalai.release.release_api_client import ReleaseAPIClient +import pytest from com.xebialabs.xlrelease.api.v1.activity_logs_api import ActivityLogsApi from com.xebialabs.xlrelease.api.v1.application_api import ApplicationApi from com.xebialabs.xlrelease.api.v1.archive_api import ArchiveApi @@ -22,9 +16,7 @@ from com.xebialabs.xlrelease.api.v1.dsl_api import DslApi from com.xebialabs.xlrelease.api.v1.environment_api import EnvironmentApi from com.xebialabs.xlrelease.api.v1.environment_label_api import EnvironmentLabelApi -from com.xebialabs.xlrelease.api.v1.environment_reservation_api import ( - EnvironmentReservationApi, -) +from com.xebialabs.xlrelease.api.v1.environment_reservation_api import EnvironmentReservationApi from com.xebialabs.xlrelease.api.v1.environment_stage_api import EnvironmentStageApi from com.xebialabs.xlrelease.api.v1.folder_api import FolderApi from com.xebialabs.xlrelease.api.v1.folder_versioning_api import FolderVersioningApi @@ -43,9 +35,14 @@ from com.xebialabs.xlrelease.api.v1.triggers_api import TriggersApi from com.xebialabs.xlrelease.api.v1.user_api import UserApi from com.xebialabs.xlrelease.api.v1.variable_api import VariableApi +from com.xebialabs.xlrelease.domain.variable import ValueProvider, Variable +from digitalai.release.integration import base_task +from digitalai.release.integration.api_base_task import ApiBaseTask +from digitalai.release.integration.input_context import AutomatedTaskAsUserContext, ReleaseContext +from tests.support import FOLDER_ID, PHASE_ID, RELEASE_ID, SERVER_URL, TASK_ID -# Mapping of every ApiBaseTask property name to the wrapper class it must return. +# Every ApiBaseTask property and the API wrapper class it must return. API_PROPERTIES = { "activityLogsApi": ActivityLogsApi, "applicationApi": ApplicationApi, @@ -80,365 +77,357 @@ } -# The live-call test below talks to a real Release server. Locally that is the -# developer's instance on localhost:5516; CI has no such server, so the test is -# skipped automatically when the port is not reachable. -LIVE_SERVER_HOST = "localhost" -LIVE_SERVER_PORT = 5516 -LIVE_SERVER_URL = f"http://{LIVE_SERVER_HOST}:{LIVE_SERVER_PORT}" +class SampleApiTask(ApiBaseTask): + def execute(self) -> None: + pass -def _server_reachable(host: str, port: int, timeout: float = 0.5) -> bool: - """Return True if a TCP connection to host:port succeeds within the timeout.""" - try: - with socket.create_connection((host, port), timeout=timeout): - return True - except OSError: - return False +@pytest.fixture +def task(): + task = SampleApiTask() + task.task_id = TASK_ID + task.release_server_url = SERVER_URL + task.release_context = ReleaseContext( + id=RELEASE_ID, + automated_task_as_user=AutomatedTaskAsUserContext(username="admin", password="admin"), + ) + return task -class _SampleApiTask(ApiBaseTask): - """Concrete ApiBaseTask used to exercise the base behaviour in tests.""" +@pytest.fixture +def client_class(monkeypatch): + """Replaces ReleaseAPIClient so no client is built; returns the stand-in class.""" + fake = mock.Mock(name="ReleaseAPIClient", side_effect=lambda *a, **kw: mock.Mock(name="client")) + monkeypatch.setattr(base_task, "ReleaseAPIClient", fake) + return fake - def execute(self) -> None: # pragma: no cover - never executed by the tests - pass +@pytest.fixture +def apis(task): + """ + Signature-checked stand-ins for the APIs the helpers use, placed in the + task's API cache so the helpers pick them up instead of building real ones. + """ + instances = {cls: mock.create_autospec(cls, instance=True) + for cls in (ReleaseApi, TaskApi, PhaseApi, FolderApi, ConfigurationApi, SettingsApi)} + task._api_instances.update(instances) + return SimpleNamespace(release=instances[ReleaseApi], task=instances[TaskApi], phase=instances[PhaseApi], + folder=instances[FolderApi], configuration=instances[ConfigurationApi], + settings=instances[SettingsApi]) -class TestApiBaseTask(unittest.TestCase): - """Integration tests for the ApiBaseTask base class.""" - - def setUp(self): - # Configure the task context the way the Release runtime would, so that - # get_release_api_client() builds a client from the "Run as user" details. - self.task = _SampleApiTask() - self.task.release_server_url = LIVE_SERVER_URL - self.task.release_context = ReleaseContext( - id="Applications/Release0000000000000000000000000000", - automated_task_as_user=AutomatedTaskAsUserContext( - username="admin", password="admin" - ), - ) - - def test_api_client_is_lazy_and_cached(self): - """apiClient is built on first access and the same instance is reused.""" - self.assertIsNone(self.task._api_client) - client = self.task.apiClient - self.assertIsInstance(client, ReleaseAPIClient) - self.assertIs(self.task.apiClient, client) - - def test_all_api_properties_return_correct_type(self): - """Every API property returns an instance of its wrapper class.""" - for name, api_class in API_PROPERTIES.items(): - with self.subTest(api=name): - instance = getattr(self.task, name) - self.assertIsInstance(instance, api_class) - - def test_api_properties_are_cached(self): - """Accessing a property twice returns the very same cached instance.""" - for name in API_PROPERTIES: - with self.subTest(api=name): - first = getattr(self.task, name) - second = getattr(self.task, name) - self.assertIs(first, second) - - def test_all_apis_share_single_client(self): - """All wrappers are built on the one shared apiClient.""" - client = self.task.apiClient - for name in API_PROPERTIES: - with self.subTest(api=name): - self.assertIs(getattr(self.task, name).api, client) - - def test_reset_api_clients_clears_caches(self): - """reset_api_clients drops the client and forces fresh instances.""" - old_client = self.task.apiClient - old_release_api = self.task.releaseApi - - self.task.reset_api_clients() - self.assertIsNone(self.task._api_client) - self.assertEqual(self.task._api_instances, {}) - - self.assertIsNot(self.task.apiClient, old_client) - self.assertIsNot(self.task.releaseApi, old_release_api) - - @unittest.skipUnless( - _server_reachable(LIVE_SERVER_HOST, LIVE_SERVER_PORT), - f"live Release server not available on {LIVE_SERVER_HOST}:{LIVE_SERVER_PORT}", - ) - def test_wired_client_performs_live_call(self): - """A property built by ApiBaseTask can talk to the live server.""" - info = self.task.settingsApi.getInstanceInformation() - self.assertIsInstance(info, dict) - self.assertIn("version", info) - print(f"ApiBaseTask wired client instance information: {info}") - - -# Realistic ids, matching the documented sample input context. The phase and -# folder ids are substrings the helpers derive from the task / release ids. -FOLDER_ID = "Applications/Folder1f65c7220b394afbb941154342fd9fc6" -RELEASE_ID = f"{FOLDER_ID}/Release31de09e95c8e4ebb95aaed29a8082d0b" -PHASE_ID = f"{RELEASE_ID}/Phase723a601c78804f7dbcaa8b05b83708f5" -TASK_ID = f"{PHASE_ID}/Task3a35b67b42b6428b854857fba470b39a" - - -class TestApiBaseTaskContextHelpers(unittest.TestCase): - """Tests for the getCurrent*/...ByTitle/getVersion convenience helpers.""" - - def _stub_task(self, release_id=RELEASE_ID, task_id=TASK_ID): - """Build an ApiBaseTask with every API wrapper replaced by a MagicMock. - - Returns the task plus the dict of stub APIs keyed by property name. The - class-level property overrides are removed again after the test. - """ - task = _SampleApiTask() - task.task_id = task_id - task.release_context = SimpleNamespace(id=release_id) - task._api_instances = {} - - apis = {name: MagicMock(name=name) for name in ( - "releaseApi", "taskApi", "phaseApi", "folderApi", "settingsApi", - "configurationApi")} - for name, stub in apis.items(): - setattr(type(task), name, property(lambda self, s=stub: s)) - self.addCleanup( - lambda: [delattr(type(task), name) for name in apis]) - return task, apis - - def test_get_current_release_and_task(self): - task, apis = self._stub_task() - apis["releaseApi"].getRelease.return_value = SimpleNamespace(title="My Release") - apis["taskApi"].getTask.return_value = SimpleNamespace(title="My Task") - - self.assertEqual(task.getCurrentRelease().title, "My Release") - self.assertEqual(task.getCurrentTask().title, "My Task") - apis["releaseApi"].getRelease.assert_called_once_with(RELEASE_ID) - apis["taskApi"].getTask.assert_called_once_with(TASK_ID) - - def test_get_current_phase_and_folder_derive_ids(self): - task, apis = self._stub_task() - apis["phaseApi"].getPhase.return_value = SimpleNamespace(title="My Phase") - apis["folderApi"].getFolder.return_value = SimpleNamespace(title="My Folder") - - self.assertEqual(task.getCurrentPhase().title, "My Phase") - self.assertEqual(task.getCurrentFolder().title, "My Folder") - apis["phaseApi"].getPhase.assert_called_once_with(PHASE_ID) - apis["folderApi"].getFolder.assert_called_once_with(FOLDER_ID) - - def test_search_helpers_default_to_current_release(self): - task, apis = self._stub_task() - apis["taskApi"].searchTasksByTitle.return_value = ["t"] - apis["phaseApi"].searchPhasesByTitle.return_value = ["p"] - apis["releaseApi"].searchReleasesByTitle.return_value = ["r"] - - self.assertEqual(task.getTasksByTitle("Deploy"), ["t"]) - self.assertEqual(task.getPhasesByTitle("Prod"), ["p"]) - self.assertEqual(task.getReleasesByTitle("Nightly"), ["r"]) - # taskApi signature is (taskTitle, releaseId, phaseTitle). - apis["taskApi"].searchTasksByTitle.assert_called_once_with("Deploy", RELEASE_ID, None) - apis["phaseApi"].searchPhasesByTitle.assert_called_once_with("Prod", RELEASE_ID) - apis["releaseApi"].searchReleasesByTitle.assert_called_once_with("Nightly") - - def test_search_helpers_honor_explicit_arguments(self): - task, apis = self._stub_task() - apis["taskApi"].searchTasksByTitle.return_value = [] - - task.getTasksByTitle("Deploy", "Prod", "Release/Other") - - apis["taskApi"].searchTasksByTitle.assert_called_once_with( - "Deploy", "Release/Other", "Prod") - - def test_get_version(self): - task, apis = self._stub_task() - apis["settingsApi"].getInstanceInformation.return_value = { - "product": "Digital.ai Release", "edition": "enterprise", "version": "25.3.0"} - - self.assertEqual(task.getVersion(), "25.3.0") - - # -- reference-variable resolution -------------------------------------- - - def test_get_release_variable_returns_stored_value_for_plain_variable(self): - """A non-reference variable returns its stored value without resolving.""" - task, apis = self._stub_task() - apis["releaseApi"].getVariables.return_value = [ - SimpleNamespace(key="relVar", value="hello", type="xlrelease.StringVariable")] - - self.assertEqual(task.getReleaseVariable("relVar"), "hello") - # No need to consult the resolved-value map for a literal value. - apis["releaseApi"].getVariableValues.assert_not_called() - - def test_get_release_variable_resolves_reference(self): - """A reference variable's value comes from the server-resolved map.""" - task, apis = self._stub_task() - # The reference variable itself carries no literal value. - apis["releaseApi"].getVariables.return_value = [ - SimpleNamespace(key="refVar", value="", type="xlrelease.ReferenceVariable")] - apis["releaseApi"].getVariableValues.return_value = {"${refVar}": "resolved-value"} - - self.assertEqual(task.getReleaseVariable("refVar"), "resolved-value") - apis["releaseApi"].getVariableValues.assert_called_once_with(RELEASE_ID) - - def test_get_folder_variable_resolves_reference(self): - """A reference folder variable resolves via listVariableValues.""" - task, apis = self._stub_task() - apis["folderApi"].listVariables.return_value = [ - SimpleNamespace(key="folder.refVar", value="", - type="xlrelease.ReferenceVariable")] - apis["folderApi"].listVariableValues.return_value = { - "${folder.refVar}": "folder-resolved"} - - self.assertEqual(task.getFolderVariable("folder.refVar"), "folder-resolved") - apis["folderApi"].listVariableValues.assert_called_once_with(FOLDER_ID) - - def test_get_global_variable_resolves_reference(self): - """A reference global variable resolves via getGlobalVariableValues.""" - task, apis = self._stub_task() - apis["configurationApi"].getGlobalVariables.return_value = [ - SimpleNamespace(key="global.refVar", value="", - type="xlrelease.ReferenceVariable")] - apis["configurationApi"].getGlobalVariableValues.return_value = { - "${global.refVar}": "global-resolved"} - - self.assertEqual(task.getGlobalVariable("global.refVar"), "global-resolved") - apis["configurationApi"].getGlobalVariableValues.assert_called_once_with() - - def test_reference_resolution_falls_back_to_stored_value(self): - """When the resolved map lacks the token, the stored value is returned.""" - task, apis = self._stub_task() - apis["releaseApi"].getVariables.return_value = [ - SimpleNamespace(key="refVar", value="stored", - type="xlrelease.ReferenceVariable")] - apis["releaseApi"].getVariableValues.return_value = {} - - self.assertEqual(task.getReleaseVariable("refVar"), "stored") - - # -- setter value-type guard -------------------------------------------- - - def test_set_release_variable_rejects_type_mismatch(self): - """Assigning a str to an existing SetStringVariable raises TypeError.""" - task, apis = self._stub_task() - apis["releaseApi"].getVariables.return_value = [ - SimpleNamespace(id="Variable1", key="relSetVar", value=["a"], - type="xlrelease.SetStringVariable")] - - with self.assertRaises(TypeError): - task.setReleaseVariable("relSetVar", "not-a-set") - # The mismatched payload is never forwarded to the server. - apis["releaseApi"].updateVariable.assert_not_called() - - def test_set_release_variable_rejects_int_for_string_variable(self): - """Assigning an int to an existing StringVariable raises TypeError.""" - task, apis = self._stub_task() - apis["releaseApi"].getVariables.return_value = [ - SimpleNamespace(id="Variable1", key="relVar", value="hello", - type="xlrelease.StringVariable")] - - with self.assertRaises(TypeError): - task.setReleaseVariable("relVar", 42) - apis["releaseApi"].updateVariable.assert_not_called() - - def test_set_release_variable_accepts_matching_type(self): - """A set value for a SetStringVariable updates and coerces to a list.""" - task, apis = self._stub_task() - variable = SimpleNamespace(id="Variable1", key="relSetVar", value=["a"], - type="xlrelease.SetStringVariable") - apis["releaseApi"].getVariables.return_value = [variable] - apis["releaseApi"].updateVariable.return_value = variable - - task.setReleaseVariable("relSetVar", {"x", "y"}) - - self.assertEqual(set(variable.value), {"x", "y"}) - self.assertIsInstance(variable.value, list) - apis["releaseApi"].updateVariable.assert_called_once_with("Variable1", variable) - - def test_set_release_variable_does_not_guard_reference_variable(self): - """Reference variables are left for the server to validate.""" - task, apis = self._stub_task() - variable = SimpleNamespace(id="Variable1", key="refVar", value="", - type="xlrelease.ReferenceVariable") - apis["releaseApi"].getVariables.return_value = [variable] - apis["releaseApi"].updateVariable.return_value = variable - - # No TypeError despite a str value against a non-string-looking type. - task.setReleaseVariable("refVar", "anything") - apis["releaseApi"].updateVariable.assert_called_once() - - def test_set_release_variable_does_not_guard_creation(self): - """A brand-new variable infers its type from the value, no guard.""" - task, apis = self._stub_task() - apis["releaseApi"].getVariables.return_value = [] - apis["releaseApi"].createVariable.return_value = SimpleNamespace( - type="xlrelease.IntegerVariable") - - task.setReleaseVariable("relNewVar", 7) - apis["releaseApi"].createVariable.assert_called_once() - apis["releaseApi"].updateVariable.assert_not_called() - - # -- date variable support ---------------------------------------------- - - def test_set_release_variable_accepts_datetime_for_date_variable(self): - """A datetime updates a DateVariable, coerced to an ISO-8601 string.""" - task, apis = self._stub_task() - when = datetime(2026, 6, 24, 10, 30, 0) - variable = SimpleNamespace(id="Variable1", key="relDateVar", value=None, - type="xlrelease.DateVariable") - apis["releaseApi"].getVariables.return_value = [variable] - apis["releaseApi"].updateVariable.return_value = variable - - task.setReleaseVariable("relDateVar", when) - - self.assertEqual(variable.value, when.isoformat()) - apis["releaseApi"].updateVariable.assert_called_once_with("Variable1", variable) - - def test_set_release_variable_accepts_iso_string_for_date_variable(self): - """An ISO-8601 string is accepted for a DateVariable (server parses it).""" - task, apis = self._stub_task() - variable = SimpleNamespace(id="Variable1", key="relDateVar", value=None, - type="xlrelease.DateVariable") - apis["releaseApi"].getVariables.return_value = [variable] - apis["releaseApi"].updateVariable.return_value = variable - - task.setReleaseVariable("relDateVar", "2026-06-24T10:30:00+00:00") - - self.assertEqual(variable.value, "2026-06-24T10:30:00+00:00") - apis["releaseApi"].updateVariable.assert_called_once() - - def test_set_release_variable_rejects_int_for_date_variable(self): - """A bare int is not a valid Python value for a DateVariable.""" - task, apis = self._stub_task() - apis["releaseApi"].getVariables.return_value = [ - SimpleNamespace(id="Variable1", key="relDateVar", value=None, - type="xlrelease.DateVariable")] - - with self.assertRaises(TypeError): - task.setReleaseVariable("relDateVar", 12345) - apis["releaseApi"].updateVariable.assert_not_called() - - def test_set_release_variable_creates_date_variable_from_date(self): - """A new variable assigned a date infers the DateVariable type.""" - task, apis = self._stub_task() - apis["releaseApi"].getVariables.return_value = [] - captured = {} - apis["releaseApi"].createVariable.side_effect = ( - lambda rid, var: captured.setdefault("var", var)) - - task.setReleaseVariable("relNewDateVar", date(2026, 6, 24)) - - self.assertEqual(captured["var"].type, "xlrelease.DateVariable") - self.assertEqual(captured["var"].value, "2026-06-24") - - def test_set_release_variable_accepts_current_datetime(self): - """The current date/time updates a DateVariable as its ISO-8601 string.""" - task, apis = self._stub_task() - now = datetime.now(timezone.utc) - variable = SimpleNamespace(id="Variable1", key="relDateVar", value=None, - type="xlrelease.DateVariable") - apis["releaseApi"].getVariables.return_value = [variable] - apis["releaseApi"].updateVariable.return_value = variable - - task.setReleaseVariable("relDateVar", now) - - self.assertEqual(variable.value, now.isoformat()) - apis["releaseApi"].updateVariable.assert_called_once_with("Variable1", variable) - - -if __name__ == "__main__": - unittest.main() + +def variable(key, value, vtype="xlrelease.StringVariable", **extra): + return Variable(id=f"{RELEASE_ID}/Variable_{key}", key=key, value=value, type=vtype, **extra) + + +# -- client and API instances ---------------------------------------------- + +def test_api_client_is_built_from_the_task_context_on_first_use(task, client_class): + client_class.assert_not_called() + + client = task.apiClient + + client_class.assert_called_once_with(SERVER_URL, "admin", "admin", timeout=None) + assert task.apiClient is client + + +@pytest.mark.parametrize("name, api_class", API_PROPERTIES.items(), ids=list(API_PROPERTIES)) +def test_api_property_returns_a_cached_wrapper_on_the_shared_client(task, client_class, name, api_class): + api = getattr(task, name) + + assert isinstance(api, api_class) + assert getattr(task, name) is api + assert api.api is task.apiClient + + +def test_all_api_properties_share_one_client(task, client_class): + for name in API_PROPERTIES: + getattr(task, name) + + client_class.assert_called_once() + + +def test_reset_api_clients_rebuilds_the_client_and_wrappers(task, client_class): + old_client, old_release_api = task.apiClient, task.releaseApi + + task.reset_api_clients() + + assert task.apiClient is not old_client + assert task.releaseApi is not old_release_api + assert task.releaseApi.api is task.apiClient + + +# -- current-context helpers ----------------------------------------------- + +@pytest.mark.parametrize("helper, api, method, expected_id", [ + ("getCurrentTask", "task", "getTask", TASK_ID), + ("getCurrentPhase", "phase", "getPhase", PHASE_ID), + ("getCurrentRelease", "release", "getRelease", RELEASE_ID), + ("getCurrentFolder", "folder", "getFolder", FOLDER_ID), +]) +def test_get_current_object_fetches_it_by_the_derived_id(task, apis, helper, api, method, expected_id): + api_method = getattr(getattr(apis, api), method) + + result = getattr(task, helper)() + + api_method.assert_called_once_with(expected_id) + assert result is api_method.return_value + + +def test_get_tasks_by_title_searches_the_current_release(task, apis): + result = task.getTasksByTitle("Deploy") + + apis.task.searchTasksByTitle.assert_called_once_with("Deploy", RELEASE_ID, None) + assert result is apis.task.searchTasksByTitle.return_value + + +def test_get_tasks_by_title_honours_phase_and_release(task, apis): + task.getTasksByTitle("Deploy", "Prod", "Applications/Release2") + + apis.task.searchTasksByTitle.assert_called_once_with("Deploy", "Applications/Release2", "Prod") + + +@pytest.mark.parametrize("release_id, expected", [(None, RELEASE_ID), ("Applications/Release2", "Applications/Release2")]) +def test_get_phases_by_title_defaults_to_the_current_release(task, apis, release_id, expected): + task.getPhasesByTitle("Prod", release_id) + + apis.phase.searchPhasesByTitle.assert_called_once_with("Prod", expected) + + +def test_get_releases_by_title_searches_all_releases(task, apis): + result = task.getReleasesByTitle("Nightly") + + apis.release.searchReleasesByTitle.assert_called_once_with("Nightly") + assert result is apis.release.searchReleasesByTitle.return_value + + +@pytest.mark.parametrize("info, expected", [({"version": "25.3.0", "edition": "enterprise"}, "25.3.0"), ({}, None)]) +def test_get_version_reads_the_instance_information(task, apis, info, expected): + apis.settings.getInstanceInformation.return_value = info + + assert task.getVersion() == expected + + +# -- variable getters and setters ------------------------------------------ + +@dataclass(frozen=True) +class Scope: + """How one variable scope maps onto ApiBaseTask helpers and API calls.""" + name: str # variable name as a task author passes it + get: str # ApiBaseTask getter + set: str # ApiBaseTask setter + api: str # attribute of the `apis` fixture + list_variables: str # API method listing the scope's variables + get_list_args: Tuple # its arguments when reading + set_list_args: Tuple # ... and when writing + set_list_kwargs: Dict[str, Any] + list_values: str # API method returning the ${key} -> value map + create: str + create_args: Tuple # arguments before the new Variable + update: str + update_args: Tuple # arguments before the variable id + + +SCOPES = { + "release": Scope("relVar", "getReleaseVariable", "setReleaseVariable", "release", + "getVariables", (RELEASE_ID,), (RELEASE_ID,), {}, "getVariableValues", + "createVariable", (RELEASE_ID,), "updateVariable", ()), + "folder": Scope("folder.myVar", "getFolderVariable", "setFolderVariable", "folder", + "listVariables", (FOLDER_ID,), (FOLDER_ID,), {"folderOnly": True}, "listVariableValues", + "createVariable", (FOLDER_ID,), "updateVariable", (FOLDER_ID,)), + "global": Scope("global.myVar", "getGlobalVariable", "setGlobalVariable", "configuration", + "getGlobalVariables", (), (), {}, "getGlobalVariableValues", + "addGlobalVariable", (), "updateGlobalVariable", ()), +} + +scopes = pytest.mark.parametrize("scope", SCOPES.values(), ids=list(SCOPES)) + + +def scope_api(apis, scope): + return getattr(apis, scope.api) + + +def stored(apis, scope, *variables): + getattr(scope_api(apis, scope), scope.list_variables).return_value = list(variables) + + +@scopes +def test_getter_returns_the_stored_value(task, apis, scope): + stored(apis, scope, variable("other", "x"), variable(scope.name, "value")) + + assert getattr(task, scope.get)(scope.name) == "value" + + getattr(scope_api(apis, scope), scope.list_variables).assert_called_once_with(*scope.get_list_args) + getattr(scope_api(apis, scope), scope.list_values).assert_not_called() + + +@scopes +def test_getter_resolves_a_reference_variable_on_the_server(task, apis, scope): + stored(apis, scope, variable(scope.name, None, "xlrelease.ReferenceVariable")) + values = getattr(scope_api(apis, scope), scope.list_values) + values.return_value = {f"${{{scope.name}}}": "resolved"} + + assert getattr(task, scope.get)(scope.name) == "resolved" + + values.assert_called_once_with(*scope.get_list_args) + + +@scopes +def test_getter_falls_back_to_the_stored_value_when_reference_is_unresolved(task, apis, scope): + stored(apis, scope, variable(scope.name, "stored", "xlrelease.ReferenceVariable")) + getattr(scope_api(apis, scope), scope.list_values).return_value = {} + + assert getattr(task, scope.get)(scope.name) == "stored" + + +@scopes +def test_getter_raises_key_error_listing_available_names(task, apis, scope): + stored(apis, scope, variable("b", 1), variable("a", 2)) + + with pytest.raises(KeyError, match=r"\['a', 'b'\]"): + getattr(task, scope.get)(scope.name) + + +@scopes +def test_setter_updates_an_existing_variable(task, apis, scope): + existing = variable(scope.name, "old") + stored(apis, scope, existing) + update = getattr(scope_api(apis, scope), scope.update) + + result = getattr(task, scope.set)(scope.name, "new") + + getattr(scope_api(apis, scope), scope.list_variables).assert_called_once_with( + *scope.set_list_args, **scope.set_list_kwargs) + update.assert_called_once_with(*scope.update_args, existing.id, existing) + assert existing.value == "new" + assert result is update.return_value + + +@scopes +def test_setter_creates_a_missing_variable_with_an_inferred_type(task, apis, scope): + stored(apis, scope, variable("other", "x")) + create = getattr(scope_api(apis, scope), scope.create) + + result = getattr(task, scope.set)(scope.name, {"b", "a"}) + + (*args, created), _ = create.call_args + assert tuple(args) == scope.create_args + assert (created.key, created.type, sorted(created.value)) == (scope.name, "xlrelease.SetStringVariable", + ["a", "b"]) + assert (created.requiresValue, created.showOnReleaseStart) == (False, False) + getattr(scope_api(apis, scope), scope.update).assert_not_called() + assert result is create.return_value + + +@scopes +def test_setter_rejects_a_value_of_the_wrong_type(task, apis, scope): + stored(apis, scope, variable(scope.name, ["a"], "xlrelease.SetStringVariable")) + + with pytest.raises(TypeError, match="Cannot assign str"): + getattr(task, scope.set)(scope.name, "not-a-set") + + getattr(scope_api(apis, scope), scope.update).assert_not_called() + + +@scopes +def test_setter_leaves_reference_variables_to_the_server(task, apis, scope): + stored(apis, scope, variable(scope.name, None, "xlrelease.ReferenceVariable")) + + getattr(task, scope.set)(scope.name, 42) + + getattr(scope_api(apis, scope), scope.update).assert_called_once() + + +@pytest.mark.parametrize("helper, args", [ + ("getFolderVariable", ("myVar",)), + ("setFolderVariable", ("myVar", "x")), + ("getGlobalVariable", ("myVar",)), + ("setGlobalVariable", ("myVar", "x")), + ("getFolderVariable", ("global.myVar",)), + ("getGlobalVariable", ("folder.myVar",)), +]) +def test_folder_and_global_helpers_require_the_scope_prefix(task, apis, helper, args): + with pytest.raises(ValueError, match="prefix"): + getattr(task, helper)(*args) + + apis.folder.listVariables.assert_not_called() + apis.configuration.getGlobalVariables.assert_not_called() + + +def test_set_release_variable_sends_a_datetime_as_iso_8601(task, apis): + existing = variable("when", None, "xlrelease.DateVariable") + stored(apis, SCOPES["release"], existing) + + task.setReleaseVariable("when", datetime(2026, 6, 24, 10, 30, tzinfo=timezone.utc)) + + assert existing.value == "2026-06-24T10:30:00+00:00" + + +# -- value typing rules ---------------------------------------------------- + +@pytest.mark.parametrize("value, expected", [ + (True, "xlrelease.BooleanVariable"), + (0, "xlrelease.IntegerVariable"), + (date(2026, 1, 1), "xlrelease.DateVariable"), + (datetime(2026, 1, 1, tzinfo=timezone.utc), "xlrelease.DateVariable"), + ({"k": "v"}, "xlrelease.MapStringStringVariable"), + (["a"], "xlrelease.SetStringVariable"), + ({"a"}, "xlrelease.SetStringVariable"), + (("a",), "xlrelease.SetStringVariable"), + ("text", "xlrelease.StringVariable"), + (1.5, "xlrelease.StringVariable"), +], ids=["bool", "int", "date", "datetime", "dict", "list", "set", "tuple", "str", "float"]) +def test_new_variable_type_is_inferred_from_the_value(value, expected): + assert ApiBaseTask._variable_type_for_value(value) == expected + + +@pytest.mark.parametrize("vtype, value", [ + ("xlrelease.BooleanVariable", False), + ("xlrelease.IntegerVariable", 3), + ("xlrelease.DateVariable", date(2026, 1, 1)), + ("xlrelease.DateVariable", "2026-01-01T00:00:00Z"), + ("xlrelease.MapStringStringVariable", {"k": "v"}), + ("xlrelease.SetStringVariable", ("a",)), + ("xlrelease.StringVariable", "text"), + ("xlrelease.PasswordStringVariable", "secret"), + ("custom.UnknownVariable", object()), +]) +def test_value_matching_the_variable_type_is_accepted(vtype, value): + assert ApiBaseTask._value_matches_type(vtype, value) is True + + +@pytest.mark.parametrize("vtype, value", [ + ("xlrelease.BooleanVariable", 1), + ("xlrelease.IntegerVariable", True), + ("xlrelease.IntegerVariable", "3"), + ("xlrelease.DateVariable", 12345), + ("xlrelease.MapStringStringVariable", [("k", "v")]), + ("xlrelease.SetStringVariable", "a"), + ("xlrelease.StringVariable", 42), + ("xlrelease.PasswordStringVariable", None), +]) +def test_value_not_matching_the_variable_type_is_rejected(vtype, value): + assert ApiBaseTask._value_matches_type(vtype, value) is False + + +@pytest.mark.parametrize("value, expected", [ + ({"a"}, ["a"]), + (("a", "b"), ["a", "b"]), + (date(2026, 6, 24), "2026-06-24"), + (datetime(2026, 6, 24, 10, 30, tzinfo=timezone(timedelta(hours=2))), "2026-06-24T10:30:00+02:00"), + (["a"], ["a"]), + ({"k": "v"}, {"k": "v"}), + ("text", "text"), +]) +def test_values_are_coerced_to_a_json_form(value, expected): + assert ApiBaseTask._coerce_value(value) == expected + + +@pytest.mark.parametrize("candidate, expected", [ + (variable("v", None, "xlrelease.ReferenceVariable"), True), + (variable("v", None, "xlrelease.StringVariable", valueProvider=ValueProvider()), True), + (variable("v", "x", "xlrelease.StringVariable"), False), + (variable("v", "x", None), False), +], ids=["by-type", "by-value-provider", "plain", "untyped"]) +def test_reference_variables_are_detected(candidate, expected): + assert ApiBaseTask._is_reference_variable(candidate) is expected diff --git a/tests/release/integration/test_api_base_task_live.py b/tests/release/integration/test_api_base_task_live.py index ce23f74..c9ee333 100644 --- a/tests/release/integration/test_api_base_task_live.py +++ b/tests/release/integration/test_api_base_task_live.py @@ -1,37 +1,24 @@ """ -Live integration tests for :class:`ApiBaseTask`. - -These tests run against a *real* Digital.ai Release server (by default the -developer instance on ``localhost:5516`` with ``admin``/``admin``). They build -an ``ApiBaseTask`` wired exactly the way the Release runtime wires it -- a task -context carrying the release id, the task id and the "Run as user" credentials --- and then exercise the convenience helpers (``getCurrent*``, ``get*ByTitle``, -the variable getters/setters and ``getVersion``) end to end. - -The suite is fully self-contained: - -* ``setUpClass`` first verifies the server is reachable, authentication works - and the API responds within a bounded timeout. If any of those checks fail - the whole suite is skipped (CI has no Release server). -* It then creates every prerequisite object -- a folder, a template (moved into - that folder), a phase, a task, and release/folder/global variables -- and - spins up a release from the template so the task context points at real ids. -* ``tearDownClass`` removes everything again; deleting the folder cascades to - the release and template, so the server is returned to its original state. -""" +ApiBaseTask helpers against a real Digital.ai Release server. + +The server defaults to the developer instance on localhost:5516 with +admin/admin; override with RELEASE_LIVE_URL, RELEASE_LIVE_USER and +RELEASE_LIVE_PASSWORD. The module is skipped when the server is unreachable or +rejects the credentials, so CI (which has no server) skips it. +A module-scoped fixture creates a folder, a template with a phase and a task, +release/folder/global variables, and a release from the template, then deletes +them afterwards. Tests that read a variable and tests that write one use +different variables, so the tests do not depend on each other's order. +""" +import os import socket -import unittest import uuid from datetime import datetime, timezone +from types import SimpleNamespace +from urllib.parse import urlparse -from digitalai.release.release_api_client import ReleaseAPIClient -from digitalai.release.integration.api_base_task import ApiBaseTask -from digitalai.release.integration.input_context import ( - AutomatedTaskAsUserContext, - ReleaseContext, -) - +import pytest from com.xebialabs.xlrelease.api.v1.configuration_api import ConfigurationApi from com.xebialabs.xlrelease.api.v1.folder_api import FolderApi from com.xebialabs.xlrelease.api.v1.phase_api import PhaseApi @@ -45,648 +32,359 @@ from com.xebialabs.xlrelease.domain.task import Task from com.xebialabs.xlrelease.domain.variable import Variable +from digitalai.release.integration.api_base_task import ApiBaseTask +from digitalai.release.integration.input_context import AutomatedTaskAsUserContext, ReleaseContext +from digitalai.release.release_api_client import ReleaseAPIClient + +pytestmark = pytest.mark.live -# Live server connection details. Locally this is the developer's instance; -# CI has no such server, so the suite skips itself when it is not reachable. -LIVE_SERVER_HOST = "localhost" -LIVE_SERVER_PORT = 5516 -LIVE_SERVER_URL = f"http://{LIVE_SERVER_HOST}:{LIVE_SERVER_PORT}" -LIVE_SERVER_USER = "admin" -LIVE_SERVER_PASSWORD = "admin" +SERVER_URL = os.getenv("RELEASE_LIVE_URL", "http://localhost:5516") +USERNAME = os.getenv("RELEASE_LIVE_USER", "admin") +PASSWORD = os.getenv("RELEASE_LIVE_PASSWORD", "admin") +# (connect, read) seconds, so an unresponsive server fails fast instead of hanging. +TIMEOUT = (3.05, 10) -# Bounded timeout (connect, read) so an unresponsive server skips rather than -# hangs the suite. -LIVE_SERVER_TIMEOUT = (3.05, 10) +PHASE_TITLE = "Test Phase" +TASK_TITLE = "Manual Task" -def _server_reachable(host: str, port: int, timeout: float = 0.5) -> bool: - """Return True if a TCP connection to host:port succeeds within the timeout.""" +def server_reachable() -> bool: + url = urlparse(SERVER_URL) try: - with socket.create_connection((host, port), timeout=timeout): + with socket.create_connection((url.hostname, url.port or 80), timeout=0.5): return True except OSError: return False -class _LiveApiTask(ApiBaseTask): - """Concrete ApiBaseTask used to drive the helpers against the live server.""" +if not server_reachable(): + pytest.skip(f"no Release server reachable at {SERVER_URL}", allow_module_level=True) + - def execute(self) -> None: # pragma: no cover - never executed by the tests +class LiveApiTask(ApiBaseTask): + def execute(self) -> None: pass -@unittest.skipUnless( - _server_reachable(LIVE_SERVER_HOST, LIVE_SERVER_PORT), - f"live Release server not available on {LIVE_SERVER_HOST}:{LIVE_SERVER_PORT}", -) -class TestApiBaseTaskLive(unittest.TestCase): - """Live integration tests for the ApiBaseTask convenience helpers.""" - - # Populated by setUpClass; defaults keep tearDownClass safe if setup aborts. - client = None - uid = None - folder_id = None - folder_title = None - template_id = None - release_id = None - release_title = None - task_id = None - global_var_id = None - global_var_name = None - # Global variables created by the "create when missing" tests; tracked here so - # _cleanup can delete them (they live outside the folder, so the folder-delete - # cascade does not remove them). - created_global_var_ids = [] - task = None - - # -- lifecycle ---------------------------------------------------------- - - @classmethod - def setUpClass(cls): - uid = uuid.uuid4().hex[:10] - cls.uid = uid - cls.created_global_var_ids = [] - cls.client = ReleaseAPIClient( - LIVE_SERVER_URL, - LIVE_SERVER_USER, - LIVE_SERVER_PASSWORD, - timeout=LIVE_SERVER_TIMEOUT, - ) - - # Skip gate: verify authentication works and the API responds. Any - # failure here (auth error, timeout, unreachable) skips the whole suite. - folder_api = FolderApi(cls.client) - try: - root_folders = folder_api.listRoot() - if not root_folders: - raise unittest.SkipTest("No root folders available on the server.") - except unittest.SkipTest: - raise - except Exception as exc: # noqa: BLE001 - any failure means "skip" - raise unittest.SkipTest( - f"Release server authentication/API check failed: {exc}" - ) - - try: - cls._create_test_data(uid, root_folders[0].id) - except Exception: - # Clean up whatever was created before re-raising; unittest does not - # call tearDownClass when setUpClass fails. - cls._cleanup() - raise - - # Build the ApiBaseTask exactly as the Release runtime would: server URL - # + "Run as user" credentials + the real release and task ids. - cls.task = _LiveApiTask() - cls.task.release_server_url = LIVE_SERVER_URL - cls.task.release_context = ReleaseContext( - id=cls.release_id, - automated_task_as_user=AutomatedTaskAsUserContext( - username=LIVE_SERVER_USER, password=LIVE_SERVER_PASSWORD - ), - ) - cls.task.task_id = cls.task_id - print(f"Live ApiBaseTask wired to release {cls.release_id}") - - @classmethod - def _create_test_data(cls, uid: str, root_id: str): - """Create the folder/template/release/variables the helpers operate on.""" - template_api = TemplateApi(cls.client) - folder_api = FolderApi(cls.client) - phase_api = PhaseApi(cls.client) - task_api = TaskApi(cls.client) - config_api = ConfigurationApi(cls.client) - - # Folder (under root) that will contain the release. - cls.folder_title = f"AbtLiveFolder_{uid}" - folder = folder_api.addFolder(root_id, Folder(title=cls.folder_title)) - cls.folder_id = folder.id - print(f"Created folder: {cls.folder_id}") - - # Template at root with a phase, a manual task and a release variable. - template_title = f"AbtLiveTemplate_{uid}" - payload = { - "id": f"Applications/Release_abt_live_{uid}", - "title": template_title, - "type": "xlrelease.Release", - "status": "TEMPLATE", - "scheduledStartDate": "2026-06-01T00:00:00+00:00", - "dueDate": "2026-07-01T00:00:00+00:00", - } - response = cls.client.post("/api/v1/templates", json=payload) - response.raise_for_status() - root_template = Release.from_dict(response.json()) - - phase = phase_api.addPhase( - root_template.id, Phase(title="Test Phase", type="xlrelease.Phase") - ) - task_api.addTask(phase.id, Task(title="Manual Task", type="xlrelease.Task")) - template_api.createVariable( - root_template.id, - Variable( - type="xlrelease.StringVariable", - key="relVar", - value="release-value", - requiresValue=False, - showOnReleaseStart=False, - ), - ) - # A set (set-of-string) release variable; its value round-trips as a list. - template_api.createVariable( - root_template.id, - Variable( - type="xlrelease.SetStringVariable", - key="relSetVar", - value=["Apples", "Pears"], - requiresValue=False, - showOnReleaseStart=False, - ), - ) - # A key/value-map release variable; its value round-trips as a dict. - template_api.createVariable( - root_template.id, - Variable( - type="xlrelease.MapStringStringVariable", - key="relMapVar", - value={"env": "dev", "region": "us"}, - requiresValue=False, - showOnReleaseStart=False, - ), - ) - # A date release variable; its value round-trips as an ISO-8601 string. - template_api.createVariable( - root_template.id, - Variable( - type="xlrelease.DateVariable", - key="relDateVar", - value="2026-01-02T03:04:05+00:00", - requiresValue=False, - showOnReleaseStart=False, - ), - ) - # A reference release variable. It holds no literal value -- the server - # resolves it from a value provider -- so getReleaseVariable must read - # its value from the server-resolved value map rather than `.value`. - template_api.createVariable( - root_template.id, - Variable( - type="xlrelease.ReferenceVariable", - key="relRefVar", - referencedType="xlrelease.StringVariable", - requiresValue=False, - showOnReleaseStart=False, - ), - ) - - # Move the template into the folder so releases created from it live - # inside the folder (required for getCurrentFolder to resolve). - folder_api.moveTemplate(cls.folder_id, root_template.id) - moved = [ - t for t in folder_api.getTemplates(cls.folder_id) - if t.title == template_title - ] - if not moved: - raise RuntimeError("Template not found in folder after move.") - cls.template_id = moved[0].id - print(f"Created template in folder: {cls.template_id}") - - # A folder-owned variable. - folder_api.createVariable( - cls.folder_id, - Variable( - type="xlrelease.StringVariable", - key="folder.folderVar", - value="folder-value", - requiresValue=False, - showOnReleaseStart=False, - ), - ) - - # A global variable (lives outside the folder; deleted explicitly later). - cls.global_var_name = f"abtLiveGlobal_{uid}" - global_var = config_api.addGlobalVariable( - Variable( - type="xlrelease.StringVariable", - key=f"global.{cls.global_var_name}", - value="global-value", - requiresValue=False, - showOnReleaseStart=False, - ) - ) - cls.global_var_id = global_var.id - print(f"Created global variable: {cls.global_var_id}") - - # Create the release from the template; it inherits the phase, task and - # release variable, and lives inside the folder. - cls.release_title = f"AbtLiveRelease_{uid}" - release = template_api.create( - cls.template_id, CreateRelease(releaseTitle=cls.release_title) - ) - cls.release_id = release.id - print(f"Created release: {cls.release_id}") - - # Resolve the task id of the release's manual task -> the task context. - tasks = task_api.searchTasksByTitle("Manual Task", cls.release_id) - if not tasks: - raise RuntimeError("Manual Task not found in created release.") - cls.task_id = tasks[0].id - print(f"Resolved task id: {cls.task_id}") - - @classmethod - def _cleanup(cls): - """Best-effort removal of every object created by the suite.""" - client = cls.client - if client is None: - return - - if cls.release_id: - release_api = ReleaseApi(client) +def as_instant(value) -> datetime: + """A date variable value as an aware datetime; the server may format it with 'Z' or its own offset.""" + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + + +def same_value(actual, expected) -> bool: + """Compares variable values; a set variable comes back as a list in server order.""" + if isinstance(expected, list): + return set(actual) == set(expected) + return actual == expected + + +def new_variable(vtype, key, value=None, **extra): + return Variable(type=vtype, key=key, value=value, requiresValue=False, showOnReleaseStart=False, **extra) + + +# Release variables seeded on the template: key -> (type, value). +RELEASE_VARIABLES = { + "relVar": ("xlrelease.StringVariable", "release-value"), + "relSetVar": ("xlrelease.SetStringVariable", ["Apples", "Pears"]), + "relMapVar": ("xlrelease.MapStringStringVariable", {"env": "dev", "region": "us"}), + "relDateVar": ("xlrelease.DateVariable", "2026-01-02T03:04:05+00:00"), + # Written by the setter tests; never read by the getter tests. + "relVarRw": ("xlrelease.StringVariable", "initial"), + "relSetVarRw": ("xlrelease.SetStringVariable", ["initial"]), + "relMapVarRw": ("xlrelease.MapStringStringVariable", {"initial": "yes"}), + "relDateVarRw": ("xlrelease.DateVariable", "2026-01-02T03:04:05+00:00"), +} + + +class Cleanup: + """Deletes what the fixture created, best effort, in reverse dependency order.""" + + def __init__(self, client): + self.client = client + self.release_id = None + self.template_id = None + self.folder_id = None + self.global_variable_ids = [] + + def run(self): + steps = [] + if self.release_id: + steps += [lambda: ReleaseApi(self.client).abort(self.release_id, "Test cleanup"), + lambda: ReleaseApi(self.client).delete(self.release_id)] + if self.template_id: + steps.append(lambda: TemplateApi(self.client).deleteTemplate(self.template_id)) + steps += [lambda v=v: ConfigurationApi(self.client).deleteGlobalVariable(v) for v in self.global_variable_ids] + if self.folder_id: + # Deleting the folder also removes anything still inside it. + steps.append(lambda: FolderApi(self.client).delete(self.folder_id)) + for step in steps: try: - release_api.abort(cls.release_id, "Aborting for test cleanup") - except Exception: - pass - try: - release_api.delete(cls.release_id) - except Exception: + step() + except Exception: # noqa: BLE001 - keep cleaning up pass - if cls.template_id: - try: - TemplateApi(client).deleteTemplate(cls.template_id) - except Exception: - pass - if cls.global_var_id: - try: - ConfigurationApi(client).deleteGlobalVariable(cls.global_var_id) - except Exception: - pass +@pytest.fixture(scope="module") +def live(): + client = ReleaseAPIClient(SERVER_URL, USERNAME, PASSWORD, timeout=TIMEOUT) + try: + root_folders = FolderApi(client).listRoot() + except Exception as exc: # noqa: BLE001 - any failure here means "no usable server" + client.close() + pytest.skip(f"Release server at {SERVER_URL} rejected the API check: {exc}") + if not root_folders: + client.close() + pytest.skip("Release server has no root folder to create test data in") + + uid = uuid.uuid4().hex[:10] + cleanup = Cleanup(client) + try: + yield create_test_data(client, cleanup, uid, root_folders[0].id) + finally: + cleanup.run() + client.close() - # Global variables created by the "create when missing" tests. - for var_id in cls.created_global_var_ids: - try: - ConfigurationApi(client).deleteGlobalVariable(var_id) - except Exception: - pass - # Deleting the folder cascades to anything still inside it (release, - # template), so this is the catch-all that restores the server state. - if cls.folder_id: - try: - FolderApi(client).delete(cls.folder_id) - except Exception: - pass +def create_test_data(client, cleanup, uid, root_folder_id): + folder_api, template_api = FolderApi(client), TemplateApi(client) + config_api = ConfigurationApi(client) + + folder_title = f"AbtLiveFolder_{uid}" + folder_id = folder_api.addFolder(root_folder_id, Folder(title=folder_title)).id + cleanup.folder_id = folder_id + + template_title = f"AbtLiveTemplate_{uid}" + response = client.post("/api/v1/templates", json={ + "id": f"Applications/Release_abt_live_{uid}", + "title": template_title, + "type": "xlrelease.Release", + "status": "TEMPLATE", + "scheduledStartDate": "2026-06-01T00:00:00+00:00", + "dueDate": "2026-07-01T00:00:00+00:00", + }) + response.raise_for_status() + root_template_id = Release.from_dict(response.json()).id + cleanup.template_id = root_template_id + + phase = PhaseApi(client).addPhase(root_template_id, Phase(title=PHASE_TITLE, type="xlrelease.Phase")) + TaskApi(client).addTask(phase.id, Task(title=TASK_TITLE, type="xlrelease.Task")) + for key, (vtype, value) in RELEASE_VARIABLES.items(): + template_api.createVariable(root_template_id, new_variable(vtype, key, value)) + # A reference variable holds no literal value: the server resolves it from a value provider. + template_api.createVariable(root_template_id, new_variable( + "xlrelease.ReferenceVariable", "relRefVar", referencedType="xlrelease.StringVariable")) + + # Releases created from a template inside the folder live in that folder. + folder_api.moveTemplate(folder_id, root_template_id) + template_id = next(t.id for t in folder_api.getTemplates(folder_id) if t.title == template_title) + cleanup.template_id = template_id + + for key, value in (("folder.folderVar", "folder-value"), ("folder.folderVarRw", "initial")): + folder_api.createVariable(folder_id, new_variable("xlrelease.StringVariable", key, value)) + + global_names = {} + for name, value in (("read", "global-value"), ("rw", "initial")): + key = f"global.abtLive_{name}_{uid}" + created = config_api.addGlobalVariable(new_variable("xlrelease.StringVariable", key, value)) + cleanup.global_variable_ids.append(created.id) + global_names[name] = key + + release_title = f"AbtLiveRelease_{uid}" + release_id = template_api.create(template_id, CreateRelease(releaseTitle=release_title)).id + cleanup.release_id = release_id + task_id = TaskApi(client).searchTasksByTitle(TASK_TITLE, release_id)[0].id + + task = LiveApiTask() + task.release_server_url = SERVER_URL + task.release_context = ReleaseContext( + id=release_id, automated_task_as_user=AutomatedTaskAsUserContext(username=USERNAME, password=PASSWORD)) + task.task_id = task_id + + return SimpleNamespace(task=task, uid=uid, folder_id=folder_id, folder_title=folder_title, + release_id=release_id, release_title=release_title, task_id=task_id, + global_read=global_names["read"], global_rw=global_names["rw"], cleanup=cleanup) + + +# -- server and current context -------------------------------------------- + +def test_get_version_returns_the_server_version(live): + version = live.task.getVersion() + + assert isinstance(version, str) and version + + +def test_get_current_release(live): + release = live.task.getCurrentRelease() + + assert (release.id, release.title) == (live.release_id, live.release_title) + + +def test_get_current_task(live): + task = live.task.getCurrentTask() + + assert (task.id, task.title) == (live.task_id, TASK_TITLE) + + +def test_get_current_phase(live): + phase = live.task.getCurrentPhase() + + assert (phase.id, phase.title) == (live.task.get_phase_id(), PHASE_TITLE) + + +def test_get_current_folder(live): + folder = live.task.getCurrentFolder() + + assert (folder.id, folder.title) == (live.folder_id, live.folder_title) + + +# -- release variables ----------------------------------------------------- + +@pytest.mark.parametrize("key, expected", [ + ("relVar", "release-value"), + ("relMapVar", {"env": "dev", "region": "us"}), +]) +def test_get_release_variable(live, key, expected): + assert live.task.getReleaseVariable(key) == expected + + +def test_get_release_set_variable_returns_its_members(live): + assert set(live.task.getReleaseVariable("relSetVar")) == {"Apples", "Pears"} + + +def test_get_release_date_variable_returns_the_stored_instant(live): + assert as_instant(live.task.getReleaseVariable("relDateVar")) == datetime(2026, 1, 2, 3, 4, 5, + tzinfo=timezone.utc) + + +def test_get_release_reference_variable_returns_the_server_resolved_value(live): + resolved = live.task.releaseApi.getVariableValues(live.release_id) + + assert "${relRefVar}" in resolved + assert live.task.getReleaseVariable("relRefVar") == resolved["${relRefVar}"] + + +def test_get_release_variable_raises_for_an_unknown_name(live): + with pytest.raises(KeyError): + live.task.getReleaseVariable("doesNotExist") + + +@pytest.mark.parametrize("key, value", [ + ("relVarRw", "updated"), + ("relMapVarRw", {"env": "prod", "tier": "gold"}), +]) +def test_set_release_variable_persists_the_value(live, key, value): + updated = live.task.setReleaseVariable(key, value) + + assert updated.value == value + assert live.task.getReleaseVariable(key) == value + + +def test_set_release_set_variable_persists_the_members(live): + members = ["Oranges", "Bananas", "Grapes"] + + updated = live.task.setReleaseVariable("relSetVarRw", members) + + assert set(updated.value) == set(members) + assert set(live.task.getReleaseVariable("relSetVarRw")) == set(members) + + +@pytest.mark.parametrize("value", [ + datetime(2027, 7, 8, 9, 10, 11, tzinfo=timezone.utc), + datetime.now(timezone.utc).replace(microsecond=0), +], ids=["fixed", "now"]) +def test_set_release_date_variable_persists_the_instant(live, value): + updated = live.task.setReleaseVariable("relDateVarRw", value) + + assert as_instant(updated.value) == value + assert as_instant(live.task.getReleaseVariable("relDateVarRw")) == value + + +@pytest.mark.parametrize("value, vtype", [ + ("created-value", "xlrelease.StringVariable"), + (["red", "green"], "xlrelease.SetStringVariable"), + ({"stage": "qa"}, "xlrelease.MapStringStringVariable"), +], ids=["string", "set", "map"]) +def test_set_release_variable_creates_a_missing_variable(live, value, vtype): + key = f"relNew_{vtype.split('.')[1]}" + + created = live.task.setReleaseVariable(key, value) + + assert (created.key, created.type) == (key, vtype) + assert same_value(live.task.getReleaseVariable(key), value) + + +# -- folder variables ------------------------------------------------------ + +def test_get_folder_variable(live): + assert live.task.getFolderVariable("folder.folderVar") == "folder-value" + + +def test_set_folder_variable_persists_the_value(live): + updated = live.task.setFolderVariable("folder.folderVarRw", "updated") + + assert updated.value == "updated" + assert live.task.getFolderVariable("folder.folderVarRw") == "updated" + + +@pytest.mark.parametrize("value, vtype", [ + ("folder-created", "xlrelease.StringVariable"), + (["alpha", "beta"], "xlrelease.SetStringVariable"), + ({"team": "infra"}, "xlrelease.MapStringStringVariable"), +], ids=["string", "set", "map"]) +def test_set_folder_variable_creates_a_missing_variable(live, value, vtype): + key = f"folder.new_{vtype.split('.')[1]}" + + created = live.task.setFolderVariable(key, value) + + assert (created.key, created.type) == (key, vtype) + assert same_value(live.task.getFolderVariable(key), value) + + +# -- global variables ------------------------------------------------------ + +def test_get_global_variable(live): + assert live.task.getGlobalVariable(live.global_read) == "global-value" + + +def test_set_global_variable_persists_the_value(live): + updated = live.task.setGlobalVariable(live.global_rw, "updated") + + assert updated.value == "updated" + assert live.task.getGlobalVariable(live.global_rw) == "updated" + + +@pytest.mark.parametrize("value, vtype", [ + ("global-created", "xlrelease.StringVariable"), + (["x", "y"], "xlrelease.SetStringVariable"), + ({"region": "ap"}, "xlrelease.MapStringStringVariable"), +], ids=["string", "set", "map"]) +def test_set_global_variable_creates_a_missing_variable(live, value, vtype): + key = f"global.abtLiveNew_{vtype.split('.')[1]}_{live.uid}" + + created = live.task.setGlobalVariable(key, value) + live.cleanup.global_variable_ids.append(created.id) + + assert (created.key, created.type) == (key, vtype) + assert same_value(live.task.getGlobalVariable(key), value) + + +# -- search by title ------------------------------------------------------- + +def test_get_phases_by_title_searches_the_current_release(live): + phases = live.task.getPhasesByTitle(PHASE_TITLE) + + assert phases and all(p.title == PHASE_TITLE for p in phases) + + +@pytest.mark.parametrize("phase_title", [None, PHASE_TITLE], ids=["release-wide", "in-phase"]) +def test_get_tasks_by_title_finds_the_current_task(live, phase_title): + tasks = live.task.getTasksByTitle(TASK_TITLE, phase_title) + + assert live.task_id in [t.id for t in tasks] + + +def test_get_releases_by_title_finds_the_current_release(live): + releases = live.task.getReleasesByTitle(live.release_title) + + assert live.release_id in [r.id for r in releases] - @classmethod - def tearDownClass(cls): - cls._cleanup() - - # -- getVersion --------------------------------------------------------- - - def test_01_get_version(self): - """getVersion returns the server's version string.""" - version = self.task.getVersion() - self.assertIsInstance(version, str) - self.assertTrue(version) - print(f"getVersion -> {version}") - - # -- getCurrent* -------------------------------------------------------- - - def test_02_get_current_release(self): - """getCurrentRelease returns the release the task belongs to.""" - release = self.task.getCurrentRelease() - self.assertEqual(release.id, self.release_id) - self.assertEqual(release.title, self.release_title) - print(f"getCurrentRelease -> {release.title}") - - def test_03_get_current_task(self): - """getCurrentTask returns the task running the code.""" - task = self.task.getCurrentTask() - self.assertEqual(task.id, self.task_id) - self.assertEqual(task.title, "Manual Task") - print(f"getCurrentTask -> {task.title}") - - def test_04_get_current_phase(self): - """getCurrentPhase returns the phase that contains the task.""" - phase = self.task.getCurrentPhase() - self.assertEqual(phase.id, self.task.get_phase_id()) - self.assertEqual(phase.title, "Test Phase") - print(f"getCurrentPhase -> {phase.title}") - - def test_05_get_current_folder(self): - """getCurrentFolder returns the folder that contains the release.""" - folder = self.task.getCurrentFolder() - self.assertEqual(folder.id, self.folder_id) - self.assertEqual(folder.title, self.folder_title) - print(f"getCurrentFolder -> {folder.title}") - - # -- release variables -------------------------------------------------- - - def test_06_get_release_variable(self): - """getReleaseVariable returns the value of a current-release variable.""" - value = self.task.getReleaseVariable("relVar") - self.assertEqual(value, "release-value") - print(f"getReleaseVariable('relVar') -> {value}") - - def test_07_set_release_variable(self): - """setReleaseVariable persists a new value, readable back via the getter.""" - updated = self.task.setReleaseVariable("relVar", "release-value-updated") - self.assertEqual(updated.value, "release-value-updated") - self.assertEqual( - self.task.getReleaseVariable("relVar"), "release-value-updated" - ) - print("setReleaseVariable('relVar') -> release-value-updated") - - def test_08_get_release_variable_missing_raises(self): - """getReleaseVariable raises KeyError for an unknown variable name.""" - with self.assertRaises(KeyError): - self.task.getReleaseVariable("doesNotExist") - print("getReleaseVariable(unknown) correctly raised KeyError") - - # -- release variables: set (set-of-string) ----------------------------- - - def test_08a_get_release_set_variable(self): - """getReleaseVariable returns a set variable's value as a list.""" - value = self.task.getReleaseVariable("relSetVar") - # A set is unordered on the server, so compare membership, not order. - self.assertEqual(set(value), {"Apples", "Pears"}) - print(f"getReleaseVariable('relSetVar') -> {value}") - - def test_08b_set_release_set_variable(self): - """setReleaseVariable persists a new set value, readable back via the getter.""" - new_value = ["Oranges", "Bananas", "Grapes"] - updated = self.task.setReleaseVariable("relSetVar", new_value) - self.assertEqual(set(updated.value), set(new_value)) - self.assertEqual( - set(self.task.getReleaseVariable("relSetVar")), set(new_value) - ) - print(f"setReleaseVariable('relSetVar') -> {new_value}") - - # -- release variables: key/value map ----------------------------------- - - def test_08c_get_release_map_variable(self): - """getReleaseVariable returns a key/value-map variable's value as a dict.""" - value = self.task.getReleaseVariable("relMapVar") - self.assertEqual(value, {"env": "dev", "region": "us"}) - print(f"getReleaseVariable('relMapVar') -> {value}") - - def test_08d_set_release_map_variable(self): - """setReleaseVariable persists a new map value, readable back via the getter.""" - new_value = {"env": "prod", "region": "eu", "tier": "gold"} - updated = self.task.setReleaseVariable("relMapVar", new_value) - self.assertEqual(updated.value, new_value) - self.assertEqual(self.task.getReleaseVariable("relMapVar"), new_value) - print(f"setReleaseVariable('relMapVar') -> {new_value}") - - # -- release variables: date -------------------------------------------- - - def test_08c1_get_release_date_variable(self): - """getReleaseVariable returns a date variable's value as an ISO-8601 string.""" - value = self.task.getReleaseVariable("relDateVar") - # The server stores the seeded instant; compare instants, not the - # literal string (the server formats in its own timezone). - stored = datetime.fromisoformat(str(value).replace("Z", "+00:00")) - self.assertEqual(stored, datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc)) - print(f"getReleaseVariable('relDateVar') -> {value}") - - def test_08c2_set_release_date_variable(self): - """setReleaseVariable persists a datetime, readable back via the getter.""" - # The server stores dates to second precision, so drop microseconds. - new_value = datetime(2027, 7, 8, 9, 10, 11, tzinfo=timezone.utc) - updated = self.task.setReleaseVariable("relDateVar", new_value) - # The SDK coerces the datetime to its ISO-8601 string before sending; the - # server echoes it back normalised to its own timezone (e.g. trailing - # 'Z'), so compare the instant rather than the literal string. - self.assertEqual( - datetime.fromisoformat(str(updated.value).replace("Z", "+00:00")), - new_value, - ) - read_back = datetime.fromisoformat( - str(self.task.getReleaseVariable("relDateVar")).replace("Z", "+00:00") - ) - self.assertEqual(read_back, new_value) - print(f"setReleaseVariable('relDateVar') -> {new_value.isoformat()}") - - def test_08c3_set_release_date_variable_current_time(self): - """setReleaseVariable accepts the current date/time and round-trips it.""" - now = datetime.now(timezone.utc).replace(microsecond=0) - self.task.setReleaseVariable("relDateVar", now) - read_back = datetime.fromisoformat( - str(self.task.getReleaseVariable("relDateVar")).replace("Z", "+00:00") - ) - self.assertEqual(read_back, now) - print(f"setReleaseVariable('relDateVar') current time -> {now.isoformat()}") - - # -- release variables: reference --------------------------------------- - - def test_08h_get_release_reference_variable(self): - """getReleaseVariable returns a reference variable's server-resolved value. - - A reference variable carries no literal value (``.value`` is empty); the - getter must read it from the resolved value map. The reference here has - no real provider target, so it resolves to an empty string -- but the - getter must return exactly what the resolved map holds for ``${key}``, - proving it delegates resolution to the server rather than returning the - raw (empty) ``.value``. - """ - resolved = self.task.releaseApi.getVariableValues(self.release_id) - expected = resolved.get("${relRefVar}") - self.assertIsNotNone(expected, "reference variable missing from value map") - self.assertEqual(self.task.getReleaseVariable("relRefVar"), expected) - print(f"getReleaseVariable('relRefVar') -> {expected!r} (server-resolved)") - - # -- release variables: create when missing ----------------------------- - - def test_08e_set_release_variable_creates_when_missing(self): - """setReleaseVariable creates a string variable when the key is unknown.""" - created = self.task.setReleaseVariable("relNewVar", "created-value") - self.assertEqual(created.key, "relNewVar") - self.assertEqual(created.type, "xlrelease.StringVariable") - self.assertEqual(created.value, "created-value") - self.assertEqual(self.task.getReleaseVariable("relNewVar"), "created-value") - print("setReleaseVariable('relNewVar') created -> created-value") - - def test_08f_set_release_set_variable_creates_when_missing(self): - """setReleaseVariable creates a set variable when the key is unknown.""" - new_value = ["red", "green", "blue"] - created = self.task.setReleaseVariable("relNewSetVar", new_value) - self.assertEqual(created.type, "xlrelease.SetStringVariable") - self.assertEqual(set(created.value), set(new_value)) - self.assertEqual( - set(self.task.getReleaseVariable("relNewSetVar")), set(new_value) - ) - print(f"setReleaseVariable('relNewSetVar') created -> {new_value}") - - def test_08g_set_release_map_variable_creates_when_missing(self): - """setReleaseVariable creates a map variable when the key is unknown.""" - new_value = {"stage": "qa", "owner": "team-a"} - created = self.task.setReleaseVariable("relNewMapVar", new_value) - self.assertEqual(created.type, "xlrelease.MapStringStringVariable") - self.assertEqual(created.value, new_value) - self.assertEqual(self.task.getReleaseVariable("relNewMapVar"), new_value) - print(f"setReleaseVariable('relNewMapVar') created -> {new_value}") - - # -- folder variables --------------------------------------------------- - - def test_09_get_folder_variable(self): - """getFolderVariable resolves a folder variable by its prefixed name.""" - self.assertEqual( - self.task.getFolderVariable("folder.folderVar"), "folder-value" - ) - print("getFolderVariable('folder.folderVar') -> folder-value") - - def test_09a_get_folder_variable_requires_prefix(self): - """getFolderVariable raises ValueError when the folder. prefix is missing.""" - with self.assertRaises(ValueError): - self.task.getFolderVariable("folderVar") - print("getFolderVariable('folderVar') correctly raised ValueError") - - def test_10_set_folder_variable(self): - """setFolderVariable persists a new value on the folder-owned variable.""" - updated = self.task.setFolderVariable( - "folder.folderVar", "folder-value-updated" - ) - self.assertEqual(updated.value, "folder-value-updated") - self.assertEqual( - self.task.getFolderVariable("folder.folderVar"), "folder-value-updated" - ) - print("setFolderVariable('folder.folderVar') -> folder-value-updated") - - def test_10_set_folder_variable_requires_prefix(self): - """setFolderVariable raises ValueError when the folder. prefix is missing.""" - with self.assertRaises(ValueError): - self.task.setFolderVariable("folderVar", "nope") - print("setFolderVariable('folderVar') correctly raised ValueError") - - # -- folder variables: create when missing ------------------------------ - - def test_10a_set_folder_variable_creates_when_missing(self): - """setFolderVariable creates a folder-owned variable when the key is unknown.""" - created = self.task.setFolderVariable("folder.folderNewVar", "folder-created") - self.assertEqual(created.key, "folder.folderNewVar") - self.assertEqual(created.type, "xlrelease.StringVariable") - self.assertEqual(created.value, "folder-created") - self.assertEqual( - self.task.getFolderVariable("folder.folderNewVar"), "folder-created" - ) - print("setFolderVariable('folder.folderNewVar') created -> folder-created") - - def test_10b_set_folder_set_variable_creates_when_missing(self): - """setFolderVariable creates a set variable when the key is unknown.""" - new_value = ["alpha", "beta"] - created = self.task.setFolderVariable("folder.folderNewSetVar", new_value) - self.assertEqual(created.type, "xlrelease.SetStringVariable") - self.assertEqual(set(created.value), set(new_value)) - self.assertEqual( - set(self.task.getFolderVariable("folder.folderNewSetVar")), set(new_value) - ) - print(f"setFolderVariable('folder.folderNewSetVar') created -> {new_value}") - - def test_10c_set_folder_map_variable_creates_when_missing(self): - """setFolderVariable creates a map variable when the key is unknown.""" - new_value = {"team": "infra", "tier": "silver"} - created = self.task.setFolderVariable("folder.folderNewMapVar", new_value) - self.assertEqual(created.type, "xlrelease.MapStringStringVariable") - self.assertEqual(created.value, new_value) - self.assertEqual( - self.task.getFolderVariable("folder.folderNewMapVar"), new_value - ) - print(f"setFolderVariable('folder.folderNewMapVar') created -> {new_value}") - - # -- global variables --------------------------------------------------- - - def test_11_get_global_variable(self): - """getGlobalVariable resolves a global variable by its prefixed name.""" - self.assertEqual( - self.task.getGlobalVariable(f"global.{self.global_var_name}"), - "global-value", - ) - print(f"getGlobalVariable('global.{self.global_var_name}') -> global-value") - - def test_11a_get_global_variable_requires_prefix(self): - """getGlobalVariable raises ValueError when the global. prefix is missing.""" - with self.assertRaises(ValueError): - self.task.getGlobalVariable(self.global_var_name) - print("getGlobalVariable(unprefixed) correctly raised ValueError") - - def test_12_set_global_variable(self): - """setGlobalVariable persists a new value on the global variable.""" - key = f"global.{self.global_var_name}" - updated = self.task.setGlobalVariable(key, "global-value-updated") - self.assertEqual(updated.value, "global-value-updated") - self.assertEqual(self.task.getGlobalVariable(key), "global-value-updated") - print(f"setGlobalVariable('{key}') -> global-value-updated") - - def test_12_set_global_variable_requires_prefix(self): - """setGlobalVariable raises ValueError when the global. prefix is missing.""" - with self.assertRaises(ValueError): - self.task.setGlobalVariable(self.global_var_name, "nope") - print("setGlobalVariable(unprefixed) correctly raised ValueError") - - # -- global variables: create when missing ------------------------------ - - def test_12a_set_global_variable_creates_when_missing(self): - """setGlobalVariable creates a global variable when the key is unknown.""" - name = f"global.abtLiveGlobalNew_{self.uid}" - created = self.task.setGlobalVariable(name, "global-created") - self.created_global_var_ids.append(created.id) - self.assertEqual(created.key, name) - self.assertEqual(created.type, "xlrelease.StringVariable") - self.assertEqual(created.value, "global-created") - self.assertEqual(self.task.getGlobalVariable(name), "global-created") - print(f"setGlobalVariable('{name}') created -> global-created") - - def test_12b_set_global_set_variable_creates_when_missing(self): - """setGlobalVariable creates a set variable when the key is unknown.""" - name = f"global.abtLiveGlobalNewSet_{self.uid}" - new_value = ["x", "y", "z"] - created = self.task.setGlobalVariable(name, new_value) - self.created_global_var_ids.append(created.id) - self.assertEqual(created.type, "xlrelease.SetStringVariable") - self.assertEqual(set(created.value), set(new_value)) - self.assertEqual(set(self.task.getGlobalVariable(name)), set(new_value)) - print(f"setGlobalVariable('{name}') created -> {new_value}") - - def test_12c_set_global_map_variable_creates_when_missing(self): - """setGlobalVariable creates a map variable when the key is unknown.""" - name = f"global.abtLiveGlobalNewMap_{self.uid}" - new_value = {"region": "ap", "tier": "bronze"} - created = self.task.setGlobalVariable(name, new_value) - self.created_global_var_ids.append(created.id) - self.assertEqual(created.type, "xlrelease.MapStringStringVariable") - self.assertEqual(created.value, new_value) - self.assertEqual(self.task.getGlobalVariable(name), new_value) - print(f"setGlobalVariable('{name}') created -> {new_value}") - - # -- ...ByTitle --------------------------------------------------------- - - def test_13_get_phases_by_title(self): - """getPhasesByTitle finds the phase by title in the current release.""" - phases = self.task.getPhasesByTitle("Test Phase") - self.assertGreaterEqual(len(phases), 1) - self.assertTrue(all(p.title == "Test Phase" for p in phases)) - print(f"getPhasesByTitle('Test Phase') -> {len(phases)} phase(s)") - - def test_14_get_tasks_by_title(self): - """getTasksByTitle finds the task by title, optionally scoped to a phase.""" - tasks = self.task.getTasksByTitle("Manual Task") - self.assertGreaterEqual(len(tasks), 1) - self.assertTrue(any(t.id == self.task_id for t in tasks)) - - scoped = self.task.getTasksByTitle("Manual Task", "Test Phase") - self.assertGreaterEqual(len(scoped), 1) - print( - f"getTasksByTitle('Manual Task') -> {len(tasks)} task(s), " - f"scoped to phase -> {len(scoped)} task(s)" - ) - - def test_15_get_releases_by_title(self): - """getReleasesByTitle finds the current release by its title.""" - releases = self.task.getReleasesByTitle(self.release_title) - self.assertGreaterEqual(len(releases), 1) - self.assertTrue(any(r.id == self.release_id for r in releases)) - print(f"getReleasesByTitle('{self.release_title}') -> {len(releases)} release(s)") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/release/integration/test_base_task.py b/tests/release/integration/test_base_task.py index b6ef8fc..04afcfd 100644 --- a/tests/release/integration/test_base_task.py +++ b/tests/release/integration/test_base_task.py @@ -1,222 +1,269 @@ -import unittest -from types import SimpleNamespace from unittest import mock +import pytest + +from digitalai.release.integration import base_task from digitalai.release.integration.base_task import BaseTask from digitalai.release.integration.exceptions import AbortException -from digitalai.release.integration.ids import Ids -from digitalai.release.integration.input_context import ( - AutomatedTaskAsUserContext, - ReleaseContext, -) +from digitalai.release.integration.input_context import AutomatedTaskAsUserContext, ReleaseContext from digitalai.release.integration.output_context import OutputContext +from tests.support import FOLDER_ID, PHASE_ID, RELEASE_ID, SERVER_URL, TASK_ID, NoOpTask + + +def make_task(task_class=NoOpTask, *, username="admin", password="secret", server_url=SERVER_URL): + task = task_class() + task.task_id = TASK_ID + task.release_server_url = server_url + task.release_context = ReleaseContext( + id=RELEASE_ID, + automated_task_as_user=AutomatedTaskAsUserContext(username=username, password=password), + ) + return task -# Realistic ids, matching the sample input context. -FOLDER_ID = 'Applications/Folder1f65c7220b394afbb941154342fd9fc6' -RELEASE_ID = f'{FOLDER_ID}/Release31de09e95c8e4ebb95aaed29a8082d0b' -PHASE_ID = f'{RELEASE_ID}/Phase723a601c78804f7dbcaa8b05b83708f5' -TASK_ID = f'{PHASE_ID}/Task3a35b67b42b6428b854857fba470b39a' +class RaisingTask(BaseTask): + error: BaseException = RuntimeError("boom") -class _StubTask(BaseTask): - """Minimal concrete task so BaseTask can be instantiated in tests.""" + def execute(self) -> None: + raise self.error - def execute(self) -> None: # pragma: no cover - never run + +class NoSuperInitTask(BaseTask): + """A task whose __init__ skips super().__init__(), as some existing plugins do.""" + + def __init__(self): pass + def execute(self) -> None: + pass -class TestIdParsing(unittest.TestCase): - def test_phase_id_from_task_id(self): - self.assertEqual(Ids.phase_id_from(TASK_ID), PHASE_ID) +# -- execute_task / abort -------------------------------------------------- - def test_find_folder_id_from_release_id(self): - self.assertEqual(Ids.find_folder_id(RELEASE_ID), FOLDER_ID) +def test_new_task_has_a_successful_empty_output_context(): + assert NoOpTask().get_output_context() == OutputContext(0, "", {}, []) - def test_phase_id_from_raises_when_absent(self): - with self.assertRaises(ValueError): - Ids.phase_id_from('Applications/Folder1/Release1') - def test_segment_name_and_parent_id(self): - self.assertEqual(Ids.segment_name(TASK_ID), 'Task3a35b67b42b6428b854857fba470b39a') - self.assertEqual(Ids.parent_id(TASK_ID), PHASE_ID) - self.assertTrue(Ids.is_root('Applications')) - self.assertFalse(Ids.is_root(RELEASE_ID)) +def test_execute_task_leaves_a_successful_result_unchanged(): + task = NoOpTask() + task.execute_task() -class TestBaseTaskIds(unittest.TestCase): + assert task.get_output_context() == OutputContext(0, "", {}, []) - def _task(self): - task = _StubTask() - task.task_id = TASK_ID - task.release_context = SimpleNamespace(id=RELEASE_ID) - return task - def test_get_phase_id_derives_from_task_id(self): - self.assertEqual(self._task().get_phase_id(), PHASE_ID) +def test_execute_task_reports_an_unexpected_error_as_a_failure(): + task = RaisingTask() - def test_get_folder_id_derives_from_release_id(self): - self.assertEqual(self._task().get_folder_id(), FOLDER_ID) + task.execute_task() + assert (task.output_context.exit_code, task.output_context.job_error_message) == (1, "boom") -class TestBaseTaskOutput(unittest.TestCase): - """Robustness coverage for output handling and credential validation.""" - def _task(self): - task = _StubTask() - task.task_id = TASK_ID - return task +def test_execute_task_turns_an_abort_into_exit_code_1(): + task = RaisingTask() + task.error = AbortException() - def test_execute_task_sets_success_output(self): - task = self._task() + with pytest.raises(SystemExit) as exit_info: task.execute_task() - ctx = task.get_output_context() - self.assertEqual(ctx.exit_code, 0) - self.assertEqual(ctx.job_error_message, "") - def test_execute_task_captures_unexpected_error(self): - class _Boom(_StubTask): - def execute(self) -> None: - raise RuntimeError("boom") + assert exit_info.value.code == 1 + assert (task.output_context.exit_code, task.output_context.job_error_message) == (1, "Abort requested") - task = _Boom() - task.execute_task() - ctx = task.get_output_context() - self.assertEqual(ctx.exit_code, 1) - self.assertEqual(ctx.job_error_message, "boom") - - def test_execute_task_handles_abort(self): - class _Abort(_StubTask): - def execute(self) -> None: - raise AbortException() - - task = _Abort() - with self.assertRaises(SystemExit) as cm: - task.execute_task() - self.assertEqual(cm.exception.code, 1) - ctx = task.get_output_context() - self.assertEqual(ctx.exit_code, 1) - self.assertEqual(ctx.job_error_message, "Abort requested") - - def test_abort_raises_abort_exception_and_marks_output_context(self): - task = self._task() - task.output_context = OutputContext(0, "", {}, []) - - with self.assertRaises(AbortException): - task.abort() - - ctx = task.get_output_context() - self.assertEqual(ctx.exit_code, 1) - self.assertEqual(ctx.job_error_message, "Abort requested") - - def test_output_context_is_never_none_immediately_after_construction(self): - # Guards against the window between task construction and the worker - # thread starting execute_task(), during which the main thread's - # SIGABRT/SIGTERM handler may already call abort(). - task = self._task() - self.assertIsNotNone(task.get_output_context()) - - def test_abort_immediately_after_construction_does_not_crash(self): - # Simulates an abort signal arriving before execute_task()/execute() - # has ever run on the worker thread. - task = self._task() - - with self.assertRaises(AbortException): - task.abort() - - ctx = task.get_output_context() - self.assertEqual(ctx.exit_code, 1) - self.assertEqual(ctx.job_error_message, "Abort requested") - - def test_is_abort_requested_defaults_to_false(self): - task = self._task() - self.assertFalse(task.is_abort_requested()) - - def test_is_abort_requested_true_after_request_abort(self): - task = self._task() - task._request_abort() - self.assertTrue(task.is_abort_requested()) - - def test_set_output_property_rejects_empty_name(self): - task = self._task() - task.execute_task() - with self.assertRaises(ValueError): - task.set_output_property("", "value") - def test_set_output_property_rejects_unsupported_type(self): - task = self._task() - task.execute_task() - with self.assertRaises(ValueError): - task.set_output_property("name", object()) - - def test_get_input_properties_requires_value(self): - task = self._task() - with self.assertRaises(ValueError): - task.get_input_properties() - - def test_get_task_user_returns_none_without_release_context(self): - task = self._task() - self.assertIsNone(task.get_task_user()) - - def test_get_release_api_client_raises_without_task_user(self): - task = self._task() - task.release_server_url = "http://localhost:5516" - with self.assertRaises(ValueError): - task.get_release_api_client() - - -class TestAutomatedTaskAsUser(unittest.TestCase): - """Coverage for the 'Run as user' (automatedTaskAsUser) username/password.""" - - SERVER_URL = "http://localhost:5516" - - def _task(self, username, password): - task = _StubTask() - task.task_id = TASK_ID - task.release_server_url = self.SERVER_URL - task.release_context = ReleaseContext( - id=RELEASE_ID, - automated_task_as_user=AutomatedTaskAsUserContext(username=username, password=password), - ) - return task - - def test_get_task_user_returns_credentials(self): - task = self._task("admin", "secret") - user = task.get_task_user() - self.assertEqual(user.username, "admin") - self.assertEqual(user.password, "secret") - - def test_get_release_api_client_passes_credentials(self): - task = self._task("admin", "secret") - with mock.patch("digitalai.release.integration.base_task.ReleaseAPIClient") as fake_client: - client = task.get_release_api_client() - fake_client.assert_called_once_with(self.SERVER_URL, "admin", "secret", timeout=None) - self.assertIs(client, fake_client.return_value) - - def test_get_release_api_client_raises_when_password_missing(self): - task = self._task("admin", None) - with self.assertRaises(ValueError): - task.get_release_api_client() - - def test_get_release_api_client_uses_password_as_token_when_username_missing(self): - task = self._task(None, "secret") - with mock.patch("digitalai.release.integration.base_task.ReleaseAPIClient") as fake_client: - client = task.get_release_api_client() - fake_client.assert_called_once_with( - self.SERVER_URL, personal_access_token="secret", timeout=None) - self.assertIs(client, fake_client.return_value) - - def test_get_release_api_client_raises_when_credentials_blank(self): - task = self._task("", "") - with self.assertRaises(ValueError): - task.get_release_api_client() - - def test_get_release_api_client_raises_without_server_url(self): - task = self._task("admin", "secret") - task.release_server_url = None - with self.assertRaises(ValueError): - task.get_release_api_client() - - -if __name__ == '__main__': - unittest.main() +def test_abort_marks_the_result_aborted_and_raises(): + task = NoOpTask() + + with pytest.raises(AbortException): + task.abort() + + assert (task.output_context.exit_code, task.output_context.job_error_message) == (1, "Abort requested") + + +def test_abort_is_not_requested_initially(): + assert NoOpTask().is_abort_requested() is False + + +def test_request_abort_makes_is_abort_requested_true(): + task = NoOpTask() + + task._request_abort() + + assert task.is_abort_requested() is True + + +def test_task_without_super_init_gets_an_output_context_on_first_use(): + task = NoSuperInitTask() + + task.set_output_property("ok", True) + + assert task.get_output_context() == OutputContext(0, "", {"ok": True}, []) + + +def test_task_without_super_init_supports_abort_requests(): + task = NoSuperInitTask() + assert task.is_abort_requested() is False + + task._request_abort() + + assert task.is_abort_requested() is True + + +# -- output ---------------------------------------------------------------- + +@pytest.mark.parametrize("value", ["text", "", 7, 0, True, False, [1, "a"], {"k": "v"}, None], + ids=lambda v: repr(v)) +def test_set_output_property_accepts_supported_types(value): + task = NoOpTask() + + task.set_output_property("name", value) + + assert task.get_output_properties() == {"name": value} + + +@pytest.mark.parametrize("value", [1.5, (1, 2), {1, 2}, object()], ids=["float", "tuple", "set", "object"]) +def test_set_output_property_rejects_unsupported_types(value): + with pytest.raises(ValueError, match="Invalid data type"): + NoOpTask().set_output_property("name", value) + + +@pytest.mark.parametrize("name", ["", None]) +def test_set_output_property_rejects_an_empty_name(name): + with pytest.raises(ValueError, match="name cannot be empty"): + NoOpTask().set_output_property(name, "value") + + +def test_set_exit_code_and_error_message_update_the_output_context(): + task = NoOpTask() + + task.set_exit_code(3) + task.set_error_message("partial failure") + + assert task.get_output_context() == OutputContext(3, "partial failure", {}, []) + + +def test_add_reporting_record_appends_in_order(): + task = NoOpTask() + + task.add_reporting_record({"n": 1}) + task.add_reporting_record({"n": 2}) + + assert task.get_output_context().reporting_records == [{"n": 1}, {"n": 2}] + + +def test_add_comment_logs_the_comment_marker(dai_log): + NoOpTask().add_comment("Deployed v1.2") + + assert dai_log.messages == ["##[start: comment]Deployed v1.2##[end: comment]"] + + +def test_set_status_line_logs_the_status_marker(dai_log): + NoOpTask().set_status_line("Waiting for approval") + + assert dai_log.messages == ["##[start: status]Waiting for approval##[end: status]"] + + +# -- input and context ----------------------------------------------------- + +def test_get_input_properties_returns_the_properties_set_by_the_wrapper(): + task = NoOpTask() + task.input_properties = {"yourName": "World"} + + assert task.get_input_properties() == {"yourName": "World"} + + +def test_get_input_properties_raises_until_set(): + with pytest.raises(ValueError, match="have not been set"): + NoOpTask().get_input_properties() + + +def test_context_getters_return_the_wired_ids(): + task = make_task() + + assert task.get_task_id() == TASK_ID + assert task.get_release_id() == RELEASE_ID + assert task.get_release_server_url() == SERVER_URL + + +def test_get_phase_id_is_derived_from_the_task_id(): + assert make_task().get_phase_id() == PHASE_ID + + +def test_get_folder_id_is_derived_from_the_release_id(): + assert make_task().get_folder_id() == FOLDER_ID + + +def test_get_task_user_returns_the_run_as_user(): + assert make_task(username="admin", password="secret").get_task_user() == AutomatedTaskAsUserContext( + "admin", "secret") + + +def test_get_task_user_is_none_without_a_release_context(): + assert NoOpTask().get_task_user() is None + + +# -- get_release_api_client ------------------------------------------------ + +@pytest.fixture +def client_class(monkeypatch): + """Replaces ReleaseAPIClient so no client is built; returns the stand-in class.""" + fake = mock.Mock(name="ReleaseAPIClient") + monkeypatch.setattr(base_task, "ReleaseAPIClient", fake) + return fake + + +def test_api_client_uses_the_run_as_user_credentials(client_class): + client = make_task(username="admin", password="secret").get_release_api_client() + + client_class.assert_called_once_with(SERVER_URL, "admin", "secret", timeout=None) + assert client is client_class.return_value + + +def test_api_client_treats_a_password_without_username_as_a_token(client_class): + make_task(username=None, password="pat-123").get_release_api_client() + + client_class.assert_called_once_with(SERVER_URL, personal_access_token="pat-123", timeout=None) + + +def test_api_client_prefers_an_explicit_token_over_credentials(client_class): + make_task().get_release_api_client(personal_access_token="pat-123") + + client_class.assert_called_once_with(SERVER_URL, personal_access_token="pat-123", timeout=None) + + +def test_api_client_arguments_override_the_task_context(client_class): + make_task().get_release_api_client("http://other:5516", "deployer", "pw", timeout=(1, 5), + headers={"X-Test": "1"}) + + client_class.assert_called_once_with("http://other:5516", "deployer", "pw", timeout=(1, 5), + headers={"X-Test": "1"}) + + +def test_api_client_works_without_a_task_context_when_given_arguments(client_class): + NoOpTask().get_release_api_client(SERVER_URL, "admin", "secret") + + client_class.assert_called_once_with(SERVER_URL, "admin", "secret", timeout=None) + + +@pytest.mark.parametrize("username, password, server_url", [ + ("admin", None, SERVER_URL), + ("", "", SERVER_URL), + (None, None, SERVER_URL), + ("admin", "secret", None), + (None, "pat-123", None), +], ids=["no-password", "blank-credentials", "no-credentials", "no-server-url", "token-without-server-url"]) +def test_api_client_requires_server_url_and_credentials(client_class, username, password, server_url): + task = make_task(username=username, password=password, server_url=server_url) + + with pytest.raises(ValueError, match="Cannot connect to Release API"): + task.get_release_api_client() + + client_class.assert_not_called() + + +def test_api_client_requires_a_release_context_or_arguments(client_class): + task = NoOpTask() + task.release_server_url = SERVER_URL + + with pytest.raises(ValueError, match="Cannot connect to Release API"): + task.get_release_api_client() diff --git a/tests/release/integration/test_ids.py b/tests/release/integration/test_ids.py new file mode 100644 index 0000000..2ae2145 --- /dev/null +++ b/tests/release/integration/test_ids.py @@ -0,0 +1,48 @@ +import pytest + +from digitalai.release.integration.ids import Ids +from tests.support import FOLDER_ID, PHASE_ID, RELEASE_ID, TASK_ID + + +@pytest.mark.parametrize("object_id, expected", [ + (TASK_ID, "Task3a35b67b42b6428b854857fba470b39a"), + ("Applications", "Applications"), +]) +def test_segment_name_is_the_last_path_segment(object_id, expected): + assert Ids.segment_name(object_id) == expected + + +def test_parent_id_drops_the_last_segment(): + assert Ids.parent_id(TASK_ID) == PHASE_ID + + +@pytest.mark.parametrize("object_id, expected", [ + ("Applications", True), + (RELEASE_ID, False), +]) +def test_is_root_is_true_only_without_a_separator(object_id, expected): + assert Ids.is_root(object_id) is expected + + +@pytest.mark.parametrize("object_id", [ + TASK_ID, + f"{PHASE_ID}/TaskGroup1/Task2", + PHASE_ID, +], ids=["task", "task-in-group", "phase-itself"]) +def test_phase_id_from_finds_the_enclosing_phase(object_id): + assert Ids.phase_id_from(object_id) == PHASE_ID + + +@pytest.mark.parametrize("object_id", [RELEASE_ID, "Task1"]) +def test_phase_id_from_raises_when_no_phase_in_path(object_id): + with pytest.raises(ValueError, match="No phase found"): + Ids.phase_id_from(object_id) + + +@pytest.mark.parametrize("object_id, expected", [ + (RELEASE_ID, FOLDER_ID), + (f"{FOLDER_ID}/Folder2/Release1", f"{FOLDER_ID}/Folder2"), + ("Applications/Release1", "Applications"), +], ids=["release-in-folder", "nested-folder", "release-at-root"]) +def test_find_folder_id_finds_the_nearest_folder(object_id, expected): + assert Ids.find_folder_id(object_id) == expected diff --git a/tests/release/integration/test_input_context.py b/tests/release/integration/test_input_context.py new file mode 100644 index 0000000..2a76030 --- /dev/null +++ b/tests/release/integration/test_input_context.py @@ -0,0 +1,101 @@ +import pytest + +from digitalai.release.integration.input_context import ( + AutomatedTaskAsUserContext, + InputContext, + PropertyDefinition, + TaskContext, +) +from tests.support import RELEASE_ID, TASK_ID, input_context, task_property + + +def ci_property(name, properties, *, password=False): + """A CI-kind property whose value is a nested CI definition.""" + return task_property(name, {"id": f"Configuration/{name}", "type": "test.Server", "properties": properties}, + kind="CI", password=password) + + +def task_context(*properties): + return TaskContext.from_dict({"id": TASK_ID, "type": "x.Task", "properties": list(properties)}) + + +def test_from_dict_reads_release_task_and_run_as_user(): + context = InputContext.from_dict(input_context(username="deployer", password="pw")) + + assert context.release.id == RELEASE_ID + assert context.release.automated_task_as_user == AutomatedTaskAsUserContext("deployer", "pw") + assert context.task.id == TASK_ID + assert context.task.type == "containerExamples.Hello" + + +def test_run_as_user_defaults_to_empty_when_absent(): + context = InputContext.from_dict({"release": {"id": RELEASE_ID}, "task": {"id": TASK_ID, "type": "x.Y"}}) + + assert context.release.automated_task_as_user == AutomatedTaskAsUserContext(None, None) + assert context.task.properties == [] + + +def test_build_locals_maps_names_to_values(): + task = task_context(task_property("name", "World"), task_property("count", 3, kind="INTEGER")) + + assert task.build_locals() == {"name": "World", "count": 3} + + +def test_build_locals_unwraps_nested_ci_properties_into_dicts(): + task = task_context(ci_property("server", [ + task_property("url", "http://jenkins"), + ci_property("proxy", [task_property("host", "proxy.local")]), + ])) + + assert task.build_locals() == {"server": {"url": "http://jenkins", "proxy": {"host": "proxy.local"}}} + + +def test_build_locals_keeps_an_empty_ci_value_as_is(): + task = task_context(task_property("server", None, kind="CI")) + + assert task.build_locals() == {"server": None} + + +def test_output_properties_lists_only_output_category(): + task = task_context(task_property("in", "a"), task_property("out1", None, category="output"), + task_property("out2", None, category="output")) + + assert task.output_properties() == ["out1", "out2"] + + +def test_secrets_collects_password_values_from_task_and_ci_properties(): + task = task_context( + task_property("token", "t0ken", password=True), + task_property("user", "admin"), + ci_property("server", [task_property("password", "srv-pw", password=True), task_property("url", "http://x")]), + ) + + assert task.secrets() == ["t0ken", "srv-pw"] + + +@pytest.mark.parametrize("value", ["", None]) +def test_secrets_skips_empty_passwords(value): + task = task_context(task_property("token", value, password=True), + ci_property("server", [task_property("password", value, password=True)])) + + assert task.secrets() == [] + + +@pytest.mark.xfail(reason="secret_value() does not recurse into a CI nested in a CI property, " + "so that password is never masked") +def test_secrets_include_passwords_of_nested_ci_properties(): + prop = PropertyDefinition.from_dict(ci_property("server", [ + ci_property("proxy", [task_property("password", "proxy-pw", password=True)]), + ])) + + assert prop.secret_value() == ["proxy-pw"] + + +def test_script_location_returns_the_property_value(): + task = task_context(task_property("scriptLocation", "sample/hello.py")) + + assert task.script_location() == "sample/hello.py" + + +def test_script_location_is_empty_when_not_set(): + assert task_context(task_property("name", "x")).script_location() == "" diff --git a/tests/release/integration/test_job_data_encryptor.py b/tests/release/integration/test_job_data_encryptor.py new file mode 100644 index 0000000..eb52759 --- /dev/null +++ b/tests/release/integration/test_job_data_encryptor.py @@ -0,0 +1,54 @@ +import base64 +import os + +import pytest +from Cryptodome.Cipher import AES + +from digitalai.release.integration.job_data_encryptor import AESJobDataEncryptor, NoOpJobDataEncryptor + + +def new_key() -> str: + """A base64-encoded 256-bit key, as the server provides it.""" + return base64.b64encode(os.urandom(32)).decode() + + +@pytest.fixture +def encryptor(): + return AESJobDataEncryptor(new_key()) + + +@pytest.mark.parametrize("content", ['{"exitCode": 0}', "", "ünïcødé ✓"]) +def test_aes_decrypt_returns_what_encrypt_was_given(encryptor, content): + assert encryptor.decrypt(encryptor.encrypt(content)) == content + + +def test_aes_encrypt_uses_a_fresh_nonce_per_call(encryptor): + assert encryptor.encrypt("same") != encryptor.encrypt("same") + + +def test_aes_decrypts_the_nonce_ciphertext_tag_layout_the_server_produces(): + key = os.urandom(32) + nonce = os.urandom(16) + cipher = AES.new(key, AES.MODE_GCM, nonce) + ciphertext, tag = cipher.encrypt_and_digest(b'{"task": {}}') + payload = base64.b64encode(nonce + ciphertext + tag).decode() + + assert AESJobDataEncryptor(base64.b64encode(key).decode()).decrypt(payload) == '{"task": {}}' + + +def test_aes_decrypt_rejects_tampered_content(encryptor): + raw = bytearray(base64.b64decode(encryptor.encrypt("payload"))) + raw[20] ^= 0xFF + + with pytest.raises(ValueError): + encryptor.decrypt(base64.b64encode(bytes(raw)).decode()) + + +def test_aes_decrypt_rejects_content_encrypted_with_another_key(encryptor): + with pytest.raises(ValueError): + AESJobDataEncryptor(new_key()).decrypt(encryptor.encrypt("payload")) + + +@pytest.mark.parametrize("method", ["encrypt", "decrypt"]) +def test_noop_returns_content_unchanged(method): + assert getattr(NoOpJobDataEncryptor(), method)('{"a": 1}') == '{"a": 1}' diff --git a/tests/release/integration/test_k8s.py b/tests/release/integration/test_k8s.py new file mode 100644 index 0000000..1cb45b1 --- /dev/null +++ b/tests/release/integration/test_k8s.py @@ -0,0 +1,90 @@ +from types import SimpleNamespace +from unittest import mock + +import pytest +from kubernetes.config.config_exception import ConfigException + +from digitalai.release.integration import k8s + + +def test_split_secret_resource_data_returns_namespace_name_and_key(): + assert k8s.split_secret_resource_data("runner-ns:result-secret:result") == ("runner-ns", "result-secret", "result") + + +@pytest.mark.parametrize("entry", ["", None]) +def test_split_secret_resource_data_rejects_an_empty_reference(entry): + with pytest.raises(ValueError, match="empty"): + k8s.split_secret_resource_data(entry) + + +@pytest.mark.parametrize("entry", ["name", "ns:name", "ns:name:key:extra"]) +def test_split_secret_resource_data_rejects_a_malformed_reference(entry): + with pytest.raises(ValueError, match="namespace:name:key"): + k8s.split_secret_resource_data(entry) + + +@pytest.fixture +def kube(monkeypatch): + """Stands in for the kubernetes config loaders and client classes, with no cached client.""" + fake = SimpleNamespace(config=mock.Mock(), client=mock.Mock()) + fake.client.Configuration.get_default_copy.return_value = SimpleNamespace(verify_ssl=True) + monkeypatch.setattr(k8s, "config", fake.config) + monkeypatch.setattr(k8s, "client", fake.client) + monkeypatch.setattr(k8s, "kubernetes_client", None) + monkeypatch.delenv("SKIP_TLS_LEGACY_K8S", raising=False) + return fake + + +def test_get_client_uses_the_in_cluster_config(kube): + client = k8s.get_client() + + kube.config.load_incluster_config.assert_called_once_with() + kube.config.load_config.assert_not_called() + assert client is kube.client.CoreV1Api.return_value + + +def test_get_client_falls_back_to_the_default_config_outside_a_cluster(kube): + kube.config.load_incluster_config.side_effect = ConfigException("not in a pod") + + k8s.get_client() + + kube.config.load_config.assert_called_once_with() + + +def test_get_client_raises_when_no_config_can_be_loaded(kube): + kube.config.load_incluster_config.side_effect = ConfigException("not in a pod") + kube.config.load_config.side_effect = ConfigException("no kubeconfig") + + with pytest.raises(RuntimeError, match="Could not configure Kubernetes client"): + k8s.get_client() + + +def test_get_client_creates_the_client_once(kube): + first = k8s.get_client() + second = k8s.get_client() + + assert first is second + kube.config.load_incluster_config.assert_called_once() + kube.client.CoreV1Api.assert_called_once() + + +@pytest.mark.parametrize("value", ["true", "TRUE"]) +def test_get_client_disables_tls_verification_for_legacy_clusters(kube, monkeypatch, value): + monkeypatch.setenv("SKIP_TLS_LEGACY_K8S", value) + + k8s.get_client() + + (configuration,), _ = kube.client.ApiClient.call_args + assert configuration.verify_ssl is False + kube.client.CoreV1Api.assert_called_once_with(kube.client.ApiClient.return_value) + + +@pytest.mark.parametrize("value", [None, "false", "yes"]) +def test_get_client_keeps_tls_verification_otherwise(kube, monkeypatch, value): + if value is not None: + monkeypatch.setenv("SKIP_TLS_LEGACY_K8S", value) + + k8s.get_client() + + kube.client.ApiClient.assert_not_called() + kube.client.CoreV1Api.assert_called_once_with() diff --git a/tests/release/integration/test_masked_io.py b/tests/release/integration/test_masked_io.py new file mode 100644 index 0000000..02f8f58 --- /dev/null +++ b/tests/release/integration/test_masked_io.py @@ -0,0 +1,73 @@ +import io + +import pytest + +from digitalai.release.integration.masked_io import MaskedIO + + +@pytest.fixture +def sink(): + return io.StringIO() + + +@pytest.fixture +def stream(sink): + stream = MaskedIO(sink) + yield stream + # A TextIOWrapper closes the stream it wraps when collected; detach so the sink stays readable. + stream.detach() + + +def test_masks_every_occurrence_of_every_secret(stream, sink): + stream.secrets = ["pw1", "tok"] + + stream.write("pw1 and tok and pw1 again") + + assert sink.getvalue() == "******** and ******** and ******** again" + + +def test_write_returns_the_length_of_the_unmasked_text(stream): + stream.secrets = ["a-long-secret"] + + assert stream.write("x a-long-secret") == len("x a-long-secret") + + +def test_passes_text_through_without_secrets(stream, sink): + stream.write("plain text") + + assert sink.getvalue() == "plain text" + + +def test_ignores_empty_secrets(stream, sink): + stream.secrets = ["", None] + + stream.write("plain text") + + assert sink.getvalue() == "plain text" + + +def test_masks_non_string_secrets_by_their_text(stream, sink): + stream.secrets = [12345] + + stream.write("pin 12345") + + assert sink.getvalue() == "pin ********" + + +def test_masks_with_the_secrets_set_most_recently(stream, sink): + stream.secrets = ["old"] + stream.secrets = ["new"] + + stream.write("old new") + + assert sink.getvalue() == "old ********" + + +@pytest.mark.xfail(reason="secrets are replaced in list order, so a secret that is a prefix of a " + "later one masks only part of it and leaks the rest") +def test_masks_a_secret_that_contains_another_secret(stream, sink): + stream.secrets = ["admin", "admin123"] + + stream.write("admin123") + + assert sink.getvalue() == "********" diff --git a/tests/release/integration/test_reporting_records.py b/tests/release/integration/test_reporting_records.py new file mode 100644 index 0000000..2b90c09 --- /dev/null +++ b/tests/release/integration/test_reporting_records.py @@ -0,0 +1,80 @@ +""" +The serialized field names are the wire format the Release server reads, so +they are pinned here exactly, including the few that are not camelCase. +""" +import json +from datetime import datetime, timezone + +import pytest + +from digitalai.release.integration import ( + BuildRecord, + CodeComplianceRecord, + DeploymentRecord, + ItsmRecord, + OutputContext, + PlanRecord, +) + +WHEN = datetime(2026, 3, 4, 5, 6, 7, tzinfo=timezone.utc) +WHEN_ISO = "2026-03-04T05:06:07+00:00" + +RECORDS = { + "build": ( + BuildRecord(target_id="T1", build="#42", build_url="http://ci/42", project="app", outcome="SUCCESS", + start_date=WHEN, end_date=WHEN, duration="PT1M"), + {"targetId": "T1", "build": "#42", "build_url": "http://ci/42", "project": "app", "outcome": "SUCCESS", + "startDate": WHEN_ISO, "endDate": WHEN_ISO, "duration": "PT1M", "type": "udm.BuildRecord", "id": "", + "serverUrl": None, "serverUser": None}, + ), + "plan": ( + PlanRecord(target_id="T1", ticket="PRJ-1", ticket_url="http://jira/PRJ-1", title="Fix", ticket_type="Bug", + status="Done", updated_date=WHEN, updated_by="dev"), + {"targetId": "T1", "ticket": "PRJ-1", "ticket_url": "http://jira/PRJ-1", "title": "Fix", + "ticketType": "Bug", "status": "Done", "updatedDate": WHEN_ISO, "type": "udm.PlanRecord", "id": "", + "updatedBy": "dev", "serverUrl": None, "serverUser": None}, + ), + "itsm": ( + ItsmRecord(target_id="T1", record="CHG1", record_url="http://snow/CHG1", title="Change", status="New", + priority="P2", created_by="ops"), + {"targetId": "T1", "record": "CHG1", "record_url": "http://snow/CHG1", "title": "Change", "status": "New", + "priority": "P2", "createdBy": "ops", "type": "udm.ItsmRecord", "id": "", "serverUrl": None, + "serverUser": None}, + ), + "code-compliance": ( + CodeComplianceRecord(target_id="T1", project="app", project_url="http://sonar/app", analysis_date=WHEN, + outcome="PASSED", compliance_data="A"), + {"targetId": "T1", "project": "app", "project_url": "http://sonar/app", "analysisDate": WHEN_ISO, + "outcome": "PASSED", "complianceData": "A", "type": "udm.CodeComplianceRecord", "id": "", + "serverUrl": None, "serverUser": None}, + ), + "deployment": ( + DeploymentRecord(target_id="T1", deployment_task="Deploy", deployment_task_url="http://cd/1", + status="DONE", version="1.0", application_name="app", environment_name="prod"), + {"targetId": "T1", "deploymentTask": "Deploy", "deploymentTask_url": "http://cd/1", "status": "DONE", + "version": "1.0", "type": "udm.DeploymentRecord", "id": "", "applicationName": "app", + "environmentName": "prod", "serverUrl": None, "serverUser": None}, + ), +} + + +@pytest.mark.parametrize("record, expected", RECORDS.values(), ids=list(RECORDS)) +def test_record_serializes_to_the_server_wire_format(record, expected): + assert record.to_dict() == expected + + +@pytest.mark.parametrize("record, _", RECORDS.values(), ids=list(RECORDS)) +def test_record_round_trips_through_its_dict_form(record, _): + assert type(record).from_dict(record.to_dict()) == record + + +def test_output_context_serializes_camel_case_with_records_as_json(): + build, expected_build = RECORDS["build"] + context = OutputContext(1, "failed", {"greeting": "Hello"}, [build, {"custom": 1}]) + + assert json.loads(json.dumps(context.to_dict())) == { + "exitCode": 1, + "jobErrorMessage": "failed", + "outputProperties": {"greeting": "Hello"}, + "reportingRecords": [expected_build, {"custom": 1}], + } diff --git a/tests/release/integration/test_watcher.py b/tests/release/integration/test_watcher.py new file mode 100644 index 0000000..67d1d6e --- /dev/null +++ b/tests/release/integration/test_watcher.py @@ -0,0 +1,119 @@ +import threading +from types import SimpleNamespace +from unittest import mock + +import pytest + +from digitalai.release.integration import k8s, watcher + +SECRET = "input-context-secret" +NAMESPACE = "runner-ns" + + +def secret_event(session_key): + data = None if session_key is None else {"session-key": session_key} + return {"object": SimpleNamespace(data=data)} + + +class FakeWatch: + """Replays a fixed list of watch events.""" + + def __init__(self, events): + self.events = events + self.consumed = 0 + self.stream_call = None + self.stopped = False + + def stream(self, func, *args, **kwargs): + self.stream_call = (func, args, kwargs) + for event in self.events: + self.consumed += 1 + yield event + + def stop(self): + self.stopped = True + + +@pytest.fixture +def kube_client(monkeypatch): + client = mock.Mock() + monkeypatch.setattr(k8s, "get_client", lambda: client) + monkeypatch.setenv("INPUT_CONTEXT_SECRET", SECRET) + monkeypatch.setenv("RUNNER_NAMESPACE", NAMESPACE) + return client + + +def watch_secret(monkeypatch, events, on_update=None, stop=None): + fake = FakeWatch(events) + monkeypatch.setattr(watcher, "watch", SimpleNamespace(Watch=lambda: fake)) + watcher.start_input_secret_watcher(on_update or mock.Mock(), stop or threading.Event()) + return fake + + +def test_watches_only_the_input_context_secret(kube_client, monkeypatch): + fake = watch_secret(monkeypatch, []) + + assert fake.stream_call == (kube_client.list_namespaced_secret, (NAMESPACE,), + {"field_selector": f"metadata.name={SECRET}"}) + + +def test_calls_back_each_time_the_session_key_changes(kube_client, monkeypatch): + on_update = mock.Mock() + + watch_secret(monkeypatch, [secret_event(k) for k in ("k1", "k1", "k2", "k2", "k3")], on_update) + + assert on_update.call_count == 2 + + +def test_first_event_only_records_the_session_key(kube_client, monkeypatch): + on_update = mock.Mock() + + watch_secret(monkeypatch, [secret_event("k1")], on_update) + + on_update.assert_not_called() + + +def test_tolerates_a_secret_without_data(kube_client, monkeypatch): + on_update = mock.Mock() + + watch_secret(monkeypatch, [secret_event(None), secret_event("k1"), secret_event("k2")], on_update) + + on_update.assert_called_once_with() + + +def test_stops_after_the_current_event_once_stop_is_set(kube_client, monkeypatch): + stop = threading.Event() + stop.set() + + fake = watch_secret(monkeypatch, [secret_event("k1"), secret_event("k2")], stop=stop) + + assert fake.consumed == 1 + assert fake.stopped + + +def test_releases_the_watch_when_the_callback_fails(kube_client, monkeypatch): + fake = FakeWatch([secret_event("k1"), secret_event("k2")]) + monkeypatch.setattr(watcher, "watch", SimpleNamespace(Watch=lambda: fake)) + + with pytest.raises(RuntimeError): + watcher.start_input_secret_watcher(mock.Mock(side_effect=RuntimeError("boom")), threading.Event()) + + assert fake.stopped + + +@pytest.mark.parametrize("missing", ["INPUT_CONTEXT_SECRET", "RUNNER_NAMESPACE"]) +def test_requires_secret_name_and_namespace(kube_client, monkeypatch, missing): + monkeypatch.delenv(missing) + + with pytest.raises(ValueError, match="INPUT_CONTEXT_SECRET and RUNNER_NAMESPACE"): + watcher.start_input_secret_watcher(mock.Mock(), threading.Event()) + + +def test_input_context_watcher_returns_when_the_secret_watcher_fails(monkeypatch): + monkeypatch.setattr(watcher, "start_input_secret_watcher", mock.Mock(side_effect=ValueError("no secret"))) + thread = threading.Thread(target=watcher.start_input_context_watcher, args=(mock.Mock(),), daemon=True) + + thread.start() + thread.join(timeout=5) + + assert not thread.is_alive() diff --git a/tests/release/integration/test_wrapper.py b/tests/release/integration/test_wrapper.py deleted file mode 100644 index 2e224f3..0000000 --- a/tests/release/integration/test_wrapper.py +++ /dev/null @@ -1,104 +0,0 @@ -import json -import os -import subprocess -import sys -import unittest - -# Directory that holds the test fixtures (input.json, hello.py, src/...). -THIS_DIR = os.path.dirname(os.path.abspath(__file__)) - -# Sample input context for a "Hello" container task (matches input.json). -SAMPLE_INPUT_CONTEXT = { - "release": { - "id": "Applications/Folder1f65c7220b394afbb941154342fd9fc6/Release31de09e95c8e4ebb95aaed29a8082d0b", - "automatedTaskAsUser": {"username": "admin", "password": "admin"}, - }, - "task": { - "id": "Applications/Folder1f65c7220b394afbb941154342fd9fc6/Release31de09e95c8e4ebb95aaed29a8082d0b/Phase723a601c78804f7dbcaa8b05b83708f5/Task3a35b67b42b6428b854857fba470b39a", - "type": "containerExamples.Hello", - "properties": [ - {"name": "capabilities", "value": ["remote"], "kind": "SET_OF_STRING", "category": "input", "password": False}, - {"name": "yourName", "value": "World", "kind": "STRING", "category": "input", "password": False}, - {"name": "greeting", "value": None, "kind": "STRING", "category": "output", "password": False}, - ], - }, -} - -# Same task, but with an explicit scriptLocation so the wrapper loads the class via -# the `if script_path:` branch (importlib) instead of walking the tree. Points at the -# src/sample/hello.py fixture, which defines the Hello1 class. -SAMPLE_INPUT_CONTEXT_WITH_SCRIPT = { - "release": SAMPLE_INPUT_CONTEXT["release"], - "task": { - "id": SAMPLE_INPUT_CONTEXT["task"]["id"], - "type": "containerExamples.Hello1", - "properties": SAMPLE_INPUT_CONTEXT["task"]["properties"] + [ - {"name": "scriptLocation", "value": "sample/hello.py", "kind": "STRING", "category": "input", "password": False}, - ], - }, -} - -EXPECTED_OUTPUT = { - "exitCode": 0, - "jobErrorMessage": "", - "outputProperties": {"greeting": "Hello World"}, - "reportingRecords": [], -} - - -class TestWrapper(unittest.TestCase): - """End-to-end test that runs the wrapper against a sample input context.""" - - def setUp(self): - self.input_path = os.path.join(THIS_DIR, "input.json") - self.output_path = os.path.join(THIS_DIR, "output.json") - # Start from a clean slate so a stale file can never mask a failure. - if os.path.exists(self.output_path): - os.remove(self.output_path) - - def _run_wrapper(self, input_context): - """Write the given input context, run the wrapper as a subprocess, and return the parsed output.""" - with open(self.input_path, "w") as f: - json.dump(input_context, f) - - env = dict(os.environ) - env["INPUT_LOCATION"] = "input.json" - env["OUTPUT_LOCATION"] = "output.json" - env["RELEASE_URL"] = "http://localhost:5516" - - result = subprocess.run( - [sys.executable, "-m", "digitalai.release.integration.wrapper"], - cwd=THIS_DIR, - env=env, - capture_output=True, - text=True, - ) - - self.assertEqual( - result.returncode, 0, - msg=f"wrapper exited with {result.returncode}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}", - ) - self.assertTrue(os.path.exists(self.output_path), "wrapper did not produce output.json") - - with open(self.output_path, "r") as json_file: - return json.load(json_file) - - def test_wrapper(self): - """ - Runs the wrapper with a sample input context that has no scriptLocation, so the - task class is resolved via the find_class_file fallback (the `else` branch of run()). - """ - actual_output = self._run_wrapper(SAMPLE_INPUT_CONTEXT) - self.assertEqual(EXPECTED_OUTPUT, actual_output) - - def test_wrapper_with_script_location(self): - """ - Runs the wrapper with a scriptLocation set, so the task class is loaded via - importlib (the `if script_path:` branch of run()). - """ - actual_output = self._run_wrapper(SAMPLE_INPUT_CONTEXT_WITH_SCRIPT) - self.assertEqual(EXPECTED_OUTPUT, actual_output) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/release/integration/test_wrapper_abort.py b/tests/release/integration/test_wrapper_abort.py index 487fcad..6d0631d 100644 --- a/tests/release/integration/test_wrapper_abort.py +++ b/tests/release/integration/test_wrapper_abort.py @@ -1,582 +1,247 @@ """ -Concurrency-aware tests for the SIGABRT worker-thread abort model -(docs/SPEC-abort-worker-thread.md). +The SIGTERM/SIGABRT handler, called directly on the test's main thread. -Most tests here exercise wrapper.py's internals directly, in-process, with -os._exit() and update_output_context() mocked out so a simulated abort does -not tear down the test runner itself or touch real files/secrets/callbacks. - -One test (`test_real_sigabrt_interrupts_blocking_task`) sends a genuine -SIGABRT to a subprocess running the real wrapper end-to-end, mirroring how the -release-runner aborts a task, to prove the full integration works. +update_output_context() and os._exit() are replaced (see the ``reported`` and +``exit_codes`` fixtures), so a simulated abort neither sends a result nor ends +the test run. Real signals are covered by test_wrapper_e2e.py. """ -import json -import os import signal -import subprocess -import sys import threading import time -import unittest -from unittest import mock -import digitalai.release.integration.wrapper as wrapper +import pytest + from digitalai.release.integration.base_task import BaseTask from digitalai.release.integration.output_context import OutputContext +from tests.support import NoOpTask -THIS_DIR = os.path.dirname(os.path.abspath(__file__)) - +ABORTED = OutputContext(1, "Abort requested", {}, []) -class _RecordingAbortTask(BaseTask): - """Stub task that counts abort() calls and lets tests set a canned output_context.""" +class CountingAbortTask(BaseTask): def __init__(self): super().__init__() - self.abort_call_count = 0 + self.abort_calls = 0 - def execute(self) -> None: # pragma: no cover - not used directly + def execute(self) -> None: pass def abort(self) -> None: - self.abort_call_count += 1 + self.abort_calls += 1 super().abort() -class WrapperAbortHandlerTests(unittest.TestCase): - """In-process tests for wrapper.abort_handler() and the single-writer flush guard.""" - - def setUp(self): - wrapper._reset_report_guard() - wrapper.dai_task_object = None - - def tearDown(self): - wrapper._reset_report_guard() - wrapper.dai_task_object = None - - def test_abort_handler_calls_abort_flushes_once_and_exits(self): - task = _RecordingAbortTask() - task.output_context = OutputContext(0, "", {}, []) - wrapper.dai_task_object = task - - with mock.patch.object(wrapper, "update_output_context") as mock_update, \ - mock.patch.object(wrapper.os, "_exit") as mock_exit: - wrapper.abort_handler(signal.SIGABRT, None) - - self.assertEqual(task.abort_call_count, 1) - mock_update.assert_called_once() - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 1) - self.assertEqual(reported_ctx.job_error_message, "Abort requested") - mock_exit.assert_called_once_with(1) - - def test_abort_handler_does_not_double_flush(self): - task = _RecordingAbortTask() - task.output_context = OutputContext(0, "", {}, []) - wrapper.dai_task_object = task - - with mock.patch.object(wrapper, "update_output_context") as mock_update, \ - mock.patch.object(wrapper.os, "_exit"): - # Simulate the worker's own normal-completion path reporting first. - self.assertTrue(wrapper._report_once()) - # Then the abort handler fires; abort() cleanup still runs, but the - # flush must not happen a second time. - wrapper.abort_handler(signal.SIGABRT, None) - - self.assertEqual(task.abort_call_count, 1) - mock_update.assert_called_once() - - def test_abort_handler_with_no_task_object_exits_cleanly(self): - wrapper.dai_task_object = None - - with mock.patch.object(wrapper, "update_output_context") as mock_update, \ - mock.patch.object(wrapper.os, "_exit") as mock_exit: - wrapper.abort_handler(signal.SIGTERM, None) - - mock_update.assert_called_once() - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 1) - mock_exit.assert_called_once_with(1) - - def test_abort_handler_before_worker_starts_still_reports_abort_requested(self): - # Simulates SIGABRT arriving right after dai_task_object is assigned, - # before the worker thread has had a chance to run execute_task()'s - # body -- i.e. before output_context was ever touched post-construction. - task = _RecordingAbortTask() - wrapper.dai_task_object = task - - with mock.patch.object(wrapper, "update_output_context") as mock_update, \ - mock.patch.object(wrapper.os, "_exit") as mock_exit: - wrapper.abort_handler(signal.SIGABRT, None) - - self.assertEqual(task.abort_call_count, 1) - mock_update.assert_called_once() - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 1) - self.assertEqual(reported_ctx.job_error_message, "Abort requested") - mock_exit.assert_called_once_with(1) - - def test_abort_handler_logs_actual_signal_name(self): - task = _RecordingAbortTask() - task.output_context = OutputContext(0, "", {}, []) - wrapper.dai_task_object = task - - with mock.patch.object(wrapper, "update_output_context"), \ - mock.patch.object(wrapper.os, "_exit"), \ - mock.patch.object(wrapper.dai_logger, "info") as mock_log_info: - wrapper.abort_handler(signal.SIGABRT, None) - - logged_messages = [call.args[0] for call in mock_log_info.call_args_list] - self.assertTrue(any("SIGABRT" in msg for msg in logged_messages)) - - -class _CooperativeTask(BaseTask): - """Task whose execute() polls is_abort_requested() and stops promptly, reporting - a distinct, custom result so tests can tell it stopped cooperatively rather - than being torn down by the abort handler.""" +def start_worker(wrapper, task): + """Runs task.execute_task() on a worker thread registered with the wrapper, as execute_task() does.""" + wrapper.dai_task_object = task + worker = threading.Thread(target=task.execute_task, daemon=True) + wrapper._worker = worker + worker.start() + return worker - def execute(self) -> None: - while not self.is_abort_requested(): - time.sleep(0.02) - self.set_output_property("stopped_cooperatively", True) - self.set_error_message("Stopped cooperatively") +@pytest.mark.parametrize("signum", [signal.SIGABRT, signal.SIGTERM], ids=["SIGABRT", "SIGTERM"]) +def test_abort_calls_abort_reports_once_and_exits_1(isolated_wrapper, reported, exit_codes, signum): + task = CountingAbortTask() + isolated_wrapper.dai_task_object = task -class WrapperExecuteTaskThreadingTests(unittest.TestCase): - """In-process tests for wrapper.execute_task()'s worker-thread + guard behavior.""" - - def setUp(self): - wrapper._reset_report_guard() - wrapper.dai_task_object = None - - def tearDown(self): - wrapper._reset_report_guard() - wrapper.dai_task_object = None - - def test_execute_task_normal_completion_reports_once(self): - class _FastTask(BaseTask): - def execute(self) -> None: - self.set_output_property("greeting", "Hello World") - - task = _FastTask() - - with mock.patch.object(wrapper, "update_output_context") as mock_update: - wrapper.execute_task(task) - - mock_update.assert_called_once() - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 0) - self.assertEqual(reported_ctx.output_properties, {"greeting": "Hello World"}) - - def test_cooperative_task_stops_promptly_when_abort_requested(self): - task = _CooperativeTask() - - def _request_abort_shortly(): - time.sleep(0.1) - task._request_abort() - - threading.Thread(target=_request_abort_shortly, daemon=True).start() - - with mock.patch.object(wrapper, "update_output_context") as mock_update: - start = time.monotonic() - wrapper.execute_task(task) - elapsed = time.monotonic() - start - - # The worker should notice is_abort_requested() and return well within - # a couple of poll intervals, without needing a real signal. - self.assertLess(elapsed, 2.0) - mock_update.assert_called_once() - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.job_error_message, "Stopped cooperatively") - self.assertTrue(reported_ctx.output_properties.get("stopped_cooperatively")) + isolated_wrapper.abort_handler(signum, None) - def test_execute_task_resets_guard_across_daemon_iterations(self): - class _FastTask(BaseTask): - def execute(self) -> None: - pass + assert task.abort_calls == 1 + assert reported == [ABORTED] + assert exit_codes == [1] - task1 = _FastTask() - task2 = _FastTask() - with mock.patch.object(wrapper, "update_output_context") as mock_update: - wrapper.execute_task(task1) - wrapper.execute_task(task2) +def test_abort_requests_a_cooperative_stop(isolated_wrapper, reported, exit_codes): + task = NoOpTask() + isolated_wrapper.dai_task_object = task - # Each iteration must flush its own task's result; the guard from the - # first iteration must not suppress the second. - self.assertEqual(mock_update.call_count, 2) - self.assertIs(wrapper.dai_task_object, task2) - - def test_abort_exception_propagates_exit_status_after_reporting(self): - from digitalai.release.integration.exceptions import AbortException - - class _AbortingTask(BaseTask): - def execute(self) -> None: - raise AbortException() - - task = _AbortingTask() - - with mock.patch.object(wrapper, "update_output_context") as mock_update: - with self.assertRaises(SystemExit) as ctx: - wrapper.execute_task(task) - - # BaseTask.execute_task() sets exit_code=1/"Abort requested" then - # sys.exit(1); the status must be propagated on the main thread. - self.assertEqual(ctx.exception.code, 1) - mock_update.assert_called_once() - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 1) - self.assertEqual(reported_ctx.job_error_message, "Abort requested") - - def test_direct_sys_exit_propagates_status_and_reports_its_code(self): - class _ExitingTask(BaseTask): - def execute(self) -> None: - sys.exit(2) - - task = _ExitingTask() - - with mock.patch.object(wrapper, "update_output_context") as mock_update: - with self.assertRaises(SystemExit) as ctx: - wrapper.execute_task(task) - - # The exact SystemExit code must be preserved. - self.assertEqual(ctx.exception.code, 2) - # ...and reported, so Release does not see a success. - mock_update.assert_called_once() - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 2) - self.assertEqual(reported_ctx.job_error_message, "Task exited with code 2") - - def test_sys_exit_with_message_reports_exit_code_1_and_the_message(self): - class _ExitingTask(BaseTask): - def execute(self) -> None: - sys.exit("Configuration missing") - - with mock.patch.object(wrapper, "update_output_context") as mock_update: - with self.assertRaises(SystemExit): - wrapper.execute_task(_ExitingTask()) - - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 1) - self.assertEqual(reported_ctx.job_error_message, "Configuration missing") - - def test_sys_exit_zero_reports_success(self): - class _ExitingTask(BaseTask): - def execute(self) -> None: - sys.exit(0) - - with mock.patch.object(wrapper, "update_output_context") as mock_update: - with self.assertRaises(SystemExit): - wrapper.execute_task(_ExitingTask()) - - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 0) - - def test_normal_completion_does_not_raise_system_exit(self): - class _FastTask(BaseTask): - def execute(self) -> None: - self.set_output_property("ok", True) - - task = _FastTask() - - with mock.patch.object(wrapper, "update_output_context") as mock_update: - # Must return normally, not raise SystemExit. - wrapper.execute_task(task) - - mock_update.assert_called_once() - - -class WrapperAbortGracePeriodTests(unittest.TestCase): - """ - abort_handler() waits for the worker for up to ABORT_GRACE_PERIOD_SECONDS - before reporting, guards an in-progress report, and ignores repeat signals. - """ - - def setUp(self): - wrapper._reset_report_guard() - wrapper.dai_task_object = None - - def tearDown(self): - wrapper._reset_report_guard() - wrapper.dai_task_object = None - - def _start_worker(self, task): - """Starts task.execute_task() on a worker thread, as execute_task() does.""" - wrapper.dai_task_object = task - worker = threading.Thread(target=task.execute_task, daemon=True) - wrapper._worker = worker - worker.start() - return worker - - def test_cooperative_stop_within_grace_period_is_reported(self): - class _PollingTask(BaseTask): - def execute(self) -> None: - while not self.is_abort_requested(): - time.sleep(0.01) - self.set_output_property("processed", 3) - self.add_reporting_record({"status": "ABORTED"}) - - task = _PollingTask() - worker = self._start_worker(task) - - with mock.patch.object(wrapper, "update_output_context") as mock_update, \ - mock.patch.object(wrapper.os, "_exit") as mock_exit: - wrapper.abort_handler(signal.SIGABRT, None) - - self.assertFalse(worker.is_alive()) - mock_update.assert_called_once() - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 1) - self.assertEqual(reported_ctx.job_error_message, "Abort requested") - self.assertEqual(reported_ctx.output_properties, {"processed": 3}) - self.assertEqual(reported_ctx.reporting_records, [{"status": "ABORTED"}]) - mock_exit.assert_called_once_with(1) - - def test_task_that_does_not_stop_is_reported_after_grace_period(self): - release = threading.Event() - - class _StuckTask(BaseTask): - def execute(self) -> None: - release.wait(10) - - task = _StuckTask() - self._start_worker(task) - - try: - with mock.patch.object(wrapper, "ABORT_GRACE_PERIOD_SECONDS", 0.2), \ - mock.patch.object(wrapper, "update_output_context") as mock_update, \ - mock.patch.object(wrapper.os, "_exit") as mock_exit: - start = time.monotonic() - wrapper.abort_handler(signal.SIGABRT, None) - elapsed = time.monotonic() - start - finally: - release.set() - - self.assertGreaterEqual(elapsed, 0.2) - self.assertLess(elapsed, 2.0) - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 1) - mock_exit.assert_called_once_with(1) - - def test_reported_result_is_a_copy_of_the_output_context(self): - task = _RecordingAbortTask() - task.set_output_property("a", 1) - wrapper.dai_task_object = task - - with mock.patch.object(wrapper, "update_output_context") as mock_update, \ - mock.patch.object(wrapper.os, "_exit"): - wrapper.abort_handler(signal.SIGABRT, None) - - (reported_ctx,), _ = mock_update.call_args - self.assertIsNot(reported_ctx, task.get_output_context()) - self.assertIsNot(reported_ctx.output_properties, task.get_output_properties()) - self.assertEqual(reported_ctx.output_properties, {"a": 1}) - - def test_signal_during_report_lets_the_report_finish_then_exits(self): - class _FastTask(BaseTask): - def execute(self) -> None: - self.set_output_property("greeting", "Hello") - - pushed = [] - - def _push_interrupted_by_signal(context): - # The signal handler runs on the main thread in the middle of the push. - wrapper.abort_handler(signal.SIGABRT, None) - pushed.append(context) - - with mock.patch.object(wrapper, "update_output_context", side_effect=_push_interrupted_by_signal), \ - mock.patch.object(wrapper.os, "_exit") as mock_exit: - wrapper.execute_task(_FastTask()) - - # The push completed with the task's own result, and only then did the process exit. - self.assertEqual(len(pushed), 1) - self.assertEqual(pushed[0].exit_code, 0) - self.assertEqual(pushed[0].output_properties, {"greeting": "Hello"}) - mock_exit.assert_called_once_with(1) - - def test_second_signal_during_abort_is_ignored(self): - class _ReentrantAbortTask(BaseTask): - def execute(self) -> None: # pragma: no cover - not used directly - pass - - def abort(self) -> None: - # A second signal arrives while the first is being handled. - wrapper.abort_handler(signal.SIGTERM, None) - super().abort() - - wrapper.dai_task_object = _ReentrantAbortTask() - - with mock.patch.object(wrapper, "update_output_context") as mock_update, \ - mock.patch.object(wrapper.os, "_exit") as mock_exit: - wrapper.abort_handler(signal.SIGABRT, None) - - mock_update.assert_called_once() - mock_exit.assert_called_once_with(1) - - def test_aborted_result_never_reports_exit_code_0(self): - class _QuietAbortTask(BaseTask): - def execute(self) -> None: # pragma: no cover - not used directly - pass - - def abort(self) -> None: - # Overrides abort() without calling super() or setting an exit code. - self.set_output_property("cleaned_up", True) - - wrapper.dai_task_object = _QuietAbortTask() - - with mock.patch.object(wrapper, "update_output_context") as mock_update, \ - mock.patch.object(wrapper.os, "_exit"): - wrapper.abort_handler(signal.SIGABRT, None) - - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 1) - self.assertEqual(reported_ctx.job_error_message, "Abort requested") - self.assertEqual(reported_ctx.output_properties, {"cleaned_up": True}) - - def test_signal_before_task_exists_reports_a_clear_message(self): - with mock.patch.object(wrapper, "update_output_context") as mock_update, \ - mock.patch.object(wrapper.os, "_exit"): - wrapper.abort_handler(signal.SIGTERM, None) - - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 1) - self.assertEqual(reported_ctx.job_error_message, "Abort requested before the task started") - - -class _NoSuperInitTask(BaseTask): - """Task whose __init__ does not call super().__init__().""" + isolated_wrapper.abort_handler(signal.SIGABRT, None) - def __init__(self): - self.calls = 0 + assert task.is_abort_requested() - def execute(self) -> None: - self.set_output_property("ok", True) - - -class WrapperTaskWithoutSuperInitTests(unittest.TestCase): - - def setUp(self): - wrapper._reset_report_guard() - wrapper.dai_task_object = None - - def tearDown(self): - wrapper._reset_report_guard() - wrapper.dai_task_object = None - - def test_runs_and_reports_outputs(self): - with mock.patch.object(wrapper, "update_output_context") as mock_update: - wrapper.execute_task(_NoSuperInitTask()) - - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 0) - self.assertEqual(reported_ctx.output_properties, {"ok": True}) - - def test_can_be_aborted(self): - task = _NoSuperInitTask() - self.assertFalse(task.is_abort_requested()) - wrapper.dai_task_object = task - - with mock.patch.object(wrapper, "update_output_context") as mock_update, \ - mock.patch.object(wrapper.os, "_exit"): - wrapper.abort_handler(signal.SIGABRT, None) - - self.assertTrue(task.is_abort_requested()) - (reported_ctx,), _ = mock_update.call_args - self.assertEqual(reported_ctx.exit_code, 1) - - -@unittest.skipIf(sys.platform == "win32", "Windows cannot deliver SIGABRT to a child process") -class WrapperRealSigabrtEndToEndTest(unittest.TestCase): - """ - Full end-to-end tests: run the real wrapper module as a subprocess, send it - a genuine SIGABRT partway through (as the release-runner does), and check - the reported result. - """ - - def setUp(self): - self.input_path = os.path.join(THIS_DIR, "abort_input.json") - self.output_path = os.path.join(THIS_DIR, "abort_output.json") - if os.path.exists(self.output_path): - os.remove(self.output_path) - - def tearDown(self): - for path in (self.input_path, self.output_path): - if os.path.exists(path): - os.remove(path) - - def _run_and_abort(self, task_type: str): - """Runs the wrapper on task_type, sends SIGABRT after 1s, and returns (seconds to exit, output).""" - input_context = { - "release": { - "id": "Applications/Folder1f65c7220b394afbb941154342fd9fc6/Release31de09e95c8e4ebb95aaed29a8082d0b", - "automatedTaskAsUser": {"username": "admin", "password": "admin"}, - }, - "task": { - "id": "Applications/Folder1f65c7220b394afbb941154342fd9fc6/Release31de09e95c8e4ebb95aaed29a8082d0b/Phase723a601c78804f7dbcaa8b05b83708f5/Task3a35b67b42b6428b854857fba470b39a", - "type": task_type, - "properties": [ - {"name": "capabilities", "value": ["remote"], "kind": "SET_OF_STRING", "category": "input", "password": False}, - ], - }, - } - with open(self.input_path, "w") as f: - json.dump(input_context, f) - - env = dict(os.environ) - env["INPUT_LOCATION"] = os.path.basename(self.input_path) - env["OUTPUT_LOCATION"] = os.path.basename(self.output_path) - env["RELEASE_URL"] = "http://localhost:5516" - - proc = subprocess.Popen( - [sys.executable, "-m", "digitalai.release.integration.wrapper"], - cwd=THIS_DIR, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - try: - # Give the task time to start running execute(). - time.sleep(1.0) - start = time.monotonic() - proc.send_signal(signal.SIGABRT) - stdout, stderr = proc.communicate(timeout=10) - elapsed = time.monotonic() - start - except subprocess.TimeoutExpired: - proc.kill() - stdout, stderr = proc.communicate() - self.fail( - f"wrapper did not exit promptly after SIGABRT\nstdout:\n{stdout}\nstderr:\n{stderr}" - ) - - self.assertEqual( - proc.returncode, 1, - msg=f"wrapper exited with {proc.returncode}\nstdout:\n{stdout}\nstderr:\n{stderr}", - ) - self.assertTrue(os.path.exists(self.output_path), "wrapper did not produce an output file") - with open(self.output_path) as f: - return elapsed, json.load(f) - - def test_real_sigabrt_interrupts_blocking_task(self): - elapsed, actual_output = self._run_and_abort("containerExamples.BlockingTask") - - # The task ignores the abort, so the wrapper reports after the grace - # period -- well under the task's 30s sleep and the runner's force-kill. - self.assertGreaterEqual(elapsed, wrapper.ABORT_GRACE_PERIOD_SECONDS) - self.assertLess(elapsed, 10.0) - self.assertEqual(actual_output["exitCode"], 1) - self.assertEqual(actual_output["jobErrorMessage"], "Abort requested") - - def test_real_sigabrt_reports_outputs_of_cooperative_task(self): - elapsed, actual_output = self._run_and_abort("containerExamples.CooperativeTask") - - # The task stops as soon as it sees the abort, well within the grace period. - self.assertLess(elapsed, wrapper.ABORT_GRACE_PERIOD_SECONDS) - self.assertEqual(actual_output["exitCode"], 1) - self.assertEqual(actual_output["jobErrorMessage"], "Abort requested") - self.assertEqual(actual_output["outputProperties"], {"stoppedCooperatively": True}) - self.assertEqual(len(actual_output["reportingRecords"]), 1) - self.assertGreater(actual_output["reportingRecords"][0]["processed"], 0) - - -if __name__ == '__main__': - unittest.main() + +def test_abort_logs_the_received_signal(isolated_wrapper, reported, exit_codes, dai_log): + isolated_wrapper.dai_task_object = NoOpTask() + + isolated_wrapper.abort_handler(signal.SIGTERM, None) + + assert "Received SIGTERM to gracefully stop the process" in dai_log.messages + + +def test_abort_before_a_task_exists_reports_a_clear_message(isolated_wrapper, reported, exit_codes): + isolated_wrapper.abort_handler(signal.SIGTERM, None) + + assert reported == [OutputContext(1, "Abort requested before the task started", {}, [])] + assert exit_codes == [1] + + +def test_abort_after_the_result_was_reported_does_not_report_again(isolated_wrapper, reported, exit_codes): + task = CountingAbortTask() + isolated_wrapper.dai_task_object = task + assert isolated_wrapper._report_once() + + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + assert task.abort_calls == 1 + assert reported == [OutputContext(0, "", {}, [])] + assert exit_codes == [1] + + +def test_cooperative_task_is_reported_with_what_it_recorded_while_stopping(isolated_wrapper, reported, exit_codes): + class PollingTask(BaseTask): + def execute(self) -> None: + deadline = time.monotonic() + 5 + while not self.is_abort_requested() and time.monotonic() < deadline: + time.sleep(0.01) + self.set_output_property("processed", 3) + self.add_reporting_record({"status": "ABORTED"}) + + worker = start_worker(isolated_wrapper, PollingTask()) + + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + assert not worker.is_alive() + assert reported == [OutputContext(1, "Abort requested", {"processed": 3}, [{"status": "ABORTED"}])] + assert exit_codes == [1] + + +def test_task_that_ignores_the_abort_is_reported_after_the_grace_period(isolated_wrapper, reported, exit_codes, + monkeypatch): + monkeypatch.setattr(isolated_wrapper, "ABORT_GRACE_PERIOD_SECONDS", 0.2) + unblock = threading.Event() + + class StuckTask(BaseTask): + def execute(self) -> None: + unblock.wait(10) + + worker = start_worker(isolated_wrapper, StuckTask()) + try: + start = time.monotonic() + isolated_wrapper.abort_handler(signal.SIGABRT, None) + elapsed = time.monotonic() - start + + assert worker.is_alive() + finally: + unblock.set() + + assert 0.2 <= elapsed < 5 + assert reported == [ABORTED] + assert exit_codes == [1] + + +def test_reported_result_is_a_snapshot_of_the_output_context(isolated_wrapper, reported, exit_codes): + task = CountingAbortTask() + task.set_output_property("a", 1) + task.add_reporting_record({"r": 1}) + isolated_wrapper.dai_task_object = task + + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + (snapshot,) = reported + assert snapshot is not task.get_output_context() + assert snapshot.output_properties is not task.get_output_properties() + assert snapshot.reporting_records is not task.get_output_context().reporting_records + assert snapshot == OutputContext(1, "Abort requested", {"a": 1}, [{"r": 1}]) + + +def test_abort_override_that_sets_no_exit_code_is_still_reported_as_failed(isolated_wrapper, reported, exit_codes): + class QuietAbortTask(BaseTask): + def execute(self) -> None: + pass + + def abort(self) -> None: + self.set_output_property("cleanedUp", True) + + isolated_wrapper.dai_task_object = QuietAbortTask() + + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + assert reported == [OutputContext(1, "Abort requested", {"cleanedUp": True}, [])] + + +def test_abort_override_keeps_its_own_error_message(isolated_wrapper, reported, exit_codes): + class CustomAbortTask(BaseTask): + def execute(self) -> None: + pass + + def abort(self) -> None: + self.set_error_message("Cancelled by operator") + + isolated_wrapper.dai_task_object = CustomAbortTask() + + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + assert reported == [OutputContext(1, "Cancelled by operator", {}, [])] + + +@pytest.mark.parametrize("error", [SystemExit(3), RuntimeError("cleanup failed")], ids=["sys-exit", "exception"]) +def test_abort_override_that_raises_still_reports_and_exits_1(isolated_wrapper, reported, exit_codes, error): + class RaisingAbortTask(BaseTask): + def execute(self) -> None: + pass + + def abort(self) -> None: + raise error + + isolated_wrapper.dai_task_object = RaisingAbortTask() + + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + assert reported == [ABORTED] + assert exit_codes == [1] + + +def test_signal_during_a_report_lets_the_report_finish_then_exits(isolated_wrapper, exit_codes, monkeypatch): + pushed = [] + + def push_interrupted_by_a_signal(context): + # Python runs signal handlers on the main thread, possibly mid-push. + isolated_wrapper.abort_handler(signal.SIGABRT, None) + pushed.append(context) + + monkeypatch.setattr(isolated_wrapper, "update_output_context", push_interrupted_by_a_signal) + + isolated_wrapper.execute_task(NoOpTask()) + + assert pushed == [OutputContext(0, "", {}, [])] + assert exit_codes == [1] + + +def test_second_signal_during_an_abort_is_ignored(isolated_wrapper, reported, exit_codes): + class ReentrantAbortTask(BaseTask): + def execute(self) -> None: + pass + + def abort(self) -> None: + isolated_wrapper.abort_handler(signal.SIGTERM, None) + super().abort() + + isolated_wrapper.dai_task_object = ReentrantAbortTask() + + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + assert reported == [ABORTED] + assert exit_codes == [1] + + +def test_task_without_super_init_can_be_aborted(isolated_wrapper, reported, exit_codes): + class NoSuperInitTask(BaseTask): + def __init__(self): + pass + + def execute(self) -> None: + pass + + task = NoSuperInitTask() + isolated_wrapper.dai_task_object = task + + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + assert task.is_abort_requested() + assert reported == [ABORTED] + + +def test_handlers_are_installed_for_sigterm_and_sigabrt(isolated_wrapper): + assert signal.getsignal(signal.SIGTERM) is isolated_wrapper.abort_handler + assert signal.getsignal(signal.SIGABRT) is isolated_wrapper.abort_handler diff --git a/tests/release/integration/test_wrapper_e2e.py b/tests/release/integration/test_wrapper_e2e.py new file mode 100644 index 0000000..922d8e5 --- /dev/null +++ b/tests/release/integration/test_wrapper_e2e.py @@ -0,0 +1,149 @@ +""" +End-to-end runs of ``python -m digitalai.release.integration.wrapper`` in a +subprocess, configured through environment variables as the runner does. +These cover what in-process tests cannot: settings read at import, the +masked standard streams, and real signal delivery. +""" +import base64 +import json +import os +import signal +import subprocess +import sys +import time + +import pytest + +from digitalai.release.integration.job_data_encryptor import AESJobDataEncryptor +from tests.release.integration.conftest import make_task_workspace +from tests.support import input_context, task_property + +pytestmark = pytest.mark.e2e + +posix_only = pytest.mark.skipif(sys.platform == "win32", reason="Windows cannot deliver SIGABRT to a child process") + + +def wrapper_env(**extra): + env = {k: v for k, v in os.environ.items() if k not in ( + "INPUT_LOCATION", "OUTPUT_LOCATION", "SESSION_KEY", "CALLBACK_URL", "RESULT_SECRET_NAME", + "INPUT_CONTEXT_SECRET", "RUNNER_NAMESPACE", "EXECUTOR_EXECUTION_MODE", "ABORT_GRACE_PERIOD_SECONDS")} + env.update(INPUT_LOCATION="input.json", OUTPUT_LOCATION="output.json", RELEASE_URL="http://localhost:5516", + PYTHONUNBUFFERED="1", **extra) + return env + + +def start_wrapper(workspace, context, *, encryptor=None, **env): + payload = json.dumps(context) + (workspace / "input.json").write_text(encryptor.encrypt(payload) if encryptor else payload) + return subprocess.Popen([sys.executable, "-m", "digitalai.release.integration.wrapper"], cwd=workspace, + env=wrapper_env(**env), stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True) + + +def finish(process, timeout=60): + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + pytest.fail(f"wrapper did not exit within {timeout}s\nstdout:\n{stdout}\nstderr:\n{stderr}") + return process.returncode, stdout, stderr + + +def read_output(workspace, encryptor=None): + path = workspace / "output.json" + assert path.exists(), "wrapper did not write output.json" + content = path.read_text() + return json.loads(encryptor.decrypt(content) if encryptor else content) + + +def run_wrapper(workspace, context, **kwargs): + returncode, stdout, stderr = finish(start_wrapper(workspace, context, **kwargs)) + assert returncode == 0, f"wrapper exited with {returncode}\nstdout:\n{stdout}\nstderr:\n{stderr}" + return stdout + + +def test_successful_task_writes_its_result_to_the_output_file(task_workspace): + run_wrapper(task_workspace, input_context("containerExamples.Hello", [task_property("yourName", "World")])) + + assert read_output(task_workspace) == { + "exitCode": 0, "jobErrorMessage": "", "outputProperties": {"greeting": "Hello World"}, "reportingRecords": [], + } + + +def test_session_key_encrypts_both_input_and_output(task_workspace): + key = base64.b64encode(os.urandom(32)).decode() + encryptor = AESJobDataEncryptor(key) + + run_wrapper(task_workspace, input_context("containerExamples.Hello", [task_property("yourName", "World")]), + encryptor=encryptor, SESSION_KEY=key) + + assert read_output(task_workspace, encryptor)["outputProperties"] == {"greeting": "Hello World"} + + +@pytest.fixture(scope="module") +def secret_task_stdout(tmp_path_factory): + """Stdout of SecretTask, which writes its secrets through print, the logger and a comment.""" + workspace = make_task_workspace(tmp_path_factory.mktemp("secret-task")) + context = input_context("containerExamples.SecretTask", [task_property("token", "t0k3n-value", password=True)], + password="run-as-pw") + return run_wrapper(workspace, context) + + +def test_printed_secrets_are_masked(secret_task_stdout): + assert "print: ******** ********" in secret_task_stdout.splitlines() + + +@pytest.mark.xfail(reason="dai_logger's handler holds the original sys.stdout, so logged lines and " + "comments bypass MaskedIO and print secrets in clear text") +@pytest.mark.parametrize("channel", ["logger:", "##[start: comment]comment:"], ids=["logger", "comment"]) +def test_logged_secrets_are_masked(secret_task_stdout, channel): + (line,) = [line for line in secret_task_stdout.splitlines() if channel in line] + + assert "t0k3n-value" not in line and "run-as-pw" not in line + + +# -- real SIGABRT ---------------------------------------------------------- + +GRACE_PERIOD = 2.0 + + +def run_and_abort(workspace, task_type): + """Runs task_type, sends SIGABRT once execute() has started; returns (seconds to exit, output).""" + process = start_wrapper(workspace, input_context(task_type, []), + ABORT_GRACE_PERIOD_SECONDS=str(GRACE_PERIOD)) + marker = workspace / "task_started" + deadline = time.monotonic() + 30 + while not marker.exists(): + if process.poll() is not None or time.monotonic() > deadline: + _, stdout, stderr = finish(process) + pytest.fail(f"task did not start\nstdout:\n{stdout}\nstderr:\n{stderr}") + time.sleep(0.05) + + start = time.monotonic() + process.send_signal(signal.SIGABRT) + returncode, stdout, stderr = finish(process, timeout=15) + elapsed = time.monotonic() - start + + assert returncode == 1, f"wrapper exited with {returncode}\nstdout:\n{stdout}\nstderr:\n{stderr}" + return elapsed, read_output(workspace) + + +@posix_only +def test_sigabrt_reports_a_blocked_task_as_aborted_after_the_grace_period(task_workspace): + elapsed, output = run_and_abort(task_workspace, "containerExamples.BlockingTask") + + assert GRACE_PERIOD <= elapsed < 10 + assert output == {"exitCode": 1, "jobErrorMessage": "Abort requested", "outputProperties": {}, + "reportingRecords": []} + + +@posix_only +def test_sigabrt_reports_what_a_cooperative_task_recorded_while_stopping(task_workspace): + elapsed, output = run_and_abort(task_workspace, "containerExamples.CooperativeTask") + + assert elapsed < GRACE_PERIOD + assert (output["exitCode"], output["jobErrorMessage"]) == (1, "Abort requested") + assert output["outputProperties"] == {"stoppedCooperatively": True} + (record,) = output["reportingRecords"] + assert record["processed"] > 0 diff --git a/tests/release/integration/test_wrapper_execute_task.py b/tests/release/integration/test_wrapper_execute_task.py new file mode 100644 index 0000000..c1f53d5 --- /dev/null +++ b/tests/release/integration/test_wrapper_execute_task.py @@ -0,0 +1,156 @@ +""" +wrapper.execute_task() runs the task on a worker thread, reports its result +exactly once, and turns a sys.exit() inside the task into the process exit +status and the reported exit code. +""" +import sys +import threading +import time + +import pytest + +from digitalai.release.integration.base_task import BaseTask +from digitalai.release.integration.exceptions import AbortException +from digitalai.release.integration.output_context import OutputContext +from tests.support import NoOpTask + + +class GreetingTask(BaseTask): + def execute(self) -> None: + self.set_output_property("greeting", "Hello World") + + +class ExitingTask(BaseTask): + def __init__(self, code): + super().__init__() + self.code = code + + def execute(self) -> None: + sys.exit(self.code) + + +def test_reports_the_task_result_once(isolated_wrapper, reported): + isolated_wrapper.execute_task(GreetingTask()) + + assert reported == [OutputContext(0, "", {"greeting": "Hello World"}, [])] + + +def test_runs_the_task_on_a_worker_thread(isolated_wrapper, reported): + threads = [] + + class RecordingTask(BaseTask): + def execute(self) -> None: + threads.append(threading.current_thread()) + + isolated_wrapper.execute_task(RecordingTask()) + + assert threads[0] is not threading.main_thread() + assert threads[0].daemon + + +def test_reports_a_failing_task_as_failed(isolated_wrapper, reported): + class FailingTask(BaseTask): + def execute(self) -> None: + raise ValueError("Deployment target unreachable") + + isolated_wrapper.execute_task(FailingTask()) + + assert reported == [OutputContext(1, "Deployment target unreachable", {}, [])] + + +def test_each_execution_reports_its_own_result(isolated_wrapper, reported): + first, second = NoOpTask(), GreetingTask() + + isolated_wrapper.execute_task(first) + isolated_wrapper.execute_task(second) + + assert reported == [OutputContext(0, "", {}, []), OutputContext(0, "", {"greeting": "Hello World"}, [])] + assert isolated_wrapper.dai_task_object is second + + +def test_abort_exception_is_reported_and_exits_with_1(isolated_wrapper, reported): + class AbortingTask(BaseTask): + def execute(self) -> None: + raise AbortException() + + with pytest.raises(SystemExit) as exit_info: + isolated_wrapper.execute_task(AbortingTask()) + + assert exit_info.value.code == 1 + assert reported == [OutputContext(1, "Abort requested", {}, [])] + + +@pytest.mark.parametrize("code, reported_code, message", [ + (2, 2, "Task exited with code 2"), + ("Configuration missing", 1, "Configuration missing"), + (0, 0, ""), + (None, 0, ""), +], ids=["int", "message", "zero", "none"]) +def test_sys_exit_is_reported_and_propagated(isolated_wrapper, reported, code, reported_code, message): + with pytest.raises(SystemExit) as exit_info: + isolated_wrapper.execute_task(ExitingTask(code)) + + assert exit_info.value.code == code + assert reported == [OutputContext(reported_code, message, {}, [])] + + +def test_sys_exit_keeps_an_exit_code_the_task_set_itself(isolated_wrapper, reported): + class FailingThenExitingTask(BaseTask): + def execute(self) -> None: + self.set_exit_code(5) + self.set_error_message("Quota exceeded") + sys.exit(2) + + with pytest.raises(SystemExit): + isolated_wrapper.execute_task(FailingThenExitingTask()) + + assert reported == [OutputContext(5, "Quota exceeded", {}, [])] + + +def test_sys_exit_keeps_an_error_message_the_task_set_itself(isolated_wrapper, reported): + class ExplainedExitTask(BaseTask): + def execute(self) -> None: + self.set_error_message("Quota exceeded") + sys.exit(2) + + with pytest.raises(SystemExit): + isolated_wrapper.execute_task(ExplainedExitTask()) + + assert reported == [OutputContext(2, "Quota exceeded", {}, [])] + + +def test_task_without_super_init_runs_and_reports(isolated_wrapper, reported): + class NoSuperInitTask(BaseTask): + def __init__(self): + pass + + def execute(self) -> None: + self.set_output_property("ok", True) + + isolated_wrapper.execute_task(NoSuperInitTask()) + + assert reported == [OutputContext(0, "", {"ok": True}, [])] + + +def test_cooperative_task_sees_an_abort_requested_from_another_thread(isolated_wrapper, reported): + started = threading.Event() + + class CooperativeTask(BaseTask): + def execute(self) -> None: + started.set() + deadline = time.monotonic() + 5 + while not self.is_abort_requested() and time.monotonic() < deadline: + time.sleep(0.01) + self.set_output_property("stoppedOnRequest", self.is_abort_requested()) + + task = CooperativeTask() + threading.Thread(target=lambda: started.wait(5) and task._request_abort(), daemon=True).start() + + isolated_wrapper.execute_task(task) + + assert reported == [OutputContext(0, "", {"stoppedOnRequest": True}, [])] + + +@pytest.mark.parametrize("code, expected", [(None, 0), (0, 0), (3, 3), ("message", 1)]) +def test_exit_code_of_matches_the_process_exit_status(isolated_wrapper, code, expected): + assert isolated_wrapper._exit_code_of(SystemExit(code)) == expected diff --git a/tests/release/integration/test_wrapper_io.py b/tests/release/integration/test_wrapper_io.py new file mode 100644 index 0000000..37d8ff8 --- /dev/null +++ b/tests/release/integration/test_wrapper_io.py @@ -0,0 +1,388 @@ +""" +How the wrapper reads the input context and delivers the result. + +With INPUT_LOCATION set, the input comes from a file and the result goes to +OUTPUT_LOCATION. On Kubernetes there is no file: the input comes from a +Secret, and the result goes to a result Secret and/or a callback URL. + +Kubernetes stores Secret values base64-encoded, and the wrapper decodes the +URLs and session key once more, so the helpers below encode values exactly as +the runner does. The Kubernetes client and HTTP session are faked. +""" +import base64 +import json +import os +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest import mock + +import pytest +import requests + +from digitalai.release.integration import BuildRecord, k8s +from digitalai.release.integration.job_data_encryptor import AESJobDataEncryptor, NoOpJobDataEncryptor +from digitalai.release.integration.output_context import OutputContext +from tests.support import RELEASE_ID, input_context, task_property + +pytestmark = pytest.mark.usefixtures("isolated_wrapper") + +INPUT_SECRET = "input-context-secret" +NAMESPACE = "runner-ns" +RESULT_SECRET = "result-secret" +RESULT_KEY = "result" +CALLBACK_URL = "http://release-server/callback" +LARGE = "x" * (1024 * 1024 + 1024) # over the 1 MiB a Secret can hold + + +def b64(value) -> str: + """base64 of a str/bytes, as text.""" + if isinstance(value, str): + value = value.encode("UTF-8") + return base64.b64encode(value).decode("UTF-8") + + +def secret_url(url: str) -> str: + """How a URL is stored in the input-context Secret: base64 of its base64.""" + return b64(b64(url)) + + +class FakeCoreV1Api: + """Serves secrets by name and records replacements.""" + + def __init__(self, secrets): + self.secrets = secrets + self.replaced = [] + + def read_namespaced_secret(self, name, namespace): + return self.secrets[name] + + def replace_namespaced_secret(self, name, namespace, body): + self.replaced.append((name, namespace, json.loads(json.dumps(body.data)))) + + +def install_kube(monkeypatch, secrets): + """Serves ``{secret name: data}`` through a fake Kubernetes client.""" + fake = FakeCoreV1Api({name: SimpleNamespace(data=data) for name, data in secrets.items()}) + monkeypatch.setattr(k8s, "get_client", lambda: fake) + return fake + + +@pytest.fixture +def on_kubernetes(isolated_wrapper, monkeypatch): + """Wrapper settings of a Kubernetes runner: input from a Secret, no files.""" + monkeypatch.setattr(isolated_wrapper, "input_context_secret", INPUT_SECRET) + monkeypatch.setattr(isolated_wrapper, "runner_namespace", NAMESPACE) + return isolated_wrapper + + +@pytest.fixture +def post(isolated_wrapper, monkeypatch): + """The callback POST, recorded instead of sent.""" + fake = mock.Mock(name="_post_callback") + monkeypatch.setattr(isolated_wrapper, "_post_callback", fake) + return fake + + +@pytest.fixture +def session_key(): + return b64(os.urandom(32)) + + +# -- encryptor ------------------------------------------------------------- + +def test_encryptor_is_aes_when_a_session_key_is_set(isolated_wrapper, monkeypatch, session_key): + monkeypatch.setattr(isolated_wrapper, "base64_session_key", session_key) + + assert isinstance(isolated_wrapper.get_encryptor(), AESJobDataEncryptor) + + +def test_encryptor_is_noop_without_a_session_key(isolated_wrapper): + assert isinstance(isolated_wrapper.get_encryptor(), NoOpJobDataEncryptor) + + +# -- input from a file ----------------------------------------------------- + +@pytest.fixture +def input_file(isolated_wrapper, monkeypatch, tmp_path): + """Writes an input context to INPUT_LOCATION; call with the context dict.""" + path = tmp_path / "input.json" + monkeypatch.setattr(isolated_wrapper, "input_context_file", str(path)) + + def write(context, encryptor=NoOpJobDataEncryptor()): + path.write_text(encryptor.encrypt(json.dumps(context))) + + return write + + +def test_task_details_from_file_return_properties_type_and_script(isolated_wrapper, input_file): + input_file(input_context("containerExamples.Hello", [ + task_property("yourName", "World"), + task_property("scriptLocation", "sample/hello.py"), + ])) + + properties, task_type, script_path = isolated_wrapper.get_task_details() + + assert properties == {"yourName": "World", "scriptLocation": "sample/hello.py"} + assert task_type == "containerExamples.Hello" + assert script_path == "sample/hello.py" + assert isolated_wrapper.input_context.release.id == RELEASE_ID + + +def test_task_details_from_file_are_decrypted_with_the_session_key(isolated_wrapper, monkeypatch, input_file, + session_key): + monkeypatch.setattr(isolated_wrapper, "base64_session_key", session_key) + input_file(input_context(), AESJobDataEncryptor(session_key)) + + properties, _, _ = isolated_wrapper.get_task_details() + + assert properties["yourName"] == "World" + + +def test_task_details_register_password_inputs_and_run_as_password_for_masking(isolated_wrapper, input_file): + input_file(input_context(properties=[task_property("token", "t0ken", password=True)], password="run-as-pw")) + + isolated_wrapper.get_task_details() + + assert isolated_wrapper.masked_std_out.secrets == ["t0ken", "run-as-pw"] + assert isolated_wrapper.masked_std_err.secrets == ["t0ken", "run-as-pw"] + + +def test_task_details_without_run_as_password_mask_only_password_inputs(isolated_wrapper, input_file): + input_file(input_context(properties=[task_property("token", "t0ken", password=True)], + username=None, password=None)) + + isolated_wrapper.get_task_details() + + assert isolated_wrapper.masked_std_out.secrets == ["t0ken"] + + +# -- input from a Kubernetes secret ---------------------------------------- + +def test_task_details_from_secret_set_the_callback_url(on_kubernetes, monkeypatch): + install_kube(monkeypatch, {INPUT_SECRET: { + "session-key": b64(""), "url": secret_url(CALLBACK_URL), "input": b64(json.dumps(input_context())), + }}) + + properties, task_type, _ = on_kubernetes.get_task_details() + + assert (properties["yourName"], task_type) == ("World", "containerExamples.Hello") + assert base64.b64decode(on_kubernetes.callback_url).decode() == CALLBACK_URL + + +def test_task_details_from_secret_are_decrypted_with_its_session_key(on_kubernetes, monkeypatch, session_key): + encrypted = AESJobDataEncryptor(session_key).encrypt(json.dumps(input_context())) + install_kube(monkeypatch, {INPUT_SECRET: { + "session-key": b64(session_key), "url": secret_url(CALLBACK_URL), "input": b64(encrypted), + }}) + + properties, _, _ = on_kubernetes.get_task_details() + + assert properties["yourName"] == "World" + + +def test_task_details_are_fetched_from_the_fetch_url_when_the_secret_has_no_input(on_kubernetes, monkeypatch): + install_kube(monkeypatch, {INPUT_SECRET: { + "session-key": b64(""), "url": secret_url(CALLBACK_URL), "input": "", + "fetchUrl": secret_url("http://release-server/input"), + }}) + response = mock.Mock(content=json.dumps(input_context()).encode()) + get = mock.Mock(return_value=response) + monkeypatch.setattr(on_kubernetes._http_session, "get", get) + + properties, _, _ = on_kubernetes.get_task_details() + + get.assert_called_once_with("http://release-server/input", timeout=on_kubernetes._HTTP_TIMEOUT) + response.raise_for_status.assert_called_once_with() + assert properties["yourName"] == "World" + + +def test_task_details_fetch_failure_propagates(on_kubernetes, monkeypatch): + install_kube(monkeypatch, {INPUT_SECRET: { + "session-key": b64(""), "url": secret_url(CALLBACK_URL), "input": "", + "fetchUrl": secret_url("http://release-server/input"), + }}) + response = mock.Mock() + response.raise_for_status.side_effect = requests.HTTPError("503 Service Unavailable") + monkeypatch.setattr(on_kubernetes._http_session, "get", mock.Mock(return_value=response)) + + with pytest.raises(requests.HTTPError): + on_kubernetes.get_task_details() + + +def test_task_details_require_a_fetch_url_when_the_secret_has_no_input(on_kubernetes, monkeypatch): + install_kube(monkeypatch, {INPUT_SECRET: { + "session-key": b64(""), "url": secret_url(CALLBACK_URL), "input": "", "fetchUrl": "", + }}) + + with pytest.raises(ValueError, match="Cannot find fetch URL"): + on_kubernetes.get_task_details() + + +def test_task_details_secret_read_failure_propagates(on_kubernetes, monkeypatch): + client = mock.Mock() + client.read_namespaced_secret.side_effect = RuntimeError("k8s unavailable") + monkeypatch.setattr(k8s, "get_client", lambda: client) + + with pytest.raises(RuntimeError, match="k8s unavailable"): + on_kubernetes.get_task_details() + + +# -- result to a file ------------------------------------------------------ + +@pytest.fixture +def output_file(isolated_wrapper, monkeypatch, tmp_path): + path = tmp_path / "output.json" + monkeypatch.setattr(isolated_wrapper, "output_context_file", str(path)) + return path + + +def test_result_is_written_to_the_output_file_as_json(isolated_wrapper, output_file): + isolated_wrapper.update_output_context(OutputContext(0, "", {"greeting": "Hello World"}, [])) + + assert json.loads(output_file.read_text()) == { + "exitCode": 0, "jobErrorMessage": "", "outputProperties": {"greeting": "Hello World"}, "reportingRecords": [], + } + + +def test_result_is_encrypted_with_the_session_key(isolated_wrapper, monkeypatch, output_file, session_key): + monkeypatch.setattr(isolated_wrapper, "base64_session_key", session_key) + + isolated_wrapper.update_output_context(OutputContext(1, "failed", {}, [])) + + assert json.loads(AESJobDataEncryptor(session_key).decrypt(output_file.read_text()))["exitCode"] == 1 + + +def test_result_serializes_reporting_record_objects(isolated_wrapper, output_file): + when = datetime(2026, 3, 4, 5, 6, 7, tzinfo=timezone.utc) + record = BuildRecord(target_id="T1", build="#42", build_url="http://ci/42", project="app", outcome="SUCCESS", + start_date=when, end_date=when, duration="PT1M") + + isolated_wrapper.update_output_context(OutputContext(0, "", {}, [record])) + + (written,) = json.loads(output_file.read_text())["reportingRecords"] + assert written == record.to_dict() + + +# -- result to a Kubernetes secret and callback ---------------------------- + +@pytest.fixture +def result_to_secret(on_kubernetes, monkeypatch, session_key): + """ + Result delivery of a Kubernetes runner: a result Secret plus the callback URL. + The runner always provides a session key there, and the wrapper relies on it: + the encrypted result is base64 text, which a real API server accepts as a + Secret ``data`` value, while plain JSON is rejected (see test_wrapper_k8s_live.py). + """ + monkeypatch.setattr(on_kubernetes, "base64_session_key", session_key) + monkeypatch.setattr(on_kubernetes, "result_secret_key", f"{NAMESPACE}:{RESULT_SECRET}:{RESULT_KEY}") + monkeypatch.setattr(on_kubernetes, "callback_url", b64(CALLBACK_URL).encode()) + return on_kubernetes + + +def test_result_is_stored_in_the_result_secret_and_posted_to_the_callback(result_to_secret, monkeypatch, post, + session_key): + kube = install_kube(monkeypatch, {RESULT_SECRET: {RESULT_KEY: "", "other": "kept"}}) + + result_to_secret.update_output_context(OutputContext(0, "", {"greeting": "Hello"}, [])) + + ((name, namespace, data),) = kube.replaced + assert (name, namespace) == (RESULT_SECRET, NAMESPACE) + assert data["other"] == "kept" + assert json.loads(AESJobDataEncryptor(session_key).decrypt(data[RESULT_KEY]))["outputProperties"] == { + "greeting": "Hello"} + post.assert_called_once_with(CALLBACK_URL, data[RESULT_KEY]) + + +def test_result_too_large_for_a_secret_is_only_posted(result_to_secret, monkeypatch, post): + kube = install_kube(monkeypatch, {RESULT_SECRET: {RESULT_KEY: ""}}) + + result_to_secret.update_output_context(OutputContext(0, "", {"big": LARGE}, [])) + + assert kube.replaced == [] + post.assert_called_once() + + +def test_result_secret_failure_is_logged_not_raised(result_to_secret, monkeypatch, post, dai_log): + client = mock.Mock() + client.read_namespaced_secret.side_effect = RuntimeError("k8s unavailable") + monkeypatch.setattr(k8s, "get_client", lambda: client) + + result_to_secret.update_output_context(OutputContext(0, "", {}, [])) + + assert "Unexpected error occurred." in dai_log.messages + post.assert_not_called() + + +def test_failed_callback_of_a_small_result_is_logged_not_retried(result_to_secret, monkeypatch, post, dai_log): + install_kube(monkeypatch, {RESULT_SECRET: {RESULT_KEY: ""}}) + post.side_effect = requests.ConnectionError("refused") + retry = mock.Mock() + monkeypatch.setattr(result_to_secret, "retry_push_result_infinitely", retry) + + result_to_secret.update_output_context(OutputContext(0, "", {}, [])) + + retry.assert_not_called() + assert "Unexpected error occurred." in dai_log.messages + + +def test_failed_callback_of_a_large_result_is_retried(result_to_secret, monkeypatch, post, session_key): + install_kube(monkeypatch, {RESULT_SECRET: {RESULT_KEY: ""}}) + post.side_effect = requests.ConnectionError("refused") + retry = mock.Mock() + monkeypatch.setattr(result_to_secret, "retry_push_result_infinitely", retry) + + result_to_secret.update_output_context(OutputContext(0, "", {"big": LARGE}, [])) + + (pushed,), _ = retry.call_args + assert json.loads(AESJobDataEncryptor(session_key).decrypt(pushed))["outputProperties"]["big"] == LARGE + + +def test_retry_rereads_the_callback_url_and_backs_off_up_to_three_minutes(on_kubernetes, monkeypatch, post): + urls = iter([f"{CALLBACK_URL}/{attempt}" for attempt in range(11)]) + client = mock.Mock() + client.read_namespaced_secret.side_effect = lambda name, ns: SimpleNamespace(data={"url": secret_url(next(urls))}) + monkeypatch.setattr(k8s, "get_client", lambda: client) + delivered = mock.Mock(name="response") + post.side_effect = [requests.ConnectionError("refused")] * 10 + [delivered] + sleeps = [] + monkeypatch.setattr(on_kubernetes.time, "sleep", sleeps.append) + + response = on_kubernetes.retry_push_result_infinitely("payload") + + assert sleeps == [1, 2, 4, 8, 16, 32, 64, 128, 180, 180] + assert [c.args for c in post.call_args_list] == [(f"{CALLBACK_URL}/{n}", "payload") for n in range(11)] + assert response is delivered + client.read_namespaced_secret.assert_called_with(INPUT_SECRET, NAMESPACE) + + +@pytest.mark.parametrize("payload, input_file, expected", [ + (LARGE, "", True), + ("small", "", False), + (LARGE, "input.json", False), +], ids=["large-without-input-file", "small", "large-with-input-file"]) +def test_should_retry_only_a_large_result_without_an_input_file(isolated_wrapper, monkeypatch, payload, input_file, + expected): + monkeypatch.setattr(isolated_wrapper, "input_context_file", input_file) + + assert isolated_wrapper.should_retry_callback_request(payload) is expected + + +def test_post_callback_returns_the_response_on_success(isolated_wrapper, monkeypatch): + response = mock.Mock() + monkeypatch.setattr(isolated_wrapper._http_session, "post", mock.Mock(return_value=response)) + + assert isolated_wrapper._post_callback(CALLBACK_URL, "payload") is response + response.raise_for_status.assert_called_once_with() + + +def test_post_callback_sends_json_with_a_timeout_and_raises_on_http_errors(isolated_wrapper, monkeypatch): + response = mock.Mock() + response.raise_for_status.side_effect = requests.HTTPError("500") + session_post = mock.Mock(return_value=response) + monkeypatch.setattr(isolated_wrapper._http_session, "post", session_post) + + with pytest.raises(requests.HTTPError): + isolated_wrapper._post_callback(CALLBACK_URL, "payload") + + session_post.assert_called_once_with(CALLBACK_URL, headers={"Content-Type": "application/json"}, + data="payload", timeout=isolated_wrapper._HTTP_TIMEOUT) diff --git a/tests/release/integration/test_wrapper_k8s.py b/tests/release/integration/test_wrapper_k8s.py deleted file mode 100644 index 78c3f7e..0000000 --- a/tests/release/integration/test_wrapper_k8s.py +++ /dev/null @@ -1,283 +0,0 @@ -""" -Tests for the wrapper's Kubernetes execution path. - -When the runner executes inside Kubernetes there is no INPUT_LOCATION file; the -input context is read from a Secret, and the result is written back to a Secret -and/or pushed to a callback URL. These tests exercise that path with a fully -mocked Kubernetes client and callback transport, so no real cluster or network -is required. - -Secret values in Kubernetes are base64-encoded strings, and the wrapper applies -its own (double) base64 encoding to the callback/fetch URLs. The helpers below -mirror that encoding so the fakes look exactly like a real Secret. -""" - -import base64 -import json -import unittest -from unittest import mock - -import digitalai.release.integration.wrapper as wrapper -from digitalai.release.integration import k8s -from digitalai.release.integration.output_context import OutputContext - -# Sample input context (Hello task) reused from the file-based wrapper test. -SAMPLE_INPUT_CONTEXT = { - "release": { - "id": "Applications/Folder1f65c7220b394afbb941154342fd9fc6/Release31de09e95c8e4ebb95aaed29a8082d0b", - "automatedTaskAsUser": {"username": "admin", "password": "admin"}, - }, - "task": { - "id": "Applications/Folder1f65c7220b394afbb941154342fd9fc6/Release31de09e95c8e4ebb95aaed29a8082d0b/Phase723a601c78804f7dbcaa8b05b83708f5/Task3a35b67b42b6428b854857fba470b39a", - "type": "containerExamples.Hello", - "properties": [ - {"name": "capabilities", "value": ["remote"], "kind": "SET_OF_STRING", "category": "input", "password": False}, - {"name": "yourName", "value": "World", "kind": "STRING", "category": "input", "password": False}, - {"name": "greeting", "value": None, "kind": "STRING", "category": "output", "password": False}, - ], - }, -} - - -def _b64(value) -> str: - """base64-encode a str/bytes the way a Kubernetes Secret stores its values.""" - if isinstance(value, str): - value = value.encode("UTF-8") - return base64.b64encode(value).decode("UTF-8") - - -def _double_b64_url(url: str) -> str: - """The wrapper expects callback/fetch URLs to be double base64-encoded.""" - return _b64(_b64(url)) - - -class FakeSecret: - """Minimal stand-in for a kubernetes V1Secret (only ``.data`` is used).""" - - def __init__(self, data): - self.data = dict(data) - - -class FakeCoreV1Api: - """Records secret reads/replacements so assertions can inspect them.""" - - def __init__(self, secrets): - self.secrets = secrets # name -> FakeSecret - self.replaced = [] # list of (name, namespace, body) - - def read_namespaced_secret(self, name, namespace): - return self.secrets[name] - - def replace_namespaced_secret(self, name, namespace, body): - self.replaced.append((name, namespace, body)) - self.secrets[name] = body - - -class _WrapperK8sTestBase(unittest.TestCase): - """Shared setup: patch the wrapper's module globals and the k8s client.""" - - INPUT_SECRET = "input-context-secret" - NAMESPACE = "runner-ns" - CALLBACK_URL = "http://release-server/callback" - - def _patch_global(self, name, value): - patcher = mock.patch.object(wrapper, name, value) - patcher.start() - self.addCleanup(patcher.stop) - - def _install_fake_client(self, secrets): - fake = FakeCoreV1Api(secrets) - patcher = mock.patch.object(k8s, "get_client", return_value=fake) - patcher.start() - self.addCleanup(patcher.stop) - return fake - - def setUp(self): - # Force the Kubernetes branch: no input/output files. - self._patch_global("input_context_file", "") - self._patch_global("output_context_file", "") - self._patch_global("input_context_secret", self.INPUT_SECRET) - self._patch_global("runner_namespace", self.NAMESPACE) - # A blank session-key decodes to b"" -> NoOp encryptor (plaintext I/O), - # which keeps the test independent of AES key material. - self._patch_global("base64_session_key", "") - - -class TestGetTaskDetailsFromSecret(_WrapperK8sTestBase): - - def test_reads_input_context_from_secret(self): - input_json = json.dumps(SAMPLE_INPUT_CONTEXT) - secret = FakeSecret({ - "session-key": _b64(""), # -> NoOp encryptor - "url": _double_b64_url(self.CALLBACK_URL), - "input": _b64(input_json), - }) - self._install_fake_client({self.INPUT_SECRET: secret}) - - task_properties, task_type, script_path = wrapper.get_task_details() - - self.assertEqual(task_type, "containerExamples.Hello") - self.assertEqual(task_properties["yourName"], "World") - self.assertEqual(script_path, "") - # callback_url global is set to the (single-decoded) callback bytes. - self.assertEqual(base64.b64decode(wrapper.callback_url).decode("UTF-8"), self.CALLBACK_URL) - # The "Run as user" password is registered as a secret to be masked. - self.assertIn("admin", wrapper.masked_std_out.secrets) - - def test_reads_input_context_via_fetch_url_when_input_empty(self): - input_json = json.dumps(SAMPLE_INPUT_CONTEXT) - fetch_url = "http://blob-store/large-input" - secret = FakeSecret({ - "session-key": _b64(""), - "url": _double_b64_url(self.CALLBACK_URL), - "input": "", # empty -> use fetchUrl - "fetchUrl": _double_b64_url(fetch_url), - }) - self._install_fake_client({self.INPUT_SECRET: secret}) - - fake_response = mock.Mock() - fake_response.content = input_json.encode("UTF-8") - fake_response.status_code = 200 - fake_response.raise_for_status = mock.Mock() - - with mock.patch.object(wrapper._http_session, "get", return_value=fake_response) as fake_get: - task_properties, task_type, _ = wrapper.get_task_details() - - fake_get.assert_called_once() - self.assertEqual(fake_get.call_args.args[0], fetch_url) - # A timeout must be supplied so a hung blob store cannot stall the runner. - self.assertIn("timeout", fake_get.call_args.kwargs) - fake_response.raise_for_status.assert_called_once() - self.assertEqual(task_type, "containerExamples.Hello") - self.assertEqual(task_properties["yourName"], "World") - - def test_missing_fetch_url_raises(self): - secret = FakeSecret({ - "session-key": _b64(""), - "url": _double_b64_url(self.CALLBACK_URL), - "input": "", - "fetchUrl": "", - }) - self._install_fake_client({self.INPUT_SECRET: secret}) - - with self.assertRaises(ValueError): - wrapper.get_task_details() - - def test_secret_read_failure_propagates(self): - # get_task_details does not swallow k8s errors: a failure to read the - # input secret must surface so run() can report the task as failed. - fake = mock.Mock() - fake.read_namespaced_secret.side_effect = RuntimeError("k8s unavailable") - patcher = mock.patch.object(k8s, "get_client", return_value=fake) - patcher.start() - self.addCleanup(patcher.stop) - - with self.assertRaises(RuntimeError): - wrapper.get_task_details() - - -class TestUpdateOutputContextToSecret(_WrapperK8sTestBase): - - RESULT_SECRET_NAME = "result-secret" - RESULT_KEY = "result" - - def setUp(self): - super().setUp() - self._patch_global("result_secret_key", f"{self.NAMESPACE}:{self.RESULT_SECRET_NAME}:{self.RESULT_KEY}") - # callback_url global as set by get_task_details: single base64 of the URL. - self._patch_global("callback_url", _b64(self.CALLBACK_URL).encode("UTF-8")) - - def test_writes_result_to_secret_and_pushes_callback(self): - result_secret = FakeSecret({self.RESULT_KEY: ""}) - fake = self._install_fake_client({self.RESULT_SECRET_NAME: result_secret}) - - output = OutputContext(0, "", {"greeting": "Hello World"}, []) - - with mock.patch.object(wrapper, "_post_callback") as fake_post: - wrapper.update_output_context(output) - - # Result written back to the secret (NoOp encryptor -> plaintext JSON). - self.assertEqual(len(fake.replaced), 1) - name, namespace, body = fake.replaced[0] - self.assertEqual(name, self.RESULT_SECRET_NAME) - self.assertEqual(namespace, self.NAMESPACE) - stored = json.loads(body.data[self.RESULT_KEY]) - self.assertEqual(stored["outputProperties"], {"greeting": "Hello World"}) - - # Callback pushed to the decoded URL with the encrypted body. - fake_post.assert_called_once() - pushed_url, pushed_body = fake_post.call_args.args - self.assertEqual(pushed_url, self.CALLBACK_URL) - self.assertEqual(json.loads(pushed_body)["exitCode"], 0) - - def test_result_too_large_skips_secret_but_still_pushes_callback(self): - result_secret = FakeSecret({self.RESULT_KEY: ""}) - fake = self._install_fake_client({self.RESULT_SECRET_NAME: result_secret}) - - # > 1Mb of output so it cannot be stored in a Secret. - big_value = "x" * (wrapper.size_of_1Mb + 1024) - output = OutputContext(0, "", {"big": big_value}, []) - - with mock.patch.object(wrapper, "_post_callback") as fake_post: - wrapper.update_output_context(output) - - # Secret write skipped because the payload is too big. - self.assertEqual(fake.replaced, []) - # Callback is still attempted. - fake_post.assert_called_once() - - def test_secret_write_failure_is_swallowed_and_logged(self): - # update_output_context must never raise: a failure to write the result - # secret is logged and swallowed so the runner exits cleanly. Because the - # failure happens before the callback step, no callback is attempted. - fake = mock.Mock() - fake.read_namespaced_secret.side_effect = RuntimeError("k8s write unavailable") - patcher = mock.patch.object(k8s, "get_client", return_value=fake) - patcher.start() - self.addCleanup(patcher.stop) - - output = OutputContext(0, "", {"greeting": "Hello World"}, []) - - with mock.patch.object(wrapper, "_post_callback") as fake_post: - # Must not raise. - wrapper.update_output_context(output) - - fake_post.assert_not_called() - - def test_callback_retries_when_too_big_and_no_output_file(self): - result_secret = FakeSecret({self.RESULT_KEY: ""}) - # The retry path re-reads the input-context secret for a fresh URL. - input_secret = FakeSecret({"url": _double_b64_url(self.CALLBACK_URL)}) - self._install_fake_client({ - self.RESULT_SECRET_NAME: result_secret, - self.INPUT_SECRET: input_secret, - }) - - big_value = "x" * (wrapper.size_of_1Mb + 1024) - output = OutputContext(0, "", {"big": big_value}, []) - - # First push fails; should_retry_callback_request() is True (too big, no - # output file) so retry_push_result_infinitely is invoked and succeeds. - with mock.patch.object(wrapper, "_post_callback", side_effect=[RuntimeError("boom"), mock.Mock()]) as fake_post: - wrapper.update_output_context(output) - - self.assertEqual(fake_post.call_count, 2) - - -class TestShouldRetryCallbackRequest(unittest.TestCase): - - def test_retries_when_too_big_and_no_output_file(self): - with mock.patch.object(wrapper, "input_context_file", ""): - self.assertTrue(wrapper.should_retry_callback_request("x" * (wrapper.size_of_1Mb + 1))) - - def test_no_retry_when_small(self): - with mock.patch.object(wrapper, "input_context_file", ""): - self.assertFalse(wrapper.should_retry_callback_request("small")) - - def test_no_retry_when_output_file_present(self): - with mock.patch.object(wrapper, "input_context_file", "output.json"): - self.assertFalse(wrapper.should_retry_callback_request("x" * (wrapper.size_of_1Mb + 1))) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/release/integration/test_wrapper_k8s_live.py b/tests/release/integration/test_wrapper_k8s_live.py new file mode 100644 index 0000000..6e72ea5 --- /dev/null +++ b/tests/release/integration/test_wrapper_k8s_live.py @@ -0,0 +1,309 @@ +""" +The wrapper's Kubernetes path against a real cluster. + +Uses only the kubeconfig context named by RELEASE_LIVE_K8S_CONTEXT (default +``docker-desktop``), selected by name, so a different current context, such as +a shared cluster, is never touched. The module is skipped when that context +is missing or its API server does not answer. + +Each run creates a namespace ``sdk-live-`` for the runner's secrets and +deletes it afterwards. Callback and fetch URLs point at an HTTP server the test +runs on 127.0.0.1, which the wrapper reaches from this host. + +Secrets are created with ``stringData`` holding what the runner stores; the +API returns them base64-encoded in ``data``, which is what the wrapper reads. +""" +import base64 +import functools +import json +import os +import subprocess +import sys +import threading +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from types import SimpleNamespace + +import pytest +import yaml +from kubernetes import client, config + +from digitalai.release.integration import k8s, watcher +from digitalai.release.integration.job_data_encryptor import AESJobDataEncryptor +from digitalai.release.integration.output_context import OutputContext +from tests.support import input_context, task_property + +pytestmark = pytest.mark.live + +CONTEXT = os.getenv("RELEASE_LIVE_K8S_CONTEXT", "docker-desktop") +RESULT_KEY = "result" + + +def b64(text: str) -> str: + return base64.b64encode(text.encode()).decode() + + +def unique(prefix: str) -> str: + return f"{prefix}-{uuid.uuid4().hex[:8]}" + + +def kubeconfig_path() -> str: + return os.path.expanduser(os.getenv("KUBECONFIG", "~/.kube/config").split(os.pathsep)[0]) + + +def write_single_context_kubeconfig(source: str, context: str, target) -> None: + """Copies just ``context`` (with its cluster and user) into a kubeconfig of its own.""" + with open(source) as f: + full = yaml.safe_load(f) + ctx = next(c for c in full["contexts"] if c["name"] == context) + cluster = next(c for c in full["clusters"] if c["name"] == ctx["context"]["cluster"]) + user = next(u for u in full["users"] if u["name"] == ctx["context"]["user"]) + target.write_text(yaml.safe_dump({"apiVersion": "v1", "kind": "Config", "current-context": context, + "contexts": [ctx], "clusters": [cluster], "users": [user]})) + + +@pytest.fixture(scope="module") +def cluster(tmp_path_factory): + source = kubeconfig_path() + try: + contexts, _ = config.list_kube_config_contexts(config_file=source) + except Exception as exc: # noqa: BLE001 - no usable kubeconfig means "skip" + pytest.skip(f"no kubeconfig at {source}: {exc}") + if CONTEXT not in [c["name"] for c in contexts]: + pytest.skip(f"kubeconfig has no context named {CONTEXT!r}") + + api_client = config.new_client_from_config(config_file=source, context=CONTEXT) + core = client.CoreV1Api(api_client) + try: + core.list_namespace(limit=1, _request_timeout=3) + except Exception as exc: # noqa: BLE001 - an unreachable cluster means "skip" + api_client.close() + pytest.skip(f"Kubernetes context {CONTEXT!r} is not reachable: {exc}") + + namespace = unique("sdk-live") + core.create_namespace(client.V1Namespace(metadata=client.V1ObjectMeta(name=namespace))) + kubeconfig = tmp_path_factory.mktemp("kube") / "config" + write_single_context_kubeconfig(source, CONTEXT, kubeconfig) + try: + yield SimpleNamespace(core=core, namespace=namespace, kubeconfig=kubeconfig) + finally: + core.delete_namespace(namespace) + api_client.close() + + +def create_secret(cluster, prefix, string_data) -> str: + name = unique(prefix) + cluster.core.create_namespaced_secret(cluster.namespace, client.V1Secret( + metadata=client.V1ObjectMeta(name=name), string_data=string_data)) + return name + + +def read_secret(cluster, name): + return cluster.core.read_namespaced_secret(name, cluster.namespace).data + + +def create_input_secret(cluster, session_key, callback_url, *, encrypted_input="", fetch_url=None) -> str: + """The input-context secret as the runner creates it.""" + data = {"session-key": session_key, "url": b64(callback_url), "input": encrypted_input} + if fetch_url: + data["fetchUrl"] = b64(fetch_url) + return create_secret(cluster, "input", data) + + +class LocalServer: + """An HTTP server on 127.0.0.1 that records requests and replies with queued responses.""" + + def __init__(self): + self.requests = [] + self._responses = {} + server = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self._handle(b"") + + def do_POST(self): + self._handle(self.rfile.read(int(self.headers.get("Content-Length", 0)))) + + def _handle(self, body): + server.requests.append((self.command, self.path, body)) + status, payload = server._next_response(self.path) + self.send_response(status) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + self._httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self._httpd.serve_forever, daemon=True).start() + + def url(self, path: str) -> str: + return f"http://127.0.0.1:{self._httpd.server_port}{path}" + + def respond(self, path, *responses): + """Replies to ``path`` with each (status, body) in turn, then keeps repeating the last.""" + self._responses[path] = list(responses) + + def _next_response(self, path): + queue = self._responses.get(path, [(200, b"")]) + return queue.pop(0) if len(queue) > 1 else queue[0] + + def close(self): + self._httpd.shutdown() + self._httpd.server_close() + + +@pytest.fixture +def server(monkeypatch): + monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost") + local = LocalServer() + yield local + local.close() + + +@pytest.fixture +def session_key(): + return base64.b64encode(os.urandom(32)).decode() + + +@pytest.fixture +def k8s_wrapper(isolated_wrapper, cluster, monkeypatch): + """The wrapper set up as on a runner pod, using the live cluster's client.""" + monkeypatch.setattr(k8s, "kubernetes_client", cluster.core) + monkeypatch.setattr(isolated_wrapper, "runner_namespace", cluster.namespace) + return isolated_wrapper + + +def encrypted_input(session_key, context): + return AESJobDataEncryptor(session_key).encrypt(json.dumps(context)) + + +# -- input ----------------------------------------------------------------- + +def test_reads_and_decrypts_the_input_secret(k8s_wrapper, cluster, monkeypatch, session_key): + context = input_context(properties=[task_property("token", "t0ken", password=True)], password="run-as-pw") + name = create_input_secret(cluster, session_key, "http://127.0.0.1:9/callback", + encrypted_input=encrypted_input(session_key, context)) + monkeypatch.setattr(k8s_wrapper, "input_context_secret", name) + + properties, task_type, _ = k8s_wrapper.get_task_details() + + assert (properties, task_type) == ({"token": "t0ken"}, "containerExamples.Hello") + assert base64.b64decode(k8s_wrapper.callback_url).decode() == "http://127.0.0.1:9/callback" + assert k8s_wrapper.masked_std_out.secrets == ["t0ken", "run-as-pw"] + + +def test_fetches_the_input_when_the_secret_has_none(k8s_wrapper, cluster, server, monkeypatch, session_key): + server.respond("/input", (200, encrypted_input(session_key, input_context()).encode())) + name = create_input_secret(cluster, session_key, server.url("/callback"), fetch_url=server.url("/input")) + monkeypatch.setattr(k8s_wrapper, "input_context_secret", name) + + properties, _, _ = k8s_wrapper.get_task_details() + + assert properties["yourName"] == "World" + assert server.requests == [("GET", "/input", b"")] + + +# -- result ---------------------------------------------------------------- + +def test_stores_the_result_in_the_result_secret_and_posts_it(k8s_wrapper, cluster, server, monkeypatch, + session_key): + result_secret = create_secret(cluster, "result", {RESULT_KEY: ""}) + monkeypatch.setattr(k8s_wrapper, "base64_session_key", session_key) + monkeypatch.setattr(k8s_wrapper, "result_secret_key", f"{cluster.namespace}:{result_secret}:{RESULT_KEY}") + monkeypatch.setattr(k8s_wrapper, "callback_url", b64(server.url("/callback"))) + + k8s_wrapper.update_output_context(OutputContext(0, "", {"greeting": "Hello"}, [])) + + stored = read_secret(cluster, result_secret)[RESULT_KEY] + assert json.loads(AESJobDataEncryptor(session_key).decrypt(stored)) == { + "exitCode": 0, "jobErrorMessage": "", "outputProperties": {"greeting": "Hello"}, "reportingRecords": [], + } + assert server.requests == [("POST", "/callback", stored.encode())] + + +def test_retry_rereads_the_callback_url_from_the_input_secret(k8s_wrapper, cluster, server, monkeypatch, + session_key): + server.respond("/first", (503, b"")) + name = create_input_secret(cluster, session_key, server.url("/second")) + monkeypatch.setattr(k8s_wrapper, "input_context_secret", name) + monkeypatch.setattr(k8s_wrapper, "base64_session_key", session_key) + monkeypatch.setattr(k8s_wrapper, "callback_url", b64(server.url("/first"))) + + # Too large for a secret, so a failed callback is retried. + k8s_wrapper.update_output_context(OutputContext(0, "", {"big": "x" * (1024 * 1024)}, [])) + + assert [(method, path) for method, path, _ in server.requests] == [("POST", "/first"), ("POST", "/second")] + + +# -- watcher --------------------------------------------------------------- + +def test_watcher_calls_back_on_each_session_key_change_until_stopped(cluster, monkeypatch): + name = create_secret(cluster, "input", {"session-key": "k1"}) + monkeypatch.setenv("INPUT_CONTEXT_SECRET", name) + monkeypatch.setenv("RUNNER_NAMESPACE", cluster.namespace) + + # Watch.stream() calls the list function to open each watch; once it returns, changes are seen. + watching = threading.Event() + list_secrets = cluster.core.list_namespaced_secret + + @functools.wraps(list_secrets) + def list_and_signal(*args, **kwargs): + response = list_secrets(*args, **kwargs) + watching.set() + return response + + monkeypatch.setattr(k8s, "kubernetes_client", SimpleNamespace(list_namespaced_secret=list_and_signal)) + + def set_session_key(value): + cluster.core.patch_namespaced_secret(name, cluster.namespace, {"stringData": {"session-key": value}}) + + updates = [] + updated = threading.Event() + stop = threading.Event() + + def on_update(): + updates.append(read_secret(cluster, name)["session-key"]) + updated.set() + + thread = threading.Thread(target=watcher.start_input_secret_watcher, args=(on_update, stop), daemon=True) + thread.start() + assert watching.wait(10), "the watch did not start" + + set_session_key("k2") + assert updated.wait(10), "no callback after the session key changed" + stop.set() + set_session_key("k3") # the watcher checks the stop flag after each event + thread.join(10) + + assert not thread.is_alive() + assert updates == [b64("k2"), b64("k3")] + + +# -- end to end ------------------------------------------------------------ + +@pytest.mark.e2e +def test_wrapper_runs_a_task_from_kubernetes_secrets(cluster, server, task_workspace, session_key): + context = input_context("containerExamples.Hello", [task_property("yourName", "World")]) + input_secret = create_input_secret(cluster, session_key, server.url("/callback"), + encrypted_input=encrypted_input(session_key, context)) + result_secret = create_secret(cluster, "result", {RESULT_KEY: ""}) + env = {k: v for k, v in os.environ.items() if k not in ( + "INPUT_LOCATION", "OUTPUT_LOCATION", "SESSION_KEY", "CALLBACK_URL", "EXECUTOR_EXECUTION_MODE", + "KUBERNETES_SERVICE_HOST", "SKIP_TLS_LEGACY_K8S")} + env.update(INPUT_CONTEXT_SECRET=input_secret, RUNNER_NAMESPACE=cluster.namespace, + RESULT_SECRET_NAME=f"{cluster.namespace}:{result_secret}:{RESULT_KEY}", + RELEASE_URL="http://localhost:5516", KUBECONFIG=str(cluster.kubeconfig)) + + process = subprocess.run([sys.executable, "-m", "digitalai.release.integration.wrapper"], cwd=task_workspace, + env=env, capture_output=True, text=True, timeout=60) + + assert process.returncode == 0, f"stdout:\n{process.stdout}\nstderr:\n{process.stderr}" + stored = read_secret(cluster, result_secret)[RESULT_KEY] + assert json.loads(AESJobDataEncryptor(session_key).decrypt(stored)) == { + "exitCode": 0, "jobErrorMessage": "", "outputProperties": {"greeting": "Hello World"}, + "reportingRecords": [], + } + assert server.requests == [("POST", "/callback", stored.encode())] diff --git a/tests/release/integration/test_wrapper_run.py b/tests/release/integration/test_wrapper_run.py new file mode 100644 index 0000000..84fdb13 --- /dev/null +++ b/tests/release/integration/test_wrapper_run.py @@ -0,0 +1,135 @@ +""" +wrapper.run(): locating and importing the task class, wiring the task context, +and reporting setup failures. Runs in-process against the fixture tasks in +tasks/, copied to a temporary runner directory (see ``importable_workspace``). +""" +import json +import os +from unittest import mock + +import pytest + +from digitalai.release.integration.output_context import OutputContext +from tests.support import RELEASE_ID, TASK_ID, input_context, task_property + + +@pytest.fixture +def run_task(isolated_wrapper, importable_workspace, reported, monkeypatch, tmp_path): + """Runs the wrapper on an input context and returns the single reported result.""" + input_path = tmp_path / "input.json" + monkeypatch.setattr(isolated_wrapper, "input_context_file", str(input_path)) + + def run(context, *, release_url="http://localhost:5516"): + monkeypatch.setattr(isolated_wrapper, "release_server_url", release_url) + input_path.write_text(json.dumps(context)) + isolated_wrapper.run() + (result,) = reported + return result + + return run + + +def hello_context(script_location=None, task_type="containerExamples.Hello"): + properties = [task_property("yourName", "World")] + if script_location is not None: + properties.append(task_property("scriptLocation", script_location)) + return input_context(task_type, properties) + + +def test_finds_the_task_class_by_name_without_a_script_location(run_task): + assert run_task(hello_context()) == OutputContext(0, "", {"greeting": "Hello World"}, []) + + +@pytest.mark.parametrize("script_location", ["sample/hello.py", "/sample/hello.py", "sample\\hello.py"], + ids=["relative", "leading-slash", "backslash"]) +def test_loads_the_task_class_from_the_script_location(run_task, script_location): + result = run_task(hello_context(script_location)) + + assert result.output_properties == {"greeting": "Hello World from sample"} + + +def test_wires_the_task_context_into_the_task(run_task): + context = input_context("containerExamples.ContextEcho", [task_property("yourName", "World")], + username="deployer", password="pw") + + result = run_task(context, release_url="http://release:5516/") + + assert result == OutputContext(0, "", { + "serverUrl": "http://release:5516", + "releaseId": RELEASE_ID, + "taskId": TASK_ID, + "username": "deployer", + "inputs": {"yourName": "World"}, + }, []) + + +def test_reports_a_failing_task_as_failed(run_task): + result = run_task(input_context("containerExamples.FailingTask", [])) + + assert result == OutputContext(1, "Deployment target unreachable", {}, []) + + +@pytest.mark.parametrize("context, message", [ + (hello_context(task_type="Hello"), + "Invalid task type 'Hello', expected format '.'"), + (hello_context(task_type="containerExamples.Missing"), + "Could not find the 'Missing' class"), + (hello_context("sample/missing.py"), + f"Script file not found at: {os.path.join('src', 'sample', 'missing.py')}"), + (hello_context("sample/hello.py", task_type="containerExamples.ContextEcho"), + f"Class 'ContextEcho' not found in script file: {os.path.join('src', 'sample', 'hello.py')}"), +], ids=["type-without-prefix", "class-not-found", "script-not-found", "class-not-in-script"]) +def test_setup_failure_is_reported_as_failed(run_task, context, message): + result = run_task(context) + + assert result == OutputContext(1, message, {}, []) + + +def test_unreadable_input_is_reported_as_failed(isolated_wrapper, reported, monkeypatch, tmp_path): + monkeypatch.setattr(isolated_wrapper, "input_context_file", str(tmp_path / "missing.json")) + + isolated_wrapper.run() + + (result,) = reported + assert result.exit_code == 1 + assert "missing.json" in result.job_error_message + + +@pytest.mark.parametrize("mode, watches", [("daemon", True), ("", False)], ids=["daemon", "single-run"]) +def test_daemon_mode_watches_for_the_next_input_after_running(isolated_wrapper, run_task, monkeypatch, mode, + watches): + monkeypatch.setattr(isolated_wrapper, "execution_mode", mode) + start_watcher = mock.Mock() + monkeypatch.setattr(isolated_wrapper.watcher, "start_input_context_watcher", start_watcher) + + run_task(hello_context()) + + if watches: + start_watcher.assert_called_once_with(isolated_wrapper.run) + else: + start_watcher.assert_not_called() + + +# -- find_class_file ------------------------------------------------------- + +def test_find_class_file_finds_a_top_level_class_in_a_nested_module(isolated_wrapper, tmp_path): + module = tmp_path / "pkg" / "deep" / "tasks.py" + module.parent.mkdir(parents=True) + module.write_text("class Helper:\n pass\n\nclass Deploy:\n pass\n") + + assert isolated_wrapper.find_class_file(str(tmp_path), "Deploy") == str(module) + + +def test_find_class_file_ignores_nested_classes_and_non_python_files(isolated_wrapper, tmp_path): + (tmp_path / "outer.py").write_text("class Outer:\n class Deploy:\n pass\n") + (tmp_path / "notes.txt").write_text("class Deploy:\n pass\n") + + assert isolated_wrapper.find_class_file(str(tmp_path), "Deploy") is None + + +def test_find_class_file_skips_files_it_cannot_parse(isolated_wrapper, tmp_path): + (tmp_path / "a_broken.py").write_text("class Broken(:\n") + (tmp_path / "b_binary.py").write_bytes(b"\xff\xfe\x00 not utf-8") + (tmp_path / "c_task.py").write_text("class Deploy:\n pass\n") + + assert isolated_wrapper.find_class_file(str(tmp_path), "Deploy") == str(tmp_path / "c_task.py") diff --git a/tests/release/test_release_api_client.py b/tests/release/test_release_api_client.py index ee5d2bc..adfeabb 100644 --- a/tests/release/test_release_api_client.py +++ b/tests/release/test_release_api_client.py @@ -1,39 +1,23 @@ -import unittest +""" +The old ``digitalai.release.release_api_client`` import path must keep working +as a drop-in for the standalone ``digitalai-release-api-client`` package. +""" +import pytest +from com.xebialabs.xlrelease.release_api_client import ReleaseAPIClient as StandaloneReleaseAPIClient -from com.xebialabs.xlrelease.release_api_client import ( - ReleaseAPIClient as ApiClientReleaseAPIClient, -) from digitalai.release.release_api_client import ReleaseAPIClient -class TestReleaseAPIClientBackwardCompatibility(unittest.TestCase): - """The old ``digitalai.release.release_api_client`` import path must keep - working as a drop-in for the standalone api-client implementation.""" +def test_shim_is_a_subclass_of_the_standalone_client(): + assert issubclass(ReleaseAPIClient, StandaloneReleaseAPIClient) - def test_is_subclass_of_api_client_class(self): - """The shim extends the standalone api-client class.""" - self.assertTrue(issubclass(ReleaseAPIClient, ApiClientReleaseAPIClient)) - def test_instance_is_recognized_as_api_client_class(self): - """Instances created via the old path are instances of both classes.""" - client = ReleaseAPIClient("http://localhost:5516", "admin", "admin") - try: - self.assertIsInstance(client, ReleaseAPIClient) - self.assertIsInstance(client, ApiClientReleaseAPIClient) - finally: - client.close() +def test_shim_normalizes_the_url_and_uses_basic_auth(): + with ReleaseAPIClient("http://localhost:5516/", "admin", "secret") as client: + assert client.server_address == "http://localhost:5516" + assert client.session.auth == ("admin", "secret") - def test_constructor_behaves_like_base(self): - """Construction, URL normalization, and auth match the base class.""" - with ReleaseAPIClient("http://localhost:5516/", "admin", "secret") as client: - self.assertEqual(client.server_address, "http://localhost:5516") - self.assertEqual(client.session.auth, ("admin", "secret")) - def test_constructor_requires_credentials(self): - """The base validation still applies through the shim.""" - with self.assertRaises(ValueError): - ReleaseAPIClient("http://localhost:5516") - - -if __name__ == "__main__": - unittest.main() +def test_shim_requires_credentials(): + with pytest.raises(ValueError): + ReleaseAPIClient("http://localhost:5516") diff --git a/tests/support.py b/tests/support.py new file mode 100644 index 0000000..380ea39 --- /dev/null +++ b/tests/support.py @@ -0,0 +1,45 @@ +"""Shared test data: realistic Release ids, input contexts and a minimal task.""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from digitalai.release.integration import BaseTask + +FOLDER_ID = "Applications/Folder1f65c7220b394afbb941154342fd9fc6" +RELEASE_ID = f"{FOLDER_ID}/Release31de09e95c8e4ebb95aaed29a8082d0b" +PHASE_ID = f"{RELEASE_ID}/Phase723a601c78804f7dbcaa8b05b83708f5" +TASK_ID = f"{PHASE_ID}/Task3a35b67b42b6428b854857fba470b39a" + +SERVER_URL = "http://localhost:5516" + + +class NoOpTask(BaseTask): + """Smallest concrete BaseTask; execute() does nothing.""" + + def execute(self) -> None: + pass + + +def task_property(name: str, value: Any, *, kind: str = "STRING", category: str = "input", + password: bool = False) -> Dict[str, Any]: + """One entry of the ``task.properties`` list in an input context.""" + return {"name": name, "value": value, "kind": kind, "category": category, "password": password} + + +def input_context(task_type: str = "containerExamples.Hello", + properties: Optional[List[Dict[str, Any]]] = None, *, + username: Optional[str] = "admin", + password: Optional[str] = "admin") -> Dict[str, Any]: + """An input context as the Release server sends it to a container task.""" + if properties is None: + properties = [ + task_property("yourName", "World"), + task_property("greeting", None, category="output"), + ] + return { + "release": { + "id": RELEASE_ID, + "automatedTaskAsUser": {"username": username, "password": password}, + }, + "task": {"id": TASK_ID, "type": task_type, "properties": properties}, + } From 5e34d24b5c568c12cf4acd702fa2dc764cdcbf54 Mon Sep 17 00:00:00 2001 From: Balaji Venkatesan Date: Thu, 29 Oct 2026 11:09:46 +0530 Subject: [PATCH 05/10] Fix for review comments --- digitalai/release/integration/wrapper.py | 52 ++++++++-- docs/abort-changes.md | 97 ++++++++----------- .../integration/test_api_base_task_live.py | 3 +- .../release/integration/test_wrapper_abort.py | 64 ++++++++++++ .../integration/test_wrapper_execute_task.py | 30 ++++++ .../integration/test_wrapper_k8s_live.py | 20 +++- 6 files changed, 200 insertions(+), 66 deletions(-) diff --git a/digitalai/release/integration/wrapper.py b/digitalai/release/integration/wrapper.py index 52e112e..9b792e4 100644 --- a/digitalai/release/integration/wrapper.py +++ b/digitalai/release/integration/wrapper.py @@ -100,6 +100,10 @@ def get_encryptor(): # for up to ABORT_GRACE_PERIOD_SECONDS. _worker: Optional[threading.Thread] = None +# How many times _snapshot_output_context() reads the output context when the +# worker changes it during the read. +_SNAPSHOT_ATTEMPTS = 5 + def _reset_report_guard() -> None: """Resets the report and abort state ahead of a new task execution.""" @@ -113,18 +117,43 @@ def _reset_report_guard() -> None: def _snapshot_output_context(task: Optional[BaseTask], no_task_message: str) -> OutputContext: """ - Returns a copy of the task's output context, so the result is serialized - from a copy the worker thread cannot change mid-serialization. + Returns a fully detached copy of the task's output context, which shares no + object with the task, so the worker thread cannot change the result while + it is reported. + + The copy is made by serializing the context. A task that ignores an abort + keeps running, and may change a nested output (e.g. a dict) in place while + it is being read, which raises RuntimeError; the copy is then retried. If + it cannot be made, the result is reported without output properties and + reporting records, and as failed. """ if task is None: dai_logger.error(no_task_message) return OutputContext(1, no_task_message, {}, []) context = task.get_output_context() + error = None + for _ in range(_SNAPSHOT_ATTEMPTS): + try: + shallow = OutputContext( + context.exit_code, + context.job_error_message, + dict(context.output_properties or {}), + list(context.reporting_records or []), + ) + return OutputContext.from_dict(json.loads(json.dumps(shallow.to_dict()))) + except RuntimeError as e: + # Changed while being read; read it again. + error = e + except Exception as e: + # Not serializable; reading it again will not help. + error = e + break + dai_logger.error(f"Reporting the result without the task's outputs, as they could not be read: {error}") return OutputContext( - context.exit_code, - context.job_error_message, - dict(context.output_properties or {}), - list(context.reporting_records or []), + context.exit_code or 1, + context.job_error_message or f"The task's outputs could not be reported: {error}", + {}, + [], ) @@ -234,8 +263,11 @@ def abort_handler(signum, frame): else: dai_logger.info("Abort requested") - _report_once(aborted=True, no_task_message="Abort requested before the task started") - _exit_process(1) + try: + _report_once(aborted=True, no_task_message="Abort requested before the task started") + finally: + # Exit even if reporting failed, so a failed report cannot keep an aborted task running. + _exit_process(1) # Register abort handler @@ -435,6 +467,10 @@ def _run_task(): worker = threading.Thread(target=_run_task, name="dai-task-worker", daemon=True) _worker = worker + # A task whose __init__ skips super().__init__() gets its output context on + # first use. Create it here, so the worker and an abort() run by the signal + # handler cannot each create one and lose what the other recorded. + task_object.get_output_context() worker.start() while worker.is_alive(): worker.join(timeout=WORKER_POLL_INTERVAL_SECONDS) diff --git a/docs/abort-changes.md b/docs/abort-changes.md index ccd03e6..64498a0 100644 --- a/docs/abort-changes.md +++ b/docs/abort-changes.md @@ -32,14 +32,14 @@ Line numbers refer to `main` at `75990ee` (column "On `main`") and to `ReportRec | # | Issue | On `main`: where and effect | Fixed on `ReportRecords`: where | How it is fixed | Tests | |---|-------|-----------------------------|---------------------------------|-----------------|-------| -| 1 | `is_abort_requested()` has no effect: no grace period | `wrapper.py` `abort_handler()`, lines 131–148: sets the abort flag, reports and calls `os._exit(1)` immediately, without waiting for `execute()`.
**Effect:** outputs, reporting records and comments a task sets while stopping are lost. | `wrapper.py`:
• [`ABORT_GRACE_PERIOD_SECONDS`, line 58](../digitalai/release/integration/wrapper.py#L58)
• [`_worker`, line 101](../digitalai/release/integration/wrapper.py#L101), set in [`execute_task()`, line 437](../digitalai/release/integration/wrapper.py#L437)
• [`abort_handler()` wait, line 231](../digitalai/release/integration/wrapper.py#L231) | After calling `abort()`, the handler waits up to `ABORT_GRACE_PERIOD_SECONDS` (environment variable, default `3`) for the worker thread with `worker.join(timeout=…)`, then reports. The wait is skipped if there is no worker or it has finished. A task that never stops is reported when the wait ends. | `test_cooperative_stop_within_grace_period_is_reported`
`test_task_that_does_not_stop_is_reported_after_grace_period`
`test_real_sigabrt_reports_outputs_of_cooperative_task` (real `SIGABRT`)
`test_real_sigabrt_interrupts_blocking_task` (real `SIGABRT`) | -| 2 | A signal during a report destroys the result | `wrapper.py` `_report_once()`, line 98 (`_result_reported` flag) and `abort_handler()`: the handler runs on the main thread, can interrupt the main thread's own report, re-enters the `RLock`, sees the flag set and calls `os._exit(1)` mid-send. A second signal during an abort does the same.
**Effect:** Release receives no result. | `wrapper.py`:
• [report state, lines 85–97](../digitalai/release/integration/wrapper.py#L85-L97)
• [`_report_once()` state changes, lines 162 and 172–174](../digitalai/release/integration/wrapper.py#L162-L174)
• [`abort_handler()` guards, lines 201–208](../digitalai/release/integration/wrapper.py#L201-L208)
• [`_reset_report_guard()`, line 104](../digitalai/release/integration/wrapper.py#L104) | The boolean flag is replaced by `_report_state` (`pending` → `in_progress` → `done`) plus `_abort_in_progress` and `_exit_after_report`. If a signal arrives while a report is `in_progress`, the handler sets `_exit_after_report` and returns, so the send completes; `_report_once()` then exits. A signal while an abort is already being handled is ignored. `_reset_report_guard()` resets all of this for each daemon-mode execution. | `test_signal_during_report_lets_the_report_finish_then_exits`
`test_second_signal_during_abort_is_ignored`
`test_execute_task_resets_guard_across_daemon_iterations` | -| 3 | The result is serialized while the task may still change it | `wrapper.py` `update_output_context()`, line 231: `json.dumps()` of the task's own output context, called from the handler while `execute()` may still add outputs on the worker thread. `json.dumps` is outside the `try`.
**Effect:** possible `RuntimeError: dictionary changed size during iteration`; the handler never reaches `os._exit`, and the task is reported as a generic failure without its outputs. | `wrapper.py`:
• [`_snapshot_output_context()`, lines 114–128](../digitalai/release/integration/wrapper.py#L114-L128), called from [`_report_once()`](../digitalai/release/integration/wrapper.py#L145) | `_report_once()` reports a new `OutputContext` with `dict(output_properties)` and `list(reporting_records)`. Each copy is a single C-level call under the GIL, so the worker cannot change it mid-copy. With the grace period, the copy is normally taken after the worker has stopped. | `test_reported_result_is_a_copy_of_the_output_context` | -| 4 | Regression: tasks whose `__init__` skips `super().__init__()` crash | `base_task.py` `__init__()`, line 24: the output context and `_abort_event` are created only here; `execute_task()` no longer creates the output context.
**Effect:** `AttributeError` in `set_output_property()`, `set_exit_code()`, `add_reporting_record()` (raised again from `execute_task()`'s error handler) and in `is_abort_requested()`. | `base_task.py`:
• [`_ensure_output_context()`, line 90](../digitalai/release/integration/base_task.py#L90), called by [`get_output_context()`, line 98](../digitalai/release/integration/base_task.py#L98)
• getters/setters use `get_output_context()`: [lines 109](../digitalai/release/integration/base_task.py#L109), [133](../digitalai/release/integration/base_task.py#L133), [139](../digitalai/release/integration/base_task.py#L139), [145](../digitalai/release/integration/base_task.py#L145), [163](../digitalai/release/integration/base_task.py#L163)
• [`is_abort_requested()`, line 78](../digitalai/release/integration/base_task.py#L78) and [`_request_abort()`, line 86](../digitalai/release/integration/base_task.py#L86) | The output context is created on first use if it is missing. `get_output_properties()`, `set_output_property()`, `set_exit_code()`, `set_error_message()` and `add_reporting_record()` go through `get_output_context()`. `is_abort_requested()` returns `False` and `_request_abort()` creates the event when `_abort_event` is missing. `__init__()` is unchanged. | `test_runs_and_reports_outputs`
`test_can_be_aborted` | -| 5a | An `abort()` override that sets no exit code is reported as succeeded | `wrapper.py` `abort_handler()`: reports the task's context as it is.
**Effect:** an override that does not call `super().abort()` reports exit code `0`, and Release records the aborted task as successful. | `wrapper.py`:
• [`_report_once(aborted=True)`, lines 166–168](../digitalai/release/integration/wrapper.py#L166-L168), called from [`abort_handler()`, line 237](../digitalai/release/integration/wrapper.py#L237) | On the abort path, an exit code of `0` in the reported copy becomes `1`, and an empty error message becomes "Abort requested". A message the task set itself is kept. | `test_aborted_result_never_reports_exit_code_0` | -| 5b | `sys.exit(n)` is reported as exit code `0` | `wrapper.py` `execute_task()`, `TODO` at line 347: the `SystemExit` is re-raised so the process exits with `n`, but the reported context keeps exit code `0`.
**Effect:** Release records the task as successful. | `wrapper.py`:
• [`_exit_code_of()`, line 385](../digitalai/release/integration/wrapper.py#L385)
• [`execute_task()`, lines 443–449](../digitalai/release/integration/wrapper.py#L443-L449) | If the worker raised `SystemExit` with a non-zero status and the task set no exit code, that status is reported. `_exit_code_of()` maps `SystemExit.code` like Python does: `None` → `0`, an `int` → itself, anything else → `1`. The error message is the string passed to `sys.exit()`, or "Task exited with code N". The `SystemExit` is still re-raised. The `TODO` is removed. | `test_direct_sys_exit_propagates_status_and_reports_its_code`
`test_sys_exit_with_message_reports_exit_code_1_and_the_message`
`test_sys_exit_zero_reports_success` | -| 6a | Output not flushed before exit | `wrapper.py` `abort_handler()`: `os._exit(1)` skips Python's stream flushing.
**Effect:** the last stdout/stderr lines, including the `##[start: comment]` markers Release reads, can be lost. | `wrapper.py`:
• [`_flush_std_streams()`, line 131](../digitalai/release/integration/wrapper.py#L131) and [`_exit_process()`, line 140](../digitalai/release/integration/wrapper.py#L140), used by `abort_handler()` and `_report_once()` | `sys.stdout` and `sys.stderr` are flushed before every `os._exit()`. | Covered by the real-`SIGABRT` end-to-end tests; live runs show the task's comments in Release. | -| 6b | Misleading message when a signal arrives before the task exists | `wrapper.py` `_report_once()`: reports "Task produced no output context". | `wrapper.py`:
• [`_snapshot_output_context()`, line 114](../digitalai/release/integration/wrapper.py#L114) with the message passed from [`abort_handler()`, line 237](../digitalai/release/integration/wrapper.py#L237) | The handler reports "Abort requested before the task started". The normal path keeps the old message. | `test_signal_before_task_exists_reports_a_clear_message` | +| 1 | `is_abort_requested()` has no effect: no grace period | `wrapper.py` `abort_handler()`, lines 131–148: sets the abort flag, reports and calls `os._exit(1)` immediately, without waiting for `execute()`.
**Effect:** outputs, reporting records and comments a task sets while stopping are lost. | `wrapper.py`:
• [`ABORT_GRACE_PERIOD_SECONDS`, line 58](../digitalai/release/integration/wrapper.py#L58)
• [`_worker`, line 101](../digitalai/release/integration/wrapper.py#L101), set in [`execute_task()`, line 469](../digitalai/release/integration/wrapper.py#L469)
• [`abort_handler()` wait, line 260](../digitalai/release/integration/wrapper.py#L260) | After calling `abort()`, the handler waits up to `ABORT_GRACE_PERIOD_SECONDS` (environment variable, default `3`) for the worker thread with `worker.join(timeout=…)`, then reports. The wait is skipped if there is no worker or it has finished. A task that never stops is reported when the wait ends. | `test_cooperative_task_is_reported_with_what_it_recorded_while_stopping`
`test_task_that_ignores_the_abort_is_reported_after_the_grace_period`
`test_sigabrt_reports_what_a_cooperative_task_recorded_while_stopping` (real `SIGABRT`)
`test_sigabrt_reports_a_blocked_task_as_aborted_after_the_grace_period` (real `SIGABRT`) | +| 2 | A signal during a report destroys the result | `wrapper.py` `_report_once()`, line 98 (`_result_reported` flag) and `abort_handler()`: the handler runs on the main thread, can interrupt the main thread's own report, re-enters the `RLock`, sees the flag set and calls `os._exit(1)` mid-send. A second signal during an abort does the same.
**Effect:** Release receives no result. | `wrapper.py`:
• [report state, lines 85–97](../digitalai/release/integration/wrapper.py#L85-L97)
• [`_report_once()` state changes, lines 191 and 201–203](../digitalai/release/integration/wrapper.py#L191-L203)
• [`abort_handler()` guards, lines 230–237](../digitalai/release/integration/wrapper.py#L230-L237)
• [`_reset_report_guard()`, line 108](../digitalai/release/integration/wrapper.py#L108) | The boolean flag is replaced by `_report_state` (`pending` → `in_progress` → `done`) plus `_abort_in_progress` and `_exit_after_report`. If a signal arrives while a report is `in_progress`, the handler sets `_exit_after_report` and returns, so the send completes; `_report_once()` then exits. A signal while an abort is already being handled is ignored. `_reset_report_guard()` resets all of this for each daemon-mode execution. | `test_signal_during_a_report_lets_the_report_finish_then_exits`
`test_second_signal_during_an_abort_is_ignored`
`test_each_execution_reports_its_own_result` | +| 3 | The result is serialized while the task may still change it | `wrapper.py` `update_output_context()`, line 231: `json.dumps()` of the task's own output context, called from the handler while `execute()` may still add outputs on the worker thread. `json.dumps` is outside the `try`.
**Effect:** possible `RuntimeError: dictionary changed size during iteration`; the handler never reaches `os._exit`, and the task is reported as a generic failure without its outputs. | `wrapper.py`:
• [`_snapshot_output_context()`, lines 118–157](../digitalai/release/integration/wrapper.py#L118-L157), called from [`_report_once()`, line 174](../digitalai/release/integration/wrapper.py#L174)
• [`abort_handler()` exit, lines 266–270](../digitalai/release/integration/wrapper.py#L266-L270) | `_report_once()` reports a fully detached copy: `_snapshot_output_context()` serializes the context and builds a new `OutputContext` from the JSON, so the result shares no object, nested or not, with the task. If a nested output changes while it is read (`RuntimeError`), it is read again, up to 5 times; if that keeps failing, or an output cannot be serialized, the result is reported as failed without outputs and records. `abort_handler()` exits in a `finally`, so a failed report cannot keep the process running. With the grace period, the copy is normally taken after the worker has stopped. | `test_reported_result_is_a_snapshot_of_the_output_context`
`test_reported_result_shares_no_nested_object_with_the_task`
`test_output_changed_while_read_is_read_again`
`test_output_that_keeps_changing_is_left_out_of_the_aborted_result`
`test_abort_exits_even_when_reporting_fails`
`test_outputs_that_cannot_be_serialized_are_reported_as_a_failure` | +| 4 | Regression: tasks whose `__init__` skips `super().__init__()` crash | `base_task.py` `__init__()`, line 24: the output context and `_abort_event` are created only here; `execute_task()` no longer creates the output context.
**Effect:** `AttributeError` in `set_output_property()`, `set_exit_code()`, `add_reporting_record()` (raised again from `execute_task()`'s error handler) and in `is_abort_requested()`. | `base_task.py`:
• [`_ensure_output_context()`, line 90](../digitalai/release/integration/base_task.py#L90), called by [`get_output_context()`, line 98](../digitalai/release/integration/base_task.py#L98)
• getters/setters use `get_output_context()`: [lines 109](../digitalai/release/integration/base_task.py#L109), [133](../digitalai/release/integration/base_task.py#L133), [139](../digitalai/release/integration/base_task.py#L139), [145](../digitalai/release/integration/base_task.py#L145), [163](../digitalai/release/integration/base_task.py#L163)
• [`is_abort_requested()`, line 78](../digitalai/release/integration/base_task.py#L78) and [`_request_abort()`, line 86](../digitalai/release/integration/base_task.py#L86) | The output context is created on first use if it is missing. `get_output_properties()`, `set_output_property()`, `set_exit_code()`, `set_error_message()` and `add_reporting_record()` go through `get_output_context()`. `is_abort_requested()` returns `False` and `_request_abort()` creates the event when `_abort_event` is missing. `__init__()` is unchanged. `execute_task()` creates the output context on the main thread before it starts the worker ([line 473](../digitalai/release/integration/wrapper.py#L473)); otherwise the worker's first setter and an `abort()` override run by the signal handler can each create one, and the worker's replaces the handler's, losing its cleanup outputs and message. | `test_task_without_super_init_runs_and_reports`
`test_task_without_super_init_can_be_aborted`
`test_task_without_super_init_has_its_output_context_before_it_runs` | +| 5a | An `abort()` override that sets no exit code is reported as succeeded | `wrapper.py` `abort_handler()`: reports the task's context as it is.
**Effect:** an override that does not call `super().abort()` reports exit code `0`, and Release records the aborted task as successful. | `wrapper.py`:
• [`_report_once(aborted=True)`, lines 195–197](../digitalai/release/integration/wrapper.py#L195-L197), called from [`abort_handler()`, line 267](../digitalai/release/integration/wrapper.py#L267) | On the abort path, an exit code of `0` in the reported copy becomes `1`, and an empty error message becomes "Abort requested". A message the task set itself is kept. | `test_abort_override_that_sets_no_exit_code_is_still_reported_as_failed` | +| 5b | `sys.exit(n)` is reported as exit code `0` | `wrapper.py` `execute_task()`, `TODO` at line 347: the `SystemExit` is re-raised so the process exits with `n`, but the reported context keeps exit code `0`.
**Effect:** Release records the task as successful. | `wrapper.py`:
• [`_exit_code_of()`, line 417](../digitalai/release/integration/wrapper.py#L417)
• [`execute_task()`, lines 479–485](../digitalai/release/integration/wrapper.py#L479-L485) | If the worker raised `SystemExit` with a non-zero status and the task set no exit code, that status is reported. `_exit_code_of()` maps `SystemExit.code` like Python does: `None` → `0`, an `int` → itself, anything else → `1`. The error message is the string passed to `sys.exit()`, or "Task exited with code N". The `SystemExit` is still re-raised. The `TODO` is removed. | `test_sys_exit_is_reported_and_propagated` (`int`, `message`, `zero` and `none` cases) | +| 6a | Output not flushed before exit | `wrapper.py` `abort_handler()`: `os._exit(1)` skips Python's stream flushing.
**Effect:** the last stdout/stderr lines, including the `##[start: comment]` markers Release reads, can be lost. | `wrapper.py`:
• [`_flush_std_streams()`, line 160](../digitalai/release/integration/wrapper.py#L160) and [`_exit_process()`, line 169](../digitalai/release/integration/wrapper.py#L169), used by `abort_handler()` and `_report_once()` | `sys.stdout` and `sys.stderr` are flushed before every `os._exit()`. | Covered by the real-`SIGABRT` end-to-end tests; live runs show the task's comments in Release. | +| 6b | Misleading message when a signal arrives before the task exists | `wrapper.py` `_report_once()`: reports "Task produced no output context". | `wrapper.py`:
• [`_snapshot_output_context()`, line 118](../digitalai/release/integration/wrapper.py#L118) with the message passed from [`abort_handler()`, line 267](../digitalai/release/integration/wrapper.py#L267) | The handler reports "Abort requested before the task started". The normal path keeps the old message. | `test_abort_before_a_task_exists_reports_a_clear_message` | | 6c | Thread safety of `abort()` overrides undocumented | `base_task.py` `abort()` docstring and `docs/classes/base_task.md`: no mention that `abort()` runs on the main thread while `execute()` runs on the worker thread. | • [`base_task.py` `abort()` docstring, lines 62–67](../digitalai/release/integration/base_task.py#L62-L67)
• [`docs/classes/base_task.md`, `abort`](classes/base_task.md#abort) and [`is_abort_requested`](classes/base_task.md#is_abort_requested)
• [`docs/examples.md`](examples.md) | The docs state that an override must be thread-safe, that outputs set in it are reported, and that they must be set before `super().abort()` because it raises. They also describe the grace period and its relation to `abortTimeout`. | Documentation only. | | 6d | Abort feature missing from the changelog | `README.md`. | [`README.md`, "Unreleased"](../README.md#unreleased) | New "Unreleased" section listing the abort feature, the grace period and the fixes. | Documentation only. | @@ -219,11 +219,21 @@ therefore always completes, and the process still exits. ### Report a copy of the output context (issue 3) -`_report_once()` now reports a copy made by `_snapshot_output_context()`: a new `OutputContext` -with `dict(output_properties)` and `list(reporting_records)`. Under CPython's GIL these copies -are taken in a single C call, so the worker thread cannot change the dictionary while it is -being serialized. With the grace period, the copy is normally taken after the worker has -already stopped. +`_report_once()` now reports a copy made by `_snapshot_output_context()`, and that copy shares +no object with the task. Copying only the outer `output_properties` dictionary and +`reporting_records` list is not enough: a nested value, such as a dictionary output, would still +be the task's own object, and a task that ignores the abort can change it while it is serialized. + +The copy is therefore made by serializing the context and building a new `OutputContext` from the +JSON. That read can still meet a nested value being changed, which raises `RuntimeError`; the +copy is then retried, up to 5 times. A deep copy would not help, as `copy.deepcopy` iterates the +same dictionary and fails the same way, and the SDK cannot lock changes a task makes directly on +its own objects. If the copy keeps failing, or an output cannot be serialized at all, the result +is reported as failed, without outputs and reporting records, and the reason is logged. + +`abort_handler()` calls `_exit_process(1)` in a `finally`, so even a report that fails cannot +leave the aborted process running. With the grace period, the copy is normally taken after the +worker has already stopped. ### Create the output context and abort flag on first use (issue 4) @@ -235,6 +245,13 @@ and getters go through `get_output_context()` instead of `self.output_context`: `is_abort_requested()` and `_request_abort()` read `_abort_event` with `getattr` and create it if it is missing. Subclasses that skip `super().__init__()` work again. +Creating the output context on first use is a check followed by an assignment, so two threads +using it for the first time at once can each create one. That happens when a cooperative task +sets its first output as it stops, while the signal handler runs the task's `abort()` override: +the worker's context replaces the handler's, and the override's outputs and message are lost. +`execute_task()` therefore calls `get_output_context()` on the main thread before it starts the +worker, so both threads use the same object. + ### Never report an aborted or exited task as succeeded (issue 5) - `_report_once(aborted=True)`: if the copy's exit code is `0`, it is set to `1`, and the error @@ -266,10 +283,10 @@ if it is missing. Subclasses that skip `super().__init__()` work again. | `ABORT_GRACE_PERIOD_SECONDS` | New setting, from the environment, default `3`. The comment explains the relation to `abortTimeout`. | | Report state | `_result_reported` replaced by `_report_state` (`_REPORT_PENDING`, `_REPORT_IN_PROGRESS`, `_REPORT_DONE`), plus `_abort_in_progress`, `_exit_after_report` and `_worker`. The comment above them explains why a signal can arrive during a report. | | `_reset_report_guard()` | Resets all four values, so each daemon-mode execution starts clean. | -| `_snapshot_output_context()` | New. Returns a copy of the task's output context, or a failure context with the given message when there is no task. | +| `_snapshot_output_context()` | New. Returns a fully detached copy of the task's output context (serialized and rebuilt, retried up to `_SNAPSHOT_ATTEMPTS` times), a copy without outputs if that fails, or a failure context with the given message when there is no task. | | `_flush_std_streams()`, `_exit_process()` | New. Flush stdout/stderr, then `os._exit()`. | | `_report_once()` | New parameters `aborted` and `no_task_message`. Moves `pending` → `in_progress` → `done`, reports the copy, forces a non-zero exit code when `aborted`, and exits afterwards if `_exit_after_report` is set. | -| `abort_handler()` | Ignores repeat signals, defers to an in-progress report, waits up to the grace period for the worker, reports with `aborted=True`, exits through `_exit_process()`. The docstring describes the three steps. The `global dai_task_object` statement is removed; the handler reads it into a local. | +| `abort_handler()` | Ignores repeat signals, defers to an in-progress report, waits up to the grace period for the worker, reports with `aborted=True`, exits through `_exit_process()`, also when reporting fails. The docstring describes the three steps. The `global dai_task_object` statement is removed; the handler reads it into a local. | | `_exit_code_of()` | New. Maps `SystemExit.code` to the process exit status. | | `execute_task()` | Stores the worker in `_worker`; copies a non-zero `sys.exit()` status into the output context when the task set none. The `TODO` is removed because it is resolved. The docstring is updated. | @@ -287,45 +304,17 @@ if it is missing. Subclasses that skip `super().__init__()` work again. `BaseTask.__init__()` still creates the output context and the abort event; the new code only covers subclasses that skip it. -### `tests/release/integration/test_wrapper_abort.py` - -Existing tests on `main` are kept, except one that asserted the old behaviour: - -- `test_direct_sys_exit_propagates_status_and_reports_context_as_is` (expected exit code `0` - reported for `sys.exit(2)`) becomes `test_direct_sys_exit_propagates_status_and_reports_its_code` - (expects `2` and "Task exited with code 2"). - -New tests: - -| Test class | Test | What it checks | -|------------|------|----------------| -| `WrapperExecuteTaskThreadingTests` | `test_sys_exit_with_message_reports_exit_code_1_and_the_message` | `sys.exit("Configuration missing")` reports exit code `1` and the message. | -| | `test_sys_exit_zero_reports_success` | `sys.exit(0)` still reports exit code `0`. | -| `WrapperAbortGracePeriodTests` | `test_cooperative_stop_within_grace_period_is_reported` | A task polling `is_abort_requested()` stops; its output property and reporting record are in the reported result, with exit code `1`. | -| | `test_task_that_does_not_stop_is_reported_after_grace_period` | A blocked task is reported after the (patched, 0.2 s) grace period, not before and not much later. | -| | `test_reported_result_is_a_copy_of_the_output_context` | The reported context and its properties dictionary are copies, not the task's own objects. | -| | `test_signal_during_report_lets_the_report_finish_then_exits` | A signal raised inside `update_output_context()` lets the push complete with the task's own result, then exits once. | -| | `test_second_signal_during_abort_is_ignored` | A signal arriving inside `abort()` is ignored; one report, one exit. | -| | `test_aborted_result_never_reports_exit_code_0` | An `abort()` override that sets no exit code is still reported with `1` and "Abort requested", keeping its outputs. | -| | `test_signal_before_task_exists_reports_a_clear_message` | Reports "Abort requested before the task started". | -| `WrapperTaskWithoutSuperInitTests` | `test_runs_and_reports_outputs` | A task without `super().__init__()` runs and reports its outputs. | -| | `test_can_be_aborted` | Such a task can be aborted, and `is_abort_requested()` works. | - -`WrapperRealSigabrtEndToEndTest` runs the real wrapper in a subprocess and sends a real -`SIGABRT`: - -- The shared logic moves into `_run_and_abort(task_type)`; input and output files are renamed - to `abort_input.json` / `abort_output.json`. -- `test_real_sigabrt_interrupts_blocking_task` now also asserts that the wrapper waits at least - the grace period before exiting. -- New `test_real_sigabrt_reports_outputs_of_cooperative_task` checks that a cooperative task - exits within the grace period and that its output property and reporting record are reported. -- The class is skipped on Windows, which cannot deliver `SIGABRT` to a child process. +### Tests -### `tests/release/integration/cooperative_task.py` +The test suite was reorganized into pytest modules (see `DEVELOPMENT.md`); the summary table +names the tests for each issue. The abort behaviour is covered by: -New fixture task for the end-to-end test. `CooperativeTask` loops until `is_abort_requested()`, -then sets `stoppedCooperatively` and adds a reporting record with the number of loop passes. +| File | What it covers | +|------|----------------| +| `tests/release/integration/test_wrapper_abort.py` | `abort_handler()` called in-process with `update_output_context()` and `os._exit()` replaced: the grace period, a cooperative stop, a task that ignores the abort, a signal during a report, a repeat signal, `abort()` overrides (no exit code, own message, raising), tasks without `super().__init__()`, and the detached snapshot, including a nested output that changes while it is read and a report that fails. | +| `tests/release/integration/test_wrapper_execute_task.py` | The worker thread, one report per execution, `AbortException`, `sys.exit()` codes and messages, and outputs that cannot be serialized. | +| `tests/release/integration/test_wrapper_e2e.py` | The real wrapper in a subprocess with a real `SIGABRT` (skipped on Windows, which cannot deliver it to a child process). It waits for the task to start before signalling, and uses a 2 s grace period. | +| `tests/release/integration/tasks/` | The tasks those subprocess runs load: `BlockingTask` ignores the abort; `CooperativeTask` loops until `is_abort_requested()`, then sets `stoppedCooperatively` and adds a reporting record with the number of loop passes. | ### `docs/classes/base_task.md` @@ -363,8 +352,8 @@ later. Set `ABORT_GRACE_PERIOD_SECONDS=0` to disable the wait. | Level | Result | |-------|--------| -| Unit tests, Windows | 126 passed, 2 skipped (the real-`SIGABRT` tests). | -| Unit and end-to-end tests, Linux (`python:3.12-slim` container) | All pass, including both real-`SIGABRT` tests. The other skipped tests need a live Release server. | +| Test suite, Windows, with a live Release server and Docker Desktop Kubernetes | 360 passed, 2 skipped (the real-`SIGABRT` tests), 4 xfailed (known secret-masking bugs, unrelated to abort). | +| Test suite, Linux (Python 3.10 container, as in CI) | 323 passed, 7 skipped (the live Release and Kubernetes tests, with no servers in the container), 4 xfailed; both real-`SIGABRT` tests pass. Three consecutive runs. | | Real wrapper, real `SIGABRT`, Linux container | Template example tasks: cooperative stop, checkpoint `abort()`, `abort()` override cleanup, and a task that ignores the abort all report exit code `1` with their outputs. The ignoring task exits 3.0 s after the signal. | | Live Release, Docker runner | All six template abort examples behave as above. Release records the outputs and the task's comments; results arrive 0.1–3.1 s after the abort. | | Live Release, Kubernetes runner (Docker Desktop) | Same results. The runner log shows `SIGABRT` delivered via `exec /bin/kill -6 1` for every aborted pod, with no fallback to pod deletion. Results arrive 0.3–3.2 s after the abort. | diff --git a/tests/release/integration/test_api_base_task_live.py b/tests/release/integration/test_api_base_task_live.py index c9ee333..353604f 100644 --- a/tests/release/integration/test_api_base_task_live.py +++ b/tests/release/integration/test_api_base_task_live.py @@ -50,8 +50,9 @@ def server_reachable() -> bool: url = urlparse(SERVER_URL) + port = url.port or (443 if url.scheme == "https" else 80) try: - with socket.create_connection((url.hostname, url.port or 80), timeout=0.5): + with socket.create_connection((url.hostname, port), timeout=0.5): return True except OSError: return False diff --git a/tests/release/integration/test_wrapper_abort.py b/tests/release/integration/test_wrapper_abort.py index 6d0631d..f1d39b2 100644 --- a/tests/release/integration/test_wrapper_abort.py +++ b/tests/release/integration/test_wrapper_abort.py @@ -8,6 +8,7 @@ import signal import threading import time +from unittest import mock import pytest @@ -145,6 +146,69 @@ def test_reported_result_is_a_snapshot_of_the_output_context(isolated_wrapper, r assert snapshot == OutputContext(1, "Abort requested", {"a": 1}, [{"r": 1}]) +def test_reported_result_shares_no_nested_object_with_the_task(isolated_wrapper, reported, exit_codes): + task = CountingAbortTask() + task.set_output_property("progress", {"done": 1}) + task.add_reporting_record({"items": [1]}) + isolated_wrapper.dai_task_object = task + + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + (snapshot,) = reported + assert snapshot.output_properties["progress"] is not task.get_output_properties()["progress"] + assert snapshot.reporting_records[0] is not task.get_output_context().reporting_records[0] + assert snapshot == OutputContext(1, "Abort requested", {"progress": {"done": 1}}, [{"items": [1]}]) + + +class ChangingDict(dict): + """A nested output the worker changes while it is read: reading it fails ``failures`` times.""" + + def __init__(self, failures, **items): + super().__init__(**items) + self.failures = failures + + def items(self): + if self.failures: + self.failures -= 1 + raise RuntimeError("dictionary changed size during iteration") + return super().items() + + +def test_output_changed_while_read_is_read_again(isolated_wrapper, reported, exit_codes): + task = CountingAbortTask() + task.set_output_property("progress", ChangingDict(failures=2, done=1)) + isolated_wrapper.dai_task_object = task + + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + assert reported == [OutputContext(1, "Abort requested", {"progress": {"done": 1}}, [])] + assert exit_codes == [1] + + +def test_output_that_keeps_changing_is_left_out_of_the_aborted_result(isolated_wrapper, reported, exit_codes, + dai_log): + task = CountingAbortTask() + task.set_output_property("progress", ChangingDict(failures=1000, done=1)) + task.add_reporting_record({"r": 1}) + isolated_wrapper.dai_task_object = task + + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + assert reported == [ABORTED] + assert exit_codes == [1] + assert any("without the task's outputs" in message for message in dai_log.messages) + + +def test_abort_exits_even_when_reporting_fails(isolated_wrapper, exit_codes, monkeypatch): + monkeypatch.setattr(isolated_wrapper, "update_output_context", mock.Mock(side_effect=RuntimeError("push failed"))) + isolated_wrapper.dai_task_object = NoOpTask() + + with pytest.raises(RuntimeError, match="push failed"): + isolated_wrapper.abort_handler(signal.SIGABRT, None) + + assert exit_codes == [1] + + def test_abort_override_that_sets_no_exit_code_is_still_reported_as_failed(isolated_wrapper, reported, exit_codes): class QuietAbortTask(BaseTask): def execute(self) -> None: diff --git a/tests/release/integration/test_wrapper_execute_task.py b/tests/release/integration/test_wrapper_execute_task.py index c1f53d5..a90c1a1 100644 --- a/tests/release/integration/test_wrapper_execute_task.py +++ b/tests/release/integration/test_wrapper_execute_task.py @@ -58,6 +58,19 @@ def execute(self) -> None: assert reported == [OutputContext(1, "Deployment target unreachable", {}, [])] +def test_outputs_that_cannot_be_serialized_are_reported_as_a_failure(isolated_wrapper, reported): + class SetOutputTask(BaseTask): + def execute(self) -> None: + self.set_output_property("deployment", {"handle": object()}) + + isolated_wrapper.execute_task(SetOutputTask()) + + (result,) = reported + assert (result.exit_code, result.output_properties, result.reporting_records) == (1, {}, []) + assert "The task's outputs could not be reported" in result.job_error_message + assert "not JSON serializable" in result.job_error_message + + def test_each_execution_reports_its_own_result(isolated_wrapper, reported): first, second = NoOpTask(), GreetingTask() @@ -132,6 +145,23 @@ def execute(self) -> None: assert reported == [OutputContext(0, "", {"ok": True}, [])] +def test_task_without_super_init_has_its_output_context_before_it_runs(isolated_wrapper, reported): + # Created on the main thread, so the worker and an abort() run by the signal + # handler share one output context instead of each creating their own. + contexts = [] + + class NoSuperInitTask(BaseTask): + def __init__(self): + pass + + def execute(self) -> None: + contexts.append(self.__dict__.get("output_context")) + + isolated_wrapper.execute_task(NoSuperInitTask()) + + assert contexts[0] is not None + + def test_cooperative_task_sees_an_abort_requested_from_another_thread(isolated_wrapper, reported): started = threading.Event() diff --git a/tests/release/integration/test_wrapper_k8s_live.py b/tests/release/integration/test_wrapper_k8s_live.py index 6e72ea5..2b1e322 100644 --- a/tests/release/integration/test_wrapper_k8s_live.py +++ b/tests/release/integration/test_wrapper_k8s_live.py @@ -52,12 +52,23 @@ def kubeconfig_path() -> str: def write_single_context_kubeconfig(source: str, context: str, target) -> None: - """Copies just ``context`` (with its cluster and user) into a kubeconfig of its own.""" + """ + Copies just ``context`` (with its cluster and user) into a kubeconfig of its own. + + Relative certificate, key and token file paths resolve against the + kubeconfig's own directory, so they are made absolute for the copy. + """ with open(source) as f: full = yaml.safe_load(f) ctx = next(c for c in full["contexts"] if c["name"] == context) cluster = next(c for c in full["clusters"] if c["name"] == ctx["context"]["cluster"]) user = next(u for u in full["users"] if u["name"] == ctx["context"]["user"]) + source_dir = os.path.dirname(os.path.abspath(source)) + for entry, keys in ((cluster["cluster"], ("certificate-authority",)), + (user["user"], ("client-certificate", "client-key", "tokenFile"))): + for key in keys: + if entry.get(key): + entry[key] = os.path.join(source_dir, os.path.expanduser(entry[key])) target.write_text(yaml.safe_dump({"apiVersion": "v1", "kind": "Config", "current-context": context, "contexts": [ctx], "clusters": [cluster], "users": [user]})) @@ -266,6 +277,10 @@ def set_session_key(value): def on_update(): updates.append(read_secret(cluster, name)["session-key"]) + if len(updates) == 2: + # The watcher checks the stop flag after each callback, so this ends the watch after k3. + # Setting it from the test thread instead could stop the watch before k3 is seen. + stop.set() updated.set() thread = threading.Thread(target=watcher.start_input_secret_watcher, args=(on_update, stop), daemon=True) @@ -274,8 +289,7 @@ def on_update(): set_session_key("k2") assert updated.wait(10), "no callback after the session key changed" - stop.set() - set_session_key("k3") # the watcher checks the stop flag after each event + set_session_key("k3") thread.join(10) assert not thread.is_alive() From 028591437b446544376009c6c711a5279d9f6496 Mon Sep 17 00:00:00 2001 From: Balaji Venkatesan Date: Thu, 29 Oct 2026 11:27:06 +0530 Subject: [PATCH 06/10] Fix for review comments --- tests/release/integration/conftest.py | 2 ++ tests/release/integration/tasks/cooperative_task.py | 6 ++++-- tests/release/integration/test_wrapper_k8s_live.py | 13 ++++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/release/integration/conftest.py b/tests/release/integration/conftest.py index f9d6f09..9e41fc1 100644 --- a/tests/release/integration/conftest.py +++ b/tests/release/integration/conftest.py @@ -38,6 +38,8 @@ def isolated_wrapper(monkeypatch): """The wrapper module with blank settings and fresh per-execution state.""" for name in _WRAPPER_SETTINGS: monkeypatch.setattr(wrapper, name, "") + # Also read from the environment; 0 would skip waiting for the worker. Tests may override it. + monkeypatch.setattr(wrapper, "ABORT_GRACE_PERIOD_SECONDS", 3.0) monkeypatch.setattr(wrapper, "input_context", None) monkeypatch.setattr(wrapper, "dai_task_object", None) monkeypatch.setattr(wrapper.masked_std_out, "secrets", []) diff --git a/tests/release/integration/tasks/cooperative_task.py b/tests/release/integration/tasks/cooperative_task.py index 5742260..e096117 100644 --- a/tests/release/integration/tasks/cooperative_task.py +++ b/tests/release/integration/tasks/cooperative_task.py @@ -7,14 +7,16 @@ class CooperativeTask(BaseTask): """ Polls is_abort_requested() and records outputs on the way out, which the - wrapper must report. Creates the ``task_started`` file once execute() runs. + wrapper must report. Creates the ``task_started`` file after its first loop + pass, so once a test sees the file, at least one pass has been counted. """ def execute(self) -> None: - Path("task_started").touch() processed = 0 while not self.is_abort_requested(): processed += 1 + if processed == 1: + Path("task_started").touch() time.sleep(0.05) self.set_output_property("stoppedCooperatively", True) self.add_reporting_record({"processed": processed}) diff --git a/tests/release/integration/test_wrapper_k8s_live.py b/tests/release/integration/test_wrapper_k8s_live.py index 2b1e322..88a5efe 100644 --- a/tests/release/integration/test_wrapper_k8s_live.py +++ b/tests/release/integration/test_wrapper_k8s_live.py @@ -92,13 +92,16 @@ def cluster(tmp_path_factory): pytest.skip(f"Kubernetes context {CONTEXT!r} is not reachable: {exc}") namespace = unique("sdk-live") - core.create_namespace(client.V1Namespace(metadata=client.V1ObjectMeta(name=namespace))) - kubeconfig = tmp_path_factory.mktemp("kube") / "config" - write_single_context_kubeconfig(source, CONTEXT, kubeconfig) try: - yield SimpleNamespace(core=core, namespace=namespace, kubeconfig=kubeconfig) + core.create_namespace(client.V1Namespace(metadata=client.V1ObjectMeta(name=namespace))) + try: + # Inside the try, so a failure here still deletes the namespace. + kubeconfig = tmp_path_factory.mktemp("kube") / "config" + write_single_context_kubeconfig(source, CONTEXT, kubeconfig) + yield SimpleNamespace(core=core, namespace=namespace, kubeconfig=kubeconfig) + finally: + core.delete_namespace(namespace) finally: - core.delete_namespace(namespace) api_client.close() From fc0118f2acd3c55ea335a41c87697e0b155aed7a Mon Sep 17 00:00:00 2001 From: Balaji Venkatesan Date: Thu, 29 Oct 2026 11:47:14 +0530 Subject: [PATCH 07/10] Fix for review comments --- digitalai/release/integration/wrapper.py | 28 +++++++++--- docs/abort-changes.md | 29 ++++++------ .../integration/tasks/stderr_flood_task.py | 17 +++++++ .../release/integration/test_wrapper_abort.py | 43 ++++++++++++++++++ tests/release/integration/test_wrapper_e2e.py | 45 +++++++++++++++++-- .../integration/test_wrapper_k8s_live.py | 14 +++--- 6 files changed, 148 insertions(+), 28 deletions(-) create mode 100644 tests/release/integration/tasks/stderr_flood_task.py diff --git a/digitalai/release/integration/wrapper.py b/digitalai/release/integration/wrapper.py index 9b792e4..5be5442 100644 --- a/digitalai/release/integration/wrapper.py +++ b/digitalai/release/integration/wrapper.py @@ -157,13 +157,29 @@ def _snapshot_output_context(task: Optional[BaseTask], no_task_message: str) -> ) +# How long _flush_std_streams() waits for stdout/stderr to flush before exiting. +FLUSH_TIMEOUT_SECONDS = 1.0 + + def _flush_std_streams() -> None: - """Flushes stdout/stderr before os._exit(), which skips Python's own flush.""" - for stream in (sys.stdout, sys.stderr): - try: - stream.flush() - except Exception: - pass + """ + Flushes stdout/stderr before os._exit(), which skips Python's own flush. + + Waits at most FLUSH_TIMEOUT_SECONDS. A task still writing to a full pipe + that nobody reads holds that stream's lock, so an unbounded flush would + keep the process from exiting. Nothing is logged on a timeout, since the + logger writes to stdout, which may be the blocked stream. + """ + def flush(): + for stream in (sys.stdout, sys.stderr): + try: + stream.flush() + except Exception: + pass + + flusher = threading.Thread(target=flush, name="dai-flush", daemon=True) + flusher.start() + flusher.join(timeout=FLUSH_TIMEOUT_SECONDS) def _exit_process(code: int) -> None: diff --git a/docs/abort-changes.md b/docs/abort-changes.md index 64498a0..31b54e3 100644 --- a/docs/abort-changes.md +++ b/docs/abort-changes.md @@ -32,14 +32,14 @@ Line numbers refer to `main` at `75990ee` (column "On `main`") and to `ReportRec | # | Issue | On `main`: where and effect | Fixed on `ReportRecords`: where | How it is fixed | Tests | |---|-------|-----------------------------|---------------------------------|-----------------|-------| -| 1 | `is_abort_requested()` has no effect: no grace period | `wrapper.py` `abort_handler()`, lines 131–148: sets the abort flag, reports and calls `os._exit(1)` immediately, without waiting for `execute()`.
**Effect:** outputs, reporting records and comments a task sets while stopping are lost. | `wrapper.py`:
• [`ABORT_GRACE_PERIOD_SECONDS`, line 58](../digitalai/release/integration/wrapper.py#L58)
• [`_worker`, line 101](../digitalai/release/integration/wrapper.py#L101), set in [`execute_task()`, line 469](../digitalai/release/integration/wrapper.py#L469)
• [`abort_handler()` wait, line 260](../digitalai/release/integration/wrapper.py#L260) | After calling `abort()`, the handler waits up to `ABORT_GRACE_PERIOD_SECONDS` (environment variable, default `3`) for the worker thread with `worker.join(timeout=…)`, then reports. The wait is skipped if there is no worker or it has finished. A task that never stops is reported when the wait ends. | `test_cooperative_task_is_reported_with_what_it_recorded_while_stopping`
`test_task_that_ignores_the_abort_is_reported_after_the_grace_period`
`test_sigabrt_reports_what_a_cooperative_task_recorded_while_stopping` (real `SIGABRT`)
`test_sigabrt_reports_a_blocked_task_as_aborted_after_the_grace_period` (real `SIGABRT`) | -| 2 | A signal during a report destroys the result | `wrapper.py` `_report_once()`, line 98 (`_result_reported` flag) and `abort_handler()`: the handler runs on the main thread, can interrupt the main thread's own report, re-enters the `RLock`, sees the flag set and calls `os._exit(1)` mid-send. A second signal during an abort does the same.
**Effect:** Release receives no result. | `wrapper.py`:
• [report state, lines 85–97](../digitalai/release/integration/wrapper.py#L85-L97)
• [`_report_once()` state changes, lines 191 and 201–203](../digitalai/release/integration/wrapper.py#L191-L203)
• [`abort_handler()` guards, lines 230–237](../digitalai/release/integration/wrapper.py#L230-L237)
• [`_reset_report_guard()`, line 108](../digitalai/release/integration/wrapper.py#L108) | The boolean flag is replaced by `_report_state` (`pending` → `in_progress` → `done`) plus `_abort_in_progress` and `_exit_after_report`. If a signal arrives while a report is `in_progress`, the handler sets `_exit_after_report` and returns, so the send completes; `_report_once()` then exits. A signal while an abort is already being handled is ignored. `_reset_report_guard()` resets all of this for each daemon-mode execution. | `test_signal_during_a_report_lets_the_report_finish_then_exits`
`test_second_signal_during_an_abort_is_ignored`
`test_each_execution_reports_its_own_result` | -| 3 | The result is serialized while the task may still change it | `wrapper.py` `update_output_context()`, line 231: `json.dumps()` of the task's own output context, called from the handler while `execute()` may still add outputs on the worker thread. `json.dumps` is outside the `try`.
**Effect:** possible `RuntimeError: dictionary changed size during iteration`; the handler never reaches `os._exit`, and the task is reported as a generic failure without its outputs. | `wrapper.py`:
• [`_snapshot_output_context()`, lines 118–157](../digitalai/release/integration/wrapper.py#L118-L157), called from [`_report_once()`, line 174](../digitalai/release/integration/wrapper.py#L174)
• [`abort_handler()` exit, lines 266–270](../digitalai/release/integration/wrapper.py#L266-L270) | `_report_once()` reports a fully detached copy: `_snapshot_output_context()` serializes the context and builds a new `OutputContext` from the JSON, so the result shares no object, nested or not, with the task. If a nested output changes while it is read (`RuntimeError`), it is read again, up to 5 times; if that keeps failing, or an output cannot be serialized, the result is reported as failed without outputs and records. `abort_handler()` exits in a `finally`, so a failed report cannot keep the process running. With the grace period, the copy is normally taken after the worker has stopped. | `test_reported_result_is_a_snapshot_of_the_output_context`
`test_reported_result_shares_no_nested_object_with_the_task`
`test_output_changed_while_read_is_read_again`
`test_output_that_keeps_changing_is_left_out_of_the_aborted_result`
`test_abort_exits_even_when_reporting_fails`
`test_outputs_that_cannot_be_serialized_are_reported_as_a_failure` | -| 4 | Regression: tasks whose `__init__` skips `super().__init__()` crash | `base_task.py` `__init__()`, line 24: the output context and `_abort_event` are created only here; `execute_task()` no longer creates the output context.
**Effect:** `AttributeError` in `set_output_property()`, `set_exit_code()`, `add_reporting_record()` (raised again from `execute_task()`'s error handler) and in `is_abort_requested()`. | `base_task.py`:
• [`_ensure_output_context()`, line 90](../digitalai/release/integration/base_task.py#L90), called by [`get_output_context()`, line 98](../digitalai/release/integration/base_task.py#L98)
• getters/setters use `get_output_context()`: [lines 109](../digitalai/release/integration/base_task.py#L109), [133](../digitalai/release/integration/base_task.py#L133), [139](../digitalai/release/integration/base_task.py#L139), [145](../digitalai/release/integration/base_task.py#L145), [163](../digitalai/release/integration/base_task.py#L163)
• [`is_abort_requested()`, line 78](../digitalai/release/integration/base_task.py#L78) and [`_request_abort()`, line 86](../digitalai/release/integration/base_task.py#L86) | The output context is created on first use if it is missing. `get_output_properties()`, `set_output_property()`, `set_exit_code()`, `set_error_message()` and `add_reporting_record()` go through `get_output_context()`. `is_abort_requested()` returns `False` and `_request_abort()` creates the event when `_abort_event` is missing. `__init__()` is unchanged. `execute_task()` creates the output context on the main thread before it starts the worker ([line 473](../digitalai/release/integration/wrapper.py#L473)); otherwise the worker's first setter and an `abort()` override run by the signal handler can each create one, and the worker's replaces the handler's, losing its cleanup outputs and message. | `test_task_without_super_init_runs_and_reports`
`test_task_without_super_init_can_be_aborted`
`test_task_without_super_init_has_its_output_context_before_it_runs` | -| 5a | An `abort()` override that sets no exit code is reported as succeeded | `wrapper.py` `abort_handler()`: reports the task's context as it is.
**Effect:** an override that does not call `super().abort()` reports exit code `0`, and Release records the aborted task as successful. | `wrapper.py`:
• [`_report_once(aborted=True)`, lines 195–197](../digitalai/release/integration/wrapper.py#L195-L197), called from [`abort_handler()`, line 267](../digitalai/release/integration/wrapper.py#L267) | On the abort path, an exit code of `0` in the reported copy becomes `1`, and an empty error message becomes "Abort requested". A message the task set itself is kept. | `test_abort_override_that_sets_no_exit_code_is_still_reported_as_failed` | -| 5b | `sys.exit(n)` is reported as exit code `0` | `wrapper.py` `execute_task()`, `TODO` at line 347: the `SystemExit` is re-raised so the process exits with `n`, but the reported context keeps exit code `0`.
**Effect:** Release records the task as successful. | `wrapper.py`:
• [`_exit_code_of()`, line 417](../digitalai/release/integration/wrapper.py#L417)
• [`execute_task()`, lines 479–485](../digitalai/release/integration/wrapper.py#L479-L485) | If the worker raised `SystemExit` with a non-zero status and the task set no exit code, that status is reported. `_exit_code_of()` maps `SystemExit.code` like Python does: `None` → `0`, an `int` → itself, anything else → `1`. The error message is the string passed to `sys.exit()`, or "Task exited with code N". The `SystemExit` is still re-raised. The `TODO` is removed. | `test_sys_exit_is_reported_and_propagated` (`int`, `message`, `zero` and `none` cases) | -| 6a | Output not flushed before exit | `wrapper.py` `abort_handler()`: `os._exit(1)` skips Python's stream flushing.
**Effect:** the last stdout/stderr lines, including the `##[start: comment]` markers Release reads, can be lost. | `wrapper.py`:
• [`_flush_std_streams()`, line 160](../digitalai/release/integration/wrapper.py#L160) and [`_exit_process()`, line 169](../digitalai/release/integration/wrapper.py#L169), used by `abort_handler()` and `_report_once()` | `sys.stdout` and `sys.stderr` are flushed before every `os._exit()`. | Covered by the real-`SIGABRT` end-to-end tests; live runs show the task's comments in Release. | -| 6b | Misleading message when a signal arrives before the task exists | `wrapper.py` `_report_once()`: reports "Task produced no output context". | `wrapper.py`:
• [`_snapshot_output_context()`, line 118](../digitalai/release/integration/wrapper.py#L118) with the message passed from [`abort_handler()`, line 267](../digitalai/release/integration/wrapper.py#L267) | The handler reports "Abort requested before the task started". The normal path keeps the old message. | `test_abort_before_a_task_exists_reports_a_clear_message` | +| 1 | `is_abort_requested()` has no effect: no grace period | `wrapper.py` `abort_handler()`, lines 131–148: sets the abort flag, reports and calls `os._exit(1)` immediately, without waiting for `execute()`.
**Effect:** outputs, reporting records and comments a task sets while stopping are lost. | `wrapper.py`:
• [`ABORT_GRACE_PERIOD_SECONDS`, line 58](../digitalai/release/integration/wrapper.py#L58)
• [`_worker`, line 101](../digitalai/release/integration/wrapper.py#L101), set in [`execute_task()`, line 485](../digitalai/release/integration/wrapper.py#L485)
• [`abort_handler()` wait, line 276](../digitalai/release/integration/wrapper.py#L276) | After calling `abort()`, the handler waits up to `ABORT_GRACE_PERIOD_SECONDS` (environment variable, default `3`) for the worker thread with `worker.join(timeout=…)`, then reports. The wait is skipped if there is no worker or it has finished. A task that never stops is reported when the wait ends. | `test_cooperative_task_is_reported_with_what_it_recorded_while_stopping`
`test_task_that_ignores_the_abort_is_reported_after_the_grace_period`
`test_sigabrt_reports_what_a_cooperative_task_recorded_while_stopping` (real `SIGABRT`)
`test_sigabrt_reports_a_blocked_task_as_aborted_after_the_grace_period` (real `SIGABRT`) | +| 2 | A signal during a report destroys the result | `wrapper.py` `_report_once()`, line 98 (`_result_reported` flag) and `abort_handler()`: the handler runs on the main thread, can interrupt the main thread's own report, re-enters the `RLock`, sees the flag set and calls `os._exit(1)` mid-send. A second signal during an abort does the same.
**Effect:** Release receives no result. | `wrapper.py`:
• [report state, lines 85–97](../digitalai/release/integration/wrapper.py#L85-L97)
• [`_report_once()` state changes, lines 207 and 217–219](../digitalai/release/integration/wrapper.py#L207-L219)
• [`abort_handler()` guards, lines 246–253](../digitalai/release/integration/wrapper.py#L246-L253)
• [`_reset_report_guard()`, line 108](../digitalai/release/integration/wrapper.py#L108) | The boolean flag is replaced by `_report_state` (`pending` → `in_progress` → `done`) plus `_abort_in_progress` and `_exit_after_report`. If a signal arrives while a report is `in_progress`, the handler sets `_exit_after_report` and returns, so the send completes; `_report_once()` then exits. A signal while an abort is already being handled is ignored. `_reset_report_guard()` resets all of this for each daemon-mode execution. | `test_signal_during_a_report_lets_the_report_finish_then_exits`
`test_second_signal_during_an_abort_is_ignored`
`test_each_execution_reports_its_own_result` | +| 3 | The result is serialized while the task may still change it | `wrapper.py` `update_output_context()`, line 231: `json.dumps()` of the task's own output context, called from the handler while `execute()` may still add outputs on the worker thread. `json.dumps` is outside the `try`.
**Effect:** possible `RuntimeError: dictionary changed size during iteration`; the handler never reaches `os._exit`, and the task is reported as a generic failure without its outputs. | `wrapper.py`:
• [`_snapshot_output_context()`, lines 118–157](../digitalai/release/integration/wrapper.py#L118-L157), called from [`_report_once()`, line 190](../digitalai/release/integration/wrapper.py#L190)
• [`abort_handler()` exit, lines 282–286](../digitalai/release/integration/wrapper.py#L282-L286) | `_report_once()` reports a fully detached copy: `_snapshot_output_context()` serializes the context and builds a new `OutputContext` from the JSON, so the result shares no object, nested or not, with the task. If a nested output changes while it is read (`RuntimeError`), it is read again, up to 5 times; if that keeps failing, or an output cannot be serialized, the result is reported as failed without outputs and records. `abort_handler()` exits in a `finally`, so a failed report cannot keep the process running. With the grace period, the copy is normally taken after the worker has stopped. | `test_reported_result_is_a_snapshot_of_the_output_context`
`test_reported_result_shares_no_nested_object_with_the_task`
`test_output_changed_while_read_is_read_again`
`test_output_that_keeps_changing_is_left_out_of_the_aborted_result`
`test_abort_exits_even_when_reporting_fails`
`test_outputs_that_cannot_be_serialized_are_reported_as_a_failure` | +| 4 | Regression: tasks whose `__init__` skips `super().__init__()` crash | `base_task.py` `__init__()`, line 24: the output context and `_abort_event` are created only here; `execute_task()` no longer creates the output context.
**Effect:** `AttributeError` in `set_output_property()`, `set_exit_code()`, `add_reporting_record()` (raised again from `execute_task()`'s error handler) and in `is_abort_requested()`. | `base_task.py`:
• [`_ensure_output_context()`, line 90](../digitalai/release/integration/base_task.py#L90), called by [`get_output_context()`, line 98](../digitalai/release/integration/base_task.py#L98)
• getters/setters use `get_output_context()`: [lines 109](../digitalai/release/integration/base_task.py#L109), [133](../digitalai/release/integration/base_task.py#L133), [139](../digitalai/release/integration/base_task.py#L139), [145](../digitalai/release/integration/base_task.py#L145), [163](../digitalai/release/integration/base_task.py#L163)
• [`is_abort_requested()`, line 78](../digitalai/release/integration/base_task.py#L78) and [`_request_abort()`, line 86](../digitalai/release/integration/base_task.py#L86) | The output context is created on first use if it is missing. `get_output_properties()`, `set_output_property()`, `set_exit_code()`, `set_error_message()` and `add_reporting_record()` go through `get_output_context()`. `is_abort_requested()` returns `False` and `_request_abort()` creates the event when `_abort_event` is missing. `__init__()` is unchanged. `execute_task()` creates the output context on the main thread before it starts the worker ([line 489](../digitalai/release/integration/wrapper.py#L489)); otherwise the worker's first setter and an `abort()` override run by the signal handler can each create one, and the worker's replaces the handler's, losing its cleanup outputs and message. | `test_task_without_super_init_runs_and_reports`
`test_task_without_super_init_can_be_aborted`
`test_task_without_super_init_has_its_output_context_before_it_runs` | +| 5a | An `abort()` override that sets no exit code is reported as succeeded | `wrapper.py` `abort_handler()`: reports the task's context as it is.
**Effect:** an override that does not call `super().abort()` reports exit code `0`, and Release records the aborted task as successful. | `wrapper.py`:
• [`_report_once(aborted=True)`, lines 211–213](../digitalai/release/integration/wrapper.py#L211-L213), called from [`abort_handler()`, line 283](../digitalai/release/integration/wrapper.py#L283) | On the abort path, an exit code of `0` in the reported copy becomes `1`, and an empty error message becomes "Abort requested". A message the task set itself is kept. | `test_abort_override_that_sets_no_exit_code_is_still_reported_as_failed` | +| 5b | `sys.exit(n)` is reported as exit code `0` | `wrapper.py` `execute_task()`, `TODO` at line 347: the `SystemExit` is re-raised so the process exits with `n`, but the reported context keeps exit code `0`.
**Effect:** Release records the task as successful. | `wrapper.py`:
• [`_exit_code_of()`, line 433](../digitalai/release/integration/wrapper.py#L433)
• [`execute_task()`, lines 495–501](../digitalai/release/integration/wrapper.py#L495-L501) | If the worker raised `SystemExit` with a non-zero status and the task set no exit code, that status is reported. `_exit_code_of()` maps `SystemExit.code` like Python does: `None` → `0`, an `int` → itself, anything else → `1`. The error message is the string passed to `sys.exit()`, or "Task exited with code N". The `SystemExit` is still re-raised. The `TODO` is removed. | `test_sys_exit_is_reported_and_propagated` (`int`, `message`, `zero` and `none` cases) | +| 6a | Output not flushed before exit | `wrapper.py` `abort_handler()`: `os._exit(1)` skips Python's stream flushing.
**Effect:** the last stdout/stderr lines, including the `##[start: comment]` markers Release reads, can be lost. | `wrapper.py`:
• [`_flush_std_streams()`, lines 160–182](../digitalai/release/integration/wrapper.py#L160-L182) and [`_exit_process()`, line 185](../digitalai/release/integration/wrapper.py#L185), used by `abort_handler()` and `_report_once()` | `sys.stdout` and `sys.stderr` are flushed before every `os._exit()`. The flush runs on a separate thread and is waited for at most `FLUSH_TIMEOUT_SECONDS` (1 s): with buffered streams (no `PYTHONUNBUFFERED`), a task blocked writing to a full stderr pipe that nobody reads holds the stream's lock, and an unbounded flush kept the process running, after reporting, until the runner force-killed it. | `test_exit_flushes_stdout_and_stderr_first`
`test_exit_does_not_wait_for_a_blocked_stream`
`test_sigabrt_exits_while_the_task_is_blocked_writing_to_stderr` (real `SIGABRT`)
Live runs show the task's comments in Release. | +| 6b | Misleading message when a signal arrives before the task exists | `wrapper.py` `_report_once()`: reports "Task produced no output context". | `wrapper.py`:
• [`_snapshot_output_context()`, line 118](../digitalai/release/integration/wrapper.py#L118) with the message passed from [`abort_handler()`, line 283](../digitalai/release/integration/wrapper.py#L283) | The handler reports "Abort requested before the task started". The normal path keeps the old message. | `test_abort_before_a_task_exists_reports_a_clear_message` | | 6c | Thread safety of `abort()` overrides undocumented | `base_task.py` `abort()` docstring and `docs/classes/base_task.md`: no mention that `abort()` runs on the main thread while `execute()` runs on the worker thread. | • [`base_task.py` `abort()` docstring, lines 62–67](../digitalai/release/integration/base_task.py#L62-L67)
• [`docs/classes/base_task.md`, `abort`](classes/base_task.md#abort) and [`is_abort_requested`](classes/base_task.md#is_abort_requested)
• [`docs/examples.md`](examples.md) | The docs state that an override must be thread-safe, that outputs set in it are reported, and that they must be set before `super().abort()` because it raises. They also describe the grace period and its relation to `abortTimeout`. | Documentation only. | | 6d | Abort feature missing from the changelog | `README.md`. | [`README.md`, "Unreleased"](../README.md#unreleased) | New "Unreleased" section listing the abort feature, the grace period and the fixes. | Documentation only. | @@ -266,7 +266,10 @@ worker, so both threads use the same object. ### Smaller issues (issue 6) - `_exit_process()` flushes `sys.stdout` and `sys.stderr` before `os._exit()`, both in the - handler and in the exit-after-report path. + handler and in the exit-after-report path. It waits for the flush at most + `FLUSH_TIMEOUT_SECONDS` (1 s), so a task blocked writing to a full, unread stderr pipe, which + holds the stream's lock when stdio is buffered, cannot keep the process from exiting. Nothing + is logged on a timeout, since the logger writes to stdout, which may be the blocked stream. - A signal before the task object exists is reported as "Abort requested before the task started". - The `abort()` docstring and `docs/classes/base_task.md` state that an override must be @@ -284,7 +287,7 @@ worker, so both threads use the same object. | Report state | `_result_reported` replaced by `_report_state` (`_REPORT_PENDING`, `_REPORT_IN_PROGRESS`, `_REPORT_DONE`), plus `_abort_in_progress`, `_exit_after_report` and `_worker`. The comment above them explains why a signal can arrive during a report. | | `_reset_report_guard()` | Resets all four values, so each daemon-mode execution starts clean. | | `_snapshot_output_context()` | New. Returns a fully detached copy of the task's output context (serialized and rebuilt, retried up to `_SNAPSHOT_ATTEMPTS` times), a copy without outputs if that fails, or a failure context with the given message when there is no task. | -| `_flush_std_streams()`, `_exit_process()` | New. Flush stdout/stderr, then `os._exit()`. | +| `_flush_std_streams()`, `_exit_process()` | New. Flush stdout/stderr, waiting at most `FLUSH_TIMEOUT_SECONDS`, then `os._exit()`. | | `_report_once()` | New parameters `aborted` and `no_task_message`. Moves `pending` → `in_progress` → `done`, reports the copy, forces a non-zero exit code when `aborted`, and exits afterwards if `_exit_after_report` is set. | | `abort_handler()` | Ignores repeat signals, defers to an in-progress report, waits up to the grace period for the worker, reports with `aborted=True`, exits through `_exit_process()`, also when reporting fails. The docstring describes the three steps. The `global dai_task_object` statement is removed; the handler reads it into a local. | | `_exit_code_of()` | New. Maps `SystemExit.code` to the process exit status. | @@ -313,7 +316,7 @@ names the tests for each issue. The abort behaviour is covered by: |------|----------------| | `tests/release/integration/test_wrapper_abort.py` | `abort_handler()` called in-process with `update_output_context()` and `os._exit()` replaced: the grace period, a cooperative stop, a task that ignores the abort, a signal during a report, a repeat signal, `abort()` overrides (no exit code, own message, raising), tasks without `super().__init__()`, and the detached snapshot, including a nested output that changes while it is read and a report that fails. | | `tests/release/integration/test_wrapper_execute_task.py` | The worker thread, one report per execution, `AbortException`, `sys.exit()` codes and messages, and outputs that cannot be serialized. | -| `tests/release/integration/test_wrapper_e2e.py` | The real wrapper in a subprocess with a real `SIGABRT` (skipped on Windows, which cannot deliver it to a child process). It waits for the task to start before signalling, and uses a 2 s grace period. | +| `tests/release/integration/test_wrapper_e2e.py` | The real wrapper in a subprocess with a real `SIGABRT` (skipped on Windows, which cannot deliver it to a child process). It waits for the task to start before signalling, and uses a 2 s grace period. One test aborts a task blocked writing to a full stderr pipe that is never read, with buffered stdio, and checks that the process still exits. | | `tests/release/integration/tasks/` | The tasks those subprocess runs load: `BlockingTask` ignores the abort; `CooperativeTask` loops until `is_abort_requested()`, then sets `stoppedCooperatively` and adds a reporting record with the number of loop passes. | ### `docs/classes/base_task.md` @@ -352,8 +355,8 @@ later. Set `ABORT_GRACE_PERIOD_SECONDS=0` to disable the wait. | Level | Result | |-------|--------| -| Test suite, Windows, with a live Release server and Docker Desktop Kubernetes | 360 passed, 2 skipped (the real-`SIGABRT` tests), 4 xfailed (known secret-masking bugs, unrelated to abort). | -| Test suite, Linux (Python 3.10 container, as in CI) | 323 passed, 7 skipped (the live Release and Kubernetes tests, with no servers in the container), 4 xfailed; both real-`SIGABRT` tests pass. Three consecutive runs. | +| Test suite, Windows, with a live Release server and Docker Desktop Kubernetes | 362 passed, 3 skipped (the real-`SIGABRT` tests), 4 xfailed (known secret-masking bugs, unrelated to abort). | +| Test suite, Linux (Python 3.10 container, as in CI) | 326 passed, 7 skipped (the live Release and Kubernetes tests, with no servers in the container), 4 xfailed; all three real-`SIGABRT` tests pass. Three consecutive runs. | | Real wrapper, real `SIGABRT`, Linux container | Template example tasks: cooperative stop, checkpoint `abort()`, `abort()` override cleanup, and a task that ignores the abort all report exit code `1` with their outputs. The ignoring task exits 3.0 s after the signal. | | Live Release, Docker runner | All six template abort examples behave as above. Release records the outputs and the task's comments; results arrive 0.1–3.1 s after the abort. | | Live Release, Kubernetes runner (Docker Desktop) | Same results. The runner log shows `SIGABRT` delivered via `exec /bin/kill -6 1` for every aborted pod, with no fallback to pod deletion. Results arrive 0.3–3.2 s after the abort. | diff --git a/tests/release/integration/tasks/stderr_flood_task.py b/tests/release/integration/tasks/stderr_flood_task.py new file mode 100644 index 0000000..16f52ff --- /dev/null +++ b/tests/release/integration/tasks/stderr_flood_task.py @@ -0,0 +1,17 @@ +import sys +from pathlib import Path + +from digitalai.release.integration import BaseTask + + +class StderrFloodTask(BaseTask): + """ + Writes to stderr without end and ignores the abort. With a stderr pipe that + nobody reads, the write blocks once the pipe is full, holding the stream's + lock. Creates the ``task_started`` file once execute() runs. + """ + + def execute(self) -> None: + Path("task_started").touch() + while True: + sys.stderr.write("x" * 1024 + "\n") diff --git a/tests/release/integration/test_wrapper_abort.py b/tests/release/integration/test_wrapper_abort.py index f1d39b2..fa075d5 100644 --- a/tests/release/integration/test_wrapper_abort.py +++ b/tests/release/integration/test_wrapper_abort.py @@ -6,6 +6,7 @@ the test run. Real signals are covered by test_wrapper_e2e.py. """ import signal +import sys import threading import time from unittest import mock @@ -209,6 +210,48 @@ def test_abort_exits_even_when_reporting_fails(isolated_wrapper, exit_codes, mon assert exit_codes == [1] +class RecordingStream: + """Stands in for sys.stdout/sys.stderr; flush() blocks until ``unblock`` is set, if given.""" + + def __init__(self, unblock=None): + self.unblock = unblock + self.flushed = False + + def write(self, text): + return len(text) + + def flush(self): + if self.unblock is not None: + self.unblock.wait(10) + self.flushed = True + + +def test_exit_flushes_stdout_and_stderr_first(isolated_wrapper, exit_codes, monkeypatch): + stdout, stderr = RecordingStream(), RecordingStream() + monkeypatch.setattr(sys, "stdout", stdout) + monkeypatch.setattr(sys, "stderr", stderr) + + isolated_wrapper._exit_process(1) + + assert stdout.flushed and stderr.flushed + assert exit_codes == [1] + + +def test_exit_does_not_wait_for_a_blocked_stream(isolated_wrapper, exit_codes, monkeypatch): + monkeypatch.setattr(isolated_wrapper, "FLUSH_TIMEOUT_SECONDS", 0.2) + unblock = threading.Event() + monkeypatch.setattr(sys, "stderr", RecordingStream(unblock)) + try: + start = time.monotonic() + isolated_wrapper._exit_process(1) + elapsed = time.monotonic() - start + finally: + unblock.set() + + assert exit_codes == [1] + assert elapsed < 5 + + def test_abort_override_that_sets_no_exit_code_is_still_reported_as_failed(isolated_wrapper, reported, exit_codes): class QuietAbortTask(BaseTask): def execute(self) -> None: diff --git a/tests/release/integration/test_wrapper_e2e.py b/tests/release/integration/test_wrapper_e2e.py index 922d8e5..c92e938 100644 --- a/tests/release/integration/test_wrapper_e2e.py +++ b/tests/release/integration/test_wrapper_e2e.py @@ -108,10 +108,8 @@ def test_logged_secrets_are_masked(secret_task_stdout, channel): GRACE_PERIOD = 2.0 -def run_and_abort(workspace, task_type): - """Runs task_type, sends SIGABRT once execute() has started; returns (seconds to exit, output).""" - process = start_wrapper(workspace, input_context(task_type, []), - ABORT_GRACE_PERIOD_SECONDS=str(GRACE_PERIOD)) +def wait_until_started(workspace, process): + """Waits for the fixture task's ``task_started`` file, failing if the wrapper exits or 30s pass.""" marker = workspace / "task_started" deadline = time.monotonic() + 30 while not marker.exists(): @@ -120,6 +118,13 @@ def run_and_abort(workspace, task_type): pytest.fail(f"task did not start\nstdout:\n{stdout}\nstderr:\n{stderr}") time.sleep(0.05) + +def run_and_abort(workspace, task_type): + """Runs task_type, sends SIGABRT once execute() has started; returns (seconds to exit, output).""" + process = start_wrapper(workspace, input_context(task_type, []), + ABORT_GRACE_PERIOD_SECONDS=str(GRACE_PERIOD)) + wait_until_started(workspace, process) + start = time.monotonic() process.send_signal(signal.SIGABRT) returncode, stdout, stderr = finish(process, timeout=15) @@ -147,3 +152,35 @@ def test_sigabrt_reports_what_a_cooperative_task_recorded_while_stopping(task_wo assert output["outputProperties"] == {"stoppedCooperatively": True} (record,) = output["reportingRecords"] assert record["processed"] > 0 + + +@posix_only +def test_sigabrt_exits_while_the_task_is_blocked_writing_to_stderr(task_workspace): + (task_workspace / "input.json").write_text(json.dumps(input_context("containerExamples.StderrFloodTask", []))) + env = wrapper_env(ABORT_GRACE_PERIOD_SECONDS=str(GRACE_PERIOD)) + del env["PYTHONUNBUFFERED"] # buffered streams have the lock a blocked write holds + read_end, write_end = os.pipe() # stderr goes here and is never read, so it fills up + try: + with open(task_workspace / "stdout.log", "w") as stdout: + process = subprocess.Popen([sys.executable, "-m", "digitalai.release.integration.wrapper"], + cwd=task_workspace, env=env, stdout=stdout, stderr=write_end) + os.close(write_end) + wait_until_started(task_workspace, process) + time.sleep(1) # the pipe fills within milliseconds; then the task's write blocks + + start = time.monotonic() + process.send_signal(signal.SIGABRT) + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + pytest.fail("wrapper did not exit after SIGABRT while the task was blocked writing to stderr") + elapsed = time.monotonic() - start + finally: + os.close(read_end) + + assert process.returncode == 1 + assert elapsed < GRACE_PERIOD + 5 + assert read_output(task_workspace) == {"exitCode": 1, "jobErrorMessage": "Abort requested", + "outputProperties": {}, "reportingRecords": []} diff --git a/tests/release/integration/test_wrapper_k8s_live.py b/tests/release/integration/test_wrapper_k8s_live.py index 88a5efe..0615d9c 100644 --- a/tests/release/integration/test_wrapper_k8s_live.py +++ b/tests/release/integration/test_wrapper_k8s_live.py @@ -53,7 +53,9 @@ def kubeconfig_path() -> str: def write_single_context_kubeconfig(source: str, context: str, target) -> None: """ - Copies just ``context`` (with its cluster and user) into a kubeconfig of its own. + Copies just ``context`` (with its cluster and user, if it has one) into a + kubeconfig of its own. A context may have no user, e.g. for a local + ``kubectl proxy``, which needs no credentials. Relative certificate, key and token file paths resolve against the kubeconfig's own directory, so they are made absolute for the copy. @@ -62,15 +64,17 @@ def write_single_context_kubeconfig(source: str, context: str, target) -> None: full = yaml.safe_load(f) ctx = next(c for c in full["contexts"] if c["name"] == context) cluster = next(c for c in full["clusters"] if c["name"] == ctx["context"]["cluster"]) - user = next(u for u in full["users"] if u["name"] == ctx["context"]["user"]) + user_name = ctx["context"].get("user") + users = [u for u in full.get("users") or [] if user_name and u["name"] == user_name] source_dir = os.path.dirname(os.path.abspath(source)) - for entry, keys in ((cluster["cluster"], ("certificate-authority",)), - (user["user"], ("client-certificate", "client-key", "tokenFile"))): + path_fields = [(cluster["cluster"], ("certificate-authority",))] + path_fields += [(u.get("user") or {}, ("client-certificate", "client-key", "tokenFile")) for u in users] + for entry, keys in path_fields: for key in keys: if entry.get(key): entry[key] = os.path.join(source_dir, os.path.expanduser(entry[key])) target.write_text(yaml.safe_dump({"apiVersion": "v1", "kind": "Config", "current-context": context, - "contexts": [ctx], "clusters": [cluster], "users": [user]})) + "contexts": [ctx], "clusters": [cluster], "users": users})) @pytest.fixture(scope="module") From 5954d51918c40352b614b7eb548f3e70167e1ba5 Mon Sep 17 00:00:00 2001 From: Balaji Venkatesan Date: Thu, 29 Oct 2026 11:58:13 +0530 Subject: [PATCH 08/10] Fix for review comments --- README.md | 1 + digitalai/release/integration/api_base_task.py | 7 ++++--- digitalai/release/integration/wrapper.py | 3 ++- docs/abort-changes.md | 7 ++++--- tests/release/integration/test_api_base_task.py | 13 +++++++++++++ .../release/integration/test_api_base_task_live.py | 4 ++++ .../integration/test_wrapper_execute_task.py | 13 ++++++++++--- 7 files changed, 38 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ea07d3a..7576d84 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,7 @@ class ShowVersion(ApiBaseTask): - A signal received while the task result is being reported no longer cuts off the report; repeat signals during an abort are ignored. - An aborted result is never reported with exit code 0, and a task calling `sys.exit(n)` is reported with exit code `n`. - Tasks whose `__init__` does not call `super().__init__()` work again. +- `setReleaseVariable()`, `setFolderVariable()` and `setGlobalVariable()` accept a list for an existing list variable (`xlrelease.ListStringVariable`); it was rejected with a `TypeError` as if the variable held a string. ### Version 26.3.1 diff --git a/digitalai/release/integration/api_base_task.py b/digitalai/release/integration/api_base_task.py index 0ad8bcf..eb7db06 100644 --- a/digitalai/release/integration/api_base_task.py +++ b/digitalai/release/integration/api_base_task.py @@ -556,8 +556,9 @@ def _value_matches_type(vtype: str, value: Any) -> bool: The accepted Python types mirror :meth:`_variable_type_for_value`. ``vtype`` is matched by suffix so subtypes are treated as their base kind (e.g. ``PasswordStringVariable`` is a string). More specific - suffixes are tested first because ``SetStringVariable`` and - ``MapStringStringVariable`` also end in ``StringVariable``. A type the + suffixes are tested first because ``ListStringVariable``, + ``SetStringVariable`` and ``MapStringStringVariable`` also end in + ``StringVariable``. A type the SDK does not recognise returns ``True`` -- it is left for the server to validate rather than blocked here. """ @@ -571,7 +572,7 @@ def _value_matches_type(vtype: str, value: Any) -> bool: return isinstance(value, (date, str)) if vtype.endswith("MapStringStringVariable"): return isinstance(value, dict) - if vtype.endswith("SetStringVariable"): + if vtype.endswith(("ListStringVariable", "SetStringVariable")): return isinstance(value, (list, set, tuple)) if vtype.endswith("StringVariable"): return isinstance(value, str) diff --git a/digitalai/release/integration/wrapper.py b/digitalai/release/integration/wrapper.py index 5be5442..dc45cf9 100644 --- a/digitalai/release/integration/wrapper.py +++ b/digitalai/release/integration/wrapper.py @@ -436,7 +436,8 @@ def _exit_code_of(system_exit: SystemExit) -> int: if code is None: return 0 if isinstance(code, int): - return code + # int() turns a bool (an int subclass), as in sys.exit(not ok), into 0 or 1. + return int(code) return 1 diff --git a/docs/abort-changes.md b/docs/abort-changes.md index 31b54e3..e3024ca 100644 --- a/docs/abort-changes.md +++ b/docs/abort-changes.md @@ -32,12 +32,12 @@ Line numbers refer to `main` at `75990ee` (column "On `main`") and to `ReportRec | # | Issue | On `main`: where and effect | Fixed on `ReportRecords`: where | How it is fixed | Tests | |---|-------|-----------------------------|---------------------------------|-----------------|-------| -| 1 | `is_abort_requested()` has no effect: no grace period | `wrapper.py` `abort_handler()`, lines 131–148: sets the abort flag, reports and calls `os._exit(1)` immediately, without waiting for `execute()`.
**Effect:** outputs, reporting records and comments a task sets while stopping are lost. | `wrapper.py`:
• [`ABORT_GRACE_PERIOD_SECONDS`, line 58](../digitalai/release/integration/wrapper.py#L58)
• [`_worker`, line 101](../digitalai/release/integration/wrapper.py#L101), set in [`execute_task()`, line 485](../digitalai/release/integration/wrapper.py#L485)
• [`abort_handler()` wait, line 276](../digitalai/release/integration/wrapper.py#L276) | After calling `abort()`, the handler waits up to `ABORT_GRACE_PERIOD_SECONDS` (environment variable, default `3`) for the worker thread with `worker.join(timeout=…)`, then reports. The wait is skipped if there is no worker or it has finished. A task that never stops is reported when the wait ends. | `test_cooperative_task_is_reported_with_what_it_recorded_while_stopping`
`test_task_that_ignores_the_abort_is_reported_after_the_grace_period`
`test_sigabrt_reports_what_a_cooperative_task_recorded_while_stopping` (real `SIGABRT`)
`test_sigabrt_reports_a_blocked_task_as_aborted_after_the_grace_period` (real `SIGABRT`) | +| 1 | `is_abort_requested()` has no effect: no grace period | `wrapper.py` `abort_handler()`, lines 131–148: sets the abort flag, reports and calls `os._exit(1)` immediately, without waiting for `execute()`.
**Effect:** outputs, reporting records and comments a task sets while stopping are lost. | `wrapper.py`:
• [`ABORT_GRACE_PERIOD_SECONDS`, line 58](../digitalai/release/integration/wrapper.py#L58)
• [`_worker`, line 101](../digitalai/release/integration/wrapper.py#L101), set in [`execute_task()`, line 486](../digitalai/release/integration/wrapper.py#L486)
• [`abort_handler()` wait, line 276](../digitalai/release/integration/wrapper.py#L276) | After calling `abort()`, the handler waits up to `ABORT_GRACE_PERIOD_SECONDS` (environment variable, default `3`) for the worker thread with `worker.join(timeout=…)`, then reports. The wait is skipped if there is no worker or it has finished. A task that never stops is reported when the wait ends. | `test_cooperative_task_is_reported_with_what_it_recorded_while_stopping`
`test_task_that_ignores_the_abort_is_reported_after_the_grace_period`
`test_sigabrt_reports_what_a_cooperative_task_recorded_while_stopping` (real `SIGABRT`)
`test_sigabrt_reports_a_blocked_task_as_aborted_after_the_grace_period` (real `SIGABRT`) | | 2 | A signal during a report destroys the result | `wrapper.py` `_report_once()`, line 98 (`_result_reported` flag) and `abort_handler()`: the handler runs on the main thread, can interrupt the main thread's own report, re-enters the `RLock`, sees the flag set and calls `os._exit(1)` mid-send. A second signal during an abort does the same.
**Effect:** Release receives no result. | `wrapper.py`:
• [report state, lines 85–97](../digitalai/release/integration/wrapper.py#L85-L97)
• [`_report_once()` state changes, lines 207 and 217–219](../digitalai/release/integration/wrapper.py#L207-L219)
• [`abort_handler()` guards, lines 246–253](../digitalai/release/integration/wrapper.py#L246-L253)
• [`_reset_report_guard()`, line 108](../digitalai/release/integration/wrapper.py#L108) | The boolean flag is replaced by `_report_state` (`pending` → `in_progress` → `done`) plus `_abort_in_progress` and `_exit_after_report`. If a signal arrives while a report is `in_progress`, the handler sets `_exit_after_report` and returns, so the send completes; `_report_once()` then exits. A signal while an abort is already being handled is ignored. `_reset_report_guard()` resets all of this for each daemon-mode execution. | `test_signal_during_a_report_lets_the_report_finish_then_exits`
`test_second_signal_during_an_abort_is_ignored`
`test_each_execution_reports_its_own_result` | | 3 | The result is serialized while the task may still change it | `wrapper.py` `update_output_context()`, line 231: `json.dumps()` of the task's own output context, called from the handler while `execute()` may still add outputs on the worker thread. `json.dumps` is outside the `try`.
**Effect:** possible `RuntimeError: dictionary changed size during iteration`; the handler never reaches `os._exit`, and the task is reported as a generic failure without its outputs. | `wrapper.py`:
• [`_snapshot_output_context()`, lines 118–157](../digitalai/release/integration/wrapper.py#L118-L157), called from [`_report_once()`, line 190](../digitalai/release/integration/wrapper.py#L190)
• [`abort_handler()` exit, lines 282–286](../digitalai/release/integration/wrapper.py#L282-L286) | `_report_once()` reports a fully detached copy: `_snapshot_output_context()` serializes the context and builds a new `OutputContext` from the JSON, so the result shares no object, nested or not, with the task. If a nested output changes while it is read (`RuntimeError`), it is read again, up to 5 times; if that keeps failing, or an output cannot be serialized, the result is reported as failed without outputs and records. `abort_handler()` exits in a `finally`, so a failed report cannot keep the process running. With the grace period, the copy is normally taken after the worker has stopped. | `test_reported_result_is_a_snapshot_of_the_output_context`
`test_reported_result_shares_no_nested_object_with_the_task`
`test_output_changed_while_read_is_read_again`
`test_output_that_keeps_changing_is_left_out_of_the_aborted_result`
`test_abort_exits_even_when_reporting_fails`
`test_outputs_that_cannot_be_serialized_are_reported_as_a_failure` | -| 4 | Regression: tasks whose `__init__` skips `super().__init__()` crash | `base_task.py` `__init__()`, line 24: the output context and `_abort_event` are created only here; `execute_task()` no longer creates the output context.
**Effect:** `AttributeError` in `set_output_property()`, `set_exit_code()`, `add_reporting_record()` (raised again from `execute_task()`'s error handler) and in `is_abort_requested()`. | `base_task.py`:
• [`_ensure_output_context()`, line 90](../digitalai/release/integration/base_task.py#L90), called by [`get_output_context()`, line 98](../digitalai/release/integration/base_task.py#L98)
• getters/setters use `get_output_context()`: [lines 109](../digitalai/release/integration/base_task.py#L109), [133](../digitalai/release/integration/base_task.py#L133), [139](../digitalai/release/integration/base_task.py#L139), [145](../digitalai/release/integration/base_task.py#L145), [163](../digitalai/release/integration/base_task.py#L163)
• [`is_abort_requested()`, line 78](../digitalai/release/integration/base_task.py#L78) and [`_request_abort()`, line 86](../digitalai/release/integration/base_task.py#L86) | The output context is created on first use if it is missing. `get_output_properties()`, `set_output_property()`, `set_exit_code()`, `set_error_message()` and `add_reporting_record()` go through `get_output_context()`. `is_abort_requested()` returns `False` and `_request_abort()` creates the event when `_abort_event` is missing. `__init__()` is unchanged. `execute_task()` creates the output context on the main thread before it starts the worker ([line 489](../digitalai/release/integration/wrapper.py#L489)); otherwise the worker's first setter and an `abort()` override run by the signal handler can each create one, and the worker's replaces the handler's, losing its cleanup outputs and message. | `test_task_without_super_init_runs_and_reports`
`test_task_without_super_init_can_be_aborted`
`test_task_without_super_init_has_its_output_context_before_it_runs` | +| 4 | Regression: tasks whose `__init__` skips `super().__init__()` crash | `base_task.py` `__init__()`, line 24: the output context and `_abort_event` are created only here; `execute_task()` no longer creates the output context.
**Effect:** `AttributeError` in `set_output_property()`, `set_exit_code()`, `add_reporting_record()` (raised again from `execute_task()`'s error handler) and in `is_abort_requested()`. | `base_task.py`:
• [`_ensure_output_context()`, line 90](../digitalai/release/integration/base_task.py#L90), called by [`get_output_context()`, line 98](../digitalai/release/integration/base_task.py#L98)
• getters/setters use `get_output_context()`: [lines 109](../digitalai/release/integration/base_task.py#L109), [133](../digitalai/release/integration/base_task.py#L133), [139](../digitalai/release/integration/base_task.py#L139), [145](../digitalai/release/integration/base_task.py#L145), [163](../digitalai/release/integration/base_task.py#L163)
• [`is_abort_requested()`, line 78](../digitalai/release/integration/base_task.py#L78) and [`_request_abort()`, line 86](../digitalai/release/integration/base_task.py#L86) | The output context is created on first use if it is missing. `get_output_properties()`, `set_output_property()`, `set_exit_code()`, `set_error_message()` and `add_reporting_record()` go through `get_output_context()`. `is_abort_requested()` returns `False` and `_request_abort()` creates the event when `_abort_event` is missing. `__init__()` is unchanged. `execute_task()` creates the output context on the main thread before it starts the worker ([line 490](../digitalai/release/integration/wrapper.py#L490)); otherwise the worker's first setter and an `abort()` override run by the signal handler can each create one, and the worker's replaces the handler's, losing its cleanup outputs and message. | `test_task_without_super_init_runs_and_reports`
`test_task_without_super_init_can_be_aborted`
`test_task_without_super_init_has_its_output_context_before_it_runs` | | 5a | An `abort()` override that sets no exit code is reported as succeeded | `wrapper.py` `abort_handler()`: reports the task's context as it is.
**Effect:** an override that does not call `super().abort()` reports exit code `0`, and Release records the aborted task as successful. | `wrapper.py`:
• [`_report_once(aborted=True)`, lines 211–213](../digitalai/release/integration/wrapper.py#L211-L213), called from [`abort_handler()`, line 283](../digitalai/release/integration/wrapper.py#L283) | On the abort path, an exit code of `0` in the reported copy becomes `1`, and an empty error message becomes "Abort requested". A message the task set itself is kept. | `test_abort_override_that_sets_no_exit_code_is_still_reported_as_failed` | -| 5b | `sys.exit(n)` is reported as exit code `0` | `wrapper.py` `execute_task()`, `TODO` at line 347: the `SystemExit` is re-raised so the process exits with `n`, but the reported context keeps exit code `0`.
**Effect:** Release records the task as successful. | `wrapper.py`:
• [`_exit_code_of()`, line 433](../digitalai/release/integration/wrapper.py#L433)
• [`execute_task()`, lines 495–501](../digitalai/release/integration/wrapper.py#L495-L501) | If the worker raised `SystemExit` with a non-zero status and the task set no exit code, that status is reported. `_exit_code_of()` maps `SystemExit.code` like Python does: `None` → `0`, an `int` → itself, anything else → `1`. The error message is the string passed to `sys.exit()`, or "Task exited with code N". The `SystemExit` is still re-raised. The `TODO` is removed. | `test_sys_exit_is_reported_and_propagated` (`int`, `message`, `zero` and `none` cases) | +| 5b | `sys.exit(n)` is reported as exit code `0` | `wrapper.py` `execute_task()`, `TODO` at line 347: the `SystemExit` is re-raised so the process exits with `n`, but the reported context keeps exit code `0`.
**Effect:** Release records the task as successful. | `wrapper.py`:
• [`_exit_code_of()`, line 433](../digitalai/release/integration/wrapper.py#L433)
• [`execute_task()`, lines 496–502](../digitalai/release/integration/wrapper.py#L496-L502) | If the worker raised `SystemExit` with a non-zero status and the task set no exit code, that status is reported. `_exit_code_of()` maps `SystemExit.code` like Python does: `None` → `0`, an `int` → itself (a `bool` → `0` or `1`, so the result holds a number, not `true`), anything else → `1`. The error message is the string passed to `sys.exit()`, or "Task exited with code N". The `SystemExit` is still re-raised. The `TODO` is removed. | `test_sys_exit_is_reported_and_propagated` (`int`, `message`, `zero`, `none`, `true` and `false` cases) | | 6a | Output not flushed before exit | `wrapper.py` `abort_handler()`: `os._exit(1)` skips Python's stream flushing.
**Effect:** the last stdout/stderr lines, including the `##[start: comment]` markers Release reads, can be lost. | `wrapper.py`:
• [`_flush_std_streams()`, lines 160–182](../digitalai/release/integration/wrapper.py#L160-L182) and [`_exit_process()`, line 185](../digitalai/release/integration/wrapper.py#L185), used by `abort_handler()` and `_report_once()` | `sys.stdout` and `sys.stderr` are flushed before every `os._exit()`. The flush runs on a separate thread and is waited for at most `FLUSH_TIMEOUT_SECONDS` (1 s): with buffered streams (no `PYTHONUNBUFFERED`), a task blocked writing to a full stderr pipe that nobody reads holds the stream's lock, and an unbounded flush kept the process running, after reporting, until the runner force-killed it. | `test_exit_flushes_stdout_and_stderr_first`
`test_exit_does_not_wait_for_a_blocked_stream`
`test_sigabrt_exits_while_the_task_is_blocked_writing_to_stderr` (real `SIGABRT`)
Live runs show the task's comments in Release. | | 6b | Misleading message when a signal arrives before the task exists | `wrapper.py` `_report_once()`: reports "Task produced no output context". | `wrapper.py`:
• [`_snapshot_output_context()`, line 118](../digitalai/release/integration/wrapper.py#L118) with the message passed from [`abort_handler()`, line 283](../digitalai/release/integration/wrapper.py#L283) | The handler reports "Abort requested before the task started". The normal path keeps the old message. | `test_abort_before_a_task_exists_reports_a_clear_message` | | 6c | Thread safety of `abort()` overrides undocumented | `base_task.py` `abort()` docstring and `docs/classes/base_task.md`: no mention that `abort()` runs on the main thread while `execute()` runs on the worker thread. | • [`base_task.py` `abort()` docstring, lines 62–67](../digitalai/release/integration/base_task.py#L62-L67)
• [`docs/classes/base_task.md`, `abort`](classes/base_task.md#abort) and [`is_abort_requested`](classes/base_task.md#is_abort_requested)
• [`docs/examples.md`](examples.md) | The docs state that an override must be thread-safe, that outputs set in it are reported, and that they must be set before `super().abort()` because it raises. They also describe the grace period and its relation to `abortTimeout`. | Documentation only. | @@ -259,6 +259,7 @@ worker, so both threads use the same object. - `execute_task()`: if the worker raised `SystemExit` with a non-zero status and the task did not set an exit code, that status is reported. The new helper `_exit_code_of()` maps `SystemExit.code` the way Python does: `None` → `0`, an `int` → itself, anything else → `1`. + A `bool`, as in `sys.exit(not ok)`, becomes `0` or `1`, so the result holds a number, not `true`. The error message is the string passed to `sys.exit()`, or "Task exited with code N". `sys.exit(0)` still reports success. The `SystemExit` is still re-raised after reporting, so the process exit status is unchanged. diff --git a/tests/release/integration/test_api_base_task.py b/tests/release/integration/test_api_base_task.py index aa309ed..f64b4f7 100644 --- a/tests/release/integration/test_api_base_task.py +++ b/tests/release/integration/test_api_base_task.py @@ -354,6 +354,16 @@ def test_folder_and_global_helpers_require_the_scope_prefix(task, apis, helper, apis.configuration.getGlobalVariables.assert_not_called() +def test_set_release_variable_updates_a_list_variable_with_a_list(task, apis): + existing = variable("trackedItems", [RELEASE_ID], "xlrelease.ListStringVariable") + stored(apis, SCOPES["release"], existing) + + task.setReleaseVariable("trackedItems", [RELEASE_ID, "Applications/Release2"]) + + apis.release.updateVariable.assert_called_once_with(existing.id, existing) + assert existing.value == [RELEASE_ID, "Applications/Release2"] + + def test_set_release_variable_sends_a_datetime_as_iso_8601(task, apis): existing = variable("when", None, "xlrelease.DateVariable") stored(apis, SCOPES["release"], existing) @@ -388,6 +398,8 @@ def test_new_variable_type_is_inferred_from_the_value(value, expected): ("xlrelease.DateVariable", "2026-01-01T00:00:00Z"), ("xlrelease.MapStringStringVariable", {"k": "v"}), ("xlrelease.SetStringVariable", ("a",)), + ("xlrelease.ListStringVariable", ["a", "a"]), + ("xlrelease.ListStringVariable", ("a",)), ("xlrelease.StringVariable", "text"), ("xlrelease.PasswordStringVariable", "secret"), ("custom.UnknownVariable", object()), @@ -403,6 +415,7 @@ def test_value_matching_the_variable_type_is_accepted(vtype, value): ("xlrelease.DateVariable", 12345), ("xlrelease.MapStringStringVariable", [("k", "v")]), ("xlrelease.SetStringVariable", "a"), + ("xlrelease.ListStringVariable", "a"), ("xlrelease.StringVariable", 42), ("xlrelease.PasswordStringVariable", None), ]) diff --git a/tests/release/integration/test_api_base_task_live.py b/tests/release/integration/test_api_base_task_live.py index 353604f..b5c7356 100644 --- a/tests/release/integration/test_api_base_task_live.py +++ b/tests/release/integration/test_api_base_task_live.py @@ -87,11 +87,13 @@ def new_variable(vtype, key, value=None, **extra): RELEASE_VARIABLES = { "relVar": ("xlrelease.StringVariable", "release-value"), "relSetVar": ("xlrelease.SetStringVariable", ["Apples", "Pears"]), + "relListVar": ("xlrelease.ListStringVariable", ["second", "first"]), "relMapVar": ("xlrelease.MapStringStringVariable", {"env": "dev", "region": "us"}), "relDateVar": ("xlrelease.DateVariable", "2026-01-02T03:04:05+00:00"), # Written by the setter tests; never read by the getter tests. "relVarRw": ("xlrelease.StringVariable", "initial"), "relSetVarRw": ("xlrelease.SetStringVariable", ["initial"]), + "relListVarRw": ("xlrelease.ListStringVariable", ["initial"]), "relMapVarRw": ("xlrelease.MapStringStringVariable", {"initial": "yes"}), "relDateVarRw": ("xlrelease.DateVariable", "2026-01-02T03:04:05+00:00"), } @@ -242,6 +244,7 @@ def test_get_current_folder(live): @pytest.mark.parametrize("key, expected", [ ("relVar", "release-value"), + ("relListVar", ["second", "first"]), ("relMapVar", {"env": "dev", "region": "us"}), ]) def test_get_release_variable(live, key, expected): @@ -271,6 +274,7 @@ def test_get_release_variable_raises_for_an_unknown_name(live): @pytest.mark.parametrize("key, value", [ ("relVarRw", "updated"), + ("relListVarRw", ["b", "a", "b"]), # a list keeps its order and duplicates ("relMapVarRw", {"env": "prod", "tier": "gold"}), ]) def test_set_release_variable_persists_the_value(live, key, value): diff --git a/tests/release/integration/test_wrapper_execute_task.py b/tests/release/integration/test_wrapper_execute_task.py index a90c1a1..1961539 100644 --- a/tests/release/integration/test_wrapper_execute_task.py +++ b/tests/release/integration/test_wrapper_execute_task.py @@ -98,13 +98,17 @@ def execute(self) -> None: ("Configuration missing", 1, "Configuration missing"), (0, 0, ""), (None, 0, ""), -], ids=["int", "message", "zero", "none"]) + (True, 1, "Task exited with code 1"), + (False, 0, ""), +], ids=["int", "message", "zero", "none", "true", "false"]) def test_sys_exit_is_reported_and_propagated(isolated_wrapper, reported, code, reported_code, message): with pytest.raises(SystemExit) as exit_info: isolated_wrapper.execute_task(ExitingTask(code)) assert exit_info.value.code == code assert reported == [OutputContext(reported_code, message, {}, [])] + # True == 1, so also check the type: the result format needs a number, not a JSON boolean. + assert type(reported[0].exit_code) is int def test_sys_exit_keeps_an_exit_code_the_task_set_itself(isolated_wrapper, reported): @@ -181,6 +185,9 @@ def execute(self) -> None: assert reported == [OutputContext(0, "", {"stoppedOnRequest": True}, [])] -@pytest.mark.parametrize("code, expected", [(None, 0), (0, 0), (3, 3), ("message", 1)]) +@pytest.mark.parametrize("code, expected", [(None, 0), (0, 0), (3, 3), ("message", 1), (True, 1), (False, 0)]) def test_exit_code_of_matches_the_process_exit_status(isolated_wrapper, code, expected): - assert isolated_wrapper._exit_code_of(SystemExit(code)) == expected + exit_code = isolated_wrapper._exit_code_of(SystemExit(code)) + + assert exit_code == expected + assert type(exit_code) is int From 6a8bf3cac15393ab3d516275df8bde340ee07fd7 Mon Sep 17 00:00:00 2001 From: Balaji Venkatesan Date: Thu, 29 Oct 2026 13:02:28 +0530 Subject: [PATCH 09/10] Updated for udm.AiUsageRecord --- README.md | 1 + digitalai/release/integration/__init__.py | 2 +- .../release/integration/reporting_records.py | 37 +++++++++++++++++++ .../integration/test_reporting_records.py | 37 +++++++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7576d84..19f1efc 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ class ShowVersion(ApiBaseTask): - Tasks can be aborted: the SDK runs `execute()` on a worker thread and handles `SIGABRT`/`SIGTERM` by calling the task's `abort()` hook. Long-running tasks can poll the new `is_abort_requested()` to stop cooperatively. - After an abort is requested, the SDK waits up to `ABORT_GRACE_PERIOD_SECONDS` (default 3) for `execute()` to return, so the outputs and reporting records a task sets on its way out are reported. +- New `AiUsageRecord` reporting record (`udm.AiUsageRecord`) for one AI agent execution: LLM provider, model, agent, termination reason, iterations, tool calls, token counts and estimated cost. Add it with `add_reporting_record()` like the other records, but only on a Release version that defines `udm.AiUsageRecord`: an older server cannot read the task's result and fails the task. #### 🐛 Fixes diff --git a/digitalai/release/integration/__init__.py b/digitalai/release/integration/__init__.py index dd7a247..747bc58 100644 --- a/digitalai/release/integration/__init__.py +++ b/digitalai/release/integration/__init__.py @@ -3,5 +3,5 @@ from .input_context import InputContext from .output_context import OutputContext from .exceptions import AbortException -from .reporting_records import BuildRecord, PlanRecord, ItsmRecord,CodeComplianceRecord, DeploymentRecord +from .reporting_records import BuildRecord, PlanRecord, ItsmRecord, CodeComplianceRecord, DeploymentRecord, AiUsageRecord from .logger import dai_logger \ No newline at end of file diff --git a/digitalai/release/integration/reporting_records.py b/digitalai/release/integration/reporting_records.py index 034a4af..436f077 100644 --- a/digitalai/release/integration/reporting_records.py +++ b/digitalai/release/integration/reporting_records.py @@ -106,3 +106,40 @@ class DeploymentRecord: environment_name: Optional[str] = None server_url: Optional[str] = None server_user: Optional[str] = None + + +def _decimal_string(value) -> Optional[str]: + """Sends a cost as the decimal string the server stores, e.g. Decimal("0.0123") -> "0.0123".""" + return None if value is None else str(value) + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class AiUsageRecord: + """ + AiUsageRecord is a data model class which represents one AI agent execution attempt of a task: + the language model provider and model, the agent, why it stopped, and its token usage and cost. + + The record's start and end times are taken from the task. The provider's endpoint and user go in + server_url and server_user. estimated_cost is a decimal string; pass a str or a Decimal, since a + float can carry rounding noise (str(0.1 + 0.2) is "0.30000000000000004"). + + Only add it on a Release version that defines udm.AiUsageRecord: an older server cannot read + the task's result and fails the task, without its output properties. + """ + target_id: str + llm_provider: Optional[str] = None + agent: Optional[str] = None + model: Optional[str] = None + termination_reason: Optional[str] = None + iterations: Optional[int] = None + tool_calls: Optional[int] = None + input_tokens: Optional[int] = None + output_tokens: Optional[int] = None + total_tokens: Optional[int] = None + estimated_cost: Optional[str] = field(default=None, metadata=config(encoder=_decimal_string)) + currency: str = "USD" + type: str = "udm.AiUsageRecord" + id: str = "" + server_url: Optional[str] = None + server_user: Optional[str] = None diff --git a/tests/release/integration/test_reporting_records.py b/tests/release/integration/test_reporting_records.py index 2b90c09..99f659d 100644 --- a/tests/release/integration/test_reporting_records.py +++ b/tests/release/integration/test_reporting_records.py @@ -4,10 +4,12 @@ """ import json from datetime import datetime, timezone +from decimal import Decimal import pytest from digitalai.release.integration import ( + AiUsageRecord, BuildRecord, CodeComplianceRecord, DeploymentRecord, @@ -55,6 +57,18 @@ "version": "1.0", "type": "udm.DeploymentRecord", "id": "", "applicationName": "app", "environmentName": "prod", "serverUrl": None, "serverUser": None}, ), + # Field names as udm.AiUsageRecord declares them in xl-release's AI reporting module. + "ai-usage": ( + AiUsageRecord(target_id="T1", llm_provider="OpenAI", agent="Release agent", model="gpt-4o", + termination_reason="COMPLETED", iterations=3, tool_calls=5, input_tokens=1200, + output_tokens=300, total_tokens=1500, estimated_cost="0.0123", currency="EUR", + server_url="https://api.openai.com/v1", server_user="svc-release"), + {"targetId": "T1", "llmProvider": "OpenAI", "agent": "Release agent", "model": "gpt-4o", + "terminationReason": "COMPLETED", "iterations": 3, "toolCalls": 5, "inputTokens": 1200, + "outputTokens": 300, "totalTokens": 1500, "estimatedCost": "0.0123", "currency": "EUR", + "type": "udm.AiUsageRecord", "id": "", "serverUrl": "https://api.openai.com/v1", + "serverUser": "svc-release"}, + ), } @@ -78,3 +92,26 @@ def test_output_context_serializes_camel_case_with_records_as_json(): "outputProperties": {"greeting": "Hello"}, "reportingRecords": [expected_build, {"custom": 1}], } + + +def test_ai_usage_record_needs_only_the_task_and_defaults_the_currency_to_usd(): + assert AiUsageRecord(target_id="T1").to_dict() == { + "targetId": "T1", "llmProvider": None, "agent": None, "model": None, "terminationReason": None, + "iterations": None, "toolCalls": None, "inputTokens": None, "outputTokens": None, "totalTokens": None, + "estimatedCost": None, "currency": "USD", "type": "udm.AiUsageRecord", "id": "", "serverUrl": None, + "serverUser": None, + } + + +def test_ai_usage_record_sends_a_decimal_cost_as_its_exact_decimal_string(): + record = AiUsageRecord(target_id="T1", estimated_cost=Decimal("0.000150")) + + assert json.loads(json.dumps(record.to_dict()))["estimatedCost"] == "0.000150" + + +def test_ai_usage_record_with_a_decimal_cost_serializes_inside_the_task_result(): + record = AiUsageRecord(target_id="T1", total_tokens=1500, estimated_cost=Decimal("0.0123")) + + (written,) = json.loads(json.dumps(OutputContext(0, "", {}, [record]).to_dict()))["reportingRecords"] + + assert (written["type"], written["totalTokens"], written["estimatedCost"]) == ("udm.AiUsageRecord", 1500, "0.0123") From e0662f0a81845ad7eeea2b3e1999c499f14292f0 Mon Sep 17 00:00:00 2001 From: Balaji Venkatesan Date: Fri, 6 Nov 2026 11:23:44 +0530 Subject: [PATCH 10/10] Updated readme --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 19f1efc..49857d0 100644 --- a/README.md +++ b/README.md @@ -80,16 +80,16 @@ class ShowVersion(ApiBaseTask): #### 🚀 Features -- Tasks can be aborted: the SDK runs `execute()` on a worker thread and handles `SIGABRT`/`SIGTERM` by calling the task's `abort()` hook. Long-running tasks can poll the new `is_abort_requested()` to stop cooperatively. -- After an abort is requested, the SDK waits up to `ABORT_GRACE_PERIOD_SECONDS` (default 3) for `execute()` to return, so the outputs and reporting records a task sets on its way out are reported. -- New `AiUsageRecord` reporting record (`udm.AiUsageRecord`) for one AI agent execution: LLM provider, model, agent, termination reason, iterations, tool calls, token counts and estimated cost. Add it with `add_reporting_record()` like the other records, but only on a Release version that defines `udm.AiUsageRecord`: an older server cannot read the task's result and fails the task. +- Tasks can be aborted. On abort, the SDK calls the task's `abort()` method; long-running tasks can check `is_abort_requested()` and stop early. +- After an abort, the SDK waits briefly (`ABORT_GRACE_PERIOD_SECONDS`, default 3) so the task's outputs and reporting records are still sent. +- Added the `AiUsageRecord` reporting record for AI agent runs (model, tokens, cost and more). Requires a Release version that supports `udm.AiUsageRecord`. #### 🐛 Fixes -- A signal received while the task result is being reported no longer cuts off the report; repeat signals during an abort are ignored. -- An aborted result is never reported with exit code 0, and a task calling `sys.exit(n)` is reported with exit code `n`. -- Tasks whose `__init__` does not call `super().__init__()` work again. -- `setReleaseVariable()`, `setFolderVariable()` and `setGlobalVariable()` accept a list for an existing list variable (`xlrelease.ListStringVariable`); it was rejected with a `TypeError` as if the variable held a string. +- The task result is no longer cut off if an abort signal arrives while it is being reported. +- Aborted tasks no longer report exit code 0; `sys.exit(n)` now reports exit code `n`. +- Tasks that don't call `super().__init__()` work again. +- `setReleaseVariable()`, `setFolderVariable()` and `setGlobalVariable()` now accept a list for list variables. ### Version 26.3.1