From fddfb3dc773d5c573840dadb892156831296616d Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 01:55:47 -0700 Subject: [PATCH 01/61] feat(camera): add best-effort set_frame_rate hint to AbstractCamera --- software/squid/abc.py | 12 ++++++++++++ software/tests/test_record_zstack_camera_fps.py | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 software/tests/test_record_zstack_camera_fps.py diff --git a/software/squid/abc.py b/software/squid/abc.py index 12cc17cdf..79bf2afa2 100644 --- a/software/squid/abc.py +++ b/software/squid/abc.py @@ -524,6 +524,18 @@ def get_total_frame_time(self) -> float: """ return self.get_exposure_time() + self.get_strobe_time() + def set_frame_rate(self, fps: float) -> float: + """Best-effort hint to run continuous/free-run acquisition near `fps`. + + Base default cannot change hardware timing, so it returns the + exposure-limited max the camera will actually deliver. Subclasses that + support an internal frame-rate (e.g. ToupTek PRECISE_FRAMERATE) override + this. Callers must still downsample to their target rate. + """ + max_fps = 1000.0 / self.get_total_frame_time() + self._log.debug(f"set_frame_rate({fps}) no-op on base camera; max≈{max_fps:.2f} fps") + return max_fps + @abc.abstractmethod def set_frame_format(self, frame_format: CameraFrameFormat): """ diff --git a/software/tests/test_record_zstack_camera_fps.py b/software/tests/test_record_zstack_camera_fps.py new file mode 100644 index 000000000..21d77b18a --- /dev/null +++ b/software/tests/test_record_zstack_camera_fps.py @@ -0,0 +1,17 @@ +import squid.config +import squid.camera.utils +from squid.abc import CameraAcquisitionMode + + +def _sim_camera(): + cfg = squid.config.get_camera_config().model_copy(update={"rotate_image_angle": None, "flip": None}) + return squid.camera.utils.get_camera(cfg, simulated=True) + + +def test_set_frame_rate_returns_achievable_and_is_clamped(): + cam = _sim_camera() + cam.set_exposure_time(10) # 10 ms -> max ~100 fps (exposure-limited) + # Requesting more than achievable returns the achievable max. + achievable = cam.set_frame_rate(10_000.0) + assert achievable <= 1000.0 / cam.get_total_frame_time() + 1e-6 + assert achievable > 0 From 1bb5e6966ce5e14ff72e9c11ddee6e60209044e5 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 01:59:44 -0700 Subject: [PATCH 02/61] feat(camera): SimulatedCamera honors set_frame_rate in continuous stream Implements set_frame_rate override on SimulatedCamera to cap the frame rate in continuous acquisition mode. When set_frame_rate is called, the streaming thread's frame cadence gate now honors both exposure time and the target frame period, using whichever is larger. The method returns the effective achievable FPS clamped by the total frame time (exposure + strobe). Co-Authored-By: Claude Haiku 4.5 --- software/squid/camera/utils.py | 22 ++++++++++++++----- .../tests/test_record_zstack_camera_fps.py | 16 ++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/software/squid/camera/utils.py b/software/squid/camera/utils.py index 0762c10fc..7fe9fc484 100644 --- a/software/squid/camera/utils.py +++ b/software/squid/camera/utils.py @@ -171,6 +171,7 @@ def __init__(self, *args, **kwargs): self._continue_streaming = False self._streaming_thread: Optional[threading.Thread] = None self._last_trigger_timestamp = 0 + self._target_frame_period_s: Optional[float] = None # This is for the migration to AbstractCamera. It helps us find methods/properties that # some cameras had in the pre-AbstractCamera days. @@ -209,6 +210,18 @@ def set_exposure_time(self, exposure_time_ms: float): def get_exposure_time(self) -> float: return self._exposure_time_ms + @debug_log + def set_frame_rate(self, fps: float) -> float: + if fps is None or fps <= 0: + self._target_frame_period_s = None + return 1000.0 / self.get_total_frame_time() + self._target_frame_period_s = 1.0 / fps + # Clamp to exposure-limited maximum (total frame time includes strobe) + total_frame_time_s = self.get_total_frame_time() / 1000.0 + effective_period_s = max(self._target_frame_period_s, total_frame_time_s) + effective = 1.0 / effective_period_s + return effective + @debug_log def get_strobe_time(self): return 3 # Just some arbitrary non-zero number so we test code that relies on this. @@ -292,12 +305,11 @@ def stream_fn(): self._log.info("Starting streaming thread...") last_frame_time = time.time() while self._continue_streaming: + period_s = self._exposure_time_ms / 1000.0 + if self._target_frame_period_s is not None: + period_s = max(period_s, self._target_frame_period_s) time_since = time.time() - last_frame_time - # use self._exposure_time and _acquisition_mode so as not to spam the logs, - # but this could case issues if subclassed for testing. - if ( - self._exposure_time_ms / 1000.0 - ) - time_since <= 0 and self._acquisition_mode == CameraAcquisitionMode.CONTINUOUS: + if time_since >= period_s and self._acquisition_mode == CameraAcquisitionMode.CONTINUOUS: self._next_frame() last_frame_time = time.time() time.sleep(0.001) diff --git a/software/tests/test_record_zstack_camera_fps.py b/software/tests/test_record_zstack_camera_fps.py index 21d77b18a..b0ad99106 100644 --- a/software/tests/test_record_zstack_camera_fps.py +++ b/software/tests/test_record_zstack_camera_fps.py @@ -1,3 +1,5 @@ +import time + import squid.config import squid.camera.utils from squid.abc import CameraAcquisitionMode @@ -15,3 +17,17 @@ def test_set_frame_rate_returns_achievable_and_is_clamped(): achievable = cam.set_frame_rate(10_000.0) assert achievable <= 1000.0 / cam.get_total_frame_time() + 1e-6 assert achievable > 0 + + +def test_simulated_camera_honors_frame_rate(): + cam = _sim_camera() + cam.set_exposure_time(1) # 1 ms exposure -> would free-run very fast + cam.set_frame_rate(20.0) # but cap to 20 fps + cam.set_acquisition_mode(CameraAcquisitionMode.CONTINUOUS) + received = [] + cam.add_frame_callback(lambda f: received.append(f.timestamp)) + cam.start_streaming() + time.sleep(1.0) + cam.stop_streaming() + # ~20 fps over ~1s -> well under 40, comfortably over 5 (loose bounds for CI). + assert 5 <= len(received) <= 40 From 1a81c53ea0d98ef66c7c5a791d59e67b0355a209 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 02:10:38 -0700 Subject: [PATCH 03/61] feat(camera): ToupcamCamera.set_frame_rate via PRECISE_FRAMERATE --- software/control/camera_toupcam.py | 46 +++++++++++++++++++ .../tests/test_record_zstack_camera_fps.py | 9 ++++ 2 files changed, 55 insertions(+) diff --git a/software/control/camera_toupcam.py b/software/control/camera_toupcam.py index 5102e7a0c..a04e0ac37 100644 --- a/software/control/camera_toupcam.py +++ b/software/control/camera_toupcam.py @@ -20,6 +20,7 @@ import threading import control.toupcam as toupcam +import control.toupcam_exceptions from control.toupcam_exceptions import hresult_checker log = squid.logging.get_logger(__name__) @@ -50,6 +51,21 @@ def get_sn_by_model(camera_model: ToupcamCameraModel): return None # return None if no device with the specified model_name is connected +def clamp_precise_framerate_tenths(fps: float, min_tenths: int, max_tenths: int) -> int: + """Clamp fps (in frames per second) to the allowed range in tenths. + + Args: + fps: Desired frame rate in frames per second + min_tenths: Minimum allowed value in tenths (0.1 fps units) + max_tenths: Maximum allowed value in tenths (0.1 fps units) + + Returns: + Clamped value in tenths of fps + """ + tenths = int(round(fps * 10.0)) + return max(min_tenths, min(max_tenths, tenths)) + + class ToupcamCamera(AbstractCamera): TOUPCAM_OPTION_RAW_RAW_VAL = 1 TOUPCAM_OPTION_RAW_RGB_VAL = 0 @@ -556,6 +572,36 @@ def get_exposure_limits(self) -> Tuple[float, float]: (min_exposure, max_exposure, default_exposure) = self._camera.get_ExpTimeRange() return min_exposure / 1000.0, max_exposure / 1000.0 # us -> ms + def set_frame_rate(self, fps: float) -> float: + """Set the frame rate via PRECISE_FRAMERATE option. + + _calculate_strobe_info (~:128-140) drives PRECISE_FRAMERATE to MAX on mode + switch; set_frame_rate must be called **after** entering CONTINUOUS to take + effect, and recording restores nothing (next acquisition resets exposure → MAX again). + + Args: + fps: Desired frame rate in frames per second. If None or <= 0, returns + current achievable frame rate without changing settings. + + Returns: + The achievable frame rate in frames per second, or current rate if not changed. + """ + if fps is None or fps <= 0: + return 1000.0 / self.get_total_frame_time() + try: + max_tenths = self._camera.get_Option(toupcam.TOUPCAM_OPTION_MAX_PRECISE_FRAMERATE) + min_tenths = self._camera.get_Option(toupcam.TOUPCAM_OPTION_MIN_PRECISE_FRAMERATE) + except toupcam.HRESULTException as ex: + self._log.warning(f"precise-framerate range read failed: {control.toupcam_exceptions.explain(ex)}") + return 1000.0 / self.get_total_frame_time() + tenths = clamp_precise_framerate_tenths(fps, min_tenths, max_tenths) + try: + self._camera.put_Option(toupcam.TOUPCAM_OPTION_PRECISE_FRAMERATE, tenths) + except toupcam.HRESULTException as ex: + self._log.warning(f"set precise-framerate failed: {control.toupcam_exceptions.explain(ex)}") + return 1000.0 / self.get_total_frame_time() + return tenths / 10.0 + @staticmethod def _user_gain_to_toupcam(user_gain): """ diff --git a/software/tests/test_record_zstack_camera_fps.py b/software/tests/test_record_zstack_camera_fps.py index b0ad99106..e4b7624ce 100644 --- a/software/tests/test_record_zstack_camera_fps.py +++ b/software/tests/test_record_zstack_camera_fps.py @@ -5,6 +5,15 @@ from squid.abc import CameraAcquisitionMode +def test_clamp_precise_framerate_tenths(): + from control.camera_toupcam import clamp_precise_framerate_tenths + + # fps -> tenths, clamped to [min,max] in tenths + assert clamp_precise_framerate_tenths(11.5, min_tenths=10, max_tenths=600) == 115 + assert clamp_precise_framerate_tenths(0.1, min_tenths=10, max_tenths=600) == 10 # below min + assert clamp_precise_framerate_tenths(999.0, min_tenths=10, max_tenths=600) == 600 # above max + + def _sim_camera(): cfg = squid.config.get_camera_config().model_copy(update={"rotate_image_angle": None, "flip": None}) return squid.camera.utils.get_camera(cfg, simulated=True) From c0fb8414a5ab2e640967cc40b6cbe72ced3ca178 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 02:36:50 -0700 Subject: [PATCH 04/61] refactor: extract MultiPointWorkerBase (no behavior change) Lift shared acquisition mechanics out of MultiPointWorker into a new MultiPointWorkerBase so a future RecordZStackWorker sibling can reuse them: camera/stage/channel-apply, single-frame capture (acquire_camera_image), the camera frame callback (_image_callback) with CaptureInfo handoff + job creation/dispatch + backpressure gating + ready-for-next-trigger event, job-result summarize/finish, move_to_z_level, _select_config, _sleep, _frame_wait_timeout_s, wait_till_operation_is_completed, update_use_piezo. MultiPointWorker now subclasses MultiPointWorkerBase, calls super().__init__() for the shared handles/state, then builds the real job runners / backpressure controller and sets all MultiPoint-specific state (NZ/deltaZ/z_range/ z_stacking_config/use_piezo/selected_configurations/scan coords/time-point/ AF + per-channel-offset). Orchestration (run, run_single_time_point, run_coordinate_acquisition, acquire_at_position) and the z-stack/offset/AF helpers stay in MultiPointWorker unchanged. _emit_plate_layout gets a base no-op stub overridden by MultiPointWorker's real implementation so the lifted frame callback stays self-contained. Method bodies moved verbatim; net MultiPointWorker state is identical. Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/core/multi_point_worker.py | 795 +++++++++++--------- 1 file changed, 433 insertions(+), 362 deletions(-) diff --git a/software/control/core/multi_point_worker.py b/software/control/core/multi_point_worker.py index 9c834ed83..430262439 100644 --- a/software/control/core/multi_point_worker.py +++ b/software/control/core/multi_point_worker.py @@ -63,7 +63,427 @@ class SummarizeResult(NamedTuple): had_results: bool # True if any results were pulled from queue -class MultiPointWorker: +class MultiPointWorkerBase: + """Shared acquisition mechanics (camera, stage, channel apply, single-frame + capture, frame callback, job dispatch, backpressure, abort, progress). + + Owns NO orchestration loop — subclasses provide run() and the acquisition + state it operates on. Subclasses MUST call super().__init__(...) first, then + build the real job runners / backpressure controller and refine the + placeholder state (use_piezo, time_point, slack notifier, etc.). + """ + + def __init__( + self, + scope: Microscope, + live_controller: LiveController, + callbacks: MultiPointControllerFunctions, + abort_requested_fn: Callable[[], bool], + request_abort_fn: Callable[[], None], + ): + # Use type(self).__name__ so subclass instances log under their own class + # name, matching the pre-refactor behavior (MultiPointWorker used + # __class__.__name__, which resolved to the subclass for its instances). + self._log = squid.logging.get_logger(type(self).__name__) + self._timing = utils.TimingManager("MultiPointWorker Timer Manager") + + # Hardware / controller handles shared by all acquisition workers. + self.microscope: Microscope = scope + self.camera: AbstractCamera = scope.camera + self.microcontroller: Microcontroller = scope.low_level_drivers.microcontroller + self.stage: squid.abc.AbstractStage = scope.stage + self.piezo: Optional[PiezoStage] = scope.addons.piezo_stage + self.liveController = live_controller + + # Callback / abort plumbing shared by all acquisition workers. + self.callbacks: MultiPointControllerFunctions = callbacks + self.abort_requested_fn: Callable[[], bool] = abort_requested_fn + self.request_abort_fn: Callable[[], None] = request_abort_fn + + # Optional SlackNotifier — subclasses set the real one if present. + self._slack_notifier = None + + # Counters touched by the shared capture / frame-callback path. Subclasses + # may reset these per timepoint, but they must exist for the base methods. + self.image_count = 0 + self._timepoint_image_count = 0 + self._acquisition_error_count = 0 + + # Capture state refined by the subclass __init__ from acquisition params. + # Defaults are placeholders so lifted methods never read an unset attribute; + # subclasses overwrite these with the real per-acquisition values. + self.use_piezo = False + self.time_point = 0 + self.z_piezo_um = 0.0 + + # Callback-based capture bookkeeping. This is for keeping track of whether + # or not we have the last image we tried to capture. + # NOTE(imo): Once we do overlapping triggering, we'll want to keep a queue of images we are expecting. + # For now, this is an improvement over blocking immediately while waiting for the next image! + self._ready_for_next_trigger = threading.Event() + # Set this to true so that the first frame capture can proceed. + self._ready_for_next_trigger.set() + # This is cleared when the image callback is no longer processing an image. If true, an image is still + # in flux and we need to make sure the object doesn't disappear. + self._image_callback_idle = threading.Event() + self._image_callback_idle.set() + # This is protected by the threading event above (aka set after clear, take copy before set) + self._current_capture_info: Optional[CaptureInfo] = None + # This is only touched via the image callback path. Don't touch it outside of there! + self._current_round_images = {} + + # Job-runner / backpressure handles. The real ones are heavily dependent on + # MultiPoint-specific acquisition state (selected configs, scan coords, zarr + # writer info, ...), so they are built by the subclass __init__. The base + # methods (_finish_jobs, _summarize_runner_outputs, _image_callback) only + # need these attributes to exist; placeholders keep them self-contained. + self._backpressure: Optional[BackpressureController] = None + self._job_runners: List[Tuple[Type[Job], JobRunner]] = [] + self._abort_on_failed_job = True + self._first_job_dispatched = False # Track if we've waited for subprocess warmup + + def update_use_piezo(self, value): + self.use_piezo = value + self._log.info(f"MultiPointWorker: updated use_piezo to {value}") + + def wait_till_operation_is_completed(self): + self.microcontroller.wait_till_operation_is_completed() + + def move_to_z_level(self, z_mm): + self._log.debug("moving z") + self.stage.move_z_to(z_mm) + self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) + + def _select_config(self, config: AcquisitionChannel): + self.callbacks.signal_current_configuration(config) + self.liveController.set_microscope_mode(config) + self.wait_till_operation_is_completed() + + def _frame_wait_timeout_s(self): + return (self.camera.get_total_frame_time() / 1e3) + 10 + + def _sleep(self, sec): + time_to_sleep = max(sec, 1e-6) + # self._log.debug(f"Sleeping for {time_to_sleep} [s]") + time.sleep(time_to_sleep) + + def _emit_plate_layout(self, image: np.ndarray) -> None: + """Hook for subclasses to emit a plate layout on the first image. + + No-op in the base; MultiPointWorker overrides this with the plate-based + scan implementation. Lives here so the shared frame callback can call it + without referencing subclass-only behavior. + """ + return + + def _summarize_runner_outputs(self, drain_all: bool = False) -> SummarizeResult: + """Process job results from output queues. + + Args: + drain_all: If True, process ALL available results. If False, process at most one per queue. + + Returns: + SummarizeResult with none_failed and had_results. + """ + none_failed = True + had_results = False + for job_class, job_runner in self._job_runners: + if job_runner is None: + continue + out_queue = job_runner.output_queue() + if out_queue is None: + # Queue was cleared during shutdown + continue + while True: + try: + job_result: JobResult = out_queue.get_nowait() + none_failed = none_failed and self._summarize_job_result(job_result) + had_results = True + if not drain_all: + break # Only process one result per queue if not draining + except queue.Empty: + break + except ValueError: + # Queue was closed during shutdown - nothing more to drain + break + + return SummarizeResult(none_failed=none_failed, had_results=had_results) + + def _summarize_job_result(self, job_result: JobResult) -> bool: + """ + Prints a summary, then returns True if the result was successful or False otherwise. + """ + if job_result.exception is not None: + self._log.error(f"Error while running job {job_result.job_id}: {job_result.exception}") + self._acquisition_error_count += 1 + + # Send Slack error notification + if self._slack_notifier is not None: + try: + context = {"job_id": job_result.job_id} + self._slack_notifier.notify_error( + str(job_result.exception), + context, + ) + except Exception as e: + self._log.warning(f"Failed to send Slack error notification: {e}") + return False + else: + self._log.info(f"Got result for job {job_result.job_id}, it completed!") + # Handle ZarrWriteResult - notify viewer that frame is written + if isinstance(job_result.result, ZarrWriteResult): + r = job_result.result + self.callbacks.signal_zarr_frame_written(r.fov, r.time_point, r.z_index, r.channel_name, r.region_idx) + return True + + def _create_job(self, job_class: Type[Job], info: CaptureInfo, image: np.ndarray) -> Optional[Job]: + """Create a job instance for the given job class. + + Returns None if the job should be skipped. + """ + return job_class(capture_info=info, capture_image=JobImage(image_array=image)) + + def _finish_jobs(self, timeout_s=10): + # Drain and summarize all currently available job results before waiting for completion + self._summarize_runner_outputs(drain_all=True) + + active_runners = [ + (job_class, job_runner) for job_class, job_runner in self._job_runners if job_runner is not None + ] + + self._log.info(f"Waiting for jobs to finish on {len(active_runners)} job runners before shutting them down...") + timeout_time = time.time() + timeout_s + + def timed_out(): + return time.time() > timeout_time + + def time_left(): + return max(timeout_time - time.time(), 0) + + # Wait for all pending jobs across all runners (round-robin to avoid blocking on one) + while not timed_out(): + any_pending = False + for job_class, job_runner in active_runners: + if job_runner.has_pending(): + any_pending = True + break + if not any_pending: + break + # Process any available results while waiting + self._summarize_runner_outputs(drain_all=True) + time.sleep(0.1) + else: + # Timed out - kill any runners that still have pending jobs + for job_class, job_runner in active_runners: + if job_runner.has_pending(): + self._log.error( + f"Timed out after {timeout_s} [s] waiting for jobs to finish. Pending jobs for {job_class.__name__} abandoned!!!" + ) + job_runner.kill() + + # Drain results before shutdown + self._summarize_runner_outputs(drain_all=True) + + # Shut down all job runners in parallel (in background to avoid blocking on subprocess termination). + # Using daemon threads is safe here because: + # 1. All jobs are complete and results are already drained + # 2. The subprocess termination is best-effort cleanup only + # 3. If app exits before threads complete, OS will terminate subprocesses anyway + # 4. This prevents slow subprocess termination from blocking acquisition completion + log = self._log # Capture for closure + + def shutdown_runner(job_runner, timeout): + try: + job_runner.shutdown(timeout) + except Exception as e: + log.error(f"Error shutting down job runner in background: {e}") + + self._log.info("Shutting down job runners (non-blocking)...") + remaining_time = time_left() + for job_class, job_runner in active_runners: + t = threading.Thread(target=shutdown_runner, args=(job_runner, remaining_time), daemon=True) + t.start() + + # Final drain of all output queues (should be empty, but check anyway) + self._summarize_runner_outputs(drain_all=True) + + # Release backpressure resources now that all jobs are complete + try: + self._backpressure.close() + except Exception as e: + self._log.error(f"Error closing backpressure controller: {e}") + + def _image_callback(self, camera_frame: CameraFrame): + try: + if self._ready_for_next_trigger.is_set(): + self._log.warning( + "Got an image in the image callback, but we didn't send a trigger. Ignoring the image." + ) + return + + self._image_callback_idle.clear() + with self._timing.get_timer("_image_callback"): + self._log.debug(f"In Image callback for frame_id={camera_frame.frame_id}") + info = self._current_capture_info + self._current_capture_info = None + + self._ready_for_next_trigger.set() + if not info: + self._log.error("In image callback, no current capture info! Something is wrong. Aborting.") + self.request_abort_fn() + return + + image = camera_frame.frame + if not camera_frame or image is None: + self._log.warning("image in frame callback is None. Something is really wrong, aborting!") + self.request_abort_fn() + return + + # Increment image counter for Slack notification stats + self._timepoint_image_count += 1 + self.image_count += 1 + + with self._timing.get_timer("job creation and dispatch"): + # Wait for subprocess to be ready before first dispatch + if not self._first_job_dispatched: + for job_class, job_runner in self._job_runners: + if job_runner is not None: + t_wait_start = time.perf_counter() + if job_runner.wait_ready(timeout_s=10.0): + t_wait_end = time.perf_counter() + wait_ms = (t_wait_end - t_wait_start) * 1000 + if wait_ms > 10: # Only log if we actually had to wait + self._log.info(f"Job runner ready (waited {wait_ms:.0f}ms for subprocess)") + else: + self._log.warning(f"Job runner for {job_class.__name__} not ready after 10s") + self._first_job_dispatched = True + + for job_class, job_runner in self._job_runners: + job = self._create_job(job_class, info, image) + if job is None: + continue # Skip if job creation returns None (e.g., downsampled views disabled for this image) + if job_runner is not None: + if not job_runner.dispatch(job): + self._log.error("Failed to dispatch multiprocessing job!") + self.request_abort_fn() + return + else: + try: + # NOTE(imo): We don't have any way of people using results, so for now just + # grab and ignore it. + result = job.run() + except Exception: + self._log.exception("Failed to execute job, abandoning acquisition!") + self.request_abort_fn() + return + + height, width = image.shape[:2] + # with self._timing.get_timer("crop_image"): + # image_to_display = utils.crop_image( + # image, + # round(width * self.display_resolution_scaling), + # round(height * self.display_resolution_scaling), + # ) + # Emit plate layout once on the first image so the unified mosaic + # widget can lay out the plate grid before tiles start arriving. + self._emit_plate_layout(image) + with self._timing.get_timer("image_to_display*.emit"): + self.callbacks.signal_new_image(camera_frame, info) + + finally: + self._image_callback_idle.set() + + def acquire_camera_image( + self, config, file_ID: str, current_path: str, k: int, region_id: int, fov: int, config_idx: int + ): + self._select_config(config) + + # trigger acquisition (including turning on the illumination) and read frame + camera_illumination_time = self.camera.get_exposure_time() + if self.liveController.trigger_mode == TriggerMode.SOFTWARE: + self.liveController.turn_on_illumination() + self.wait_till_operation_is_completed() + camera_illumination_time = None + elif self.liveController.trigger_mode == TriggerMode.HARDWARE: + if "Fluorescence" in config.name and ENABLE_NL5 and NL5_USE_DOUT: + # TODO(imo): This used to use the "reset_image_ready_flag=False" on the read_frame, but oinly the toupcam camera implementation had the + # "reset_image_ready_flag" arg, so this is broken for all other cameras. Also this used to do some other funky stuff like setting internal camera flags. + # I am pretty sure this is broken! + self.microscope.addons.nl5.start_acquisition() + # This is some large timeout that we use just so as to not block forever + with self._timing.get_timer("_ready_for_next_trigger.wait"): + if not self._ready_for_next_trigger.wait(self._frame_wait_timeout_s()): + self._log.error("Frame callback never set _have_last_triggered_image callback! Aborting acquisition.") + self.request_abort_fn() + return + + # Backpressure check AFTER previous frame dispatched, BEFORE next trigger + # This is when we know the previous image's jobs have been dispatched (and counters incremented) + if self._backpressure.should_throttle(): + with self._timing.get_timer("backpressure.wait_for_capacity"): + got_capacity = self._backpressure.wait_for_capacity() + if not got_capacity: + self._log.error( + f"Backpressure timeout - disk I/O cannot keep up. Stats: {self._backpressure.get_stats()}" + ) + + with self._timing.get_timer("get_ready_for_trigger re-check"): + # This should be a noop - we have the frame already. Still, check! + while not self.camera.get_ready_for_trigger(): + self._sleep(0.001) + + self._ready_for_next_trigger.clear() + with self._timing.get_timer("current_capture_info ="): + # Even though the capture time will be slightly after this, we need to capture and set the capture info + # before the trigger to be 100% sure the callback doesn't stomp on it. + # NOTE(imo): One level up from acquire_camera_image, we have acquire_pos. We're careful to use that as + # much as we can, but don't use it here because we'd rather take the position as close as possible to the + # real capture time for the image info. Ideally we'd use this position for the caller's acquire_pos as well. + current_capture_info = CaptureInfo( + position=self.stage.get_pos(), + z_index=k, + capture_time=time.time(), + z_piezo_um=(self.z_piezo_um if self.use_piezo else None), + configuration=config, + save_directory=current_path, + file_id=file_ID, + region_id=region_id, + fov=fov, + configuration_idx=config_idx, + time_point=self.time_point, + ) + self._current_capture_info = current_capture_info + with self._timing.get_timer("send_trigger"): + self.camera.send_trigger(illumination_time=camera_illumination_time) + + with self._timing.get_timer("exposure_time_done_sleep_hw or wait_for_image_sw"): + if self.liveController.trigger_mode == TriggerMode.HARDWARE: + exposure_done_time = time.time() + self.camera.get_total_frame_time() / 1e3 + # Even though we can do overlapping triggers, we want to make sure that we don't move before our exposure + # is done. So we still need to at least sleep for the total frame time corresponding to this exposure. + self._sleep(max(0.0, exposure_done_time - time.time())) + else: + # In SW trigger mode (or anything not HARDWARE mode), there's indeterminism in the trigger timing. + # To overcome this, just wait until the frame for this capture actually comes into the image + # callback. That way we know we have it. This also helps by making sure the illumination for this + # frame is on from before the trigger until after we get the frame (which guarantees it will be on + # for the full exposure). + # + # If we wait for longer than 5x the exposure + 2 seconds, abort the acquisition because something is + # wrong. + non_hw_frame_timeout = 5 * self.camera.get_total_frame_time() / 1e3 + 2 + if not self._ready_for_next_trigger.wait(non_hw_frame_timeout): + self._log.error("Timed out waiting {non_hw_frame_timeout} [s] for a frame, aborting acquisition.") + self.request_abort_fn() + # Let this fall through so we still turn off illumination. Let the caller actually break out + # of the acquisition. + + # turn off the illumination if using software trigger + if self.liveController.trigger_mode == TriggerMode.SOFTWARE: + self.liveController.turn_off_illumination() + + +class MultiPointWorker(MultiPointWorkerBase): def __init__( self, scope: Microscope, @@ -82,8 +502,13 @@ def __init__( prewarmed_job_runner: Optional[JobRunner] = None, prewarmed_bp_values: Optional["BackpressureValues"] = None, ): - self._log = squid.logging.get_logger(__class__.__name__) - self._timing = utils.TimingManager("MultiPointWorker Timer Manager") + super().__init__( + scope=scope, + live_controller=live_controller, + callbacks=callbacks, + abort_requested_fn=abort_requested_fn, + request_abort_fn=request_abort_fn, + ) self._alignment_widget = alignment_widget # Optional AlignmentWidget for coordinate offset self._slack_notifier = slack_notifier # Optional SlackNotifier for notifications @@ -95,21 +520,12 @@ def __init__( self._laser_af_successes = 0 self._laser_af_failures = 0 self._current_z_offset_um: float = 0.0 - self.microscope: Microscope = scope - self.camera: AbstractCamera = scope.camera - self.microcontroller: Microcontroller = scope.low_level_drivers.microcontroller - self.stage: squid.abc.AbstractStage = scope.stage - self.piezo: Optional[PiezoStage] = scope.addons.piezo_stage - self.liveController = live_controller self.autofocusController: Optional[AutoFocusController] = auto_focus_controller self.laser_auto_focus_controller: Optional[LaserAutofocusController] = laser_auto_focus_controller self.objectiveStore: ObjectiveStore = objective_store self.fluidics = scope.addons.fluidics self.use_fluidics = acquisition_parameters.use_fluidics - self.callbacks: MultiPointControllerFunctions = callbacks - self.abort_requested_fn: Callable[[], bool] = abort_requested_fn - self.request_abort_fn: Callable[[], None] = request_abort_fn self.NZ = acquisition_parameters.NZ self.deltaZ = acquisition_parameters.deltaZ @@ -153,7 +569,7 @@ def __init__( physical_size_y_um=self._pixel_size_um, ) - self.time_point = 0 + self.time_point = 0 # also set in base __init__; re-set here for clarity self.af_fov_count = 0 self.num_fovs = 0 self.total_scans = 0 @@ -175,22 +591,8 @@ def __init__( self.count = 0 self.merged_image = None - self.image_count = 0 - - # This is for keeping track of whether or not we have the last image we tried to capture. - # NOTE(imo): Once we do overlapping triggering, we'll want to keep a queue of images we are expecting. - # For now, this is an improvement over blocking immediately while waiting for the next image! - self._ready_for_next_trigger = threading.Event() - # Set this to true so that the first frame capture can proceed. - self._ready_for_next_trigger.set() - # This is cleared when the image callback is no longer processing an image. If true, an image is still - # in flux and we need to make sure the object doesn't disappear. - self._image_callback_idle = threading.Event() - self._image_callback_idle.set() - # This is protected by the threading event above (aka set after clear, take copy before set) - self._current_capture_info: Optional[CaptureInfo] = None - # This is only touched via the image callback path. Don't touch it outside of there! - self._current_round_images = {} + # image_count, the trigger-ready/idle events, and capture-info bookkeeping + # are initialized in MultiPointWorkerBase.__init__. self.skip_saving = acquisition_parameters.skip_saving job_classes = [] @@ -353,10 +755,6 @@ def __init__( self._abort_on_failed_job = abort_on_failed_jobs self._first_job_dispatched = False # Track if we've waited for subprocess warmup - def update_use_piezo(self, value): - self.use_piezo = value - self._log.info(f"MultiPointWorker: updated use_piezo to {value}") - def _is_well_based_acquisition(self) -> bool: """Check if regions represent a valid well-based acquisition. @@ -600,81 +998,8 @@ def _wait_for_outstanding_callback_images(self): self._log.warning("Timed out waiting for the last image to process!") # No matter what, set the flags so things can continue - self._ready_for_next_trigger.set() - self._image_callback_idle.set() - - def _finish_jobs(self, timeout_s=10): - # Drain and summarize all currently available job results before waiting for completion - self._summarize_runner_outputs(drain_all=True) - - active_runners = [ - (job_class, job_runner) for job_class, job_runner in self._job_runners if job_runner is not None - ] - - self._log.info(f"Waiting for jobs to finish on {len(active_runners)} job runners before shutting them down...") - timeout_time = time.time() + timeout_s - - def timed_out(): - return time.time() > timeout_time - - def time_left(): - return max(timeout_time - time.time(), 0) - - # Wait for all pending jobs across all runners (round-robin to avoid blocking on one) - while not timed_out(): - any_pending = False - for job_class, job_runner in active_runners: - if job_runner.has_pending(): - any_pending = True - break - if not any_pending: - break - # Process any available results while waiting - self._summarize_runner_outputs(drain_all=True) - time.sleep(0.1) - else: - # Timed out - kill any runners that still have pending jobs - for job_class, job_runner in active_runners: - if job_runner.has_pending(): - self._log.error( - f"Timed out after {timeout_s} [s] waiting for jobs to finish. Pending jobs for {job_class.__name__} abandoned!!!" - ) - job_runner.kill() - - # Drain results before shutdown - self._summarize_runner_outputs(drain_all=True) - - # Shut down all job runners in parallel (in background to avoid blocking on subprocess termination). - # Using daemon threads is safe here because: - # 1. All jobs are complete and results are already drained - # 2. The subprocess termination is best-effort cleanup only - # 3. If app exits before threads complete, OS will terminate subprocesses anyway - # 4. This prevents slow subprocess termination from blocking acquisition completion - log = self._log # Capture for closure - - def shutdown_runner(job_runner, timeout): - try: - job_runner.shutdown(timeout) - except Exception as e: - log.error(f"Error shutting down job runner in background: {e}") - - self._log.info("Shutting down job runners (non-blocking)...") - remaining_time = time_left() - for job_class, job_runner in active_runners: - t = threading.Thread(target=shutdown_runner, args=(job_runner, remaining_time), daemon=True) - t.start() - - # Final drain of all output queues (should be empty, but check anyway) - self._summarize_runner_outputs(drain_all=True) - - # Release backpressure resources now that all jobs are complete - try: - self._backpressure.close() - except Exception as e: - self._log.error(f"Error closing backpressure controller: {e}") - - def wait_till_operation_is_completed(self): - self.microcontroller.wait_till_operation_is_completed() + self._ready_for_next_trigger.set() + self._image_callback_idle.set() def run_single_time_point(self): try: @@ -800,78 +1125,6 @@ def move_to_coordinate(self, coordinate_mm, region_id, fov): z_mm = coordinate_mm[2] self.move_to_z_level(z_mm) - def move_to_z_level(self, z_mm): - self._log.debug("moving z") - self.stage.move_z_to(z_mm) - self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) - - def _summarize_runner_outputs(self, drain_all: bool = False) -> SummarizeResult: - """Process job results from output queues. - - Args: - drain_all: If True, process ALL available results. If False, process at most one per queue. - - Returns: - SummarizeResult with none_failed and had_results. - """ - none_failed = True - had_results = False - for job_class, job_runner in self._job_runners: - if job_runner is None: - continue - out_queue = job_runner.output_queue() - if out_queue is None: - # Queue was cleared during shutdown - continue - while True: - try: - job_result: JobResult = out_queue.get_nowait() - none_failed = none_failed and self._summarize_job_result(job_result) - had_results = True - if not drain_all: - break # Only process one result per queue if not draining - except queue.Empty: - break - except ValueError: - # Queue was closed during shutdown - nothing more to drain - break - - return SummarizeResult(none_failed=none_failed, had_results=had_results) - - def _summarize_job_result(self, job_result: JobResult) -> bool: - """ - Prints a summary, then returns True if the result was successful or False otherwise. - """ - if job_result.exception is not None: - self._log.error(f"Error while running job {job_result.job_id}: {job_result.exception}") - self._acquisition_error_count += 1 - - # Send Slack error notification - if self._slack_notifier is not None: - try: - context = {"job_id": job_result.job_id} - self._slack_notifier.notify_error( - str(job_result.exception), - context, - ) - except Exception as e: - self._log.warning(f"Failed to send Slack error notification: {e}") - return False - else: - self._log.info(f"Got result for job {job_result.job_id}, it completed!") - # Handle ZarrWriteResult - notify viewer that frame is written - if isinstance(job_result.result, ZarrWriteResult): - r = job_result.result - self.callbacks.signal_zarr_frame_written(r.fov, r.time_point, r.z_index, r.channel_name, r.region_idx) - return True - - def _create_job(self, job_class: Type[Job], info: CaptureInfo, image: np.ndarray) -> Optional[Job]: - """Create a job instance for the given job class. - - Returns None if the job should be skipped. - """ - return job_class(capture_info=info, capture_image=JobImage(image_array=image)) - def _emit_plate_layout(self, image: np.ndarray) -> None: """Emit plate_view_init for the unified mosaic widget on the first image. @@ -1125,11 +1378,6 @@ def acquire_at_position(self, region_id, current_path, fov): # Increment FOV counter for Slack notification stats self._timepoint_fov_count += 1 - def _select_config(self, config: AcquisitionChannel): - self.callbacks.signal_current_configuration(config) - self.liveController.set_microscope_mode(config) - self.wait_till_operation_is_completed() - def perform_autofocus(self, region_id, fov): if not self.do_reflection_af: # contrast-based AF; perform AF only if when not taking z stack or doing z stack from center @@ -1304,183 +1552,6 @@ def _log_ignored_offsets(self) -> None: reason = "laser AF off" if not self.do_reflection_af else "'Apply channel offset' unchecked" self._log.info(f"[multi-point] {reason} — ignoring non-zero z-offsets on channels: [{summary}]") - def _image_callback(self, camera_frame: CameraFrame): - try: - if self._ready_for_next_trigger.is_set(): - self._log.warning( - "Got an image in the image callback, but we didn't send a trigger. Ignoring the image." - ) - return - - self._image_callback_idle.clear() - with self._timing.get_timer("_image_callback"): - self._log.debug(f"In Image callback for frame_id={camera_frame.frame_id}") - info = self._current_capture_info - self._current_capture_info = None - - self._ready_for_next_trigger.set() - if not info: - self._log.error("In image callback, no current capture info! Something is wrong. Aborting.") - self.request_abort_fn() - return - - image = camera_frame.frame - if not camera_frame or image is None: - self._log.warning("image in frame callback is None. Something is really wrong, aborting!") - self.request_abort_fn() - return - - # Increment image counter for Slack notification stats - self._timepoint_image_count += 1 - self.image_count += 1 - - with self._timing.get_timer("job creation and dispatch"): - # Wait for subprocess to be ready before first dispatch - if not self._first_job_dispatched: - for job_class, job_runner in self._job_runners: - if job_runner is not None: - t_wait_start = time.perf_counter() - if job_runner.wait_ready(timeout_s=10.0): - t_wait_end = time.perf_counter() - wait_ms = (t_wait_end - t_wait_start) * 1000 - if wait_ms > 10: # Only log if we actually had to wait - self._log.info(f"Job runner ready (waited {wait_ms:.0f}ms for subprocess)") - else: - self._log.warning(f"Job runner for {job_class.__name__} not ready after 10s") - self._first_job_dispatched = True - - for job_class, job_runner in self._job_runners: - job = self._create_job(job_class, info, image) - if job is None: - continue # Skip if job creation returns None (e.g., downsampled views disabled for this image) - if job_runner is not None: - if not job_runner.dispatch(job): - self._log.error("Failed to dispatch multiprocessing job!") - self.request_abort_fn() - return - else: - try: - # NOTE(imo): We don't have any way of people using results, so for now just - # grab and ignore it. - result = job.run() - except Exception: - self._log.exception("Failed to execute job, abandoning acquisition!") - self.request_abort_fn() - return - - height, width = image.shape[:2] - # with self._timing.get_timer("crop_image"): - # image_to_display = utils.crop_image( - # image, - # round(width * self.display_resolution_scaling), - # round(height * self.display_resolution_scaling), - # ) - # Emit plate layout once on the first image so the unified mosaic - # widget can lay out the plate grid before tiles start arriving. - self._emit_plate_layout(image) - with self._timing.get_timer("image_to_display*.emit"): - self.callbacks.signal_new_image(camera_frame, info) - - finally: - self._image_callback_idle.set() - - def _frame_wait_timeout_s(self): - return (self.camera.get_total_frame_time() / 1e3) + 10 - - def acquire_camera_image( - self, config, file_ID: str, current_path: str, k: int, region_id: int, fov: int, config_idx: int - ): - self._select_config(config) - - # trigger acquisition (including turning on the illumination) and read frame - camera_illumination_time = self.camera.get_exposure_time() - if self.liveController.trigger_mode == TriggerMode.SOFTWARE: - self.liveController.turn_on_illumination() - self.wait_till_operation_is_completed() - camera_illumination_time = None - elif self.liveController.trigger_mode == TriggerMode.HARDWARE: - if "Fluorescence" in config.name and ENABLE_NL5 and NL5_USE_DOUT: - # TODO(imo): This used to use the "reset_image_ready_flag=False" on the read_frame, but oinly the toupcam camera implementation had the - # "reset_image_ready_flag" arg, so this is broken for all other cameras. Also this used to do some other funky stuff like setting internal camera flags. - # I am pretty sure this is broken! - self.microscope.addons.nl5.start_acquisition() - # This is some large timeout that we use just so as to not block forever - with self._timing.get_timer("_ready_for_next_trigger.wait"): - if not self._ready_for_next_trigger.wait(self._frame_wait_timeout_s()): - self._log.error("Frame callback never set _have_last_triggered_image callback! Aborting acquisition.") - self.request_abort_fn() - return - - # Backpressure check AFTER previous frame dispatched, BEFORE next trigger - # This is when we know the previous image's jobs have been dispatched (and counters incremented) - if self._backpressure.should_throttle(): - with self._timing.get_timer("backpressure.wait_for_capacity"): - got_capacity = self._backpressure.wait_for_capacity() - if not got_capacity: - self._log.error( - f"Backpressure timeout - disk I/O cannot keep up. Stats: {self._backpressure.get_stats()}" - ) - - with self._timing.get_timer("get_ready_for_trigger re-check"): - # This should be a noop - we have the frame already. Still, check! - while not self.camera.get_ready_for_trigger(): - self._sleep(0.001) - - self._ready_for_next_trigger.clear() - with self._timing.get_timer("current_capture_info ="): - # Even though the capture time will be slightly after this, we need to capture and set the capture info - # before the trigger to be 100% sure the callback doesn't stomp on it. - # NOTE(imo): One level up from acquire_camera_image, we have acquire_pos. We're careful to use that as - # much as we can, but don't use it here because we'd rather take the position as close as possible to the - # real capture time for the image info. Ideally we'd use this position for the caller's acquire_pos as well. - current_capture_info = CaptureInfo( - position=self.stage.get_pos(), - z_index=k, - capture_time=time.time(), - z_piezo_um=(self.z_piezo_um if self.use_piezo else None), - configuration=config, - save_directory=current_path, - file_id=file_ID, - region_id=region_id, - fov=fov, - configuration_idx=config_idx, - time_point=self.time_point, - ) - self._current_capture_info = current_capture_info - with self._timing.get_timer("send_trigger"): - self.camera.send_trigger(illumination_time=camera_illumination_time) - - with self._timing.get_timer("exposure_time_done_sleep_hw or wait_for_image_sw"): - if self.liveController.trigger_mode == TriggerMode.HARDWARE: - exposure_done_time = time.time() + self.camera.get_total_frame_time() / 1e3 - # Even though we can do overlapping triggers, we want to make sure that we don't move before our exposure - # is done. So we still need to at least sleep for the total frame time corresponding to this exposure. - self._sleep(max(0.0, exposure_done_time - time.time())) - else: - # In SW trigger mode (or anything not HARDWARE mode), there's indeterminism in the trigger timing. - # To overcome this, just wait until the frame for this capture actually comes into the image - # callback. That way we know we have it. This also helps by making sure the illumination for this - # frame is on from before the trigger until after we get the frame (which guarantees it will be on - # for the full exposure). - # - # If we wait for longer than 5x the exposure + 2 seconds, abort the acquisition because something is - # wrong. - non_hw_frame_timeout = 5 * self.camera.get_total_frame_time() / 1e3 + 2 - if not self._ready_for_next_trigger.wait(non_hw_frame_timeout): - self._log.error("Timed out waiting {non_hw_frame_timeout} [s] for a frame, aborting acquisition.") - self.request_abort_fn() - # Let this fall through so we still turn off illumination. Let the caller actually break out - # of the acquisition. - - # turn off the illumination if using software trigger - if self.liveController.trigger_mode == TriggerMode.SOFTWARE: - self.liveController.turn_off_illumination() - - def _sleep(self, sec): - time_to_sleep = max(sec, 1e-6) - # self._log.debug(f"Sleeping for {time_to_sleep} [s]") - time.sleep(time_to_sleep) - def acquire_rgb_image(self, config, file_ID, current_path, k, region_id, fov): # go through the channels rgb_channels = ["BF LED matrix full_R", "BF LED matrix full_G", "BF LED matrix full_B"] From 4b8cc1fd78f5d25edac5da90d5682033e64dc357 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 03:46:49 -0700 Subject: [PATCH 05/61] refactor: extract create_experiment_dir helper into acquisition_setup.py (no behavior change) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the experiment-ID timestamping + directory creation logic from MultiPointController.start_new_experiment into a free function create_experiment_dir(base_path, experiment_id) -> (resolved_id, dir_path) in control/core/acquisition_setup.py so a future RecordZStackController can reuse it without duplicating code. Pre-warmed JobRunner methods left in MultiPointController — they mutate instance state and do not factor out cleanly as free functions. Co-Authored-By: Claude Sonnet 4.6 --- software/control/core/acquisition_setup.py | 33 +++++++++++++++++++ .../control/core/multi_point_controller.py | 9 ++--- 2 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 software/control/core/acquisition_setup.py diff --git a/software/control/core/acquisition_setup.py b/software/control/core/acquisition_setup.py new file mode 100644 index 000000000..8d5b0547f --- /dev/null +++ b/software/control/core/acquisition_setup.py @@ -0,0 +1,33 @@ +"""Shared acquisition setup helpers. + +Free functions used by MultiPointController (and future controllers such as +RecordZStackController) to set up experiment directories without duplicating +logic across controller classes. +""" + +import os +from datetime import datetime +from typing import Tuple + +from control import utils + + +def create_experiment_dir(base_path: str, experiment_id: str) -> Tuple[str, str]: + """Resolve a unique experiment ID and create its output directory. + + Appends a timestamp to *experiment_id* (spaces replaced with underscores) + to guarantee uniqueness, then creates the directory tree under *base_path*. + + Args: + base_path: Root directory for all experiments. + experiment_id: Human-readable experiment name supplied by the user. + + Returns: + A ``(resolved_id, dir_path)`` tuple where *resolved_id* is the + timestamped identifier and *dir_path* is the absolute path of the + newly created directory. + """ + resolved_id = experiment_id.replace(" ", "_") + "_" + datetime.now().strftime("%Y-%m-%d_%H-%M-%S.%f") + dir_path = os.path.join(base_path, resolved_id) + utils.ensure_directory_exists(dir_path) + return resolved_id, dir_path diff --git a/software/control/core/multi_point_controller.py b/software/control/core/multi_point_controller.py index 680c22b8d..ac2609b99 100644 --- a/software/control/core/multi_point_controller.py +++ b/software/control/core/multi_point_controller.py @@ -6,7 +6,6 @@ import tempfile import time import yaml -from datetime import datetime from enum import Enum from threading import Thread from typing import Optional, Tuple, Any @@ -17,6 +16,7 @@ from control import utils, utils_acquisition import control._def from control.core.auto_focus_controller import AutoFocusController +from control.core.acquisition_setup import create_experiment_dir from control.core.multi_point_utils import MultiPointControllerFunctions, ScanPositionInformation, AcquisitionParameters from control.core.scan_coordinates import ScanCoordinates from control.core.laser_auto_focus_controller import LaserAutofocusController @@ -437,12 +437,9 @@ def set_overlap_percent(self, overlap_percent: float): self.overlap_percent = overlap_percent def start_new_experiment(self, experiment_ID): # @@@ to do: change name to prepare_folder_for_new_experiment - # generate unique experiment ID - self.experiment_ID = experiment_ID.replace(" ", "_") + "_" + datetime.now().strftime("%Y-%m-%d_%H-%M-%S.%f") + # generate unique experiment ID and create its output directory + self.experiment_ID, experiment_dir = create_experiment_dir(self.base_path, experiment_ID) self.recording_start_time = time.time() - # create a new folder - experiment_dir = os.path.join(self.base_path, self.experiment_ID) - utils.ensure_directory_exists(experiment_dir) # Save acquisition configuration via ConfigRepository self.liveController.microscope.config_repo.save_acquisition_output( output_dir=experiment_dir, From 9df259a227a10c7f2d25d270ce262c3b36e8bd2a Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 03:52:01 -0700 Subject: [PATCH 06/61] feat(streaming): add CountStop + RecordingRouter (pure) Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/core/streaming_capture.py | 26 +++++++++++++++++++ software/tests/core/__init__.py | 0 software/tests/core/test_streaming_capture.py | 16 ++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 software/control/core/streaming_capture.py create mode 100644 software/tests/core/__init__.py create mode 100644 software/tests/core/test_streaming_capture.py diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py new file mode 100644 index 000000000..bc5e13089 --- /dev/null +++ b/software/control/core/streaming_capture.py @@ -0,0 +1,26 @@ +from typing import Optional, Tuple + + +class CountStop: + def __init__(self, target: int): + self.target = target + + def met(self, emitted: int) -> bool: + return emitted >= self.target + + +class RecordingRouter: + """Maps incoming frames to (t,c,z)=(t_index,0,0), downsampling to `fps`.""" + + def __init__(self, fps: float): + self._min_period = 1.0 / fps if fps and fps > 0 else 0.0 + self._t_index = 0 + self._last_emit_ts: Optional[float] = None + + def route(self, timestamp: float) -> Optional[Tuple[int, int, int]]: + if self._last_emit_ts is not None and (timestamp - self._last_emit_ts) < self._min_period - 1e-9: + return None + idx = (self._t_index, 0, 0) + self._t_index += 1 + self._last_emit_ts = timestamp + return idx diff --git a/software/tests/core/__init__.py b/software/tests/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py new file mode 100644 index 000000000..1b072e483 --- /dev/null +++ b/software/tests/core/test_streaming_capture.py @@ -0,0 +1,16 @@ +from control.core.streaming_capture import CountStop, RecordingRouter + + +def test_count_stop(): + s = CountStop(3) + assert not s.met(2) + assert s.met(3) + assert s.met(4) + + +def test_recording_router_downsamples_and_indexes(): + r = RecordingRouter(fps=10.0) # min spacing 0.1 s + assert r.route(100.00) == (0, 0, 0) # first frame always emits + assert r.route(100.05) is None # 50 ms later -> skip + assert r.route(100.10) == (1, 0, 0) # 100 ms later -> emit, t=1 + assert r.route(100.30) == (2, 0, 0) # emit, t=2 From deae236e3ab1d9ef849107ef2f31c7ac2c57abf9 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 03:55:28 -0700 Subject: [PATCH 07/61] feat(streaming): RecordingWriter (bounded queue + thread -> ZarrWriter) Co-Authored-By: Claude Sonnet 4.6 --- software/control/core/streaming_capture.py | 66 +++++++++++++++++++ software/tests/core/test_streaming_capture.py | 31 +++++++++ 2 files changed, 97 insertions(+) diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index bc5e13089..e5d142982 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -1,5 +1,15 @@ +import queue +import threading from typing import Optional, Tuple +import numpy as np + +import squid.logging +from control.core.zarr_writer import ZarrAcquisitionConfig, ZarrWriter + +_log = squid.logging.get_logger("RecordingWriter") +_SENTINEL = object() + class CountStop: def __init__(self, target: int): @@ -24,3 +34,59 @@ def route(self, timestamp: float) -> Optional[Tuple[int, int, int]]: self._t_index += 1 self._last_emit_ts = timestamp return idx + + +class RecordingWriter: + """Bounded-queue writer that drains frames to a ZarrWriter on a background thread. + + The hot camera callback calls `enqueue` (non-blocking); the background thread + calls `ZarrWriter.write_frame` which may block on I/O. The queue is bounded so + that a slow disk eventually provides backpressure: `enqueue` will block for up + to 0.5 s before logging a drop and returning. + """ + + def __init__(self, config: ZarrAcquisitionConfig, max_queue: int = 64): + self._writer = ZarrWriter(config) + self._q: "queue.Queue" = queue.Queue(maxsize=max_queue) + self._thread = threading.Thread(target=self._drain, daemon=True) + self._dropped = 0 + + def start(self) -> None: + """Initialize the underlying ZarrWriter and start the drain thread.""" + self._writer.initialize() + self._thread.start() + + def enqueue(self, frame: np.ndarray, t: int, c: int, z: int) -> None: + """Non-blocking enqueue. Blocks briefly as backpressure; drops on full.""" + try: + self._q.put((frame, t, c, z), timeout=0.5) + except queue.Full: + self._dropped += 1 + _log.warning(f"recording queue full; dropped frame t={t} (total dropped={self._dropped})") + + def _drain(self) -> None: + """Background thread: pulls items and writes them via ZarrWriter.""" + while True: + item = self._q.get() + if item is _SENTINEL: + return + frame, t, c, z = item + try: + self._writer.write_frame(frame, t=t, c=c, z=z) + except Exception as e: + _log.error(f"recording write_frame failed t={t}: {e}") + + def finalize(self) -> None: + """Flush the queue, join the drain thread, and finalize the ZarrWriter.""" + self._q.put(_SENTINEL) + self._thread.join() + self._writer.finalize() + + def abort(self) -> None: + """Signal the drain thread to stop, then abort the ZarrWriter.""" + try: + self._q.put_nowait(_SENTINEL) + except queue.Full: + pass + self._thread.join(timeout=2.0) + self._writer.abort() diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py index 1b072e483..6fb545480 100644 --- a/software/tests/core/test_streaming_capture.py +++ b/software/tests/core/test_streaming_capture.py @@ -1,3 +1,5 @@ +import numpy as np + from control.core.streaming_capture import CountStop, RecordingRouter @@ -14,3 +16,32 @@ def test_recording_router_downsamples_and_indexes(): assert r.route(100.05) is None # 50 ms later -> skip assert r.route(100.10) == (1, 0, 0) # 100 ms later -> emit, t=1 assert r.route(100.30) == (2, 0, 0) # emit, t=2 + + +def test_recording_writer_roundtrip(tmp_path): + T, Y, X = 4, 16, 12 + from control.core.zarr_writer import ZarrAcquisitionConfig + from control.core.streaming_capture import RecordingWriter + + cfg = ZarrAcquisitionConfig( + output_path=str(tmp_path / "rec.ome.zarr"), + shape=(T, 1, 1, Y, X), + dtype=np.uint16, + pixel_size_um=1.0, + z_step_um=None, + time_increment_s=0.1, + channel_names=["BF"], + channel_colors=["#FFFFFF"], + channel_wavelengths=[None], + is_hcs=False, + ) + w = RecordingWriter(cfg) + w.start() + for t in range(T): + w.enqueue(np.full((Y, X), t + 1, dtype=np.uint16), t, 0, 0) + w.finalize() + import tensorstore as ts + + ds = ts.open({"driver": "zarr3", "kvstore": {"driver": "file", "path": cfg.output_path}}).result() + assert tuple(ds.shape) == (T, 1, 1, Y, X) + assert int(ds[2, 0, 0, 0, 0].read().result()) == 3 From 83fa68a3ae26363fe25d8627717202c54f4c743e Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 03:59:32 -0700 Subject: [PATCH 08/61] fix(streaming): drain thread solely owns ZarrWriter lifecycle (no abort/write race) Co-Authored-By: Claude Sonnet 4.6 --- software/control/core/streaming_capture.py | 57 ++++++++++++++++------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index e5d142982..a17237a23 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -43,6 +43,12 @@ class RecordingWriter: calls `ZarrWriter.write_frame` which may block on I/O. The queue is bounded so that a slow disk eventually provides backpressure: `enqueue` will block for up to 0.5 s before logging a drop and returning. + + After `start()` the drain thread is the SOLE owner of the ZarrWriter: only it + calls `write_frame`, `finalize`, and `abort`. The main thread only calls + `initialize()` (before the thread starts) and then enqueues items / signals stop. + This prevents the data race where `abort()` used to call `self._writer.abort()` + concurrently with the drain thread still inside `write_frame`. """ def __init__(self, config: ZarrAcquisitionConfig, max_queue: int = 64): @@ -50,6 +56,7 @@ def __init__(self, config: ZarrAcquisitionConfig, max_queue: int = 64): self._q: "queue.Queue" = queue.Queue(maxsize=max_queue) self._thread = threading.Thread(target=self._drain, daemon=True) self._dropped = 0 + self._abort_requested = threading.Event() def start(self) -> None: """Initialize the underlying ZarrWriter and start the drain thread.""" @@ -65,28 +72,46 @@ def enqueue(self, frame: np.ndarray, t: int, c: int, z: int) -> None: _log.warning(f"recording queue full; dropped frame t={t} (total dropped={self._dropped})") def _drain(self) -> None: - """Background thread: pulls items and writes them via ZarrWriter.""" - while True: - item = self._q.get() - if item is _SENTINEL: - return - frame, t, c, z = item - try: - self._writer.write_frame(frame, t=t, c=c, z=z) - except Exception as e: - _log.error(f"recording write_frame failed t={t}: {e}") + """Background thread: sole owner of ZarrWriter after start(). + + Reads the queue with a short timeout so it can notice an abort between + frames. On exit, calls writer.abort() or writer.finalize() as appropriate. + """ + try: + while True: + if self._abort_requested.is_set(): + break + try: + item = self._q.get(timeout=0.1) + except queue.Empty: + continue + if item is _SENTINEL: + break + frame, t, c, z = item + try: + self._writer.write_frame(frame, t=t, c=c, z=z) + except Exception as e: + _log.error(f"recording write_frame failed t={t}: {e}") + finally: + if self._abort_requested.is_set(): + self._writer.abort() + else: + self._writer.finalize() def finalize(self) -> None: - """Flush the queue, join the drain thread, and finalize the ZarrWriter.""" + """Flush the queue, join the drain thread (which finalizes the ZarrWriter).""" self._q.put(_SENTINEL) - self._thread.join() - self._writer.finalize() + self._thread.join(timeout=30.0) + if self._thread.is_alive(): + _log.warning("drain thread still alive after finalize() join timeout") def abort(self) -> None: - """Signal the drain thread to stop, then abort the ZarrWriter.""" + """Signal the drain thread to stop (which aborts the ZarrWriter).""" + self._abort_requested.set() try: self._q.put_nowait(_SENTINEL) except queue.Full: pass - self._thread.join(timeout=2.0) - self._writer.abort() + self._thread.join(timeout=5.0) + if self._thread.is_alive(): + _log.warning("drain thread still alive after abort() join timeout") From 3be2db40de211398f5f0c6d1eeda24e137a47b77 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 04:04:12 -0700 Subject: [PATCH 09/61] feat(streaming): ContinuousFrameSource + StreamingCapture orchestrator Appends ContinuousFrameSource (wraps AbstractCamera into start/stop source) and StreamingCapture (orchestrates source, router, stop-condition, writer) to streaming_capture.py. run() accepts an optional timeout parameter so Task D can bound wall-clock wait without hanging on a stalled camera. Adds fake-source test confirming 5-frame emit + downsampling + finalize. Co-Authored-By: Claude Sonnet 4.6 --- software/control/core/streaming_capture.py | 90 ++++++++++++++++++- software/tests/core/test_streaming_capture.py | 62 ++++++++++++- 2 files changed, 150 insertions(+), 2 deletions(-) diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index a17237a23..7b0395469 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -1,6 +1,6 @@ import queue import threading -from typing import Optional, Tuple +from typing import Callable, Optional, Tuple import numpy as np @@ -115,3 +115,91 @@ def abort(self) -> None: self._thread.join(timeout=5.0) if self._thread.is_alive(): _log.warning("drain thread still alive after abort() join timeout") + + +# --------------------------------------------------------------------------- +# Task C3: ContinuousFrameSource + StreamingCapture +# --------------------------------------------------------------------------- + +from squid.abc import CameraAcquisitionMode # noqa: E402 — kept at bottom to avoid circular import + + +class ContinuousFrameSource: + """Wraps a camera and delivers frames via callback. + + Calls set_frame_rate, set_acquisition_mode(CONTINUOUS), registers a frame + callback, and starts/stops streaming. + """ + + def __init__(self, camera, fps: float): + self._camera = camera + self._fps = fps + self._cb_id: Optional[int] = None + + def start(self, on_frame: Callable) -> None: + self._camera.set_frame_rate(self._fps) + self._camera.set_acquisition_mode(CameraAcquisitionMode.CONTINUOUS) + self._cb_id = self._camera.add_frame_callback(on_frame) + self._camera.start_streaming() + + def stop(self) -> None: + self._camera.stop_streaming() + if self._cb_id is not None: + self._camera.remove_frame_callback(self._cb_id) + self._cb_id = None + + +class StreamingCapture: + """Orchestrates a frame source, router, stop condition, and writer. + + ``run()`` starts the source, routes each incoming frame through the router, + enqueues accepted frames to the writer, and stops when the stop condition is + met or ``abort_fn`` returns True. + + The ``_on_frame`` callback runs on the hot camera thread — it must stay cheap + (route + enqueue only, no blocking I/O). + + Args: + frame_source: Any object with ``start(on_frame)`` / ``stop()`` interface. + router: ``RecordingRouter`` (or compatible) — maps timestamps to (t,c,z). + stop_condition: ``CountStop`` (or compatible) — ``met(emitted)`` returns bool. + writer: Object with ``start()``, ``enqueue(frame,t,c,z)``, ``finalize()``, ``abort()``. + abort_fn: Zero-argument callable; returns True to abort early. + timeout: Optional seconds to wait for completion. If the source does not + trigger the done event within this time ``run()`` still stops and + finalizes (returns frames emitted so far). None means wait forever. + """ + + def __init__(self, frame_source, router, stop_condition, writer, abort_fn: Callable[[], bool]): + self._source = frame_source + self._router = router + self._stop = stop_condition + self._writer = writer + self._abort_fn = abort_fn + self._emitted = 0 + self._done = threading.Event() + + def _on_frame(self, camera_frame) -> None: + """Hot-thread callback: route + enqueue only. Must not block.""" + if self._done.is_set(): + return + if self._abort_fn(): + self._done.set() + return + idx = self._router.route(camera_frame.timestamp) + if idx is not None: + self._writer.enqueue(camera_frame.frame, *idx) + self._emitted += 1 + if self._stop.met(self._emitted): + self._done.set() + + def run(self, timeout: Optional[float] = None) -> int: + """Start capture, block until done (or timeout), and return emitted count.""" + self._writer.start() + try: + self._source.start(self._on_frame) + self._done.wait(timeout) # FakeSource sets _done synchronously; real camera via callback + finally: + self._source.stop() + self._writer.finalize() + return self._emitted diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py index 6fb545480..2710fa21c 100644 --- a/software/tests/core/test_streaming_capture.py +++ b/software/tests/core/test_streaming_capture.py @@ -1,6 +1,6 @@ import numpy as np -from control.core.streaming_capture import CountStop, RecordingRouter +from control.core.streaming_capture import CountStop, RecordingRouter, StreamingCapture def test_count_stop(): @@ -45,3 +45,63 @@ def test_recording_writer_roundtrip(tmp_path): ds = ts.open({"driver": "zarr3", "kvstore": {"driver": "file", "path": cfg.output_path}}).result() assert tuple(ds.shape) == (T, 1, 1, Y, X) assert int(ds[2, 0, 0, 0, 0].read().result()) == 3 + + +# --------------------------------------------------------------------------- +# Task C3: StreamingCapture + ContinuousFrameSource tests (fake source) +# --------------------------------------------------------------------------- + + +class _FakeFrame: + def __init__(self, ts, arr): + self.timestamp = ts + self.frame = arr + + +class _FakeSource: + """Delivers N frames synchronously when started.""" + + def __init__(self, frames): + self._frames = frames + self._cb = None + + def start(self, on_frame): + self._cb = on_frame + for f in self._frames: + self._cb(f) + + def stop(self): + pass + + +class _ListWriter: + def __init__(self): + self.writes = [] + + def start(self): + pass + + def enqueue(self, frame, t, c, z): + self.writes.append((t, c, z)) + + def finalize(self): + self.finalized = True + + def abort(self): + pass + + +def test_streaming_capture_counts_and_downsamples(): + frames = [_FakeFrame(100.0 + i * 0.05, np.zeros((4, 4), np.uint16)) for i in range(20)] + w = _ListWriter() + cap = StreamingCapture( + _FakeSource(frames), + RecordingRouter(fps=10.0), + CountStop(5), + w, + abort_fn=lambda: False, + ) + emitted = cap.run() + assert emitted == 5 + assert w.writes == [(0, 0, 0), (1, 0, 0), (2, 0, 0), (3, 0, 0), (4, 0, 0)] + assert getattr(w, "finalized", False) is True From 5d787738b8aaaa6e483f0965fa28de3f09281bfd Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 04:11:54 -0700 Subject: [PATCH 10/61] fix(streaming): move CameraAcquisitionMode import to top (no real cycle); document stop/finalize assumption Co-Authored-By: Claude Sonnet 4.6 --- software/control/core/streaming_capture.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index 7b0395469..d55c5048c 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -6,6 +6,7 @@ import squid.logging from control.core.zarr_writer import ZarrAcquisitionConfig, ZarrWriter +from squid.abc import CameraAcquisitionMode _log = squid.logging.get_logger("RecordingWriter") _SENTINEL = object() @@ -121,8 +122,6 @@ def abort(self) -> None: # Task C3: ContinuousFrameSource + StreamingCapture # --------------------------------------------------------------------------- -from squid.abc import CameraAcquisitionMode # noqa: E402 — kept at bottom to avoid circular import - class ContinuousFrameSource: """Wraps a camera and delivers frames via callback. @@ -200,6 +199,11 @@ def run(self, timeout: Optional[float] = None) -> int: self._source.start(self._on_frame) self._done.wait(timeout) # FakeSource sets _done synchronously; real camera via callback finally: + # Assumes source.stop() quiesces the camera delivery thread. With cameras + # that don't join their callback thread on stop, a final in-flight frame may + # reach writer.enqueue after finalize — harmless with RecordingWriter (the + # drain thread has exited, so the put times out and the frame is logged as + # dropped, not corrupted). self._source.stop() self._writer.finalize() return self._emitted From aac45186c9ef38b46b6cc565cc2a2753564f91a9 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 04:13:55 -0700 Subject: [PATCH 11/61] feat(record-zstack): params + pure planning helpers Add RecordZStackAcquisitionParameters dataclass and pure helper functions frame_count, zstack_plane_count, and zstack_offsets_um for z-stack and recording acquisition planning. These helpers handle frame counting, z-plane calculation with epsilon tolerance, and offset list generation. Includes full test coverage. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../control/core/record_zstack_controller.py | 40 +++++++++++++++++++ .../tests/core/test_record_zstack_worker.py | 13 ++++++ 2 files changed, 53 insertions(+) create mode 100644 software/control/core/record_zstack_controller.py create mode 100644 software/tests/core/test_record_zstack_worker.py diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py new file mode 100644 index 000000000..8195c0292 --- /dev/null +++ b/software/control/core/record_zstack_controller.py @@ -0,0 +1,40 @@ +import math +from dataclasses import dataclass, field +from typing import List, Optional + +from control.models.acquisition_config import AcquisitionChannel + + +def frame_count(fps: float, duration_s: float) -> int: + return int(round(fps * duration_s)) + + +def zstack_plane_count(z_min_um: float, z_max_um: float, step_um: float) -> int: + if step_um <= 0 or z_max_um < z_min_um: + raise ValueError("require step>0 and z_max>=z_min") + return int(math.floor((z_max_um - z_min_um) / step_um + 1e-9)) + 1 + + +def zstack_offsets_um(z_min_um: float, z_max_um: float, step_um: float) -> List[float]: + return [round(z_min_um + i * step_um, 6) for i in range(zstack_plane_count(z_min_um, z_max_um, step_um))] + + +@dataclass +class RecordZStackAcquisitionParameters: + base_path: str + experiment_id: str + Nt: int = 1 + dt_s: float = 0.0 + use_laser_af: bool = False + # recording phase + recording_enabled: bool = False + recording_channel: Optional[AcquisitionChannel] = None + fps: float = 10.0 + duration_s: float = 1.0 + recording_z_offset_um: float = 0.0 + # z-stack phase + zstack_enabled: bool = False + zstack_channels: List[AcquisitionChannel] = field(default_factory=list) + z_min_um: float = -3.0 + z_max_um: float = 3.0 + z_step_um: float = 1.0 diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py new file mode 100644 index 000000000..318c622e2 --- /dev/null +++ b/software/tests/core/test_record_zstack_worker.py @@ -0,0 +1,13 @@ +from control.core.record_zstack_controller import frame_count, zstack_plane_count, zstack_offsets_um + + +def test_frame_count(): + assert frame_count(10.0, 30.0) == 300 + assert frame_count(7.5, 2.0) == 15 + + +def test_zstack_plane_count_and_offsets(): + assert zstack_plane_count(-3.0, 3.0, 1.0) == 7 + assert zstack_offsets_um(-3.0, 3.0, 1.0) == [-3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0] + assert zstack_plane_count(0.0, 5.0, 2.0) == 3 # 0,2,4 + assert zstack_offsets_um(0.0, 5.0, 2.0) == [0.0, 2.0, 4.0] From e0eb6bd160b57d60a61bc7d32f1f160423fa6f4b Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 04:16:00 -0700 Subject: [PATCH 12/61] test(record-zstack): cover validation paths + document epsilon Address code review: add ValueError tests for invalid z-stack inputs (z_max --- software/control/core/record_zstack_controller.py | 1 + software/tests/core/test_record_zstack_worker.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py index 8195c0292..c2d37a482 100644 --- a/software/control/core/record_zstack_controller.py +++ b/software/control/core/record_zstack_controller.py @@ -12,6 +12,7 @@ def frame_count(fps: float, duration_s: float) -> int: def zstack_plane_count(z_min_um: float, z_max_um: float, step_um: float) -> int: if step_um <= 0 or z_max_um < z_min_um: raise ValueError("require step>0 and z_max>=z_min") + # epsilon absorbs float representation error so e.g. 6.0/1.0 -> 5.999... still floors to 6 return int(math.floor((z_max_um - z_min_um) / step_um + 1e-9)) + 1 diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index 318c622e2..7537b0979 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -1,3 +1,5 @@ +import pytest + from control.core.record_zstack_controller import frame_count, zstack_plane_count, zstack_offsets_um @@ -11,3 +13,12 @@ def test_zstack_plane_count_and_offsets(): assert zstack_offsets_um(-3.0, 3.0, 1.0) == [-3.0, -2.0, -1.0, 0.0, 1.0, 2.0, 3.0] assert zstack_plane_count(0.0, 5.0, 2.0) == 3 # 0,2,4 assert zstack_offsets_um(0.0, 5.0, 2.0) == [0.0, 2.0, 4.0] + + +def test_zstack_plane_count_validation(): + with pytest.raises(ValueError): + zstack_plane_count(3.0, -3.0, 1.0) # z_max < z_min + with pytest.raises(ValueError): + zstack_plane_count(0.0, 3.0, 0.0) # step == 0 + with pytest.raises(ValueError): + zstack_plane_count(0.0, 3.0, -1.0) # step < 0 From 691369d66a163a67437ac3732d8b7fc8e986d83a Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 04:30:46 -0700 Subject: [PATCH 13/61] feat(record-zstack): RecordZStackWorker (record + zstack per FOV) Add RecordZStackWorker(MultiPointWorkerBase) orchestrating per-FOV record + z-stack acquisition: - record() streams a continuous high-fps capture to a per-FOV recording .ome.zarr via the C3 StreamingCapture primitive - zstack() reuses the inherited triggered-capture + SaveZarrJob dispatch path (builds its own JobRunner + BackpressureController like MultiPointWorker), managing camera streaming/callback lifecycle locally - establish_reference() uses laser AF when available, falls back to current stage Z - run() loops time points -> regions -> FOVs, abort- and dt-aware Smoke test (simulated microscope, 2 wells x 2 FOV x 2 t, both phases) verifies recording dataset count/shape and z-stack per-FOV zarr shape. Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/core/record_zstack_worker.py | 473 ++++++++++++++++++ .../tests/core/test_record_zstack_worker.py | 135 +++++ 2 files changed, 608 insertions(+) create mode 100644 software/control/core/record_zstack_worker.py diff --git a/software/control/core/record_zstack_worker.py b/software/control/core/record_zstack_worker.py new file mode 100644 index 000000000..7425a797f --- /dev/null +++ b/software/control/core/record_zstack_worker.py @@ -0,0 +1,473 @@ +"""RecordZStackWorker — per-FOV orchestration for "Record + Z-Stack" acquisitions. + +For each (time point, region, FOV) the worker: + 1. moves XY to the FOV, + 2. establishes a Z reference (laser AF if requested+available, else current Z), + 3. (optional) records a continuous high-fps stream to a per-FOV recording zarr, + 4. (optional) acquires a software-triggered z-stack saved via the inherited + ``SaveZarrJob`` dispatch path. + +The recording phase reuses the C3 ``StreamingCapture`` primitive +(``ContinuousFrameSource`` + ``RecordingRouter`` + ``CountStop`` + +``RecordingWriter``). The z-stack phase reuses ``MultiPointWorkerBase``'s +shared single-frame capture + frame callback + job dispatch machinery, so the +worker builds its own ``JobRunner`` (or accepts a pre-warmed one) plus a +``BackpressureController`` exactly the way ``MultiPointWorker`` does. +""" + +import os +import time +from typing import Callable, Dict, List, Optional, Tuple, Type + +import numpy as np + +import squid.logging +import control._def +from control._def import ( + Acquisition, + SCAN_STABILIZATION_TIME_MS_X, + SCAN_STABILIZATION_TIME_MS_Y, + SCAN_STABILIZATION_TIME_MS_Z, +) +from squid.abc import CameraAcquisitionMode +from control.core.multi_point_worker import MultiPointWorkerBase +from control.core.streaming_capture import ( + StreamingCapture, + ContinuousFrameSource, + RecordingRouter, + CountStop, + RecordingWriter, +) +from control.core.zarr_writer import ZarrAcquisitionConfig +from control.core.record_zstack_controller import ( + RecordZStackAcquisitionParameters, + frame_count, + zstack_offsets_um, + zstack_plane_count, +) +from control.core.job_processing import ( + AcquisitionInfo, + Job, + JobRunner, + SaveZarrJob, + ZarrWriterInfo, +) +from control.core.backpressure import BackpressureController, BackpressureValues + +log = squid.logging.get_logger("RecordZStackWorker") + + +class RecordZStackWorker(MultiPointWorkerBase): + """Per-FOV record-and-z-stack acquisition worker. + + Scan coords are supplied as ``{region_id: [(x_mm, y_mm[, z_mm]), ...]}``. + """ + + def __init__( + self, + scope, + live_controller, + laser_auto_focus_controller, + objective_store, + params: RecordZStackAcquisitionParameters, + callbacks, + abort_requested_fn: Callable[[], bool], + request_abort_fn: Callable[[], None], + scan_region_fov_coords: Dict[object, List[Tuple]], + prewarmed_job_runner: Optional[JobRunner] = None, + prewarmed_bp_values: Optional[BackpressureValues] = None, + ): + super().__init__( + scope=scope, + live_controller=live_controller, + callbacks=callbacks, + abort_requested_fn=abort_requested_fn, + request_abort_fn=request_abort_fn, + ) + + self.params = params + self.laser_af = laser_auto_focus_controller + self.objectiveStore = objective_store + self._scan: Dict[object, List[Tuple]] = scan_region_fov_coords or {} + + # This worker drives the stage directly (no piezo z-stacking), and uses a + # single time_point at a time — refine the base placeholders. + self.use_piezo = False + self.time_point = 0 + + # Experiment output layout (mirrors MultiPointWorker.experiment_path). + self.base_path = params.base_path + self.experiment_ID = params.experiment_id + self.experiment_path = os.path.join(self.base_path or "", self.experiment_ID or "") + + # Z-stack channels drive the inherited SaveZarrJob path. + self.zstack_channels: List = list(params.zstack_channels or []) + self._NZ = ( + zstack_plane_count(params.z_min_um, params.z_max_um, params.z_step_um) if params.zstack_enabled else 1 + ) + + # Pre-compute acquisition-wide metadata (pixel size etc.) for zarr/job info. + try: + pixel_factor = self.objectiveStore.get_pixel_size_factor() + sensor_pixel_um = self.camera.get_pixel_size_binned_um() + if pixel_factor is not None and sensor_pixel_um is not None: + self._pixel_size_um = float(pixel_factor) * float(sensor_pixel_um) + else: + self._pixel_size_um = None + except Exception: + self._pixel_size_um = None + + self._time_increment_s = params.dt_s if params.Nt > 1 and params.dt_s > 0 else None + self._physical_size_z_um = abs(params.z_step_um) if self._NZ > 1 else None + + # Build the z-stack job runner + backpressure (only when a z-stack will run). + # Mirrors MultiPointWorker.__init__: per-FOV 5D non-HCS zarr output. + self.acquisition_info = AcquisitionInfo( + total_time_points=params.Nt, + total_z_levels=self._NZ, + total_channels=len(self.zstack_channels), + channel_names=[c.name for c in self.zstack_channels], + experiment_path=self.experiment_path, + time_increment_s=self._time_increment_s, + physical_size_z_um=self._physical_size_z_um, + physical_size_x_um=self._pixel_size_um, + physical_size_y_um=self._pixel_size_um, + ) + + self._backpressure: Optional[BackpressureController] = None + self._job_runners: List[Tuple[Type[Job], JobRunner]] = [] + self._abort_on_failed_job = True + self._first_job_dispatched = False + + if params.zstack_enabled and self.zstack_channels: + self._setup_zstack_job_runner(prewarmed_job_runner, prewarmed_bp_values) + else: + # Still need a (disabled) backpressure controller so base methods that + # reference self._backpressure don't crash if ever reached. + self._backpressure = BackpressureController(enabled=False) + + # ------------------------------------------------------------------ setup + def _setup_zstack_job_runner(self, prewarmed_job_runner, prewarmed_bp_values) -> None: + bp_kwargs = { + "max_jobs": control._def.ACQUISITION_MAX_PENDING_JOBS, + "max_mb": control._def.ACQUISITION_MAX_PENDING_MB, + "timeout_s": control._def.ACQUISITION_THROTTLE_TIMEOUT_S, + "enabled": control._def.ACQUISITION_THROTTLING_ENABLED, + } + if prewarmed_bp_values is not None: + bp_kwargs["bp_values"] = prewarmed_bp_values + self._backpressure = BackpressureController(**bp_kwargs) + + # Channel metadata for zarr output. + channel_names = [c.name for c in self.zstack_channels] + channel_colors = [c.display_color for c in self.zstack_channels] + channel_wavelengths: List[Optional[int]] = [] + illumination_config = self.microscope.config_repo.get_illumination_config() + for c in self.zstack_channels: + try: + w = c.get_illumination_wavelength(illumination_config) if illumination_config else None + except Exception: + w = None + channel_wavelengths.append(w) + + # FOV counts per region (non-HCS per-FOV 5D output -> count only used for 6D). + region_fov_counts = {str(region_id): len(coords) for region_id, coords in self._scan.items()} + + zarr_writer_info = ZarrWriterInfo( + base_path=self.experiment_path, + t_size=self.params.Nt, + c_size=len(self.zstack_channels), + z_size=self._NZ, + is_hcs=False, + use_6d_fov=False, + region_fov_counts=region_fov_counts, + pixel_size_um=self._pixel_size_um, + z_step_um=self._physical_size_z_um, + time_increment_s=self._time_increment_s, + channel_names=channel_names, + channel_colors=channel_colors, + channel_wavelengths=channel_wavelengths, + ) + + log_file_path = squid.logging.get_current_log_file_path() + can_use_prewarmed = prewarmed_job_runner is not None and prewarmed_bp_values is not None + + job_runner: Optional[JobRunner] = None + if Acquisition.USE_MULTIPROCESSING: + if can_use_prewarmed and prewarmed_job_runner.is_ready(): + log.info("Using pre-warmed job runner for SaveZarrJob jobs") + job_runner = prewarmed_job_runner + job_runner.set_acquisition_info(self.acquisition_info) + job_runner.set_zarr_writer_info(zarr_writer_info) + else: + if can_use_prewarmed: + log.warning("Pre-warmed job runner not ready; shutting it down and creating a new one") + try: + prewarmed_job_runner.shutdown(timeout_s=1.0) + except Exception as e: + log.error(f"Error shutting down hung pre-warmed runner: {e}") + log.info("Creating job runner for SaveZarrJob jobs") + job_runner = JobRunner( + self.acquisition_info, + cleanup_stale_ome_files=False, + log_file_path=log_file_path, + bp_pending_jobs=self._backpressure.pending_jobs_value, + bp_pending_bytes=self._backpressure.pending_bytes_value, + bp_capacity_event=self._backpressure.capacity_event, + zarr_writer_info=zarr_writer_info, + ) + job_runner.start() + + self._job_runners = [(SaveZarrJob, job_runner)] + + # -------------------------------------------------------------------- run + def run(self): + """Top-level orchestration loop (runs on the acquisition thread). + + Recording manages its own CONTINUOUS streaming/callback via + ``ContinuousFrameSource``; the z-stack phase manages its own + software-trigger streaming + frame callback inside ``zstack()``. The + camera is left stopped between FOVs/phases, so this loop owns no camera + callback of its own. + """ + try: + if self.params.zstack_enabled and self._job_runners: + self._backpressure.reset() + + for t_idx in range(self.params.Nt): + self.time_point = t_idx + if t_idx > 0 and self.params.dt_s > 0: + if not self._wait_for_dt(t_idx): + break + if self.abort_requested_fn(): + break + + for region_id, fovs in self._scan.items(): + for fov_idx, coord in enumerate(fovs): + if self.abort_requested_fn(): + return + self._move_xy(coord) + z_ref = self.establish_reference() + + if self.params.recording_enabled: + self.record(t_idx, region_id, fov_idx, z_ref) + if self.params.zstack_enabled and self.zstack_channels: + self.zstack(t_idx, region_id, fov_idx, z_ref) + except Exception as e: + log.exception(e) + self.request_abort_fn() + finally: + # Drain + shut down z-stack job runners (also closes backpressure). + if self._job_runners: + try: + self._finish_jobs() + except Exception: + log.exception("Error finishing z-stack jobs") + try: + self.callbacks.signal_acquisition_finished() + except Exception: + log.exception("signal_acquisition_finished callback failed") + + # ------------------------------------------------------------- reference + def establish_reference(self) -> float: + """Return the Z reference (mm) for this FOV. + + Uses laser AF ``move_to_target(0)`` when requested and a reference is set; + on failure (raise or soft-fail return) falls back to the current stage Z. + """ + if self.params.use_laser_af and self.laser_af is not None and self._laser_af_has_reference(): + try: + ok = self.laser_af.move_to_target(0.0) + if not ok: + log.warning("laser AF move_to_target(0) reported failure; using current Z") + except Exception as e: + log.warning(f"laser AF failed at FOV, falling back to current Z: {e}") + return self.stage.get_pos().z_mm + + def _laser_af_has_reference(self) -> bool: + try: + return bool(self.laser_af.laser_af_properties.has_reference) + except Exception: + return False + + # ---------------------------------------------------------------- record + def record(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> int: + """Record a continuous stream to a per-FOV recording zarr, then restore Z. + + Returns the number of frames emitted. + """ + # Quiesce live view if one is running (guarded for the headless/test case). + was_live = bool(getattr(self.liveController, "is_live", False)) + if was_live: + try: + self.liveController.stop_live() + except Exception: + log.exception("Failed to stop live view before recording") + + # Apply the recording channel (exposure/gain/illumination). + if self.params.recording_channel is not None: + self._select_config(self.params.recording_channel) + + # Move to z_ref + recording offset. + self._move_z_to_offset(z_ref, self.params.recording_z_offset_um) + + T = frame_count(self.params.fps, self.params.duration_s) + out = self._recording_path(t_idx, region_id, fov_idx) + y, x, dtype = self._probe_frame_shape() + + rec_channel_name = self.params.recording_channel.name if self.params.recording_channel is not None else "REC" + rec_color = ( + self.params.recording_channel.display_color if self.params.recording_channel is not None else "#FFFFFF" + ) + cfg = ZarrAcquisitionConfig( + output_path=out, + shape=(T, 1, 1, y, x), + dtype=dtype, + pixel_size_um=self._pixel_size_um if self._pixel_size_um is not None else 1.0, + z_step_um=None, + time_increment_s=(1.0 / self.params.fps) if self.params.fps and self.params.fps > 0 else None, + channel_names=[rec_channel_name], + channel_colors=[rec_color], + channel_wavelengths=[None], + is_hcs=False, + ) + writer = RecordingWriter(cfg) + cap = StreamingCapture( + ContinuousFrameSource(self.camera, self.params.fps), + RecordingRouter(self.params.fps), + CountStop(T), + writer, + abort_fn=self.abort_requested_fn, + ) + # Generous timeout: enough to gather T frames even at a slow effective rate. + timeout = self.params.duration_s * 3 + 5 + emitted = cap.run(timeout=timeout) + log.info(f"recording done t={t_idx} region={region_id} fov={fov_idx}: {emitted}/{T} frames") + + # Restore the camera to a software-trigger-friendly state and Z reference. + try: + self.camera.set_acquisition_mode(CameraAcquisitionMode.SOFTWARE_TRIGGER) + except Exception: + log.exception("Failed to restore software-trigger acquisition mode after recording") + self.stage.move_z_to(z_ref) + self.wait_till_operation_is_completed() + self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) + return emitted + + # ---------------------------------------------------------------- zstack + def zstack(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> None: + """Acquire a software-triggered z-stack via the inherited capture path. + + Each plane/channel goes through ``acquire_camera_image`` -> ``_image_callback`` + -> ``SaveZarrJob`` dispatch. Restores Z to ``z_ref`` at the end. + """ + self.time_point = t_idx + offsets = zstack_offsets_um(self.params.z_min_um, self.params.z_max_um, self.params.z_step_um) + + # The inherited capture path needs the camera streaming, in software-trigger + # mode, with our frame callback registered. Manage that lifecycle locally so + # it never interferes with the recording phase's CONTINUOUS streaming. + try: + self.camera.set_acquisition_mode(CameraAcquisitionMode.SOFTWARE_TRIGGER) + except Exception: + log.exception("Failed to set software-trigger mode for z-stack") + self.camera.start_streaming() + cb_id = self.camera.add_frame_callback(self._image_callback) + # Make sure the trigger gate starts open. + self._ready_for_next_trigger.set() + + current_path = self._zstack_dir(region_id) + + try: + for z_idx, off_um in enumerate(offsets): + self._move_z_to_offset(z_ref, off_um) + for c_idx, config in enumerate(self.zstack_channels): + if self.abort_requested_fn(): + return + file_id = f"{region_id}_{fov_idx}" + self.acquire_camera_image( + config=config, + file_ID=file_id, + current_path=current_path, + k=z_idx, + region_id=region_id, + fov=fov_idx, + config_idx=c_idx, + ) + finally: + # Wait for the last in-flight frame, then detach our callback. + self._wait_for_outstanding_callback_images() + try: + self.camera.remove_frame_callback(cb_id) + except Exception: + log.exception("Failed to remove z-stack frame callback") + self.stage.move_z_to(z_ref) + self.wait_till_operation_is_completed() + self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) + + # --------------------------------------------------------------- helpers + def _wait_for_outstanding_callback_images(self) -> None: + """Block until any in-flight triggered frame has been dispatched/processed.""" + if not self._ready_for_next_trigger.wait(self._frame_wait_timeout_s()): + log.warning("Timed out waiting for outstanding z-stack frames") + if not self._image_callback_idle.wait(self._frame_wait_timeout_s()): + log.warning("Timed out waiting for last z-stack image to process") + self._ready_for_next_trigger.set() + self._image_callback_idle.set() + + def _move_xy(self, coord) -> None: + self.stage.move_x_to(coord[0]) + self._sleep(SCAN_STABILIZATION_TIME_MS_X / 1000) + self.stage.move_y_to(coord[1]) + self._sleep(SCAN_STABILIZATION_TIME_MS_Y / 1000) + + def _move_z_to_offset(self, z_ref: float, offset_um: float) -> None: + """Move to ``z_ref + offset_um`` (offset in µm, z_ref in mm) via the stage.""" + target_mm = z_ref + offset_um / 1000.0 + self.stage.move_z_to(target_mm) + self.wait_till_operation_is_completed() + self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) + + def _probe_frame_shape(self) -> Tuple[int, int, np.dtype]: + """Return (Y, X, dtype) for the recording dataset from the camera resolution.""" + width, height = self.camera.get_resolution() + # Map pixel format to numpy dtype (MONO8 -> uint8, else uint16). + try: + from squid.config import CameraPixelFormat + + fmt = self.camera.get_pixel_format() + dtype = np.uint8 if fmt == CameraPixelFormat.MONO8 else np.uint16 + except Exception: + dtype = np.uint16 + return int(height), int(width), np.dtype(dtype) + + def _recording_path(self, t_idx: int, region_id, fov_idx: int) -> str: + """Per-(t, region, fov) recording dataset path under {experiment}/recording.""" + return os.path.join( + self.experiment_path, + "recording", + f"t{t_idx}", + str(region_id), + f"fov_{fov_idx}.ome.zarr", + ) + + def _zstack_dir(self, region_id) -> str: + """Directory passed as ``current_path`` to the inherited capture path. + + SaveZarrJob ignores this (it builds its own path from ZarrWriterInfo), but + SaveImageJob/SaveOMETiffJob would use it, so keep it valid. + """ + path = os.path.join(self.experiment_path, "zstack", str(region_id)) + return path + + def _wait_for_dt(self, t_idx: int) -> bool: + """Sleep until it's time for time point ``t_idx`` (abort-aware). + + Returns False if an abort was requested while waiting. + """ + deadline = time.time() + self.params.dt_s + while time.time() < deadline: + if self.abort_requested_fn(): + return False + self._sleep(min(0.1, max(0.0, deadline - time.time()))) + return True diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index 7537b0979..729531675 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from control.core.record_zstack_controller import frame_count, zstack_plane_count, zstack_offsets_um @@ -22,3 +24,136 @@ def test_zstack_plane_count_validation(): zstack_plane_count(0.0, 3.0, 0.0) # step == 0 with pytest.raises(ValueError): zstack_plane_count(0.0, 3.0, -1.0) # step < 0 + + +# --------------------------------------------------------------------------- +# Task D2: RecordZStackWorker smoke test (simulated microscope + JobRunner) +# +# 2 wells x 2 FOV x 2 time points, both phases (recording + z-stack) enabled. +# Asserts: +# - one recording .ome.zarr per (t, well, fov) with shape (T,1,1,Y,X) +# - the z-stack per-FOV zarr datasets exist +# --------------------------------------------------------------------------- + + +def _build_simulated_microscope(crop_w: int, crop_h: int): + """Build a simulated microscope and shrink the camera frame for a fast test. + + The simulated camera reports its resolution from crop_width/crop_height, so + mutating the (loaded) config shrinks every captured/recorded frame. + """ + import control.microscope + + scope = control.microscope.Microscope.build_from_global_config(True) + scope.camera._config.crop_width = crop_w + scope.camera._config.crop_height = crop_h + return scope + + +def test_record_zstack_worker_smoke(tmp_path): + import control._def + import tests.control.test_stubs as ts + from control.core.multi_point_controller import NoOpCallbacks + from control.core.record_zstack_controller import RecordZStackAcquisitionParameters, frame_count + from control.core.record_zstack_worker import RecordZStackWorker + + # Use real Zarr v3 saving for the z-stack phase (the path under test). + control._def.FILE_SAVING_OPTION = control._def.FileSavingOption.ZARR_V3 + + crop_w, crop_h = 64, 48 + scope = _build_simulated_microscope(crop_w, crop_h) + live_controller = ts.get_test_live_controller(scope, scope.objective_store.default_objective) + laser_af = ts.get_test_laser_autofocus_controller(scope) + + channels = live_controller.get_channels(scope.objective_store.default_objective) + assert len(channels) >= 2 + recording_channel = channels[0] + zstack_channels = channels[:2] + + # Keep exposure short so capture/recording is fast. + scope.camera.set_exposure_time(1) + + # Move the stage to a safe mid-Z so z_ref +/- offsets stay in range. + z_cfg = scope.stage.get_config().Z_AXIS + z_mid = (z_cfg.MAX_POSITION + z_cfg.MIN_POSITION) / 2.0 + scope.stage.move_z_to(z_mid) + scope.low_level_drivers.microcontroller.wait_till_operation_is_completed() + + x_cfg = scope.stage.get_config().X_AXIS + y_cfg = scope.stage.get_config().Y_AXIS + x0 = x_cfg.MIN_POSITION + 1.0 + y0 = y_cfg.MIN_POSITION + 1.0 + # 2 wells x 2 FOV + scan_region_fov_coords = { + "A1": [(x0, y0), (x0 + 0.5, y0)], + "A2": [(x0 + 1.0, y0), (x0 + 1.5, y0)], + } + n_wells = len(scan_region_fov_coords) + n_fov = 2 + Nt = 2 + + fps = 10.0 + duration_s = 0.3 + T = frame_count(fps, duration_s) + assert T >= 1 + + params = RecordZStackAcquisitionParameters( + base_path=str(tmp_path), + experiment_id="rec_zstack_smoke", + Nt=Nt, + dt_s=0.0, + use_laser_af=False, # no reference set -> would fall back anyway + recording_enabled=True, + recording_channel=recording_channel, + fps=fps, + duration_s=duration_s, + recording_z_offset_um=0.0, + zstack_enabled=True, + zstack_channels=zstack_channels, + z_min_um=-1.0, + z_max_um=1.0, + z_step_um=1.0, # 3 planes + ) + + aborted = {"v": False} + + worker = RecordZStackWorker( + scope=scope, + live_controller=live_controller, + laser_auto_focus_controller=laser_af, + objective_store=scope.objective_store, + params=params, + callbacks=NoOpCallbacks, + abort_requested_fn=lambda: aborted["v"], + request_abort_fn=lambda: aborted.__setitem__("v", True), + scan_region_fov_coords=scan_region_fov_coords, + ) + + worker.run() + + assert not aborted["v"], "worker requested abort during the smoke run" + + base = Path(params.base_path) / params.experiment_id + + # Recording: one dataset per (t, well, fov). + rec = sorted((base / "recording").rglob("*.ome.zarr")) + assert len(rec) == Nt * n_wells * n_fov, f"expected {Nt * n_wells * n_fov} recordings, got {len(rec)}: {rec}" + + # Each recording dataset has shape (T, 1, 1, Y, X). + import tensorstore as tstore + + for path in rec: + ds = tstore.open({"driver": "zarr3", "kvstore": {"driver": "file", "path": str(path)}}).result() + assert tuple(ds.shape) == (T, 1, 1, crop_h, crop_w), f"bad recording shape {ds.shape} for {path}" + + # Z-stack: per-FOV zarr datasets exist under {experiment}/zarr (non-HCS per-FOV layout). + zstack_zarrs = list((base / "zarr").rglob("fov_*.ome.zarr")) + assert zstack_zarrs, f"no z-stack zarr datasets found under {base / 'zarr'}" + # One per (well, fov) — written across all time points/z/channels. + assert len(zstack_zarrs) == n_wells * n_fov, f"expected {n_wells * n_fov} z-stack fovs, got {len(zstack_zarrs)}" + + # Verify z-stack shape (T, C, Z, Y, X). + n_z = len(zstack_offsets_um(params.z_min_um, params.z_max_um, params.z_step_um)) + for path in zstack_zarrs: + ds = tstore.open({"driver": "zarr3", "kvstore": {"driver": "file", "path": str(path / "0")}}).result() + assert tuple(ds.shape) == (Nt, len(zstack_channels), n_z, crop_h, crop_w), f"bad z-stack shape {ds.shape}" From fb93711866af18eac070a41739b64900e6af0972 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 04:41:51 -0700 Subject: [PATCH 14/61] feat(record-zstack): RecordZStackController + ZarrWriter daemon-thread fix - Add RecordZStackController to record_zstack_controller.py: builds params via setters, pre-warms a JobRunner subprocess (mirrors MultiPointController), resolves a timestamped experiment dir via create_experiment_dir, constructs RecordZStackWorker, and spawns it on a daemon thread. Exposes request_abort(), join(), close(), and per-param setters for the widget. - Fix ZarrWriter._get_loop(): after calling get_event_loop() check that the returned loop is not already closed; if it is, fall through to new_event_loop() so recordings on daemon threads across multiple time-points don't raise 'Event loop is closed'. - Add test_record_zstack_controller_smoke: controller-driven test covering Nt=2 with dt_s=0.1, both recording and z-stack phases, shape assertions. Co-Authored-By: Claude Sonnet 4.6 --- .../control/core/record_zstack_controller.py | 240 ++++++++++++++++++ software/control/core/zarr_writer.py | 6 +- .../tests/core/test_record_zstack_worker.py | 118 +++++++++ 3 files changed, 363 insertions(+), 1 deletion(-) diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py index c2d37a482..cec0b5236 100644 --- a/software/control/core/record_zstack_controller.py +++ b/software/control/core/record_zstack_controller.py @@ -1,9 +1,14 @@ import math from dataclasses import dataclass, field +from threading import Thread from typing import List, Optional +import squid.logging +import control._def from control.models.acquisition_config import AcquisitionChannel +log = squid.logging.get_logger("RecordZStackController") + def frame_count(fps: float, duration_s: float) -> int: return int(round(fps * duration_s)) @@ -39,3 +44,238 @@ class RecordZStackAcquisitionParameters: z_min_um: float = -3.0 z_max_um: float = 3.0 z_step_um: float = 1.0 + + +class RecordZStackController: + """Controller that builds params, sets up the pre-warmed JobRunner, and spawns + RecordZStackWorker on a daemon thread. + + Constructor arguments mirror MultiPointController's signature where the concept + overlaps; widget setters are provided for every RecordZStackAcquisitionParameters + field so the GUI (or tests) can push values in before calling run_acquisition(). + """ + + def __init__( + self, + microscope, + live_controller, + laser_autofocus_controller, + objective_store, + scan_coordinates, + callbacks, + ): + self._microscope = microscope + self._live_controller = live_controller + self._laser_af = laser_autofocus_controller + self._objective_store = objective_store + self._scan_coordinates = scan_coordinates + self._callbacks = callbacks + + # Mutable params pushed by the widget before run_acquisition(). + self.base_path: Optional[str] = None + self.experiment_id: str = "record_zstack" + self.Nt: int = 1 + self.dt_s: float = 0.0 + self.use_laser_af: bool = False + self.recording_enabled: bool = False + self.recording_channel: Optional[AcquisitionChannel] = None + self.fps: float = 10.0 + self.duration_s: float = 1.0 + self.recording_z_offset_um: float = 0.0 + self.zstack_enabled: bool = False + self.zstack_channels: List[AcquisitionChannel] = [] + self.z_min_um: float = -3.0 + self.z_max_um: float = 3.0 + self.z_step_um: float = 1.0 + + self._abort_requested: bool = False + self._worker = None + self._thread: Optional[Thread] = None + + # Pre-warm a job runner subprocess at init so it is ready when the user + # clicks "Start Acquisition" (mirrors MultiPointController.__init__). + self._prewarmed_job_runner = None + self._prewarmed_bp_values = None + if control._def.Acquisition.USE_MULTIPROCESSING: + self._start_prewarmed_job_runner() + + # ---------------------------------------------------------------------- pre-warm + + def _start_prewarmed_job_runner(self) -> None: + from control.core.job_processing import JobRunner + from control.core.backpressure import create_backpressure_values + + log.info("Pre-warming job runner subprocess for RecordZStack...") + self._prewarmed_bp_values = create_backpressure_values() + self._prewarmed_job_runner = JobRunner( + bp_pending_jobs=self._prewarmed_bp_values[0], + bp_pending_bytes=self._prewarmed_bp_values[1], + bp_capacity_event=self._prewarmed_bp_values[2], + ) + self._prewarmed_job_runner.start() + + def _get_prewarmed_job_runner(self): + """Consume the pre-warmed runner (start a fresh one for next time). + + Returns (runner, bp_values) or (None, None) when multiprocessing is off. + """ + runner = self._prewarmed_job_runner + bp_values = self._prewarmed_bp_values + self._prewarmed_job_runner = None + self._prewarmed_bp_values = None + if control._def.Acquisition.USE_MULTIPROCESSING: + self._start_prewarmed_job_runner() + return runner, bp_values + + def _cleanup_prewarmed_runner(self, runner, timeout_s: float = 1.0, context: str = "") -> None: + if runner is not None: + try: + runner.shutdown(timeout_s=timeout_s) + except Exception as e: + log.error(f"Error shutting down pre-warmed runner {context}: {e}") + + # ---------------------------------------------------------------------- widget setters + + def set_base_path(self, path: str) -> None: + self.base_path = path + + def set_experiment_id(self, experiment_id: str) -> None: + self.experiment_id = experiment_id + + def set_Nt(self, Nt: int) -> None: + self.Nt = Nt + + def set_dt_s(self, dt_s: float) -> None: + self.dt_s = dt_s + + def set_use_laser_af(self, flag: bool) -> None: + self.use_laser_af = flag + + def set_recording_enabled(self, flag: bool) -> None: + self.recording_enabled = flag + + def set_recording_channel(self, channel: Optional[AcquisitionChannel]) -> None: + self.recording_channel = channel + + def set_fps(self, fps: float) -> None: + self.fps = fps + + def set_duration_s(self, duration_s: float) -> None: + self.duration_s = duration_s + + def set_recording_z_offset_um(self, offset_um: float) -> None: + self.recording_z_offset_um = offset_um + + def set_zstack_enabled(self, flag: bool) -> None: + self.zstack_enabled = flag + + def set_zstack_channels(self, channels: List[AcquisitionChannel]) -> None: + self.zstack_channels = list(channels) + + def set_z_min_um(self, z_min_um: float) -> None: + self.z_min_um = z_min_um + + def set_z_max_um(self, z_max_um: float) -> None: + self.z_max_um = z_max_um + + def set_z_step_um(self, z_step_um: float) -> None: + self.z_step_um = z_step_um + + # ---------------------------------------------------------------------- acquisition + + def acquisition_in_progress(self) -> bool: + return self._thread is not None and self._thread.is_alive() + + def request_abort(self) -> None: + """Signal the running worker to stop after the current FOV.""" + self._abort_requested = True + + def run_acquisition(self) -> None: + """Build params, set up the pre-warmed JobRunner, and spawn the worker thread.""" + from control.core.acquisition_setup import create_experiment_dir + from control.core.record_zstack_worker import RecordZStackWorker + + # Resolve and create a timestamped unique output directory. + resolved_id, _dir = create_experiment_dir(self.base_path, self.experiment_id) + + params = RecordZStackAcquisitionParameters( + base_path=self.base_path, + experiment_id=resolved_id, + Nt=self.Nt, + dt_s=self.dt_s, + use_laser_af=self.use_laser_af, + recording_enabled=self.recording_enabled, + recording_channel=self.recording_channel, + fps=self.fps, + duration_s=self.duration_s, + recording_z_offset_um=self.recording_z_offset_um, + zstack_enabled=self.zstack_enabled, + zstack_channels=list(self.zstack_channels), + z_min_um=self.z_min_um, + z_max_um=self.z_max_um, + z_step_um=self.z_step_um, + ) + + # Collect scan coordinates: {region_id: [(x_mm, y_mm[, z_mm]), ...]} + scan_region_fov_coords = {} + if self._scan_coordinates is not None and hasattr(self._scan_coordinates, "region_fov_coordinates"): + scan_region_fov_coords = dict(self._scan_coordinates.region_fov_coordinates) + + # Reset abort flag for this run. + self._abort_requested = False + + # Consume the pre-warmed runner; only pass it to the worker when + # USE_MULTIPROCESSING is True (otherwise it's None and passing it would + # create a resource leak if the worker ignores non-multiprocessing paths). + prewarmed_runner, prewarmed_bp_values = self._get_prewarmed_job_runner() + + try: + self._worker = RecordZStackWorker( + scope=self._microscope, + live_controller=self._live_controller, + laser_auto_focus_controller=self._laser_af, + objective_store=self._objective_store, + params=params, + callbacks=self._callbacks, + abort_requested_fn=lambda: self._abort_requested, + request_abort_fn=self.request_abort, + scan_region_fov_coords=scan_region_fov_coords, + prewarmed_job_runner=prewarmed_runner if control._def.Acquisition.USE_MULTIPROCESSING else None, + prewarmed_bp_values=prewarmed_bp_values if control._def.Acquisition.USE_MULTIPROCESSING else None, + ) + except Exception: + # Clean up the pre-warmed runner if worker construction failed. + self._cleanup_prewarmed_runner(prewarmed_runner, context="after worker creation failure") + raise + + self._thread = Thread(target=self._worker.run, name="RecordZStack-acquisition", daemon=True) + self._thread.start() + + def join(self, timeout: Optional[float] = None) -> None: + """Wait for the acquisition thread to finish (useful in tests and scripts).""" + if self._thread is not None: + self._thread.join(timeout=timeout) + + # ---------------------------------------------------------------------- cleanup + + def close(self, timeout_s: float = 5.0) -> None: + """Abort any running acquisition and shut down the pre-warmed job runner.""" + if self._prewarmed_job_runner is not None: + log.info("Shutting down pre-warmed job runner for RecordZStackController...") + self._cleanup_prewarmed_runner( + self._prewarmed_job_runner, + timeout_s=1.0, + context="during close", + ) + self._prewarmed_job_runner = None + self._prewarmed_bp_values = None + + if self.acquisition_in_progress(): + self.request_abort() + if self._thread is not None: + self._thread.join(timeout=timeout_s) + if self._thread.is_alive(): + log.warning(f"RecordZStack acquisition thread did not stop within {timeout_s}s") + + self._worker = None + self._thread = None diff --git a/software/control/core/zarr_writer.py b/software/control/core/zarr_writer.py index 483c7a66e..5d34bd3e4 100644 --- a/software/control/core/zarr_writer.py +++ b/software/control/core/zarr_writer.py @@ -363,7 +363,11 @@ def _get_loop(self) -> asyncio.AbstractEventLoop: """Get or create the event loop (only used for init/finalize).""" if self._loop is None or self._loop.is_closed(): try: - self._loop = asyncio.get_event_loop() + candidate = asyncio.get_event_loop() + if candidate.is_closed(): + # The thread's current loop is closed; create a fresh one. + raise RuntimeError("current event loop is closed") + self._loop = candidate self._owns_loop = False # Using existing loop except RuntimeError: self._loop = asyncio.new_event_loop() diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index 729531675..22438d514 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -157,3 +157,121 @@ def test_record_zstack_worker_smoke(tmp_path): for path in zstack_zarrs: ds = tstore.open({"driver": "zarr3", "kvstore": {"driver": "file", "path": str(path / "0")}}).result() assert tuple(ds.shape) == (Nt, len(zstack_channels), n_z, crop_h, crop_w), f"bad z-stack shape {ds.shape}" + + +# --------------------------------------------------------------------------- +# Task D3: RecordZStackController smoke test +# +# Same 2-well x 2-FOV geometry as the D2 test but driven through +# RecordZStackController.run_acquisition() + join(). +# Also exercises Nt=2 with a short dt_s=0.1 (two time-points). +# --------------------------------------------------------------------------- + + +def test_record_zstack_controller_smoke(tmp_path): + import control._def + import tests.control.test_stubs as ts + from control.core.multi_point_controller import NoOpCallbacks + from control.core.record_zstack_controller import RecordZStackController, frame_count + from control.core.scan_coordinates import ScanCoordinates + + control._def.FILE_SAVING_OPTION = control._def.FileSavingOption.ZARR_V3 + + crop_w, crop_h = 64, 48 + scope = _build_simulated_microscope(crop_w, crop_h) + live_controller = ts.get_test_live_controller(scope, scope.objective_store.default_objective) + laser_af = ts.get_test_laser_autofocus_controller(scope) + + channels = live_controller.get_channels(scope.objective_store.default_objective) + assert len(channels) >= 2 + recording_channel = channels[0] + zstack_channels = channels[:2] + + scope.camera.set_exposure_time(1) + + z_cfg = scope.stage.get_config().Z_AXIS + z_mid = (z_cfg.MAX_POSITION + z_cfg.MIN_POSITION) / 2.0 + scope.stage.move_z_to(z_mid) + scope.low_level_drivers.microcontroller.wait_till_operation_is_completed() + + x_cfg = scope.stage.get_config().X_AXIS + y_cfg = scope.stage.get_config().Y_AXIS + x0 = x_cfg.MIN_POSITION + 1.0 + y0 = y_cfg.MIN_POSITION + 1.0 + + # Build a minimal ScanCoordinates with two regions, two FOVs each. + scan_coords = ScanCoordinates( + objectiveStore=scope.objective_store, + stage=scope.stage, + camera=scope.camera, + ) + scan_coords.region_fov_coordinates = { + "A1": [(x0, y0), (x0 + 0.5, y0)], + "A2": [(x0 + 1.0, y0), (x0 + 1.5, y0)], + } + n_wells = 2 + n_fov = 2 + Nt = 2 + + fps = 10.0 + duration_s = 0.3 + T = frame_count(fps, duration_s) + assert T >= 1 + + controller = RecordZStackController( + microscope=scope, + live_controller=live_controller, + laser_autofocus_controller=laser_af, + objective_store=scope.objective_store, + scan_coordinates=scan_coords, + callbacks=NoOpCallbacks, + ) + + # Push params via setters (as a widget would). + controller.set_base_path(str(tmp_path)) + controller.set_experiment_id("ctrl_smoke") + controller.set_Nt(Nt) + controller.set_dt_s(0.1) # short inter-timepoint delay + controller.set_use_laser_af(False) + controller.set_recording_enabled(True) + controller.set_recording_channel(recording_channel) + controller.set_fps(fps) + controller.set_duration_s(duration_s) + controller.set_recording_z_offset_um(0.0) + controller.set_zstack_enabled(True) + controller.set_zstack_channels(zstack_channels) + controller.set_z_min_um(-1.0) + controller.set_z_max_um(1.0) + controller.set_z_step_um(1.0) # 3 planes + + controller.run_acquisition() + # Wait up to 120 s for the worker thread to finish. + controller.join(timeout=120) + assert not controller.acquisition_in_progress(), "worker thread did not finish in time" + + # Determine the resolved experiment_id (has a timestamp appended). + subdirs = [d for d in Path(tmp_path).iterdir() if d.is_dir()] + assert len(subdirs) == 1, f"expected exactly one experiment dir, got {subdirs}" + base = subdirs[0] + + # Recording: one dataset per (t, well, fov). + rec = sorted((base / "recording").rglob("*.ome.zarr")) + assert len(rec) == Nt * n_wells * n_fov, f"expected {Nt * n_wells * n_fov} recordings, got {len(rec)}: {rec}" + + import tensorstore as tstore + + for path in rec: + ds = tstore.open({"driver": "zarr3", "kvstore": {"driver": "file", "path": str(path)}}).result() + assert tuple(ds.shape) == (T, 1, 1, crop_h, crop_w), f"bad recording shape {ds.shape} for {path}" + + # Z-stack: per-FOV zarr datasets exist. + zstack_zarrs = list((base / "zarr").rglob("fov_*.ome.zarr")) + assert zstack_zarrs, f"no z-stack zarr datasets found under {base / 'zarr'}" + assert len(zstack_zarrs) == n_wells * n_fov, f"expected {n_wells * n_fov} z-stack fovs, got {len(zstack_zarrs)}" + + n_z = len(zstack_offsets_um(-1.0, 1.0, 1.0)) + for path in zstack_zarrs: + ds = tstore.open({"driver": "zarr3", "kvstore": {"driver": "file", "path": str(path / "0")}}).result() + assert tuple(ds.shape) == (Nt, len(zstack_channels), n_z, crop_h, crop_w), f"bad z-stack shape {ds.shape}" + + controller.close() From 3d2a62cee92e49312ef01a284c289d78d5cf58be Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 04:53:31 -0700 Subject: [PATCH 15/61] feat(record-zstack): RecordZStackMultiPointWidget + validation Implements Task E1: widget skeleton (Option-A single-column layout with Output, Wells & FOV, Time-lapse/Focus, Recording phase, Z-Stack phase, Start groups), pure _validate_record_zstack_params() helper, validate(), build_parameters(), and _add_zstack_channel_row(). 18 tests pass (13 pure-helper, 5 widget via qtbot). Co-Authored-By: Claude Sonnet 4.6 --- software/control/widgets.py | 434 ++++++++++++++++++++ software/tests/test_record_zstack_widget.py | 280 +++++++++++++ 2 files changed, 714 insertions(+) create mode 100644 software/tests/test_record_zstack_widget.py diff --git a/software/control/widgets.py b/software/control/widgets.py index e1305e0ac..f13e7a166 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -16926,3 +16926,437 @@ def _format_display_message(self, message: str) -> str: if len(msg) > 60: msg = msg[:57] + "..." return msg + + +# --------------------------------------------------------------------------- +# Pure validation helper — factored out so tests can call it without a real +# QWidget (Qt display is optional; a QApplication is still needed for the +# widget itself, but not for the rules below). +# --------------------------------------------------------------------------- + + +def _validate_record_zstack_params( + *, + base_path: str, + selected_well_count: int, + recording_enabled: bool, + fps: float, + duration_s: float, + recording_channel_name: Optional[str], + zstack_enabled: bool, + z_min: float, + z_max: float, + step: float, + zstack_channel_names: List[str], + use_laser_af: bool, + laser_af_has_reference: bool, +) -> Optional[str]: + """Return an error string describing the first validation failure, or None if valid.""" + if not base_path: + return "Base path must be set before starting acquisition." + if selected_well_count < 1: + return "At least one well must be selected." + if not recording_enabled and not zstack_enabled: + return "At least one phase (Recording or Z-Stack) must be enabled." + if recording_enabled: + if fps <= 0: + return "Recording FPS must be greater than 0." + if duration_s <= 0: + return "Recording duration must be greater than 0." + if not recording_channel_name: + return "A channel must be chosen for the Recording phase." + if zstack_enabled: + if z_max <= z_min: + return "Z-Stack: z_max must be greater than z_min." + if step <= 0: + return "Z-Stack: step must be greater than 0." + if not zstack_channel_names: + return "Z-Stack: at least one channel must be selected." + if use_laser_af and not laser_af_has_reference: + return "Laser AF is enabled but no reference position has been captured." + return None + + +class RecordZStackMultiPointWidget(QFrame): + """Single-column 'Record + Z-Stack' acquisition tab (Option-A layout). + + Construction pattern mirrors WellplateMultiPointWidget: + - group boxes stacked vertically in a scroll area + - reads channels via liveController.get_channels(objectiveStore.current_objective) + - base-path / experiment-ID fields identical to the wellplate widget + + E1 scope: skeleton + input validation + build_parameters(). + Inline channel-editor wiring, Copy-from-Live, and Start-button handoff are E2/E3. + """ + + def __init__( + self, + stage, + navigationViewer, + recordZStackController, + liveController, + objectiveStore, + scanCoordinates, + well_selection_widget=None, + tab_widget=None, + laser_autofocus_controller=None, + *args, + **kwargs, + ): + super().__init__(*args, **kwargs) + self._log = squid.logging.get_logger(self.__class__.__name__) + self.stage = stage + self.navigationViewer = navigationViewer + self.recordZStackController = recordZStackController + self.liveController = liveController + self.objectiveStore = objectiveStore + self.scanCoordinates = scanCoordinates + self.well_selection_widget = well_selection_widget + self.tab_widget: Optional[QTabWidget] = tab_widget + self.laser_autofocus_controller = laser_autofocus_controller + + # Track z-stack channel names added via _add_zstack_channel_row() + self._zstack_channel_names: List[str] = [] + + self.setFrameStyle(QFrame.Panel | QFrame.Raised) + self._add_components() + + # ---------------------------------------------------------------------- build UI + + def _add_components(self) -> None: + outer_layout = QVBoxLayout(self) + outer_layout.setContentsMargins(4, 4, 4, 4) + + scroll = QScrollArea() + scroll.setWidgetResizable(True) + scroll.setFrameShape(QFrame.NoFrame) + inner = QWidget() + layout = QVBoxLayout(inner) + layout.setContentsMargins(4, 4, 4, 4) + layout.setSpacing(6) + + layout.addWidget(self._build_output_group()) + layout.addWidget(self._build_wells_fov_group()) + layout.addWidget(self._build_timelapse_focus_group()) + layout.addWidget(self._build_recording_group()) + layout.addWidget(self._build_zstack_group()) + layout.addWidget(self._build_start_group()) + layout.addStretch(1) + + scroll.setWidget(inner) + outer_layout.addWidget(scroll) + + def _build_output_group(self) -> QGroupBox: + grp = QGroupBox("Output") + layout = QGridLayout(grp) + + layout.addWidget(QLabel("Base path:"), 0, 0) + self.lineEdit_savingDir = QLineEdit() + last_path = get_last_used_saving_path() + self.lineEdit_savingDir.setText(last_path) + layout.addWidget(self.lineEdit_savingDir, 0, 1) + + self.btn_setSavingDir = QPushButton("Browse") + self.btn_setSavingDir.setIcon(QIcon("icon/folder.png")) + self.btn_setSavingDir.clicked.connect(self._browse_saving_dir) + layout.addWidget(self.btn_setSavingDir, 0, 2) + + layout.addWidget(QLabel("Experiment ID:"), 1, 0) + self.lineEdit_experimentID = QLineEdit() + self.lineEdit_experimentID.setPlaceholderText("record_zstack") + layout.addWidget(self.lineEdit_experimentID, 1, 1, 1, 2) + + return grp + + def _build_wells_fov_group(self) -> QGroupBox: + grp = QGroupBox("Wells & FOV Grid") + layout = QFormLayout(grp) + + self.label_selected_wells = QLabel("0 well(s) selected") + layout.addRow("Wells:", self.label_selected_wells) + + self.entry_overlap = QDoubleSpinBox() + self.entry_overlap.setRange(-1000, 99) + self.entry_overlap.setValue(10) + self.entry_overlap.setSuffix("%") + self.entry_overlap.setKeyboardTracking(False) + layout.addRow("FOV overlap:", self.entry_overlap) + + self.combobox_shape = QComboBox() + self.combobox_shape.addItems(["Square", "Circle", "Rectangle"]) + layout.addRow("Region shape:", self.combobox_shape) + + return grp + + def _build_timelapse_focus_group(self) -> QGroupBox: + grp = QGroupBox("Time-lapse & Focus") + layout = QFormLayout(grp) + + self.entry_Nt = QSpinBox() + self.entry_Nt.setMinimum(1) + self.entry_Nt.setMaximum(5000) + self.entry_Nt.setValue(1) + layout.addRow("Time points (Nt):", self.entry_Nt) + + self.entry_dt = QDoubleSpinBox() + self.entry_dt.setRange(0, 24 * 3600) + self.entry_dt.setValue(0) + self.entry_dt.setSuffix(" s") + self.entry_dt.setKeyboardTracking(False) + layout.addRow("Interval (dt):", self.entry_dt) + + self.checkbox_laser_af = QCheckBox("Use laser reflection AF") + self.checkbox_laser_af.setChecked(False) + layout.addRow("", self.checkbox_laser_af) + + return grp + + def _build_recording_group(self) -> QGroupBox: + grp = QGroupBox("Recording phase") + grp.setCheckable(True) + grp.setChecked(False) + self.checkbox_recording = grp # expose as checkbox_recording for callers + + layout = QFormLayout(grp) + + self.combobox_recording_channel = QComboBox() + self._populate_channel_combo(self.combobox_recording_channel) + layout.addRow("Channel:", self.combobox_recording_channel) + + self.entry_fps = QDoubleSpinBox() + self.entry_fps.setRange(0.1, 1000) + self.entry_fps.setValue(10.0) + self.entry_fps.setSuffix(" fps") + self.entry_fps.setKeyboardTracking(False) + layout.addRow("FPS:", self.entry_fps) + + self.entry_duration = QDoubleSpinBox() + self.entry_duration.setRange(0.01, 3600) + self.entry_duration.setValue(1.0) + self.entry_duration.setSuffix(" s") + self.entry_duration.setKeyboardTracking(False) + layout.addRow("Duration:", self.entry_duration) + + self.label_recording_frames = QLabel("-- frames") + layout.addRow("Computed:", self.label_recording_frames) + + self.entry_recording_z_offset = QDoubleSpinBox() + self.entry_recording_z_offset.setRange(-1000, 1000) + self.entry_recording_z_offset.setValue(0.0) + self.entry_recording_z_offset.setSuffix(" μm") + self.entry_recording_z_offset.setKeyboardTracking(False) + layout.addRow("Z-offset:", self.entry_recording_z_offset) + + # Wire up live frame count update + self.entry_fps.valueChanged.connect(self._update_recording_frames_label) + self.entry_duration.valueChanged.connect(self._update_recording_frames_label) + self._update_recording_frames_label() + + return grp + + def _build_zstack_group(self) -> QGroupBox: + grp = QGroupBox("Z-Stack phase") + grp.setCheckable(True) + grp.setChecked(False) + self.checkbox_zstack = grp # expose as checkbox_zstack for callers + + layout = QFormLayout(grp) + + self.entry_zmin = QDoubleSpinBox() + self.entry_zmin.setRange(-5000, 5000) + self.entry_zmin.setValue(-3.0) + self.entry_zmin.setSuffix(" μm") + self.entry_zmin.setDecimals(3) + self.entry_zmin.setSingleStep(0.5) + self.entry_zmin.setKeyboardTracking(False) + layout.addRow("Z-min:", self.entry_zmin) + + self.entry_zmax = QDoubleSpinBox() + self.entry_zmax.setRange(-5000, 5000) + self.entry_zmax.setValue(3.0) + self.entry_zmax.setSuffix(" μm") + self.entry_zmax.setDecimals(3) + self.entry_zmax.setSingleStep(0.5) + self.entry_zmax.setKeyboardTracking(False) + layout.addRow("Z-max:", self.entry_zmax) + + self.entry_step = QDoubleSpinBox() + self.entry_step.setRange(0.001, 1000) + self.entry_step.setValue(1.0) + self.entry_step.setSuffix(" μm") + self.entry_step.setDecimals(3) + self.entry_step.setSingleStep(0.1) + self.entry_step.setKeyboardTracking(False) + layout.addRow("Step:", self.entry_step) + + self.label_zstack_planes = QLabel("-- planes") + layout.addRow("Computed:", self.label_zstack_planes) + + # Z-stack channel table (rows added via _add_zstack_channel_row) + self.zstack_channel_table = QTableWidget(0, 1) + self.zstack_channel_table.setHorizontalHeaderLabels(["Channel"]) + self.zstack_channel_table.horizontalHeader().setStretchLastSection(True) + self.zstack_channel_table.setSelectionBehavior(QAbstractItemView.SelectRows) + self.zstack_channel_table.setMaximumHeight(120) + layout.addRow("Channels:", self.zstack_channel_table) + + # Wire up live plane count update + self.entry_zmin.valueChanged.connect(self._update_zstack_planes_label) + self.entry_zmax.valueChanged.connect(self._update_zstack_planes_label) + self.entry_step.valueChanged.connect(self._update_zstack_planes_label) + self._update_zstack_planes_label() + + return grp + + def _build_start_group(self) -> QGroupBox: + grp = QGroupBox() + layout = QVBoxLayout(grp) + self.btn_startAcquisition = QPushButton("Start Acquisition") + self.btn_startAcquisition.setStyleSheet("background-color: #C2C2FF") + self.btn_startAcquisition.setCheckable(True) + self.btn_startAcquisition.setChecked(False) + layout.addWidget(self.btn_startAcquisition) + return grp + + # ---------------------------------------------------------------------- helpers + + def _populate_channel_combo(self, combo: QComboBox) -> None: + """Populate a channel QComboBox from liveController.""" + combo.clear() + try: + channels = self.liveController.get_channels(self.objectiveStore.current_objective) + for ch in channels: + combo.addItem(ch.name) + except Exception as exc: + self._log.warning(f"Could not populate channel combo: {exc}") + + def _browse_saving_dir(self) -> None: + path = QFileDialog.getExistingDirectory(self, "Select Saving Directory", self.lineEdit_savingDir.text()) + if path: + self.lineEdit_savingDir.setText(path) + + def _update_recording_frames_label(self) -> None: + from control.core.record_zstack_controller import frame_count + + try: + n = frame_count(self.entry_fps.value(), self.entry_duration.value()) + self.label_recording_frames.setText(f"{n} frames") + except Exception: + self.label_recording_frames.setText("-- frames") + + def _update_zstack_planes_label(self) -> None: + from control.core.record_zstack_controller import zstack_plane_count + + try: + n = zstack_plane_count(self.entry_zmin.value(), self.entry_zmax.value(), self.entry_step.value()) + self.label_zstack_planes.setText(f"{n} planes") + except Exception: + self.label_zstack_planes.setText("-- planes") + + def _add_zstack_channel_row(self, name: str) -> None: + """Add a channel row to the Z-Stack channel table (also updates internal list). + + E2 will call this from the inline-editor wiring; tests may call it directly. + """ + if name not in self._zstack_channel_names: + self._zstack_channel_names.append(name) + row = self.zstack_channel_table.rowCount() + self.zstack_channel_table.insertRow(row) + item = QTableWidgetItem(name) + item.setFlags(item.flags() & ~Qt.ItemIsEditable) + self.zstack_channel_table.setItem(row, 0, item) + + def _get_selected_well_count(self) -> int: + """Return the number of currently selected wells.""" + if self.well_selection_widget is not None and hasattr(self.well_selection_widget, "get_selected_wells"): + return len(self.well_selection_widget.get_selected_wells()) + if self.scanCoordinates is not None and hasattr(self.scanCoordinates, "get_selected_wells"): + return len(self.scanCoordinates.get_selected_wells()) + # Fallback: treat as 1 well (current position) so the widget is usable + # without a well-selection panel attached. + return 1 + + def _laser_af_has_reference(self) -> bool: + """Return True if the laser autofocus controller has a captured reference.""" + ctrl = self.laser_autofocus_controller + if ctrl is None: + return False + # LaserAutofocusController sets reference_object_distance_mm when captured. + return getattr(ctrl, "reference_object_distance_mm", None) is not None + + # ---------------------------------------------------------------------- public API + + def validate(self) -> Optional[str]: + """Return an error string if the current widget state is invalid, or None. + + Delegates to the pure helper _validate_record_zstack_params so the + rules can be tested independently of Qt. + """ + return _validate_record_zstack_params( + base_path=self.lineEdit_savingDir.text().strip(), + selected_well_count=self._get_selected_well_count(), + recording_enabled=self.checkbox_recording.isChecked(), + fps=self.entry_fps.value(), + duration_s=self.entry_duration.value(), + recording_channel_name=( + self.combobox_recording_channel.currentText() if self.combobox_recording_channel.count() > 0 else None + ), + zstack_enabled=self.checkbox_zstack.isChecked(), + z_min=self.entry_zmin.value(), + z_max=self.entry_zmax.value(), + step=self.entry_step.value(), + zstack_channel_names=list(self._zstack_channel_names), + use_laser_af=self.checkbox_laser_af.isChecked(), + laser_af_has_reference=self._laser_af_has_reference(), + ) + + def build_parameters(self): + """Read widget fields and return a RecordZStackAcquisitionParameters instance. + + Transient AcquisitionChannel objects are constructed from the inline + channel selectors (full per-channel editor wiring is E2). + """ + from control.core.record_zstack_controller import RecordZStackAcquisitionParameters + from control.models.acquisition_config import AcquisitionChannel, CameraSettings, IlluminationSettings + + def _make_channel(name: str) -> AcquisitionChannel: + """Build a minimal transient AcquisitionChannel from the channel name.""" + channels = self.liveController.get_channels(self.objectiveStore.current_objective) + for ch in channels: + if ch.name == name: + return ch + # Fallback: construct a bare-bones channel (E2 will provide richer wiring) + return AcquisitionChannel( + name=name, + camera_settings=CameraSettings(exposure_time_ms=50.0, gain_mode=0.0), + illumination_settings=IlluminationSettings(intensity=50.0), + ) + + rec_channel_name = ( + self.combobox_recording_channel.currentText() + if self.combobox_recording_channel.count() > 0 and self.checkbox_recording.isChecked() + else None + ) + recording_channel = _make_channel(rec_channel_name) if rec_channel_name else None + + zstack_channels = ( + [_make_channel(name) for name in self._zstack_channel_names] if self.checkbox_zstack.isChecked() else [] + ) + + return RecordZStackAcquisitionParameters( + base_path=self.lineEdit_savingDir.text().strip(), + experiment_id=self.lineEdit_experimentID.text().strip() or "record_zstack", + Nt=self.entry_Nt.value(), + dt_s=self.entry_dt.value(), + use_laser_af=self.checkbox_laser_af.isChecked(), + recording_enabled=self.checkbox_recording.isChecked(), + recording_channel=recording_channel, + fps=self.entry_fps.value(), + duration_s=self.entry_duration.value(), + recording_z_offset_um=self.entry_recording_z_offset.value(), + zstack_enabled=self.checkbox_zstack.isChecked(), + zstack_channels=zstack_channels, + z_min_um=self.entry_zmin.value(), + z_max_um=self.entry_zmax.value(), + z_step_um=self.entry_step.value(), + ) diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py new file mode 100644 index 000000000..1e4198a90 --- /dev/null +++ b/software/tests/test_record_zstack_widget.py @@ -0,0 +1,280 @@ +"""Tests for RecordZStackMultiPointWidget (Task E1). + +Two test layers: +1. Pure validation helper (_validate_record_zstack_params) — no Qt needed. +2. Full widget instantiation using qtbot (pytest-qt manages QApplication). + +The validation rules are extracted into a pure helper so they can be tested +without any Qt machinery. The widget tests verify that the widget reads its +fields correctly and delegates to the same rules. + +NOTE on testability (as required by the task brief): +The validation logic was factored into _validate_record_zstack_params() in +widgets.py so that all constraint rules can be exercised without instantiating +QWidget. Creating a QApplication manually via PyQt5.QtWidgets.QApplication +aborts because pytest-qt (loaded by napari's conftest) creates a PyQt6 +QApplication before the test body runs. Using qtbot solves this — it is +available in the test suite and is the pattern used by all other widget tests +in tests/control/. +""" + +import sys +from typing import List, Optional +from unittest.mock import MagicMock + +import pytest + +from control.widgets import _validate_record_zstack_params + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_channel(name: str = "BF LED matrix full"): + """Return a minimal AcquisitionChannel with the given name.""" + from control.models.acquisition_config import AcquisitionChannel, CameraSettings, IlluminationSettings + + return AcquisitionChannel( + name=name, + camera_settings=CameraSettings(exposure_time_ms=50.0, gain_mode=0.0), + illumination_settings=IlluminationSettings(intensity=50.0), + ) + + +def _base_params(**overrides): + """Return a dict of valid _validate_record_zstack_params kwargs, with optional overrides.""" + defaults = dict( + base_path="/tmp/test", + selected_well_count=1, + recording_enabled=False, + fps=10.0, + duration_s=1.0, + recording_channel_name="BF LED matrix full", + zstack_enabled=True, + z_min=-3.0, + z_max=3.0, + step=1.0, + zstack_channel_names=["BF LED matrix full"], + use_laser_af=False, + laser_af_has_reference=False, + ) + defaults.update(overrides) + return defaults + + +# --------------------------------------------------------------------------- +# Pure-function validation tests (no Qt required) +# --------------------------------------------------------------------------- + + +def test_validate_helper_no_base_path(): + params = _base_params(base_path="") + assert _validate_record_zstack_params(**params) is not None + + +def test_validate_helper_no_wells(): + params = _base_params(selected_well_count=0) + assert _validate_record_zstack_params(**params) is not None + + +def test_validate_helper_no_phases_enabled(): + params = _base_params(recording_enabled=False, zstack_enabled=False) + assert _validate_record_zstack_params(**params) is not None + + +def test_validate_helper_bad_zstack_range(): + params = _base_params(z_min=3.0, z_max=-3.0) + assert _validate_record_zstack_params(**params) is not None + + +def test_validate_helper_bad_zstack_step(): + params = _base_params(step=0.0) + assert _validate_record_zstack_params(**params) is not None + + +def test_validate_helper_zstack_no_channels(): + params = _base_params(zstack_channel_names=[]) + assert _validate_record_zstack_params(**params) is not None + + +def test_validate_helper_recording_no_channel(): + params = _base_params(recording_enabled=True, recording_channel_name=None, zstack_enabled=False) + assert _validate_record_zstack_params(**params) is not None + + +def test_validate_helper_recording_bad_fps(): + params = _base_params(recording_enabled=True, fps=0.0, zstack_enabled=False) + assert _validate_record_zstack_params(**params) is not None + + +def test_validate_helper_laser_af_no_reference(): + params = _base_params(use_laser_af=True, laser_af_has_reference=False) + assert _validate_record_zstack_params(**params) is not None + + +def test_validate_helper_laser_af_with_reference(): + params = _base_params(use_laser_af=True, laser_af_has_reference=True) + assert _validate_record_zstack_params(**params) is None + + +def test_validate_helper_valid_zstack_only(): + params = _base_params() + assert _validate_record_zstack_params(**params) is None + + +def test_validate_helper_valid_recording_only(): + params = _base_params( + recording_enabled=True, + fps=10.0, + duration_s=1.0, + recording_channel_name="BF LED matrix full", + zstack_enabled=False, + ) + assert _validate_record_zstack_params(**params) is None + + +def test_validate_helper_valid_both_phases(): + params = _base_params( + recording_enabled=True, + fps=10.0, + duration_s=1.0, + recording_channel_name="BF LED matrix full", + zstack_enabled=True, + ) + assert _validate_record_zstack_params(**params) is None + + +# --------------------------------------------------------------------------- +# Widget-level fixtures +# --------------------------------------------------------------------------- + + +def _make_stub_live_controller(): + """Stub liveController whose get_channels() returns two fake AcquisitionChannels.""" + ctrl = MagicMock() + ctrl.get_channels.return_value = [ + _make_channel("BF LED matrix full"), + _make_channel("Fluorescence 488 nm Ex"), + ] + ctrl.currentConfiguration = _make_channel("BF LED matrix full") + return ctrl + + +def _make_stub_objective_store(): + store = MagicMock() + store.current_objective = "10x" + return store + + +def _make_stub_scan_coordinates(): + """Stub scanCoordinates reporting 1 selected well.""" + sc = MagicMock() + sc.get_selected_wells.return_value = ["A1"] + return sc + + +@pytest.fixture +def simulated_widget_deps(tmp_path): + """Provide lightweight stub dependencies for RecordZStackMultiPointWidget.""" + stage = MagicMock() + stage.get_pos.return_value = MagicMock(z_mm=0.0) + + return dict( + stage=stage, + navigationViewer=MagicMock(), + recordZStackController=MagicMock(), + liveController=_make_stub_live_controller(), + objectiveStore=_make_stub_objective_store(), + scanCoordinates=_make_stub_scan_coordinates(), + well_selection_widget=None, + tab_widget=None, + ) + + +# --------------------------------------------------------------------------- +# Widget-level tests — use qtbot (pytest-qt) for QApplication management. +# --------------------------------------------------------------------------- + + +def test_validate_rejects_bad_zstack_and_builds_params(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + # Must set base path so validation can reach the z-stack checks + w.lineEdit_savingDir.setText("/tmp/test") + + w.checkbox_zstack.setChecked(True) + w.entry_zmin.setValue(3.0) + w.entry_zmax.setValue(-3.0) # invalid: max < min + assert w.validate() is not None # returns an error string + + w.entry_zmin.setValue(-3.0) + w.entry_zmax.setValue(3.0) + w.entry_step.setValue(1.0) + # Add one z-stack channel row so the phase is valid + w._add_zstack_channel_row(w.liveController.get_channels(w.objectiveStore.current_objective)[0].name) + assert w.validate() is None + + params = w.build_parameters() + assert params.zstack_enabled and len(params.zstack_channels) == 1 + + +def test_validate_requires_base_path(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("") + w.checkbox_zstack.setChecked(True) + w.entry_zmin.setValue(-3.0) + w.entry_zmax.setValue(3.0) + w.entry_step.setValue(1.0) + w._add_zstack_channel_row("BF LED matrix full") + assert w.validate() is not None + + +def test_validate_requires_phase_enabled(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("/tmp/test") + w.checkbox_zstack.setChecked(False) + w.checkbox_recording.setChecked(False) + assert w.validate() is not None + + +def test_build_parameters_recording_phase(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("/tmp/test") + w.lineEdit_experimentID.setText("my_exp") + w.checkbox_recording.setChecked(True) + w.entry_fps.setValue(20.0) + w.entry_duration.setValue(5.0) + w.checkbox_zstack.setChecked(False) + + params = w.build_parameters() + assert params.recording_enabled is True + assert params.fps == pytest.approx(20.0) + assert params.duration_s == pytest.approx(5.0) + assert params.experiment_id == "my_exp" + assert params.zstack_enabled is False + assert params.recording_channel is not None + + +def test_add_zstack_channel_row_deduplicates(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w._add_zstack_channel_row("BF LED matrix full") + w._add_zstack_channel_row("BF LED matrix full") # duplicate + # Internal list should deduplicate; table may have 2 rows (visual only) + assert w._zstack_channel_names.count("BF LED matrix full") == 1 From d66885c2cdc4f8e47b917e1ed2059edfdb9c6e42 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 04:59:15 -0700 Subject: [PATCH 16/61] fix(record-zstack): correct laser-AF reference check + dedup zstack row table Co-Authored-By: Claude Sonnet 4.6 --- software/control/widgets.py | 8 ++++---- software/tests/test_record_zstack_widget.py | 8 +++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index f13e7a166..d5f9baed6 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17258,8 +17258,9 @@ def _add_zstack_channel_row(self, name: str) -> None: E2 will call this from the inline-editor wiring; tests may call it directly. """ - if name not in self._zstack_channel_names: - self._zstack_channel_names.append(name) + if name in self._zstack_channel_names: + return + self._zstack_channel_names.append(name) row = self.zstack_channel_table.rowCount() self.zstack_channel_table.insertRow(row) item = QTableWidgetItem(name) @@ -17281,8 +17282,7 @@ def _laser_af_has_reference(self) -> bool: ctrl = self.laser_autofocus_controller if ctrl is None: return False - # LaserAutofocusController sets reference_object_distance_mm when captured. - return getattr(ctrl, "reference_object_distance_mm", None) is not None + return bool(getattr(getattr(ctrl, "laser_af_properties", None), "has_reference", False)) # ---------------------------------------------------------------------- public API diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 1e4198a90..deb25a8f5 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -276,5 +276,11 @@ def test_add_zstack_channel_row_deduplicates(qtbot, simulated_widget_deps): qtbot.addWidget(w) w._add_zstack_channel_row("BF LED matrix full") w._add_zstack_channel_row("BF LED matrix full") # duplicate - # Internal list should deduplicate; table may have 2 rows (visual only) + # Both the internal list AND the table must stay at 1 entry after dedup. assert w._zstack_channel_names.count("BF LED matrix full") == 1 + assert w.zstack_channel_table.rowCount() == 1 + + +def test_validate_helper_recording_bad_duration(): + params = _base_params(recording_enabled=True, duration_s=0.0, zstack_enabled=False) + assert _validate_record_zstack_params(**params) is not None From ae5af7182285630dfbfa62c139b7317093413881 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 05:06:44 -0700 Subject: [PATCH 17/61] feat(record-zstack): inline channel editors, Copy-from-Live, computed labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Recording group: add exposure/gain/illumination spinboxes and a '⟳ Live' button that copies currentConfiguration into the recording channel dropdown + inline editors - Z-stack table expanded to 5 columns (channel, exposure, gain, illumination, actions); each row has per-channel spinboxes, a '⟳' copy-from-live button, and a '✕' remove button - '+ Add channel' combo+button below the z-stack table - Added _remove_zstack_channel_row(), _set_zstack_row_values(), _get_zstack_row_values(), _copy_recording_from_live(), _copy_zstack_row_from_live(), _on_zstack_add_channel_clicked() - build_parameters() now reads inline editor values for both the recording channel and each z-stack row; uses model_copy(deep=True) to avoid mutating source channel objects - 7 new qtbot tests added (26 total, all passing) Co-Authored-By: Claude Sonnet 4.6 --- software/control/widgets.py | 259 ++++++++++++++++++-- software/tests/test_record_zstack_widget.py | 153 ++++++++++++ 2 files changed, 387 insertions(+), 25 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index d5f9baed6..4b52caa8a 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17119,9 +17119,43 @@ def _build_recording_group(self) -> QGroupBox: layout = QFormLayout(grp) + # Channel row with Copy-from-Live button + channel_row = QWidget() + channel_row_layout = QHBoxLayout(channel_row) + channel_row_layout.setContentsMargins(0, 0, 0, 0) self.combobox_recording_channel = QComboBox() self._populate_channel_combo(self.combobox_recording_channel) - layout.addRow("Channel:", self.combobox_recording_channel) + channel_row_layout.addWidget(self.combobox_recording_channel, 1) + self.btn_copy_from_live = QPushButton("⟳ Live") + self.btn_copy_from_live.setToolTip("Copy current live channel settings (channel, exposure, gain, illumination)") + self.btn_copy_from_live.setMaximumWidth(70) + self.btn_copy_from_live.clicked.connect(self._copy_recording_from_live) + channel_row_layout.addWidget(self.btn_copy_from_live) + layout.addRow("Channel:", channel_row) + + # Inline exposure/gain/illumination spinboxes + self.entry_recording_exposure = QDoubleSpinBox() + self.entry_recording_exposure.setRange(0.01, 60000) + self.entry_recording_exposure.setValue(50.0) + self.entry_recording_exposure.setSuffix(" ms") + self.entry_recording_exposure.setDecimals(1) + self.entry_recording_exposure.setKeyboardTracking(False) + layout.addRow("Exposure:", self.entry_recording_exposure) + + self.entry_recording_gain = QDoubleSpinBox() + self.entry_recording_gain.setRange(0.0, 100.0) + self.entry_recording_gain.setValue(0.0) + self.entry_recording_gain.setDecimals(2) + self.entry_recording_gain.setKeyboardTracking(False) + layout.addRow("Gain:", self.entry_recording_gain) + + self.entry_recording_illumination = QDoubleSpinBox() + self.entry_recording_illumination.setRange(0.0, 100.0) + self.entry_recording_illumination.setValue(50.0) + self.entry_recording_illumination.setSuffix(" %") + self.entry_recording_illumination.setDecimals(1) + self.entry_recording_illumination.setKeyboardTracking(False) + layout.addRow("Illumination:", self.entry_recording_illumination) self.entry_fps = QDoubleSpinBox() self.entry_fps.setRange(0.1, 1000) @@ -17193,13 +17227,33 @@ def _build_zstack_group(self) -> QGroupBox: layout.addRow("Computed:", self.label_zstack_planes) # Z-stack channel table (rows added via _add_zstack_channel_row) - self.zstack_channel_table = QTableWidget(0, 1) - self.zstack_channel_table.setHorizontalHeaderLabels(["Channel"]) - self.zstack_channel_table.horizontalHeader().setStretchLastSection(True) + # Columns: Channel | Exposure (ms) | Gain | Illumination (%) | Actions + self.zstack_channel_table = QTableWidget(0, 5) + self.zstack_channel_table.setHorizontalHeaderLabels(["Channel", "Exp (ms)", "Gain", "Illum (%)", ""]) + hdr = self.zstack_channel_table.horizontalHeader() + hdr.setSectionResizeMode(0, QHeaderView.Stretch) + hdr.setSectionResizeMode(1, QHeaderView.ResizeToContents) + hdr.setSectionResizeMode(2, QHeaderView.ResizeToContents) + hdr.setSectionResizeMode(3, QHeaderView.ResizeToContents) + hdr.setSectionResizeMode(4, QHeaderView.ResizeToContents) self.zstack_channel_table.setSelectionBehavior(QAbstractItemView.SelectRows) - self.zstack_channel_table.setMaximumHeight(120) + self.zstack_channel_table.setMinimumHeight(80) + self.zstack_channel_table.setMaximumHeight(200) layout.addRow("Channels:", self.zstack_channel_table) + # "+ Add channel" row below the table + add_ch_row = QWidget() + add_ch_row_layout = QHBoxLayout(add_ch_row) + add_ch_row_layout.setContentsMargins(0, 0, 0, 0) + self.combobox_zstack_add_channel = QComboBox() + self._populate_channel_combo(self.combobox_zstack_add_channel) + add_ch_row_layout.addWidget(self.combobox_zstack_add_channel, 1) + self.btn_zstack_add_channel = QPushButton("+ Add") + self.btn_zstack_add_channel.setMaximumWidth(60) + self.btn_zstack_add_channel.clicked.connect(self._on_zstack_add_channel_clicked) + add_ch_row_layout.addWidget(self.btn_zstack_add_channel) + layout.addRow("Add channel:", add_ch_row) + # Wire up live plane count update self.entry_zmin.valueChanged.connect(self._update_zstack_planes_label) self.entry_zmax.valueChanged.connect(self._update_zstack_planes_label) @@ -17253,20 +17307,160 @@ def _update_zstack_planes_label(self) -> None: except Exception: self.label_zstack_planes.setText("-- planes") - def _add_zstack_channel_row(self, name: str) -> None: + def _add_zstack_channel_row( + self, name: str, exposure: float = 50.0, gain: float = 0.0, illumination: float = 50.0 + ) -> None: """Add a channel row to the Z-Stack channel table (also updates internal list). - E2 will call this from the inline-editor wiring; tests may call it directly. + Each row contains: channel name (read-only) | exposure spinbox | gain spinbox | + illumination spinbox | ⟳ Live button + ✕ remove button. + Skips silently if ``name`` is already present (dedup guard). """ if name in self._zstack_channel_names: return self._zstack_channel_names.append(name) row = self.zstack_channel_table.rowCount() self.zstack_channel_table.insertRow(row) + + # Col 0: channel name (read-only) item = QTableWidgetItem(name) item.setFlags(item.flags() & ~Qt.ItemIsEditable) self.zstack_channel_table.setItem(row, 0, item) + # Col 1: exposure spinbox + exp_spin = QDoubleSpinBox() + exp_spin.setRange(0.01, 60000) + exp_spin.setValue(exposure) + exp_spin.setSuffix(" ms") + exp_spin.setDecimals(1) + exp_spin.setKeyboardTracking(False) + self.zstack_channel_table.setCellWidget(row, 1, exp_spin) + + # Col 2: gain spinbox + gain_spin = QDoubleSpinBox() + gain_spin.setRange(0.0, 100.0) + gain_spin.setValue(gain) + gain_spin.setDecimals(2) + gain_spin.setKeyboardTracking(False) + self.zstack_channel_table.setCellWidget(row, 2, gain_spin) + + # Col 3: illumination spinbox + illum_spin = QDoubleSpinBox() + illum_spin.setRange(0.0, 100.0) + illum_spin.setValue(illumination) + illum_spin.setSuffix(" %") + illum_spin.setDecimals(1) + illum_spin.setKeyboardTracking(False) + self.zstack_channel_table.setCellWidget(row, 3, illum_spin) + + # Col 4: action buttons (⟳ Live + ✕) + btn_container = QWidget() + btn_layout = QHBoxLayout(btn_container) + btn_layout.setContentsMargins(1, 1, 1, 1) + btn_layout.setSpacing(2) + + btn_live = QPushButton("⟳") + btn_live.setToolTip(f"Copy current live settings into '{name}' row") + btn_live.setMaximumWidth(28) + btn_live.clicked.connect(lambda checked=False, n=name: self._copy_zstack_row_from_live(n)) + btn_layout.addWidget(btn_live) + + btn_remove = QPushButton("✕") + btn_remove.setToolTip(f"Remove '{name}' from z-stack channels") + btn_remove.setMaximumWidth(28) + btn_remove.clicked.connect(lambda checked=False, n=name: self._remove_zstack_channel_row(n)) + btn_layout.addWidget(btn_remove) + + self.zstack_channel_table.setCellWidget(row, 4, btn_container) + + def _remove_zstack_channel_row(self, name: str) -> None: + """Remove the row for *name* from the table and internal list. + + No-op if *name* is not present. + """ + if name not in self._zstack_channel_names: + return + # Find the row by scanning col-0 items + for row in range(self.zstack_channel_table.rowCount()): + item = self.zstack_channel_table.item(row, 0) + if item is not None and item.text() == name: + self.zstack_channel_table.removeRow(row) + break + self._zstack_channel_names.remove(name) + + def _set_zstack_row_values(self, name: str, exposure: float, gain: float, illumination: float) -> None: + """Set inline editor values for the z-stack row identified by *name*. + + Used by ⟳ Live button and by tests. No-op if *name* is not present. + """ + for row in range(self.zstack_channel_table.rowCount()): + item = self.zstack_channel_table.item(row, 0) + if item is not None and item.text() == name: + exp_spin = self.zstack_channel_table.cellWidget(row, 1) + gain_spin = self.zstack_channel_table.cellWidget(row, 2) + illum_spin = self.zstack_channel_table.cellWidget(row, 3) + if exp_spin is not None: + exp_spin.setValue(exposure) + if gain_spin is not None: + gain_spin.setValue(gain) + if illum_spin is not None: + illum_spin.setValue(illumination) + return + + def _get_zstack_row_values(self, name: str): + """Return (exposure, gain, illumination) for the z-stack row identified by *name*. + + Returns (50.0, 0.0, 50.0) as defaults if the row or spinboxes are missing. + """ + for row in range(self.zstack_channel_table.rowCount()): + item = self.zstack_channel_table.item(row, 0) + if item is not None and item.text() == name: + exp_spin = self.zstack_channel_table.cellWidget(row, 1) + gain_spin = self.zstack_channel_table.cellWidget(row, 2) + illum_spin = self.zstack_channel_table.cellWidget(row, 3) + exposure = exp_spin.value() if exp_spin is not None else 50.0 + gain = gain_spin.value() if gain_spin is not None else 0.0 + illum = illum_spin.value() if illum_spin is not None else 50.0 + return exposure, gain, illum + return 50.0, 0.0, 50.0 + + def _copy_recording_from_live(self) -> None: + """Copy current live channel settings into the recording inline editors.""" + try: + live_ch = self.liveController.currentConfiguration + if live_ch is None: + return + # Update channel dropdown + idx = self.combobox_recording_channel.findText(live_ch.name) + if idx >= 0: + self.combobox_recording_channel.setCurrentIndex(idx) + # Update inline spinboxes + self.entry_recording_exposure.setValue(live_ch.exposure_time) + self.entry_recording_gain.setValue(live_ch.analog_gain) + self.entry_recording_illumination.setValue(live_ch.illumination_intensity) + except Exception as exc: + self._log.warning(f"Copy-from-Live failed: {exc}") + + def _copy_zstack_row_from_live(self, name: str) -> None: + """Copy current live channel settings into the z-stack row for *name*.""" + try: + live_ch = self.liveController.currentConfiguration + if live_ch is None: + return + self._set_zstack_row_values( + name, live_ch.exposure_time, live_ch.analog_gain, live_ch.illumination_intensity + ) + except Exception as exc: + self._log.warning(f"Copy-from-Live for z-stack row '{name}' failed: {exc}") + + def _on_zstack_add_channel_clicked(self) -> None: + """Add the currently selected channel in the add-channel combo to the z-stack table.""" + if self.combobox_zstack_add_channel is None: + return + name = self.combobox_zstack_add_channel.currentText() + if name: + self._add_zstack_channel_row(name) + def _get_selected_well_count(self) -> int: """Return the number of currently selected wells.""" if self.well_selection_widget is not None and hasattr(self.well_selection_widget, "get_selected_wells"): @@ -17314,34 +17508,49 @@ def build_parameters(self): """Read widget fields and return a RecordZStackAcquisitionParameters instance. Transient AcquisitionChannel objects are constructed from the inline - channel selectors (full per-channel editor wiring is E2). + channel editors so that the caller receives the exact settings the user + entered, not the defaults from the channel config. """ from control.core.record_zstack_controller import RecordZStackAcquisitionParameters from control.models.acquisition_config import AcquisitionChannel, CameraSettings, IlluminationSettings - def _make_channel(name: str) -> AcquisitionChannel: - """Build a minimal transient AcquisitionChannel from the channel name.""" - channels = self.liveController.get_channels(self.objectiveStore.current_objective) - for ch in channels: - if ch.name == name: - return ch - # Fallback: construct a bare-bones channel (E2 will provide richer wiring) + def _make_channel_base(name: str) -> AcquisitionChannel: + """Return a copy of the named channel from liveController, or a bare-bones fallback.""" + try: + channels = self.liveController.get_channels(self.objectiveStore.current_objective) + for ch in channels: + if ch.name == name: + # Return a copy so inline-editor mutations don't corrupt the source + return ch.model_copy(deep=True) + except Exception: + pass return AcquisitionChannel( name=name, camera_settings=CameraSettings(exposure_time_ms=50.0, gain_mode=0.0), illumination_settings=IlluminationSettings(intensity=50.0), ) - rec_channel_name = ( - self.combobox_recording_channel.currentText() - if self.combobox_recording_channel.count() > 0 and self.checkbox_recording.isChecked() - else None - ) - recording_channel = _make_channel(rec_channel_name) if rec_channel_name else None - - zstack_channels = ( - [_make_channel(name) for name in self._zstack_channel_names] if self.checkbox_zstack.isChecked() else [] - ) + # Build recording channel from inline editors + recording_channel = None + if self.combobox_recording_channel.count() > 0 and self.checkbox_recording.isChecked(): + rec_name = self.combobox_recording_channel.currentText() + if rec_name: + ch = _make_channel_base(rec_name) + ch.exposure_time = self.entry_recording_exposure.value() + ch.analog_gain = self.entry_recording_gain.value() + ch.illumination_intensity = self.entry_recording_illumination.value() + recording_channel = ch + + # Build z-stack channels from per-row inline editors + zstack_channels = [] + if self.checkbox_zstack.isChecked(): + for name in self._zstack_channel_names: + ch = _make_channel_base(name) + exposure, gain, illum = self._get_zstack_row_values(name) + ch.exposure_time = exposure + ch.analog_gain = gain + ch.illumination_intensity = illum + zstack_channels.append(ch) return RecordZStackAcquisitionParameters( base_path=self.lineEdit_savingDir.text().strip(), diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index deb25a8f5..5a9af31e4 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -284,3 +284,156 @@ def test_add_zstack_channel_row_deduplicates(qtbot, simulated_widget_deps): def test_validate_helper_recording_bad_duration(): params = _base_params(recording_enabled=True, duration_s=0.0, zstack_enabled=False) assert _validate_record_zstack_params(**params) is not None + + +# --------------------------------------------------------------------------- +# E2: Copy-from-Live, dynamic z-stack rows, computed labels +# --------------------------------------------------------------------------- + + +def _make_live_channel(name: str, exposure: float, gain: float, intensity: float): + """Return an AcquisitionChannel stub with specific settings.""" + from control.models.acquisition_config import AcquisitionChannel, CameraSettings, IlluminationSettings + + return AcquisitionChannel( + name=name, + camera_settings=CameraSettings(exposure_time_ms=exposure, gain_mode=gain), + illumination_settings=IlluminationSettings(intensity=intensity), + ) + + +def test_copy_from_live_populates_recording_fields(qtbot, simulated_widget_deps): + """Copy-from-Live reads currentConfiguration and sets recording fields.""" + from control.widgets import RecordZStackMultiPointWidget + + live_ch = _make_live_channel("Fluorescence 488 nm Ex", exposure=33.0, gain=2.5, intensity=75.0) + simulated_widget_deps["liveController"].currentConfiguration = live_ch + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + # Enable the recording group so its child widgets are interactive + w.checkbox_recording.setChecked(True) + w.btn_copy_from_live.click() + + # Channel dropdown should be updated to the live channel name + assert w.combobox_recording_channel.currentText() == "Fluorescence 488 nm Ex" + # Exposure, gain, intensity spinboxes should reflect live channel values + assert w.entry_recording_exposure.value() == pytest.approx(33.0) + assert w.entry_recording_gain.value() == pytest.approx(2.5) + assert w.entry_recording_illumination.value() == pytest.approx(75.0) + + +def test_add_remove_zstack_channel_row_syncs_list_and_table(qtbot, simulated_widget_deps): + """_add_zstack_channel_row and _remove_zstack_channel_row keep list+table in sync.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + assert w.zstack_channel_table.rowCount() == 0 + assert len(w._zstack_channel_names) == 0 + + w._add_zstack_channel_row("BF LED matrix full") + assert w.zstack_channel_table.rowCount() == 1 + assert len(w._zstack_channel_names) == 1 + + w._add_zstack_channel_row("Fluorescence 488 nm Ex") + assert w.zstack_channel_table.rowCount() == 2 + assert len(w._zstack_channel_names) == 2 + + w._remove_zstack_channel_row("BF LED matrix full") + assert w.zstack_channel_table.rowCount() == 1 + assert len(w._zstack_channel_names) == 1 + assert "BF LED matrix full" not in w._zstack_channel_names + assert "Fluorescence 488 nm Ex" in w._zstack_channel_names + + +def test_computed_frame_label_updates_on_spinbox_change(qtbot, simulated_widget_deps): + """label_recording_frames updates to '→ N frames' when fps/duration change.""" + from control.core.record_zstack_controller import frame_count + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.entry_fps.setValue(5.0) + w.entry_duration.setValue(4.0) + expected = frame_count(5.0, 4.0) # 20 + assert str(expected) in w.label_recording_frames.text() + + +def test_computed_plane_label_updates_on_spinbox_change(qtbot, simulated_widget_deps): + """label_zstack_planes updates to '→ N planes' when zmin/zmax/step change.""" + from control.core.record_zstack_controller import zstack_plane_count + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.entry_zmin.setValue(-2.0) + w.entry_zmax.setValue(2.0) + w.entry_step.setValue(1.0) + expected = zstack_plane_count(-2.0, 2.0, 1.0) # 5 + assert str(expected) in w.label_zstack_planes.text() + + +def test_computed_plane_label_degrades_gracefully_on_invalid_range(qtbot, simulated_widget_deps): + """label_zstack_planes shows '—' or '--' when z_max < z_min (invalid range).""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.entry_zmin.setValue(3.0) + w.entry_zmax.setValue(-3.0) # invalid: max < min + # Label should show a placeholder, not crash + text = w.label_zstack_planes.text() + assert "planes" not in text.lower() or text.startswith("--") or "—" in text or text == "— planes" + + +def test_build_parameters_uses_inline_editor_values(qtbot, simulated_widget_deps): + """build_parameters() reflects inline editor values, not just channel name lookup.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("/tmp/test") + w.checkbox_recording.setChecked(True) + w.checkbox_zstack.setChecked(False) + + # Set inline editor values + w.entry_recording_exposure.setValue(99.0) + w.entry_recording_gain.setValue(1.5) + w.entry_recording_illumination.setValue(60.0) + + params = w.build_parameters() + assert params.recording_channel is not None + assert params.recording_channel.exposure_time == pytest.approx(99.0) + assert params.recording_channel.analog_gain == pytest.approx(1.5) + assert params.recording_channel.illumination_intensity == pytest.approx(60.0) + + +def test_zstack_row_inline_editors_reflected_in_build_parameters(qtbot, simulated_widget_deps): + """Z-stack row inline editor values are used when build_parameters() is called.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("/tmp/test") + w.checkbox_zstack.setChecked(True) + w.checkbox_recording.setChecked(False) + w.entry_zmin.setValue(-1.0) + w.entry_zmax.setValue(1.0) + w.entry_step.setValue(1.0) + + w._add_zstack_channel_row("BF LED matrix full") + # Set inline editor values for the row + w._set_zstack_row_values("BF LED matrix full", exposure=77.0, gain=3.0, illumination=45.0) + + params = w.build_parameters() + assert len(params.zstack_channels) == 1 + ch = params.zstack_channels[0] + assert ch.exposure_time == pytest.approx(77.0) + assert ch.analog_gain == pytest.approx(3.0) + assert ch.illumination_intensity == pytest.approx(45.0) From ba30f7d13c68908aaee31a0fbe01c4edbbbdf6d2 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 05:11:33 -0700 Subject: [PATCH 18/61] test(record-zstack): tighten invalid-range plane-label assertion + docstring Co-Authored-By: Claude Sonnet 4.6 --- software/tests/test_record_zstack_widget.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 5a9af31e4..e417be0d7 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -1,4 +1,6 @@ -"""Tests for RecordZStackMultiPointWidget (Task E1). +"""Tests for RecordZStackMultiPointWidget (Tasks E1 + E2). + +Covers: widget skeleton, validation, inline editors, Copy-from-Live, computed labels. Two test layers: 1. Pure validation helper (_validate_record_zstack_params) — no Qt needed. @@ -389,7 +391,7 @@ def test_computed_plane_label_degrades_gracefully_on_invalid_range(qtbot, simula w.entry_zmax.setValue(-3.0) # invalid: max < min # Label should show a placeholder, not crash text = w.label_zstack_planes.text() - assert "planes" not in text.lower() or text.startswith("--") or "—" in text or text == "— planes" + assert text == "-- planes" def test_build_parameters_uses_inline_editor_values(qtbot, simulated_widget_deps): From 843175592cd708aac2a0c24a993cb96e7f7dfcd8 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 05:14:19 -0700 Subject: [PATCH 19/61] feat(record-zstack): Start/Stop handoff to RecordZStackController Co-Authored-By: Claude Sonnet 4.6 --- software/control/widgets.py | 48 ++++++++ software/tests/test_record_zstack_widget.py | 121 +++++++++++++++++++- 2 files changed, 168 insertions(+), 1 deletion(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index 4b52caa8a..300c689b8 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17270,6 +17270,7 @@ def _build_start_group(self) -> QGroupBox: self.btn_startAcquisition.setCheckable(True) self.btn_startAcquisition.setChecked(False) layout.addWidget(self.btn_startAcquisition) + self.btn_startAcquisition.clicked.connect(self.toggle_acquisition) return grp # ---------------------------------------------------------------------- helpers @@ -17569,3 +17570,50 @@ def _make_channel_base(name: str) -> AcquisitionChannel: z_max_um=self.entry_zmax.value(), z_step_um=self.entry_step.value(), ) + + def toggle_acquisition(self, pressed: bool) -> None: + """Handle Start/Stop button press. + + On start (pressed=True): + - validate(); show QMessageBox.warning and un-check button on failure. + - push all parameters to recordZStackController via its setters. + - call recordZStackController.run_acquisition(). + + On stop (pressed=False): + - call recordZStackController.request_abort(). + """ + self._log.debug(f"RecordZStackMultiPointWidget.toggle_acquisition, {pressed=}") + if pressed: + if self.recordZStackController.acquisition_in_progress(): + self._log.warning("Acquisition already in progress, cannot start another.") + self.btn_startAcquisition.setChecked(False) + return + + error = self.validate() + if error is not None: + self.btn_startAcquisition.setChecked(False) + QMessageBox.warning(self, "Invalid Parameters", error) + return + + params = self.build_parameters() + + # Push all parameters to the controller via its individual setters. + self.recordZStackController.set_base_path(params.base_path) + self.recordZStackController.set_experiment_id(params.experiment_id) + self.recordZStackController.set_Nt(params.Nt) + self.recordZStackController.set_dt_s(params.dt_s) + self.recordZStackController.set_use_laser_af(params.use_laser_af) + self.recordZStackController.set_recording_enabled(params.recording_enabled) + self.recordZStackController.set_recording_channel(params.recording_channel) + self.recordZStackController.set_fps(params.fps) + self.recordZStackController.set_duration_s(params.duration_s) + self.recordZStackController.set_recording_z_offset_um(params.recording_z_offset_um) + self.recordZStackController.set_zstack_enabled(params.zstack_enabled) + self.recordZStackController.set_zstack_channels(params.zstack_channels) + self.recordZStackController.set_z_min_um(params.z_min_um) + self.recordZStackController.set_z_max_um(params.z_max_um) + self.recordZStackController.set_z_step_um(params.z_step_um) + + self.recordZStackController.run_acquisition() + else: + self.recordZStackController.request_abort() diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index e417be0d7..e6c1ebff6 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -22,7 +22,7 @@ import sys from typing import List, Optional -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest @@ -439,3 +439,122 @@ def test_zstack_row_inline_editors_reflected_in_build_parameters(qtbot, simulate assert ch.exposure_time == pytest.approx(77.0) assert ch.analog_gain == pytest.approx(3.0) assert ch.illumination_intensity == pytest.approx(45.0) + + +# --------------------------------------------------------------------------- +# E3: Start/Stop handoff to RecordZStackController +# --------------------------------------------------------------------------- + + +def _make_stub_controller(): + """Stub RecordZStackController that records calls and reports not-in-progress.""" + ctrl = MagicMock() + ctrl.acquisition_in_progress.return_value = False + return ctrl + + +def _make_valid_widget(qtbot, deps): + """Return a widget pre-configured for a valid z-stack-only acquisition.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("/tmp/test_e3") + w.checkbox_zstack.setChecked(True) + w.checkbox_recording.setChecked(False) + w.entry_zmin.setValue(-2.0) + w.entry_zmax.setValue(2.0) + w.entry_step.setValue(1.0) + w._add_zstack_channel_row("BF LED matrix full") + return w + + +def test_toggle_acquisition_start_valid_calls_run_acquisition(qtbot, simulated_widget_deps): + """Clicking Start with valid params calls run_acquisition() exactly once.""" + ctrl = _make_stub_controller() + simulated_widget_deps["recordZStackController"] = ctrl + + w = _make_valid_widget(qtbot, simulated_widget_deps) + + # Simulate pressing the Start button (checked=True) + w.toggle_acquisition(True) + + ctrl.run_acquisition.assert_called_once() + + +def test_toggle_acquisition_start_valid_pushes_params_to_controller(qtbot, simulated_widget_deps): + """Clicking Start with valid params pushes all parameters via controller setters.""" + ctrl = _make_stub_controller() + simulated_widget_deps["recordZStackController"] = ctrl + + w = _make_valid_widget(qtbot, simulated_widget_deps) + w.toggle_acquisition(True) + + # Verify key setters were called + ctrl.set_base_path.assert_called_once_with("/tmp/test_e3") + ctrl.set_zstack_enabled.assert_called_once_with(True) + ctrl.set_recording_enabled.assert_called_once_with(False) + ctrl.set_z_min_um.assert_called_once_with(-2.0) + ctrl.set_z_max_um.assert_called_once_with(2.0) + ctrl.set_z_step_um.assert_called_once_with(1.0) + + +def test_toggle_acquisition_start_invalid_no_phase_does_not_call_run(qtbot, simulated_widget_deps): + """Clicking Start with both phases disabled does NOT call run_acquisition().""" + ctrl = _make_stub_controller() + simulated_widget_deps["recordZStackController"] = ctrl + + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("/tmp/test_e3") + w.checkbox_zstack.setChecked(False) + w.checkbox_recording.setChecked(False) + + with patch("control.widgets.QMessageBox.warning") as mock_warn: + w.toggle_acquisition(True) + + ctrl.run_acquisition.assert_not_called() + mock_warn.assert_called_once() + # Button must be un-checked after a failed start + assert not w.btn_startAcquisition.isChecked() + + +def test_toggle_acquisition_start_invalid_bad_zrange_shows_warning(qtbot, simulated_widget_deps): + """Clicking Start with z_max <= z_min pops a warning and does not start.""" + ctrl = _make_stub_controller() + simulated_widget_deps["recordZStackController"] = ctrl + + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("/tmp/test_e3") + w.checkbox_zstack.setChecked(True) + w.entry_zmin.setValue(3.0) + w.entry_zmax.setValue(-3.0) # invalid: max < min + w.entry_step.setValue(1.0) + w._add_zstack_channel_row("BF LED matrix full") + + warning_args = [] + with patch("control.widgets.QMessageBox.warning", side_effect=lambda *a: warning_args.append(a)): + w.toggle_acquisition(True) + + ctrl.run_acquisition.assert_not_called() + assert len(warning_args) == 1 + assert not w.btn_startAcquisition.isChecked() + + +def test_toggle_acquisition_stop_calls_request_abort(qtbot, simulated_widget_deps): + """Un-pressing the Start button (pressed=False) calls request_abort().""" + ctrl = _make_stub_controller() + simulated_widget_deps["recordZStackController"] = ctrl + + w = _make_valid_widget(qtbot, simulated_widget_deps) + + # Simulate pressing stop (unchecked) + w.toggle_acquisition(False) + + ctrl.request_abort.assert_called_once() + ctrl.run_acquisition.assert_not_called() From e6447873a83247388a1d7394cb711fa424bb1405 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 12:08:41 -0700 Subject: [PATCH 20/61] feat(record-zstack): add Record + Z-Stack tab to gui_hcs (ENABLE_RECORDING) Co-Authored-By: Claude Sonnet 4.6 --- software/control/gui_hcs.py | 71 +++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/software/control/gui_hcs.py b/software/control/gui_hcs.py index 361f76836..42b4a3bcf 100644 --- a/software/control/gui_hcs.py +++ b/software/control/gui_hcs.py @@ -48,6 +48,7 @@ class NDViewerMode(Enum): from control.core.live_controller import LiveController from control.core.multi_point_controller import MultiPointController from control.core.mosaic_utils import parse_well_id +from control.core.record_zstack_controller import RecordZStackController from control.core.multi_point_utils import ( MultiPointControllerFunctions, AcquisitionParameters, @@ -598,6 +599,52 @@ def _detect_hcs_mode(self, scan_info: ScanCoordinates) -> bool: return len(scan_info.scan_region_names) > 0 +class QtRecordZStackController(RecordZStackController, QObject): + """Qt-aware wrapper for RecordZStackController. + + Emits ``acquisition_finished`` as a thread-safe Qt signal so the widget + can reset its Start button via a queued connection, regardless of which + thread the worker runs on. + """ + + acquisition_finished = Signal() + + def __init__( + self, + microscope, + live_controller, + laser_autofocus_controller, + objective_store, + scan_coordinates, + ): + # Build a MultiPointControllerFunctions callbacks object whose + # signal_acquisition_finished emits our Qt signal. All other + # callbacks are no-ops because RecordZStackWorker only ever calls + # signal_acquisition_finished. + callbacks = MultiPointControllerFunctions( + signal_acquisition_start=lambda *a, **kw: None, + signal_acquisition_finished=self._on_acquisition_finished, + signal_new_image=lambda *a, **kw: None, + signal_current_configuration=lambda *a, **kw: None, + signal_current_fov=lambda *a, **kw: None, + signal_overall_progress=lambda *a, **kw: None, + signal_region_progress=lambda *a, **kw: None, + ) + RecordZStackController.__init__( + self, + microscope=microscope, + live_controller=live_controller, + laser_autofocus_controller=laser_autofocus_controller, + objective_store=objective_store, + scan_coordinates=scan_coordinates, + callbacks=callbacks, + ) + QObject.__init__(self) + + def _on_acquisition_finished(self): + self.acquisition_finished.emit() + + class HighContentScreeningGui(QMainWindow): fps_software_trigger = 100 LASER_BASED_FOCUS_TAB_NAME = "Laser-Based Focus" @@ -676,6 +723,7 @@ def __init__( self.well_selector_visible = False # Add this line to track well selector visibility self.multipointController: QtMultiPointController = None + self.recordZStackController: Optional[QtRecordZStackController] = None self.streamHandler: core.QtStreamHandler = None self.autofocusController: AutoFocusController = None self.imageSaver: core.ImageSaver = core.ImageSaver() @@ -884,6 +932,14 @@ def scan_coordinate_callback(update: ScanCoordinatesUpdate): laser_autofocus_controller=self.laserAutofocusController, fluidics=self.fluidics, ) + if ENABLE_RECORDING: + self.recordZStackController = QtRecordZStackController( + self.microscope, + self.liveController, + self.laserAutofocusController, + self.objectiveStore, + self.scanCoordinates, + ) def setup_hardware(self): self.camera.add_frame_callback(self.streamHandler.get_frame_callback()) @@ -1077,6 +1133,20 @@ def load_widgets(self): show_configurations=TRACKING_SHOW_MICROSCOPE_CONFIGURATIONS, ) + if ENABLE_RECORDING: + self.recordZStackWidget = widgets.RecordZStackMultiPointWidget( + self.stage, + self.navigationViewer, + self.recordZStackController, + self.liveController, + self.objectiveStore, + self.scanCoordinates, + well_selection_widget=self.wellSelectionWidget, + tab_widget=self.recordTabWidget, + laser_autofocus_controller=self.laserAutofocusController, + ) + self.recordZStackController.acquisition_finished.connect(self.recordZStackWidget.acquisition_is_finished) + self.setupRecordTabWidget() self.setupCameraTabWidget() @@ -1270,6 +1340,7 @@ def setupRecordTabWidget(self): self.recordTabWidget.addTab(self.trackingControlWidget, "Tracking") if ENABLE_RECORDING: self.recordTabWidget.addTab(self.recordingControlWidget, "Simple Recording") + self.recordTabWidget.addTab(self.recordZStackWidget, "Record + Z-Stack") self.recordTabWidget.currentChanged.connect(lambda: self.resizeCurrentTab(self.recordTabWidget)) self.resizeCurrentTab(self.recordTabWidget) From 30148936c5edf4cb7c9f6d7460f1a04149f22eb4 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 12:10:36 -0700 Subject: [PATCH 21/61] feat(record-zstack): widget finished/progress slots for gui_hcs wiring Completes E4: gui_hcs.py:1148 connects acquisition_finished -> recordZStackWidget.acquisition_is_finished; these slots were authored with E4 but omitted from its commit. Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/widgets.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/software/control/widgets.py b/software/control/widgets.py index 300c689b8..c86550d71 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17617,3 +17617,14 @@ def toggle_acquisition(self, pressed: bool) -> None: self.recordZStackController.run_acquisition() else: self.recordZStackController.request_abort() + + def acquisition_is_finished(self): + """Called (thread-safe via Qt signal) when the acquisition worker finishes.""" + self.btn_startAcquisition.setChecked(False) + + def display_progress_bar(self, show: bool) -> None: + """No-op stub: RecordZStackMultiPointWidget has no progress bar. + + Required so ``toggleAcquisitionStart`` in gui_hcs can call this on + whatever widget is the current record tab without checking the type. + """ From 25131f463b445a737d0350ee383d12b933bc8ce9 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 12:15:59 -0700 Subject: [PATCH 22/61] fix(record-zstack): add recordZStackWidget None sentinel (gui_hcs) Mirrors the recordZStackController sentinel; avoids a latent AttributeError if recordZStackWidget is referenced when ENABLE_RECORDING is False. Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/gui_hcs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/software/control/gui_hcs.py b/software/control/gui_hcs.py index 42b4a3bcf..5556b1c60 100644 --- a/software/control/gui_hcs.py +++ b/software/control/gui_hcs.py @@ -724,6 +724,7 @@ def __init__( self.multipointController: QtMultiPointController = None self.recordZStackController: Optional[QtRecordZStackController] = None + self.recordZStackWidget: Optional[widgets.RecordZStackMultiPointWidget] = None self.streamHandler: core.QtStreamHandler = None self.autofocusController: AutoFocusController = None self.imageSaver: core.ImageSaver = core.ImageSaver() From 27694cba22d36db804578ca0cfb1b84d48423a00 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 12:35:03 -0700 Subject: [PATCH 23/61] fix(record-zstack): illumination, trigger mode, streaming & live-view lifecycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CRITICAL 1: recording captured dark frames — turn illumination on around cap.run() (off in finally), since set_microscope_mode only energizes illumination when live and the CONTINUOUS stream does not gate it. CRITICAL 2: z-stack used the wrong acquire_camera_image branch — set liveController.set_trigger_mode(SOFTWARE) (capturing/restoring the previous mode) so camera mode and liveController.trigger_mode agree. IMPORTANT 6: zstack() now calls camera.stop_streaming() in the finally, mirroring ContinuousFrameSource.stop(), so the next FOV's recording starts on a stopped camera. IMPORTANT 9a: hoist stop-live to run() start (once, capturing was_live) and restart live once in run()'s finally; removed per-FOV stop-live from record(). MEDIUM: precompute zstack offsets and frame shape once in __init__. SIMPLIFICATION: resolve recording_channel to a local in record(). Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/core/record_zstack_worker.py | 92 ++++++++++++++----- 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/software/control/core/record_zstack_worker.py b/software/control/core/record_zstack_worker.py index 7425a797f..ccbb237fb 100644 --- a/software/control/core/record_zstack_worker.py +++ b/software/control/core/record_zstack_worker.py @@ -28,6 +28,7 @@ SCAN_STABILIZATION_TIME_MS_X, SCAN_STABILIZATION_TIME_MS_Y, SCAN_STABILIZATION_TIME_MS_Z, + TriggerMode, ) from squid.abc import CameraAcquisitionMode from control.core.multi_point_worker import MultiPointWorkerBase @@ -139,6 +140,13 @@ def __init__( self._abort_on_failed_job = True self._first_job_dispatched = False + # Per-acquisition fixed geometry — compute once and reuse for every FOV + # (z-stack offsets and the recording frame shape don't change mid-run). + self._zstack_offsets: List[float] = ( + zstack_offsets_um(params.z_min_um, params.z_max_um, params.z_step_um) if params.zstack_enabled else [] + ) + self._frame_shape: Tuple[int, int, np.dtype] = self._probe_frame_shape() + if params.zstack_enabled and self.zstack_channels: self._setup_zstack_job_runner(prewarmed_job_runner, prewarmed_bp_values) else: @@ -230,6 +238,14 @@ def run(self): camera is left stopped between FOVs/phases, so this loop owns no camera callback of its own. """ + # Quiesce live view once for the whole acquisition (restored in finally). + was_live = bool(getattr(self.liveController, "is_live", False)) + if was_live: + try: + self.liveController.stop_live() + except Exception: + log.exception("Failed to stop live view before acquisition") + try: if self.params.zstack_enabled and self._job_runners: self._backpressure.reset() @@ -263,6 +279,12 @@ def run(self): self._finish_jobs() except Exception: log.exception("Error finishing z-stack jobs") + # Restart live view once, only if it was running before the acquisition. + if was_live: + try: + self.liveController.start_live() + except Exception: + log.exception("Failed to restart live view after acquisition") try: self.callbacks.signal_acquisition_finished() except Exception: @@ -296,29 +318,20 @@ def record(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> int: Returns the number of frames emitted. """ - # Quiesce live view if one is running (guarded for the headless/test case). - was_live = bool(getattr(self.liveController, "is_live", False)) - if was_live: - try: - self.liveController.stop_live() - except Exception: - log.exception("Failed to stop live view before recording") - - # Apply the recording channel (exposure/gain/illumination). - if self.params.recording_channel is not None: - self._select_config(self.params.recording_channel) + # Apply the recording channel (exposure/gain/illumination settings). + rec_channel = self.params.recording_channel + if rec_channel is not None: + self._select_config(rec_channel) # Move to z_ref + recording offset. self._move_z_to_offset(z_ref, self.params.recording_z_offset_um) T = frame_count(self.params.fps, self.params.duration_s) out = self._recording_path(t_idx, region_id, fov_idx) - y, x, dtype = self._probe_frame_shape() + y, x, dtype = self._frame_shape - rec_channel_name = self.params.recording_channel.name if self.params.recording_channel is not None else "REC" - rec_color = ( - self.params.recording_channel.display_color if self.params.recording_channel is not None else "#FFFFFF" - ) + rec_channel_name = rec_channel.name if rec_channel is not None else "REC" + rec_color = rec_channel.display_color if rec_channel is not None else "#FFFFFF" cfg = ZarrAcquisitionConfig( output_path=out, shape=(T, 1, 1, y, x), @@ -341,7 +354,16 @@ def record(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> int: ) # Generous timeout: enough to gather T frames even at a slow effective rate. timeout = self.params.duration_s * 3 + 5 - emitted = cap.run(timeout=timeout) + + # The CONTINUOUS stream does not gate illumination per-frame, and + # set_microscope_mode only energizes illumination when live (we are not + # live here). Turn illumination on for the whole recording, off in finally, + # so the recorded frames are not dark. + self.liveController.turn_on_illumination() + try: + emitted = cap.run(timeout=timeout) + finally: + self.liveController.turn_off_illumination() log.info(f"recording done t={t_idx} region={region_id} fov={fov_idx}: {emitted}/{T} frames") # Restore the camera to a software-trigger-friendly state and Z reference. @@ -362,15 +384,25 @@ def zstack(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> None: -> ``SaveZarrJob`` dispatch. Restores Z to ``z_ref`` at the end. """ self.time_point = t_idx - offsets = zstack_offsets_um(self.params.z_min_um, self.params.z_max_um, self.params.z_step_um) - - # The inherited capture path needs the camera streaming, in software-trigger - # mode, with our frame callback registered. Manage that lifecycle locally so - # it never interferes with the recording phase's CONTINUOUS streaming. + offsets = self._zstack_offsets + + # The inherited capture path (acquire_camera_image) branches on + # liveController.trigger_mode, so the LiveController and the camera must agree + # on software-trigger mode. set_trigger_mode(SOFTWARE) sets both the camera + # acquisition mode and the microcontroller trigger mode; capture the previous + # LiveController mode and restore it in the finally. Manage the streaming + # lifecycle locally so it never interferes with the recording phase's + # CONTINUOUS streaming. + prev_trigger_mode = getattr(self.liveController, "trigger_mode", None) try: - self.camera.set_acquisition_mode(CameraAcquisitionMode.SOFTWARE_TRIGGER) + self.liveController.set_trigger_mode(TriggerMode.SOFTWARE) except Exception: log.exception("Failed to set software-trigger mode for z-stack") + # Fall back to setting the camera directly so capture can still proceed. + try: + self.camera.set_acquisition_mode(CameraAcquisitionMode.SOFTWARE_TRIGGER) + except Exception: + log.exception("Failed to set camera software-trigger mode for z-stack") self.camera.start_streaming() cb_id = self.camera.add_frame_callback(self._image_callback) # Make sure the trigger gate starts open. @@ -395,12 +427,24 @@ def zstack(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> None: config_idx=c_idx, ) finally: - # Wait for the last in-flight frame, then detach our callback. + # Wait for the last in-flight frame, then detach our callback and stop + # streaming (mirrors ContinuousFrameSource.stop()), so the next FOV's + # recording phase starts ContinuousFrameSource on a stopped camera. self._wait_for_outstanding_callback_images() try: self.camera.remove_frame_callback(cb_id) except Exception: log.exception("Failed to remove z-stack frame callback") + try: + self.camera.stop_streaming() + except Exception: + log.exception("Failed to stop streaming after z-stack") + # Restore the LiveController trigger mode captured before the z-stack. + if prev_trigger_mode is not None and prev_trigger_mode != TriggerMode.SOFTWARE: + try: + self.liveController.set_trigger_mode(prev_trigger_mode) + except Exception: + log.exception("Failed to restore LiveController trigger mode after z-stack") self.stage.move_z_to(z_ref) self.wait_till_operation_is_completed() self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) From da939bf165dc593b6c798526b641d6fc517d38ac Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 12:42:52 -0700 Subject: [PATCH 24/61] fix: harden streaming_capture (fps order, start error path, OOB gating, non-blocking enqueue, abort sealing, partial warning) - ContinuousFrameSource.start: set CONTINUOUS mode before set_frame_rate so the toupcam PRECISE_FRAMERATE hint survives the mode switch (IMPORTANT 5). - RecordingWriter: track _started, set only after thread.start(); finalize/abort skip the join when not started so a failing initialize() propagates its real error instead of a 'cannot join thread' crash (IMPORTANT 7). - StreamingCapture._on_frame: re-check stop condition before routing so no frame is enqueued at/past t-index T into a (T,...)-shaped dataset (IMPORTANT 8). - RecordingWriter.enqueue: put_nowait + drop-on-full (no 0.5s block on the camera hot thread); default queue maxsize 64 -> 256 (MEDIUM). - StreamingCapture: track _aborted; run() finally calls writer.abort() on abort (seals zarr incomplete) else finalize() (MEDIUM). - StreamingCapture.run: WARNING when emitted < expected (CountStop.expected()); trailing zarr planes are blank fill (MEDIUM). - Document that the post-stop _emitted read is safe (MINOR). - Tests: start-error path, OOB gating, abort->writer.abort, complete->finalize, partial warning. Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/core/streaming_capture.py | 78 +++++++-- software/tests/core/test_streaming_capture.py | 152 ++++++++++++++++++ 2 files changed, 219 insertions(+), 11 deletions(-) diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index d55c5048c..e5be40da8 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -19,6 +19,10 @@ def __init__(self, target: int): def met(self, emitted: int) -> bool: return emitted >= self.target + def expected(self) -> Optional[int]: + """Expected total frame count, used for partial-capture warnings.""" + return self.target + class RecordingRouter: """Maps incoming frames to (t,c,z)=(t_index,0,0), downsampling to `fps`.""" @@ -40,10 +44,10 @@ def route(self, timestamp: float) -> Optional[Tuple[int, int, int]]: class RecordingWriter: """Bounded-queue writer that drains frames to a ZarrWriter on a background thread. - The hot camera callback calls `enqueue` (non-blocking); the background thread - calls `ZarrWriter.write_frame` which may block on I/O. The queue is bounded so - that a slow disk eventually provides backpressure: `enqueue` will block for up - to 0.5 s before logging a drop and returning. + The hot camera callback calls `enqueue` (truly non-blocking); the background + thread calls `ZarrWriter.write_frame` which may block on I/O. The queue is + bounded so that a slow disk eventually fills it: when full, `enqueue` drops the + frame immediately (never blocks the camera delivery thread) and logs a warning. After `start()` the drain thread is the SOLE owner of the ZarrWriter: only it calls `write_frame`, `finalize`, and `abort`. The main thread only calls @@ -52,22 +56,38 @@ class RecordingWriter: concurrently with the drain thread still inside `write_frame`. """ - def __init__(self, config: ZarrAcquisitionConfig, max_queue: int = 64): + def __init__(self, config: ZarrAcquisitionConfig, max_queue: int = 256): self._writer = ZarrWriter(config) self._q: "queue.Queue" = queue.Queue(maxsize=max_queue) self._thread = threading.Thread(target=self._drain, daemon=True) self._dropped = 0 self._abort_requested = threading.Event() + # True only once the drain thread has actually been started. finalize()/ + # abort() must not join (or push the sentinel to) a thread that never + # started, otherwise a failure in start()'s initialize() would surface as + # "cannot join thread before it is started" and mask the real error. + self._started = False def start(self) -> None: - """Initialize the underlying ZarrWriter and start the drain thread.""" + """Initialize the underlying ZarrWriter and start the drain thread. + + ``initialize()`` runs BEFORE the thread starts. If it raises, the thread + is never started and ``_started`` stays False, so a later finalize()/abort() + cleanly no-ops the join and the original ``initialize()`` error propagates. + """ self._writer.initialize() self._thread.start() + self._started = True def enqueue(self, frame: np.ndarray, t: int, c: int, z: int) -> None: - """Non-blocking enqueue. Blocks briefly as backpressure; drops on full.""" + """Truly non-blocking enqueue: drops the frame on a full queue. + + Runs on the hot camera delivery thread, so it must never block. When the + bounded queue is full (drain thread cannot keep up with disk I/O) the frame + is dropped and counted rather than waiting for space. + """ try: - self._q.put((frame, t, c, z), timeout=0.5) + self._q.put_nowait((frame, t, c, z)) except queue.Full: self._dropped += 1 _log.warning(f"recording queue full; dropped frame t={t} (total dropped={self._dropped})") @@ -101,6 +121,10 @@ def _drain(self) -> None: def finalize(self) -> None: """Flush the queue, join the drain thread (which finalizes the ZarrWriter).""" + if not self._started: + # start() never got the thread running (e.g. initialize() raised). + # Nothing to flush or join; let the original error propagate. + return self._q.put(_SENTINEL) self._thread.join(timeout=30.0) if self._thread.is_alive(): @@ -108,6 +132,10 @@ def finalize(self) -> None: def abort(self) -> None: """Signal the drain thread to stop (which aborts the ZarrWriter).""" + if not self._started: + # start() never got the thread running (e.g. initialize() raised). + self._abort_requested.set() + return self._abort_requested.set() try: self._q.put_nowait(_SENTINEL) @@ -126,7 +154,7 @@ def abort(self) -> None: class ContinuousFrameSource: """Wraps a camera and delivers frames via callback. - Calls set_frame_rate, set_acquisition_mode(CONTINUOUS), registers a frame + Calls set_acquisition_mode(CONTINUOUS), set_frame_rate, registers a frame callback, and starts/stops streaming. """ @@ -136,8 +164,11 @@ def __init__(self, camera, fps: float): self._cb_id: Optional[int] = None def start(self, on_frame: Callable) -> None: - self._camera.set_frame_rate(self._fps) + # Order matters: switching to CONTINUOUS resets the frame-rate strategy to + # MAX on toupcam, wiping any earlier fps hint. Set the mode FIRST, then the + # frame rate, so the PRECISE_FRAMERATE hint survives the mode switch. self._camera.set_acquisition_mode(CameraAcquisitionMode.CONTINUOUS) + self._camera.set_frame_rate(self._fps) self._cb_id = self._camera.add_frame_callback(on_frame) self._camera.start_streaming() @@ -177,12 +208,22 @@ def __init__(self, frame_source, router, stop_condition, writer, abort_fn: Calla self._abort_fn = abort_fn self._emitted = 0 self._done = threading.Event() + self._aborted = False def _on_frame(self, camera_frame) -> None: """Hot-thread callback: route + enqueue only. Must not block.""" if self._done.is_set(): return if self._abort_fn(): + self._aborted = True + self._done.set() + return + # Out-of-bounds guard: if the stop condition is already met for the current + # emitted count, do not route/enqueue. A frame that arrives in-flight after + # CountStop(T) is satisfied would otherwise route to t-index == T and enqueue + # into a (T, ...)-shaped dataset (out of bounds). Re-check here, not just at + # entry, so once _emitted == T no further frame is ever emitted. + if self._stop.met(self._emitted): self._done.set() return idx = self._router.route(camera_frame.timestamp) @@ -205,5 +246,20 @@ def run(self, timeout: Optional[float] = None) -> int: # drain thread has exited, so the put times out and the frame is logged as # dropped, not corrupted). self._source.stop() - self._writer.finalize() + # source.stop() above quiesces the camera delivery thread, so reading + # self._emitted here is safe without a lock: no callback thread mutates + # it after this point (and CPython int load/store is atomic anyway). + expected = self._stop.expected() if hasattr(self._stop, "expected") else None + if self._aborted: + # Aborted mid-capture: seal the recording as incomplete, not complete. + self._writer.abort() + else: + self._writer.finalize() + if expected is not None and self._emitted < expected: + # Stop condition was not met (slow camera / timeout): the trailing + # zarr planes are fill values, not real data. Warn loudly. + _log.warning( + f"streaming capture incomplete: captured {self._emitted}/{expected} " + f"frames; trailing planes are blank fill" + ) return self._emitted diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py index 2710fa21c..cbb7a1bf8 100644 --- a/software/tests/core/test_streaming_capture.py +++ b/software/tests/core/test_streaming_capture.py @@ -105,3 +105,155 @@ def test_streaming_capture_counts_and_downsamples(): assert emitted == 5 assert w.writes == [(0, 0, 0), (1, 0, 0), (2, 0, 0), (3, 0, 0), (4, 0, 0)] assert getattr(w, "finalized", False) is True + + +# --------------------------------------------------------------------------- +# Fix-batch tests: start() error path, OOB gating, abort path, partial warning +# --------------------------------------------------------------------------- + + +def test_recording_writer_start_failure_propagates_without_join_crash(tmp_path, monkeypatch): + """If ZarrWriter.initialize() raises, start() propagates the original error and + finalize()/abort() must NOT crash trying to join an unstarted thread.""" + from control.core.zarr_writer import ZarrAcquisitionConfig, ZarrWriter + from control.core.streaming_capture import RecordingWriter + + cfg = ZarrAcquisitionConfig( + output_path=str(tmp_path / "rec.ome.zarr"), + shape=(2, 1, 1, 4, 4), + dtype=np.uint16, + pixel_size_um=1.0, + z_step_um=None, + time_increment_s=0.1, + channel_names=["BF"], + channel_colors=["#FFFFFF"], + channel_wavelengths=[None], + is_hcs=False, + ) + + sentinel_error = RuntimeError("boom from initialize") + + def boom(self): + raise sentinel_error + + monkeypatch.setattr(ZarrWriter, "initialize", boom) + + w = RecordingWriter(cfg) + import pytest + + with pytest.raises(RuntimeError, match="boom from initialize"): + w.start() + + assert w._started is False + # finalize() and abort() must be safe no-ops (no "cannot join thread" crash). + w.finalize() + w.abort() + + +class _CountingFakeSource: + """Delivers all frames synchronously, even past the stop count, to exercise the + out-of-bounds guard in _on_frame.""" + + def __init__(self, frames): + self._frames = frames + + def start(self, on_frame): + for f in self._frames: + on_frame(f) + + def stop(self): + pass + + +def test_streaming_capture_no_enqueue_past_T_with_extra_frames(): + """Frames arriving after the stop count is met must never be enqueued (would be + out of bounds for a (T, ...)-shaped dataset).""" + # No downsampling (fps=0 -> emit every frame); 10 frames but T=3. + frames = [_FakeFrame(100.0 + i, np.zeros((4, 4), np.uint16)) for i in range(10)] + w = _ListWriter() + cap = StreamingCapture( + _CountingFakeSource(frames), + RecordingRouter(fps=0.0), + CountStop(3), + w, + abort_fn=lambda: False, + ) + emitted = cap.run() + assert emitted == 3 + # Only t-indices 0..2 — nothing at or beyond T=3. + assert w.writes == [(0, 0, 0), (1, 0, 0), (2, 0, 0)] + assert all(t < 3 for (t, _, _) in w.writes) + + +class _RecordingStubWriter: + """Records which of finalize()/abort() was called.""" + + def __init__(self): + self.writes = [] + self.finalized = False + self.aborted = False + + def start(self): + pass + + def enqueue(self, frame, t, c, z): + self.writes.append((t, c, z)) + + def finalize(self): + self.finalized = True + + def abort(self): + self.aborted = True + + +def test_streaming_capture_abort_calls_writer_abort_not_finalize(): + frames = [_FakeFrame(100.0 + i, np.zeros((4, 4), np.uint16)) for i in range(10)] + w = _RecordingStubWriter() + cap = StreamingCapture( + _CountingFakeSource(frames), + RecordingRouter(fps=0.0), + CountStop(100), # never reached + w, + abort_fn=lambda: True, # abort on first frame + ) + emitted = cap.run() + assert emitted == 0 + assert w.aborted is True + assert w.finalized is False + + +def test_streaming_capture_complete_calls_finalize_not_abort(): + frames = [_FakeFrame(100.0 + i, np.zeros((4, 4), np.uint16)) for i in range(5)] + w = _RecordingStubWriter() + cap = StreamingCapture( + _CountingFakeSource(frames), + RecordingRouter(fps=0.0), + CountStop(3), + w, + abort_fn=lambda: False, + ) + emitted = cap.run() + assert emitted == 3 + assert w.finalized is True + assert w.aborted is False + + +def test_streaming_capture_partial_warns(caplog): + """Fewer frames than T -> finalize + a loud WARNING about partial capture.""" + import logging + + frames = [_FakeFrame(100.0 + i, np.zeros((4, 4), np.uint16)) for i in range(2)] + w = _RecordingStubWriter() + cap = StreamingCapture( + _CountingFakeSource(frames), + RecordingRouter(fps=0.0), + CountStop(5), # expect 5, only 2 delivered + w, + abort_fn=lambda: False, + ) + with caplog.at_level(logging.WARNING): + emitted = cap.run(timeout=0.1) + assert emitted == 2 + assert w.finalized is True + assert w.aborted is False + assert any("incomplete" in rec.getMessage() and "2/5" in rec.getMessage() for rec in caplog.records) From 2e9cf5791c3be582954959ffdfb246da7352ca1c Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 12:53:58 -0700 Subject: [PATCH 25/61] fix(record-zstack): batch-3 correctness, threading, and simplification fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - IMPORTANT 3+4: fix _get_selected_well_count to resolve lazily via scanCoordinates.get_selected_wells() instead of a stale cached well_selection_widget; handles None (glass-slide → count=1) and empty dict (no wells selected → count=0 → validation rejects) - IMPORTANT 9b: reject recording configs where frame_count(fps, duration_s) < 1 in _validate_record_zstack_params with clear message - MEDIUM: replace _abort_requested bool with threading.Event (_abort_event); set/clear/is_set are thread-safe; abort_requested_fn passed to worker uses lambda: self._abort_event.is_set() - REUSE: add run_acquisition(params) direct-params path; toggle_acquisition now calls run_acquisition(self.build_parameters()) — no 15-setter fan-out; setters kept as shims for test_record_zstack_worker.py compatibility - SIMPLIFICATION: remove unreachable None guard in _on_zstack_add_channel_clicked; log warnings in _get_zstack_row_values instead of silently substituting defaults - Tests: 39 pass (8 new); cover frame_count=0, glass-slide/None well count, lazy well-selector resolution, and abort Event lifecycle Co-Authored-By: Claude Sonnet 4.6 --- .../control/core/record_zstack_controller.py | 65 ++++--- software/control/widgets.py | 66 +++---- software/tests/test_record_zstack_widget.py | 161 +++++++++++++++++- 3 files changed, 224 insertions(+), 68 deletions(-) diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py index cec0b5236..b865a4310 100644 --- a/software/control/core/record_zstack_controller.py +++ b/software/control/core/record_zstack_controller.py @@ -1,6 +1,6 @@ import math from dataclasses import dataclass, field -from threading import Thread +from threading import Event, Thread from typing import List, Optional import squid.logging @@ -88,7 +88,7 @@ def __init__( self.z_max_um: float = 3.0 self.z_step_um: float = 1.0 - self._abort_requested: bool = False + self._abort_event: Event = Event() self._worker = None self._thread: Optional[Thread] = None @@ -188,41 +188,50 @@ def acquisition_in_progress(self) -> bool: def request_abort(self) -> None: """Signal the running worker to stop after the current FOV.""" - self._abort_requested = True + self._abort_event.set() - def run_acquisition(self) -> None: - """Build params, set up the pre-warmed JobRunner, and spawn the worker thread.""" + def run_acquisition(self, params: Optional[RecordZStackAcquisitionParameters] = None) -> None: + """Build params, set up the pre-warmed JobRunner, and spawn the worker thread. + + If *params* is provided the acquisition uses those values directly; + otherwise the instance attributes set via the individual setters are used + (legacy path, kept for backwards compatibility with tests/scripts that + still call the setters before invoking run_acquisition()). + """ from control.core.acquisition_setup import create_experiment_dir from control.core.record_zstack_worker import RecordZStackWorker + # Resolve params: use supplied object or build one from instance attrs. + if params is None: + params = RecordZStackAcquisitionParameters( + base_path=self.base_path, + experiment_id=self.experiment_id, + Nt=self.Nt, + dt_s=self.dt_s, + use_laser_af=self.use_laser_af, + recording_enabled=self.recording_enabled, + recording_channel=self.recording_channel, + fps=self.fps, + duration_s=self.duration_s, + recording_z_offset_um=self.recording_z_offset_um, + zstack_enabled=self.zstack_enabled, + zstack_channels=list(self.zstack_channels), + z_min_um=self.z_min_um, + z_max_um=self.z_max_um, + z_step_um=self.z_step_um, + ) + # Resolve and create a timestamped unique output directory. - resolved_id, _dir = create_experiment_dir(self.base_path, self.experiment_id) - - params = RecordZStackAcquisitionParameters( - base_path=self.base_path, - experiment_id=resolved_id, - Nt=self.Nt, - dt_s=self.dt_s, - use_laser_af=self.use_laser_af, - recording_enabled=self.recording_enabled, - recording_channel=self.recording_channel, - fps=self.fps, - duration_s=self.duration_s, - recording_z_offset_um=self.recording_z_offset_um, - zstack_enabled=self.zstack_enabled, - zstack_channels=list(self.zstack_channels), - z_min_um=self.z_min_um, - z_max_um=self.z_max_um, - z_step_um=self.z_step_um, - ) + resolved_id, _dir = create_experiment_dir(params.base_path, params.experiment_id) + params.experiment_id = resolved_id # Collect scan coordinates: {region_id: [(x_mm, y_mm[, z_mm]), ...]} scan_region_fov_coords = {} if self._scan_coordinates is not None and hasattr(self._scan_coordinates, "region_fov_coordinates"): scan_region_fov_coords = dict(self._scan_coordinates.region_fov_coordinates) - # Reset abort flag for this run. - self._abort_requested = False + # Clear abort event for this run (thread-safe: Event.clear() is atomic). + self._abort_event.clear() # Consume the pre-warmed runner; only pass it to the worker when # USE_MULTIPROCESSING is True (otherwise it's None and passing it would @@ -237,7 +246,7 @@ def run_acquisition(self) -> None: objective_store=self._objective_store, params=params, callbacks=self._callbacks, - abort_requested_fn=lambda: self._abort_requested, + abort_requested_fn=lambda: self._abort_event.is_set(), request_abort_fn=self.request_abort, scan_region_fov_coords=scan_region_fov_coords, prewarmed_job_runner=prewarmed_runner if control._def.Acquisition.USE_MULTIPROCESSING else None, @@ -271,7 +280,7 @@ def close(self, timeout_s: float = 5.0) -> None: self._prewarmed_bp_values = None if self.acquisition_in_progress(): - self.request_abort() + self._abort_event.set() if self._thread is not None: self._thread.join(timeout=timeout_s) if self._thread.is_alive(): diff --git a/software/control/widgets.py b/software/control/widgets.py index c86550d71..30355afa4 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -16963,6 +16963,10 @@ def _validate_record_zstack_params( return "Recording FPS must be greater than 0." if duration_s <= 0: return "Recording duration must be greater than 0." + from control.core.record_zstack_controller import frame_count as _frame_count + + if _frame_count(fps, duration_s) < 1: + return f"Recording: fps×duration yields 0 frames (fps={fps}, duration={duration_s}s). Increase one or both." if not recording_channel_name: return "A channel must be chosen for the Recording phase." if zstack_enabled: @@ -17411,7 +17415,10 @@ def _set_zstack_row_values(self, name: str, exposure: float, gain: float, illumi def _get_zstack_row_values(self, name: str): """Return (exposure, gain, illumination) for the z-stack row identified by *name*. - Returns (50.0, 0.0, 50.0) as defaults if the row or spinboxes are missing. + Returns (50.0, 0.0, 50.0) as defaults if the row is not found (logs a warning). + Spinbox widgets are always present when a row exists (created by + _add_zstack_channel_row), so missing-spinbox branches are not expected; + a warning is logged rather than silently substituting defaults. """ for row in range(self.zstack_channel_table.rowCount()): item = self.zstack_channel_table.item(row, 0) @@ -17419,10 +17426,14 @@ def _get_zstack_row_values(self, name: str): exp_spin = self.zstack_channel_table.cellWidget(row, 1) gain_spin = self.zstack_channel_table.cellWidget(row, 2) illum_spin = self.zstack_channel_table.cellWidget(row, 3) - exposure = exp_spin.value() if exp_spin is not None else 50.0 - gain = gain_spin.value() if gain_spin is not None else 0.0 - illum = illum_spin.value() if illum_spin is not None else 50.0 - return exposure, gain, illum + if exp_spin is None or gain_spin is None or illum_spin is None: + self._log.warning( + f"_get_zstack_row_values: spinbox(es) missing for row '{name}'; " + "returning defaults (50.0, 0.0, 50.0)" + ) + return 50.0, 0.0, 50.0 + return exp_spin.value(), gain_spin.value(), illum_spin.value() + self._log.warning(f"_get_zstack_row_values: row '{name}' not found; returning defaults") return 50.0, 0.0, 50.0 def _copy_recording_from_live(self) -> None: @@ -17456,20 +17467,29 @@ def _copy_zstack_row_from_live(self, name: str) -> None: def _on_zstack_add_channel_clicked(self) -> None: """Add the currently selected channel in the add-channel combo to the z-stack table.""" - if self.combobox_zstack_add_channel is None: - return name = self.combobox_zstack_add_channel.currentText() if name: self._add_zstack_channel_row(name) def _get_selected_well_count(self) -> int: - """Return the number of currently selected wells.""" - if self.well_selection_widget is not None and hasattr(self.well_selection_widget, "get_selected_wells"): - return len(self.well_selection_widget.get_selected_wells()) + """Return the number of currently selected wells. + + Resolves lazily via ``self.scanCoordinates`` so that plate-format + changes (which replace ``gui_hcs.wellSelectionWidget`` and call + ``scanCoordinates.add_well_selector()``) are always reflected + without needing to update a cached widget reference. + + Returns 0 when a wellplate format is active but no wells are + selected. Returns 1 for glass-slide (current-position imaging) + and when no scanCoordinates is attached. + """ if self.scanCoordinates is not None and hasattr(self.scanCoordinates, "get_selected_wells"): - return len(self.scanCoordinates.get_selected_wells()) - # Fallback: treat as 1 well (current position) so the widget is usable - # without a well-selection panel attached. + selected = self.scanCoordinates.get_selected_wells() + if selected is None: + # glass-slide: imaging at current position — count as 1 + return 1 + return len(selected) + # No scanCoordinates attached: treat as single-position (glass-slide-like). return 1 def _laser_af_has_reference(self) -> bool: @@ -17596,25 +17616,7 @@ def toggle_acquisition(self, pressed: bool) -> None: return params = self.build_parameters() - - # Push all parameters to the controller via its individual setters. - self.recordZStackController.set_base_path(params.base_path) - self.recordZStackController.set_experiment_id(params.experiment_id) - self.recordZStackController.set_Nt(params.Nt) - self.recordZStackController.set_dt_s(params.dt_s) - self.recordZStackController.set_use_laser_af(params.use_laser_af) - self.recordZStackController.set_recording_enabled(params.recording_enabled) - self.recordZStackController.set_recording_channel(params.recording_channel) - self.recordZStackController.set_fps(params.fps) - self.recordZStackController.set_duration_s(params.duration_s) - self.recordZStackController.set_recording_z_offset_um(params.recording_z_offset_um) - self.recordZStackController.set_zstack_enabled(params.zstack_enabled) - self.recordZStackController.set_zstack_channels(params.zstack_channels) - self.recordZStackController.set_z_min_um(params.z_min_um) - self.recordZStackController.set_z_max_um(params.z_max_um) - self.recordZStackController.set_z_step_um(params.z_step_um) - - self.recordZStackController.run_acquisition() + self.recordZStackController.run_acquisition(params) else: self.recordZStackController.request_abort() diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index e6c1ebff6..557ca7b6b 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -483,20 +483,26 @@ def test_toggle_acquisition_start_valid_calls_run_acquisition(qtbot, simulated_w def test_toggle_acquisition_start_valid_pushes_params_to_controller(qtbot, simulated_widget_deps): - """Clicking Start with valid params pushes all parameters via controller setters.""" + """Clicking Start with valid params calls run_acquisition(params) with correct values.""" + from control.core.record_zstack_controller import RecordZStackAcquisitionParameters + ctrl = _make_stub_controller() simulated_widget_deps["recordZStackController"] = ctrl w = _make_valid_widget(qtbot, simulated_widget_deps) w.toggle_acquisition(True) - # Verify key setters were called - ctrl.set_base_path.assert_called_once_with("/tmp/test_e3") - ctrl.set_zstack_enabled.assert_called_once_with(True) - ctrl.set_recording_enabled.assert_called_once_with(False) - ctrl.set_z_min_um.assert_called_once_with(-2.0) - ctrl.set_z_max_um.assert_called_once_with(2.0) - ctrl.set_z_step_um.assert_called_once_with(1.0) + # run_acquisition should be called once with a RecordZStackAcquisitionParameters object + ctrl.run_acquisition.assert_called_once() + call_args = ctrl.run_acquisition.call_args + params = call_args.args[0] if call_args.args else call_args.kwargs.get("params") + assert isinstance(params, RecordZStackAcquisitionParameters) + assert params.base_path == "/tmp/test_e3" + assert params.zstack_enabled is True + assert params.recording_enabled is False + assert params.z_min_um == pytest.approx(-2.0) + assert params.z_max_um == pytest.approx(2.0) + assert params.z_step_um == pytest.approx(1.0) def test_toggle_acquisition_start_invalid_no_phase_does_not_call_run(qtbot, simulated_widget_deps): @@ -558,3 +564,142 @@ def test_toggle_acquisition_stop_calls_request_abort(qtbot, simulated_widget_dep ctrl.request_abort.assert_called_once() ctrl.run_acquisition.assert_not_called() + + +# --------------------------------------------------------------------------- +# New tests added by fix-batch3 +# --------------------------------------------------------------------------- + + +# --- IMPORTANT 9b: frame_count < 1 rejected --- + + +def test_validate_helper_recording_zero_frame_count(): + """fps × duration rounds to 0 frames — must be rejected.""" + # 0.1 fps × 1.0 s = 0.1 → round → 0 frames + params = _base_params(recording_enabled=True, fps=0.1, duration_s=1.0, zstack_enabled=False) + err = _validate_record_zstack_params(**params) + assert err is not None + assert "0 frames" in err or "frame" in err.lower() + + +def test_validate_helper_recording_one_frame_is_valid(): + """fps × duration = exactly 1 frame — must pass.""" + # 1.0 fps × 1.0 s = 1 frame + params = _base_params(recording_enabled=True, fps=1.0, duration_s=1.0, zstack_enabled=False) + assert _validate_record_zstack_params(**params) is None + + +def test_validate_helper_recording_borderline_zero_frames(): + """fps × duration just below 0.5 → rounds to 0 — must be rejected.""" + params = _base_params(recording_enabled=True, fps=0.4, duration_s=1.0, zstack_enabled=False) + err = _validate_record_zstack_params(**params) + assert err is not None + + +# --- IMPORTANT 3+4: well count handles None / glass-slide --- + + +def test_get_selected_well_count_glass_slide_returns_one(qtbot, simulated_widget_deps): + """_get_selected_well_count returns 1 (not 0) when scanCoordinates.get_selected_wells() is None (glass-slide).""" + from unittest.mock import MagicMock + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + sc.get_selected_wells.return_value = None # glass-slide: returns None + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + assert w._get_selected_well_count() == 1 + + +def test_get_selected_well_count_empty_wellplate_returns_zero(qtbot, simulated_widget_deps): + """_get_selected_well_count returns 0 when no wells are selected on a wellplate.""" + from unittest.mock import MagicMock + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + sc.get_selected_wells.return_value = {} # wellplate, no wells selected + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + assert w._get_selected_well_count() == 0 + + +def test_get_selected_well_count_wellplate_selection(qtbot, simulated_widget_deps): + """_get_selected_well_count reflects current well_selector, not a stale cached widget.""" + from unittest.mock import MagicMock + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + sc.get_selected_wells.return_value = {"A1": (0.0, 0.0), "A2": (1.0, 0.0)} + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + assert w._get_selected_well_count() == 2 + + # Simulate a plate-format change that updates the scanCoordinates well_selector + sc.get_selected_wells.return_value = {"B1": (0.0, 1.0)} + assert w._get_selected_well_count() == 1 # picks up new selection, no stale cache + + +def test_validate_rejects_no_wells_selected(qtbot, simulated_widget_deps): + """validate() returns an error when no wells are selected (dict empty from scanCoordinates).""" + from unittest.mock import MagicMock + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + sc.get_selected_wells.return_value = {} # no wells + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("/tmp/test") + w.checkbox_zstack.setChecked(True) + w.entry_zmin.setValue(-1.0) + w.entry_zmax.setValue(1.0) + w.entry_step.setValue(1.0) + w._add_zstack_channel_row("BF LED matrix full") + + err = w.validate() + assert err is not None + assert "well" in err.lower() + + +# --- MEDIUM: _abort_event threading.Event path --- + + +def test_abort_event_set_on_request_abort(): + """request_abort() sets the abort event; run_acquisition() clears it before starting.""" + import threading + from unittest.mock import MagicMock, patch + + ctrl_kwargs = dict( + microscope=MagicMock(), + live_controller=MagicMock(), + laser_autofocus_controller=MagicMock(), + objective_store=MagicMock(), + scan_coordinates=MagicMock(), + callbacks=MagicMock(), + ) + + with patch("control._def.Acquisition.USE_MULTIPROCESSING", False): + from control.core.record_zstack_controller import RecordZStackController + + ctrl = RecordZStackController(**ctrl_kwargs) + + # Event should start clear + assert not ctrl._abort_event.is_set() + + ctrl.request_abort() + assert ctrl._abort_event.is_set() + + # Simulate run_acquisition clearing the event before spawning the worker + ctrl._abort_event.clear() + assert not ctrl._abort_event.is_set() From 103bcd99dec7d92ad91ac5bc74308d985de6056d Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 13:05:17 -0700 Subject: [PATCH 26/61] =?UTF-8?q?refactor:=20DRY=20batch=204=20=E2=80=94?= =?UTF-8?q?=20lift=20=5Fwait=5Ffor=5Foutstanding=5Fcallback=5Fimages=20to?= =?UTF-8?q?=20base,=20extract=20compute=5Fpixel=5Fsize=5Fum,=20drop=20redu?= =?UTF-8?q?ndant=20=5Fjob=5Frunners=20re-assignments,=20fix=20misleading?= =?UTF-8?q?=20=5F=5Fclass=5F=5F=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - REUSE#1: move _wait_for_outstanding_callback_images verbatim to MultiPointWorkerBase; remove subclass overrides from MultiPointWorker and RecordZStackWorker (both inherit the shared implementation) - REUSE#2: add compute_pixel_size_um(objective_store, camera) to acquisition_setup.py; call from both worker __init__s - REUSE#5: deferred — ZarrWriterInfo constructions differ materially (is_hcs, use_6d_fov) between MultiPointWorker and RecordZStackWorker - SIMP#1: drop redundant self._job_runners = [] in MultiPointWorker and self._job_runners/_backpressure/_abort_on_failed_job/ _first_job_dispatched in RecordZStackWorker (base __init__ sets them) - B1: correct misleading comment — __class__ in a method body always refers to the defining class, not the runtime type; type(self) is what produces per-subclass logger names Co-Authored-By: Claude Sonnet 4.6 --- software/control/core/acquisition_setup.py | 28 ++++++++++- software/control/core/multi_point_worker.py | 46 ++++++++----------- software/control/core/record_zstack_worker.py | 25 +--------- 3 files changed, 49 insertions(+), 50 deletions(-) diff --git a/software/control/core/acquisition_setup.py b/software/control/core/acquisition_setup.py index 8d5b0547f..44498bf10 100644 --- a/software/control/core/acquisition_setup.py +++ b/software/control/core/acquisition_setup.py @@ -7,11 +7,37 @@ import os from datetime import datetime -from typing import Tuple +from typing import Optional, Tuple from control import utils +def compute_pixel_size_um(objective_store, camera) -> Optional[float]: + """Compute the physical pixel size in µm from objective and camera metadata. + + Returns the product of the objective's pixel-size factor and the camera's + binned pixel size in µm, or None if either value is unavailable or an + exception is raised. + + Args: + objective_store: ObjectiveStore (or compatible object) with + ``get_pixel_size_factor() -> Optional[float]``. + camera: AbstractCamera (or compatible) with + ``get_pixel_size_binned_um() -> Optional[float]``. + + Returns: + Pixel size in µm, or None. + """ + try: + pixel_factor = objective_store.get_pixel_size_factor() + sensor_pixel_um = camera.get_pixel_size_binned_um() + if pixel_factor is not None and sensor_pixel_um is not None: + return float(pixel_factor) * float(sensor_pixel_um) + return None + except Exception: + return None + + def create_experiment_dir(base_path: str, experiment_id: str) -> Tuple[str, str]: """Resolve a unique experiment ID and create its output directory. diff --git a/software/control/core/multi_point_worker.py b/software/control/core/multi_point_worker.py index 430262439..54dd7d6dd 100644 --- a/software/control/core/multi_point_worker.py +++ b/software/control/core/multi_point_worker.py @@ -45,6 +45,7 @@ JobRunner, JobResult, ) +from control.core.acquisition_setup import compute_pixel_size_um from control.core.mosaic_utils import ( calculate_overlap_pixels, parse_well_id, @@ -81,9 +82,11 @@ def __init__( abort_requested_fn: Callable[[], bool], request_abort_fn: Callable[[], None], ): - # Use type(self).__name__ so subclass instances log under their own class - # name, matching the pre-refactor behavior (MultiPointWorker used - # __class__.__name__, which resolved to the subclass for its instances). + # Use type(self).__name__ so each subclass instance gets a logger named + # after its own concrete class (e.g. "RecordZStackWorker"), rather than + # after the class where the expression is written. __class__ inside a + # method always refers to the defining class (MultiPointWorkerBase), not + # the runtime type, so type(self) is required to produce per-subclass names. self._log = squid.logging.get_logger(type(self).__name__) self._timing = utils.TimingManager("MultiPointWorker Timer Manager") @@ -162,6 +165,19 @@ def _select_config(self, config: AcquisitionChannel): def _frame_wait_timeout_s(self): return (self.camera.get_total_frame_time() / 1e3) + 10 + def _wait_for_outstanding_callback_images(self): + """Block until any in-flight triggered frame has been dispatched/processed.""" + self._log.info("Waiting for any outstanding frames.") + if not self._ready_for_next_trigger.wait(self._frame_wait_timeout_s()): + self._log.warning("Timed out waiting for the last outstanding frames at end of acquisition!") + + if not self._image_callback_idle.wait(self._frame_wait_timeout_s()): + self._log.warning("Timed out waiting for the last image to process!") + + # No matter what, set the flags so things can continue + self._ready_for_next_trigger.set() + self._image_callback_idle.set() + def _sleep(self, sec): time_to_sleep = max(sec, 1e-6) # self._log.debug(f"Sleeping for {time_to_sleep} [s]") @@ -544,15 +560,7 @@ def __init__( self.selected_configurations = acquisition_parameters.selected_configurations # Pre-compute acquisition metadata that remains constant throughout the run. - try: - pixel_factor = self.objectiveStore.get_pixel_size_factor() - sensor_pixel_um = self.camera.get_pixel_size_binned_um() - if pixel_factor is not None and sensor_pixel_um is not None: - self._pixel_size_um = float(pixel_factor) * float(sensor_pixel_um) - else: - self._pixel_size_um = None - except Exception: - self._pixel_size_um = None + self._pixel_size_um = compute_pixel_size_um(self.objectiveStore, self.camera) self._time_increment_s = self.dt if self.Nt > 1 and self.dt > 0 else None self._physical_size_z_um = abs(self.deltaZ) * 1000 if self.NZ > 1 else None self.timestamp_acquisition_started = acquisition_parameters.acquisition_start_time @@ -646,7 +654,6 @@ def __init__( # For now, use 1 runner per job class. There's no real reason/rationale behind this, though. The runners # can all run any job type. But 1 per is a reasonable arbitrary arrangement while we don't have a lot # of job types. If we have a lot of custom jobs, this could cause problems via resource hogging. - self._job_runners: List[Tuple[Type[Job], JobRunner]] = [] self._log.info(f"Acquisition.USE_MULTIPROCESSING = {Acquisition.USE_MULTIPROCESSING}") # Get the current log file path to share with subprocess workers @@ -988,19 +995,6 @@ def _wait_for_laser_engine(self) -> None: f"while waiting on channel(s) {channels}; aborting acquisition" ) - def _wait_for_outstanding_callback_images(self): - # If there are outstanding frames, wait for them to come in. - self._log.info("Waiting for any outstanding frames.") - if not self._ready_for_next_trigger.wait(self._frame_wait_timeout_s()): - self._log.warning("Timed out waiting for the last outstanding frames at end of acquisition!") - - if not self._image_callback_idle.wait(self._frame_wait_timeout_s()): - self._log.warning("Timed out waiting for the last image to process!") - - # No matter what, set the flags so things can continue - self._ready_for_next_trigger.set() - self._image_callback_idle.set() - def run_single_time_point(self): try: start = time.time() diff --git a/software/control/core/record_zstack_worker.py b/software/control/core/record_zstack_worker.py index ccbb237fb..1aa3d2dd2 100644 --- a/software/control/core/record_zstack_worker.py +++ b/software/control/core/record_zstack_worker.py @@ -31,6 +31,7 @@ TriggerMode, ) from squid.abc import CameraAcquisitionMode +from control.core.acquisition_setup import compute_pixel_size_um from control.core.multi_point_worker import MultiPointWorkerBase from control.core.streaming_capture import ( StreamingCapture, @@ -108,15 +109,7 @@ def __init__( ) # Pre-compute acquisition-wide metadata (pixel size etc.) for zarr/job info. - try: - pixel_factor = self.objectiveStore.get_pixel_size_factor() - sensor_pixel_um = self.camera.get_pixel_size_binned_um() - if pixel_factor is not None and sensor_pixel_um is not None: - self._pixel_size_um = float(pixel_factor) * float(sensor_pixel_um) - else: - self._pixel_size_um = None - except Exception: - self._pixel_size_um = None + self._pixel_size_um = compute_pixel_size_um(self.objectiveStore, self.camera) self._time_increment_s = params.dt_s if params.Nt > 1 and params.dt_s > 0 else None self._physical_size_z_um = abs(params.z_step_um) if self._NZ > 1 else None @@ -135,11 +128,6 @@ def __init__( physical_size_y_um=self._pixel_size_um, ) - self._backpressure: Optional[BackpressureController] = None - self._job_runners: List[Tuple[Type[Job], JobRunner]] = [] - self._abort_on_failed_job = True - self._first_job_dispatched = False - # Per-acquisition fixed geometry — compute once and reuse for every FOV # (z-stack offsets and the recording frame shape don't change mid-run). self._zstack_offsets: List[float] = ( @@ -450,15 +438,6 @@ def zstack(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> None: self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) # --------------------------------------------------------------- helpers - def _wait_for_outstanding_callback_images(self) -> None: - """Block until any in-flight triggered frame has been dispatched/processed.""" - if not self._ready_for_next_trigger.wait(self._frame_wait_timeout_s()): - log.warning("Timed out waiting for outstanding z-stack frames") - if not self._image_callback_idle.wait(self._frame_wait_timeout_s()): - log.warning("Timed out waiting for last z-stack image to process") - self._ready_for_next_trigger.set() - self._image_callback_idle.set() - def _move_xy(self, coord) -> None: self.stage.move_x_to(coord[0]) self._sleep(SCAN_STABILIZATION_TIME_MS_X / 1000) From 82d8166a6b9045ddcafcf4300c3b8ea4be952946 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 13:12:19 -0700 Subject: [PATCH 27/61] fix: Wire signal_acquisition_started for Record+Z-Stack UI lockout (fix-batch5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add signal_acquisition_started(bool) to RecordZStackMultiPointWidget, emitting True on valid start and False in acquisition_is_finished(), mirroring the pattern used by WellplateMultiPointWidget and FlexibleMultiPointWidget. - Connect the signal to gui_hcs.toggleAcquisitionStart inside the ENABLE_RECORDING guard so other tabs, click-to-move, and live-scan-grid are locked during acquisition. - Update toggle_acquisition docstring to reflect build_parameters() call path. - Add comment in record_zstack_controller explaining the abort_event clear→start window. - Add dropped_count property to RecordingWriter and a summary WARNING log in StreamingCapture.run so slow-disk dropped frames are diagnosable without grepping. - Add 5 new tests covering the signal and dropped-count behaviour. Co-Authored-By: Claude Sonnet 4.6 --- .../control/core/record_zstack_controller.py | 3 + software/control/core/streaming_capture.py | 12 +++ software/control/gui_hcs.py | 3 + software/control/widgets.py | 8 +- software/tests/core/test_streaming_capture.py | 73 +++++++++++++++++++ software/tests/test_record_zstack_widget.py | 57 +++++++++++++++ 6 files changed, 154 insertions(+), 2 deletions(-) diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py index b865a4310..8c7711673 100644 --- a/software/control/core/record_zstack_controller.py +++ b/software/control/core/record_zstack_controller.py @@ -231,6 +231,9 @@ def run_acquisition(self, params: Optional[RecordZStackAcquisitionParameters] = scan_region_fov_coords = dict(self._scan_coordinates.region_fov_coordinates) # Clear abort event for this run (thread-safe: Event.clear() is atomic). + # The small window between clear() and the worker thread starting is safe + # under the single-acquisition-at-a-time assumption enforced by the widget + # (toggle_acquisition checks acquisition_in_progress() before calling here). self._abort_event.clear() # Consume the pre-warmed runner; only pass it to the worker when diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index e5be40da8..f76670ebd 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -119,6 +119,11 @@ def _drain(self) -> None: else: self._writer.finalize() + @property + def dropped_count(self) -> int: + """Total frames dropped due to a full queue (diagnosable in slow-disk runs).""" + return self._dropped + def finalize(self) -> None: """Flush the queue, join the drain thread (which finalizes the ZarrWriter).""" if not self._started: @@ -262,4 +267,11 @@ def run(self, timeout: Optional[float] = None) -> int: f"streaming capture incomplete: captured {self._emitted}/{expected} " f"frames; trailing planes are blank fill" ) + # Surface total dropped frames so slow-disk runs are diagnosable without + # grepping individual per-frame warnings. + dropped = self._writer.dropped_count if hasattr(self._writer, "dropped_count") else 0 + if dropped > 0: + _log.warning( + f"streaming capture finished: {dropped} frame(s) dropped total " f"(queue full / slow disk)" + ) return self._emitted diff --git a/software/control/gui_hcs.py b/software/control/gui_hcs.py index 5556b1c60..a1ed3776d 100644 --- a/software/control/gui_hcs.py +++ b/software/control/gui_hcs.py @@ -1547,6 +1547,9 @@ def make_connections(self): self.fluidicsWidget.fluidics_initialized_signal.connect(self.multiPointWithFluidicsWidget.init_fluidics) self.signal_performance_mode_changed.connect(self.multiPointWithFluidicsWidget.set_performance_mode) + if ENABLE_RECORDING: + self.recordZStackWidget.signal_acquisition_started.connect(self.toggleAcquisitionStart) + self.profileWidget.signal_profile_changed.connect(self.liveControlWidget.refresh_mode_list) self.liveControlWidget.signal_newExposureTime.connect(self.cameraSettingWidget.set_exposure_time) diff --git a/software/control/widgets.py b/software/control/widgets.py index 30355afa4..1addbfa73 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -16993,6 +16993,8 @@ class RecordZStackMultiPointWidget(QFrame): Inline channel-editor wiring, Copy-from-Live, and Start-button handoff are E2/E3. """ + signal_acquisition_started = Signal(bool) # True = started, False = finished + def __init__( self, stage, @@ -17596,8 +17598,8 @@ def toggle_acquisition(self, pressed: bool) -> None: On start (pressed=True): - validate(); show QMessageBox.warning and un-check button on failure. - - push all parameters to recordZStackController via its setters. - - call recordZStackController.run_acquisition(). + - call recordZStackController.run_acquisition(self.build_parameters()). + - emit signal_acquisition_started(True) so gui_hcs can lock down the UI. On stop (pressed=False): - call recordZStackController.request_abort(). @@ -17617,12 +17619,14 @@ def toggle_acquisition(self, pressed: bool) -> None: params = self.build_parameters() self.recordZStackController.run_acquisition(params) + self.signal_acquisition_started.emit(True) else: self.recordZStackController.request_abort() def acquisition_is_finished(self): """Called (thread-safe via Qt signal) when the acquisition worker finishes.""" self.btn_startAcquisition.setChecked(False) + self.signal_acquisition_started.emit(False) def display_progress_bar(self, show: bool) -> None: """No-op stub: RecordZStackMultiPointWidget has no progress bar. diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py index cbb7a1bf8..e9cc918cc 100644 --- a/software/tests/core/test_streaming_capture.py +++ b/software/tests/core/test_streaming_capture.py @@ -257,3 +257,76 @@ def test_streaming_capture_partial_warns(caplog): assert w.finalized is True assert w.aborted is False assert any("incomplete" in rec.getMessage() and "2/5" in rec.getMessage() for rec in caplog.records) + + +# --------------------------------------------------------------------------- +# Fix-batch5: dropped_count accessor + summary log +# --------------------------------------------------------------------------- + + +def test_recording_writer_dropped_count_accessor(tmp_path): + """dropped_count property returns the number of frames dropped due to a full queue.""" + from control.core.zarr_writer import ZarrAcquisitionConfig + from control.core.streaming_capture import RecordingWriter + + cfg = ZarrAcquisitionConfig( + output_path=str(tmp_path / "rec.ome.zarr"), + shape=(4, 1, 1, 4, 4), + dtype=np.uint16, + pixel_size_um=1.0, + z_step_um=None, + time_increment_s=0.1, + channel_names=["BF"], + channel_colors=["#FFFFFF"], + channel_wavelengths=[None], + is_hcs=False, + ) + # Use a queue of size 1 so extra enqueues are dropped immediately. + w = RecordingWriter(cfg, max_queue=1) + assert w.dropped_count == 0 + w.start() + # Fill the queue slot with first frame, then overflow with two more. + frame = np.zeros((4, 4), dtype=np.uint16) + w.enqueue(frame, 0, 0, 0) + w.enqueue(frame, 1, 0, 0) # may or may not drop depending on drain speed + w.enqueue(frame, 2, 0, 0) # likely dropped + w.finalize() + # At least one drop should have occurred; exact count is timing-dependent. + # Just verify the property exists and returns an int. + assert isinstance(w.dropped_count, int) + + +def test_streaming_capture_logs_dropped_summary(caplog): + """When frames are dropped, StreamingCapture logs a summary WARNING at end.""" + import logging + + class _DroppingWriter: + """Writer that pretends to drop every frame (dropped_count always > 0).""" + + dropped_count = 3 + + def start(self): + pass + + def enqueue(self, frame, t, c, z): + pass + + def finalize(self): + pass + + def abort(self): + pass + + frames = [_FakeFrame(100.0 + i, np.zeros((4, 4), np.uint16)) for i in range(3)] + w = _DroppingWriter() + cap = StreamingCapture( + _CountingFakeSource(frames), + RecordingRouter(fps=0.0), + CountStop(3), + w, + abort_fn=lambda: False, + ) + with caplog.at_level(logging.WARNING): + cap.run() + + assert any("dropped" in rec.getMessage() and "3" in rec.getMessage() for rec in caplog.records) diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 557ca7b6b..41d459905 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -703,3 +703,60 @@ def test_abort_event_set_on_request_abort(): # Simulate run_acquisition clearing the event before spawning the worker ctrl._abort_event.clear() assert not ctrl._abort_event.is_set() + + +# --------------------------------------------------------------------------- +# Fix-batch5: signal_acquisition_started wiring +# --------------------------------------------------------------------------- + + +def test_signal_acquisition_started_emits_true_on_start(qtbot, simulated_widget_deps): + """signal_acquisition_started(True) is emitted when toggle_acquisition(True) succeeds.""" + ctrl = _make_stub_controller() + simulated_widget_deps["recordZStackController"] = ctrl + + w = _make_valid_widget(qtbot, simulated_widget_deps) + + emitted = [] + w.signal_acquisition_started.connect(emitted.append) + + w.toggle_acquisition(True) + + assert emitted == [True] + + +def test_signal_acquisition_started_not_emitted_on_invalid_start(qtbot, simulated_widget_deps): + """signal_acquisition_started must NOT emit when validation rejects the start.""" + ctrl = _make_stub_controller() + simulated_widget_deps["recordZStackController"] = ctrl + + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("/tmp/test") + w.checkbox_zstack.setChecked(False) + w.checkbox_recording.setChecked(False) + + emitted = [] + w.signal_acquisition_started.connect(emitted.append) + + with patch("control.widgets.QMessageBox.warning"): + w.toggle_acquisition(True) + + assert emitted == [] + + +def test_signal_acquisition_started_emits_false_on_finish(qtbot, simulated_widget_deps): + """signal_acquisition_started(False) is emitted when acquisition_is_finished() is called.""" + ctrl = _make_stub_controller() + simulated_widget_deps["recordZStackController"] = ctrl + + w = _make_valid_widget(qtbot, simulated_widget_deps) + + emitted = [] + w.signal_acquisition_started.connect(emitted.append) + + w.acquisition_is_finished() + + assert emitted == [False] From d4dc9d5a2fde381fd0c964ee6c55e1647124a457 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 13:16:57 -0700 Subject: [PATCH 28/61] fix: emit signal_acquisition_started(True) before spawning worker thread Code review found a race: toggle_acquisition emitted True AFTER run_acquisition() returned, but run_acquisition spawns a worker thread that on a fast/one-frame acquisition could finish (acquisition_is_finished -> emit False) before the GUI thread reached the emit(True) line. The GUI would then process False then True, permanently locking all tabs. Fix mirrors WellplateMultiPointWidget (which emits True via _set_ui_acquisition_running before run_acquisition): - Emit True before calling run_acquisition. - Wrap run_acquisition in try/except; on failure un-check the button and emit False to unlock the UI (the worker never started, so acquisition_finished will not fire). - Document the emit(False) paths in acquisition_is_finished docstring. - Add 2 tests: emit-ordering (True before run) and run-raises (True then False). Co-Authored-By: Claude Sonnet 4.6 --- software/control/widgets.py | 30 ++++++++++++++-- software/tests/test_record_zstack_widget.py | 39 +++++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index 1addbfa73..f4271463c 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17598,11 +17598,19 @@ def toggle_acquisition(self, pressed: bool) -> None: On start (pressed=True): - validate(); show QMessageBox.warning and un-check button on failure. - - call recordZStackController.run_acquisition(self.build_parameters()). - emit signal_acquisition_started(True) so gui_hcs can lock down the UI. + - call recordZStackController.run_acquisition(self.build_parameters()). On stop (pressed=False): - call recordZStackController.request_abort(). + + Ordering note: signal_acquisition_started(True) is emitted BEFORE + run_acquisition() spawns the worker thread. Otherwise a fast/one-frame + acquisition could finish (firing acquisition_is_finished -> emit(False)) + before this method reaches the emit(True) line, leaving the GUI to process + False then True and permanently locking all tabs. This mirrors + WellplateMultiPointWidget, which emits True (via _set_ui_acquisition_running) + before calling run_acquisition(). """ self._log.debug(f"RecordZStackMultiPointWidget.toggle_acquisition, {pressed=}") if pressed: @@ -17618,13 +17626,29 @@ def toggle_acquisition(self, pressed: bool) -> None: return params = self.build_parameters() - self.recordZStackController.run_acquisition(params) + # Lock the UI before the worker thread can possibly finish (see docstring). self.signal_acquisition_started.emit(True) + try: + self.recordZStackController.run_acquisition(params) + except Exception as e: + self._log.error(f"Failed to start acquisition: {e}", exc_info=True) + self.btn_startAcquisition.setChecked(False) + # Unlock the UI: the worker never started, so acquisition_is_finished + # will not fire to emit(False) for us. + self.signal_acquisition_started.emit(False) else: self.recordZStackController.request_abort() def acquisition_is_finished(self): - """Called (thread-safe via Qt signal) when the acquisition worker finishes.""" + """Called (thread-safe via Qt signal) when the acquisition worker finishes. + + Connected in gui_hcs to recordZStackController.acquisition_finished, so this + is the single place that emits signal_acquisition_started(False) on normal + completion AND on the stop-button/abort path (request_abort eventually drives + the worker to completion, which fires acquisition_finished -> here). The only + other emit(False) is the failure-to-start path in toggle_acquisition, where the + worker thread never launched so acquisition_finished will not fire. + """ self.btn_startAcquisition.setChecked(False) self.signal_acquisition_started.emit(False) diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 41d459905..dfeb84f22 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -760,3 +760,42 @@ def test_signal_acquisition_started_emits_false_on_finish(qtbot, simulated_widge w.acquisition_is_finished() assert emitted == [False] + + +def test_signal_acquisition_started_emitted_before_run_acquisition(qtbot, simulated_widget_deps): + """emit(True) must happen BEFORE run_acquisition() spawns the worker thread. + + Otherwise a fast/one-frame acquisition could finish (emit False) before this + widget reaches the emit(True) line, leaving the UI permanently locked. + """ + events = [] + + ctrl = _make_stub_controller() + ctrl.run_acquisition.side_effect = lambda *a, **k: events.append("run") + simulated_widget_deps["recordZStackController"] = ctrl + + w = _make_valid_widget(qtbot, simulated_widget_deps) + w.signal_acquisition_started.connect(lambda started: events.append(f"emit({started})")) + + w.toggle_acquisition(True) + + # The True emit must come first, then run_acquisition. + assert events == ["emit(True)", "run"] + + +def test_signal_acquisition_started_emits_false_when_run_raises(qtbot, simulated_widget_deps): + """If run_acquisition() raises, the UI is unlocked: emit(True) then emit(False).""" + ctrl = _make_stub_controller() + ctrl.run_acquisition.side_effect = RuntimeError("boom starting worker") + simulated_widget_deps["recordZStackController"] = ctrl + + w = _make_valid_widget(qtbot, simulated_widget_deps) + + emitted = [] + w.signal_acquisition_started.connect(emitted.append) + + w.toggle_acquisition(True) + + assert emitted == [True, False] + # Button must be un-checked after a failed start. + assert not w.btn_startAcquisition.isChecked() From df68321c785d3f83fd65819a33ece4a598f3eaf7 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 19:46:11 -0700 Subject: [PATCH 29/61] refactor: Remove setter shims and duplicate attrs from RecordZStackController The widget already uses the clean run_acquisition(params) path, so the 15 one-line setter methods, the corresponding duplicate per-field instance attributes, and the legacy params=None fallback are all dead code. Remove them and update the controller test (test_record_zstack_controller_smoke) to build RecordZStackAcquisitionParameters directly and pass it to run_acquisition(params), matching how the widget works. Co-Authored-By: Claude Sonnet 4.6 --- .../control/core/record_zstack_controller.py | 102 ++---------------- .../tests/core/test_record_zstack_worker.py | 44 ++++---- 2 files changed, 33 insertions(+), 113 deletions(-) diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py index 8c7711673..0c8ea86ed 100644 --- a/software/control/core/record_zstack_controller.py +++ b/software/control/core/record_zstack_controller.py @@ -47,12 +47,12 @@ class RecordZStackAcquisitionParameters: class RecordZStackController: - """Controller that builds params, sets up the pre-warmed JobRunner, and spawns - RecordZStackWorker on a daemon thread. + """Controller that sets up the pre-warmed JobRunner and spawns RecordZStackWorker + on a daemon thread. Constructor arguments mirror MultiPointController's signature where the concept - overlaps; widget setters are provided for every RecordZStackAcquisitionParameters - field so the GUI (or tests) can push values in before calling run_acquisition(). + overlaps. Call run_acquisition(params) with a fully-built + RecordZStackAcquisitionParameters object (as the widget does via build_parameters()). """ def __init__( @@ -71,23 +71,6 @@ def __init__( self._scan_coordinates = scan_coordinates self._callbacks = callbacks - # Mutable params pushed by the widget before run_acquisition(). - self.base_path: Optional[str] = None - self.experiment_id: str = "record_zstack" - self.Nt: int = 1 - self.dt_s: float = 0.0 - self.use_laser_af: bool = False - self.recording_enabled: bool = False - self.recording_channel: Optional[AcquisitionChannel] = None - self.fps: float = 10.0 - self.duration_s: float = 1.0 - self.recording_z_offset_um: float = 0.0 - self.zstack_enabled: bool = False - self.zstack_channels: List[AcquisitionChannel] = [] - self.z_min_um: float = -3.0 - self.z_max_um: float = 3.0 - self.z_step_um: float = 1.0 - self._abort_event: Event = Event() self._worker = None self._thread: Optional[Thread] = None @@ -134,53 +117,6 @@ def _cleanup_prewarmed_runner(self, runner, timeout_s: float = 1.0, context: str except Exception as e: log.error(f"Error shutting down pre-warmed runner {context}: {e}") - # ---------------------------------------------------------------------- widget setters - - def set_base_path(self, path: str) -> None: - self.base_path = path - - def set_experiment_id(self, experiment_id: str) -> None: - self.experiment_id = experiment_id - - def set_Nt(self, Nt: int) -> None: - self.Nt = Nt - - def set_dt_s(self, dt_s: float) -> None: - self.dt_s = dt_s - - def set_use_laser_af(self, flag: bool) -> None: - self.use_laser_af = flag - - def set_recording_enabled(self, flag: bool) -> None: - self.recording_enabled = flag - - def set_recording_channel(self, channel: Optional[AcquisitionChannel]) -> None: - self.recording_channel = channel - - def set_fps(self, fps: float) -> None: - self.fps = fps - - def set_duration_s(self, duration_s: float) -> None: - self.duration_s = duration_s - - def set_recording_z_offset_um(self, offset_um: float) -> None: - self.recording_z_offset_um = offset_um - - def set_zstack_enabled(self, flag: bool) -> None: - self.zstack_enabled = flag - - def set_zstack_channels(self, channels: List[AcquisitionChannel]) -> None: - self.zstack_channels = list(channels) - - def set_z_min_um(self, z_min_um: float) -> None: - self.z_min_um = z_min_um - - def set_z_max_um(self, z_max_um: float) -> None: - self.z_max_um = z_max_um - - def set_z_step_um(self, z_step_um: float) -> None: - self.z_step_um = z_step_um - # ---------------------------------------------------------------------- acquisition def acquisition_in_progress(self) -> bool: @@ -190,37 +126,15 @@ def request_abort(self) -> None: """Signal the running worker to stop after the current FOV.""" self._abort_event.set() - def run_acquisition(self, params: Optional[RecordZStackAcquisitionParameters] = None) -> None: - """Build params, set up the pre-warmed JobRunner, and spawn the worker thread. + def run_acquisition(self, params: RecordZStackAcquisitionParameters) -> None: + """Set up the pre-warmed JobRunner and spawn the worker thread. - If *params* is provided the acquisition uses those values directly; - otherwise the instance attributes set via the individual setters are used - (legacy path, kept for backwards compatibility with tests/scripts that - still call the setters before invoking run_acquisition()). + *params* must be a fully-built RecordZStackAcquisitionParameters (as produced + by the widget's build_parameters() method). """ from control.core.acquisition_setup import create_experiment_dir from control.core.record_zstack_worker import RecordZStackWorker - # Resolve params: use supplied object or build one from instance attrs. - if params is None: - params = RecordZStackAcquisitionParameters( - base_path=self.base_path, - experiment_id=self.experiment_id, - Nt=self.Nt, - dt_s=self.dt_s, - use_laser_af=self.use_laser_af, - recording_enabled=self.recording_enabled, - recording_channel=self.recording_channel, - fps=self.fps, - duration_s=self.duration_s, - recording_z_offset_um=self.recording_z_offset_um, - zstack_enabled=self.zstack_enabled, - zstack_channels=list(self.zstack_channels), - z_min_um=self.z_min_um, - z_max_um=self.z_max_um, - z_step_um=self.z_step_um, - ) - # Resolve and create a timestamped unique output directory. resolved_id, _dir = create_experiment_dir(params.base_path, params.experiment_id) params.experiment_id = resolved_id diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index 22438d514..8f60ac191 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -172,7 +172,11 @@ def test_record_zstack_controller_smoke(tmp_path): import control._def import tests.control.test_stubs as ts from control.core.multi_point_controller import NoOpCallbacks - from control.core.record_zstack_controller import RecordZStackController, frame_count + from control.core.record_zstack_controller import ( + RecordZStackAcquisitionParameters, + RecordZStackController, + frame_count, + ) from control.core.scan_coordinates import ScanCoordinates control._def.FILE_SAVING_OPTION = control._def.FileSavingOption.ZARR_V3 @@ -227,24 +231,26 @@ def test_record_zstack_controller_smoke(tmp_path): callbacks=NoOpCallbacks, ) - # Push params via setters (as a widget would). - controller.set_base_path(str(tmp_path)) - controller.set_experiment_id("ctrl_smoke") - controller.set_Nt(Nt) - controller.set_dt_s(0.1) # short inter-timepoint delay - controller.set_use_laser_af(False) - controller.set_recording_enabled(True) - controller.set_recording_channel(recording_channel) - controller.set_fps(fps) - controller.set_duration_s(duration_s) - controller.set_recording_z_offset_um(0.0) - controller.set_zstack_enabled(True) - controller.set_zstack_channels(zstack_channels) - controller.set_z_min_um(-1.0) - controller.set_z_max_um(1.0) - controller.set_z_step_um(1.0) # 3 planes - - controller.run_acquisition() + # Build params directly and pass to run_acquisition (same path as the widget). + params = RecordZStackAcquisitionParameters( + base_path=str(tmp_path), + experiment_id="ctrl_smoke", + Nt=Nt, + dt_s=0.1, # short inter-timepoint delay + use_laser_af=False, + recording_enabled=True, + recording_channel=recording_channel, + fps=fps, + duration_s=duration_s, + recording_z_offset_um=0.0, + zstack_enabled=True, + zstack_channels=zstack_channels, + z_min_um=-1.0, + z_max_um=1.0, + z_step_um=1.0, # 3 planes + ) + + controller.run_acquisition(params) # Wait up to 120 s for the worker thread to finish. controller.join(timeout=120) assert not controller.acquisition_in_progress(), "worker thread did not finish in time" From 2aacdb6de4b73f6918277445bab261bae3c2322d Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 21 Jun 2026 21:56:29 -0700 Subject: [PATCH 30/61] style(record-zstack): compact widget layout to match WellplateMultiPoint Co-Authored-By: Claude Sonnet 4.6 --- software/control/widgets.py | 105 ++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 41 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index f4271463c..b52f42758 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17031,15 +17031,15 @@ def __init__( def _add_components(self) -> None: outer_layout = QVBoxLayout(self) - outer_layout.setContentsMargins(4, 4, 4, 4) + outer_layout.setContentsMargins(4, 2, 4, 2) scroll = QScrollArea() scroll.setWidgetResizable(True) scroll.setFrameShape(QFrame.NoFrame) inner = QWidget() layout = QVBoxLayout(inner) - layout.setContentsMargins(4, 4, 4, 4) - layout.setSpacing(6) + layout.setContentsMargins(4, 2, 4, 2) + layout.setSpacing(4) layout.addWidget(self._build_output_group()) layout.addWidget(self._build_wells_fov_group()) @@ -17055,6 +17055,8 @@ def _add_components(self) -> None: def _build_output_group(self) -> QGroupBox: grp = QGroupBox("Output") layout = QGridLayout(grp) + layout.setContentsMargins(4, 2, 4, 2) + layout.setSpacing(4) layout.addWidget(QLabel("Base path:"), 0, 0) self.lineEdit_savingDir = QLineEdit() @@ -17076,44 +17078,53 @@ def _build_output_group(self) -> QGroupBox: def _build_wells_fov_group(self) -> QGroupBox: grp = QGroupBox("Wells & FOV Grid") - layout = QFormLayout(grp) + layout = QGridLayout(grp) + layout.setContentsMargins(4, 2, 4, 2) + layout.setSpacing(4) self.label_selected_wells = QLabel("0 well(s) selected") - layout.addRow("Wells:", self.label_selected_wells) + layout.addWidget(QLabel("Wells:"), 0, 0) + layout.addWidget(self.label_selected_wells, 0, 1) self.entry_overlap = QDoubleSpinBox() self.entry_overlap.setRange(-1000, 99) self.entry_overlap.setValue(10) self.entry_overlap.setSuffix("%") self.entry_overlap.setKeyboardTracking(False) - layout.addRow("FOV overlap:", self.entry_overlap) + layout.addWidget(QLabel("FOV overlap:"), 0, 2) + layout.addWidget(self.entry_overlap, 0, 3) self.combobox_shape = QComboBox() self.combobox_shape.addItems(["Square", "Circle", "Rectangle"]) - layout.addRow("Region shape:", self.combobox_shape) + layout.addWidget(QLabel("Region shape:"), 1, 0) + layout.addWidget(self.combobox_shape, 1, 1, 1, 3) return grp def _build_timelapse_focus_group(self) -> QGroupBox: grp = QGroupBox("Time-lapse & Focus") - layout = QFormLayout(grp) + layout = QGridLayout(grp) + layout.setContentsMargins(4, 2, 4, 2) + layout.setSpacing(4) self.entry_Nt = QSpinBox() self.entry_Nt.setMinimum(1) self.entry_Nt.setMaximum(5000) self.entry_Nt.setValue(1) - layout.addRow("Time points (Nt):", self.entry_Nt) + layout.addWidget(QLabel("Nt:"), 0, 0) + layout.addWidget(self.entry_Nt, 0, 1) self.entry_dt = QDoubleSpinBox() self.entry_dt.setRange(0, 24 * 3600) self.entry_dt.setValue(0) self.entry_dt.setSuffix(" s") self.entry_dt.setKeyboardTracking(False) - layout.addRow("Interval (dt):", self.entry_dt) + layout.addWidget(QLabel("dt:"), 0, 2) + layout.addWidget(self.entry_dt, 0, 3) self.checkbox_laser_af = QCheckBox("Use laser reflection AF") self.checkbox_laser_af.setChecked(False) - layout.addRow("", self.checkbox_laser_af) + layout.addWidget(self.checkbox_laser_af, 0, 4) return grp @@ -17123,37 +17134,38 @@ def _build_recording_group(self) -> QGroupBox: grp.setChecked(False) self.checkbox_recording = grp # expose as checkbox_recording for callers - layout = QFormLayout(grp) + layout = QGridLayout(grp) + layout.setContentsMargins(4, 2, 4, 2) + layout.setSpacing(4) - # Channel row with Copy-from-Live button - channel_row = QWidget() - channel_row_layout = QHBoxLayout(channel_row) - channel_row_layout.setContentsMargins(0, 0, 0, 0) + # Row 0: Channel combo + Copy-from-Live button self.combobox_recording_channel = QComboBox() self._populate_channel_combo(self.combobox_recording_channel) - channel_row_layout.addWidget(self.combobox_recording_channel, 1) + layout.addWidget(QLabel("Channel:"), 0, 0) + layout.addWidget(self.combobox_recording_channel, 0, 1, 1, 3) self.btn_copy_from_live = QPushButton("⟳ Live") self.btn_copy_from_live.setToolTip("Copy current live channel settings (channel, exposure, gain, illumination)") self.btn_copy_from_live.setMaximumWidth(70) self.btn_copy_from_live.clicked.connect(self._copy_recording_from_live) - channel_row_layout.addWidget(self.btn_copy_from_live) - layout.addRow("Channel:", channel_row) + layout.addWidget(self.btn_copy_from_live, 0, 4) - # Inline exposure/gain/illumination spinboxes + # Row 1: Exposure | Gain | Illumination on one row self.entry_recording_exposure = QDoubleSpinBox() self.entry_recording_exposure.setRange(0.01, 60000) self.entry_recording_exposure.setValue(50.0) self.entry_recording_exposure.setSuffix(" ms") self.entry_recording_exposure.setDecimals(1) self.entry_recording_exposure.setKeyboardTracking(False) - layout.addRow("Exposure:", self.entry_recording_exposure) + layout.addWidget(QLabel("Exp:"), 1, 0) + layout.addWidget(self.entry_recording_exposure, 1, 1) self.entry_recording_gain = QDoubleSpinBox() self.entry_recording_gain.setRange(0.0, 100.0) self.entry_recording_gain.setValue(0.0) self.entry_recording_gain.setDecimals(2) self.entry_recording_gain.setKeyboardTracking(False) - layout.addRow("Gain:", self.entry_recording_gain) + layout.addWidget(QLabel("Gain:"), 1, 2) + layout.addWidget(self.entry_recording_gain, 1, 3) self.entry_recording_illumination = QDoubleSpinBox() self.entry_recording_illumination.setRange(0.0, 100.0) @@ -17161,31 +17173,37 @@ def _build_recording_group(self) -> QGroupBox: self.entry_recording_illumination.setSuffix(" %") self.entry_recording_illumination.setDecimals(1) self.entry_recording_illumination.setKeyboardTracking(False) - layout.addRow("Illumination:", self.entry_recording_illumination) + layout.addWidget(QLabel("Illum:"), 1, 4) + layout.addWidget(self.entry_recording_illumination, 1, 5) + # Row 2: FPS | Duration | Computed frames on one row self.entry_fps = QDoubleSpinBox() self.entry_fps.setRange(0.1, 1000) self.entry_fps.setValue(10.0) self.entry_fps.setSuffix(" fps") self.entry_fps.setKeyboardTracking(False) - layout.addRow("FPS:", self.entry_fps) + layout.addWidget(QLabel("FPS:"), 2, 0) + layout.addWidget(self.entry_fps, 2, 1) self.entry_duration = QDoubleSpinBox() self.entry_duration.setRange(0.01, 3600) self.entry_duration.setValue(1.0) self.entry_duration.setSuffix(" s") self.entry_duration.setKeyboardTracking(False) - layout.addRow("Duration:", self.entry_duration) + layout.addWidget(QLabel("Dur:"), 2, 2) + layout.addWidget(self.entry_duration, 2, 3) self.label_recording_frames = QLabel("-- frames") - layout.addRow("Computed:", self.label_recording_frames) + layout.addWidget(self.label_recording_frames, 2, 4, 1, 2) + # Row 3: Z-offset self.entry_recording_z_offset = QDoubleSpinBox() self.entry_recording_z_offset.setRange(-1000, 1000) self.entry_recording_z_offset.setValue(0.0) self.entry_recording_z_offset.setSuffix(" μm") self.entry_recording_z_offset.setKeyboardTracking(False) - layout.addRow("Z-offset:", self.entry_recording_z_offset) + layout.addWidget(QLabel("Z-offset:"), 3, 0) + layout.addWidget(self.entry_recording_z_offset, 3, 1, 1, 2) # Wire up live frame count update self.entry_fps.valueChanged.connect(self._update_recording_frames_label) @@ -17200,8 +17218,11 @@ def _build_zstack_group(self) -> QGroupBox: grp.setChecked(False) self.checkbox_zstack = grp # expose as checkbox_zstack for callers - layout = QFormLayout(grp) + layout = QGridLayout(grp) + layout.setContentsMargins(4, 2, 4, 2) + layout.setSpacing(4) + # Row 0: Z-min | Z-max | Step | Computed planes on one row self.entry_zmin = QDoubleSpinBox() self.entry_zmin.setRange(-5000, 5000) self.entry_zmin.setValue(-3.0) @@ -17209,7 +17230,8 @@ def _build_zstack_group(self) -> QGroupBox: self.entry_zmin.setDecimals(3) self.entry_zmin.setSingleStep(0.5) self.entry_zmin.setKeyboardTracking(False) - layout.addRow("Z-min:", self.entry_zmin) + layout.addWidget(QLabel("Z-min:"), 0, 0) + layout.addWidget(self.entry_zmin, 0, 1) self.entry_zmax = QDoubleSpinBox() self.entry_zmax.setRange(-5000, 5000) @@ -17218,7 +17240,8 @@ def _build_zstack_group(self) -> QGroupBox: self.entry_zmax.setDecimals(3) self.entry_zmax.setSingleStep(0.5) self.entry_zmax.setKeyboardTracking(False) - layout.addRow("Z-max:", self.entry_zmax) + layout.addWidget(QLabel("Z-max:"), 0, 2) + layout.addWidget(self.entry_zmax, 0, 3) self.entry_step = QDoubleSpinBox() self.entry_step.setRange(0.001, 1000) @@ -17227,12 +17250,13 @@ def _build_zstack_group(self) -> QGroupBox: self.entry_step.setDecimals(3) self.entry_step.setSingleStep(0.1) self.entry_step.setKeyboardTracking(False) - layout.addRow("Step:", self.entry_step) + layout.addWidget(QLabel("Step:"), 0, 4) + layout.addWidget(self.entry_step, 0, 5) self.label_zstack_planes = QLabel("-- planes") - layout.addRow("Computed:", self.label_zstack_planes) + layout.addWidget(self.label_zstack_planes, 0, 6) - # Z-stack channel table (rows added via _add_zstack_channel_row) + # Row 1: Z-stack channel table (rows added via _add_zstack_channel_row) # Columns: Channel | Exposure (ms) | Gain | Illumination (%) | Actions self.zstack_channel_table = QTableWidget(0, 5) self.zstack_channel_table.setHorizontalHeaderLabels(["Channel", "Exp (ms)", "Gain", "Illum (%)", ""]) @@ -17245,20 +17269,17 @@ def _build_zstack_group(self) -> QGroupBox: self.zstack_channel_table.setSelectionBehavior(QAbstractItemView.SelectRows) self.zstack_channel_table.setMinimumHeight(80) self.zstack_channel_table.setMaximumHeight(200) - layout.addRow("Channels:", self.zstack_channel_table) + layout.addWidget(self.zstack_channel_table, 1, 0, 1, 7) - # "+ Add channel" row below the table - add_ch_row = QWidget() - add_ch_row_layout = QHBoxLayout(add_ch_row) - add_ch_row_layout.setContentsMargins(0, 0, 0, 0) + # Row 2: Add channel dropdown + button self.combobox_zstack_add_channel = QComboBox() self._populate_channel_combo(self.combobox_zstack_add_channel) - add_ch_row_layout.addWidget(self.combobox_zstack_add_channel, 1) + layout.addWidget(QLabel("Add:"), 2, 0) + layout.addWidget(self.combobox_zstack_add_channel, 2, 1, 1, 5) self.btn_zstack_add_channel = QPushButton("+ Add") self.btn_zstack_add_channel.setMaximumWidth(60) self.btn_zstack_add_channel.clicked.connect(self._on_zstack_add_channel_clicked) - add_ch_row_layout.addWidget(self.btn_zstack_add_channel) - layout.addRow("Add channel:", add_ch_row) + layout.addWidget(self.btn_zstack_add_channel, 2, 6) # Wire up live plane count update self.entry_zmin.valueChanged.connect(self._update_zstack_planes_label) @@ -17271,6 +17292,8 @@ def _build_zstack_group(self) -> QGroupBox: def _build_start_group(self) -> QGroupBox: grp = QGroupBox() layout = QVBoxLayout(grp) + layout.setContentsMargins(4, 2, 4, 2) + layout.setSpacing(4) self.btn_startAcquisition = QPushButton("Start Acquisition") self.btn_startAcquisition.setStyleSheet("background-color: #C2C2FF") self.btn_startAcquisition.setCheckable(True) From 407f1d5f325e47af39731e631ea470bc0fdaa2f4 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 28 Jun 2026 00:41:58 -0700 Subject: [PATCH 31/61] feat(record-zstack): wire FOV grid and compact the acquisition widget Wire the FOV grid controls so they drive the scan: overlap, region shape, and region size now call scanCoordinates.set_well_coordinates() on change and at acquisition start. Compact RecordZStackMultiPointWidget to match WellplateMultiPoint (715px -> 552px): single-column layout; recording channel as a single-row table aligned with the z-stack table (both 530px); one-decimal z-range fields; 'Laser AF' checkbox merged onto the Nt/dt row; left-aligned labels with the Experiment ID field stretched to fill the row. Fix channel-name and fps-suffix truncation in the channel tables. Add FOV-grid wiring test coverage (50 widget tests passing). Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/widgets.py | 409 ++++++++++++++------ software/tests/test_record_zstack_widget.py | 176 +++++++-- 2 files changed, 432 insertions(+), 153 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index b52f42758..ac22306b2 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17043,7 +17043,6 @@ def _add_components(self) -> None: layout.addWidget(self._build_output_group()) layout.addWidget(self._build_wells_fov_group()) - layout.addWidget(self._build_timelapse_focus_group()) layout.addWidget(self._build_recording_group()) layout.addWidget(self._build_zstack_group()) layout.addWidget(self._build_start_group()) @@ -17053,78 +17052,146 @@ def _add_components(self) -> None: outer_layout.addWidget(scroll) def _build_output_group(self) -> QGroupBox: - grp = QGroupBox("Output") - layout = QGridLayout(grp) - layout.setContentsMargins(4, 2, 4, 2) - layout.setSpacing(4) - - layout.addWidget(QLabel("Base path:"), 0, 0) + grp = QGroupBox() + grp.setFlat(True) + vbox = QVBoxLayout(grp) + vbox.setContentsMargins(4, 2, 4, 2) + vbox.setSpacing(6) + + # Fixed-width label column so both input fields start at the same x + # (sized for the wider "Experiment ID:" label). + label_col_w = 112 + + # Row 1: Base path lineedit + Browse button + row1 = QHBoxLayout() + row1.setSpacing(6) + base_label = QLabel("Base path:") + base_label.setFixedWidth(label_col_w) + base_label.setAlignment(Qt.AlignLeft | Qt.AlignVCenter) + row1.addWidget(base_label) self.lineEdit_savingDir = QLineEdit() last_path = get_last_used_saving_path() self.lineEdit_savingDir.setText(last_path) - layout.addWidget(self.lineEdit_savingDir, 0, 1) - + row1.addWidget(self.lineEdit_savingDir, 1) # stretch=1 → fills available space self.btn_setSavingDir = QPushButton("Browse") self.btn_setSavingDir.setIcon(QIcon("icon/folder.png")) self.btn_setSavingDir.clicked.connect(self._browse_saving_dir) - layout.addWidget(self.btn_setSavingDir, 0, 2) - - layout.addWidget(QLabel("Experiment ID:"), 1, 0) + row1.addWidget(self.btn_setSavingDir) + vbox.addLayout(row1) + + # Row 2: Experiment ID on its own row + row2 = QHBoxLayout() + row2.setSpacing(6) + exp_label = QLabel("Experiment ID:") + exp_label.setFixedWidth(label_col_w) + exp_label.setAlignment(Qt.AlignLeft | Qt.AlignVCenter) + row2.addWidget(exp_label) self.lineEdit_experimentID = QLineEdit() self.lineEdit_experimentID.setPlaceholderText("record_zstack") - layout.addWidget(self.lineEdit_experimentID, 1, 1, 1, 2) + row2.addWidget(self.lineEdit_experimentID, 1) # stretch=1 → fills to the right edge + vbox.addLayout(row2) return grp def _build_wells_fov_group(self) -> QGroupBox: - grp = QGroupBox("Wells & FOV Grid") - layout = QGridLayout(grp) - layout.setContentsMargins(4, 2, 4, 2) - layout.setSpacing(4) - - self.label_selected_wells = QLabel("0 well(s) selected") - layout.addWidget(QLabel("Wells:"), 0, 0) - layout.addWidget(self.label_selected_wells, 0, 1) - + grp = QGroupBox() + grp.setFlat(True) + vbox = QVBoxLayout(grp) + vbox.setContentsMargins(4, 2, 4, 2) + vbox.setSpacing(6) + + # Consistent widths so label+field pairs line up cleanly across rows. + field_w = 90 # spinbox / combo width + pair_gap = 14 # horizontal gap between adjacent label+field pairs + + def _pair_label(text: str) -> QLabel: + lbl = QLabel(text) + lbl.setAlignment(Qt.AlignLeft | Qt.AlignVCenter) + return lbl + + # First-column labels share a width so the first field of each row aligns + # (sized for the wider "Overlap:" label). + first_label_w = 66 + + # Row 1: FOV overlap + Region shape + Region size + row1 = QHBoxLayout() + row1.setSpacing(4) + + overlap_label = _pair_label("Overlap:") + overlap_label.setFixedWidth(first_label_w) + row1.addWidget(overlap_label) self.entry_overlap = QDoubleSpinBox() self.entry_overlap.setRange(-1000, 99) self.entry_overlap.setValue(10) - self.entry_overlap.setSuffix("%") + self.entry_overlap.setSuffix(" %") self.entry_overlap.setKeyboardTracking(False) - layout.addWidget(QLabel("FOV overlap:"), 0, 2) - layout.addWidget(self.entry_overlap, 0, 3) + self.entry_overlap.setFixedWidth(field_w) + row1.addWidget(self.entry_overlap) + row1.addSpacing(pair_gap) + row1.addWidget(_pair_label("Shape:")) self.combobox_shape = QComboBox() self.combobox_shape.addItems(["Square", "Circle", "Rectangle"]) - layout.addWidget(QLabel("Region shape:"), 1, 0) - layout.addWidget(self.combobox_shape, 1, 1, 1, 3) + self.combobox_shape.setFixedWidth(field_w) + row1.addWidget(self.combobox_shape) - return grp + row1.addSpacing(pair_gap) + row1.addWidget(_pair_label("Size:")) + self.entry_scan_size = QDoubleSpinBox() + self.entry_scan_size.setRange(0.1, 100) + self.entry_scan_size.setValue(0.1) + self.entry_scan_size.setSuffix(" mm") + self.entry_scan_size.setDecimals(2) + self.entry_scan_size.setKeyboardTracking(False) + self.entry_scan_size.setFixedWidth(field_w) + row1.addWidget(self.entry_scan_size) - def _build_timelapse_focus_group(self) -> QGroupBox: - grp = QGroupBox("Time-lapse & Focus") - layout = QGridLayout(grp) - layout.setContentsMargins(4, 2, 4, 2) - layout.setSpacing(4) + row1.addStretch(1) + vbox.addLayout(row1) + # Row 2: Nt + dt + row2 = QHBoxLayout() + row2.setSpacing(4) + + nt_label = _pair_label("Nt:") + nt_label.setFixedWidth(first_label_w) + row2.addWidget(nt_label) self.entry_Nt = QSpinBox() self.entry_Nt.setMinimum(1) self.entry_Nt.setMaximum(5000) self.entry_Nt.setValue(1) - layout.addWidget(QLabel("Nt:"), 0, 0) - layout.addWidget(self.entry_Nt, 0, 1) + self.entry_Nt.setFixedWidth(field_w) + row2.addWidget(self.entry_Nt) + row2.addSpacing(pair_gap) + row2.addWidget(_pair_label("dt:")) self.entry_dt = QDoubleSpinBox() self.entry_dt.setRange(0, 24 * 3600) self.entry_dt.setValue(0) self.entry_dt.setSuffix(" s") self.entry_dt.setKeyboardTracking(False) - layout.addWidget(QLabel("dt:"), 0, 2) - layout.addWidget(self.entry_dt, 0, 3) + self.entry_dt.setFixedWidth(field_w) + row2.addWidget(self.entry_dt) - self.checkbox_laser_af = QCheckBox("Use laser reflection AF") + # Laser reflection AF checkbox shares the Nt/dt row (room to the right of dt). + row2.addSpacing(pair_gap) + self.checkbox_laser_af = QCheckBox("Laser AF") self.checkbox_laser_af.setChecked(False) - layout.addWidget(self.checkbox_laser_af, 0, 4) + row2.addWidget(self.checkbox_laser_af) + + row2.addStretch(1) + vbox.addLayout(row2) + + # Separator below this section, before the Recording phase group. + sep = QFrame() + sep.setFrameShape(QFrame.HLine) + sep.setFrameShadow(QFrame.Sunken) + vbox.addWidget(sep) + + # Wire FOV-grid signals + self.entry_overlap.valueChanged.connect(self._update_scan_regions) + self.entry_scan_size.valueChanged.connect(self._update_scan_regions) + self.combobox_shape.currentIndexChanged.connect(self._update_scan_regions) return grp @@ -17134,81 +17201,120 @@ def _build_recording_group(self) -> QGroupBox: grp.setChecked(False) self.checkbox_recording = grp # expose as checkbox_recording for callers - layout = QGridLayout(grp) - layout.setContentsMargins(4, 2, 4, 2) - layout.setSpacing(4) + vbox = QVBoxLayout(grp) + vbox.setContentsMargins(4, 2, 4, 2) + vbox.setSpacing(4) - # Row 0: Channel combo + Copy-from-Live button - self.combobox_recording_channel = QComboBox() - self._populate_channel_combo(self.combobox_recording_channel) - layout.addWidget(QLabel("Channel:"), 0, 0) - layout.addWidget(self.combobox_recording_channel, 0, 1, 1, 3) - self.btn_copy_from_live = QPushButton("⟳ Live") + # Row 0: single-row channel table (Channel | Exp (ms) | Gain | Illum (%) | ↻) + self.recording_channel_table = QTableWidget(1, 5) + self.recording_channel_table.setHorizontalHeaderLabels(["Channel", "Exp (ms)", "Gain", "Illum (%)", ""]) + hdr = self.recording_channel_table.horizontalHeader() + hdr.setSectionResizeMode(0, QHeaderView.Stretch) + hdr.setSectionResizeMode(1, QHeaderView.ResizeToContents) + hdr.setSectionResizeMode(2, QHeaderView.ResizeToContents) + hdr.setSectionResizeMode(3, QHeaderView.ResizeToContents) + hdr.setSectionResizeMode(4, QHeaderView.ResizeToContents) + self.recording_channel_table.verticalHeader().setVisible(False) + self.recording_channel_table.setSelectionBehavior(QAbstractItemView.SelectRows) + # Use a generous fixed height so the single data row and header are fully visible. + self.recording_channel_table.setFixedHeight( + self.recording_channel_table.horizontalHeader().height() + self.recording_channel_table.rowHeight(0) + 8 + ) + # Bound table width to match the z-stack table and force the 5 columns to + # fit (Channel truncates via Stretch) instead of showing a horizontal scrollbar. + self.recording_channel_table.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) + # Match the z-stack channel table width so the two tables align as one column. + self.recording_channel_table.setFixedWidth(530) + + # Col 0: channel combo. Let it shrink within the Stretch column instead of + # demanding its full text width (otherwise the long channel name forces the + # column wide and pushes Exp/Gain/Illum behind a horizontal scrollbar). + self._recording_ch_combo = QComboBox() + self._populate_channel_combo(self._recording_ch_combo) + self._recording_ch_combo.setSizeAdjustPolicy(QComboBox.AdjustToMinimumContentsLengthWithIcon) + self._recording_ch_combo.setMinimumContentsLength(6) + self._recording_ch_combo.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Fixed) + self.recording_channel_table.setCellWidget(0, 0, self._recording_ch_combo) + + # Col 1: exposure spinbox (cap width so the Channel column keeps room for full names) + self._recording_exp_spin = QDoubleSpinBox() + self._recording_exp_spin.setRange(0.01, 60000) + self._recording_exp_spin.setValue(50.0) + self._recording_exp_spin.setSuffix(" ms") + self._recording_exp_spin.setDecimals(1) + self._recording_exp_spin.setKeyboardTracking(False) + self._recording_exp_spin.setMaximumWidth(85) + self.recording_channel_table.setCellWidget(0, 1, self._recording_exp_spin) + + # Col 2: gain spinbox + self._recording_gain_spin = QDoubleSpinBox() + self._recording_gain_spin.setRange(0.0, 100.0) + self._recording_gain_spin.setValue(0.0) + self._recording_gain_spin.setDecimals(2) + self._recording_gain_spin.setKeyboardTracking(False) + self._recording_gain_spin.setMaximumWidth(60) + self.recording_channel_table.setCellWidget(0, 2, self._recording_gain_spin) + + # Col 3: illumination spinbox + self._recording_illum_spin = QDoubleSpinBox() + self._recording_illum_spin.setRange(0.0, 100.0) + self._recording_illum_spin.setValue(50.0) + self._recording_illum_spin.setSuffix(" %") + self._recording_illum_spin.setDecimals(1) + self._recording_illum_spin.setKeyboardTracking(False) + self._recording_illum_spin.setMaximumWidth(76) + self.recording_channel_table.setCellWidget(0, 3, self._recording_illum_spin) + + # Col 4: ↻ copy-from-live button + self.btn_copy_from_live = QPushButton("↻") self.btn_copy_from_live.setToolTip("Copy current live channel settings (channel, exposure, gain, illumination)") - self.btn_copy_from_live.setMaximumWidth(70) + self.btn_copy_from_live.setMaximumWidth(28) self.btn_copy_from_live.clicked.connect(self._copy_recording_from_live) - layout.addWidget(self.btn_copy_from_live, 0, 4) - - # Row 1: Exposure | Gain | Illumination on one row - self.entry_recording_exposure = QDoubleSpinBox() - self.entry_recording_exposure.setRange(0.01, 60000) - self.entry_recording_exposure.setValue(50.0) - self.entry_recording_exposure.setSuffix(" ms") - self.entry_recording_exposure.setDecimals(1) - self.entry_recording_exposure.setKeyboardTracking(False) - layout.addWidget(QLabel("Exp:"), 1, 0) - layout.addWidget(self.entry_recording_exposure, 1, 1) - - self.entry_recording_gain = QDoubleSpinBox() - self.entry_recording_gain.setRange(0.0, 100.0) - self.entry_recording_gain.setValue(0.0) - self.entry_recording_gain.setDecimals(2) - self.entry_recording_gain.setKeyboardTracking(False) - layout.addWidget(QLabel("Gain:"), 1, 2) - layout.addWidget(self.entry_recording_gain, 1, 3) - - self.entry_recording_illumination = QDoubleSpinBox() - self.entry_recording_illumination.setRange(0.0, 100.0) - self.entry_recording_illumination.setValue(50.0) - self.entry_recording_illumination.setSuffix(" %") - self.entry_recording_illumination.setDecimals(1) - self.entry_recording_illumination.setKeyboardTracking(False) - layout.addWidget(QLabel("Illum:"), 1, 4) - layout.addWidget(self.entry_recording_illumination, 1, 5) - - # Row 2: FPS | Duration | Computed frames on one row + self.recording_channel_table.setCellWidget(0, 4, self.btn_copy_from_live) + + # Bound the table width to match the z-stack table: wrap in an HBox + # with a trailing stretch so it doesn't expand to fill the panel. + table_row = QHBoxLayout() + table_row.setContentsMargins(0, 0, 0, 0) + table_row.addWidget(self.recording_channel_table) + table_row.addStretch(1) + vbox.addLayout(table_row) + + # Row 1: FPS | Duration | Z-offset + fps_row = QHBoxLayout() + fps_row.setSpacing(4) + + fps_row.addWidget(QLabel("FPS:")) self.entry_fps = QDoubleSpinBox() self.entry_fps.setRange(0.1, 1000) self.entry_fps.setValue(10.0) self.entry_fps.setSuffix(" fps") self.entry_fps.setKeyboardTracking(False) - layout.addWidget(QLabel("FPS:"), 2, 0) - layout.addWidget(self.entry_fps, 2, 1) + self.entry_fps.setMaximumWidth(100) + fps_row.addWidget(self.entry_fps) + fps_row.addSpacing(4) + fps_row.addWidget(QLabel("Dur:")) self.entry_duration = QDoubleSpinBox() self.entry_duration.setRange(0.01, 3600) self.entry_duration.setValue(1.0) self.entry_duration.setSuffix(" s") self.entry_duration.setKeyboardTracking(False) - layout.addWidget(QLabel("Dur:"), 2, 2) - layout.addWidget(self.entry_duration, 2, 3) - - self.label_recording_frames = QLabel("-- frames") - layout.addWidget(self.label_recording_frames, 2, 4, 1, 2) + self.entry_duration.setMaximumWidth(90) + fps_row.addWidget(self.entry_duration) - # Row 3: Z-offset + fps_row.addSpacing(4) + fps_row.addWidget(QLabel("Z-offset:")) self.entry_recording_z_offset = QDoubleSpinBox() self.entry_recording_z_offset.setRange(-1000, 1000) self.entry_recording_z_offset.setValue(0.0) self.entry_recording_z_offset.setSuffix(" μm") self.entry_recording_z_offset.setKeyboardTracking(False) - layout.addWidget(QLabel("Z-offset:"), 3, 0) - layout.addWidget(self.entry_recording_z_offset, 3, 1, 1, 2) + self.entry_recording_z_offset.setMaximumWidth(95) + fps_row.addWidget(self.entry_recording_z_offset) - # Wire up live frame count update - self.entry_fps.valueChanged.connect(self._update_recording_frames_label) - self.entry_duration.valueChanged.connect(self._update_recording_frames_label) - self._update_recording_frames_label() + fps_row.addStretch(1) + vbox.addLayout(fps_row) return grp @@ -17227,9 +17333,10 @@ def _build_zstack_group(self) -> QGroupBox: self.entry_zmin.setRange(-5000, 5000) self.entry_zmin.setValue(-3.0) self.entry_zmin.setSuffix(" μm") - self.entry_zmin.setDecimals(3) + self.entry_zmin.setDecimals(1) self.entry_zmin.setSingleStep(0.5) self.entry_zmin.setKeyboardTracking(False) + self.entry_zmin.setMaximumWidth(85) layout.addWidget(QLabel("Z-min:"), 0, 0) layout.addWidget(self.entry_zmin, 0, 1) @@ -17237,9 +17344,10 @@ def _build_zstack_group(self) -> QGroupBox: self.entry_zmax.setRange(-5000, 5000) self.entry_zmax.setValue(3.0) self.entry_zmax.setSuffix(" μm") - self.entry_zmax.setDecimals(3) + self.entry_zmax.setDecimals(1) self.entry_zmax.setSingleStep(0.5) self.entry_zmax.setKeyboardTracking(False) + self.entry_zmax.setMaximumWidth(85) layout.addWidget(QLabel("Z-max:"), 0, 2) layout.addWidget(self.entry_zmax, 0, 3) @@ -17247,14 +17355,17 @@ def _build_zstack_group(self) -> QGroupBox: self.entry_step.setRange(0.001, 1000) self.entry_step.setValue(1.0) self.entry_step.setSuffix(" μm") - self.entry_step.setDecimals(3) + self.entry_step.setDecimals(1) self.entry_step.setSingleStep(0.1) self.entry_step.setKeyboardTracking(False) + self.entry_step.setMaximumWidth(85) layout.addWidget(QLabel("Step:"), 0, 4) layout.addWidget(self.entry_step, 0, 5) self.label_zstack_planes = QLabel("-- planes") layout.addWidget(self.label_zstack_planes, 0, 6) + # Stretch column so the row left-packs + layout.setColumnStretch(7, 1) # Row 1: Z-stack channel table (rows added via _add_zstack_channel_row) # Columns: Channel | Exposure (ms) | Gain | Illumination (%) | Actions @@ -17269,17 +17380,27 @@ def _build_zstack_group(self) -> QGroupBox: self.zstack_channel_table.setSelectionBehavior(QAbstractItemView.SelectRows) self.zstack_channel_table.setMinimumHeight(80) self.zstack_channel_table.setMaximumHeight(200) + # Match the recording channel table width so the two tables align as one column. + self.zstack_channel_table.setFixedWidth(530) + self.zstack_channel_table.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) + self.zstack_channel_table.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) layout.addWidget(self.zstack_channel_table, 1, 0, 1, 7) - # Row 2: Add channel dropdown + button + # Row 2: Add channel dropdown + button (capped combo with "+ Add" right after, + # then a trailing stretch so the row doesn't span the full panel width) + add_row = QHBoxLayout() + add_row.setContentsMargins(0, 0, 0, 0) + add_row.setSpacing(4) self.combobox_zstack_add_channel = QComboBox() self._populate_channel_combo(self.combobox_zstack_add_channel) - layout.addWidget(QLabel("Add:"), 2, 0) - layout.addWidget(self.combobox_zstack_add_channel, 2, 1, 1, 5) + self.combobox_zstack_add_channel.setMaximumWidth(260) + add_row.addWidget(self.combobox_zstack_add_channel) self.btn_zstack_add_channel = QPushButton("+ Add") self.btn_zstack_add_channel.setMaximumWidth(60) self.btn_zstack_add_channel.clicked.connect(self._on_zstack_add_channel_clicked) - layout.addWidget(self.btn_zstack_add_channel, 2, 6) + add_row.addWidget(self.btn_zstack_add_channel) + add_row.addStretch(1) + layout.addLayout(add_row, 2, 0, 1, 7) # Wire up live plane count update self.entry_zmin.valueChanged.connect(self._update_zstack_planes_label) @@ -17314,20 +17435,32 @@ def _populate_channel_combo(self, combo: QComboBox) -> None: except Exception as exc: self._log.warning(f"Could not populate channel combo: {exc}") + # ---------------------------------------------------------------------- recording table accessors + + def _recording_channel_name(self) -> Optional[str]: + """Return the selected channel name from the recording table row, or None if empty.""" + combo = self.recording_channel_table.cellWidget(0, 0) + if combo is None or combo.count() == 0: + return None + return combo.currentText() or None + + def _recording_exposure(self) -> float: + spin = self.recording_channel_table.cellWidget(0, 1) + return spin.value() if spin is not None else 50.0 + + def _recording_gain(self) -> float: + spin = self.recording_channel_table.cellWidget(0, 2) + return spin.value() if spin is not None else 0.0 + + def _recording_illumination(self) -> float: + spin = self.recording_channel_table.cellWidget(0, 3) + return spin.value() if spin is not None else 50.0 + def _browse_saving_dir(self) -> None: path = QFileDialog.getExistingDirectory(self, "Select Saving Directory", self.lineEdit_savingDir.text()) if path: self.lineEdit_savingDir.setText(path) - def _update_recording_frames_label(self) -> None: - from control.core.record_zstack_controller import frame_count - - try: - n = frame_count(self.entry_fps.value(), self.entry_duration.value()) - self.label_recording_frames.setText(f"{n} frames") - except Exception: - self.label_recording_frames.setText("-- frames") - def _update_zstack_planes_label(self) -> None: from control.core.record_zstack_controller import zstack_plane_count @@ -17364,6 +17497,7 @@ def _add_zstack_channel_row( exp_spin.setSuffix(" ms") exp_spin.setDecimals(1) exp_spin.setKeyboardTracking(False) + exp_spin.setMaximumWidth(85) self.zstack_channel_table.setCellWidget(row, 1, exp_spin) # Col 2: gain spinbox @@ -17372,6 +17506,7 @@ def _add_zstack_channel_row( gain_spin.setValue(gain) gain_spin.setDecimals(2) gain_spin.setKeyboardTracking(False) + gain_spin.setMaximumWidth(60) self.zstack_channel_table.setCellWidget(row, 2, gain_spin) # Col 3: illumination spinbox @@ -17381,6 +17516,7 @@ def _add_zstack_channel_row( illum_spin.setSuffix(" %") illum_spin.setDecimals(1) illum_spin.setKeyboardTracking(False) + illum_spin.setMaximumWidth(76) self.zstack_channel_table.setCellWidget(row, 3, illum_spin) # Col 4: action buttons (⟳ Live + ✕) @@ -17462,19 +17598,19 @@ def _get_zstack_row_values(self, name: str): return 50.0, 0.0, 50.0 def _copy_recording_from_live(self) -> None: - """Copy current live channel settings into the recording inline editors.""" + """Copy current live channel settings into the recording table row.""" try: live_ch = self.liveController.currentConfiguration if live_ch is None: return - # Update channel dropdown - idx = self.combobox_recording_channel.findText(live_ch.name) - if idx >= 0: - self.combobox_recording_channel.setCurrentIndex(idx) - # Update inline spinboxes - self.entry_recording_exposure.setValue(live_ch.exposure_time) - self.entry_recording_gain.setValue(live_ch.analog_gain) - self.entry_recording_illumination.setValue(live_ch.illumination_intensity) + combo = self.recording_channel_table.cellWidget(0, 0) + if combo is not None: + idx = combo.findText(live_ch.name) + if idx >= 0: + combo.setCurrentIndex(idx) + self._recording_exp_spin.setValue(live_ch.exposure_time) + self._recording_gain_spin.setValue(live_ch.analog_gain) + self._recording_illum_spin.setValue(live_ch.illumination_intensity) except Exception as exc: self._log.warning(f"Copy-from-Live failed: {exc}") @@ -17517,6 +17653,23 @@ def _get_selected_well_count(self) -> int: # No scanCoordinates attached: treat as single-position (glass-slide-like). return 1 + def _update_scan_regions(self) -> None: + """Update the FOV grid from the current overlap/shape/scan-size settings. + + Mirrors WellplateMultiPointWidget.update_coordinates. Called whenever + entry_overlap, entry_scan_size, or combobox_shape changes, and also at + the start of toggle_acquisition to ensure the grid is current. + """ + if self.scanCoordinates is None: + return + scan_size_mm = self.entry_scan_size.value() + overlap_percent = self.entry_overlap.value() + shape = self.combobox_shape.currentText() + try: + self.scanCoordinates.set_well_coordinates(scan_size_mm, overlap_percent, shape) + except Exception as exc: + self._log.warning(f"_update_scan_regions: set_well_coordinates failed: {exc}") + def _laser_af_has_reference(self) -> bool: """Return True if the laser autofocus controller has a captured reference.""" ctrl = self.laser_autofocus_controller @@ -17538,9 +17691,7 @@ def validate(self) -> Optional[str]: recording_enabled=self.checkbox_recording.isChecked(), fps=self.entry_fps.value(), duration_s=self.entry_duration.value(), - recording_channel_name=( - self.combobox_recording_channel.currentText() if self.combobox_recording_channel.count() > 0 else None - ), + recording_channel_name=self._recording_channel_name(), zstack_enabled=self.checkbox_zstack.isChecked(), z_min=self.entry_zmin.value(), z_max=self.entry_zmax.value(), @@ -17576,15 +17727,15 @@ def _make_channel_base(name: str) -> AcquisitionChannel: illumination_settings=IlluminationSettings(intensity=50.0), ) - # Build recording channel from inline editors + # Build recording channel from the single-row recording table recording_channel = None - if self.combobox_recording_channel.count() > 0 and self.checkbox_recording.isChecked(): - rec_name = self.combobox_recording_channel.currentText() + if self.checkbox_recording.isChecked(): + rec_name = self._recording_channel_name() if rec_name: ch = _make_channel_base(rec_name) - ch.exposure_time = self.entry_recording_exposure.value() - ch.analog_gain = self.entry_recording_gain.value() - ch.illumination_intensity = self.entry_recording_illumination.value() + ch.exposure_time = self._recording_exposure() + ch.analog_gain = self._recording_gain() + ch.illumination_intensity = self._recording_illumination() recording_channel = ch # Build z-stack channels from per-row inline editors @@ -17648,6 +17799,10 @@ def toggle_acquisition(self, pressed: bool) -> None: QMessageBox.warning(self, "Invalid Parameters", error) return + # Refresh the per-well FOV grid before building parameters so the + # scan regions reflect the current overlap/shape/region-size settings. + self._update_scan_regions() + params = self.build_parameters() # Lock the UI before the worker thread can possibly finish (see docstring). self.signal_acquisition_started.emit(True) diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index dfeb84f22..68efa72c9 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -305,7 +305,7 @@ def _make_live_channel(name: str, exposure: float, gain: float, intensity: float def test_copy_from_live_populates_recording_fields(qtbot, simulated_widget_deps): - """Copy-from-Live reads currentConfiguration and sets recording fields.""" + """Copy-from-Live reads currentConfiguration and sets the recording table row.""" from control.widgets import RecordZStackMultiPointWidget live_ch = _make_live_channel("Fluorescence 488 nm Ex", exposure=33.0, gain=2.5, intensity=75.0) @@ -318,12 +318,12 @@ def test_copy_from_live_populates_recording_fields(qtbot, simulated_widget_deps) w.checkbox_recording.setChecked(True) w.btn_copy_from_live.click() - # Channel dropdown should be updated to the live channel name - assert w.combobox_recording_channel.currentText() == "Fluorescence 488 nm Ex" - # Exposure, gain, intensity spinboxes should reflect live channel values - assert w.entry_recording_exposure.value() == pytest.approx(33.0) - assert w.entry_recording_gain.value() == pytest.approx(2.5) - assert w.entry_recording_illumination.value() == pytest.approx(75.0) + # Channel combo in table row should be updated to the live channel name + assert w._recording_channel_name() == "Fluorescence 488 nm Ex" + # Spinboxes in table row should reflect live channel values + assert w._recording_exposure() == pytest.approx(33.0) + assert w._recording_gain() == pytest.approx(2.5) + assert w._recording_illumination() == pytest.approx(75.0) def test_add_remove_zstack_channel_row_syncs_list_and_table(qtbot, simulated_widget_deps): @@ -351,20 +351,6 @@ def test_add_remove_zstack_channel_row_syncs_list_and_table(qtbot, simulated_wid assert "Fluorescence 488 nm Ex" in w._zstack_channel_names -def test_computed_frame_label_updates_on_spinbox_change(qtbot, simulated_widget_deps): - """label_recording_frames updates to '→ N frames' when fps/duration change.""" - from control.core.record_zstack_controller import frame_count - from control.widgets import RecordZStackMultiPointWidget - - w = RecordZStackMultiPointWidget(**simulated_widget_deps) - qtbot.addWidget(w) - - w.entry_fps.setValue(5.0) - w.entry_duration.setValue(4.0) - expected = frame_count(5.0, 4.0) # 20 - assert str(expected) in w.label_recording_frames.text() - - def test_computed_plane_label_updates_on_spinbox_change(qtbot, simulated_widget_deps): """label_zstack_planes updates to '→ N planes' when zmin/zmax/step change.""" from control.core.record_zstack_controller import zstack_plane_count @@ -395,7 +381,7 @@ def test_computed_plane_label_degrades_gracefully_on_invalid_range(qtbot, simula def test_build_parameters_uses_inline_editor_values(qtbot, simulated_widget_deps): - """build_parameters() reflects inline editor values, not just channel name lookup.""" + """build_parameters() reflects recording table row values, not just channel name lookup.""" from control.widgets import RecordZStackMultiPointWidget w = RecordZStackMultiPointWidget(**simulated_widget_deps) @@ -404,10 +390,10 @@ def test_build_parameters_uses_inline_editor_values(qtbot, simulated_widget_deps w.checkbox_recording.setChecked(True) w.checkbox_zstack.setChecked(False) - # Set inline editor values - w.entry_recording_exposure.setValue(99.0) - w.entry_recording_gain.setValue(1.5) - w.entry_recording_illumination.setValue(60.0) + # Set values via the table's spinbox cell widgets + w._recording_exp_spin.setValue(99.0) + w._recording_gain_spin.setValue(1.5) + w._recording_illum_spin.setValue(60.0) params = w.build_parameters() assert params.recording_channel is not None @@ -799,3 +785,141 @@ def test_signal_acquisition_started_emits_false_when_run_raises(qtbot, simulated assert emitted == [True, False] # Button must be un-checked after a failed start. assert not w.btn_startAcquisition.isChecked() + + +# --------------------------------------------------------------------------- +# FOV-grid wiring tests (entry_scan_size / entry_overlap / combobox_shape) +# --------------------------------------------------------------------------- + + +def test_entry_scan_size_exists(qtbot, simulated_widget_deps): + """entry_scan_size widget is created on the widget.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + assert hasattr(w, "entry_scan_size") + + +def test_fov_grid_wired_overlap_calls_set_well_coordinates(qtbot, simulated_widget_deps): + """Changing entry_overlap triggers scanCoordinates.set_well_coordinates with correct args.""" + from unittest.mock import MagicMock + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + sc.reset_mock() + w.entry_overlap.setValue(20.0) + + sc.set_well_coordinates.assert_called() + call_args = sc.set_well_coordinates.call_args + _scan_size_mm, overlap_pct, _shape = call_args.args + assert overlap_pct == pytest.approx(20.0) + + +def test_fov_grid_wired_scan_size_calls_set_well_coordinates(qtbot, simulated_widget_deps): + """Changing entry_scan_size triggers scanCoordinates.set_well_coordinates with correct args.""" + from unittest.mock import MagicMock + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + sc.reset_mock() + w.entry_scan_size.setValue(2.5) + + sc.set_well_coordinates.assert_called() + call_args = sc.set_well_coordinates.call_args + scan_size_mm, _overlap_pct, _shape = call_args.args + assert scan_size_mm == pytest.approx(2.5) + + +def test_fov_grid_wired_shape_calls_set_well_coordinates(qtbot, simulated_widget_deps): + """Changing combobox_shape triggers scanCoordinates.set_well_coordinates with correct shape.""" + from unittest.mock import MagicMock + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + sc.reset_mock() + w.combobox_shape.setCurrentText("Circle") + + sc.set_well_coordinates.assert_called() + call_args = sc.set_well_coordinates.call_args + _scan_size_mm, _overlap_pct, shape = call_args.args + assert shape == "Circle" + + +def test_fov_grid_wired_set_well_coordinates_receives_all_three_args(qtbot, simulated_widget_deps): + """_update_scan_regions passes scan_size_mm, overlap_percent, shape to set_well_coordinates.""" + from unittest.mock import MagicMock + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.entry_scan_size.setValue(1.5) + w.entry_overlap.setValue(15.0) + w.combobox_shape.setCurrentText("Rectangle") + + sc.reset_mock() + w._update_scan_regions() + + sc.set_well_coordinates.assert_called_once_with(pytest.approx(1.5), pytest.approx(15.0), "Rectangle") + + +def test_fov_grid_no_crash_without_scan_coordinates(qtbot, simulated_widget_deps): + """_update_scan_regions is a no-op when scanCoordinates is None.""" + from control.widgets import RecordZStackMultiPointWidget + + simulated_widget_deps["scanCoordinates"] = None + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + # Must not raise + w._update_scan_regions() + + +def test_toggle_acquisition_calls_update_scan_regions_before_run(qtbot, simulated_widget_deps): + """toggle_acquisition calls _update_scan_regions before run_acquisition.""" + from unittest.mock import MagicMock + + sc = MagicMock() + # Non-empty well selection so validate() passes (no modal warning dialog). + sc.get_selected_wells.return_value = {"A1": (0.0, 0.0)} + simulated_widget_deps["scanCoordinates"] = sc + + ctrl = _make_stub_controller() + simulated_widget_deps["recordZStackController"] = ctrl + + w = _make_valid_widget(qtbot, simulated_widget_deps) + w.scanCoordinates = sc + + sc.reset_mock() + ctrl.reset_mock() + ctrl.acquisition_in_progress.return_value = False + + call_order = [] + sc.set_well_coordinates.side_effect = lambda *a: call_order.append("set_well_coordinates") + ctrl.run_acquisition.side_effect = lambda *a: call_order.append("run_acquisition") + + w.toggle_acquisition(True) + + assert "set_well_coordinates" in call_order + assert "run_acquisition" in call_order + assert call_order.index("set_well_coordinates") < call_order.index("run_acquisition") From 3120d0142cc2079a9f8128741afa1250cd2532b6 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 28 Jun 2026 16:47:15 -0700 Subject: [PATCH 32/61] fix(record-zstack): release RecordingWriter if drain thread fails to start If ZarrWriter.initialize() succeeds but thread.start() then raises, the drain thread (the writer's sole owner after start()) never runs to close the writer, leaking the open ZarrWriter. abort() the writer on a failed start() before propagating. Adds an error-path test. Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/core/streaming_capture.py | 9 +++- software/tests/core/test_streaming_capture.py | 44 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index f76670ebd..10097c2fb 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -76,7 +76,14 @@ def start(self) -> None: cleanly no-ops the join and the original ``initialize()`` error propagates. """ self._writer.initialize() - self._thread.start() + try: + self._thread.start() + except Exception: + # initialize() already opened the writer, but the drain thread (its + # sole owner after start()) will never run to close it — release it + # here before propagating so we don't leak the ZarrWriter. + self._writer.abort() + raise self._started = True def enqueue(self, frame: np.ndarray, t: int, c: int, z: int) -> None: diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py index e9cc918cc..d8d36c634 100644 --- a/software/tests/core/test_streaming_capture.py +++ b/software/tests/core/test_streaming_capture.py @@ -150,6 +150,50 @@ def boom(self): w.abort() +def test_recording_writer_aborts_writer_when_thread_start_fails(tmp_path, monkeypatch): + """If initialize() succeeds but the drain thread fails to start, start() must + abort the already-opened ZarrWriter (no leak) and propagate the error.""" + import pytest + from control.core.zarr_writer import ZarrAcquisitionConfig, ZarrWriter + from control.core.streaming_capture import RecordingWriter + + cfg = ZarrAcquisitionConfig( + output_path=str(tmp_path / "rec.ome.zarr"), + shape=(2, 1, 1, 4, 4), + dtype=np.uint16, + pixel_size_um=1.0, + z_step_um=None, + time_increment_s=0.1, + channel_names=["BF"], + channel_colors=["#FFFFFF"], + channel_wavelengths=[None], + is_hcs=False, + ) + + # initialize() succeeds (opens the writer); the drain thread then fails to start. + monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) + aborted = {"called": False} + monkeypatch.setattr(ZarrWriter, "abort", lambda self: aborted.__setitem__("called", True)) + + w = RecordingWriter(cfg) + + class _BoomThread: + def start(self): + raise RuntimeError("boom from thread start") + + w._thread = _BoomThread() + + with pytest.raises(RuntimeError, match="boom from thread start"): + w.start() + + # The writer that initialize() opened must be released, not leaked. + assert aborted["called"] is True + assert w._started is False + # subsequent finalize()/abort() stay safe no-ops. + w.finalize() + w.abort() + + class _CountingFakeSource: """Delivers all frames synchronously, even past the stop count, to exercise the out-of-bounds guard in _on_frame.""" From 14ddb18860805dc4a08f15202e38e48b292180b8 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 28 Jun 2026 20:06:41 -0700 Subject: [PATCH 33/61] fix(record-zstack): clear stale FOV regions before re-geometrying + allow sub-0.1um z-step Code review of the FOV-grid wiring found two issues. (1) _update_scan_regions() called set_well_coordinates() without clearing first; set_well_coordinates only adds wells not already present, so changing region size/overlap/shape was silently ignored for already-selected wells (and stale geometry from the shared scanCoordinates persisted). Clear regions first, mirroring WellplateMultiPointWidget.update_coordinates; adds a call-order regression test (the MagicMock-based tests could not catch this). (2) entry_step setDecimals(1)->(2) so sub-0.1um z-steps are enterable again, widening the step spinbox to 98px so '1.00 um' does not clip; z-min/z-max stay 1-decimal. Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/widgets.py | 10 +++++-- software/tests/test_record_zstack_widget.py | 29 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index ac22306b2..8e9dba9aa 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17355,10 +17355,11 @@ def _build_zstack_group(self) -> QGroupBox: self.entry_step.setRange(0.001, 1000) self.entry_step.setValue(1.0) self.entry_step.setSuffix(" μm") - self.entry_step.setDecimals(1) + self.entry_step.setDecimals(2) self.entry_step.setSingleStep(0.1) self.entry_step.setKeyboardTracking(False) - self.entry_step.setMaximumWidth(85) + # Slightly wider than zmin/zmax: 2 decimals ("1.00 μm") need ~1 extra char. + self.entry_step.setMaximumWidth(98) layout.addWidget(QLabel("Step:"), 0, 4) layout.addWidget(self.entry_step, 0, 5) @@ -17666,6 +17667,11 @@ def _update_scan_regions(self) -> None: overlap_percent = self.entry_overlap.value() shape = self.combobox_shape.currentText() try: + # Clear first: set_well_coordinates only adds wells not already present, + # so without clearing, already-selected wells keep their old tile geometry + # and the new size/overlap/shape would be silently ignored. + if self.scanCoordinates.has_regions(): + self.scanCoordinates.clear_regions() self.scanCoordinates.set_well_coordinates(scan_size_mm, overlap_percent, shape) except Exception as exc: self._log.warning(f"_update_scan_regions: set_well_coordinates failed: {exc}") diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 68efa72c9..fec3a3c37 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -882,6 +882,35 @@ def test_fov_grid_wired_set_well_coordinates_receives_all_three_args(qtbot, simu sc.set_well_coordinates.assert_called_once_with(pytest.approx(1.5), pytest.approx(15.0), "Rectangle") +def test_fov_grid_clears_regions_before_set_well_coordinates(qtbot, simulated_widget_deps): + """_update_scan_regions must clear_regions() BEFORE set_well_coordinates(). + + Regression guard: set_well_coordinates only adds wells not already present, so + without clearing first, already-selected wells keep their old tile geometry and + a new size/overlap/shape is silently ignored. + """ + from unittest.mock import MagicMock, call + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + sc.has_regions.return_value = True # there is an existing region to clear + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + sc.reset_mock() + sc.has_regions.return_value = True + w._update_scan_regions() + + # clear_regions must be called, and must precede set_well_coordinates. + sc.clear_regions.assert_called_once() + sc.set_well_coordinates.assert_called_once() + relevant = [c for c in sc.method_calls if c[0] in ("clear_regions", "set_well_coordinates")] + assert relevant[0] == call.clear_regions() + assert relevant[1][0] == "set_well_coordinates" + + def test_fov_grid_no_crash_without_scan_coordinates(qtbot, simulated_widget_deps): """_update_scan_regions is a no-op when scanCoordinates is None.""" from control.widgets import RecordZStackMultiPointWidget From 9744776605d8255d994ecd0883a6c1c1db54ff98 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Tue, 30 Jun 2026 09:19:19 -0700 Subject: [PATCH 34/61] test(record-zstack): skip tensorstore-dependent tests when it's absent (CI) tensorstore is an optional dependency (zarr_writer imports it lazily; not in pyproject) and is not installed in CI, so the feature's real-ZarrWriter tests errored there. Guard them with pytest.importorskip('tensorstore'), matching the existing convention in tests/control/core/test_zarr_writer.py. Per-test (not module-level) so the non-tensorstore tests in these files keep running in CI. Verified: tensorstore absent -> the 4 skip (13 passed, 4 skipped, exit 0); present -> all 17 pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- software/tests/core/test_record_zstack_worker.py | 2 ++ software/tests/core/test_streaming_capture.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index 8f60ac191..8af51d355 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -51,6 +51,7 @@ def _build_simulated_microscope(crop_w: int, crop_h: int): def test_record_zstack_worker_smoke(tmp_path): + pytest.importorskip("tensorstore") # optional dep; the worker writes real Zarr import control._def import tests.control.test_stubs as ts from control.core.multi_point_controller import NoOpCallbacks @@ -169,6 +170,7 @@ def test_record_zstack_worker_smoke(tmp_path): def test_record_zstack_controller_smoke(tmp_path): + pytest.importorskip("tensorstore") # optional dep; the worker writes real Zarr import control._def import tests.control.test_stubs as ts from control.core.multi_point_controller import NoOpCallbacks diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py index d8d36c634..3bf7e4e16 100644 --- a/software/tests/core/test_streaming_capture.py +++ b/software/tests/core/test_streaming_capture.py @@ -19,6 +19,9 @@ def test_recording_router_downsamples_and_indexes(): def test_recording_writer_roundtrip(tmp_path): + import pytest + + pytest.importorskip("tensorstore") # optional dep; real ZarrWriter needs it T, Y, X = 4, 16, 12 from control.core.zarr_writer import ZarrAcquisitionConfig from control.core.streaming_capture import RecordingWriter @@ -310,6 +313,9 @@ def test_streaming_capture_partial_warns(caplog): def test_recording_writer_dropped_count_accessor(tmp_path): """dropped_count property returns the number of frames dropped due to a full queue.""" + import pytest + + pytest.importorskip("tensorstore") # optional dep; real ZarrWriter needs it from control.core.zarr_writer import ZarrAcquisitionConfig from control.core.streaming_capture import RecordingWriter From 9b5be65d4446d3d3e030579f6e316f4a9bf6fde3 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Fri, 3 Jul 2026 22:12:05 -0400 Subject: [PATCH 35/61] fix(record-zstack): add emit_selected_channels stub + keep well selector on tab Switching to the Record + Z-Stack tab raised AttributeError: 'RecordZStackMultiPointWidget' object has no attribute 'emit_selected_channels' from gui_hcs.onTabChanged, which duck-types the method on whichever record tab becomes current (the widget already carried the display_progress_bar stub for the same contract). Add the missing no-op stub. onTabChanged also hid the well selector dock when entering the tab (it only kept it for Wellplate Multipoint), while the tab's own validation requires selected wells. Treat Record + Z-Stack like Wellplate Multipoint when deciding well-selector visibility. Both found by driving the GUI end-to-end in simulation; covered by a widget-level stub test and a gui_hcs-level tab-switch test. Co-Authored-By: Claude Fable 5 --- software/control/gui_hcs.py | 12 ++++++- software/control/widgets.py | 8 +++++ .../control/test_HighContentScreeningGui.py | 31 +++++++++++++++++++ software/tests/test_record_zstack_widget.py | 12 +++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/software/control/gui_hcs.py b/software/control/gui_hcs.py index a1ed3776d..f9b44fe5f 100644 --- a/software/control/gui_hcs.py +++ b/software/control/gui_hcs.py @@ -2407,6 +2407,13 @@ def onTabChanged(self, index): if ENABLE_WELLPLATE_MULTIPOINT else False ) + # Record + Z-Stack selects wells like Wellplate Multipoint, so it needs the + # well selector too (its validation rejects acquisitions with no wells). + is_record_zstack_acquisition = ( + (index == self.recordTabWidget.indexOf(self.recordZStackWidget)) + if self.recordZStackWidget is not None + else False + ) self.scanCoordinates.clear_regions() if is_wellplate_acquisition: @@ -2421,7 +2428,10 @@ def onTabChanged(self, index): # trigger flexible regions update self.flexibleMultiPointWidget.update_fov_positions() - self.toggleWellSelector(is_wellplate_acquisition and self.wellSelectionWidget.format != "glass slide") + self.toggleWellSelector( + (is_wellplate_acquisition or is_record_zstack_acquisition) + and self.wellSelectionWidget.format != "glass slide" + ) acquisitionWidget = self.recordTabWidget.widget(index) acquisitionWidget.emit_selected_channels() diff --git a/software/control/widgets.py b/software/control/widgets.py index 8e9dba9aa..bd8c14c77 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17836,6 +17836,14 @@ def acquisition_is_finished(self): self.btn_startAcquisition.setChecked(False) self.signal_acquisition_started.emit(False) + def emit_selected_channels(self) -> None: + """No-op stub: RecordZStackMultiPointWidget has no channel list to broadcast. + + Required so ``onTabChanged`` in gui_hcs can call this on whatever widget + is the current record tab without checking the type (same contract as + ``display_progress_bar`` below). + """ + def display_progress_bar(self, show: bool) -> None: """No-op stub: RecordZStackMultiPointWidget has no progress bar. diff --git a/software/tests/control/test_HighContentScreeningGui.py b/software/tests/control/test_HighContentScreeningGui.py index 35a3ff961..5a33fe7c3 100644 --- a/software/tests/control/test_HighContentScreeningGui.py +++ b/software/tests/control/test_HighContentScreeningGui.py @@ -62,3 +62,34 @@ def confirm_exit(parent, title, text, *args, **kwargs): assert len(z_calls) == 1, f"signal_z_um_delta wired {len(z_calls)} times, expected 1" assert len(click_calls) == 1, f"image_click_coordinates wired {len(click_calls)} times, expected 1" + + +def test_record_zstack_tab_keeps_well_selector_visible(qtbot, monkeypatch): + """Switching to the Record + Z-Stack tab must not hide the well selector dock. + + The tab's own validation requires selected wells, so onTabChanged has to + treat it like Wellplate Multipoint when deciding well-selector visibility. + Also exercises the tab switch end-to-end, which duck-calls + emit_selected_channels() on the widget. + """ + + def confirm_exit(parent, title, text, *args, **kwargs): + if title == "Confirm Exit": + return QMessageBox.Yes + raise RuntimeError(f"Unexpected QMessageBox: {title} - {text}") + + monkeypatch.setattr(QMessageBox, "question", confirm_exit) + # The tab is gated by ENABLE_RECORDING; force it on regardless of the local INI. + monkeypatch.setattr(control.gui_hcs, "ENABLE_RECORDING", True) + + scope = control.microscope.Microscope.build_from_global_config(True) + win = control.gui_hcs.HighContentScreeningGui(microscope=scope, is_simulation=True) + qtbot.add_widget(win) + + tabw = win.recordTabWidget + labels = [tabw.tabText(i) for i in range(tabw.count())] + assert "Record + Z-Stack" in labels + + tabw.setCurrentIndex(labels.index("Record + Z-Stack")) + + assert not win.dock_wellSelection.isHidden(), "well selector dock must stay available on the Record + Z-Stack tab" diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index fec3a3c37..26cd0d3af 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -952,3 +952,15 @@ def test_toggle_acquisition_calls_update_scan_regions_before_run(qtbot, simulate assert "set_well_coordinates" in call_order assert "run_acquisition" in call_order assert call_order.index("set_well_coordinates") < call_order.index("run_acquisition") + + +def test_emit_selected_channels_is_a_safe_noop(qtbot, simulated_widget_deps): + """gui_hcs.onTabChanged duck-types emit_selected_channels() on whichever record + tab widget becomes current; the widget must provide it (same contract as + display_progress_bar) or every switch to the tab raises AttributeError.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.emit_selected_channels() # must not raise From 3f0ccc90f68413bd3a41b8060b47f740ee7f7e2e Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Fri, 3 Jul 2026 23:54:52 -0400 Subject: [PATCH 36/61] =?UTF-8?q?fix(record-zstack):=20critical=20review?= =?UTF-8?q?=20fixes=20=E2=80=94=20data=20loss,=20deadlock,=20abort,=20stat?= =?UTF-8?q?e=20restore,=20fps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the critical/high findings from the streaming/channel-control code review (worktrees/docs/code-review-record-zstack-streaming-2026-07-03.md): - Size the recording dataset from one real processed frame instead of camera.get_resolution(): real cameras crop/rotate frames, so every write failed shape-check and the recording was silently blank. Probe runs in run() after live view stops. - Drain thread counts write_frame failures and seals the store as incomplete (abort) instead of stamping acquisition_complete=True over blank fill; the worker fails fast (aborts the acquisition) on write errors instead of recording blank data at every remaining FOV. - finalize() no longer deadlocks pushing the sentinel into a full queue while the drain thread is wedged in a stalled write (bounded put with fallback to the abort path). - StreamingCapture.run() polls abort_fn while waiting, so Stop works when the camera delivers no frames, and the aborted capture is sealed aborted instead of complete. - run() captures and restores the pre-acquisition trigger mode and channel configuration (record-only runs left the camera stuck in SOFTWARE_TRIGGER; z-stack runs left the last z-stack channel applied with the GUI showing the old one). - zstack()'s trigger-mode fallback keeps liveController.trigger_mode in sync with the camera so acquire_camera_image still gates illumination (a stale mode captured the whole stack dark). - RecordingRouter anchors emission slots to the first frame with a quarter-period tolerance: at-rate delivery jitter no longer rejects alternate frames (halving the capture rate); faster-than-target cameras still downsample correctly. - record() sizes T, pacing, and time_increment_s from the achievable fps returned by set_frame_rate() instead of stalling to timeout and leaving blank trailing planes when the camera clamps the rate. TDD: each fix has a test that failed before it (10 new tests). Co-Authored-By: Claude Fable 5 --- software/control/core/record_zstack_worker.py | 97 +++++++- software/control/core/streaming_capture.py | 80 ++++++- .../tests/core/test_record_zstack_worker.py | 218 ++++++++++++++++++ software/tests/core/test_streaming_capture.py | 125 ++++++++++ 4 files changed, 504 insertions(+), 16 deletions(-) diff --git a/software/control/core/record_zstack_worker.py b/software/control/core/record_zstack_worker.py index 1aa3d2dd2..e371ac15e 100644 --- a/software/control/core/record_zstack_worker.py +++ b/software/control/core/record_zstack_worker.py @@ -133,7 +133,8 @@ def __init__( self._zstack_offsets: List[float] = ( zstack_offsets_um(params.z_min_um, params.z_max_um, params.z_step_um) if params.zstack_enabled else [] ) - self._frame_shape: Tuple[int, int, np.dtype] = self._probe_frame_shape() + # Probing captures a frame, so it happens in run() after live view stops. + self._frame_shape: Optional[Tuple[int, int, np.dtype]] = None if params.zstack_enabled and self.zstack_channels: self._setup_zstack_job_runner(prewarmed_job_runner, prewarmed_bp_values) @@ -228,6 +229,12 @@ def run(self): """ # Quiesce live view once for the whole acquisition (restored in finally). was_live = bool(getattr(self.liveController, "is_live", False)) + # Capture pre-acquisition hardware state so the finally can put the + # camera/MCU/LiveController back the way the user had them: both phases + # change the trigger mode, and every z-stack channel apply overwrites + # the current channel configuration (exposure/gain/illumination). + prev_trigger_mode = getattr(self.liveController, "trigger_mode", None) + prev_configuration = getattr(self.liveController, "currentConfiguration", None) if was_live: try: self.liveController.stop_live() @@ -235,6 +242,12 @@ def run(self): log.exception("Failed to stop live view before acquisition") try: + if self.params.recording_enabled: + # Size the recording datasets from one real processed frame. + # Deferred to here (not __init__) so the probe capture only + # touches the camera after live view has been stopped. + self._frame_shape = self._probe_frame_shape() + if self.params.zstack_enabled and self._job_runners: self._backpressure.reset() @@ -267,6 +280,20 @@ def run(self): self._finish_jobs() except Exception: log.exception("Error finishing z-stack jobs") + # Restore pre-acquisition hardware state: channel configuration first + # (exposure/gain/illumination source), then trigger mode (which for + # HARDWARE reads currentConfiguration.exposure_time), so camera + + # LiveController + MCU agree again before live view resumes. + if prev_configuration is not None: + try: + self.liveController.set_microscope_mode(prev_configuration) + except Exception: + log.exception("Failed to restore channel configuration after acquisition") + if prev_trigger_mode is not None: + try: + self.liveController.set_trigger_mode(prev_trigger_mode) + except Exception: + log.exception("Failed to restore trigger mode after acquisition") # Restart live view once, only if it was running before the acquisition. if was_live: try: @@ -314,7 +341,26 @@ def record(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> int: # Move to z_ref + recording offset. self._move_z_to_offset(z_ref, self.params.recording_z_offset_um) - T = frame_count(self.params.fps, self.params.duration_s) + # Size the dataset, pacing, and time metadata from the fps the camera can + # actually deliver: a camera clamped below the requested rate (exposure + # limit, PRECISE_FRAMERATE max) can never fill fps*duration frames within + # duration seconds — the run would stall to the timeout and leave the + # trailing planes blank. The mode switch happens first because toupcam + # resets its frame-rate strategy on mode change. + self.camera.set_acquisition_mode(CameraAcquisitionMode.CONTINUOUS) + effective_fps = self.params.fps + try: + achievable_fps = self.camera.set_frame_rate(self.params.fps) + if achievable_fps and 0 < achievable_fps < self.params.fps: + log.warning( + f"camera cannot deliver {self.params.fps:g} fps " + f"(achievable ≈ {achievable_fps:.2f}); recording at the achievable rate" + ) + effective_fps = achievable_fps + except Exception: + log.exception("set_frame_rate probe failed; assuming the requested fps") + + T = max(1, frame_count(effective_fps, self.params.duration_s)) out = self._recording_path(t_idx, region_id, fov_idx) y, x, dtype = self._frame_shape @@ -326,7 +372,7 @@ def record(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> int: dtype=dtype, pixel_size_um=self._pixel_size_um if self._pixel_size_um is not None else 1.0, z_step_um=None, - time_increment_s=(1.0 / self.params.fps) if self.params.fps and self.params.fps > 0 else None, + time_increment_s=(1.0 / effective_fps) if effective_fps and effective_fps > 0 else None, channel_names=[rec_channel_name], channel_colors=[rec_color], channel_wavelengths=[None], @@ -334,8 +380,8 @@ def record(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> int: ) writer = RecordingWriter(cfg) cap = StreamingCapture( - ContinuousFrameSource(self.camera, self.params.fps), - RecordingRouter(self.params.fps), + ContinuousFrameSource(self.camera, effective_fps), + RecordingRouter(effective_fps), CountStop(T), writer, abort_fn=self.abort_requested_fn, @@ -362,6 +408,14 @@ def record(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> int: self.stage.move_z_to(z_ref) self.wait_till_operation_is_completed() self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) + # Fail fast on write errors: the cause is almost always systematic (full + # disk, shape mismatch), so continuing would record blank data at every + # remaining FOV. run() catches this, aborts, and signals finished. + if writer.write_error_count > 0: + raise RuntimeError( + f"{writer.write_error_count} recording write error(s) at t={t_idx} " + f"region={region_id} fov={fov_idx}; store sealed incomplete: {out}" + ) return emitted # ---------------------------------------------------------------- zstack @@ -389,6 +443,11 @@ def zstack(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> None: # Fall back to setting the camera directly so capture can still proceed. try: self.camera.set_acquisition_mode(CameraAcquisitionMode.SOFTWARE_TRIGGER) + # Keep the LiveController's view of the mode in sync with the + # camera: the inherited acquire_camera_image branches on + # liveController.trigger_mode to gate illumination, so a stale + # mode here would capture the entire z-stack dark. + self.liveController.trigger_mode = TriggerMode.SOFTWARE except Exception: log.exception("Failed to set camera software-trigger mode for z-stack") self.camera.start_streaming() @@ -452,7 +511,33 @@ def _move_z_to_offset(self, z_ref: float, offset_um: float) -> None: self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) def _probe_frame_shape(self) -> Tuple[int, int, np.dtype]: - """Return (Y, X, dtype) for the recording dataset from the camera resolution.""" + """Return (Y, X, dtype) for the recording dataset from one processed frame. + + ``get_resolution()`` reports the sensor/binned size, but frames delivered + to callbacks pass through ``_process_raw_frame`` (software crop, rotation, + ROI). On cameras where the two differ, a dataset sized from + ``get_resolution()`` makes every ``write_frame`` fail — a blank recording. + Capture one real frame and size the dataset from it; fall back to + ``get_resolution()`` only if the probe capture fails. + """ + frame = None + try: + self.camera.set_acquisition_mode(CameraAcquisitionMode.SOFTWARE_TRIGGER) + self.camera.start_streaming() + self.camera.send_trigger() + cam_frame = self.camera.read_camera_frame() + if cam_frame is not None: + frame = cam_frame.frame + except Exception: + log.exception("probe-frame capture failed; falling back to get_resolution()") + finally: + try: + self.camera.stop_streaming() + except Exception: + log.exception("failed to stop streaming after probe frame") + if frame is not None: + return int(frame.shape[0]), int(frame.shape[1]), frame.dtype + log.warning("sizing recording dataset from get_resolution(); may mismatch delivered frames") width, height = self.camera.get_resolution() # Map pixel format to numpy dtype (MONO8 -> uint8, else uint16). try: diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index 10097c2fb..9e380acdf 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -1,5 +1,6 @@ import queue import threading +import time from typing import Callable, Optional, Tuple import numpy as np @@ -25,19 +26,29 @@ def expected(self) -> Optional[int]: class RecordingRouter: - """Maps incoming frames to (t,c,z)=(t_index,0,0), downsampling to `fps`.""" + """Maps incoming frames to (t,c,z)=(t_index,0,0), downsampling to `fps`. + + Emission slots are anchored to the FIRST frame's timestamp (slot k opens at + ``first_ts + k*period``), not to the previous emission: with the camera + running at the target rate, host-side delivery jitter around the period + would otherwise reject roughly alternate frames and halve the effective + capture rate. A quarter-period tolerance absorbs per-frame jitter, and + absolute slots self-correct — one late frame never delays later slots, so + the long-run rate stays at ``fps``. + """ def __init__(self, fps: float): - self._min_period = 1.0 / fps if fps and fps > 0 else 0.0 + self._period = 1.0 / fps if fps and fps > 0 else 0.0 self._t_index = 0 - self._last_emit_ts: Optional[float] = None + self._first_ts: Optional[float] = None def route(self, timestamp: float) -> Optional[Tuple[int, int, int]]: - if self._last_emit_ts is not None and (timestamp - self._last_emit_ts) < self._min_period - 1e-9: + if self._first_ts is None: + self._first_ts = timestamp + elif self._period > 0 and (timestamp - self._first_ts) < (self._t_index - 0.25) * self._period: return None idx = (self._t_index, 0, 0) self._t_index += 1 - self._last_emit_ts = timestamp return idx @@ -61,6 +72,7 @@ def __init__(self, config: ZarrAcquisitionConfig, max_queue: int = 256): self._q: "queue.Queue" = queue.Queue(maxsize=max_queue) self._thread = threading.Thread(target=self._drain, daemon=True) self._dropped = 0 + self._write_errors = 0 self._abort_requested = threading.Event() # True only once the drain thread has actually been started. finalize()/ # abort() must not join (or push the sentinel to) a thread that never @@ -119,10 +131,20 @@ def _drain(self) -> None: try: self._writer.write_frame(frame, t=t, c=c, z=z) except Exception as e: + self._write_errors += 1 _log.error(f"recording write_frame failed t={t}: {e}") finally: if self._abort_requested.is_set(): self._writer.abort() + elif self._write_errors > 0: + # Never stamp acquisition_complete=True on a store with failed + # writes: the missing planes are silent fill values. abort() + # seals it acquisition_complete=False so readers can tell. + _log.error( + f"{self._write_errors} write error(s) during recording; " + f"sealing store as incomplete instead of complete" + ) + self._writer.abort() else: self._writer.finalize() @@ -131,14 +153,39 @@ def dropped_count(self) -> int: """Total frames dropped due to a full queue (diagnosable in slow-disk runs).""" return self._dropped - def finalize(self) -> None: - """Flush the queue, join the drain thread (which finalizes the ZarrWriter).""" + @property + def write_error_count(self) -> int: + """Total write_frame failures on the drain thread (0 on a healthy run).""" + return self._write_errors + + def finalize(self, timeout_s: float = 30.0) -> None: + """Flush the queue, join the drain thread (which seals the ZarrWriter). + + The sentinel push is bounded: with the drain thread wedged inside a + stalled write and the queue full, a bare ``put`` would block the + acquisition thread forever (the join timeout below would never be + reached). After ``timeout_s`` of no queue space, fall back to the + abort path and return — the daemon drain thread seals the store + whenever the stalled write finally returns. + """ if not self._started: # start() never got the thread running (e.g. initialize() raised). # Nothing to flush or join; let the original error propagate. return - self._q.put(_SENTINEL) - self._thread.join(timeout=30.0) + deadline = time.monotonic() + timeout_s + while True: + try: + self._q.put(_SENTINEL, timeout=min(1.0, max(0.1, timeout_s))) + break + except queue.Full: + if time.monotonic() >= deadline: + _log.error( + f"drain thread wedged (queue full for {timeout_s:.0f}s); " + f"switching to abort — recording will be sealed incomplete" + ) + self._abort_requested.set() + return + self._thread.join(timeout=timeout_s) if self._thread.is_alive(): _log.warning("drain thread still alive after finalize() join timeout") @@ -250,7 +297,20 @@ def run(self, timeout: Optional[float] = None) -> int: self._writer.start() try: self._source.start(self._on_frame) - self._done.wait(timeout) # FakeSource sets _done synchronously; real camera via callback + # Poll abort_fn while waiting: the frame callback also samples it, but + # if the camera delivers no frames at all (stall, misconfigured + # trigger) the callback never runs and a bare wait(timeout) would + # ignore Stop for the full timeout — and then seal the store as + # complete. (FakeSource sets _done synchronously; the first wait() + # returns immediately in that case.) + deadline = (time.monotonic() + timeout) if timeout is not None else None + while not self._done.wait(0.2): + if self._abort_fn(): + self._aborted = True + self._done.set() + break + if deadline is not None and time.monotonic() >= deadline: + break finally: # Assumes source.stop() quiesces the camera delivery thread. With cameras # that don't join their callback thread on stop, a final in-flight frame may diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index 8af51d355..6f44164b7 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -283,3 +283,221 @@ def test_record_zstack_controller_smoke(tmp_path): assert tuple(ds.shape) == (Nt, len(zstack_channels), n_z, crop_h, crop_w), f"bad z-stack shape {ds.shape}" controller.close() + + +def test_probe_frame_shape_uses_processed_frame_not_resolution(): + """The recording dataset must be sized from a delivered (processed) frame: + real cameras crop/rotate frames in _process_raw_frame, so get_resolution() + (sensor/binned size) is the wrong source and yields blank recordings.""" + from types import SimpleNamespace + + import numpy as np + + from control.core.record_zstack_worker import RecordZStackWorker + + processed = np.zeros((50, 60), dtype=np.uint8) # crop/rotation already applied + + class _FakeCamera: + def get_resolution(self): + return (100, 80) # (width, height) sensor size — differs from frames + + def get_pixel_format(self): + raise NotImplementedError # force the dtype to come from the frame + + def set_acquisition_mode(self, mode): + pass + + def start_streaming(self): + pass + + def stop_streaming(self): + pass + + def send_trigger(self): + pass + + def read_camera_frame(self): + return SimpleNamespace(frame=processed) + + fake_self = SimpleNamespace(camera=_FakeCamera()) + y, x, dtype = RecordZStackWorker._probe_frame_shape(fake_self) + + assert (y, x) == (50, 60), f"expected processed-frame shape, got ({y}, {x})" + assert dtype == np.uint8 + + +# --------------------------------------------------------------------------- +# Review-fix tests: post-run hardware state restore (F8, F9, F10) +# --------------------------------------------------------------------------- + + +def _build_worker_harness(tmp_path, recording_enabled, zstack_enabled, zstack_channel_slice=slice(1, 2)): + """Simulated scope + live controller + a 1-FOV worker with tiny params.""" + import control._def + import tests.control.test_stubs as ts + from control.core.multi_point_controller import NoOpCallbacks + from control.core.record_zstack_controller import RecordZStackAcquisitionParameters + from control.core.record_zstack_worker import RecordZStackWorker + + control._def.FILE_SAVING_OPTION = control._def.FileSavingOption.ZARR_V3 + scope = _build_simulated_microscope(64, 48) + live_controller = ts.get_test_live_controller(scope, scope.objective_store.default_objective) + laser_af = ts.get_test_laser_autofocus_controller(scope) + channels = live_controller.get_channels(scope.objective_store.default_objective) + scope.camera.set_exposure_time(1) + + z_cfg = scope.stage.get_config().Z_AXIS + scope.stage.move_z_to((z_cfg.MAX_POSITION + z_cfg.MIN_POSITION) / 2.0) + scope.low_level_drivers.microcontroller.wait_till_operation_is_completed() + x0 = scope.stage.get_config().X_AXIS.MIN_POSITION + 1.0 + y0 = scope.stage.get_config().Y_AXIS.MIN_POSITION + 1.0 + + params = RecordZStackAcquisitionParameters( + base_path=str(tmp_path), + experiment_id="state_restore", + Nt=1, + dt_s=0.0, + use_laser_af=False, + recording_enabled=recording_enabled, + recording_channel=channels[0], + fps=10.0, + duration_s=0.2, + recording_z_offset_um=0.0, + zstack_enabled=zstack_enabled, + zstack_channels=list(channels[zstack_channel_slice]) if zstack_enabled else [], + z_min_um=0.0, + z_max_um=1.0, + z_step_um=1.0, + ) + aborted = {"v": False} + worker = RecordZStackWorker( + scope=scope, + live_controller=live_controller, + laser_auto_focus_controller=laser_af, + objective_store=scope.objective_store, + params=params, + callbacks=NoOpCallbacks, + abort_requested_fn=lambda: aborted["v"], + request_abort_fn=lambda: aborted.__setitem__("v", True), + scan_region_fov_coords={"A1": [(x0, y0)]}, + ) + return scope, live_controller, channels, worker, aborted + + +def test_record_only_restores_trigger_mode(tmp_path): + """F8: a record-only run must not leave the camera in SOFTWARE_TRIGGER when + the LiveController was in CONTINUOUS (or HARDWARE) before the acquisition.""" + pytest.importorskip("tensorstore") + from control._def import TriggerMode + from squid.abc import CameraAcquisitionMode + + scope, live_controller, channels, worker, aborted = _build_worker_harness( + tmp_path, recording_enabled=True, zstack_enabled=False + ) + live_controller.set_microscope_mode(channels[0]) + live_controller.set_trigger_mode(TriggerMode.CONTINUOUS) + + worker.run() + + assert not aborted["v"] + assert live_controller.trigger_mode == TriggerMode.CONTINUOUS + assert ( + scope.camera.get_acquisition_mode() == CameraAcquisitionMode.CONTINUOUS + ), "camera left out of sync with LiveController trigger mode after record-only run" + + +def test_run_restores_pre_acquisition_channel_config(tmp_path): + """F10: after the acquisition, the hardware must be back on the channel the + user was viewing, not the last z-stack channel.""" + pytest.importorskip("tensorstore") + + scope, live_controller, channels, worker, aborted = _build_worker_harness( + tmp_path, recording_enabled=False, zstack_enabled=True, zstack_channel_slice=slice(1, 2) + ) + live_controller.set_microscope_mode(channels[0]) # user was viewing channel 0 + + worker.run() + + assert not aborted["v"] + current = live_controller.currentConfiguration + assert current is not None and current.name == channels[0].name, ( + f"expected channel config restored to {channels[0].name!r}, " f"got {current.name if current else None!r}" + ) + + +def test_zstack_trigger_fallback_keeps_livecontroller_in_sync(tmp_path, monkeypatch): + """F9: if set_trigger_mode(SOFTWARE) fails once, the fallback must keep + liveController.trigger_mode in sync with the camera — otherwise + acquire_camera_image takes neither illumination branch and the whole + z-stack is captured dark.""" + pytest.importorskip("tensorstore") + from control._def import TriggerMode + + scope, live_controller, channels, worker, aborted = _build_worker_harness( + tmp_path, recording_enabled=False, zstack_enabled=True + ) + live_controller.set_microscope_mode(channels[0]) + live_controller.set_trigger_mode(TriggerMode.CONTINUOUS) + + orig_set_trigger_mode = live_controller.set_trigger_mode + calls = {"n": 0} + + def flaky(mode): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("simulated MCU ACK timeout") + return orig_set_trigger_mode(mode) + + monkeypatch.setattr(live_controller, "set_trigger_mode", flaky) + + seen_modes = [] + orig_acquire = worker.acquire_camera_image + + def spy(*args, **kwargs): + seen_modes.append(live_controller.trigger_mode) + return orig_acquire(*args, **kwargs) + + monkeypatch.setattr(worker, "acquire_camera_image", spy) + + worker.run() + + assert seen_modes, "no z-stack captures happened" + assert all(m == TriggerMode.SOFTWARE for m in seen_modes), ( + f"z-stack captured with stale liveController.trigger_mode {seen_modes[0]} " + f"— illumination branch skipped (dark images)" + ) + + +def test_recording_uses_achievable_fps_when_camera_clamps(tmp_path): + """F6: when the camera clamps below the requested fps (exposure-limited), + the dataset size and time metadata must reflect the achievable rate — + otherwise the run stalls to the timeout and trailing planes are blank.""" + pytest.importorskip("tensorstore") + import json + + scope, live_controller, channels, worker, aborted = _build_worker_harness( + tmp_path, recording_enabled=True, zstack_enabled=False + ) + # record() applies the recording channel (exposure included) before probing + # the frame rate, so the long exposure must live on the channel itself. + channels[0].exposure_time = 200.0 # ms → achievable ≈ a few fps, well below 10 + scope.camera.set_exposure_time(200) + achievable = scope.camera.set_frame_rate(10.0) + assert achievable < 10.0, "precondition: camera must clamp below the requested rate" + + worker.run() + assert not aborted["v"] + + rec = sorted((Path(tmp_path) / "state_restore" / "recording").rglob("*.ome.zarr")) + assert len(rec) == 1 + meta = json.load(open(rec[0] / "zarr.json")) + squid_attrs = meta["attributes"]["_squid"] + expected_T = max(1, frame_count(achievable, 0.2)) + assert squid_attrs["shape"][0] == expected_T, ( + f"dataset sized for the requested fps (T={squid_attrs['shape'][0]}), " + f"expected achievable-rate T={expected_T}" + ) + assert abs(squid_attrs["time_increment_s"] - 1.0 / achievable) < 1e-6, ( + f"time_increment_s={squid_attrs['time_increment_s']} does not match the " + f"achievable rate (1/{achievable:.2f})" + ) diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py index 3bf7e4e16..dc09ced96 100644 --- a/software/tests/core/test_streaming_capture.py +++ b/software/tests/core/test_streaming_capture.py @@ -380,3 +380,128 @@ def abort(self): cap.run() assert any("dropped" in rec.getMessage() and "3" in rec.getMessage() for rec in caplog.records) + + +# --------------------------------------------------------------------------- +# Review-fix tests: fail-loud drain (F2), bounded finalize (F3), abort wake (F4) +# --------------------------------------------------------------------------- + + +def _stub_zarr_cfg(tmp_path): + from control.core.zarr_writer import ZarrAcquisitionConfig + + return ZarrAcquisitionConfig( + output_path=str(tmp_path / "rec.ome.zarr"), + shape=(4, 1, 1, 4, 4), + dtype=np.uint16, + pixel_size_um=1.0, + z_step_um=None, + time_increment_s=0.1, + channel_names=["BF"], + channel_colors=["#FFFFFF"], + channel_wavelengths=[None], + is_hcs=False, + ) + + +def test_recording_writer_write_errors_seal_incomplete(tmp_path, monkeypatch): + """If write_frame fails, the store must NOT be sealed acquisition_complete=True: + the drain thread must count the failures and seal via abort() instead.""" + from control.core.zarr_writer import ZarrWriter + from control.core.streaming_capture import RecordingWriter + + calls = [] + monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) + + def boom(self, image, t, c, z, fov=None): + raise RuntimeError("disk full") + + monkeypatch.setattr(ZarrWriter, "write_frame", boom) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self: calls.append("finalize")) + monkeypatch.setattr(ZarrWriter, "abort", lambda self: calls.append("abort")) + + rw = RecordingWriter(_stub_zarr_cfg(tmp_path)) + rw.start() + rw.enqueue(np.zeros((4, 4), np.uint16), 0, 0, 0) + rw.enqueue(np.zeros((4, 4), np.uint16), 1, 0, 0) + rw.finalize() + + assert rw.write_error_count == 2 + assert calls == ["abort"], f"expected incomplete seal via abort(), got {calls}" + + +def test_recording_writer_finalize_bounded_when_drain_wedged(tmp_path, monkeypatch): + """finalize() must not block forever pushing the sentinel onto a full queue + while the drain thread is wedged inside a stalled write.""" + import threading + from control.core.zarr_writer import ZarrWriter + from control.core.streaming_capture import RecordingWriter + + release = threading.Event() + monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: release.wait(20)) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "abort", lambda self: None) + + rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=2) + rw.start() + # First frame wedges the drain thread inside write_frame; two more fill the queue. + for i in range(3): + rw.enqueue(np.zeros((4, 4), np.uint16), i, 0, 0) + + t = threading.Thread(target=lambda: rw.finalize(timeout_s=1.0), daemon=True) + t.start() + t.join(timeout=5.0) + still_stuck = t.is_alive() + release.set() # let the drain thread exit before asserting + assert not still_stuck, "finalize() deadlocked on the full bounded queue" + + +def test_streaming_capture_abort_wakes_without_frames(): + """Stop/abort must work even when the camera delivers no frames at all: + run() must poll abort_fn rather than sampling it only in the frame callback.""" + import time as _time + + class _NoFrameSource: + def start(self, on_frame): + pass + + def stop(self): + pass + + w = _RecordingStubWriter() + cap = StreamingCapture( + _NoFrameSource(), + RecordingRouter(fps=10.0), + CountStop(5), + w, + abort_fn=lambda: True, # user pressed Stop + ) + t0 = _time.monotonic() + emitted = cap.run(timeout=10.0) + took = _time.monotonic() - t0 + + assert emitted == 0 + assert took < 2.0, f"abort took {took:.1f}s — run() ignored abort while no frames arrived" + assert w.aborted is True, "aborted capture must be sealed as aborted" + assert w.finalized is False, "aborted capture must not be sealed as complete" + + +def test_recording_router_tolerates_delivery_jitter(): + """F5: with the camera running AT the target rate, ms-level host delivery + jitter must not reject frames — anchoring to the previous emission made + every slightly-early frame fail the gate and halved the effective rate.""" + r = RecordingRouter(fps=10.0) + # 10 fps arrivals with a 1 ms early wobble on every other frame. + stamps = [100.0 + i * 0.1 - (0.001 if i % 2 else 0.0) for i in range(10)] + accepted = [s for s in stamps if r.route(s) is not None] + assert len(accepted) == 10, f"only {len(accepted)}/10 at-rate frames accepted (jitter rejected frames)" + + +def test_recording_router_still_downsamples_faster_camera(): + """The jitter fix must not break downsampling: a camera at 2x the target + rate should still have roughly half its frames rejected.""" + r = RecordingRouter(fps=10.0) + stamps = [100.0 + i * 0.05 for i in range(20)] # 20 fps camera, 10 fps target + accepted = [s for s in stamps if r.route(s) is not None] + assert len(accepted) == 10, f"expected 10/20 accepted, got {len(accepted)}" From 80b4c85560d13c2d87b7accfa454c95dbf9d4c50 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 00:02:29 -0400 Subject: [PATCH 37/61] =?UTF-8?q?fix(record-zstack):=20medium=20review=20f?= =?UTF-8?q?ixes=20=E2=80=94=20byte=20cap,=20per-FOV=20z,=20dt=20pacing,=20?= =?UTF-8?q?refresh,=20cleanup,=20bookkeeping?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remaining findings from the streaming/channel-control review: - RecordingWriter queue is bounded by bytes (default 2 GiB) as well as count: 256 full-resolution 16-bit frames is ~13 GB, so a count-only bound could OOM the process on a slow disk. - _move_xy honors the optional z of (x, y, z) scan coordinates like MultiPointWorker.move_to_coordinate — stored per-FOV focus planes were silently ignored (out of focus on tilted samples without laser AF). - _wait_for_dt paces timepoint STARTS at acquisition_start + k*dt (matching multipoint and the recorded time_increment_s), instead of appending dt after each timepoint's work; logs when work overruns dt. - RecordZStackMultiPointWidget.refresh_channel_list() repopulates the channel combos on objective/profile/config changes (wired in gui_hcs alongside the other multipoint widgets), and the _make_channel_base fallback warns loudly instead of silently producing a channel with no illumination mapping (dark images). - gui_hcs._cleanup_common closes recordZStackController on exit so the JobRunner subprocess finalizes Zarr writers instead of being killed mid-write. - Each experiment directory now gets the acquisition_channels.yaml settings snapshot (controller) and the .done completion marker (worker), matching every multipoint acquisition. TDD: 8 new tests, each watched failing first (except the two gui_hcs wiring changes, covered by the CI gui test file and runtime driver). Co-Authored-By: Claude Fable 5 --- .../control/core/record_zstack_controller.py | 21 +++- software/control/core/record_zstack_worker.py | 36 ++++++- software/control/core/streaming_capture.py | 24 ++++- software/control/gui_hcs.py | 15 +++ software/control/widgets.py | 29 ++++++ .../tests/core/test_record_zstack_worker.py | 99 +++++++++++++++++++ software/tests/core/test_streaming_capture.py | 24 +++++ software/tests/test_record_zstack_widget.py | 34 +++++++ 8 files changed, 277 insertions(+), 5 deletions(-) diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py index 0c8ea86ed..0bc5a3f59 100644 --- a/software/control/core/record_zstack_controller.py +++ b/software/control/core/record_zstack_controller.py @@ -136,9 +136,28 @@ def run_acquisition(self, params: RecordZStackAcquisitionParameters) -> None: from control.core.record_zstack_worker import RecordZStackWorker # Resolve and create a timestamped unique output directory. - resolved_id, _dir = create_experiment_dir(params.base_path, params.experiment_id) + resolved_id, experiment_dir = create_experiment_dir(params.base_path, params.experiment_id) params.experiment_id = resolved_id + # Snapshot the acquisition settings into the experiment directory + # (mirrors MultiPointController.start_new_experiment) so the run is + # reproducible/auditable: acquisition_channels.yaml records objective + + # every channel used by either phase. + try: + channels = [] + if params.recording_enabled and params.recording_channel is not None: + channels.append(params.recording_channel) + if params.zstack_enabled: + channels.extend(params.zstack_channels) + self._microscope.config_repo.save_acquisition_output( + output_dir=experiment_dir, + objective=self._objective_store.current_objective, + channels=channels, + confocal_mode=self._live_controller.is_confocal_mode(), + ) + except Exception: + log.exception("Failed to save acquisition settings snapshot to the experiment directory") + # Collect scan coordinates: {region_id: [(x_mm, y_mm[, z_mm]), ...]} scan_region_fov_coords = {} if self._scan_coordinates is not None and hasattr(self._scan_coordinates, "region_fov_coordinates"): diff --git a/software/control/core/record_zstack_worker.py b/software/control/core/record_zstack_worker.py index e371ac15e..27bf67db8 100644 --- a/software/control/core/record_zstack_worker.py +++ b/software/control/core/record_zstack_worker.py @@ -135,6 +135,8 @@ def __init__( ) # Probing captures a frame, so it happens in run() after live view stops. self._frame_shape: Optional[Tuple[int, int, np.dtype]] = None + # Set at the top of run(); _wait_for_dt paces timepoint STARTS from it. + self._acq_start_time: Optional[float] = None if params.zstack_enabled and self.zstack_channels: self._setup_zstack_job_runner(prewarmed_job_runner, prewarmed_bp_values) @@ -251,6 +253,7 @@ def run(self): if self.params.zstack_enabled and self._job_runners: self._backpressure.reset() + self._acq_start_time = time.time() for t_idx in range(self.params.Nt): self.time_point = t_idx if t_idx > 0 and self.params.dt_s > 0: @@ -300,6 +303,16 @@ def run(self): self.liveController.start_live() except Exception: log.exception("Failed to restart live view after acquisition") + # Completion marker for downstream watchers (mirrors multipoint's + # _on_acquisition_completed). Written on abort too: the directory is + # final either way, and the zarr attrs record completeness. + try: + from control.utils import create_done_file + + if os.path.isdir(self.experiment_path): + create_done_file(self.experiment_path) + except Exception: + log.exception("Failed to write completion marker (.done)") try: self.callbacks.signal_acquisition_finished() except Exception: @@ -502,6 +515,13 @@ def _move_xy(self, coord) -> None: self._sleep(SCAN_STABILIZATION_TIME_MS_X / 1000) self.stage.move_y_to(coord[1]) self._sleep(SCAN_STABILIZATION_TIME_MS_Y / 1000) + # (x, y, z) coords carry a stored per-FOV focus plane (flexible regions, + # update_fov_z) — honor it like MultiPointWorker.move_to_coordinate, or + # establish_reference() would reuse the previous FOV's Z on tilted samples. + if len(coord) > 2 and coord[2] is not None: + self.stage.move_z_to(coord[2]) + self.wait_till_operation_is_completed() + self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) def _move_z_to_offset(self, z_ref: float, offset_um: float) -> None: """Move to ``z_ref + offset_um`` (offset in µm, z_ref in mm) via the stage.""" @@ -569,11 +589,21 @@ def _zstack_dir(self, region_id) -> str: return path def _wait_for_dt(self, t_idx: int) -> bool: - """Sleep until it's time for time point ``t_idx`` (abort-aware). + """Sleep until time point ``t_idx``'s scheduled start (abort-aware). - Returns False if an abort was requested while waiting. + Starts are paced at ``acquisition_start + t_idx * dt`` (matching + MultiPointWorker and the recorded ``time_increment_s`` metadata) — + NOT ``dt`` after the previous timepoint's work finished, which would + stretch the real sampling interval to ``dt + work`` while the metadata + still claimed ``dt``. Returns False if an abort was requested. """ - deadline = time.time() + self.params.dt_s + start = self._acq_start_time if self._acq_start_time is not None else time.time() + deadline = start + t_idx * self.params.dt_s + if time.time() > deadline: + log.warning( + f"time point {t_idx} starting late: per-timepoint work exceeded dt={self.params.dt_s:g}s; " + f"recorded time_increment_s underestimates the real interval" + ) while time.time() < deadline: if self.abort_requested_fn(): return False diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index 9e380acdf..0dde47678 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -67,12 +67,17 @@ class RecordingWriter: concurrently with the drain thread still inside `write_frame`. """ - def __init__(self, config: ZarrAcquisitionConfig, max_queue: int = 256): + def __init__(self, config: ZarrAcquisitionConfig, max_queue: int = 256, max_bytes: int = 2 * 1024**3): self._writer = ZarrWriter(config) self._q: "queue.Queue" = queue.Queue(maxsize=max_queue) self._thread = threading.Thread(target=self._drain, daemon=True) self._dropped = 0 self._write_errors = 0 + # Byte cap alongside the count cap: 256 full-resolution 16-bit frames is + # ~13 GB, so a count-only bound can OOM the process on a slow disk. + self._max_bytes = max_bytes + self._held_bytes = 0 + self._bytes_lock = threading.Lock() self._abort_requested = threading.Event() # True only once the drain thread has actually been started. finalize()/ # abort() must not join (or push the sentinel to) a thread that never @@ -105,9 +110,23 @@ def enqueue(self, frame: np.ndarray, t: int, c: int, z: int) -> None: bounded queue is full (drain thread cannot keep up with disk I/O) the frame is dropped and counted rather than waiting for space. """ + nbytes = int(getattr(frame, "nbytes", 0)) + with self._bytes_lock: + over_cap = self._held_bytes + nbytes > self._max_bytes + if not over_cap: + self._held_bytes += nbytes + if over_cap: + self._dropped += 1 + _log.warning( + f"recording byte cap reached ({self._max_bytes} B); dropped frame t={t} " + f"(total dropped={self._dropped})" + ) + return try: self._q.put_nowait((frame, t, c, z)) except queue.Full: + with self._bytes_lock: + self._held_bytes -= nbytes self._dropped += 1 _log.warning(f"recording queue full; dropped frame t={t} (total dropped={self._dropped})") @@ -133,6 +152,9 @@ def _drain(self) -> None: except Exception as e: self._write_errors += 1 _log.error(f"recording write_frame failed t={t}: {e}") + finally: + with self._bytes_lock: + self._held_bytes -= int(getattr(frame, "nbytes", 0)) finally: if self._abort_requested.is_set(): self._writer.abort() diff --git a/software/control/gui_hcs.py b/software/control/gui_hcs.py index f9b44fe5f..30e50cfd5 100644 --- a/software/control/gui_hcs.py +++ b/software/control/gui_hcs.py @@ -1650,6 +1650,10 @@ def make_connections(self): self.objectivesWidget.signal_objective_changed.connect( self.wellplateMultiPointWidget.handle_objective_change ) + if ENABLE_RECORDING and self.recordZStackWidget is not None: + # Channel sets are per-objective: repopulate the tab's channel combos + # so stale names don't fall back to a no-illumination bare channel. + self.objectivesWidget.signal_objective_changed.connect(self.recordZStackWidget.refresh_channel_list) self.profileWidget.signal_profile_changed.connect( lambda: self.liveControlWidget.select_new_microscope_mode_by_name( @@ -2395,6 +2399,8 @@ def _refresh_channel_lists(self): self.wellplateMultiPointWidget.refresh_channel_list() if self.multiPointWithFluidicsWidget: self.multiPointWithFluidicsWidget.refresh_channel_list() + if self.recordZStackWidget is not None: + self.recordZStackWidget.refresh_channel_list() def onTabChanged(self, index): is_flexible_acquisition = ( @@ -2896,6 +2902,15 @@ def _cleanup_common(self, for_restart: bool = False): except Exception: self.log.exception(f"Error closing multipoint controller during {context}") + # Clean up record+z-stack controller: aborts any running acquisition and + # shuts down its JobRunner subprocess so Zarr writers finalize instead of + # being killed mid-write (corrupted store). + if getattr(self, "recordZStackController", None) is not None: + try: + self.recordZStackController.close() + except Exception: + self.log.exception(f"Error closing record z-stack controller during {context}") + # Clean up NDViewer if self.ndviewerTab is not None: try: diff --git a/software/control/widgets.py b/software/control/widgets.py index bd8c14c77..d7e99cda0 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17727,6 +17727,13 @@ def _make_channel_base(name: str) -> AcquisitionChannel: return ch.model_copy(deep=True) except Exception: pass + # The fallback has no illumination-source mapping, so the acquisition + # would run dark — warn loudly so the cause is diagnosable. + self._log.warning( + f"channel {name!r} not found for objective " + f"{getattr(self.objectiveStore, 'current_objective', '?')!r}; " + f"using a bare fallback with no illumination mapping (images may be dark)" + ) return AcquisitionChannel( name=name, camera_settings=CameraSettings(exposure_time_ms=50.0, gain_mode=0.0), @@ -17836,6 +17843,28 @@ def acquisition_is_finished(self): self.btn_startAcquisition.setChecked(False) self.signal_acquisition_started.emit(False) + def refresh_channel_list(self) -> None: + """Repopulate the channel combos from liveController. + + Channel sets are per-objective (and per-profile): after an objective or + profile change, stale names would silently fall back to a bare-bones + channel with no illumination source and acquire dark images. Mirrors + WellplateMultiPointWidget.refresh_channel_list. The previous recording + selection is kept when it still exists; z-stack rows whose channel no + longer exists are removed. + """ + prev_recording = self._recording_channel_name() + self._populate_channel_combo(self._recording_ch_combo) + if prev_recording: + idx = self._recording_ch_combo.findText(prev_recording) + if idx >= 0: + self._recording_ch_combo.setCurrentIndex(idx) + self._populate_channel_combo(self.combobox_zstack_add_channel) + available = {self._recording_ch_combo.itemText(i) for i in range(self._recording_ch_combo.count())} + for name in [n for n in self._zstack_channel_names if n not in available]: + self._log.info(f"removing z-stack channel row {name!r}: not available for the current objective") + self._remove_zstack_channel_row(name) + def emit_selected_channels(self) -> None: """No-op stub: RecordZStackMultiPointWidget has no channel list to broadcast. diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index 6f44164b7..353e6a19e 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -501,3 +501,102 @@ def test_recording_uses_achievable_fps_when_camera_clamps(tmp_path): f"time_increment_s={squid_attrs['time_increment_s']} does not match the " f"achievable rate (1/{achievable:.2f})" ) + + +def test_move_xy_honors_z_component(): + """F11: (x, y, z) scan coordinates carry a stored per-FOV focus plane + (flexible regions, update_fov_z); dropping z means recording/z-stacking at + the previous FOV's focus on tilted samples.""" + from types import SimpleNamespace + from unittest.mock import MagicMock + + from control.core.record_zstack_worker import RecordZStackWorker + + stage = MagicMock() + fake_self = SimpleNamespace(stage=stage, _sleep=lambda s: None, wait_till_operation_is_completed=lambda: None) + RecordZStackWorker._move_xy(fake_self, (1.0, 2.0, 3.5)) + stage.move_x_to.assert_called_once_with(1.0) + stage.move_y_to.assert_called_once_with(2.0) + stage.move_z_to.assert_called_once_with(3.5) + + +def test_move_xy_two_tuple_does_not_move_z(): + from types import SimpleNamespace + from unittest.mock import MagicMock + + from control.core.record_zstack_worker import RecordZStackWorker + + stage = MagicMock() + fake_self = SimpleNamespace(stage=stage, _sleep=lambda s: None, wait_till_operation_is_completed=lambda: None) + RecordZStackWorker._move_xy(fake_self, (1.0, 2.0)) + stage.move_z_to.assert_not_called() + + +def test_wait_for_dt_paces_from_acquisition_start(tmp_path): + """F12: dt is the interval between timepoint STARTS (t0 + k*dt, matching + MultiPointWorker and the recorded time_increment_s metadata), not a wait + appended after each timepoint's work.""" + import time as _time + + pytest.importorskip("tensorstore") + scope, live_controller, channels, worker, aborted = _build_worker_harness( + tmp_path, recording_enabled=True, zstack_enabled=False + ) + worker.params.dt_s = 2.0 + worker._acq_start_time = _time.time() - 100.0 # the work already overran the interval + + t0 = _time.monotonic() + ok = worker._wait_for_dt(1) + took = _time.monotonic() - t0 + + assert ok + assert took < 1.0, f"_wait_for_dt slept {took:.1f}s though t=1's start time is long past" + + +def test_controller_writes_config_snapshot_and_done_file(tmp_path): + """F15: every experiment dir must carry the settings snapshot + (acquisition_channels.yaml) and completion marker (.done) that every + multipoint acquisition produces — downstream watchers depend on both.""" + pytest.importorskip("tensorstore") + import control._def + import tests.control.test_stubs as ts + from control.core.multi_point_controller import NoOpCallbacks + from control.core.record_zstack_controller import ( + RecordZStackAcquisitionParameters, + RecordZStackController, + ) + + control._def.FILE_SAVING_OPTION = control._def.FileSavingOption.ZARR_V3 + scope = _build_simulated_microscope(64, 48) + live_controller = ts.get_test_live_controller(scope, scope.objective_store.default_objective) + laser_af = ts.get_test_laser_autofocus_controller(scope) + channels = live_controller.get_channels(scope.objective_store.default_objective) + scope.camera.set_exposure_time(1) + + controller = RecordZStackController( + microscope=scope, + live_controller=live_controller, + laser_autofocus_controller=laser_af, + objective_store=scope.objective_store, + scan_coordinates=None, # no FOVs — completion bookkeeping is what's under test + callbacks=NoOpCallbacks, + ) + params = RecordZStackAcquisitionParameters( + base_path=str(tmp_path), + experiment_id="bookkeeping", + Nt=1, + recording_enabled=True, + recording_channel=channels[0], + fps=10.0, + duration_s=0.2, + ) + try: + controller.run_acquisition(params) + controller.join(timeout=60) + finally: + controller.close() + + exp = Path(params.base_path) / params.experiment_id + assert exp.is_dir() + assert (exp / "acquisition_channels.yaml").exists(), "settings snapshot missing from experiment dir" + assert (exp / ".done").exists(), "completion marker missing from experiment dir" diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py index dc09ced96..ac65bdbb2 100644 --- a/software/tests/core/test_streaming_capture.py +++ b/software/tests/core/test_streaming_capture.py @@ -505,3 +505,27 @@ def test_recording_router_still_downsamples_faster_camera(): stamps = [100.0 + i * 0.05 for i in range(20)] # 20 fps camera, 10 fps target accepted = [s for s in stamps if r.route(s) is not None] assert len(accepted) == 10, f"expected 10/20 accepted, got {len(accepted)}" + + +def test_recording_writer_byte_bound_drops(tmp_path, monkeypatch): + """F7: the queue must bound MEMORY, not just frame count — 256 full-res + 16-bit frames is ~13 GB. Frames beyond max_bytes drop like a full queue.""" + import threading + from control.core.zarr_writer import ZarrWriter + from control.core.streaming_capture import RecordingWriter + + release = threading.Event() + monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: release.wait(20)) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "abort", lambda self: None) + + frame = np.zeros((100, 100), np.uint16) # 20 kB each + rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=256, max_bytes=50_000) + rw.start() + for i in range(6): # 120 kB total >> 50 kB cap; first frame wedges in write + rw.enqueue(frame, i, 0, 0) + dropped = rw.dropped_count + release.set() + rw.finalize(timeout_s=2.0) + assert dropped >= 3, f"byte cap not enforced: only {dropped} frames dropped" diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 26cd0d3af..e725b9d31 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -964,3 +964,37 @@ def test_emit_selected_channels_is_a_safe_noop(qtbot, simulated_widget_deps): qtbot.addWidget(w) w.emit_selected_channels() # must not raise + + +def test_refresh_channel_list_repopulates_combos(qtbot, simulated_widget_deps): + """Channel sets are per-objective: after an objective/profile change the + combos must repopulate, or stale names silently fall back to a bare + channel with no illumination source (dark acquisition).""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.liveController.get_channels.return_value = [ + _make_channel("New Channel A"), + _make_channel("New Channel B"), + ] + w.refresh_channel_list() + + rec_names = [w._recording_ch_combo.itemText(i) for i in range(w._recording_ch_combo.count())] + add_names = [w.combobox_zstack_add_channel.itemText(i) for i in range(w.combobox_zstack_add_channel.count())] + assert rec_names == ["New Channel A", "New Channel B"] + assert add_names == ["New Channel A", "New Channel B"] + + +def test_refresh_channel_list_drops_stale_zstack_rows(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w._add_zstack_channel_row("BF LED matrix full") # valid for the old objective + + w.liveController.get_channels.return_value = [_make_channel("New Channel A")] + w.refresh_channel_list() + + assert "BF LED matrix full" not in w._zstack_channel_names From b745fa801cbf938905fca2c675942afaebe2a5b6 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 00:47:36 -0400 Subject: [PATCH 38/61] fix(record-zstack): round-2 review fixes on the fix commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inline-verified findings from the second review pass (the workflow's verify phase hit API limits; candidates were verified by hand): - Dropped frames (byte cap / queue full) no longer seal the store acquisition_complete=True: StreamingCapture counts routed frames, so the stop condition was satisfied even when enqueue dropped them and the completeness attribute lied over fill-value planes. Any drop or write error now seals incomplete. - finalize() exposes finalize_wedged; the worker's fail-fast treats a wedged drain as failure (write_error_count reads 0 when finalize returns before the errors happen, so the check alone was bypassable — a stalled disk would have burned the timeout at all remaining FOVs). - finalize()'s sentinel put and join share one timeout budget (a late sentinel followed by a full-length join blocked callers ~2x). - Timepoint pacing uses time.monotonic() (NTP steps skewed wall-clock pacing) and skips slots that already passed, mirroring MultiPointWorker's grid-preserving skip; dt=0 remains continuous. - refresh_channel_list fetches before clearing: a transient get_channels failure (or empty set) no longer wipes the user's configured z-stack rows and combos. - Profile changes now also refresh the tab's channel combos (was objective-only; profile switches replace the channel set too). - _probe_frame_shape rejects color (Y,X,3) frames with a clear error instead of building a dataset every write fails against. - zstack() no longer restores the trigger mode per FOV — run() restores once at the end, removing 2 camera mode switches + 2 MCU commands per FOV. - Hot-path drop warnings are rate-limited (first + every 100th) — per-frame logging at fps rate on the camera delivery thread. Deferred (documented in the review doc): NTP-step handling in RecordingRouter's absolute-slot anchor, a GUI-visible error dialog for fail-fast aborts, folding the achievable-fps probe into ContinuousFrameSource, and app-exit-vs-running-worker hardware races. TDD: 6 new tests watched failing first; controller smoke updated to dt=0 (grid-skip semantics make dt --- software/control/core/record_zstack_worker.py | 87 +++++++++++-------- software/control/core/streaming_capture.py | 47 +++++++--- software/control/gui_hcs.py | 6 +- software/control/widgets.py | 31 +++++-- .../tests/core/test_record_zstack_worker.py | 65 +++++++++++++- software/tests/core/test_streaming_capture.py | 79 +++++++++++++++++ software/tests/test_record_zstack_widget.py | 23 +++++ 7 files changed, 280 insertions(+), 58 deletions(-) diff --git a/software/control/core/record_zstack_worker.py b/software/control/core/record_zstack_worker.py index 27bf67db8..8d609ea36 100644 --- a/software/control/core/record_zstack_worker.py +++ b/software/control/core/record_zstack_worker.py @@ -253,12 +253,14 @@ def run(self): if self.params.zstack_enabled and self._job_runners: self._backpressure.reset() - self._acq_start_time = time.time() + self._acq_start_time = time.monotonic() for t_idx in range(self.params.Nt): self.time_point = t_idx - if t_idx > 0 and self.params.dt_s > 0: - if not self._wait_for_dt(t_idx): - break + action = self._pace_timepoint(t_idx) + if action == "skip": + continue + if action == "abort": + break if self.abort_requested_fn(): break @@ -421,13 +423,17 @@ def record(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> int: self.stage.move_z_to(z_ref) self.wait_till_operation_is_completed() self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) - # Fail fast on write errors: the cause is almost always systematic (full - # disk, shape mismatch), so continuing would record blank data at every - # remaining FOV. run() catches this, aborts, and signals finished. - if writer.write_error_count > 0: + # Fail fast on write errors OR a wedged drain thread: either way the + # cause is almost always systematic (full disk, stalled mount), so + # continuing would burn the timeout at every remaining FOV producing + # blank data. A wedged finalize returns before errors are countable, + # which is why write_error_count alone is not sufficient. + # run() catches this, aborts, and signals finished. + if writer.write_error_count > 0 or writer.finalize_wedged: raise RuntimeError( - f"{writer.write_error_count} recording write error(s) at t={t_idx} " - f"region={region_id} fov={fov_idx}; store sealed incomplete: {out}" + f"recording failed at t={t_idx} region={region_id} fov={fov_idx} " + f"(write errors={writer.write_error_count}, drain wedged={writer.finalize_wedged}); " + f"store sealed incomplete: {out}" ) return emitted @@ -444,11 +450,11 @@ def zstack(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> None: # The inherited capture path (acquire_camera_image) branches on # liveController.trigger_mode, so the LiveController and the camera must agree # on software-trigger mode. set_trigger_mode(SOFTWARE) sets both the camera - # acquisition mode and the microcontroller trigger mode; capture the previous - # LiveController mode and restore it in the finally. Manage the streaming - # lifecycle locally so it never interferes with the recording phase's - # CONTINUOUS streaming. - prev_trigger_mode = getattr(self.liveController, "trigger_mode", None) + # acquisition mode and the microcontroller trigger mode. No per-FOV restore: + # run()'s finally restores the user's trigger mode once at the end of the + # acquisition — restoring per FOV would flip-flop the camera and MCU 2x per + # FOV for no benefit. Manage the streaming lifecycle locally so it never + # interferes with the recording phase's CONTINUOUS streaming. try: self.liveController.set_trigger_mode(TriggerMode.SOFTWARE) except Exception: @@ -499,12 +505,6 @@ def zstack(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> None: self.camera.stop_streaming() except Exception: log.exception("Failed to stop streaming after z-stack") - # Restore the LiveController trigger mode captured before the z-stack. - if prev_trigger_mode is not None and prev_trigger_mode != TriggerMode.SOFTWARE: - try: - self.liveController.set_trigger_mode(prev_trigger_mode) - except Exception: - log.exception("Failed to restore LiveController trigger mode after z-stack") self.stage.move_z_to(z_ref) self.wait_till_operation_is_completed() self._sleep(SCAN_STABILIZATION_TIME_MS_Z / 1000) @@ -556,6 +556,13 @@ def _probe_frame_shape(self) -> Tuple[int, int, np.dtype]: except Exception: log.exception("failed to stop streaming after probe frame") if frame is not None: + if frame.ndim != 2: + # A color frame (Y, X, 3) would silently produce a 2-D dataset + # that every write then fails against — reject it up front. + raise ValueError( + f"recording supports monochrome frames only; camera delivered shape {frame.shape} " + f"(set the camera to a mono pixel format for the recording phase)" + ) return int(frame.shape[0]), int(frame.shape[1]), frame.dtype log.warning("sizing recording dataset from get_resolution(); may mismatch delivered frames") width, height = self.camera.get_resolution() @@ -588,24 +595,36 @@ def _zstack_dir(self, region_id) -> str: path = os.path.join(self.experiment_path, "zstack", str(region_id)) return path + def _pace_timepoint(self, t_idx: int) -> str: + """Decide how to handle time point ``t_idx``: 'run', 'skip', or 'abort'. + + Starts are paced on the absolute grid ``acquisition_start + t_idx*dt``. + A slot whose start already passed is SKIPPED (grid-preserving, mirrors + MultiPointWorker's skip loop) — running it late would silently stretch + the real sampling interval while the recorded ``time_increment_s`` + metadata still claims ``dt``. + """ + if t_idx == 0 or self.params.dt_s <= 0: + return "run" + start = self._acq_start_time if self._acq_start_time is not None else time.monotonic() + if time.monotonic() > start + t_idx * self.params.dt_s: + log.warning( + f"skipping time point {t_idx}: per-timepoint work exceeded dt={self.params.dt_s:g}s " + f"(grid-preserving skip, mirrors MultiPointWorker)" + ) + return "skip" + return "run" if self._wait_for_dt(t_idx) else "abort" + def _wait_for_dt(self, t_idx: int) -> bool: """Sleep until time point ``t_idx``'s scheduled start (abort-aware). - Starts are paced at ``acquisition_start + t_idx * dt`` (matching - MultiPointWorker and the recorded ``time_increment_s`` metadata) — - NOT ``dt`` after the previous timepoint's work finished, which would - stretch the real sampling interval to ``dt + work`` while the metadata - still claimed ``dt``. Returns False if an abort was requested. + Uses time.monotonic() so an NTP clock step mid-acquisition cannot skew + or collapse the remaining intervals. Returns False on abort. """ - start = self._acq_start_time if self._acq_start_time is not None else time.time() + start = self._acq_start_time if self._acq_start_time is not None else time.monotonic() deadline = start + t_idx * self.params.dt_s - if time.time() > deadline: - log.warning( - f"time point {t_idx} starting late: per-timepoint work exceeded dt={self.params.dt_s:g}s; " - f"recorded time_increment_s underestimates the real interval" - ) - while time.time() < deadline: + while time.monotonic() < deadline: if self.abort_requested_fn(): return False - self._sleep(min(0.1, max(0.0, deadline - time.time()))) + self._sleep(min(0.1, max(0.0, deadline - time.monotonic()))) return True diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index 0dde47678..a19458964 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -79,6 +79,10 @@ def __init__(self, config: ZarrAcquisitionConfig, max_queue: int = 256, max_byte self._held_bytes = 0 self._bytes_lock = threading.Lock() self._abort_requested = threading.Event() + # True when finalize() gave up on a wedged drain thread: write errors + # may still be accruing after finalize() returns, so callers must not + # trust write_error_count == 0 as "healthy" — check this flag too. + self._finalize_wedged = False # True only once the drain thread has actually been started. finalize()/ # abort() must not join (or push the sentinel to) a thread that never # started, otherwise a failure in start()'s initialize() would surface as @@ -117,10 +121,13 @@ def enqueue(self, frame: np.ndarray, t: int, c: int, z: int) -> None: self._held_bytes += nbytes if over_cap: self._dropped += 1 - _log.warning( - f"recording byte cap reached ({self._max_bytes} B); dropped frame t={t} " - f"(total dropped={self._dropped})" - ) + # Rate-limited: this runs on the hot camera delivery thread, and a + # stalled disk would otherwise log (and do handler I/O) at fps rate. + if self._dropped == 1 or self._dropped % 100 == 0: + _log.warning( + f"recording byte cap reached ({self._max_bytes} B); dropped frame t={t} " + f"(total dropped={self._dropped})" + ) return try: self._q.put_nowait((frame, t, c, z)) @@ -128,7 +135,8 @@ def enqueue(self, frame: np.ndarray, t: int, c: int, z: int) -> None: with self._bytes_lock: self._held_bytes -= nbytes self._dropped += 1 - _log.warning(f"recording queue full; dropped frame t={t} (total dropped={self._dropped})") + if self._dropped == 1 or self._dropped % 100 == 0: + _log.warning(f"recording queue full; dropped frame t={t} (total dropped={self._dropped})") def _drain(self) -> None: """Background thread: sole owner of ZarrWriter after start(). @@ -158,13 +166,16 @@ def _drain(self) -> None: finally: if self._abort_requested.is_set(): self._writer.abort() - elif self._write_errors > 0: + elif self._write_errors > 0 or self._dropped > 0: # Never stamp acquisition_complete=True on a store with failed - # writes: the missing planes are silent fill values. abort() - # seals it acquisition_complete=False so readers can tell. + # writes OR dropped frames: either way some planes are silent + # fill values (StreamingCapture counts routed frames, so its + # stop condition is satisfied even when enqueue dropped them). + # abort() seals acquisition_complete=False so readers can tell; + # the queue was already drained, so no captured data is lost. _log.error( - f"{self._write_errors} write error(s) during recording; " - f"sealing store as incomplete instead of complete" + f"recording store sealed INCOMPLETE: {self._write_errors} write error(s), " + f"{self._dropped} dropped frame(s)" ) self._writer.abort() else: @@ -180,6 +191,15 @@ def write_error_count(self) -> int: """Total write_frame failures on the drain thread (0 on a healthy run).""" return self._write_errors + @property + def finalize_wedged(self) -> bool: + """True if finalize() gave up on a wedged drain thread. + + In that state write_error_count may still be 0 (the errors happen after + finalize() returned), so fail-fast callers must treat wedged as failure. + """ + return self._finalize_wedged + def finalize(self, timeout_s: float = 30.0) -> None: """Flush the queue, join the drain thread (which seals the ZarrWriter). @@ -205,10 +225,15 @@ def finalize(self, timeout_s: float = 30.0) -> None: f"drain thread wedged (queue full for {timeout_s:.0f}s); " f"switching to abort — recording will be sealed incomplete" ) + self._finalize_wedged = True self._abort_requested.set() return - self._thread.join(timeout=timeout_s) + # The put and the join share ONE budget: a sentinel accepted late must + # not be followed by a fresh full-length join (callers would block for + # up to ~2x timeout_s). + self._thread.join(timeout=max(0.1, deadline - time.monotonic())) if self._thread.is_alive(): + self._finalize_wedged = True _log.warning("drain thread still alive after finalize() join timeout") def abort(self) -> None: diff --git a/software/control/gui_hcs.py b/software/control/gui_hcs.py index 30e50cfd5..75ec5825f 100644 --- a/software/control/gui_hcs.py +++ b/software/control/gui_hcs.py @@ -1651,9 +1651,11 @@ def make_connections(self): self.wellplateMultiPointWidget.handle_objective_change ) if ENABLE_RECORDING and self.recordZStackWidget is not None: - # Channel sets are per-objective: repopulate the tab's channel combos - # so stale names don't fall back to a no-illumination bare channel. + # Channel sets are per-objective AND per-profile: repopulate the tab's + # channel combos so stale names don't fall back to a no-illumination + # bare channel (dark acquisition). self.objectivesWidget.signal_objective_changed.connect(self.recordZStackWidget.refresh_channel_list) + self.profileWidget.signal_profile_changed.connect(self.recordZStackWidget.refresh_channel_list) self.profileWidget.signal_profile_changed.connect( lambda: self.liveControlWidget.select_new_microscope_mode_by_name( diff --git a/software/control/widgets.py b/software/control/widgets.py index d7e99cda0..2b3390f6c 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17853,15 +17853,30 @@ def refresh_channel_list(self) -> None: selection is kept when it still exists; z-stack rows whose channel no longer exists are removed. """ + # Fetch FIRST and bail out on failure or an empty result: clearing the + # combos before a failed fetch would leave them empty and the stale-row + # pruning below would then wipe every configured z-stack row (with the + # user's per-row exposure/gain/illumination edits) on a transient error. + try: + channels = self.liveController.get_channels(self.objectiveStore.current_objective) + except Exception as exc: + self._log.warning(f"refresh_channel_list: get_channels failed; keeping existing lists: {exc}") + return + if not channels: + self._log.warning( + "refresh_channel_list: no channels for the current objective/profile; keeping existing lists" + ) + return + names = [ch.name for ch in channels] + prev_recording = self._recording_channel_name() - self._populate_channel_combo(self._recording_ch_combo) - if prev_recording: - idx = self._recording_ch_combo.findText(prev_recording) - if idx >= 0: - self._recording_ch_combo.setCurrentIndex(idx) - self._populate_channel_combo(self.combobox_zstack_add_channel) - available = {self._recording_ch_combo.itemText(i) for i in range(self._recording_ch_combo.count())} - for name in [n for n in self._zstack_channel_names if n not in available]: + self._recording_ch_combo.clear() + self._recording_ch_combo.addItems(names) + if prev_recording and prev_recording in names: + self._recording_ch_combo.setCurrentIndex(names.index(prev_recording)) + self.combobox_zstack_add_channel.clear() + self.combobox_zstack_add_channel.addItems(names) + for name in [n for n in self._zstack_channel_names if n not in names]: self._log.info(f"removing z-stack channel row {name!r}: not available for the current objective") self._remove_zstack_channel_row(name) diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index 353e6a19e..cdf672455 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -165,7 +165,9 @@ def test_record_zstack_worker_smoke(tmp_path): # # Same 2-well x 2-FOV geometry as the D2 test but driven through # RecordZStackController.run_acquisition() + join(). -# Also exercises Nt=2 with a short dt_s=0.1 (two time-points). +# Also exercises Nt=2 with dt_s=0.0 (continuous: no grid pacing, so both +# time-points run even though the per-timepoint work exceeds any short dt — +# with dt>0 the worker now SKIPS missed slots, mirroring MultiPointWorker). # --------------------------------------------------------------------------- @@ -238,7 +240,7 @@ def test_record_zstack_controller_smoke(tmp_path): base_path=str(tmp_path), experiment_id="ctrl_smoke", Nt=Nt, - dt_s=0.1, # short inter-timepoint delay + dt_s=0.0, # continuous: dt>0 would skip slots missed while working use_laser_af=False, recording_enabled=True, recording_channel=recording_channel, @@ -543,7 +545,7 @@ def test_wait_for_dt_paces_from_acquisition_start(tmp_path): tmp_path, recording_enabled=True, zstack_enabled=False ) worker.params.dt_s = 2.0 - worker._acq_start_time = _time.time() - 100.0 # the work already overran the interval + worker._acq_start_time = _time.monotonic() - 100.0 # the work already overran the interval t0 = _time.monotonic() ok = worker._wait_for_dt(1) @@ -600,3 +602,60 @@ def test_controller_writes_config_snapshot_and_done_file(tmp_path): assert exp.is_dir() assert (exp / "acquisition_channels.yaml").exists(), "settings snapshot missing from experiment dir" assert (exp / ".done").exists(), "completion marker missing from experiment dir" + + +def test_pace_timepoint_skips_missed_slots(tmp_path): + """Round-2 parity: MultiPointWorker skips timepoints whose slot already + passed (grid-preserving); running them late back-to-back silently breaks + the recorded time_increment_s for the rest of the run.""" + import time as _time + + pytest.importorskip("tensorstore") + scope, live_controller, channels, worker, aborted = _build_worker_harness( + tmp_path, recording_enabled=True, zstack_enabled=False + ) + worker.params.dt_s = 2.0 + + worker._acq_start_time = _time.monotonic() - 100.0 + assert worker._pace_timepoint(1) == "skip" + + worker._acq_start_time = _time.monotonic() + assert worker._pace_timepoint(0) == "run" + + aborted["v"] = True + assert worker._pace_timepoint(1) in ("abort", "skip") # never 'run' while aborted + + +def test_probe_frame_shape_rejects_color_frames(): + """Round-2: a color (Y,X,3) probe frame silently produced a 2-D dataset + that every write then failed against; reject it with a clear error.""" + from types import SimpleNamespace + + import numpy as np + + from control.core.record_zstack_worker import RecordZStackWorker + + color = np.zeros((50, 60, 3), dtype=np.uint8) + + class _FakeColorCamera: + def get_resolution(self): + return (60, 50) + + def set_acquisition_mode(self, mode): + pass + + def start_streaming(self): + pass + + def stop_streaming(self): + pass + + def send_trigger(self): + pass + + def read_camera_frame(self): + return SimpleNamespace(frame=color) + + fake_self = SimpleNamespace(camera=_FakeColorCamera()) + with pytest.raises(ValueError, match="monochrome"): + RecordZStackWorker._probe_frame_shape(fake_self) diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py index ac65bdbb2..11364afbb 100644 --- a/software/tests/core/test_streaming_capture.py +++ b/software/tests/core/test_streaming_capture.py @@ -529,3 +529,82 @@ def test_recording_writer_byte_bound_drops(tmp_path, monkeypatch): release.set() rw.finalize(timeout_s=2.0) assert dropped >= 3, f"byte cap not enforced: only {dropped} frames dropped" + + +def test_dropped_frames_seal_store_incomplete(tmp_path, monkeypatch): + """Round-2: frames dropped by backpressure leave fill-value holes, so the + store must NOT be sealed acquisition_complete=True (CountStop still counts + routed frames, so the completeness attribute was lying).""" + import threading + from control.core.zarr_writer import ZarrWriter + from control.core.streaming_capture import RecordingWriter + + release = threading.Event() + calls = [] + monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: release.wait(10)) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self: calls.append("finalize")) + monkeypatch.setattr(ZarrWriter, "abort", lambda self: calls.append("abort")) + + rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=1) + rw.start() + for i in range(4): # first wedges in write, second queues, rest drop + rw.enqueue(np.zeros((4, 4), np.uint16), i, 0, 0) + assert rw.dropped_count > 0 + release.set() + rw.finalize(timeout_s=5.0) + + assert calls == ["abort"], f"store with dropped frames sealed via {calls}, expected incomplete seal" + assert rw.finalize_wedged is False + + +def test_finalize_wedged_flag_feeds_fail_fast(tmp_path, monkeypatch): + """Round-2: when finalize() takes the wedged-drain fallback it returns with + the drain thread still running — the caller's write_error_count check reads + 0 and the fail-fast never fires. The writer must expose the wedged state.""" + import threading + from control.core.zarr_writer import ZarrWriter + from control.core.streaming_capture import RecordingWriter + + release = threading.Event() + monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: release.wait(20)) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "abort", lambda self: None) + + rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=2) + rw.start() + for i in range(3): + rw.enqueue(np.zeros((4, 4), np.uint16), i, 0, 0) + rw.finalize(timeout_s=1.0) # sentinel can't be queued -> wedged fallback + still_wedged = rw.finalize_wedged + release.set() + assert still_wedged is True + + +def test_finalize_total_time_respects_timeout_budget(tmp_path, monkeypatch): + """Round-2: the sentinel put and the join must share ONE timeout budget — + a late-accepted sentinel followed by a full-length join blocked callers + for up to ~2x timeout_s.""" + import time as _time + from control.core.zarr_writer import ZarrWriter + from control.core.streaming_capture import RecordingWriter + + monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: _time.sleep(0.95)) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "abort", lambda self: None) + + rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=2) + rw.start() + # f0 wedges the drain mid-write; f1+f2 then fill the queue, so the sentinel + # is accepted LATE (~0.75s into the 1.0s put deadline) — the join must use + # only the remaining budget, not a fresh full timeout. + rw.enqueue(np.zeros((4, 4), np.uint16), 0, 0, 0) + _time.sleep(0.2) # let the drain thread take f0 and start the slow write + rw.enqueue(np.zeros((4, 4), np.uint16), 1, 0, 0) + rw.enqueue(np.zeros((4, 4), np.uint16), 2, 0, 0) + t0 = _time.monotonic() + rw.finalize(timeout_s=1.0) + took = _time.monotonic() - t0 + assert took < 1.5, f"finalize(timeout_s=1.0) blocked {took:.1f}s — put+join must share one budget" diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index e725b9d31..f7d1b1f27 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -998,3 +998,26 @@ def test_refresh_channel_list_drops_stale_zstack_rows(qtbot, simulated_widget_de w.refresh_channel_list() assert "BF LED matrix full" not in w._zstack_channel_names + + +def test_refresh_channel_list_preserves_state_on_failure(qtbot, simulated_widget_deps): + """Round-2: a transient get_channels failure (or empty result) during an + objective/profile switch must NOT wipe the user's configured z-stack rows + and channel combos — keep the existing lists until a successful refresh.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w._add_zstack_channel_row("BF LED matrix full") + combo_count_before = w._recording_ch_combo.count() + + w.liveController.get_channels.side_effect = RuntimeError("config repo hiccup") + w.refresh_channel_list() + assert w._zstack_channel_names == ["BF LED matrix full"], "rows wiped on transient failure" + assert w._recording_ch_combo.count() == combo_count_before, "combo wiped on transient failure" + + w.liveController.get_channels.side_effect = None + w.liveController.get_channels.return_value = [] + w.refresh_channel_list() + assert w._zstack_channel_names == ["BF LED matrix full"], "rows wiped on empty channel list" + assert w._recording_ch_combo.count() == combo_count_before From 9a24c5394ab0d46c6ac635e7c5a3e70766dffb82 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 02:09:21 -0400 Subject: [PATCH 39/61] fix(record-zstack): data-integrity cluster from completed round-2 verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R1-R5 + R9 from the full adversarial verification pass (see worktrees/docs/code-review-record-zstack-streaming-2026-07-03.md): - Under-delivered captures (camera stall, no user abort) no longer seal acquisition_complete=True: StreamingCapture calls mark_incomplete() and the drain seals with captured/expected counts. - record() fails fast on dropped frames too — sustained backpressure drops are the systematic slow-disk case the guard exists for. - RecordingRouter maps frames to the slot nearest their arrival time: a burst after a delivery stall leaves the stall as fill holes instead of back-filling consecutive slots and compressing the time axis; slots past T stop the capture instead of writing out of bounds. - A wedged finalize() now sets flush-and-exit instead of abort: the captured backlog is written whenever the stall clears rather than discarded; join-timeout on a slow-but-progressing drain no longer sets finalize_wedged (false-positive fail-fast aborted whole runs over stores that finished fine). - Non-abort incomplete seals no longer stamp aborted=true: ZarrWriter.abort(mark_aborted=False, extra_attrs=...) writes incomplete=True with error/drop/captured counts so tooling can tell "user pressed Stop" from "finished with missing planes". Wedge-style tests join their drain threads before returning (the class-level ZarrWriter monkeypatches made leftover daemon drains call the next test's stubs — order-dependent flakes). TDD: 6 new tests watched failing first; router/index test updated to the honest-timeline semantics. Co-Authored-By: Claude Fable 5 --- software/control/core/record_zstack_worker.py | 6 +- software/control/core/streaming_capture.py | 114 ++++++++---- software/control/core/zarr_writer.py | 19 +- .../tests/core/test_record_zstack_worker.py | 40 ++++ software/tests/core/test_streaming_capture.py | 176 ++++++++++++++++-- 5 files changed, 297 insertions(+), 58 deletions(-) diff --git a/software/control/core/record_zstack_worker.py b/software/control/core/record_zstack_worker.py index 8d609ea36..847ccbd49 100644 --- a/software/control/core/record_zstack_worker.py +++ b/software/control/core/record_zstack_worker.py @@ -429,11 +429,11 @@ def record(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> int: # blank data. A wedged finalize returns before errors are countable, # which is why write_error_count alone is not sufficient. # run() catches this, aborts, and signals finished. - if writer.write_error_count > 0 or writer.finalize_wedged: + if writer.write_error_count > 0 or writer.finalize_wedged or writer.dropped_count > 0: raise RuntimeError( f"recording failed at t={t_idx} region={region_id} fov={fov_idx} " - f"(write errors={writer.write_error_count}, drain wedged={writer.finalize_wedged}); " - f"store sealed incomplete: {out}" + f"(write errors={writer.write_error_count}, dropped={writer.dropped_count}, " + f"drain wedged={writer.finalize_wedged}); store sealed incomplete: {out}" ) return emitted diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index a19458964..07ba8fc3d 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -26,29 +26,37 @@ def expected(self) -> Optional[int]: class RecordingRouter: - """Maps incoming frames to (t,c,z)=(t_index,0,0), downsampling to `fps`. - - Emission slots are anchored to the FIRST frame's timestamp (slot k opens at - ``first_ts + k*period``), not to the previous emission: with the camera - running at the target rate, host-side delivery jitter around the period - would otherwise reject roughly alternate frames and halve the effective - capture rate. A quarter-period tolerance absorbs per-frame jitter, and - absolute slots self-correct — one late frame never delays later slots, so - the long-run rate stays at ``fps``. + """Maps incoming frames to (t,c,z)=(slot,0,0), downsampling to `fps`. + + Each accepted frame goes to the slot NEAREST its arrival time relative to + the first frame (``slot = round(elapsed / period)``, ties rounding down); + a frame whose nearest slot is already filled is rejected. This anchors + pacing absolutely (host-delivery jitter around the period cannot reject + at-rate frames or halve the capture rate) and, after a delivery stall, a + burst of frames does NOT back-fill the missed slots — the stall stays in + the data as fill-value holes, keeping the time axis honest (the store is + then sealed incomplete by the under-delivery path). """ def __init__(self, fps: float): self._period = 1.0 / fps if fps and fps > 0 else 0.0 - self._t_index = 0 + self._t_index = 0 # next unfilled slot self._first_ts: Optional[float] = None def route(self, timestamp: float) -> Optional[Tuple[int, int, int]]: if self._first_ts is None: self._first_ts = timestamp - elif self._period > 0 and (timestamp - self._first_ts) < (self._t_index - 0.25) * self._period: - return None - idx = (self._t_index, 0, 0) - self._t_index += 1 + slot = 0 + elif self._period > 0: + # Nearest slot; the epsilon makes exact half-period ties round DOWN + # so a 2x-rate camera still has alternate frames rejected. + slot = int((timestamp - self._first_ts) / self._period + 0.5 - 1e-9) + if slot < self._t_index: + return None + else: + slot = self._t_index + idx = (slot, 0, 0) + self._t_index = slot + 1 return idx @@ -83,6 +91,14 @@ def __init__(self, config: ZarrAcquisitionConfig, max_queue: int = 256, max_byte # may still be accruing after finalize() returns, so callers must not # trust write_error_count == 0 as "healthy" — check this flag too. self._finalize_wedged = False + # Set instead of the abort flag when finalize() gives up on a wedged + # drain: the drain writes the remaining CAPTURED frames whenever the + # stall clears, then exits and seals (abort would discard them). + self._flush_and_exit = threading.Event() + # (captured, expected) reported by the capture when frames never + # arrived (camera stall) — drops/errors are counted here, but only the + # capture knows about frames it expected and never saw. + self._incomplete_info: Optional[Tuple[int, int]] = None # True only once the drain thread has actually been started. finalize()/ # abort() must not join (or push the sentinel to) a thread that never # started, otherwise a failure in start()'s initialize() would surface as @@ -151,6 +167,8 @@ def _drain(self) -> None: try: item = self._q.get(timeout=0.1) except queue.Empty: + if self._flush_and_exit.is_set(): + break # backlog flushed after a wedged finalize() continue if item is _SENTINEL: break @@ -166,18 +184,22 @@ def _drain(self) -> None: finally: if self._abort_requested.is_set(): self._writer.abort() - elif self._write_errors > 0 or self._dropped > 0: + elif self._write_errors > 0 or self._dropped > 0 or self._incomplete_info is not None: # Never stamp acquisition_complete=True on a store with failed - # writes OR dropped frames: either way some planes are silent - # fill values (StreamingCapture counts routed frames, so its - # stop condition is satisfied even when enqueue dropped them). - # abort() seals acquisition_complete=False so readers can tell; - # the queue was already drained, so no captured data is lost. - _log.error( - f"recording store sealed INCOMPLETE: {self._write_errors} write error(s), " - f"{self._dropped} dropped frame(s)" - ) - self._writer.abort() + # writes, dropped frames, or frames that never arrived: some + # planes are silent fill values. Seal acquisition_complete=False + # with the counts, but do NOT mark it "aborted" — this was not a + # user abort, and the queue was drained so no captured data is + # lost. + extra = { + "incomplete": True, + "write_errors": self._write_errors, + "dropped_frames": self._dropped, + } + if self._incomplete_info is not None: + extra["captured_frames"], extra["expected_frames"] = self._incomplete_info + _log.error(f"recording store sealed INCOMPLETE: {extra}") + self._writer.abort(mark_aborted=False, extra_attrs=extra) else: self._writer.finalize() @@ -200,6 +222,16 @@ def finalize_wedged(self) -> bool: """ return self._finalize_wedged + def mark_incomplete(self, captured: int, expected: int) -> None: + """Record that the capture under-delivered (frames never arrived). + + Called by StreamingCapture before finalize() when the stop condition + was not met with no user abort — drops and write errors are counted + internally, but only the capture knows about frames it expected and + never saw. Makes the drain seal the store acquisition_complete=False. + """ + self._incomplete_info = (int(captured), int(expected)) + def finalize(self, timeout_s: float = 30.0) -> None: """Flush the queue, join the drain thread (which seals the ZarrWriter). @@ -222,19 +254,24 @@ def finalize(self, timeout_s: float = 30.0) -> None: except queue.Full: if time.monotonic() >= deadline: _log.error( - f"drain thread wedged (queue full for {timeout_s:.0f}s); " - f"switching to abort — recording will be sealed incomplete" + f"drain thread wedged (queue full for {timeout_s:.0f}s); giving up the wait — " + f"the captured backlog will be flushed and sealed whenever the stall clears" ) self._finalize_wedged = True - self._abort_requested.set() + # Flush-and-exit, NOT abort: the queued frames are captured + # data; the daemon drain writes them once the stalled write + # returns, then exits and seals the store. + self._flush_and_exit.set() return # The put and the join share ONE budget: a sentinel accepted late must # not be followed by a fresh full-length join (callers would block for # up to ~2x timeout_s). self._thread.join(timeout=max(0.1, deadline - time.monotonic())) if self._thread.is_alive(): - self._finalize_wedged = True - _log.warning("drain thread still alive after finalize() join timeout") + # Slow but progressing backlog — NOT wedged (flagging it would + # fail-fast whole multi-well runs over stores that finish fine). + # The daemon drain seals the store when it finishes. + _log.warning("drain thread still writing after finalize() join timeout; store seals when it finishes") def abort(self) -> None: """Signal the drain thread to stop (which aborts the ZarrWriter).""" @@ -334,6 +371,12 @@ def _on_frame(self, camera_frame) -> None: return idx = self._router.route(camera_frame.timestamp) if idx is not None: + expected = self._stop.expected() if hasattr(self._stop, "expected") else None + if expected is not None and idx[0] >= expected: + # The router's slot ran past the dataset (a delivery stall + # pushed the timeline beyond T): nothing left to record into. + self._done.set() + return self._writer.enqueue(camera_frame.frame, *idx) self._emitted += 1 if self._stop.met(self._emitted): @@ -373,14 +416,17 @@ def run(self, timeout: Optional[float] = None) -> int: # Aborted mid-capture: seal the recording as incomplete, not complete. self._writer.abort() else: - self._writer.finalize() if expected is not None and self._emitted < expected: - # Stop condition was not met (slow camera / timeout): the trailing - # zarr planes are fill values, not real data. Warn loudly. + # Stop condition was not met (slow camera / stall / timeout): + # some zarr planes are fill values, not real data. Tell the + # writer so the store is sealed acquisition_complete=False. _log.warning( f"streaming capture incomplete: captured {self._emitted}/{expected} " - f"frames; trailing planes are blank fill" + f"frames; missing planes are blank fill" ) + if hasattr(self._writer, "mark_incomplete"): + self._writer.mark_incomplete(self._emitted, expected) + self._writer.finalize() # Surface total dropped frames so slow-disk runs are diagnosable without # grepping individual per-frame warnings. dropped = self._writer.dropped_count if hasattr(self._writer, "dropped_count") else 0 diff --git a/software/control/core/zarr_writer.py b/software/control/core/zarr_writer.py index 5d34bd3e4..e89221a0c 100644 --- a/software/control/core/zarr_writer.py +++ b/software/control/core/zarr_writer.py @@ -779,13 +779,21 @@ def finalize(self) -> None: self._cleanup_event_loop() log.info(f"Zarr v3 dataset finalized: {self._config.output_path}") - def abort(self) -> None: - """Abort and clean up (blocking). + def abort(self, mark_aborted: bool = True, extra_attrs: Optional[Dict[str, object]] = None) -> None: + """Seal the store as incomplete and clean up (blocking). + + Args: + mark_aborted: stamp ``aborted: True`` (a user/programmatic abort). + Pass False for non-abort incomplete seals (write errors, + dropped frames, under-delivery) so tooling can distinguish + "user pressed Stop" from "finished with missing planes". + extra_attrs: extra keys merged into ``_squid`` (e.g. error/drop + counts) alongside ``acquisition_complete: False``. Uses try-finally to ensure cleanup always happens, even if an unexpected exception occurs during abort. """ - log.warning("Aborting Zarr writer...") + log.warning("Aborting Zarr writer..." if mark_aborted else "Sealing Zarr writer as incomplete...") try: # Clear pending futures (don't wait for them) @@ -800,7 +808,10 @@ def abort(self) -> None: attrs = zarr_json.get("attributes", {}) if "_squid" in attrs: attrs["_squid"]["acquisition_complete"] = False - attrs["_squid"]["aborted"] = True + if mark_aborted: + attrs["_squid"]["aborted"] = True + if extra_attrs: + attrs["_squid"].update(extra_attrs) zarr_json["attributes"] = attrs with open(zarr_json_path, "w") as f: json.dump(zarr_json, f, indent=2) diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index cdf672455..d01d6dac6 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -659,3 +659,43 @@ def read_camera_frame(self): fake_self = SimpleNamespace(camera=_FakeColorCamera()) with pytest.raises(ValueError, match="monochrome"): RecordZStackWorker._probe_frame_shape(fake_self) + + +def test_record_fails_fast_on_dropped_frames(tmp_path, monkeypatch): + """R2: sustained backpressure drops are the systematic slow-disk condition + the fail-fast was written for — record() must abort the acquisition, not + grind through hours of half-blank FOVs.""" + pytest.importorskip("tensorstore") + import control.core.record_zstack_worker as worker_mod + + class _DroppyWriter: + """Stand-in RecordingWriter that reports backpressure drops.""" + + def __init__(self, cfg, **kwargs): + self.dropped_count = 5 + self.write_error_count = 0 + self.finalize_wedged = False + + def start(self): + pass + + def enqueue(self, frame, t, c, z): + pass + + def mark_incomplete(self, captured, expected): + pass + + def finalize(self, timeout_s=30.0): + pass + + def abort(self): + pass + + monkeypatch.setattr(worker_mod, "RecordingWriter", _DroppyWriter) + + scope, live_controller, channels, worker, aborted = _build_worker_harness( + tmp_path, recording_enabled=True, zstack_enabled=False + ) + worker.run() + + assert aborted["v"] is True, "acquisition continued despite dropped recording frames" diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py index 11364afbb..f4b173c2e 100644 --- a/software/tests/core/test_streaming_capture.py +++ b/software/tests/core/test_streaming_capture.py @@ -15,7 +15,9 @@ def test_recording_router_downsamples_and_indexes(): assert r.route(100.00) == (0, 0, 0) # first frame always emits assert r.route(100.05) is None # 50 ms later -> skip assert r.route(100.10) == (1, 0, 0) # 100 ms later -> emit, t=1 - assert r.route(100.30) == (2, 0, 0) # emit, t=2 + # 300 ms after the anchor -> slot 3 (slot 2 stays a fill hole: frames map + # to the slot matching their arrival time; gaps are not compressed). + assert r.route(100.30) == (3, 0, 0) def test_recording_writer_roundtrip(tmp_path): @@ -417,8 +419,8 @@ def boom(self, image, t, c, z, fov=None): raise RuntimeError("disk full") monkeypatch.setattr(ZarrWriter, "write_frame", boom) - monkeypatch.setattr(ZarrWriter, "finalize", lambda self: calls.append("finalize")) - monkeypatch.setattr(ZarrWriter, "abort", lambda self: calls.append("abort")) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self, *a, **k: calls.append("finalize")) + monkeypatch.setattr(ZarrWriter, "abort", lambda self, *a, **k: calls.append("abort")) rw = RecordingWriter(_stub_zarr_cfg(tmp_path)) rw.start() @@ -440,20 +442,25 @@ def test_recording_writer_finalize_bounded_when_drain_wedged(tmp_path, monkeypat release = threading.Event() monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: release.wait(20)) - monkeypatch.setattr(ZarrWriter, "finalize", lambda self: None) - monkeypatch.setattr(ZarrWriter, "abort", lambda self: None) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self, *a, **k: None) + monkeypatch.setattr(ZarrWriter, "abort", lambda self, *a, **k: None) rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=2) rw.start() # First frame wedges the drain thread inside write_frame; two more fill the queue. - for i in range(3): - rw.enqueue(np.zeros((4, 4), np.uint16), i, 0, 0) + rw.enqueue(np.zeros((4, 4), np.uint16), 0, 0, 0) + import time as _t + + _t.sleep(0.2) # let the drain take f0 so f1+f2 truly fill the queue + rw.enqueue(np.zeros((4, 4), np.uint16), 1, 0, 0) + rw.enqueue(np.zeros((4, 4), np.uint16), 2, 0, 0) t = threading.Thread(target=lambda: rw.finalize(timeout_s=1.0), daemon=True) t.start() t.join(timeout=5.0) still_stuck = t.is_alive() release.set() # let the drain thread exit before asserting + rw._thread.join(timeout=5.0) # don't leak the drain into the next test assert not still_stuck, "finalize() deadlocked on the full bounded queue" @@ -517,8 +524,8 @@ def test_recording_writer_byte_bound_drops(tmp_path, monkeypatch): release = threading.Event() monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: release.wait(20)) - monkeypatch.setattr(ZarrWriter, "finalize", lambda self: None) - monkeypatch.setattr(ZarrWriter, "abort", lambda self: None) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self, *a, **k: None) + monkeypatch.setattr(ZarrWriter, "abort", lambda self, *a, **k: None) frame = np.zeros((100, 100), np.uint16) # 20 kB each rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=256, max_bytes=50_000) @@ -528,6 +535,7 @@ def test_recording_writer_byte_bound_drops(tmp_path, monkeypatch): dropped = rw.dropped_count release.set() rw.finalize(timeout_s=2.0) + rw._thread.join(timeout=5.0) # don't leak the drain into the next test assert dropped >= 3, f"byte cap not enforced: only {dropped} frames dropped" @@ -543,8 +551,8 @@ def test_dropped_frames_seal_store_incomplete(tmp_path, monkeypatch): calls = [] monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: release.wait(10)) - monkeypatch.setattr(ZarrWriter, "finalize", lambda self: calls.append("finalize")) - monkeypatch.setattr(ZarrWriter, "abort", lambda self: calls.append("abort")) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self, *a, **k: calls.append("finalize")) + monkeypatch.setattr(ZarrWriter, "abort", lambda self, *a, **k: calls.append("abort")) rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=1) rw.start() @@ -569,16 +577,21 @@ def test_finalize_wedged_flag_feeds_fail_fast(tmp_path, monkeypatch): release = threading.Event() monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: release.wait(20)) - monkeypatch.setattr(ZarrWriter, "finalize", lambda self: None) - monkeypatch.setattr(ZarrWriter, "abort", lambda self: None) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self, *a, **k: None) + monkeypatch.setattr(ZarrWriter, "abort", lambda self, *a, **k: None) rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=2) rw.start() - for i in range(3): - rw.enqueue(np.zeros((4, 4), np.uint16), i, 0, 0) + rw.enqueue(np.zeros((4, 4), np.uint16), 0, 0, 0) + import time as _t + + _t.sleep(0.2) # drain takes f0 (wedges); f1+f2 then fill the queue + rw.enqueue(np.zeros((4, 4), np.uint16), 1, 0, 0) + rw.enqueue(np.zeros((4, 4), np.uint16), 2, 0, 0) rw.finalize(timeout_s=1.0) # sentinel can't be queued -> wedged fallback still_wedged = rw.finalize_wedged release.set() + rw._thread.join(timeout=5.0) # don't leak the drain into the next test assert still_wedged is True @@ -592,8 +605,8 @@ def test_finalize_total_time_respects_timeout_budget(tmp_path, monkeypatch): monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: _time.sleep(0.95)) - monkeypatch.setattr(ZarrWriter, "finalize", lambda self: None) - monkeypatch.setattr(ZarrWriter, "abort", lambda self: None) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self, *a, **k: None) + monkeypatch.setattr(ZarrWriter, "abort", lambda self, *a, **k: None) rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=2) rw.start() @@ -607,4 +620,133 @@ def test_finalize_total_time_respects_timeout_budget(tmp_path, monkeypatch): t0 = _time.monotonic() rw.finalize(timeout_s=1.0) took = _time.monotonic() - t0 + rw._thread.join(timeout=10.0) # drain finishes backlog; don't leak into the next test assert took < 1.5, f"finalize(timeout_s=1.0) blocked {took:.1f}s — put+join must share one budget" + + +# --------------------------------------------------------------------------- +# Round-2-complete fixes (R1, R3, R4, R5, R9) +# --------------------------------------------------------------------------- + + +def test_under_delivery_marks_store_incomplete(tmp_path, monkeypatch): + """R1: a capture that times out with emitted < expected must not let the + drain seal acquisition_complete=True — StreamingCapture must tell the + writer the capture is incomplete.""" + from control.core.zarr_writer import ZarrWriter + from control.core.streaming_capture import RecordingWriter + + calls = [] + monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: None) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self, *a, **k: calls.append(("finalize",))) + monkeypatch.setattr(ZarrWriter, "abort", lambda self, *a, **k: calls.append(("abort", k))) + + rw = RecordingWriter(_stub_zarr_cfg(tmp_path)) + rw.start() + rw.enqueue(np.zeros((4, 4), np.uint16), 0, 0, 0) + rw.mark_incomplete(captured=1, expected=4) # camera stalled at 1/4 frames + rw.finalize(timeout_s=5.0) + + assert len(calls) == 1 and calls[0][0] == "abort", f"expected incomplete seal, got {calls}" + extra = calls[0][1].get("extra_attrs") or {} + assert extra.get("captured_frames") == 1 and extra.get("expected_frames") == 4 + assert calls[0][1].get("mark_aborted") is False, "under-delivery is not a user abort" + + +def test_streaming_capture_calls_mark_incomplete_on_timeout(): + """R1 (capture side): run() must call writer.mark_incomplete when the stop + condition was not met (frames never arrived — no drops, no write errors).""" + frames = [_FakeFrame(100.0 + i, np.zeros((4, 4), np.uint16)) for i in range(3)] + + class _StubWriterWithIncomplete(_RecordingStubWriter): + def __init__(self): + super().__init__() + self.incomplete = None + + def mark_incomplete(self, captured, expected): + self.incomplete = (captured, expected) + + w = _StubWriterWithIncomplete() + cap = StreamingCapture( + _FakeSource(frames), + RecordingRouter(fps=0.0), + CountStop(10), # only 3 will arrive + w, + abort_fn=lambda: False, + ) + emitted = cap.run(timeout=0.5) + assert emitted == 3 + assert w.incomplete == (3, 10), "run() must flag under-delivery to the writer" + assert w.finalized is True and w.aborted is False + + +def test_router_leaves_holes_after_stall(): + """R3: a burst after a delivery stall must NOT back-fill consecutive slots + (compressing the time axis) — frames map to the slot matching their actual + arrival time, leaving fill holes for the stall.""" + r = RecordingRouter(fps=10.0) + stamps = [100.0, 100.1, 102.0, 102.001, 102.002, 102.1] + slots = [r.route(s) for s in stamps] + accepted = [s[0] for s in slots if s is not None] + assert accepted == [0, 1, 20, 21], f"expected stall gap preserved as holes (slots [0, 1, 20, 21]), got {accepted}" + + +def test_wedged_finalize_flushes_backlog_instead_of_discarding(tmp_path, monkeypatch): + """R4: when finalize() gives up on a wedged drain, the already-captured + queued frames must still be written once the stall clears — the old abort + path discarded them.""" + import threading + import time as _time + from control.core.zarr_writer import ZarrWriter + from control.core.streaming_capture import RecordingWriter + + release = threading.Event() + writing = threading.Event() + written = [] + + def slow_write(self, image, t, c, z, fov=None): + writing.set() # drain dequeued a frame and entered the write + release.wait(20) + written.append(t) + + monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "write_frame", slow_write) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self, *a, **k: None) + monkeypatch.setattr(ZarrWriter, "abort", lambda self, *a, **k: None) + + rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=2) + rw.start() + rw.enqueue(np.zeros((4, 4), np.uint16), 0, 0, 0) + assert writing.wait(5.0) # drain took f0 and is wedged; f1, f2 now queue + rw.enqueue(np.zeros((4, 4), np.uint16), 1, 0, 0) + rw.enqueue(np.zeros((4, 4), np.uint16), 2, 0, 0) + rw.finalize(timeout_s=0.5) + assert rw.finalize_wedged is True + + release.set() # stall clears + deadline = _time.monotonic() + 5.0 + while len(written) < 3 and _time.monotonic() < deadline: + _time.sleep(0.05) + rw._thread.join(timeout=5.0) # don't leak the drain into the next test + assert written == [0, 1, 2], f"queued captured frames discarded after wedge: only {written} written" + + +def test_join_timeout_is_not_wedged(tmp_path, monkeypatch): + """R5: a drain that is slow but steadily writing a backlog is NOT wedged — + flagging it aborts whole multi-well runs over stores that finish fine.""" + import time as _time + from control.core.zarr_writer import ZarrWriter + from control.core.streaming_capture import RecordingWriter + + monkeypatch.setattr(ZarrWriter, "initialize", lambda self: None) + monkeypatch.setattr(ZarrWriter, "write_frame", lambda self, image, t, c, z, fov=None: _time.sleep(0.4)) + monkeypatch.setattr(ZarrWriter, "finalize", lambda self, *a, **k: None) + monkeypatch.setattr(ZarrWriter, "abort", lambda self, *a, **k: None) + + rw = RecordingWriter(_stub_zarr_cfg(tmp_path), max_queue=8) + rw.start() + for i in range(4): # ~1.6s of backlog, sentinel enqueues immediately + rw.enqueue(np.zeros((4, 4), np.uint16), i, 0, 0) + rw.finalize(timeout_s=0.5) # join times out while the drain makes progress + assert rw.finalize_wedged is False, "slow-but-progressing drain wrongly flagged wedged" From ebea578f2023e77fee8596ea07b0392d4c925827 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 02:14:31 -0400 Subject: [PATCH 40/61] fix(record-zstack): GUI + cleanup findings from completed round-2 verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R6-R8, R10-R13 (R1-R5+R9 landed in the previous commit): - toggleAcquisitionStart shares the tab-needs-well-selector decision with onTabChanged (_tab_uses_well_selector): finishing a Record + Z-Stack acquisition no longer hides the well selector permanently, and transient acquisition hides no longer overwrite the user's remembered visibility preference. - Entering the Record + Z-Stack tab rebuilds the FOV grid (regions were cleared but never rebuilt), and well clicks on this tab rebuild it too (wellplate's handler early-returns for other tabs), so the navigation viewer shows scan coverage before Start. - refresh_channel_list warns loudly when the previously selected recording channel vanishes after an objective/profile change (the combo silently swapped to the first channel while the exposure/gain/ illumination spins kept the old channel's values). - acquisition_channels.yaml dedupes channels by name (recording entry wins) — a channel used by both phases appeared twice. - ContinuousFrameSource(already_configured=True): record() already probes CONTINUOUS mode + frame rate for the achievable fps, so the source no longer repeats both per FOV (mode switch + strobe/exposure re-send on toupcam); the clamped rate is re-applied when it differs. - _populate_channel_combo(names=...) shared by construction and refresh (two divergent population paths). - _cleanup_common closes recordZStackController with a 30s budget — 5s let app teardown race a worker still unwinding on a slow disk (partial fix; closeEvent acquisition guard remains a follow-up). TDD: 4 new tests watched failing first (well-selector restore test rides in the CI-only gui test file). Co-Authored-By: Claude Fable 5 --- .../control/core/record_zstack_controller.py | 13 ++++- software/control/core/record_zstack_worker.py | 9 ++- software/control/core/streaming_capture.py | 12 +++- software/control/gui_hcs.py | 48 ++++++++++++---- software/control/widgets.py | 35 ++++++++++-- .../control/test_HighContentScreeningGui.py | 27 +++++++++ .../tests/core/test_record_zstack_worker.py | 55 +++++++++++++++++++ software/tests/core/test_streaming_capture.py | 45 +++++++++++++++ software/tests/test_record_zstack_widget.py | 40 ++++++++++++++ 9 files changed, 261 insertions(+), 23 deletions(-) diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py index 0bc5a3f59..f3b560886 100644 --- a/software/control/core/record_zstack_controller.py +++ b/software/control/core/record_zstack_controller.py @@ -144,11 +144,20 @@ def run_acquisition(self, params: RecordZStackAcquisitionParameters) -> None: # reproducible/auditable: acquisition_channels.yaml records objective + # every channel used by either phase. try: + # Dedupe by name (recording entry wins): channels are identified by + # name, so a channel used by both phases must not appear twice with + # different settings in the snapshot. channels = [] + seen_names = set() + candidates = [] if params.recording_enabled and params.recording_channel is not None: - channels.append(params.recording_channel) + candidates.append(params.recording_channel) if params.zstack_enabled: - channels.extend(params.zstack_channels) + candidates.extend(params.zstack_channels) + for ch in candidates: + if ch.name not in seen_names: + seen_names.add(ch.name) + channels.append(ch) self._microscope.config_repo.save_acquisition_output( output_dir=experiment_dir, objective=self._objective_store.current_objective, diff --git a/software/control/core/record_zstack_worker.py b/software/control/core/record_zstack_worker.py index 847ccbd49..7d1cec2dc 100644 --- a/software/control/core/record_zstack_worker.py +++ b/software/control/core/record_zstack_worker.py @@ -393,9 +393,16 @@ def record(self, t_idx: int, region_id, fov_idx: int, z_ref: float) -> int: channel_wavelengths=[None], is_hcs=False, ) + if effective_fps != self.params.fps: + # The probe requested the original fps; align the camera hint with + # the clamped rate we will actually pace/size against. + try: + self.camera.set_frame_rate(effective_fps) + except Exception: + log.exception("failed to re-apply clamped frame rate") writer = RecordingWriter(cfg) cap = StreamingCapture( - ContinuousFrameSource(self.camera, effective_fps), + ContinuousFrameSource(self.camera, effective_fps, already_configured=True), RecordingRouter(effective_fps), CountStop(T), writer, diff --git a/software/control/core/streaming_capture.py b/software/control/core/streaming_capture.py index 07ba8fc3d..a57f38b2e 100644 --- a/software/control/core/streaming_capture.py +++ b/software/control/core/streaming_capture.py @@ -301,17 +301,23 @@ class ContinuousFrameSource: callback, and starts/stops streaming. """ - def __init__(self, camera, fps: float): + def __init__(self, camera, fps: float, already_configured: bool = False): self._camera = camera self._fps = fps + # True when the caller already set CONTINUOUS mode + frame rate (the + # achievable-fps probe in record() does exactly that): skip repeating + # both, which on toupcam costs a mode switch plus a strobe/exposure + # re-send per FOV. + self._already_configured = already_configured self._cb_id: Optional[int] = None def start(self, on_frame: Callable) -> None: # Order matters: switching to CONTINUOUS resets the frame-rate strategy to # MAX on toupcam, wiping any earlier fps hint. Set the mode FIRST, then the # frame rate, so the PRECISE_FRAMERATE hint survives the mode switch. - self._camera.set_acquisition_mode(CameraAcquisitionMode.CONTINUOUS) - self._camera.set_frame_rate(self._fps) + if not self._already_configured: + self._camera.set_acquisition_mode(CameraAcquisitionMode.CONTINUOUS) + self._camera.set_frame_rate(self._fps) self._cb_id = self._camera.add_frame_callback(on_frame) self._camera.start_streaming() diff --git a/software/control/gui_hcs.py b/software/control/gui_hcs.py index 75ec5825f..27a2fac58 100644 --- a/software/control/gui_hcs.py +++ b/software/control/gui_hcs.py @@ -1656,6 +1656,10 @@ def make_connections(self): # bare channel (dark acquisition). self.objectivesWidget.signal_objective_changed.connect(self.recordZStackWidget.refresh_channel_list) self.profileWidget.signal_profile_changed.connect(self.recordZStackWidget.refresh_channel_list) + # Well clicks rebuild this tab's FOV grid (wellplate's handler + # early-returns when its own tab is not current, so without this the + # selector gives no coverage feedback on the Record + Z-Stack tab). + self.wellSelectionWidget.signal_wellSelected.connect(self.recordZStackWidget.on_well_selection_changed) self.profileWidget.signal_profile_changed.connect( lambda: self.liveControlWidget.select_new_microscope_mode_by_name( @@ -2436,9 +2440,13 @@ def onTabChanged(self, index): # trigger flexible regions update self.flexibleMultiPointWidget.update_fov_positions() + if is_record_zstack_acquisition: + # Regions were cleared above; rebuild the FOV grid for the current + # well selection so the navigation viewer shows scan coverage. + self.recordZStackWidget._update_scan_regions() + self.toggleWellSelector( - (is_wellplate_acquisition or is_record_zstack_acquisition) - and self.wellSelectionWidget.format != "glass slide" + self._tab_uses_well_selector(index) and self.wellSelectionWidget.format != "glass slide" ) acquisitionWidget = self.recordTabWidget.widget(index) acquisitionWidget.emit_selected_channels() @@ -2558,6 +2566,23 @@ def connectWellSelectionWidget(self): if ENABLE_WELLPLATE_MULTIPOINT: self.wellSelectionWidget.signal_wellSelected.connect(self.wellplateMultiPointWidget.update_well_coordinates) + def _tab_uses_well_selector(self, index) -> bool: + """True if the record tab at ``index`` drives acquisitions from the well + selector (Wellplate Multipoint and Record + Z-Stack): both onTabChanged + and toggleAcquisitionStart must agree on this, or the selector is hidden + and never restored after an acquisition on one of these tabs.""" + is_wellplate = ( + (index == self.recordTabWidget.indexOf(self.wellplateMultiPointWidget)) + if ENABLE_WELLPLATE_MULTIPOINT + else False + ) + is_record_zstack = ( + (index == self.recordTabWidget.indexOf(self.recordZStackWidget)) + if self.recordZStackWidget is not None + else False + ) + return is_wellplate or is_record_zstack + def toggleWellSelector(self, show, remember_state=True): if show and self.imageDisplayTabs.tabText(self.imageDisplayTabs.currentIndex()) == "Live View": self.dock_wellSelection.setVisible(True) @@ -2604,16 +2629,14 @@ def toggleAcquisitionStart(self, acquisition_started): if acquisition_started: self.liveControlWidget.toggle_autolevel(not acquisition_started) - # hide well selector during acquisition - is_wellplate_acquisition = ( - (current_index == self.recordTabWidget.indexOf(self.wellplateMultiPointWidget)) - if ENABLE_WELLPLATE_MULTIPOINT - else False - ) - if is_wellplate_acquisition and self.wellSelectionWidget.format != "glass slide": + # hide well selector during acquisition; restore it afterwards on tabs + # that drive acquisitions from it. remember_state=False everywhere: + # acquisition start/stop is transient and must not overwrite the user's + # remembered visibility preference. + if self._tab_uses_well_selector(current_index) and self.wellSelectionWidget.format != "glass slide": self.toggleWellSelector(not acquisition_started, remember_state=False) else: - self.toggleWellSelector(False) + self.toggleWellSelector(False, remember_state=False) # display acquisition progress bar during acquisition self.recordTabWidget.currentWidget().display_progress_bar(acquisition_started) @@ -2909,7 +2932,10 @@ def _cleanup_common(self, for_restart: bool = False): # being killed mid-write (corrupted store). if getattr(self, "recordZStackController", None) is not None: try: - self.recordZStackController.close() + # 30s: the worker's unwind can legitimately take this long on a + # slow disk (RecordingWriter finalize budget + job drain + stage + # restores); a 5s join let teardown race the still-running worker. + self.recordZStackController.close(timeout_s=30.0) except Exception: self.log.exception(f"Error closing record z-stack controller during {context}") diff --git a/software/control/widgets.py b/software/control/widgets.py index 2b3390f6c..f562672c6 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17426,9 +17426,16 @@ def _build_start_group(self) -> QGroupBox: # ---------------------------------------------------------------------- helpers - def _populate_channel_combo(self, combo: QComboBox) -> None: - """Populate a channel QComboBox from liveController.""" + def _populate_channel_combo(self, combo: QComboBox, names: Optional[List[str]] = None) -> None: + """Populate a channel QComboBox from liveController (or a pre-fetched list). + + ``names`` lets refresh_channel_list share this path with its fetch-first + bail-out semantics instead of duplicating the population code. + """ combo.clear() + if names is not None: + combo.addItems(names) + return try: channels = self.liveController.get_channels(self.objectiveStore.current_objective) for ch in channels: @@ -17843,6 +17850,18 @@ def acquisition_is_finished(self): self.btn_startAcquisition.setChecked(False) self.signal_acquisition_started.emit(False) + def on_well_selection_changed(self) -> None: + """Rebuild the FOV grid when the well selection changes on this tab. + + Connected in gui_hcs to wellSelectionWidget.signal_wellSelected — + WellplateMultiPointWidget's equivalent handler early-returns when its + own tab is not current, so this widget needs its own. No-op when + another record tab is current. + """ + if self.tab_widget is not None and self.tab_widget.currentWidget() is not self: + return + self._update_scan_regions() + def refresh_channel_list(self) -> None: """Repopulate the channel combos from liveController. @@ -17870,12 +17889,16 @@ def refresh_channel_list(self) -> None: names = [ch.name for ch in channels] prev_recording = self._recording_channel_name() - self._recording_ch_combo.clear() - self._recording_ch_combo.addItems(names) + self._populate_channel_combo(self._recording_ch_combo, names=names) if prev_recording and prev_recording in names: self._recording_ch_combo.setCurrentIndex(names.index(prev_recording)) - self.combobox_zstack_add_channel.clear() - self.combobox_zstack_add_channel.addItems(names) + elif prev_recording: + self._log.warning( + f"recording channel {prev_recording!r} is not available for the current " + f"objective/profile; selection changed to {names[0]!r} — review the recording " + f"exposure/gain/illumination before starting an acquisition" + ) + self._populate_channel_combo(self.combobox_zstack_add_channel, names=names) for name in [n for n in self._zstack_channel_names if n not in names]: self._log.info(f"removing z-stack channel row {name!r}: not available for the current objective") self._remove_zstack_channel_row(name) diff --git a/software/tests/control/test_HighContentScreeningGui.py b/software/tests/control/test_HighContentScreeningGui.py index 5a33fe7c3..a537ef475 100644 --- a/software/tests/control/test_HighContentScreeningGui.py +++ b/software/tests/control/test_HighContentScreeningGui.py @@ -93,3 +93,30 @@ def confirm_exit(parent, title, text, *args, **kwargs): tabw.setCurrentIndex(labels.index("Record + Z-Stack")) assert not win.dock_wellSelection.isHidden(), "well selector dock must stay available on the Record + Z-Stack tab" + + +def test_record_zstack_acquisition_finish_keeps_well_selector(qtbot, monkeypatch): + """R6: toggleAcquisitionStart(False) after a Record + Z-Stack acquisition + must not hide the well selector dock (only the wellplate branch kept it).""" + + def confirm_exit(parent, title, text, *args, **kwargs): + if title == "Confirm Exit": + return QMessageBox.Yes + raise RuntimeError(f"Unexpected QMessageBox: {title} - {text}") + + monkeypatch.setattr(QMessageBox, "question", confirm_exit) + monkeypatch.setattr(control.gui_hcs, "ENABLE_RECORDING", True) + + scope = control.microscope.Microscope.build_from_global_config(True) + win = control.gui_hcs.HighContentScreeningGui(microscope=scope, is_simulation=True) + qtbot.add_widget(win) + + tabw = win.recordTabWidget + labels = [tabw.tabText(i) for i in range(tabw.count())] + tabw.setCurrentIndex(labels.index("Record + Z-Stack")) + assert not win.dock_wellSelection.isHidden() + + win.toggleAcquisitionStart(True) # acquisition starts: selector hides + assert win.dock_wellSelection.isHidden() + win.toggleAcquisitionStart(False) # acquisition ends: selector must return + assert not win.dock_wellSelection.isHidden(), "well selector not restored after Record+Z-Stack acquisition" diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index d01d6dac6..8f83e367a 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -699,3 +699,58 @@ def abort(self): worker.run() assert aborted["v"] is True, "acquisition continued despite dropped recording frames" + + +def test_config_snapshot_dedupes_channels_by_name(tmp_path): + """R10: recording the same channel that is also z-stacked must not produce + two same-name entries in acquisition_channels.yaml (channels are identified + by name; duplicate names make the snapshot ambiguous).""" + pytest.importorskip("tensorstore") + import yaml + + import control._def + import tests.control.test_stubs as ts + from control.core.multi_point_controller import NoOpCallbacks + from control.core.record_zstack_controller import ( + RecordZStackAcquisitionParameters, + RecordZStackController, + ) + + control._def.FILE_SAVING_OPTION = control._def.FileSavingOption.ZARR_V3 + scope = _build_simulated_microscope(64, 48) + live_controller = ts.get_test_live_controller(scope, scope.objective_store.default_objective) + laser_af = ts.get_test_laser_autofocus_controller(scope) + channels = live_controller.get_channels(scope.objective_store.default_objective) + scope.camera.set_exposure_time(1) + + controller = RecordZStackController( + microscope=scope, + live_controller=live_controller, + laser_autofocus_controller=laser_af, + objective_store=scope.objective_store, + scan_coordinates=None, + callbacks=NoOpCallbacks, + ) + params = RecordZStackAcquisitionParameters( + base_path=str(tmp_path), + experiment_id="dedupe", + Nt=1, + recording_enabled=True, + recording_channel=channels[0], + fps=10.0, + duration_s=0.2, + zstack_enabled=True, + zstack_channels=[channels[0], channels[1]], # channel 0 in BOTH phases + z_min_um=0.0, + z_max_um=1.0, + z_step_um=1.0, + ) + try: + controller.run_acquisition(params) + controller.join(timeout=60) + finally: + controller.close() + + snapshot = yaml.safe_load(open(Path(tmp_path) / params.experiment_id / "acquisition_channels.yaml")) + names = [c["name"] for c in snapshot["channels"]] + assert len(names) == len(set(names)), f"duplicate channel names in snapshot: {names}" diff --git a/software/tests/core/test_streaming_capture.py b/software/tests/core/test_streaming_capture.py index f4b173c2e..9b9fbb8f8 100644 --- a/software/tests/core/test_streaming_capture.py +++ b/software/tests/core/test_streaming_capture.py @@ -750,3 +750,48 @@ def test_join_timeout_is_not_wedged(tmp_path, monkeypatch): rw.enqueue(np.zeros((4, 4), np.uint16), i, 0, 0) rw.finalize(timeout_s=0.5) # join times out while the drain makes progress assert rw.finalize_wedged is False, "slow-but-progressing drain wrongly flagged wedged" + + +def test_frame_source_skips_reconfig_when_already_configured(): + """R11: record() already probes set_acquisition_mode + set_frame_rate for + the achievable fps; ContinuousFrameSource repeating both doubles the camera + reconfiguration (mode switch + strobe/exposure re-send) on every FOV.""" + from control.core.streaming_capture import ContinuousFrameSource + + class _CountingCamera: + def __init__(self): + self.mode_calls = 0 + self.rate_calls = 0 + + def set_acquisition_mode(self, mode): + self.mode_calls += 1 + + def set_frame_rate(self, fps): + self.rate_calls += 1 + return fps + + def add_frame_callback(self, cb): + return 1 + + def remove_frame_callback(self, cb_id): + pass + + def start_streaming(self): + pass + + def stop_streaming(self): + pass + + cam = _CountingCamera() + src = ContinuousFrameSource(cam, fps=10.0, already_configured=True) + src.start(lambda f: None) + src.stop() + assert cam.mode_calls == 0 and cam.rate_calls == 0, ( + f"already-configured source still reconfigured the camera " f"(mode={cam.mode_calls}, rate={cam.rate_calls})" + ) + + cam2 = _CountingCamera() + src2 = ContinuousFrameSource(cam2, fps=10.0) + src2.start(lambda f: None) + src2.stop() + assert cam2.mode_calls == 1 and cam2.rate_calls == 1 # default behavior unchanged diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index f7d1b1f27..40b808833 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -1021,3 +1021,43 @@ def test_refresh_channel_list_preserves_state_on_failure(qtbot, simulated_widget w.refresh_channel_list() assert w._zstack_channel_names == ["BF LED matrix full"], "rows wiped on empty channel list" assert w._recording_ch_combo.count() == combo_count_before + + +def test_on_well_selection_changed_rebuilds_regions(qtbot, simulated_widget_deps): + """R7: well clicks while this tab is current must rebuild the FOV grid so + the navigation viewer shows scan coverage (wellplate's handler early-returns + for other tabs, leaving this tab's preview inert).""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + sc = w.scanCoordinates + sc.reset_mock() + + w.on_well_selection_changed() + + assert sc.set_well_coordinates.called, "well selection change did not rebuild the FOV grid" + + +def test_refresh_channel_list_warns_on_silent_selection_swap(qtbot, simulated_widget_deps, caplog): + """R8: when the previously selected recording channel vanishes after an + objective/profile change, the combo falls back to the first channel — that + swap must be loud, or the next Start records the wrong channel.""" + import logging + + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + # Select the second channel, then refresh with a set that lacks it. + w._recording_ch_combo.setCurrentIndex(1) + prev = w._recording_ch_combo.currentText() + w.liveController.get_channels.return_value = [_make_channel("Only Channel")] + + with caplog.at_level(logging.WARNING): + w.refresh_channel_list() + + assert w._recording_ch_combo.currentText() == "Only Channel" + assert any( + prev in rec.message and "Only Channel" in rec.message for rec in caplog.records + ), f"no warning about the recording selection changing from {prev!r}" From 3a78ace9c246dee19ffc479644282551ad0252c7 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 20:03:20 -0400 Subject: [PATCH 41/61] feat(record-zstack): XY/Time tabbed row, Select Wells mode, channel seeding, narrower layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors WellplateMultiPointWidget's XY/Time tab row (checkbox + combo in a colored frame that shows/hides its controls), skipping Z since this widget's own Z-Stack phase group already covers z-stacking: - checkbox_xy + combobox_xy_mode now offer two real modes: "Select Wells" (today's tile-over-selected-wells behavior, kept as default) and "Current Position" (new: single FOV at the live stage position, bypassing well selection — validate() no longer requires wells in this mode). Unchecking XY forces Current Position and disables the combo; re-checking restores the previous mode, mirroring WellplateMultiPointWidget exactly. - checkbox_time wraps Nt/dt: unchecked forces a single timepoint and hides the controls, restoring the previous Nt/dt on re-check. - checkbox_laser_af relocated into this row as a plain checkbox, matching how WellplateMultiPointWidget places its own Laser AF toggle. Channel-add now seeds from the channel's own configured settings (via a new _channel_settings/_find_channel lookup) instead of hardcoded 50ms/0 gain/50%, for both the z-stack "+ Add" button and the recording-channel combo (including its initial row and on selection change). Layout: removed the channel tables' fixed 530px width (fighting against whatever width the main window happens to lock in for this tab) in favor of a responsive fill plus tightened field widths throughout, so the panel no longer needs a horizontal scrollbar. Z-Stack channel table now spans the group's full width to match the Recording table. Z-Stack channel names get a tooltip showing the full name since the Channel column truncates. ~20 new tests covering the XY/Time toggling, mode switching, channel seeding, and table behavior. --- software/control/widgets.py | 345 ++++++++++++++++---- software/tests/test_record_zstack_widget.py | 299 ++++++++++++++++- 2 files changed, 583 insertions(+), 61 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index 5f678d5f2..8139143b8 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -6,7 +6,7 @@ import logging import sys from pathlib import Path -from typing import Dict, List, Optional, TYPE_CHECKING +from typing import Dict, List, Optional, Tuple, TYPE_CHECKING import psutil @@ -17109,8 +17109,8 @@ def _build_wells_fov_group(self) -> QGroupBox: vbox.setSpacing(6) # Consistent widths so label+field pairs line up cleanly across rows. - field_w = 90 # spinbox / combo width - pair_gap = 14 # horizontal gap between adjacent label+field pairs + field_w = 70 # spinbox / combo width + pair_gap = 6 # horizontal gap between adjacent label+field pairs def _pair_label(text: str) -> QLabel: lbl = QLabel(text) @@ -17119,10 +17119,53 @@ def _pair_label(text: str) -> QLabel: # First-column labels share a width so the first field of each row aligns # (sized for the wider "Overlap:" label). - first_label_w = 66 + first_label_w = 52 + + # --- XY / Time tabs row: mirrors WellplateMultiPointWidget's XY/Z/Time + # tabs (checkbox + combo in a frame that highlights when active). Z is + # skipped here since the "Z-Stack phase" group below already covers + # z-stacking for this widget. Laser AF is a standalone toggle, so it's + # placed as a plain checkbox in this row rather than in its own tab + # frame — mirroring how WellplateMultiPointWidget places its own Laser + # AF checkbox as a bare widget outside the tabs. + self.checkbox_xy = QCheckBox("XY") + self.checkbox_xy.setChecked(True) + self.combobox_xy_mode = QComboBox() + self.combobox_xy_mode.addItems(["Current Position", "Select Wells"]) + self.combobox_xy_mode.setMaximumWidth(130) + # Select Wells (tile a grid over the selected wells) is the default so + # existing tiling behavior is unchanged out of the box; Current + # Position is a single FOV at the live stage position, bypassing + # well selection entirely. + self.combobox_xy_mode.setCurrentText("Select Wells") + self.xy_frame = QFrame() + xy_layout = QHBoxLayout(self.xy_frame) + xy_layout.setContentsMargins(8, 4, 8, 4) + xy_layout.addWidget(self.checkbox_xy) + xy_layout.addWidget(self.combobox_xy_mode) - # Row 1: FOV overlap + Region shape + Region size - row1 = QHBoxLayout() + self.checkbox_time = QCheckBox("Time") + self.checkbox_time.setChecked(False) + self.time_frame = QFrame() + time_frame_layout = QHBoxLayout(self.time_frame) + time_frame_layout.setContentsMargins(8, 4, 8, 4) + time_frame_layout.addWidget(self.checkbox_time) + time_frame_layout.addStretch() + + self.checkbox_laser_af = QCheckBox("Laser AF") + self.checkbox_laser_af.setChecked(False) + + tabs_row = QHBoxLayout() + tabs_row.setSpacing(4) + tabs_row.addWidget(self.xy_frame, 2) + tabs_row.addWidget(self.time_frame, 1) + tabs_row.addWidget(self.checkbox_laser_af) + vbox.addLayout(tabs_row) + + # Row 1: FOV overlap + Region shape + Region size (hidden when XY unchecked) + self.xy_controls_frame = QFrame() + row1 = QHBoxLayout(self.xy_controls_frame) + row1.setContentsMargins(0, 0, 0, 0) row1.setSpacing(4) overlap_label = _pair_label("Overlap:") @@ -17140,7 +17183,9 @@ def _pair_label(text: str) -> QLabel: row1.addWidget(_pair_label("Shape:")) self.combobox_shape = QComboBox() self.combobox_shape.addItems(["Square", "Circle", "Rectangle"]) - self.combobox_shape.setFixedWidth(field_w) + # Wider than field_w: word options ("Rectangle") need more room than + # the numeric+suffix spinboxes elsewhere in this row. + self.combobox_shape.setFixedWidth(90) row1.addWidget(self.combobox_shape) row1.addSpacing(pair_gap) @@ -17155,10 +17200,12 @@ def _pair_label(text: str) -> QLabel: row1.addWidget(self.entry_scan_size) row1.addStretch(1) - vbox.addLayout(row1) + vbox.addWidget(self.xy_controls_frame) - # Row 2: Nt + dt - row2 = QHBoxLayout() + # Row 2: Nt + dt (hidden when Time unchecked) + self.time_controls_frame = QFrame() + row2 = QHBoxLayout(self.time_controls_frame) + row2.setContentsMargins(0, 0, 0, 0) row2.setSpacing(4) nt_label = _pair_label("Nt:") @@ -17181,14 +17228,8 @@ def _pair_label(text: str) -> QLabel: self.entry_dt.setFixedWidth(field_w) row2.addWidget(self.entry_dt) - # Laser reflection AF checkbox shares the Nt/dt row (room to the right of dt). - row2.addSpacing(pair_gap) - self.checkbox_laser_af = QCheckBox("Laser AF") - self.checkbox_laser_af.setChecked(False) - row2.addWidget(self.checkbox_laser_af) - row2.addStretch(1) - vbox.addLayout(row2) + vbox.addWidget(self.time_controls_frame) # Separator below this section, before the Recording phase group. sep = QFrame() @@ -17201,8 +17242,118 @@ def _pair_label(text: str) -> QLabel: self.entry_scan_size.valueChanged.connect(self._update_scan_regions) self.combobox_shape.currentIndexChanged.connect(self._update_scan_regions) + # Wire XY/Time toggle behavior and initialize their visibility/styling. + self._stored_time_params = None + self._stored_xy_mode = None + self.checkbox_xy.toggled.connect(self._on_xy_toggled) + self.combobox_xy_mode.currentTextChanged.connect(self._on_xy_mode_changed) + self.checkbox_time.toggled.connect(self._on_time_toggled) + self._on_xy_toggled(self.checkbox_xy.isChecked()) + self._on_time_toggled(self.checkbox_time.isChecked()) + return grp + def _on_xy_toggled(self, checked: bool) -> None: + """Enable/disable the mode combo, forcing Current Position (single + stage-position FOV) when unchecked and restoring the previously + selected mode when re-checked — mirrors WellplateMultiPointWidget's + on_xy_toggled. + """ + self.combobox_xy_mode.setEnabled(checked) + old_mode = self.combobox_xy_mode.currentText() + if checked: + if self._stored_xy_mode is not None: + self.combobox_xy_mode.setCurrentText(self._stored_xy_mode) + else: + self._stored_xy_mode = self.combobox_xy_mode.currentText() + self.combobox_xy_mode.setCurrentText("Current Position") + self._update_tab_styles() + if self.combobox_xy_mode.currentText() == old_mode: + # currentTextChanged didn't fire (mode unchanged), so + # _on_xy_mode_changed never rebuilt the region — do it here. + self._update_scan_regions() + + def _on_xy_mode_changed(self, mode: str) -> None: + """Show the FOV-tiling controls only for Select Wells; Current + Position always uses a fixed single FOV at the live stage position. + """ + self.xy_controls_frame.setVisible(self.checkbox_xy.isChecked() and mode == "Select Wells") + self._update_scan_regions() + + def _on_time_toggled(self, checked: bool) -> None: + """Show/hide the Nt/dt controls, resetting to a single timepoint when + unchecked and restoring the previous Nt/dt when re-checked (mirrors + WellplateMultiPointWidget's store/restore of Time parameters). + """ + self.time_controls_frame.setVisible(checked) + if checked: + if self._stored_time_params is not None: + nt, dt = self._stored_time_params + self.entry_Nt.setValue(nt) + self.entry_dt.setValue(dt) + else: + self._stored_time_params = (self.entry_Nt.value(), self.entry_dt.value()) + self.entry_Nt.setValue(1) + self.entry_dt.setValue(0) + self._update_tab_styles() + + def _update_tab_styles(self) -> None: + """Border/background styling for the XY/Time tabs (colors mirror + WellplateMultiPointWidget.update_tab_styles: orange for XY, green for Time). + """ + xy_active_style = """ + QFrame { + border: 1px solid #FF8C00; + border-radius: 2px; + } + """ + xy_controls_style = """ + QFrame { + background-color: rgba(255, 140, 0, 0.15); + } + QFrame QComboBox, QFrame QSpinBox, QFrame QDoubleSpinBox { + background-color: white; + color: black; + } + QFrame QComboBox QAbstractItemView { + background-color: white; + color: black; + selection-background-color: palette(highlight); + selection-color: palette(highlighted-text); + } + QFrame QLabel { + background-color: transparent; + } + """ + time_active_style = """ + QFrame { + border: 1px solid #00A000; + border-radius: 2px; + } + """ + time_controls_style = """ + QFrame { + background-color: rgba(0, 160, 0, 0.15); + } + QFrame QComboBox, QFrame QSpinBox, QFrame QDoubleSpinBox { + background-color: white; + color: black; + } + QFrame QLabel { + background-color: transparent; + } + """ + inactive_style = """ + QFrame { + border: 1px solid palette(mid); + border-radius: 2px; + } + """ + self.xy_frame.setStyleSheet(xy_active_style if self.checkbox_xy.isChecked() else inactive_style) + self.xy_controls_frame.setStyleSheet(xy_controls_style if self.checkbox_xy.isChecked() else "") + self.time_frame.setStyleSheet(time_active_style if self.checkbox_time.isChecked() else inactive_style) + self.time_controls_frame.setStyleSheet(time_controls_style if self.checkbox_time.isChecked() else "") + def _build_recording_group(self) -> QGroupBox: grp = QGroupBox("Recording phase") grp.setCheckable(True) @@ -17228,11 +17379,13 @@ def _build_recording_group(self) -> QGroupBox: self.recording_channel_table.setFixedHeight( self.recording_channel_table.horizontalHeader().height() + self.recording_channel_table.rowHeight(0) + 8 ) - # Bound table width to match the z-stack table and force the 5 columns to - # fit (Channel truncates via Stretch) instead of showing a horizontal scrollbar. + # Force the 5 columns to fit (Channel truncates via Stretch) instead of + # showing a horizontal scrollbar. No fixed width: the panel's actual + # available width varies with which other tab last drove the main + # window's width, so the table fills whatever space it's given rather + # than gambling on a specific pixel value. self.recording_channel_table.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) - # Match the z-stack channel table width so the two tables align as one column. - self.recording_channel_table.setFixedWidth(530) + self.recording_channel_table.setMinimumWidth(300) # Col 0: channel combo. Let it shrink within the Stretch column instead of # demanding its full text width (otherwise the long channel name forces the @@ -17251,7 +17404,7 @@ def _build_recording_group(self) -> QGroupBox: self._recording_exp_spin.setSuffix(" ms") self._recording_exp_spin.setDecimals(1) self._recording_exp_spin.setKeyboardTracking(False) - self._recording_exp_spin.setMaximumWidth(85) + self._recording_exp_spin.setMaximumWidth(68) self.recording_channel_table.setCellWidget(0, 1, self._recording_exp_spin) # Col 2: gain spinbox @@ -17260,7 +17413,7 @@ def _build_recording_group(self) -> QGroupBox: self._recording_gain_spin.setValue(0.0) self._recording_gain_spin.setDecimals(2) self._recording_gain_spin.setKeyboardTracking(False) - self._recording_gain_spin.setMaximumWidth(60) + self._recording_gain_spin.setMaximumWidth(48) self.recording_channel_table.setCellWidget(0, 2, self._recording_gain_spin) # Col 3: illumination spinbox @@ -17270,7 +17423,7 @@ def _build_recording_group(self) -> QGroupBox: self._recording_illum_spin.setSuffix(" %") self._recording_illum_spin.setDecimals(1) self._recording_illum_spin.setKeyboardTracking(False) - self._recording_illum_spin.setMaximumWidth(76) + self._recording_illum_spin.setMaximumWidth(60) self.recording_channel_table.setCellWidget(0, 3, self._recording_illum_spin) # Col 4: ↻ copy-from-live button @@ -17280,13 +17433,13 @@ def _build_recording_group(self) -> QGroupBox: self.btn_copy_from_live.clicked.connect(self._copy_recording_from_live) self.recording_channel_table.setCellWidget(0, 4, self.btn_copy_from_live) - # Bound the table width to match the z-stack table: wrap in an HBox - # with a trailing stretch so it doesn't expand to fill the panel. - table_row = QHBoxLayout() - table_row.setContentsMargins(0, 0, 0, 0) - table_row.addWidget(self.recording_channel_table) - table_row.addStretch(1) - vbox.addLayout(table_row) + # Seed the row from the selected channel's own configured settings + # (not the hardcoded defaults above) whenever the selection changes, + # including the initial population. + self._recording_ch_combo.currentIndexChanged.connect(self._on_recording_channel_changed) + self._on_recording_channel_changed(self._recording_ch_combo.currentIndex()) + + vbox.addWidget(self.recording_channel_table) # Row 1: FPS | Duration | Z-offset fps_row = QHBoxLayout() @@ -17298,7 +17451,7 @@ def _build_recording_group(self) -> QGroupBox: self.entry_fps.setValue(10.0) self.entry_fps.setSuffix(" fps") self.entry_fps.setKeyboardTracking(False) - self.entry_fps.setMaximumWidth(100) + self.entry_fps.setMaximumWidth(78) fps_row.addWidget(self.entry_fps) fps_row.addSpacing(4) @@ -17308,7 +17461,7 @@ def _build_recording_group(self) -> QGroupBox: self.entry_duration.setValue(1.0) self.entry_duration.setSuffix(" s") self.entry_duration.setKeyboardTracking(False) - self.entry_duration.setMaximumWidth(90) + self.entry_duration.setMaximumWidth(65) fps_row.addWidget(self.entry_duration) fps_row.addSpacing(4) @@ -17318,7 +17471,7 @@ def _build_recording_group(self) -> QGroupBox: self.entry_recording_z_offset.setValue(0.0) self.entry_recording_z_offset.setSuffix(" μm") self.entry_recording_z_offset.setKeyboardTracking(False) - self.entry_recording_z_offset.setMaximumWidth(95) + self.entry_recording_z_offset.setMaximumWidth(70) fps_row.addWidget(self.entry_recording_z_offset) fps_row.addStretch(1) @@ -17344,7 +17497,7 @@ def _build_zstack_group(self) -> QGroupBox: self.entry_zmin.setDecimals(1) self.entry_zmin.setSingleStep(0.5) self.entry_zmin.setKeyboardTracking(False) - self.entry_zmin.setMaximumWidth(85) + self.entry_zmin.setMaximumWidth(68) # a hair more than zmax: room for the "-" sign layout.addWidget(QLabel("Z-min:"), 0, 0) layout.addWidget(self.entry_zmin, 0, 1) @@ -17355,7 +17508,7 @@ def _build_zstack_group(self) -> QGroupBox: self.entry_zmax.setDecimals(1) self.entry_zmax.setSingleStep(0.5) self.entry_zmax.setKeyboardTracking(False) - self.entry_zmax.setMaximumWidth(85) + self.entry_zmax.setMaximumWidth(62) layout.addWidget(QLabel("Z-max:"), 0, 2) layout.addWidget(self.entry_zmax, 0, 3) @@ -17367,7 +17520,7 @@ def _build_zstack_group(self) -> QGroupBox: self.entry_step.setSingleStep(0.1) self.entry_step.setKeyboardTracking(False) # Slightly wider than zmin/zmax: 2 decimals ("1.00 μm") need ~1 extra char. - self.entry_step.setMaximumWidth(98) + self.entry_step.setMaximumWidth(68) layout.addWidget(QLabel("Step:"), 0, 4) layout.addWidget(self.entry_step, 0, 5) @@ -17389,11 +17542,14 @@ def _build_zstack_group(self) -> QGroupBox: self.zstack_channel_table.setSelectionBehavior(QAbstractItemView.SelectRows) self.zstack_channel_table.setMinimumHeight(80) self.zstack_channel_table.setMaximumHeight(200) - # Match the recording channel table width so the two tables align as one column. - self.zstack_channel_table.setFixedWidth(530) + # No fixed width — see the matching comment on recording_channel_table. + self.zstack_channel_table.setMinimumWidth(300) self.zstack_channel_table.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) self.zstack_channel_table.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) - layout.addWidget(self.zstack_channel_table, 1, 0, 1, 7) + # Span into the stretch column (7) too, so the table fills the full + # group width instead of stopping at the Z-min/Z-max/Step row's + # narrower natural width. + layout.addWidget(self.zstack_channel_table, 1, 0, 1, 8) # Row 2: Add channel dropdown + button (capped combo with "+ Add" right after, # then a trailing stretch so the row doesn't span the full panel width) @@ -17402,7 +17558,7 @@ def _build_zstack_group(self) -> QGroupBox: add_row.setSpacing(4) self.combobox_zstack_add_channel = QComboBox() self._populate_channel_combo(self.combobox_zstack_add_channel) - self.combobox_zstack_add_channel.setMaximumWidth(260) + self.combobox_zstack_add_channel.setMaximumWidth(170) add_row.addWidget(self.combobox_zstack_add_channel) self.btn_zstack_add_channel = QPushButton("+ Add") self.btn_zstack_add_channel.setMaximumWidth(60) @@ -17451,6 +17607,45 @@ def _populate_channel_combo(self, combo: QComboBox, names: Optional[List[str]] = except Exception as exc: self._log.warning(f"Could not populate channel combo: {exc}") + def _find_channel(self, name: str): + """Look up channel *name* in liveController.get_channels() for the + current objective. Returns None if not found or unavailable. + + Shared by _channel_settings (used to seed new rows) and + build_parameters._make_channel_base (used to build the acquisition + channel), so both see the same channel for the same name. + """ + try: + channels = self.liveController.get_channels(self.objectiveStore.current_objective) + for ch in channels: + if ch.name == name: + return ch + except Exception as exc: + self._log.warning(f"_find_channel: could not fetch channel {name!r}: {exc}") + return None + + def _channel_settings(self, name: str) -> Tuple[float, float, float]: + """Return (exposure, gain, illumination) configured for channel *name*. + + Falls back to (50.0, 0.0, 50.0) if the channel can't be found, so newly + added rows are seeded from the channel's own settings (as known by + liveController) instead of an arbitrary hardcoded default. + """ + ch = self._find_channel(name) + if ch is not None: + return ch.exposure_time, ch.analog_gain, ch.illumination_intensity + return 50.0, 0.0, 50.0 + + def _on_recording_channel_changed(self, index: int) -> None: + """Reseed the recording row's exposure/gain/illum from the newly selected channel.""" + name = self._recording_ch_combo.currentText() + if not name: + return + exposure, gain, illum = self._channel_settings(name) + self._recording_exp_spin.setValue(exposure) + self._recording_gain_spin.setValue(gain) + self._recording_illum_spin.setValue(illum) + # ---------------------------------------------------------------------- recording table accessors def _recording_channel_name(self) -> Optional[str]: @@ -17501,9 +17696,11 @@ def _add_zstack_channel_row( row = self.zstack_channel_table.rowCount() self.zstack_channel_table.insertRow(row) - # Col 0: channel name (read-only) + # Col 0: channel name (read-only). Tooltip shows the full name since + # the Stretch column truncates long names visually. item = QTableWidgetItem(name) item.setFlags(item.flags() & ~Qt.ItemIsEditable) + item.setToolTip(name) self.zstack_channel_table.setItem(row, 0, item) # Col 1: exposure spinbox @@ -17646,7 +17843,8 @@ def _on_zstack_add_channel_clicked(self) -> None: """Add the currently selected channel in the add-channel combo to the z-stack table.""" name = self.combobox_zstack_add_channel.currentText() if name: - self._add_zstack_channel_row(name) + exposure, gain, illum = self._channel_settings(name) + self._add_zstack_channel_row(name, exposure, gain, illum) def _get_selected_well_count(self) -> int: """Return the number of currently selected wells. @@ -17670,26 +17868,36 @@ def _get_selected_well_count(self) -> int: return 1 def _update_scan_regions(self) -> None: - """Update the FOV grid from the current overlap/shape/scan-size settings. + """Update the FOV grid from the current XY mode. Mirrors WellplateMultiPointWidget.update_coordinates. Called whenever - entry_overlap, entry_scan_size, or combobox_shape changes, and also at - the start of toggle_acquisition to ensure the grid is current. + entry_overlap, entry_scan_size, or combobox_shape changes, when the XY + checkbox/mode changes, and also at the start of toggle_acquisition to + ensure the grid is current. + + Select Wells mode tiles a grid (per overlap/shape/size) over the + currently selected wells. Current Position mode (also forced when XY + is unchecked) ignores well selection entirely and uses a single small + FOV at the live stage position, mirroring + WellplateMultiPointWidget.set_coordinates_to_current_position. """ if self.scanCoordinates is None: return - scan_size_mm = self.entry_scan_size.value() - overlap_percent = self.entry_overlap.value() - shape = self.combobox_shape.currentText() try: # Clear first: set_well_coordinates only adds wells not already present, # so without clearing, already-selected wells keep their old tile geometry # and the new size/overlap/shape would be silently ignored. if self.scanCoordinates.has_regions(): self.scanCoordinates.clear_regions() - self.scanCoordinates.set_well_coordinates(scan_size_mm, overlap_percent, shape) + if self.combobox_xy_mode.currentText() == "Select Wells": + self.scanCoordinates.set_well_coordinates( + self.entry_scan_size.value(), self.entry_overlap.value(), self.combobox_shape.currentText() + ) + else: # "Current Position" + pos = self.stage.get_pos() + self.scanCoordinates.add_region("current", pos.x_mm, pos.y_mm, 0.01, 0, "Square") except Exception as exc: - self._log.warning(f"_update_scan_regions: set_well_coordinates failed: {exc}") + self._log.warning(f"_update_scan_regions: failed: {exc}") def _laser_af_has_reference(self) -> bool: """Return True if the laser autofocus controller has a captured reference.""" @@ -17705,10 +17913,18 @@ def validate(self) -> Optional[str]: Delegates to the pure helper _validate_record_zstack_params so the rules can be tested independently of Qt. + + Current Position mode doesn't use well selection at all (it acquires + at the live stage position), so it's treated as always having its one + implicit "well" satisfied. """ + if self.combobox_xy_mode.currentText() == "Current Position": + selected_well_count = 1 + else: + selected_well_count = self._get_selected_well_count() return _validate_record_zstack_params( base_path=self.lineEdit_savingDir.text().strip(), - selected_well_count=self._get_selected_well_count(), + selected_well_count=selected_well_count, recording_enabled=self.checkbox_recording.isChecked(), fps=self.entry_fps.value(), duration_s=self.entry_duration.value(), @@ -17734,14 +17950,10 @@ def build_parameters(self): def _make_channel_base(name: str) -> AcquisitionChannel: """Return a copy of the named channel from liveController, or a bare-bones fallback.""" - try: - channels = self.liveController.get_channels(self.objectiveStore.current_objective) - for ch in channels: - if ch.name == name: - # Return a copy so inline-editor mutations don't corrupt the source - return ch.model_copy(deep=True) - except Exception: - pass + ch = self._find_channel(name) + if ch is not None: + # Return a copy so inline-editor mutations don't corrupt the source + return ch.model_copy(deep=True) # The fallback has no illumination-source mapping, so the acquisition # would run dark — warn loudly so the cause is diagnosable. self._log.warning( @@ -17896,16 +18108,29 @@ def refresh_channel_list(self) -> None: return names = [ch.name for ch in channels] + # Repopulating transiently selects other channels (clear() then + # addItems() auto-selects index 0), firing currentIndexChanged along + # the way and reseeding the exposure/gain/illum spinboxes from their + # base config. Block signals so only a genuine selection change (the + # previous channel no longer being available) triggers a reseed — + # otherwise the user's manually edited values would be silently lost + # even though the selected channel is unchanged from their perspective. prev_recording = self._recording_channel_name() + self._recording_ch_combo.blockSignals(True) self._populate_channel_combo(self._recording_ch_combo, names=names) + channel_changed = True if prev_recording and prev_recording in names: self._recording_ch_combo.setCurrentIndex(names.index(prev_recording)) + channel_changed = False elif prev_recording: self._log.warning( f"recording channel {prev_recording!r} is not available for the current " f"objective/profile; selection changed to {names[0]!r} — review the recording " f"exposure/gain/illumination before starting an acquisition" ) + self._recording_ch_combo.blockSignals(False) + if channel_changed: + self._on_recording_channel_changed(self._recording_ch_combo.currentIndex()) self._populate_channel_combo(self.combobox_zstack_add_channel, names=names) for name in [n for n in self._zstack_channel_names if n not in names]: self._log.info(f"removing z-stack channel row {name!r}: not available for the current objective") diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 40b808833..8bc474e3f 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -181,7 +181,7 @@ def _make_stub_scan_coordinates(): def simulated_widget_deps(tmp_path): """Provide lightweight stub dependencies for RecordZStackMultiPointWidget.""" stage = MagicMock() - stage.get_pos.return_value = MagicMock(z_mm=0.0) + stage.get_pos.return_value = MagicMock(x_mm=1.5, y_mm=2.5, z_mm=0.0) return dict( stage=stage, @@ -283,6 +283,19 @@ def test_add_zstack_channel_row_deduplicates(qtbot, simulated_widget_deps): assert w.zstack_channel_table.rowCount() == 1 +def test_zstack_channel_row_tooltip_shows_full_name(qtbot, simulated_widget_deps): + """The Channel column truncates long names visually (narrow Stretch column), + so the full name must be available as a tooltip on hover.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w._add_zstack_channel_row("Fluorescence 488 nm Ex") + + item = w.zstack_channel_table.item(0, 0) + assert item.toolTip() == "Fluorescence 488 nm Ex" + + def test_validate_helper_recording_bad_duration(): params = _base_params(recording_enabled=True, duration_s=0.0, zstack_enabled=False) assert _validate_record_zstack_params(**params) is not None @@ -1061,3 +1074,287 @@ def test_refresh_channel_list_warns_on_silent_selection_swap(qtbot, simulated_wi assert any( prev in rec.message and "Only Channel" in rec.message for rec in caplog.records ), f"no warning about the recording selection changing from {prev!r}" + + +def test_refresh_channel_list_preserves_user_edited_recording_settings(qtbot, simulated_widget_deps): + """R9: refresh_channel_list() must not silently reset the user's manually + edited recording exposure/gain/illumination when the selected channel is + still available afterward — even though repopulating the combo transiently + fires currentIndexChanged for other channels along the way.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w._recording_exp_spin.setValue(999.0) + w._recording_gain_spin.setValue(77.0) + w._recording_illum_spin.setValue(88.0) + + # Same channel list, same selection still available (e.g. an objective + # switch that doesn't change the configured channels). + w.refresh_channel_list() + + assert w._recording_exposure() == pytest.approx(999.0) + assert w._recording_gain() == pytest.approx(77.0) + assert w._recording_illumination() == pytest.approx(88.0) + + +# --------------------------------------------------------------------------- +# Channel-add seeds from the channel's configured live-controller settings +# (instead of the hardcoded 50 ms / 0 gain / 50% defaults). +# --------------------------------------------------------------------------- + + +def test_zstack_add_channel_seeds_from_channel_config(qtbot, simulated_widget_deps): + """Clicking + Add on a channel must seed its row from that channel's own + configured exposure/gain/illumination, not the hardcoded (50, 0, 50).""" + from control.widgets import RecordZStackMultiPointWidget + + simulated_widget_deps["liveController"].get_channels.return_value = [ + _make_live_channel("Fluorescence 488 nm Ex", exposure=123.0, gain=4.0, intensity=77.0), + ] + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.checkbox_zstack.setChecked(True) # unchecked group box disables its children + w.combobox_zstack_add_channel.setCurrentText("Fluorescence 488 nm Ex") + w.btn_zstack_add_channel.click() + + exposure, gain, illum = w._get_zstack_row_values("Fluorescence 488 nm Ex") + assert exposure == pytest.approx(123.0) + assert gain == pytest.approx(4.0) + assert illum == pytest.approx(77.0) + + +def test_recording_row_seeds_from_channel_config_on_construction(qtbot, simulated_widget_deps): + """The recording table's initial row must reflect the first channel's own + configured settings, not the hardcoded (50, 0, 50) defaults.""" + from control.widgets import RecordZStackMultiPointWidget + + simulated_widget_deps["liveController"].get_channels.return_value = [ + _make_live_channel("BF LED matrix full", exposure=12.0, gain=1.5, intensity=33.0), + _make_live_channel("Fluorescence 488 nm Ex", exposure=123.0, gain=4.0, intensity=77.0), + ] + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + assert w._recording_channel_name() == "BF LED matrix full" + assert w._recording_exposure() == pytest.approx(12.0) + assert w._recording_gain() == pytest.approx(1.5) + assert w._recording_illumination() == pytest.approx(33.0) + + +def test_recording_row_seeds_from_channel_config_on_selection_change(qtbot, simulated_widget_deps): + """Switching the recording channel combo must reseed exposure/gain/illum + from the newly selected channel's own configured settings.""" + from control.widgets import RecordZStackMultiPointWidget + + simulated_widget_deps["liveController"].get_channels.return_value = [ + _make_live_channel("BF LED matrix full", exposure=12.0, gain=1.5, intensity=33.0), + _make_live_channel("Fluorescence 488 nm Ex", exposure=123.0, gain=4.0, intensity=77.0), + ] + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w._recording_ch_combo.setCurrentText("Fluorescence 488 nm Ex") + + assert w._recording_exposure() == pytest.approx(123.0) + assert w._recording_gain() == pytest.approx(4.0) + assert w._recording_illumination() == pytest.approx(77.0) + + +# --------------------------------------------------------------------------- +# XY / Time tabbed row (mirrors WellplateMultiPointWidget, skipping Z). +# --------------------------------------------------------------------------- + + +def test_checkbox_xy_exists_and_defaults_checked(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + assert w.checkbox_xy.isChecked() is True + assert w.combobox_xy_mode.isEnabled() is True + # isHidden() reflects the explicit show/hide flag; isVisible() would be + # False regardless since the top-level widget itself is never shown(). + assert w.xy_controls_frame.isHidden() is False + + +def test_unchecking_xy_hides_scan_grid_controls(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.checkbox_xy.setChecked(False) + assert w.xy_controls_frame.isHidden() is True + assert w.combobox_xy_mode.isEnabled() is False + + w.checkbox_xy.setChecked(True) + assert w.xy_controls_frame.isHidden() is False + assert w.combobox_xy_mode.isEnabled() is True + + +def test_combobox_xy_mode_has_current_position_and_select_wells(qtbot, simulated_widget_deps): + """The XY mode combo offers both modes, defaulting to Select Wells so + existing tiling behavior is unchanged out of the box.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + items = [w.combobox_xy_mode.itemText(i) for i in range(w.combobox_xy_mode.count())] + assert items == ["Current Position", "Select Wells"] + assert w.combobox_xy_mode.currentText() == "Select Wells" + + +def test_selecting_current_position_mode_hides_scan_grid_controls(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.combobox_xy_mode.setCurrentText("Current Position") + assert w.xy_controls_frame.isHidden() is True + + w.combobox_xy_mode.setCurrentText("Select Wells") + assert w.xy_controls_frame.isHidden() is False + + +def test_current_position_mode_uses_live_stage_position(qtbot, simulated_widget_deps): + """Current Position mode must build a single region at the live stage + position via add_region, bypassing well selection entirely.""" + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + sc.has_regions.return_value = False + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + sc.set_well_coordinates.reset_mock() # construction seeds the default Select Wells mode + + w.combobox_xy_mode.setCurrentText("Current Position") + + sc.add_region.assert_called_with("current", 1.5, 2.5, 0.01, 0, "Square") + sc.set_well_coordinates.assert_not_called() + + +def test_unchecking_xy_forces_current_position_and_restores_on_recheck(qtbot, simulated_widget_deps): + """Mirrors WellplateMultiPointWidget: unchecking XY forces Current + Position (single stage-position FOV) and disables the mode combo; + re-checking restores whatever mode was previously selected.""" + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + sc.has_regions.return_value = False + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.checkbox_xy.setChecked(False) + assert w.combobox_xy_mode.currentText() == "Current Position" + assert w.combobox_xy_mode.isEnabled() is False + sc.add_region.assert_called_with("current", 1.5, 2.5, 0.01, 0, "Square") + + w.checkbox_xy.setChecked(True) + assert w.combobox_xy_mode.currentText() == "Select Wells" + assert w.combobox_xy_mode.isEnabled() is True + + +def test_toggling_xy_rebuilds_scan_region_exactly_once(qtbot, simulated_widget_deps): + """Toggling XY changes combobox_xy_mode's text, which already triggers + _update_scan_regions() via _on_xy_mode_changed — _on_xy_toggled must not + also call it unconditionally, or the region gets rebuilt twice (including + an extra stage.get_pos() query) per toggle.""" + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + sc.has_regions.return_value = False + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.stage.get_pos.reset_mock() + w.checkbox_xy.setChecked(False) + w.stage.get_pos.assert_called_once() + + w.stage.get_pos.reset_mock() + sc.set_well_coordinates.reset_mock() + w.checkbox_xy.setChecked(True) + sc.set_well_coordinates.assert_called_once() + + +def test_validate_current_position_mode_bypasses_well_selection(qtbot, simulated_widget_deps): + """With no wells selected, Current Position mode must still validate + (it doesn't need any wells), while Select Wells mode still requires one.""" + from control.widgets import RecordZStackMultiPointWidget + + sc = MagicMock() + sc.get_selected_wells.return_value = {} + simulated_widget_deps["scanCoordinates"] = sc + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("/tmp/test") + w.checkbox_zstack.setChecked(True) + w.entry_zmin.setValue(-1.0) + w.entry_zmax.setValue(1.0) + w.entry_step.setValue(1.0) + w._add_zstack_channel_row("BF LED matrix full") + + assert w.validate() is not None # Select Wells (default) with 0 wells: still rejected + + w.combobox_xy_mode.setCurrentText("Current Position") + assert w.validate() is None + + +# --------------------------------------------------------------------------- +# Time tabbed row (mirrors WellplateMultiPointWidget). +# --------------------------------------------------------------------------- + + +def test_checkbox_time_exists_and_defaults_unchecked(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + assert w.checkbox_time.isChecked() is False + assert w.time_controls_frame.isHidden() is True + + +def test_checking_time_shows_nt_dt_controls(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.checkbox_time.setChecked(True) + assert w.time_controls_frame.isHidden() is False + + +def test_unchecking_time_resets_and_restores_nt_dt(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + w.checkbox_time.setChecked(True) + w.entry_Nt.setValue(5) + w.entry_dt.setValue(10.0) + + w.checkbox_time.setChecked(False) + assert w.entry_Nt.value() == 1 + assert w.entry_dt.value() == pytest.approx(0.0) + assert w.time_controls_frame.isHidden() is True + + w.checkbox_time.setChecked(True) + assert w.entry_Nt.value() == 5 + assert w.entry_dt.value() == pytest.approx(10.0) From 82a4676d3cabd18a6093647133d76aa7dd3fd6b8 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 22:18:36 -0400 Subject: [PATCH 42/61] feat(record-zstack): add xy_mode/scan_size_mm/overlap_percent to acquisition params Co-Authored-By: Claude Sonnet 5 --- .../control/core/record_zstack_controller.py | 4 ++++ software/control/widgets.py | 3 +++ software/tests/test_record_zstack_widget.py | 16 ++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py index f3b560886..b254f953e 100644 --- a/software/control/core/record_zstack_controller.py +++ b/software/control/core/record_zstack_controller.py @@ -44,6 +44,10 @@ class RecordZStackAcquisitionParameters: z_min_um: float = -3.0 z_max_um: float = 3.0 z_step_um: float = 1.0 + # XY / well-selection state (needed to save a full reusable settings snapshot) + xy_mode: str = "Select Wells" + scan_size_mm: float = 0.1 + overlap_percent: float = 10.0 class RecordZStackController: diff --git a/software/control/widgets.py b/software/control/widgets.py index 8139143b8..5efee2fba 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -18005,6 +18005,9 @@ def _make_channel_base(name: str) -> AcquisitionChannel: z_min_um=self.entry_zmin.value(), z_max_um=self.entry_zmax.value(), z_step_um=self.entry_step.value(), + xy_mode=self.combobox_xy_mode.currentText(), + scan_size_mm=self.entry_scan_size.value(), + overlap_percent=self.entry_overlap.value(), ) def toggle_acquisition(self, pressed: bool) -> None: diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 8bc474e3f..d18b07719 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -271,6 +271,22 @@ def test_build_parameters_recording_phase(qtbot, simulated_widget_deps): assert params.recording_channel is not None +def test_build_parameters_includes_xy_mode_and_scan_settings(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText("/tmp/test") + w.combobox_xy_mode.setCurrentText("Current Position") + w.entry_scan_size.setValue(1.5) + w.entry_overlap.setValue(12.0) + + params = w.build_parameters() + assert params.xy_mode == "Current Position" + assert params.scan_size_mm == pytest.approx(1.5) + assert params.overlap_percent == pytest.approx(12.0) + + def test_add_zstack_channel_row_deduplicates(qtbot, simulated_widget_deps): from control.widgets import RecordZStackMultiPointWidget From 09f6dd26eb2500046b033612f9f737b3cce36034 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 22:22:59 -0400 Subject: [PATCH 43/61] feat(record-zstack): add _build_objective_info helper for acquisition.yaml --- .../control/core/record_zstack_controller.py | 28 +++++++++++++++ .../tests/core/test_record_zstack_worker.py | 36 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py index b254f953e..1e8ba52e2 100644 --- a/software/control/core/record_zstack_controller.py +++ b/software/control/core/record_zstack_controller.py @@ -25,6 +25,34 @@ def zstack_offsets_um(z_min_um: float, z_max_um: float, step_um: float) -> List[ return [round(z_min_um + i * step_um, 6) for i in range(zstack_plane_count(z_min_um, z_max_um, step_um))] +def _build_objective_info(objective_store, camera) -> dict: + """Build the informational `objective:` YAML section. + + Mirrors the dict multi_point_controller.py builds before calling + _save_acquisition_yaml, adapted to tolerate camera=None (the record widget's + Save-button path may not always have a live camera reference). + """ + current_objective = objective_store.current_objective + objective_dict = getattr(objective_store, "objectives_dict", {}).get(current_objective, {}) + + camera_binning = None + pixel_size_um = None + if camera is not None and hasattr(camera, "get_binning"): + camera_binning = list(camera.get_binning()) + if camera is not None and hasattr(camera, "get_pixel_size_binned_um"): + try: + pixel_size_um = objective_store.get_pixel_size_factor() * camera.get_pixel_size_binned_um() + except Exception: + pixel_size_um = None + + return { + "name": current_objective, + "magnification": objective_dict.get("magnification"), + "pixel_size_um": pixel_size_um, + "camera_binning": camera_binning, + } + + @dataclass class RecordZStackAcquisitionParameters: base_path: str diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index 8f83e367a..94859c0f9 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -754,3 +754,39 @@ def test_config_snapshot_dedupes_channels_by_name(tmp_path): snapshot = yaml.safe_load(open(Path(tmp_path) / params.experiment_id / "acquisition_channels.yaml")) names = [c["name"] for c in snapshot["channels"]] assert len(names) == len(set(names)), f"duplicate channel names in snapshot: {names}" + + +def test_build_objective_info_reads_objective_and_camera(): + from unittest.mock import MagicMock + from control.core.record_zstack_controller import _build_objective_info + + objective_store = MagicMock() + objective_store.current_objective = "20x" + objective_store.objectives_dict = {"20x": {"magnification": 20.0}} + objective_store.get_pixel_size_factor.return_value = 1.0 + + camera = MagicMock() + camera.get_binning.return_value = (2, 2) + camera.get_pixel_size_binned_um.return_value = 0.5 + + info = _build_objective_info(objective_store, camera) + + assert info["name"] == "20x" + assert info["magnification"] == 20.0 + assert info["camera_binning"] == [2, 2] + assert info["pixel_size_um"] == pytest.approx(0.5) + + +def test_build_objective_info_handles_missing_camera(): + from unittest.mock import MagicMock + from control.core.record_zstack_controller import _build_objective_info + + objective_store = MagicMock() + objective_store.current_objective = "10x" + objective_store.objectives_dict = {} + + info = _build_objective_info(objective_store, None) + + assert info["name"] == "10x" + assert info["camera_binning"] is None + assert info["pixel_size_um"] is None From d1fd8f4775797fe6d392e9a13d6fe9457a95c38d Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 22:32:21 -0400 Subject: [PATCH 44/61] feat(record-zstack): write full acquisition.yaml snapshot alongside acquisition_channels.yaml Adds _save_record_zstack_yaml(), wired into run_acquisition() right after the existing acquisition_channels.yaml snapshot call. Writes a superset settings snapshot (recording phase, z-stack phase, wellplate scan region list when xy_mode == "Select Wells") to acquisition.yaml in the same experiment directory, without touching the existing snapshot call. --- .../control/core/record_zstack_controller.py | 83 +++++++++++ .../tests/core/test_record_zstack_worker.py | 134 ++++++++++++++++++ 2 files changed, 217 insertions(+) diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py index 1e8ba52e2..f7f4cd68a 100644 --- a/software/control/core/record_zstack_controller.py +++ b/software/control/core/record_zstack_controller.py @@ -1,11 +1,16 @@ import math +import os +import time from dataclasses import dataclass, field from threading import Event, Thread from typing import List, Optional +import yaml + import squid.logging import control._def from control.models.acquisition_config import AcquisitionChannel +from control.utils import serialize_for_yaml as _serialize_for_yaml log = squid.logging.get_logger("RecordZStackController") @@ -53,6 +58,71 @@ def _build_objective_info(objective_store, camera) -> dict: } +def _save_record_zstack_yaml( + params: "RecordZStackAcquisitionParameters", + yaml_path: str, + scan_coordinates=None, + objective_info: dict = None, +) -> None: + """Save full record/z-stack acquisition settings to *yaml_path*. + + Mirrors multi_point_controller.py's _save_acquisition_yaml for the + wellplate/flexible widgets, but kept separate: this widget's shape + (recording phase with fps/duration/z-offset, z-stack as min/max/step, + two channel lists) doesn't fit the shared builder without polluting it + with fields the other two widgets have no reason to know about. + """ + yaml_dict = { + "acquisition": { + "experiment_id": params.experiment_id, + "start_time": time.strftime("%Y-%m-%d %H:%M:%S"), + "widget_type": "record_zstack", + "xy_mode": params.xy_mode, + }, + "objective": objective_info or {}, + "time_series": { + "nt": params.Nt, + "delta_t_s": params.dt_s, + }, + "autofocus": { + "laser_af": params.use_laser_af, + }, + "recording": { + "enabled": params.recording_enabled, + "channel": _serialize_for_yaml(params.recording_channel) if params.recording_channel else None, + "fps": params.fps, + "duration_s": params.duration_s, + "z_offset_um": params.recording_z_offset_um, + }, + "z_stack": { + "enabled": params.zstack_enabled, + "channels": [_serialize_for_yaml(ch) for ch in params.zstack_channels], + "z_min_um": params.z_min_um, + "z_max_um": params.z_max_um, + "z_step_um": params.z_step_um, + }, + } + + if params.xy_mode == "Select Wells": + region_centers = getattr(scan_coordinates, "region_centers", {}) or {} + region_shapes = getattr(scan_coordinates, "region_shapes", {}) or {} + yaml_dict["wellplate_scan"] = { + "scan_size_mm": params.scan_size_mm, + "overlap_percent": params.overlap_percent, + "regions": [ + {"name": name, "center_mm": _serialize_for_yaml(center), "shape": region_shapes.get(name)} + for name, center in region_centers.items() + ], + } + + try: + with open(yaml_path, "w", encoding="utf-8") as f: + f.write(f"# Record/Z-Stack Acquisition Parameters - {params.experiment_id}\n\n") + yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False, allow_unicode=True) + except (OSError, yaml.YAMLError) as exc: + log.error("Failed to write record_zstack acquisition YAML file '%s': %s", yaml_path, exc) + + @dataclass class RecordZStackAcquisitionParameters: base_path: str @@ -199,6 +269,19 @@ def run_acquisition(self, params: RecordZStackAcquisitionParameters) -> None: except Exception: log.exception("Failed to save acquisition settings snapshot to the experiment directory") + # Full reusable settings snapshot (superset of acquisition_channels.yaml above, + # written alongside it — not a replacement; see design doc's "Snapshot files" decision). + try: + objective_info = _build_objective_info(self._objective_store, getattr(self._microscope, "camera", None)) + _save_record_zstack_yaml( + params, + os.path.join(experiment_dir, "acquisition.yaml"), + self._scan_coordinates, + objective_info, + ) + except Exception: + log.exception("Failed to save full record_zstack acquisition.yaml snapshot") + # Collect scan coordinates: {region_id: [(x_mm, y_mm[, z_mm]), ...]} scan_region_fov_coords = {} if self._scan_coordinates is not None and hasattr(self._scan_coordinates, "region_fov_coordinates"): diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index 94859c0f9..e5cea11b0 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -790,3 +790,137 @@ def test_build_objective_info_handles_missing_camera(): assert info["name"] == "10x" assert info["camera_binning"] is None assert info["pixel_size_um"] is None + + +def test_save_record_zstack_yaml_writes_full_schema(tmp_path): + from control.core.record_zstack_controller import RecordZStackAcquisitionParameters, _save_record_zstack_yaml + from control.models.acquisition_config import AcquisitionChannel, CameraSettings, IlluminationSettings + + channel = AcquisitionChannel( + name="BF LED matrix full", + camera_settings=CameraSettings(exposure_time_ms=50.0, gain_mode=0.0), + illumination_settings=IlluminationSettings(intensity=50.0), + ) + params = RecordZStackAcquisitionParameters( + base_path=str(tmp_path), + experiment_id="my_exp", + Nt=3, + dt_s=5.0, + use_laser_af=True, + recording_enabled=True, + recording_channel=channel, + fps=15.0, + duration_s=2.0, + recording_z_offset_um=1.5, + zstack_enabled=True, + zstack_channels=[channel], + z_min_um=-2.0, + z_max_um=2.0, + z_step_um=0.5, + xy_mode="Select Wells", + scan_size_mm=1.2, + overlap_percent=10.0, + ) + + scan_coordinates = type( + "FakeScanCoordinates", (), {"region_centers": {"A1": [1.0, 2.0, 0.1]}, "region_shapes": {"A1": "Square"}} + )() + + yaml_path = tmp_path / "acquisition.yaml" + _save_record_zstack_yaml(params, str(yaml_path), scan_coordinates, {"name": "20x"}) + + import yaml as pyyaml + + data = pyyaml.safe_load(yaml_path.read_text()) + + assert data["acquisition"]["widget_type"] == "record_zstack" + assert data["acquisition"]["xy_mode"] == "Select Wells" + assert data["objective"]["name"] == "20x" + assert data["time_series"] == {"nt": 3, "delta_t_s": 5.0} + assert data["autofocus"] == {"laser_af": True} + assert data["recording"]["enabled"] is True + assert data["recording"]["channel"]["name"] == "BF LED matrix full" + assert data["recording"]["fps"] == 15.0 + assert data["recording"]["duration_s"] == 2.0 + assert data["recording"]["z_offset_um"] == 1.5 + assert data["z_stack"]["enabled"] is True + assert len(data["z_stack"]["channels"]) == 1 + assert data["z_stack"]["z_min_um"] == -2.0 + assert data["z_stack"]["z_max_um"] == 2.0 + assert data["z_stack"]["z_step_um"] == 0.5 + assert data["wellplate_scan"]["scan_size_mm"] == 1.2 + assert data["wellplate_scan"]["regions"] == [{"name": "A1", "center_mm": [1.0, 2.0, 0.1], "shape": "Square"}] + + +def test_save_record_zstack_yaml_omits_wellplate_scan_for_current_position(tmp_path): + from control.core.record_zstack_controller import RecordZStackAcquisitionParameters, _save_record_zstack_yaml + + params = RecordZStackAcquisitionParameters( + base_path=str(tmp_path), experiment_id="my_exp", xy_mode="Current Position" + ) + yaml_path = tmp_path / "acquisition.yaml" + _save_record_zstack_yaml(params, str(yaml_path)) + + import yaml as pyyaml + + data = pyyaml.safe_load(yaml_path.read_text()) + assert "wellplate_scan" not in data + + +def test_run_acquisition_writes_both_yaml_files(tmp_path, monkeypatch): + """acquisition_channels.yaml (existing) and acquisition.yaml (new) both land in the experiment dir. + + Stubs RecordZStackWorker entirely (a no-op .run()) so this test only exercises the + synchronous setup code in run_acquisition() — the YAML-writing calls happen before the + worker/thread are created. Exercising the real worker with mocked hardware would be slow + and flaky; that's covered by this module's existing worker-level tests instead. + """ + from unittest.mock import MagicMock + import control.core.record_zstack_controller as rzc + + monkeypatch.setattr("control._def.Acquisition.USE_MULTIPROCESSING", False) + + class DummyWorker: + def __init__(self, **kwargs): + pass + + def run(self): + pass + + # run_acquisition() does `from control.core.record_zstack_worker import RecordZStackWorker` + # as a local import at call time, so patching the source module's attribute (not + # record_zstack_controller's namespace) is what actually intercepts it. + monkeypatch.setattr("control.core.record_zstack_worker.RecordZStackWorker", DummyWorker) + + def _write_channels_yaml(output_dir, **kw): + # microscope is a full MagicMock, so save_acquisition_output's real disk write + # (which the existing acquisition_channels.yaml tests exercise for real) doesn't + # happen here; write a stand-in so the "both files exist" assertion reflects the + # real run_acquisition() call order rather than the (irrelevant) file contents. + (Path(output_dir) / "acquisition_channels.yaml").write_text("channels: []\n") + + microscope = MagicMock() + microscope.config_repo.save_acquisition_output.side_effect = _write_channels_yaml + microscope.camera.get_binning.return_value = (1, 1) + microscope.camera.get_pixel_size_binned_um.return_value = 0.5 + objective_store = MagicMock() + objective_store.current_objective = "20x" + objective_store.objectives_dict = {} + objective_store.get_pixel_size_factor.return_value = 1.0 + callbacks = MagicMock() + + controller = rzc.RecordZStackController( + microscope=microscope, + live_controller=MagicMock(), + laser_autofocus_controller=None, + objective_store=objective_store, + scan_coordinates=None, + callbacks=callbacks, + ) + params = rzc.RecordZStackAcquisitionParameters(base_path=str(tmp_path), experiment_id="my_exp") + controller.run_acquisition(params) + controller.join(timeout=5.0) + + experiment_dir = next(tmp_path.iterdir()) + assert (experiment_dir / "acquisition_channels.yaml").exists() + assert (experiment_dir / "acquisition.yaml").exists() From 6fea97fa4738913de18c0a51db2f04c7b65491f0 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 22:39:21 -0400 Subject: [PATCH 45/61] feat(record-zstack): parse record_zstack acquisition.yaml into RecordZStackYAMLData --- software/control/acquisition_yaml_loader.py | 76 +++++++++++++- .../control/test_acquisition_yaml_loader.py | 99 +++++++++++++++++++ 2 files changed, 174 insertions(+), 1 deletion(-) diff --git a/software/control/acquisition_yaml_loader.py b/software/control/acquisition_yaml_loader.py index e0e81ae25..eb50ac3a2 100644 --- a/software/control/acquisition_yaml_loader.py +++ b/software/control/acquisition_yaml_loader.py @@ -51,6 +51,77 @@ class AcquisitionYAMLData: flexible_positions: Optional[List[Dict]] = None # [{name, center_mm}, ...] +@dataclass +class RecordZStackYAMLData: + """Parsed record/z-stack acquisition YAML data structure.""" + + widget_type: str # "record_zstack" + xy_mode: str = "Select Wells" + + objective_name: Optional[str] = None + camera_binning: Optional[Tuple[int, int]] = None + + nt: int = 1 + delta_t_s: float = 0.0 + + laser_af: bool = False + + recording_enabled: bool = False + recording_channel: Optional[Dict] = None + fps: float = 10.0 + duration_s: float = 1.0 + recording_z_offset_um: float = 0.0 + + zstack_enabled: bool = False + zstack_channels: List[Dict] = field(default_factory=list) + z_min_um: float = -3.0 + z_max_um: float = 3.0 + z_step_um: float = 1.0 + + scan_size_mm: Optional[float] = None + overlap_percent: float = 10.0 + wellplate_regions: Optional[List[Dict]] = None + + +def _parse_camera_binning(obj: dict) -> Optional[Tuple[int, int]]: + binning = obj.get("camera_binning") + if binning and isinstance(binning, list) and len(binning) == 2: + return tuple(binning) + return None + + +def _parse_record_zstack_yaml_data(data: dict, acq: dict) -> RecordZStackYAMLData: + obj = data.get("objective", {}) + time_series = data.get("time_series", {}) + autofocus = data.get("autofocus", {}) + recording = data.get("recording", {}) + z_stack = data.get("z_stack", {}) + wellplate_scan = data.get("wellplate_scan", {}) + + return RecordZStackYAMLData( + widget_type="record_zstack", + xy_mode=acq.get("xy_mode", "Select Wells"), + objective_name=obj.get("name"), + camera_binning=_parse_camera_binning(obj), + nt=time_series.get("nt", 1), + delta_t_s=time_series.get("delta_t_s", 0.0), + laser_af=autofocus.get("laser_af", False), + recording_enabled=recording.get("enabled", False), + recording_channel=recording.get("channel"), + fps=recording.get("fps", 10.0), + duration_s=recording.get("duration_s", 1.0), + recording_z_offset_um=recording.get("z_offset_um", 0.0), + zstack_enabled=z_stack.get("enabled", False), + zstack_channels=z_stack.get("channels", []), + z_min_um=z_stack.get("z_min_um", -3.0), + z_max_um=z_stack.get("z_max_um", 3.0), + z_step_um=z_stack.get("z_step_um", 1.0), + scan_size_mm=wellplate_scan.get("scan_size_mm"), + overlap_percent=wellplate_scan.get("overlap_percent", 10.0), + wellplate_regions=wellplate_scan.get("regions"), + ) + + def parse_acquisition_yaml(file_path: str) -> AcquisitionYAMLData: """Parse acquisition YAML file and return structured data. @@ -82,11 +153,14 @@ def parse_acquisition_yaml(file_path: str) -> AcquisitionYAMLData: flexible_scan = data.get("flexible_scan", {}) # Validate widget_type - VALID_WIDGET_TYPES = ("wellplate", "flexible") + VALID_WIDGET_TYPES = ("wellplate", "flexible", "record_zstack") widget_type = acq.get("widget_type", "wellplate") if widget_type not in VALID_WIDGET_TYPES: raise ValueError(f"Invalid widget_type '{widget_type}'. Must be one of: {VALID_WIDGET_TYPES}") + if widget_type == "record_zstack": + return _parse_record_zstack_yaml_data(data, acq) + # Parse camera binning binning = obj.get("camera_binning") if binning and isinstance(binning, list) and len(binning) == 2: diff --git a/software/tests/control/test_acquisition_yaml_loader.py b/software/tests/control/test_acquisition_yaml_loader.py index 574abdad0..7c25e13b0 100644 --- a/software/tests/control/test_acquisition_yaml_loader.py +++ b/software/tests/control/test_acquisition_yaml_loader.py @@ -270,6 +270,105 @@ def test_parse_channels_with_missing_names(self, tmp_path): assert result.channel_names == ["Valid Channel", "Another Valid"] +class TestParseRecordZstackYAML: + """Tests for parse_acquisition_yaml with widget_type: record_zstack.""" + + def test_parse_record_zstack_yaml_select_wells(self, tmp_path): + yaml_content = """ +acquisition: + widget_type: record_zstack + xy_mode: Select Wells +objective: + name: 20x + camera_binning: + - 1 + - 1 +time_series: + nt: 3 + delta_t_s: 5.0 +autofocus: + laser_af: true +recording: + enabled: true + channel: + name: BF LED matrix full + fps: 15.0 + duration_s: 2.0 + z_offset_um: 1.5 +z_stack: + enabled: true + channels: + - name: Fluorescence 488 nm Ex + z_min_um: -2.0 + z_max_um: 2.0 + z_step_um: 0.5 +wellplate_scan: + scan_size_mm: 1.2 + overlap_percent: 12.0 + regions: + - name: A1 + center_mm: [1.0, 2.0, 0.1] + shape: Square +""" + yaml_file = tmp_path / "test_record.yaml" + yaml_file.write_text(yaml_content) + + result = parse_acquisition_yaml(str(yaml_file)) + + assert result.widget_type == "record_zstack" + assert result.xy_mode == "Select Wells" + assert result.objective_name == "20x" + assert result.camera_binning == (1, 1) + assert result.nt == 3 + assert result.delta_t_s == 5.0 + assert result.laser_af is True + assert result.recording_enabled is True + assert result.recording_channel == {"name": "BF LED matrix full"} + assert result.fps == 15.0 + assert result.duration_s == 2.0 + assert result.recording_z_offset_um == 1.5 + assert result.zstack_enabled is True + assert result.zstack_channels == [{"name": "Fluorescence 488 nm Ex"}] + assert result.z_min_um == -2.0 + assert result.z_max_um == 2.0 + assert result.z_step_um == 0.5 + assert result.scan_size_mm == 1.2 + assert result.overlap_percent == 12.0 + assert result.wellplate_regions == [{"name": "A1", "center_mm": [1.0, 2.0, 0.1], "shape": "Square"}] + + def test_parse_record_zstack_yaml_minimal_defaults(self, tmp_path): + yaml_content = """ +acquisition: + widget_type: record_zstack +""" + yaml_file = tmp_path / "test_minimal_record.yaml" + yaml_file.write_text(yaml_content) + + result = parse_acquisition_yaml(str(yaml_file)) + + assert result.widget_type == "record_zstack" + assert result.recording_enabled is False + assert result.recording_channel is None + assert result.zstack_channels == [] + assert result.wellplate_regions is None + + def test_validate_hardware_accepts_record_zstack_yaml_data(self, tmp_path): + """validate_hardware() duck-types .objective_name/.camera_binning; must work unchanged.""" + yaml_content = """ +acquisition: + widget_type: record_zstack +objective: + name: 20x + camera_binning: [1, 1] +""" + yaml_file = tmp_path / "test_record_hw.yaml" + yaml_file.write_text(yaml_content) + result = parse_acquisition_yaml(str(yaml_file)) + + validation = validate_hardware(result, current_objective="20x", current_binning=(1, 1)) + assert validation.is_valid is True + + class TestValidateHardware: """Tests for validate_hardware function.""" From d4198b1cd80eba889a64a79000fa3d6f5efc8535 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 22:45:51 -0400 Subject: [PATCH 46/61] refactor(widgets): generalize AcquisitionYAMLDropMixin for a 3rd widget_type Replace the hardcoded multipointController.camera lookup with an overridable _get_camera_for_binning_check() hook, and change _get_other_widget_name() to map an actual widget_type string to a display name via a lookup table instead of guessing a binary wellplate/flexible opposite. Behavior-preserving for WellplateMultiPointWidget and FlexibleMultiPointWidget; needed so RecordZStackMultiPointWidget can reuse the mixin. Co-Authored-By: Claude Sonnet 5 --- software/control/widgets.py | 33 ++++++++++++++----- .../tests/test_acquisition_yaml_drop_mixin.py | 27 +++++++++++++++ 2 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 software/tests/test_acquisition_yaml_drop_mixin.py diff --git a/software/control/widgets.py b/software/control/widgets.py index 5efee2fba..865a0a4ab 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -826,13 +826,21 @@ def apply_and_exit(self): self.close() +_ACQUISITION_WIDGET_TYPE_DISPLAY_NAMES = { + "wellplate": "Wellplate Multipoint", + "flexible": "Flexible Multipoint", + "record_zstack": "Record/Z-Stack", +} + + class AcquisitionYAMLDropMixin: """Mixin class providing drag-and-drop functionality for loading acquisition YAML files. Widgets using this mixin must: 1. Call `self.setAcceptDrops(True)` in __init__ - 2. Have `self._log`, `self.multipointController`, `self.objectiveStore` attributes - 3. Implement `_get_expected_widget_type()` returning "wellplate" or "flexible" + 2. Have `self._log`, `self.objectiveStore` attributes, and override + `_get_camera_for_binning_check()` unless they have `self.multipointController.camera` + 3. Implement `_get_expected_widget_type()` returning "wellplate", "flexible", or "record_zstack" 4. Implement `_apply_yaml_settings(yaml_data)` to apply settings to the widget """ @@ -900,11 +908,18 @@ def _get_expected_widget_type(self) -> str: """Return the expected widget_type for this widget. Override in subclass.""" raise NotImplementedError("Subclass must implement _get_expected_widget_type()") - def _get_other_widget_name(self) -> str: - """Return the name of the other widget type for error messages.""" - if self._get_expected_widget_type() == "wellplate": - return "Flexible Multipoint" - return "Wellplate Multipoint" + def _get_other_widget_name(self, actual_widget_type: str) -> str: + """Return the display name of the widget that handles *actual_widget_type* files.""" + return _ACQUISITION_WIDGET_TYPE_DISPLAY_NAMES.get(actual_widget_type, actual_widget_type) + + def _get_camera_for_binning_check(self): + """Return the camera used for the binning-mismatch check on load. + + Default assumes self.multipointController.camera (wellplate/flexible). + Widgets without a multipointController (e.g. RecordZStackMultiPointWidget) + must override this. + """ + return getattr(self.multipointController, "camera", None) def _load_acquisition_yaml(self, file_path: str) -> bool: """Load acquisition settings from YAML file. @@ -927,14 +942,14 @@ def _load_acquisition_yaml(self, file_path: str) -> bool: self, "Widget Type Mismatch", f"This YAML is for '{yaml_data.widget_type}' mode.\n" - f"Please drop this file on the {self._get_other_widget_name()} widget instead.", + f"Please drop this file on the {self._get_other_widget_name(yaml_data.widget_type)} widget instead.", ) return False # Validate hardware current_binning = (1, 1) try: - camera = getattr(self.multipointController, "camera", None) + camera = self._get_camera_for_binning_check() if camera and hasattr(camera, "get_binning"): current_binning = tuple(camera.get_binning()) except Exception as e: diff --git a/software/tests/test_acquisition_yaml_drop_mixin.py b/software/tests/test_acquisition_yaml_drop_mixin.py new file mode 100644 index 000000000..a22a7ecc7 --- /dev/null +++ b/software/tests/test_acquisition_yaml_drop_mixin.py @@ -0,0 +1,27 @@ +"""Tests for the two AcquisitionYAMLDropMixin generalizations needed to support a 3rd +widget_type (record_zstack) without breaking the existing wellplate/flexible behavior.""" + +from unittest.mock import MagicMock + +from control.widgets import AcquisitionYAMLDropMixin + + +class _HostWithMultipointController(AcquisitionYAMLDropMixin): + def __init__(self, camera): + self.multipointController = MagicMock(camera=camera) + + def _get_expected_widget_type(self): + return "wellplate" + + +def test_default_camera_hook_reads_multipoint_controller_camera(): + camera = object() + host = _HostWithMultipointController(camera) + assert host._get_camera_for_binning_check() is camera + + +def test_get_other_widget_name_maps_all_three_types(): + host = _HostWithMultipointController(camera=None) + assert host._get_other_widget_name("wellplate") == "Wellplate Multipoint" + assert host._get_other_widget_name("flexible") == "Flexible Multipoint" + assert host._get_other_widget_name("record_zstack") == "Record/Z-Stack" From 3ff22116f982b6c2502a11925c770d356b34f56a Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 22:51:03 -0400 Subject: [PATCH 47/61] fix(record-zstack): match mismatch-dialog display name to actual tab label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 5's review caught that "Record/Z-Stack" doesn't match the real tab label "Record + Z-Stack" (gui_hcs.py) — would confuse users once Task 7 wires the widget into the dialog. --- software/control/widgets.py | 2 +- software/tests/test_acquisition_yaml_drop_mixin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index 865a0a4ab..bb279f728 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -829,7 +829,7 @@ def apply_and_exit(self): _ACQUISITION_WIDGET_TYPE_DISPLAY_NAMES = { "wellplate": "Wellplate Multipoint", "flexible": "Flexible Multipoint", - "record_zstack": "Record/Z-Stack", + "record_zstack": "Record + Z-Stack", } diff --git a/software/tests/test_acquisition_yaml_drop_mixin.py b/software/tests/test_acquisition_yaml_drop_mixin.py index a22a7ecc7..f35721e99 100644 --- a/software/tests/test_acquisition_yaml_drop_mixin.py +++ b/software/tests/test_acquisition_yaml_drop_mixin.py @@ -24,4 +24,4 @@ def test_get_other_widget_name_maps_all_three_types(): host = _HostWithMultipointController(camera=None) assert host._get_other_widget_name("wellplate") == "Wellplate Multipoint" assert host._get_other_widget_name("flexible") == "Flexible Multipoint" - assert host._get_other_widget_name("record_zstack") == "Record/Z-Stack" + assert host._get_other_widget_name("record_zstack") == "Record + Z-Stack" From c0a246dbc1d27dd2fda96e1c0e94e41a9cc519fa Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 22:53:48 -0400 Subject: [PATCH 48/61] refactor(widgets): extract well-region YAML apply logic into shared functions --- software/control/widgets.py | 101 ++++++++---------- .../tests/test_acquisition_yaml_drop_mixin.py | 29 +++++ 2 files changed, 76 insertions(+), 54 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index bb279f728..bcc13013c 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -833,6 +833,52 @@ def apply_and_exit(self): } +def _parse_well_name(well_name: str): + """Parse well name like 'C4' to (row, col) indices. Returns (None, None) if unparseable.""" + match = re.match(r"^([A-Z]+)(\d+)$", well_name.upper()) + if not match: + return None, None + + row_str, col_str = match.groups() + row = 0 + for char in row_str: + row = row * 26 + (ord(char) - ord("A") + 1) + row -= 1 + col = int(col_str) - 1 + return row, col + + +def _load_well_regions(well_selection_widget, regions) -> None: + """Select *regions* (from a dropped acquisition YAML) in *well_selection_widget*. + + Shared by WellplateMultiPointWidget and RecordZStackMultiPointWidget, both of which + receive the same shared well-selection grid instance via gui_hcs.py. No-op when + well_selection_widget is None (glass-slide mode / not yet wired). + """ + if not well_selection_widget: + return + + well_selection_widget.blockSignals(True) + try: + well_selection_widget.clearSelection() + has_selection = False + for region in regions: + well_name = region.get("name", "") + if not well_name: + continue + row, col = _parse_well_name(well_name) + if row is not None and col is not None: + if row < well_selection_widget.rowCount() and col < well_selection_widget.columnCount(): + item = well_selection_widget.item(row, col) + if item: + item.setSelected(True) + has_selection = True + finally: + well_selection_widget.blockSignals(False) + + well_selection_widget.signal_wellSelected.emit(has_selection) + + class AcquisitionYAMLDropMixin: """Mixin class providing drag-and-drop functionality for loading acquisition YAML files. @@ -9297,7 +9343,7 @@ def _apply_yaml_settings(self, yaml_data): # Load well regions if present and update XY checkbox state if yaml_data.wellplate_regions: - self._load_well_regions(yaml_data.wellplate_regions) + _load_well_regions(self.well_selection_widget, yaml_data.wellplate_regions) self.checkbox_xy.setChecked(True) else: self.checkbox_xy.setChecked(False) @@ -9317,59 +9363,6 @@ def _apply_yaml_settings(self, yaml_data): self.update_tab_styles() self.update_coordinates() - def _load_well_regions(self, regions): - """Load well regions from YAML and select them in the well selector.""" - if not self.well_selection_widget: - return - - # Block signals during batch selection to prevent multiple updates - self.well_selection_widget.blockSignals(True) - - try: - # Clear current selection - self.well_selection_widget.clearSelection() - - has_selection = False - # Parse well names and select them - for region in regions: - well_name = region.get("name", "") - if not well_name: - continue - - # Parse well name (e.g., "C4" -> row=2, col=3) - row, col = self._parse_well_name(well_name) - if row is not None and col is not None: - # Check bounds - if row < self.well_selection_widget.rowCount() and col < self.well_selection_widget.columnCount(): - item = self.well_selection_widget.item(row, col) - if item: - item.setSelected(True) - has_selection = True - finally: - # Unblock signals - self.well_selection_widget.blockSignals(False) - - # Emit signal once to trigger coordinate update - self.well_selection_widget.signal_wellSelected.emit(has_selection) - - def _parse_well_name(self, well_name: str): - """Parse well name like 'C4' to (row, col) indices.""" - match = re.match(r"^([A-Z]+)(\d+)$", well_name.upper()) - if not match: - return None, None - - row_str, col_str = match.groups() - - # Convert row letters to index (A=0, B=1, ..., AA=26, etc.) - row = 0 - for char in row_str: - row = row * 26 + (ord(char) - ord("A") + 1) - row -= 1 # Convert to 0-based index - - col = int(col_str) - 1 # Convert to 0-based index - - return row, col - class MultiPointWithFluidicsWidget(_ApplyChannelOffsetMixin, QFrame): """A simplified version of WellplateMultiPointWidget for use with fluidics""" diff --git a/software/tests/test_acquisition_yaml_drop_mixin.py b/software/tests/test_acquisition_yaml_drop_mixin.py index f35721e99..0207ff814 100644 --- a/software/tests/test_acquisition_yaml_drop_mixin.py +++ b/software/tests/test_acquisition_yaml_drop_mixin.py @@ -25,3 +25,32 @@ def test_get_other_widget_name_maps_all_three_types(): assert host._get_other_widget_name("wellplate") == "Wellplate Multipoint" assert host._get_other_widget_name("flexible") == "Flexible Multipoint" assert host._get_other_widget_name("record_zstack") == "Record + Z-Stack" + + +def test_parse_well_name_basic(): + from control.widgets import _parse_well_name + + assert _parse_well_name("C4") == (2, 3) + assert _parse_well_name("A1") == (0, 0) + assert _parse_well_name("not-a-well") == (None, None) + + +def test_load_well_regions_selects_items_and_emits_signal(): + from control.widgets import _load_well_regions + + well_widget = MagicMock() + well_widget.rowCount.return_value = 8 + well_widget.columnCount.return_value = 12 + item = MagicMock() + well_widget.item.return_value = item + + _load_well_regions(well_widget, [{"name": "C4", "center_mm": [1, 2, 3], "shape": "Square"}]) + + item.setSelected.assert_called_once_with(True) + well_widget.signal_wellSelected.emit.assert_called_once_with(True) + + +def test_load_well_regions_noop_when_widget_is_none(): + from control.widgets import _load_well_regions + + _load_well_regions(None, [{"name": "C4"}]) # must not raise From 9c1ae70d125f64729eb731df5bef9f538701cb85 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 23:01:39 -0400 Subject: [PATCH 49/61] feat(record-zstack): wire RecordZStackMultiPointWidget into AcquisitionYAMLDropMixin --- software/control/widgets.py | 83 +++++++++++++++++++- software/tests/test_record_zstack_widget.py | 84 +++++++++++++++++++++ 2 files changed, 166 insertions(+), 1 deletion(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index bcc13013c..81ba4c03d 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -16997,7 +16997,7 @@ def _validate_record_zstack_params( return None -class RecordZStackMultiPointWidget(QFrame): +class RecordZStackMultiPointWidget(AcquisitionYAMLDropMixin, QFrame): """Single-column 'Record + Z-Stack' acquisition tab (Option-A layout). Construction pattern mirrors WellplateMultiPointWidget: @@ -17026,6 +17026,7 @@ def __init__( **kwargs, ): super().__init__(*args, **kwargs) + self.setAcceptDrops(True) # Enable drag-and-drop for loading acquisition YAML self._log = squid.logging.get_logger(self.__class__.__name__) self.stage = stage self.navigationViewer = navigationViewer @@ -17907,6 +17908,86 @@ def _update_scan_regions(self) -> None: except Exception as exc: self._log.warning(f"_update_scan_regions: failed: {exc}") + # ---------------------------------------------------------------------- AcquisitionYAMLDropMixin + + def _get_expected_widget_type(self) -> str: + return "record_zstack" + + def _get_camera_for_binning_check(self): + """RecordZStackMultiPointWidget has no multipointController; use liveController's camera.""" + return getattr(self.liveController, "camera", None) + + def _apply_yaml_settings(self, yaml_data) -> None: + """Apply parsed RecordZStackYAMLData to widget controls.""" + from control.models.acquisition_config import AcquisitionChannel + + widgets_to_block = [ + self.entry_Nt, + self.entry_dt, + self.checkbox_laser_af, + self.checkbox_recording, + self._recording_ch_combo, + self._recording_exp_spin, + self._recording_gain_spin, + self._recording_illum_spin, + self.entry_fps, + self.entry_duration, + self.entry_recording_z_offset, + self.checkbox_zstack, + self.entry_zmin, + self.entry_zmax, + self.entry_step, + self.combobox_xy_mode, + self.checkbox_xy, + self.entry_overlap, + self.entry_scan_size, + ] + for widget in widgets_to_block: + widget.blockSignals(True) + + try: + self.entry_Nt.setValue(yaml_data.nt) + self.entry_dt.setValue(yaml_data.delta_t_s) + self.checkbox_laser_af.setChecked(yaml_data.laser_af) + + self.checkbox_recording.setChecked(yaml_data.recording_enabled) + if yaml_data.recording_channel: + ch = AcquisitionChannel.model_validate(yaml_data.recording_channel) + idx = self._recording_ch_combo.findText(ch.name) + if idx >= 0: + self._recording_ch_combo.setCurrentIndex(idx) + self._recording_exp_spin.setValue(ch.exposure_time) + self._recording_gain_spin.setValue(ch.analog_gain) + self._recording_illum_spin.setValue(ch.illumination_intensity) + self.entry_fps.setValue(yaml_data.fps) + self.entry_duration.setValue(yaml_data.duration_s) + self.entry_recording_z_offset.setValue(yaml_data.recording_z_offset_um) + + self.checkbox_zstack.setChecked(yaml_data.zstack_enabled) + for name in list(self._zstack_channel_names): + self._remove_zstack_channel_row(name) + for ch_data in yaml_data.zstack_channels: + ch = AcquisitionChannel.model_validate(ch_data) + self._add_zstack_channel_row(ch.name, ch.exposure_time, ch.analog_gain, ch.illumination_intensity) + self.entry_zmin.setValue(yaml_data.z_min_um) + self.entry_zmax.setValue(yaml_data.z_max_um) + self.entry_step.setValue(yaml_data.z_step_um) + + if yaml_data.xy_mode in ("Current Position", "Select Wells"): + self.combobox_xy_mode.setCurrentText(yaml_data.xy_mode) + if yaml_data.scan_size_mm is not None: + self.entry_scan_size.setValue(yaml_data.scan_size_mm) + self.entry_overlap.setValue(yaml_data.overlap_percent) + + if yaml_data.wellplate_regions: + _load_well_regions(self.well_selection_widget, yaml_data.wellplate_regions) + self.checkbox_xy.setChecked(True) + finally: + for widget in widgets_to_block: + widget.blockSignals(False) + self._update_zstack_planes_label() + self._update_scan_regions() + def _laser_af_has_reference(self) -> bool: """Return True if the laser autofocus controller has a captured reference.""" ctrl = self.laser_autofocus_controller diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index d18b07719..3c23a44a0 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -1374,3 +1374,87 @@ def test_unchecking_time_resets_and_restores_nt_dt(qtbot, simulated_widget_deps) w.checkbox_time.setChecked(True) assert w.entry_Nt.value() == 5 assert w.entry_dt.value() == pytest.approx(10.0) + + +# --------------------------------------------------------------------------- +# AcquisitionYAMLDropMixin integration (Task 7) +# --------------------------------------------------------------------------- + + +def test_apply_yaml_settings_round_trips_all_fields(qtbot, simulated_widget_deps): + from control.acquisition_yaml_loader import RecordZStackYAMLData + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + yaml_data = RecordZStackYAMLData( + widget_type="record_zstack", + xy_mode="Current Position", + nt=4, + delta_t_s=2.5, + laser_af=True, + recording_enabled=True, + recording_channel={ + "name": "BF LED matrix full", + "camera_settings": {"exposure_time_ms": 33.0, "gain_mode": 1.0}, + "illumination_settings": {"intensity": 60.0}, + }, + fps=25.0, + duration_s=3.0, + recording_z_offset_um=2.0, + zstack_enabled=True, + zstack_channels=[ + { + "name": "Fluorescence 488 nm Ex", + "camera_settings": {"exposure_time_ms": 80.0, "gain_mode": 0.5}, + "illumination_settings": {"intensity": 40.0}, + } + ], + z_min_um=-4.0, + z_max_um=4.0, + z_step_um=2.0, + scan_size_mm=2.0, + overlap_percent=15.0, + ) + + w._apply_yaml_settings(yaml_data) + + assert w.entry_Nt.value() == 4 + assert w.entry_dt.value() == pytest.approx(2.5) + assert w.checkbox_laser_af.isChecked() is True + assert w.checkbox_recording.isChecked() is True + assert w._recording_channel_name() == "BF LED matrix full" + assert w._recording_exposure() == pytest.approx(33.0) + assert w._recording_gain() == pytest.approx(1.0) + assert w._recording_illumination() == pytest.approx(60.0) + assert w.entry_fps.value() == pytest.approx(25.0) + assert w.entry_duration.value() == pytest.approx(3.0) + assert w.entry_recording_z_offset.value() == pytest.approx(2.0) + assert w.checkbox_zstack.isChecked() is True + assert w._zstack_channel_names == ["Fluorescence 488 nm Ex"] + assert w._get_zstack_row_values("Fluorescence 488 nm Ex") == pytest.approx((80.0, 0.5, 40.0)) + assert w.entry_zmin.value() == pytest.approx(-4.0) + assert w.entry_zmax.value() == pytest.approx(4.0) + assert w.entry_step.value() == pytest.approx(2.0) + assert w.combobox_xy_mode.currentText() == "Current Position" + assert w.entry_scan_size.value() == pytest.approx(2.0) + assert w.entry_overlap.value() == pytest.approx(15.0) + + +def test_get_expected_widget_type_is_record_zstack(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + assert w._get_expected_widget_type() == "record_zstack" + + +def test_get_camera_for_binning_check_uses_live_controller(qtbot, simulated_widget_deps): + from control.widgets import RecordZStackMultiPointWidget + + camera = object() + simulated_widget_deps["liveController"].camera = camera + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + assert w._get_camera_for_binning_check() is camera From d15695120dcf15fc7150d218cc9d3eb69b6ba6c1 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 23:17:48 -0400 Subject: [PATCH 50/61] fix: RecordZStackMultiPointWidget._apply_yaml_settings channel/time bugs Task review findings (PR #564): - Recording channel exp/gain/illum spinboxes were set unconditionally even when the YAML channel name wasn't found in the current combo, silently pairing a stale display name with mismatched numeric settings. Now only applied when found; a warning is logged otherwise and the row is left untouched. - checkbox_time was never synced from yaml_data.nt, leaving the Time checkbox unchecked (and time_controls_frame hidden) after loading a multi-timepoint YAML even though entry_Nt/entry_dt were updated. Now checkbox_time is blocked/set like the sibling WellplateMultiPointWidget, and the frame's visibility is refreshed directly in the finally block (not via _on_time_toggled, which would clobber the just-loaded Nt/dt via its stored/restore side effect). Adds two regression tests in tests/test_record_zstack_widget.py. Co-Authored-By: Claude Sonnet 5 --- software/control/widgets.py | 20 +++++- software/tests/test_record_zstack_widget.py | 68 +++++++++++++++++++++ 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index 81ba4c03d..f6bc0c961 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17924,6 +17924,7 @@ def _apply_yaml_settings(self, yaml_data) -> None: widgets_to_block = [ self.entry_Nt, self.entry_dt, + self.checkbox_time, self.checkbox_laser_af, self.checkbox_recording, self._recording_ch_combo, @@ -17946,6 +17947,7 @@ def _apply_yaml_settings(self, yaml_data) -> None: widget.blockSignals(True) try: + self.checkbox_time.setChecked(yaml_data.nt > 1) self.entry_Nt.setValue(yaml_data.nt) self.entry_dt.setValue(yaml_data.delta_t_s) self.checkbox_laser_af.setChecked(yaml_data.laser_af) @@ -17956,9 +17958,14 @@ def _apply_yaml_settings(self, yaml_data) -> None: idx = self._recording_ch_combo.findText(ch.name) if idx >= 0: self._recording_ch_combo.setCurrentIndex(idx) - self._recording_exp_spin.setValue(ch.exposure_time) - self._recording_gain_spin.setValue(ch.analog_gain) - self._recording_illum_spin.setValue(ch.illumination_intensity) + self._recording_exp_spin.setValue(ch.exposure_time) + self._recording_gain_spin.setValue(ch.analog_gain) + self._recording_illum_spin.setValue(ch.illumination_intensity) + else: + self._log.warning( + f"_apply_yaml_settings: recording channel {ch.name!r} not found in current " + "objective's channels; keeping existing recording row settings" + ) self.entry_fps.setValue(yaml_data.fps) self.entry_duration.setValue(yaml_data.duration_s) self.entry_recording_z_offset.setValue(yaml_data.recording_z_offset_um) @@ -17985,6 +17992,13 @@ def _apply_yaml_settings(self, yaml_data) -> None: finally: for widget in widgets_to_block: widget.blockSignals(False) + # checkbox_time's toggled signal was blocked above, so the normal + # _on_time_toggled-driven visibility refresh didn't fire. Set the + # frame's visibility directly rather than calling _on_time_toggled + # itself, since that method also stores/restores Nt/dt via + # _stored_time_params — invoking it here could clobber the Nt/dt + # values just loaded from the YAML. + self.time_controls_frame.setVisible(self.checkbox_time.isChecked()) self._update_zstack_planes_label() self._update_scan_regions() diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 3c23a44a0..6ac37654d 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -1440,6 +1440,74 @@ def test_apply_yaml_settings_round_trips_all_fields(qtbot, simulated_widget_deps assert w.combobox_xy_mode.currentText() == "Current Position" assert w.entry_scan_size.value() == pytest.approx(2.0) assert w.entry_overlap.value() == pytest.approx(15.0) + assert w.checkbox_time.isChecked() is True + + +def test_apply_yaml_settings_unknown_recording_channel_keeps_existing_row(qtbot, simulated_widget_deps, caplog): + """Fix Round 1 / Finding 1: when the YAML's recording channel name isn't in the + current combo (e.g. objective changed, or the channel was renamed/removed since + the YAML was saved), the exposure/gain/illumination spinboxes must NOT be + silently paired with a channel that doesn't match the combo's selection.""" + import logging + + from control.acquisition_yaml_loader import RecordZStackYAMLData + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + # Pre-set the recording row to known, distinct values that must survive untouched. + w._recording_ch_combo.setCurrentIndex(0) + pre_channel_name = w._recording_ch_combo.currentText() + w._recording_exp_spin.setValue(11.0) + w._recording_gain_spin.setValue(2.0) + w._recording_illum_spin.setValue(22.0) + + yaml_data = RecordZStackYAMLData( + widget_type="record_zstack", + recording_enabled=True, + recording_channel={ + "name": "Channel Not In Combo", + "camera_settings": {"exposure_time_ms": 99.0, "gain_mode": 9.0}, + "illumination_settings": {"intensity": 99.0}, + }, + ) + + with caplog.at_level(logging.WARNING): + w._apply_yaml_settings(yaml_data) + + # Combo selection and spinbox values are unchanged -- no name/value mismatch. + assert w._recording_ch_combo.currentText() == pre_channel_name + assert w._recording_exp_spin.value() == pytest.approx(11.0) + assert w._recording_gain_spin.value() == pytest.approx(2.0) + assert w._recording_illum_spin.value() == pytest.approx(22.0) + assert any( + "Channel Not In Combo" in rec.message for rec in caplog.records + ), "expected a warning naming the missing recording channel" + + +def test_apply_yaml_settings_checks_time_checkbox_and_shows_frame(qtbot, simulated_widget_deps): + """Fix Round 1 / Finding 2: loading a multi-timepoint YAML (nt > 1) must check + checkbox_time and make the time-controls frame visible immediately, not just + update entry_Nt/entry_dt under the hood.""" + from control.acquisition_yaml_loader import RecordZStackYAMLData + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + # Sanity check on the default (unchecked) state before loading. + assert w.checkbox_time.isChecked() is False + assert w.time_controls_frame.isHidden() is True + + yaml_data = RecordZStackYAMLData(widget_type="record_zstack", nt=4, delta_t_s=2.0) + + w._apply_yaml_settings(yaml_data) + + assert w.checkbox_time.isChecked() is True + assert w.entry_Nt.value() == 4 + assert w.entry_dt.value() == pytest.approx(2.0) + assert w.time_controls_frame.isHidden() is False def test_get_expected_widget_type_is_record_zstack(qtbot, simulated_widget_deps): From e6d85c76f1f667f556079243a45e041e619960b8 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 23:28:21 -0400 Subject: [PATCH 51/61] fix: refresh Time tab styling in RecordZStackMultiPointWidget YAML load _apply_yaml_settings blocks checkbox_time's toggled signal while loading, so the normal _on_time_toggled -> _update_tab_styles path never fires. Fix Round 1 already restored checkbox state and frame visibility but left the frame's border/background stylesheet stale. _update_tab_styles() has no interaction with Nt/dt or _stored_time_params, so it's safe to call directly in the finally block. --- software/control/widgets.py | 8 ++++++ software/tests/test_record_zstack_widget.py | 32 +++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/software/control/widgets.py b/software/control/widgets.py index f6bc0c961..ef9029537 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17999,6 +17999,14 @@ def _apply_yaml_settings(self, yaml_data) -> None: # _stored_time_params — invoking it here could clobber the Nt/dt # values just loaded from the YAML. self.time_controls_frame.setVisible(self.checkbox_time.isChecked()) + # _update_tab_styles() only refreshes stylesheets on + # xy_frame/xy_controls_frame/time_frame/time_controls_frame based on + # the current checkbox states — it has no interaction with Nt/dt or + # _stored_time_params, so (unlike _on_time_toggled) it's safe to call + # directly here. Without it, the Time tab's border/background stays + # in its stale "inactive" styling even after the checkbox/frame + # visibility above are updated to reflect the loaded YAML. + self._update_tab_styles() self._update_zstack_planes_label() self._update_scan_regions() diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 6ac37654d..28f644046 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -1510,6 +1510,38 @@ def test_apply_yaml_settings_checks_time_checkbox_and_shows_frame(qtbot, simulat assert w.time_controls_frame.isHidden() is False +def test_apply_yaml_settings_refreshes_time_tab_styling(qtbot, simulated_widget_deps): + """Fix Round 2: loading a multi-timepoint YAML (nt > 1) must also refresh the + Time tab's stylesheet (border/background), not just the checkbox state and + frame visibility. checkbox_time.toggled is blocked during the load, so the + normal _on_time_toggled -> _update_tab_styles path never fires; the fix calls + _update_tab_styles() directly in the finally block. Compare against a second + widget where checkbox_time is toggled normally (unblocked) to avoid hardcoding + the expected stylesheet string.""" + from control.acquisition_yaml_loader import RecordZStackYAMLData + from control.widgets import RecordZStackMultiPointWidget + + w_loaded = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w_loaded) + + yaml_data = RecordZStackYAMLData(widget_type="record_zstack", nt=4, delta_t_s=2.0) + w_loaded._apply_yaml_settings(yaml_data) + + # Reference widget: toggle checkbox_time normally (signals not blocked), so + # _on_time_toggled -> _update_tab_styles runs through its ordinary path. + w_reference = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w_reference) + w_reference.checkbox_time.setChecked(True) + + assert w_reference.checkbox_time.isChecked() is True + assert w_loaded.time_frame.styleSheet() == w_reference.time_frame.styleSheet() + assert w_loaded.time_controls_frame.styleSheet() == w_reference.time_controls_frame.styleSheet() + # Guard against both sides trivially being empty strings (which would make + # the equality assertions above vacuous rather than a real regression check). + assert w_reference.time_frame.styleSheet() != "" + assert w_reference.time_controls_frame.styleSheet() != "" + + def test_get_expected_widget_type_is_record_zstack(qtbot, simulated_widget_deps): from control.widgets import RecordZStackMultiPointWidget From 7ed09076dad8c8e07ca5ba25abeea9a0419c88b2 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sat, 4 Jul 2026 23:47:28 -0400 Subject: [PATCH 52/61] feat(record-zstack): add Save Settings/Load Settings buttons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement Task 8 of the Record/Z-Stack feature. Add two buttons to the output group that allow users to save and load full acquisition settings via file dialogs: - "Save Settings…" button calls _on_save_settings_clicked to save current widget state (calls _update_scan_regions, builds parameters, and calls _save_record_zstack_yaml) - "Load Settings…" button calls _on_load_settings_clicked to load from a file (calls the mixin's _load_acquisition_yaml) Both buttons use QFileDialog for file selection. Error handling shows warning dialog on save failure. Tests verify button clicks are wired correctly and call underlying save/load machinery. Co-Authored-By: Claude Sonnet 5 --- software/control/widgets.py | 38 ++++++++++ software/tests/test_record_zstack_widget.py | 77 +++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/software/control/widgets.py b/software/control/widgets.py index ef9029537..36386adb9 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -17108,6 +17108,19 @@ def _build_output_group(self) -> QGroupBox: row2.addWidget(self.lineEdit_experimentID, 1) # stretch=1 → fills to the right edge vbox.addLayout(row2) + # Row 3: Save/Load full settings (reusable across acquisitions, unlike the + # per-run acquisition_channels.yaml audit snapshot). + row3 = QHBoxLayout() + row3.setSpacing(6) + self.btn_saveSettings = QPushButton("Save Settings…") + self.btn_saveSettings.clicked.connect(self._on_save_settings_clicked) + row3.addWidget(self.btn_saveSettings) + self.btn_loadSettings = QPushButton("Load Settings…") + self.btn_loadSettings.clicked.connect(self._on_load_settings_clicked) + row3.addWidget(self.btn_loadSettings) + row3.addStretch(1) + vbox.addLayout(row3) + return grp def _build_wells_fov_group(self) -> QGroupBox: @@ -17681,6 +17694,31 @@ def _browse_saving_dir(self) -> None: if path: self.lineEdit_savingDir.setText(path) + def _on_save_settings_clicked(self) -> None: + """Save full current settings to a user-chosen YAML file (no acquisition run required).""" + from control.core.record_zstack_controller import _build_objective_info, _save_record_zstack_yaml + + path, _ = QFileDialog.getSaveFileName( + self, "Save Record/Z-Stack Settings", "acquisition.yaml", "YAML Files (*.yaml *.yml)" + ) + if not path: + return + self._update_scan_regions() + params = self.build_parameters() + objective_info = _build_objective_info(self.objectiveStore, getattr(self.liveController, "camera", None)) + try: + _save_record_zstack_yaml(params, path, self.scanCoordinates, objective_info) + self._log.info(f"Settings saved to {path}") + except Exception as exc: + self._log.error(f"Failed to save settings: {exc}", exc_info=True) + QMessageBox.warning(self, "Save Error", f"Failed to save settings:\n{exc}") + + def _on_load_settings_clicked(self) -> None: + """Load full settings from a user-chosen YAML file via the same path as drag-and-drop.""" + path, _ = QFileDialog.getOpenFileName(self, "Load Record/Z-Stack Settings", "", "YAML Files (*.yaml *.yml)") + if path: + self._load_acquisition_yaml(path) + def _update_zstack_planes_label(self) -> None: from control.core.record_zstack_controller import zstack_plane_count diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 28f644046..900106fb2 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -174,6 +174,8 @@ def _make_stub_scan_coordinates(): """Stub scanCoordinates reporting 1 selected well.""" sc = MagicMock() sc.get_selected_wells.return_value = ["A1"] + sc.region_centers = {} + sc.region_shapes = {} return sc @@ -1558,3 +1560,78 @@ def test_get_camera_for_binning_check_uses_live_controller(qtbot, simulated_widg w = RecordZStackMultiPointWidget(**simulated_widget_deps) qtbot.addWidget(w) assert w._get_camera_for_binning_check() is camera + + +def test_save_settings_button_writes_yaml(qtbot, simulated_widget_deps, tmp_path, monkeypatch): + """Verify that clicking Save Settings button calls _save_record_zstack_yaml with correct parameters.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText(str(tmp_path)) + w.checkbox_zstack.setChecked(True) + w._add_zstack_channel_row("BF LED matrix full") + + # Mock the underlying save function to verify it gets called + save_called = [] + + def mock_save(params, path, scan_coords, objective_info): + save_called.append((params, path)) + # Write a minimal YAML file to satisfy the test assertion + with open(path, "w") as f: + f.write("acquisition:\n widget_type: record_zstack\n") + + monkeypatch.setattr("control.core.record_zstack_controller._save_record_zstack_yaml", mock_save) + + save_path = tmp_path / "my_preset.yaml" + monkeypatch.setattr("control.widgets.QFileDialog.getSaveFileName", lambda *a, **kw: (str(save_path), "")) + + w.btn_saveSettings.click() + + # Verify save was called with correct path + assert len(save_called) == 1 + assert save_called[0][1] == str(save_path) + + # Verify file was created with correct structure + assert save_path.exists() + import yaml as pyyaml + + data = pyyaml.safe_load(save_path.read_text()) + assert data["acquisition"]["widget_type"] == "record_zstack" + + +def test_load_settings_button_applies_yaml(qtbot, simulated_widget_deps, tmp_path, monkeypatch): + """Verify that clicking Load Settings button calls _load_acquisition_yaml with correct path.""" + from control.widgets import RecordZStackMultiPointWidget + from unittest.mock import patch + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + # Mock _load_acquisition_yaml to track the call and apply test data + load_called = [] + + def mock_load(self, path): + load_called.append(path) + # Simulate loading by calling _apply_yaml_settings with test data + from control.acquisition_yaml_loader import RecordZStackYAMLData + + yaml_data = RecordZStackYAMLData(widget_type="record_zstack", xy_mode="Current Position", nt=7, delta_t_s=1.0) + self._apply_yaml_settings(yaml_data) + + yaml_path = tmp_path / "preset.yaml" + yaml_path.write_text("dummy") # File just needs to exist + + monkeypatch.setattr("control.widgets.QFileDialog.getOpenFileName", lambda *a, **kw: (str(yaml_path), "")) + + # Patch the instance method + with patch.object(w, "_load_acquisition_yaml", mock_load.__get__(w, type(w))): + w.btn_loadSettings.click() + + # Verify _load_acquisition_yaml was called with correct path + assert len(load_called) == 1 + assert load_called[0] == str(yaml_path) + + # Verify settings were applied + assert w.entry_Nt.value() == 7 + assert w.combobox_xy_mode.currentText() == "Current Position" From 573d3692002a3d0bbdbe9153a270d9e0dfbfc86d Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 5 Jul 2026 00:00:18 -0400 Subject: [PATCH 53/61] fix(tests): exercise real save/load chains in record_zstack settings tests Save test now hardens its own objectiveStore/camera stubs so the real _build_objective_info() -> _save_record_zstack_yaml() runs unmocked and writes a real, parseable YAML file, instead of mocking the save function itself (a workaround for the shared fixture's un-serializable MagicMocks). Load test now writes a real, valid record_zstack YAML file and only mocks QFileDialog.getOpenFileName, so the full parse_acquisition_yaml -> validate_hardware -> _apply_yaml_settings chain is actually exercised instead of a hand-rolled substitute. Also drops a redundant local `from unittest.mock import patch` now that the load test no longer needs to patch the instance method. Co-Authored-By: Claude Sonnet 5 --- software/tests/test_record_zstack_widget.py | 88 ++++++++++++--------- 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 900106fb2..64df24a8d 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -1563,7 +1563,20 @@ def test_get_camera_for_binning_check_uses_live_controller(qtbot, simulated_widg def test_save_settings_button_writes_yaml(qtbot, simulated_widget_deps, tmp_path, monkeypatch): - """Verify that clicking Save Settings button calls _save_record_zstack_yaml with correct parameters.""" + """Verify that clicking Save Settings runs the REAL save chain end to end: + _on_save_settings_clicked -> build_parameters -> _build_objective_info -> + _save_record_zstack_yaml -> a real YAML file on disk. Only QFileDialog is mocked + (real file dialogs can't run in tests). + + simulated_widget_deps's stub objectiveStore/liveController are plain MagicMocks + with no objectives_dict/camera attributes configured, so the real + _build_objective_info() would otherwise produce MagicMock leaf values (e.g. for + magnification/pixel_size_um) that yaml.dump() cannot serialize -- a failure that + _save_record_zstack_yaml() swallows internally (logs and returns) via its own + pre-existing try/except. Hardening just these two stubs (not the shared fixture) + gives _build_objective_info() cleanly-serializable values so the real save runs + for real and actually writes the file. + """ from control.widgets import RecordZStackMultiPointWidget w = RecordZStackMultiPointWidget(**simulated_widget_deps) @@ -1572,66 +1585,67 @@ def test_save_settings_button_writes_yaml(qtbot, simulated_widget_deps, tmp_path w.checkbox_zstack.setChecked(True) w._add_zstack_channel_row("BF LED matrix full") - # Mock the underlying save function to verify it gets called - save_called = [] - - def mock_save(params, path, scan_coords, objective_info): - save_called.append((params, path)) - # Write a minimal YAML file to satisfy the test assertion - with open(path, "w") as f: - f.write("acquisition:\n widget_type: record_zstack\n") + # Harden this test's own objectiveStore/camera stubs so _build_objective_info() + # returns real, YAML-serializable values instead of MagicMock leaves. + w.objectiveStore.objectives_dict = {"10x": {"magnification": 10.0}} + w.objectiveStore.current_objective = "10x" + w.objectiveStore.get_pixel_size_factor.return_value = 1.0 - monkeypatch.setattr("control.core.record_zstack_controller._save_record_zstack_yaml", mock_save) + camera = MagicMock() + camera.get_binning.return_value = (1, 1) + camera.get_pixel_size_binned_um.return_value = 0.5 + w.liveController.camera = camera save_path = tmp_path / "my_preset.yaml" monkeypatch.setattr("control.widgets.QFileDialog.getSaveFileName", lambda *a, **kw: (str(save_path), "")) w.btn_saveSettings.click() - # Verify save was called with correct path - assert len(save_called) == 1 - assert save_called[0][1] == str(save_path) - - # Verify file was created with correct structure + # Verify the REAL file was written by the REAL _save_record_zstack_yaml call. assert save_path.exists() import yaml as pyyaml data = pyyaml.safe_load(save_path.read_text()) assert data["acquisition"]["widget_type"] == "record_zstack" + assert data["z_stack"]["enabled"] is True + assert data["z_stack"]["channels"][0]["name"] == "BF LED matrix full" + assert data["objective"]["name"] == "10x" + assert data["objective"]["magnification"] == pytest.approx(10.0) + assert data["objective"]["pixel_size_um"] == pytest.approx(0.5) + assert data["objective"]["camera_binning"] == [1, 1] def test_load_settings_button_applies_yaml(qtbot, simulated_widget_deps, tmp_path, monkeypatch): - """Verify that clicking Load Settings button calls _load_acquisition_yaml with correct path.""" + """Verify that clicking Load Settings runs the REAL load chain end to end: + _on_load_settings_clicked -> _load_acquisition_yaml -> parse_acquisition_yaml -> + validate_hardware -> _apply_yaml_settings. Only QFileDialog is mocked (real file + dialogs can't run in tests); a real, valid record_zstack YAML file is written to + disk and actually read back. + + The YAML has no objective:/camera_binning keys, so yaml_data.objective_name and + yaml_data.camera_binning are both falsy and validate_hardware() reports + is_valid=True -- no mismatch dialog blocks the test. + """ from control.widgets import RecordZStackMultiPointWidget - from unittest.mock import patch w = RecordZStackMultiPointWidget(**simulated_widget_deps) qtbot.addWidget(w) - # Mock _load_acquisition_yaml to track the call and apply test data - load_called = [] - - def mock_load(self, path): - load_called.append(path) - # Simulate loading by calling _apply_yaml_settings with test data - from control.acquisition_yaml_loader import RecordZStackYAMLData - - yaml_data = RecordZStackYAMLData(widget_type="record_zstack", xy_mode="Current Position", nt=7, delta_t_s=1.0) - self._apply_yaml_settings(yaml_data) - yaml_path = tmp_path / "preset.yaml" - yaml_path.write_text("dummy") # File just needs to exist + yaml_path.write_text( + "acquisition:\n" + " widget_type: record_zstack\n" + " xy_mode: Current Position\n" + "time_series:\n" + " nt: 7\n" + " delta_t_s: 1.0\n" + ) monkeypatch.setattr("control.widgets.QFileDialog.getOpenFileName", lambda *a, **kw: (str(yaml_path), "")) - # Patch the instance method - with patch.object(w, "_load_acquisition_yaml", mock_load.__get__(w, type(w))): - w.btn_loadSettings.click() - - # Verify _load_acquisition_yaml was called with correct path - assert len(load_called) == 1 - assert load_called[0] == str(yaml_path) + w.btn_loadSettings.click() - # Verify settings were applied + # Verify settings were applied via the REAL parse -> validate -> apply chain. assert w.entry_Nt.value() == 7 + assert w.entry_dt.value() == pytest.approx(1.0) assert w.combobox_xy_mode.currentText() == "Current Position" From 2ec6cf61699f1f97639c73a8824e3a79b5d62092 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 5 Jul 2026 00:11:10 -0400 Subject: [PATCH 54/61] test(record-zstack): add full save/load round-trip regression test --- software/tests/test_record_zstack_widget.py | 42 +++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 64df24a8d..02acb8faa 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -1649,3 +1649,45 @@ def test_load_settings_button_applies_yaml(qtbot, simulated_widget_deps, tmp_pat assert w.entry_Nt.value() == 7 assert w.entry_dt.value() == pytest.approx(1.0) assert w.combobox_xy_mode.currentText() == "Current Position" + + +def test_full_save_load_round_trip_preserves_settings(qtbot, simulated_widget_deps, tmp_path): + """Save via build_parameters()+_save_record_zstack_yaml, load into a fresh widget, + and confirm the fresh widget's build_parameters() matches (excluding base_path/experiment_id).""" + from control.core.record_zstack_controller import _save_record_zstack_yaml + from control.acquisition_yaml_loader import parse_acquisition_yaml + from control.widgets import RecordZStackMultiPointWidget + + w1 = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w1) + w1.lineEdit_savingDir.setText(str(tmp_path)) + w1.checkbox_recording.setChecked(True) + w1.entry_fps.setValue(30.0) + w1.entry_duration.setValue(4.0) + w1.checkbox_zstack.setChecked(True) + w1._add_zstack_channel_row("Fluorescence 488 nm Ex", exposure=80.0, gain=1.0, illumination=45.0) + w1.entry_zmin.setValue(-5.0) + w1.entry_zmax.setValue(5.0) + w1.entry_step.setValue(2.5) + w1.combobox_xy_mode.setCurrentText("Current Position") + + params1 = w1.build_parameters() + yaml_path = tmp_path / "roundtrip.yaml" + _save_record_zstack_yaml(params1, str(yaml_path)) + + yaml_data = parse_acquisition_yaml(str(yaml_path)) + + w2 = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w2) + w2._apply_yaml_settings(yaml_data) + params2 = w2.build_parameters() + + assert params2.recording_enabled == params1.recording_enabled + assert params2.fps == pytest.approx(params1.fps) + assert params2.duration_s == pytest.approx(params1.duration_s) + assert params2.zstack_enabled == params1.zstack_enabled + assert [c.name for c in params2.zstack_channels] == [c.name for c in params1.zstack_channels] + assert params2.z_min_um == pytest.approx(params1.z_min_um) + assert params2.z_max_um == pytest.approx(params1.z_max_um) + assert params2.z_step_um == pytest.approx(params1.z_step_um) + assert params2.xy_mode == params1.xy_mode From 8f8f2b6177a03826d5ef93c2cc59f0f75964e41f Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 5 Jul 2026 00:33:51 -0400 Subject: [PATCH 55/61] test: Add assertions for recording_channel and z-stack channel numeric fields The test_full_save_load_round_trip_preserves_settings test was only checking channel names and basic parameters, missing critical assertions on: - recording_channel object equality (the actual channel object round-trips) - z-stack channels' numeric fields (exposure_time, analog_gain, illumination_intensity) Added comprehensive assertions to verify that these values survive YAML serialization/deserialization round-trips. Updated docstring to accurately describe what's now tested. Co-Authored-By: Claude Sonnet 5 --- software/tests/test_record_zstack_widget.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 02acb8faa..a9800dfb8 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -1653,7 +1653,13 @@ def test_load_settings_button_applies_yaml(qtbot, simulated_widget_deps, tmp_pat def test_full_save_load_round_trip_preserves_settings(qtbot, simulated_widget_deps, tmp_path): """Save via build_parameters()+_save_record_zstack_yaml, load into a fresh widget, - and confirm the fresh widget's build_parameters() matches (excluding base_path/experiment_id).""" + and confirm the fresh widget's build_parameters() output matches exactly (excluding base_path/experiment_id). + + Tests round-trip serialization and deserialization via YAML for all user-facing settings: + - recording channel and its numeric settings (exposure_time, analog_gain, illumination_intensity) + - z-stack channels and their numeric settings + - acquisition parameters (FPS, duration, z-range, XY mode) + """ from control.core.record_zstack_controller import _save_record_zstack_yaml from control.acquisition_yaml_loader import parse_acquisition_yaml from control.widgets import RecordZStackMultiPointWidget @@ -1682,12 +1688,23 @@ def test_full_save_load_round_trip_preserves_settings(qtbot, simulated_widget_de w2._apply_yaml_settings(yaml_data) params2 = w2.build_parameters() + # Acquisition control parameters assert params2.recording_enabled == params1.recording_enabled assert params2.fps == pytest.approx(params1.fps) assert params2.duration_s == pytest.approx(params1.duration_s) assert params2.zstack_enabled == params1.zstack_enabled - assert [c.name for c in params2.zstack_channels] == [c.name for c in params1.zstack_channels] assert params2.z_min_um == pytest.approx(params1.z_min_um) assert params2.z_max_um == pytest.approx(params1.z_max_um) assert params2.z_step_um == pytest.approx(params1.z_step_um) assert params2.xy_mode == params1.xy_mode + + # Recording channel (enabled above, so should be non-None) + assert params2.recording_channel == params1.recording_channel + + # Z-stack channels: check channel names and numeric settings + assert [c.name for c in params2.zstack_channels] == [c.name for c in params1.zstack_channels] + assert len(params2.zstack_channels) == len(params1.zstack_channels) + for ch2, ch1 in zip(params2.zstack_channels, params1.zstack_channels): + assert ch2.exposure_time == pytest.approx(ch1.exposure_time) + assert ch2.analog_gain == pytest.approx(ch1.analog_gain) + assert ch2.illumination_intensity == pytest.approx(ch1.illumination_intensity) From 538573520700be05ec79f956c52fa0b22adbb1ef Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 5 Jul 2026 01:04:28 -0400 Subject: [PATCH 56/61] fix(record-zstack): resync xy_controls_frame visibility after YAML load _apply_yaml_settings()'s finally block resynced time_controls_frame after signal-blocked mutation (checkbox_time.toggled was blocked during load), but missed the equivalent for XY: checkbox_xy.toggled and combobox_xy_mode.currentTextChanged were also blocked, so _on_xy_mode_changed's visibility refresh never fired. A loaded YAML with xy_mode="Current Position" left the FOV overlap/shape/size controls visible even though they don't apply in that mode. Co-Authored-By: Claude Sonnet 5 --- software/control/widgets.py | 11 +++++++++ software/tests/test_record_zstack_widget.py | 27 +++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/software/control/widgets.py b/software/control/widgets.py index 36386adb9..27d259c6f 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -18037,6 +18037,17 @@ def _apply_yaml_settings(self, yaml_data) -> None: # _stored_time_params — invoking it here could clobber the Nt/dt # values just loaded from the YAML. self.time_controls_frame.setVisible(self.checkbox_time.isChecked()) + # checkbox_xy's toggled signal and combobox_xy_mode's currentTextChanged + # signal were both blocked above too, so the normal + # _on_xy_toggled/_on_xy_mode_changed-driven visibility refresh didn't + # fire either. Mirror _on_xy_mode_changed's condition directly (rather + # than calling it) for the same reason as checkbox_time above: calling + # the handlers could re-trigger stored-mode restore logic that would + # clobber the xy_mode just loaded from the YAML. + self.combobox_xy_mode.setEnabled(self.checkbox_xy.isChecked()) + self.xy_controls_frame.setVisible( + self.checkbox_xy.isChecked() and self.combobox_xy_mode.currentText() == "Select Wells" + ) # _update_tab_styles() only refreshes stylesheets on # xy_frame/xy_controls_frame/time_frame/time_controls_frame based on # the current checkbox states — it has no interaction with Nt/dt or diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index a9800dfb8..cbc956632 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -1544,6 +1544,33 @@ def test_apply_yaml_settings_refreshes_time_tab_styling(qtbot, simulated_widget_ assert w_reference.time_controls_frame.styleSheet() != "" +def test_apply_yaml_settings_resyncs_xy_controls_frame_visibility(qtbot, simulated_widget_deps): + """Final-review Finding 1: like checkbox_time's toggled signal, checkbox_xy's + toggled signal (and combobox_xy_mode's currentTextChanged) are blocked during + the load, so the normal _on_xy_mode_changed-driven visibility refresh never + fires. Loading a YAML with xy_mode="Current Position" onto a widget that starts + in its default state (XY checked, Select Wells, controls visible) must still + hide xy_controls_frame and disable combobox_xy_mode's peer state correctly.""" + from control.acquisition_yaml_loader import RecordZStackYAMLData + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + # Sanity check on the default state before loading. + assert w.checkbox_xy.isChecked() is True + assert w.combobox_xy_mode.currentText() == "Select Wells" + assert w.xy_controls_frame.isHidden() is False + + yaml_data = RecordZStackYAMLData(widget_type="record_zstack", xy_mode="Current Position") + + w._apply_yaml_settings(yaml_data) + + assert w.combobox_xy_mode.currentText() == "Current Position" + # The FOV overlap/shape/size controls don't apply in Current Position mode. + assert w.xy_controls_frame.isHidden() is True + + def test_get_expected_widget_type_is_record_zstack(qtbot, simulated_widget_deps): from control.widgets import RecordZStackMultiPointWidget From ae60e1ef8ac7e2a1f5e1d0bdae4a35639412d34f Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 5 Jul 2026 01:05:17 -0400 Subject: [PATCH 57/61] fix(record-zstack): catch _apply_yaml_settings failures in the shared drop mixin _load_acquisition_yaml() (AcquisitionYAMLDropMixin, shared by wellplate/ flexible/record_zstack widgets) called self._apply_yaml_settings(yaml_data) with no guard. RecordZStackMultiPointWidget is the first (and only) consumer whose _apply_yaml_settings() calls AcquisitionChannel.model_validate() on embedded channel dicts, so a syntactically-valid YAML with a malformed channel (e.g. missing a required field) raised a pydantic ValidationError straight through the drop/button Qt slot instead of showing a warning. Wrap the call the same way the existing parse-error handling above it does: log, show a "Load Error" QMessageBox, and return False. Purely additive for the success path of all three widgets. Co-Authored-By: Claude Sonnet 5 --- software/control/widgets.py | 14 +++++++-- software/tests/test_record_zstack_widget.py | 32 +++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index 27d259c6f..a84a33347 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -1012,8 +1012,18 @@ def _load_acquisition_yaml(self, file_path: str) -> bool: dialog.exec_() return False - # Apply settings with signal blocking - self._apply_yaml_settings(yaml_data) + # Apply settings with signal blocking. Subclasses (currently only + # RecordZStackMultiPointWidget) may validate embedded channel dicts via + # pydantic inside _apply_yaml_settings(); a malformed-but-syntactically-valid + # YAML can raise there. Catch broadly here (mirroring the parse-error + # handling above) so a bad drop/button-load degrades to a warning dialog + # instead of crashing the Qt slot. + try: + self._apply_yaml_settings(yaml_data) + except Exception as e: + self._log.error(f"Failed to apply YAML settings from {file_path}: {e}") + QMessageBox.warning(self, "Load Error", f"Failed to apply YAML settings:\n{e}") + return False self._log.info(f"Loaded acquisition settings from: {file_path}") return True diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index cbc956632..997e33cdc 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -1678,6 +1678,38 @@ def test_load_settings_button_applies_yaml(qtbot, simulated_widget_deps, tmp_pat assert w.combobox_xy_mode.currentText() == "Current Position" +def test_load_acquisition_yaml_malformed_channel_shows_warning_not_exception(qtbot, simulated_widget_deps, tmp_path): + """Final-review Finding 2: _apply_yaml_settings() calls + AcquisitionChannel.model_validate() on recording_channel/zstack_channels with no + guard. A YAML with valid syntax/widget_type but a malformed channel dict (here, + missing the required camera_settings field) must not crash the drop/button slot + with an uncaught pydantic ValidationError -- the shared + AcquisitionYAMLDropMixin._load_acquisition_yaml() must catch it, log, warn, and + return False, exactly like the existing YAML-parse-error path a few lines above it.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + + yaml_path = tmp_path / "malformed.yaml" + yaml_path.write_text( + "acquisition:\n" + " widget_type: record_zstack\n" + " xy_mode: Current Position\n" + "recording:\n" + " enabled: true\n" + " channel:\n" + " name: Foo\n" # missing required camera_settings -> pydantic ValidationError + ) + + with patch("control.widgets.QMessageBox.warning") as mock_warn: + result = w._load_acquisition_yaml(str(yaml_path)) + + assert result is False + mock_warn.assert_called_once() + assert mock_warn.call_args[0][1] == "Load Error" + + def test_full_save_load_round_trip_preserves_settings(qtbot, simulated_widget_deps, tmp_path): """Save via build_parameters()+_save_record_zstack_yaml, load into a fresh widget, and confirm the fresh widget's build_parameters() output matches exactly (excluding base_path/experiment_id). From 0397d47f60108e3fc27cc8ed6e9943a07f9f393b Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 5 Jul 2026 01:07:33 -0400 Subject: [PATCH 58/61] fix(record-zstack): let real save-YAML write failures reach the caller _save_record_zstack_yaml() wrapped its open()/yaml.dump() call in a try/except (OSError, yaml.YAMLError): log.error(...) that swallowed the error and returned normally. That's correct for nothing on its own -- it made the interactive "Save Settings..." button handler (_on_save_settings_clicked) log "Settings saved" and show no warning even when the file was never written, because its own try/except QMessageBox.warning guard never got a chance to fire. Remove the inner swallow and let OSError/yaml.YAMLError propagate. Both real call sites already handle this correctly one level up: - run_acquisition()'s snapshot call site already wraps this in try/except Exception: log.exception(...), so a failed settings snapshot still can't abort a real acquisition. - _on_save_settings_clicked() already wraps this in try/except Exception: QMessageBox.warning(...), which is now reachable. Co-Authored-By: Claude Sonnet 5 --- .../control/core/record_zstack_controller.py | 16 ++-- .../tests/core/test_record_zstack_worker.py | 74 +++++++++++++++++++ software/tests/test_record_zstack_widget.py | 35 +++++++++ 3 files changed, 119 insertions(+), 6 deletions(-) diff --git a/software/control/core/record_zstack_controller.py b/software/control/core/record_zstack_controller.py index f7f4cd68a..0d88a88fb 100644 --- a/software/control/core/record_zstack_controller.py +++ b/software/control/core/record_zstack_controller.py @@ -115,12 +115,16 @@ def _save_record_zstack_yaml( ], } - try: - with open(yaml_path, "w", encoding="utf-8") as f: - f.write(f"# Record/Z-Stack Acquisition Parameters - {params.experiment_id}\n\n") - yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False, allow_unicode=True) - except (OSError, yaml.YAMLError) as exc: - log.error("Failed to write record_zstack acquisition YAML file '%s': %s", yaml_path, exc) + # Let OSError/yaml.YAMLError propagate: both real call sites already handle + # failures appropriately one level up -- run_acquisition()'s snapshot call site + # wraps this in try/except Exception: log.exception(...) so a failed settings + # snapshot never aborts a real acquisition, and the Save Settings button + # handler (_on_save_settings_clicked) wraps this in try/except Exception: + # QMessageBox.warning(...) so the user is told the save failed. Swallowing the + # error here made that button's warning dialog unreachable. + with open(yaml_path, "w", encoding="utf-8") as f: + f.write(f"# Record/Z-Stack Acquisition Parameters - {params.experiment_id}\n\n") + yaml.dump(yaml_dict, f, default_flow_style=False, sort_keys=False, allow_unicode=True) @dataclass diff --git a/software/tests/core/test_record_zstack_worker.py b/software/tests/core/test_record_zstack_worker.py index e5cea11b0..dcf37aa9a 100644 --- a/software/tests/core/test_record_zstack_worker.py +++ b/software/tests/core/test_record_zstack_worker.py @@ -867,6 +867,80 @@ def test_save_record_zstack_yaml_omits_wellplate_scan_for_current_position(tmp_p assert "wellplate_scan" not in data +def test_save_record_zstack_yaml_raises_on_write_failure(tmp_path): + """Final-review Finding 3: _save_record_zstack_yaml() must let a real write + failure (e.g. target directory doesn't exist) propagate rather than swallowing + it internally. The two real call sites (run_acquisition()'s snapshot and the + Save Settings button handler) each already have their own appropriate + try/except one level up; swallowing here just made the button handler's + warning dialog unreachable.""" + from control.core.record_zstack_controller import RecordZStackAcquisitionParameters, _save_record_zstack_yaml + + params = RecordZStackAcquisitionParameters(base_path=str(tmp_path), experiment_id="my_exp") + # Parent directory doesn't exist -> open() raises FileNotFoundError (an OSError). + yaml_path = tmp_path / "does_not_exist" / "acquisition.yaml" + + with pytest.raises(OSError): + _save_record_zstack_yaml(params, str(yaml_path)) + + +def test_run_acquisition_survives_snapshot_write_failure(tmp_path, monkeypatch): + """Final-review Finding 3: run_acquisition()'s own try/except around + _save_record_zstack_yaml (a real acquisition's settings snapshot) must still + degrade gracefully -- i.e. NOT raise out of run_acquisition() -- now that the + inner swallow inside _save_record_zstack_yaml itself has been removed.""" + from unittest.mock import MagicMock + import control.core.record_zstack_controller as rzc + + monkeypatch.setattr("control._def.Acquisition.USE_MULTIPROCESSING", False) + + class DummyWorker: + def __init__(self, **kwargs): + pass + + def run(self): + pass + + monkeypatch.setattr("control.core.record_zstack_worker.RecordZStackWorker", DummyWorker) + + def _raise_on_save(*args, **kwargs): + raise OSError("simulated disk failure") + + monkeypatch.setattr(rzc, "_save_record_zstack_yaml", _raise_on_save) + + def _write_channels_yaml(output_dir, **kw): + (Path(output_dir) / "acquisition_channels.yaml").write_text("channels: []\n") + + microscope = MagicMock() + microscope.config_repo.save_acquisition_output.side_effect = _write_channels_yaml + microscope.camera.get_binning.return_value = (1, 1) + microscope.camera.get_pixel_size_binned_um.return_value = 0.5 + objective_store = MagicMock() + objective_store.current_objective = "20x" + objective_store.objectives_dict = {} + objective_store.get_pixel_size_factor.return_value = 1.0 + callbacks = MagicMock() + + controller = rzc.RecordZStackController( + microscope=microscope, + live_controller=MagicMock(), + laser_autofocus_controller=None, + objective_store=objective_store, + scan_coordinates=None, + callbacks=callbacks, + ) + params = rzc.RecordZStackAcquisitionParameters(base_path=str(tmp_path), experiment_id="my_exp") + + # Must not raise, even though _save_record_zstack_yaml raises internally. + controller.run_acquisition(params) + controller.join(timeout=5.0) + + experiment_dir = next(tmp_path.iterdir()) + assert (experiment_dir / "acquisition_channels.yaml").exists() + # The failed snapshot must not have produced a (possibly partial) acquisition.yaml. + assert not (experiment_dir / "acquisition.yaml").exists() + + def test_run_acquisition_writes_both_yaml_files(tmp_path, monkeypatch): """acquisition_channels.yaml (existing) and acquisition.yaml (new) both land in the experiment dir. diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index 997e33cdc..bf4ff3b3e 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -1642,6 +1642,41 @@ def test_save_settings_button_writes_yaml(qtbot, simulated_widget_deps, tmp_path assert data["objective"]["camera_binning"] == [1, 1] +def test_save_settings_button_shows_warning_when_write_fails(qtbot, simulated_widget_deps, tmp_path, monkeypatch): + """Final-review Finding 3: when the underlying _save_record_zstack_yaml() write + genuinely fails, _on_save_settings_clicked()'s own try/except must reach the user + via QMessageBox.warning -- it must NOT log "Settings saved" and silently show no + warning. Mirrors test_save_settings_button_writes_yaml's structure but points the + save at a path inside a non-existent directory so the real open() raises.""" + from control.widgets import RecordZStackMultiPointWidget + + w = RecordZStackMultiPointWidget(**simulated_widget_deps) + qtbot.addWidget(w) + w.lineEdit_savingDir.setText(str(tmp_path)) + w.checkbox_zstack.setChecked(True) + w._add_zstack_channel_row("BF LED matrix full") + + w.objectiveStore.objectives_dict = {"10x": {"magnification": 10.0}} + w.objectiveStore.current_objective = "10x" + w.objectiveStore.get_pixel_size_factor.return_value = 1.0 + + camera = MagicMock() + camera.get_binning.return_value = (1, 1) + camera.get_pixel_size_binned_um.return_value = 0.5 + w.liveController.camera = camera + + # Parent directory doesn't exist -> the real open() call raises OSError. + save_path = tmp_path / "does_not_exist" / "my_preset.yaml" + monkeypatch.setattr("control.widgets.QFileDialog.getSaveFileName", lambda *a, **kw: (str(save_path), "")) + + with patch("control.widgets.QMessageBox.warning") as mock_warn: + w.btn_saveSettings.click() + + assert not save_path.exists() + mock_warn.assert_called_once() + assert mock_warn.call_args[0][1] == "Save Error" + + def test_load_settings_button_applies_yaml(qtbot, simulated_widget_deps, tmp_path, monkeypatch): """Verify that clicking Load Settings runs the REAL load chain end to end: _on_load_settings_clicked -> _load_acquisition_yaml -> parse_acquisition_yaml -> From a585eb34efc693a7f6aa3ca485f79b36e5bf737d Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 5 Jul 2026 19:43:11 -0400 Subject: [PATCH 59/61] feat(record-zstack): cap Hamamatsu recording frame rate via DCAM INTERNALFRAMERATE Add HamamatsuCamera.set_frame_rate() (was inheriting the AbstractCamera no-op) so Record+Z-Stack recording caps the ORCA's internal free-run rate instead of free-running at the exposure/readout-limited max and downsampling in software. - Forces fast readout (DCAMPROP.READOUTSPEED.FASTEST) before reading the INTERNALFRAMERATE ceiling, so high (>30 fps) targets aren't pinned to the slow/low-noise mode's limit. - Clamps the request to the property's valid range, sets it, and returns the achieved rate (base contract). Any failure falls back to the exposure-limited max (== prior behavior), so recording stays correct even if the props are unsupported. Untested on hardware locally (no DCAM lib on the dev Mac); verified py_compile + black only. Hardware verification pending on ORCA-Fusion BT (C15440-20UP). Known follow-up: readout speed is left in fast mode after recording (not restored); make it configurable / restore for low-noise low-fps recording. Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/camera_hamamatsu.py | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/software/control/camera_hamamatsu.py b/software/control/camera_hamamatsu.py index 2d6c3d45c..fdb9e7b2b 100644 --- a/software/control/camera_hamamatsu.py +++ b/software/control/camera_hamamatsu.py @@ -210,6 +210,54 @@ def get_strobe_time(self) -> float: return (line_interval_s + trigger_delay_s) * 1000.0 + def set_frame_rate(self, fps: float) -> float: + """Cap the internal free-run rate (DCAM INTERNALFRAMERATE) for recording. + + In CONTINUOUS (internal-trigger) mode the sensor otherwise free-runs at its + exposure/readout-limited maximum, forcing the read thread to process frames + the recording pipeline only downsamples away. Setting INTERNALFRAMERATE caps + delivery near the requested rate. Exposure is unaffected: the frame period is + constrained to be >= exposure + readout, so a too-fast request is clamped by + the camera (reflected in the read-back) while a slower request just adds + inter-frame dead time. + + Returns the rate the camera accepted, or the exposure-limited max on any + failure, so callers can still size/pace against a real number (base contract). + """ + fallback = 1000.0 / self.get_total_frame_time() + if fps is None or fps <= 0: + return fallback + try: + with self._pause_streaming(): + # Fast readout FIRST: high frame rates need it, and the sensor's + # default may be the low-noise (slow) mode whose INTERNALFRAMERATE + # ceiling is far below `fps`. It also raises the valuemax we read + # next, so the clamp isn't pinned to the slow-mode limit. Best- + # effort — log if it doesn't take, but still try to set the rate. + # NOTE: this leaves the camera in fast readout after a recording + # (not restored); acceptable for throughput-oriented recording, see + # PR notes / follow-up to make readout speed configurable. + if not self._set_prop(DCAM_IDPROP.READOUTSPEED, DCAMPROP.READOUTSPEED.FASTEST): + self._log.warning("Could not set fast readout speed; achievable fps may be limited.") + + attr = self._camera.prop_getattr(DCAM_IDPROP.INTERNALFRAMERATE) + if isinstance(attr, bool) or attr is None: + self._log.warning( + "INTERNALFRAMERATE not available on this camera; leaving free-run rate at the camera default." + ) + return fallback + target = max(attr.valuemin, min(float(fps), attr.valuemax)) + if not self._set_prop(DCAM_IDPROP.INTERNALFRAMERATE, target): + return fallback + achieved = self._camera.prop_getvalue(DCAM_IDPROP.INTERNALFRAMERATE) + if isinstance(achieved, bool): + return fallback + self._log.info(f"set_frame_rate({fps}) set INTERNALFRAMERATE -> {float(achieved):.3f} fps") + return float(achieved) + except Exception: + self._log.exception("set_frame_rate failed; falling back to exposure-limited max.") + return fallback + def set_frame_format(self, frame_format: CameraFrameFormat): if frame_format != CameraFrameFormat.RAW: raise ValueError("Only the RAW frame format is supported by this camera.") From 01c07aa9af002cc8b73d9799792f9d4718df91c3 Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 5 Jul 2026 19:59:47 -0400 Subject: [PATCH 60/61] fix(record-zstack): honor set_frame_rate never-raises contract + review nits Address review of the Hamamatsu set_frame_rate override: - Guard the exposure-limited-max fallback: get_strobe_time() can raise on a DCAM read error, which previously escaped the method despite the "returns a usable rate on any failure" docstring. Now falls back to the requested rate. - Drop the unreachable `attr is None` branch (prop_getattr returns attr or False). - Fix the misleading "camera default" log (readout speed is already set by then). - Drop redundant float() on an already-float prop_getvalue result. Co-Authored-By: Claude Opus 4.8 (1M context) --- software/control/camera_hamamatsu.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/software/control/camera_hamamatsu.py b/software/control/camera_hamamatsu.py index fdb9e7b2b..f66a479bc 100644 --- a/software/control/camera_hamamatsu.py +++ b/software/control/camera_hamamatsu.py @@ -224,7 +224,14 @@ def set_frame_rate(self, fps: float) -> float: Returns the rate the camera accepted, or the exposure-limited max on any failure, so callers can still size/pace against a real number (base contract). """ - fallback = 1000.0 / self.get_total_frame_time() + try: + fallback = 1000.0 / self.get_total_frame_time() + except Exception: + # get_strobe_time() raises on a DCAM read error; honor the "always + # returns a usable rate" contract by assuming the requested rate (the + # caller then paces/sizes against it, same as if we'd never capped). + self._log.exception("could not read frame timing; assuming the requested rate.") + fallback = float(fps) if fps and fps > 0 else 0.0 if fps is None or fps <= 0: return fallback try: @@ -241,9 +248,9 @@ def set_frame_rate(self, fps: float) -> float: self._log.warning("Could not set fast readout speed; achievable fps may be limited.") attr = self._camera.prop_getattr(DCAM_IDPROP.INTERNALFRAMERATE) - if isinstance(attr, bool) or attr is None: + if isinstance(attr, bool): self._log.warning( - "INTERNALFRAMERATE not available on this camera; leaving free-run rate at the camera default." + "INTERNALFRAMERATE not available on this camera; leaving the internal frame rate uncapped." ) return fallback target = max(attr.valuemin, min(float(fps), attr.valuemax)) @@ -252,8 +259,8 @@ def set_frame_rate(self, fps: float) -> float: achieved = self._camera.prop_getvalue(DCAM_IDPROP.INTERNALFRAMERATE) if isinstance(achieved, bool): return fallback - self._log.info(f"set_frame_rate({fps}) set INTERNALFRAMERATE -> {float(achieved):.3f} fps") - return float(achieved) + self._log.info(f"set_frame_rate({fps}) set INTERNALFRAMERATE -> {achieved:.3f} fps") + return achieved except Exception: self._log.exception("set_frame_rate failed; falling back to exposure-limited max.") return fallback From 4621d9df7bf69541b1350d01fd080113331769bd Mon Sep 17 00:00:00 2001 From: Hongquan Li Date: Sun, 5 Jul 2026 21:41:52 -0400 Subject: [PATCH 61/61] refactor: drop dead no-op stubs after master's duck-typing removal (#584) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit master (PR #584) removed the duck-typed emit_selected_channels() / display_progress_bar() calls from gui_hcs.onTabChanged and toggleAcquisitionStart — channels are now emitted by each multipoint widget at acquisition start. RecordZStackMultiPointWidget's no-op stubs (added in 9b5be65d to satisfy the old contract) and their regression test are now unreachable dead code; remove them. Co-Authored-By: Claude Fable 5 --- software/control/widgets.py | 15 --------------- software/tests/test_record_zstack_widget.py | 12 ------------ 2 files changed, 27 deletions(-) diff --git a/software/control/widgets.py b/software/control/widgets.py index 60d120596..f8dc4f627 100644 --- a/software/control/widgets.py +++ b/software/control/widgets.py @@ -18311,18 +18311,3 @@ def refresh_channel_list(self) -> None: for name in [n for n in self._zstack_channel_names if n not in names]: self._log.info(f"removing z-stack channel row {name!r}: not available for the current objective") self._remove_zstack_channel_row(name) - - def emit_selected_channels(self) -> None: - """No-op stub: RecordZStackMultiPointWidget has no channel list to broadcast. - - Required so ``onTabChanged`` in gui_hcs can call this on whatever widget - is the current record tab without checking the type (same contract as - ``display_progress_bar`` below). - """ - - def display_progress_bar(self, show: bool) -> None: - """No-op stub: RecordZStackMultiPointWidget has no progress bar. - - Required so ``toggleAcquisitionStart`` in gui_hcs can call this on - whatever widget is the current record tab without checking the type. - """ diff --git a/software/tests/test_record_zstack_widget.py b/software/tests/test_record_zstack_widget.py index bf4ff3b3e..1938a809a 100644 --- a/software/tests/test_record_zstack_widget.py +++ b/software/tests/test_record_zstack_widget.py @@ -985,18 +985,6 @@ def test_toggle_acquisition_calls_update_scan_regions_before_run(qtbot, simulate assert call_order.index("set_well_coordinates") < call_order.index("run_acquisition") -def test_emit_selected_channels_is_a_safe_noop(qtbot, simulated_widget_deps): - """gui_hcs.onTabChanged duck-types emit_selected_channels() on whichever record - tab widget becomes current; the widget must provide it (same contract as - display_progress_bar) or every switch to the tab raises AttributeError.""" - from control.widgets import RecordZStackMultiPointWidget - - w = RecordZStackMultiPointWidget(**simulated_widget_deps) - qtbot.addWidget(w) - - w.emit_selected_channels() # must not raise - - def test_refresh_channel_list_repopulates_combos(qtbot, simulated_widget_deps): """Channel sets are per-objective: after an objective/profile change the combos must repopulate, or stale names silently fall back to a bare