From 3d0cec62fa4dbc65b50256bf5801b1a3a717bcff Mon Sep 17 00:00:00 2001 From: Tanny Chavez Date: Fri, 2 Oct 2026 17:32:56 -0700 Subject: [PATCH 1/2] Add FITS adapter for ALS BL11.0.1.2 (RSoXS) Reads ALS CCD FITS frames where the primary HDU is metadata-only and the image lives in an extension named 'Images'. Primary-header cards (including HIERARCH keys) become node metadata, the image header is nested under 'image_hdu', and small tables such as 'Izeros' are copied into 'extensions'. Adds a bl11012 extra (astropy), maps .fits to application/fits in the server config, installs the extra in the container, and type-checks the adapter strictly like bl733. Co-Authored-By: Claude Opus 5.5 --- .mypy.ini | 8 + Containerfile | 2 +- README.md | 15 ++ pixi.toml | 7 +- pyproject.toml | 4 + src/splash_tiled/bl11012/__init__.py | 0 src/splash_tiled/bl11012/adapters/__init__.py | 0 src/splash_tiled/bl11012/adapters/fits.py | 239 ++++++++++++++++++ tiled_config/config.yaml | 2 + 9 files changed, 274 insertions(+), 3 deletions(-) create mode 100644 src/splash_tiled/bl11012/__init__.py create mode 100644 src/splash_tiled/bl11012/adapters/__init__.py create mode 100644 src/splash_tiled/bl11012/adapters/fits.py diff --git a/.mypy.ini b/.mypy.ini index 6b46f35..129df9b 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -13,6 +13,14 @@ disallow_untyped_defs = True disallow_incomplete_defs = True disallow_untyped_calls = True +[mypy-splash_tiled.bl11012.adapters.*] +ignore_errors = False +ignore_missing_imports = False +check_untyped_defs = True +disallow_untyped_defs = True +disallow_incomplete_defs = True +disallow_untyped_calls = True + [mypy-tests.*] ignore_errors = False disallow_untyped_defs = False diff --git a/Containerfile b/Containerfile index 8c8f9fb..b5502d6 100755 --- a/Containerfile +++ b/Containerfile @@ -10,5 +10,5 @@ ENV PYTHONPATH=/app/src:/tiled_deploy/config COPY --chown=app:app pyproject.toml README.md ./ COPY --chown=app:app src/ ./src/ -RUN uv pip install --python /app/bin/python --no-cache ".[bl733]" +RUN uv pip install --python /app/bin/python --no-cache ".[bl733,bl11012]" COPY --chown=app:app scripts/ ./scripts/ diff --git a/README.md b/README.md index 92c38db..ab39276 100755 --- a/README.md +++ b/README.md @@ -189,6 +189,21 @@ The application can be configured through environment variables: - `TILED_SERVER_ENABLE_ORIGINS`: Configure CORS origins - `PYTHONPATH`: Python module search path +## Beamline Adapters + +Beamline-specific Tiled adapters live in `src/splash_tiled/bl/adapters/`. Each beamline +has a pip extra for its file-format dependencies, and its mimetypes are mapped in +`tiled_config/config.yaml`. + +| Beamline | Extra | File types | Adapter | +|---|---|---|---| +| 7.3.3 (SAXS/WAXS) | `bl733` | `.edf`, `.gb` | `EDFAdapter`, `GeneralBinaryPilatus2MAdapter` | +| 11.0.1.2 (RSoXS) | `bl11012` | `.fits` | `FITSAdapter` | + +```bash +pip install -e ".[bl733,bl11012]" +``` + ## ESAF Sync Script The repository includes a small CLI for loading ESAFs into SQLite. diff --git a/pixi.toml b/pixi.toml index 058e247..ab16637 100644 --- a/pixi.toml +++ b/pixi.toml @@ -5,8 +5,8 @@ channels = ["conda-forge"] platforms = ["linux-64", "osx-64", "osx-arm64", "linux-aarch64"] [environments] -default = { features = ["bl733"] } -dev = { features = ["bl733", "dev"] } +default = { features = ["bl733", "bl11012"] } +dev = { features = ["bl733", "bl11012", "dev"] } [dependencies] python = "3.13.*" @@ -35,6 +35,9 @@ pytest-asyncio = "*" [feature.bl733.dependencies] fabio = "*" +[feature.bl11012.dependencies] +astropy = "*" + [tasks] test = "pytest --cov=splash_tiled --cov-report=term-missing" test_user_office = "pytest -m user_office" diff --git a/pyproject.toml b/pyproject.toml index da0a1ab..7ed5d84 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,9 @@ tiled-all = [ bl733 = [ "fabio", ] +bl11012 = [ + "astropy>=6", +] dev = [ "flake8", "black", @@ -45,6 +48,7 @@ dev = [ "pytest-cov", "pytest-asyncio", "fabio", + "astropy>=6", "tiled[server]>=0.2.8" ] diff --git a/src/splash_tiled/bl11012/__init__.py b/src/splash_tiled/bl11012/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/splash_tiled/bl11012/adapters/__init__.py b/src/splash_tiled/bl11012/adapters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/splash_tiled/bl11012/adapters/fits.py b/src/splash_tiled/bl11012/adapters/fits.py new file mode 100644 index 0000000..522c1ab --- /dev/null +++ b/src/splash_tiled/bl11012/adapters/fits.py @@ -0,0 +1,239 @@ +"""Adapter for ALS CCD FITS files from beamline 11.0.1.2 (RSoXS). + +The primary HDU only holds metadata (NAXIS=0, about 120 motor and analog +cards). The detector frame lives in an IMAGE extension named ``Images``, and +small BINTABLE extensions such as ``Izeros`` hold per-frame readbacks. The +same layout is written at 7.3.3. Plain single-image FITS files are handled by +the fallbacks in ``_pick_image_hdu``. +""" + +import logging +from typing import Any, Optional + +import numpy as np +from astropy.io import fits +from tiled.adapters.array import ArrayAdapter +from tiled.adapters.utils import init_adapter_from_catalog +from tiled.catalog.orm import Node +from tiled.structures.array import ArrayStructure +from tiled.structures.core import Spec, StructureFamily +from tiled.structures.data_source import DataSource +from tiled.type_aliases import JSON +from tiled.utils import path_from_uri + +logger = logging.getLogger(__name__) +LOADING_MESSAGE = "Loading FITS file produced by ALS beamline 11.0.1.2: %s" + +#: EXTNAME of the image extension these detectors write (matched case-insensitively). +PREFERRED_EXTNAME = "images" + +#: Non-image extensions with at most this many rows are copied into metadata. +MAX_TABLE_ROWS = 1024 + +_COMMENTARY_KEYWORDS = ("COMMENT", "HISTORY") + + +def _is_2d_image(hdu: Any) -> bool: + """Return True if ``hdu`` is an image HDU with 2-D data.""" + return ( + bool(getattr(hdu, "is_image", False)) + and len(getattr(hdu, "shape", ()) or ()) == 2 + ) + + +def _pick_image_hdu(hdulist: fits.HDUList) -> tuple[int, Any]: + """Locate the HDU holding the detector frame. + + Order of preference: + 1. An image HDU whose EXTNAME is ``Images`` (case-insensitive). + 2. The first 2-D image extension. + 3. The primary HDU, if it carries at least 2-D data. + + Args: + hdulist: An open FITS HDU list. + + Returns: + The index of the chosen HDU and the HDU itself. + + Raises: + ValueError: If no HDU contains image data. + """ + for index, hdu in enumerate(hdulist): + extname = str(hdu.header.get("EXTNAME", "")).strip().lower() + if extname == PREFERRED_EXTNAME and _is_2d_image(hdu): + return index, hdu + for index, hdu in enumerate(hdulist): + if index == 0: + continue + if _is_2d_image(hdu): + return index, hdu + primary = hdulist[0] + if ( + getattr(primary, "is_image", False) + and len(getattr(primary, "shape", ()) or ()) >= 2 + ): + return 0, primary + extensions = ", ".join( + f"{i}:{h.header.get('EXTNAME', h.__class__.__name__)}" + for i, h in enumerate(hdulist) + ) + raise ValueError(f"No image HDU found in FITS file (extensions: {extensions})") + + +def _safe_str(value: Any) -> str: + """Stringify ``value``, replacing lone surrogates that orjson rejects.""" + return str(value).encode("utf-8", "replace").decode("utf-8") + + +def _card_value(value: Any) -> Any: + """Convert a FITS card value into a JSON-safe Python value.""" + if isinstance(value, fits.card.Undefined): + return None + if isinstance(value, bool): # must precede int: bool is an int + return value + if isinstance(value, str): + return _safe_str(value) + if isinstance(value, (int, float)): + return value + if isinstance(value, complex): + return [value.real, value.imag] + return _safe_str(value) + + +def _header_to_dict(header: fits.Header) -> dict[str, Any]: + """Flatten an astropy Header into a JSON-safe dict. + + Iterates over ``header.cards`` rather than ``dict(header)`` so that + HIERARCH cards keep their full long keyword ("EPU Polarization") and so + that repeated COMMENT/HISTORY cards collect into a list. + + Args: + header: The FITS header to convert. + + Returns: + A mapping of keyword to value. + """ + out: dict[str, Any] = {} + commentary: dict[str, list[str]] = {} + for card in header.cards: + try: + keyword, value = card.keyword, card.value + except Exception: # unparseable card: skip rather than fail the read + logger.warning( + "Skipping unparseable FITS card: %r", getattr(card, "image", card) + ) + continue + keyword = str(keyword).strip() + if not keyword: + continue + if keyword in _COMMENTARY_KEYWORDS: + commentary.setdefault(keyword, []).append(_safe_str(value).strip()) + continue + out[keyword] = _card_value(value) + out.update(commentary) + return out + + +def _table_to_dict(hdu: Any, index: int) -> dict[str, Any]: + """Summarise a TABLE/BINTABLE extension, including its data if small. + + Args: + hdu: The table HDU. + index: Position of the HDU in the file. + + Returns: + A summary with ``index``, ``extname``, ``rows`` and ``columns``, plus + ``data`` (column name to values) when the table has at most + ``MAX_TABLE_ROWS`` rows. + """ + columns = [str(n) for n in (hdu.columns.names if hdu.columns else [])] + rows = int(hdu.header.get("NAXIS2", 0)) + summary: dict[str, Any] = { + "index": index, + "extname": str(hdu.header.get("EXTNAME", "")).strip() or None, + "rows": rows, + "columns": columns, + } + if 0 < rows <= MAX_TABLE_ROWS: + data = hdu.data + summary["data"] = {name: np.asarray(data[name]).tolist() for name in columns} + return summary + + +class FITSAdapter(ArrayAdapter): + structure_family = StructureFamily.array + + def __init__( + self, + data_uri: str, + structure: Optional[ArrayStructure] = None, + metadata: Optional[JSON] = None, + specs: Optional[list[Spec]] = None, + **kwargs: Optional[Any], + ) -> None: + """Adapter for `.fits` CCD frames (e.g. RSoXS) at ALS beamline 11.0.1.2.""" + filepath = path_from_uri(data_uri) + logger.debug(LOADING_MESSAGE, filepath) + + # memmap=False is required: astropy refuses to memory-map an image HDU + # that carries BZERO/BSCALE, and these frames set BZERO=32768 to store + # unsigned counts in a signed int16 HDU. + with fits.open(filepath, memmap=False, lazy_load_hdus=False) as hdulist: + index, image_hdu = _pick_image_hdu(hdulist) + array = np.asarray(image_hdu.data) + + file_metadata = _header_to_dict(hdulist[0].header) + image_header: dict[str, Any] = _header_to_dict(image_hdu.header) + image_header["index"] = index + extensions: dict[str, Any] = {} + for other_index, hdu in enumerate(hdulist): + if other_index == index or getattr(hdu, "is_image", False): + continue + if getattr(hdu, "columns", None) is None: + continue + name = str(hdu.header.get("EXTNAME", "")).strip() or f"HDU{other_index}" + extensions[name] = _table_to_dict(hdu, other_index) + + # Tiled's JSON serializer rejects non-native byte order, and astropy + # returns big-endian arrays for HDUs it does not rescale. + native_dtype = array.dtype.newbyteorder("=") + if array.dtype != native_dtype: + array = array.astype(native_dtype) + array = np.ascontiguousarray(array) + + # Nest the image header so its BITPIX/NAXIS/EXTNAME cards do not + # clobber the primary header's, which is where the motors live. + fits_metadata: dict[str, Any] = {**file_metadata, "image_hdu": image_header} + if extensions: + fits_metadata["extensions"] = extensions + metadata = {**fits_metadata, **(metadata or {})} + + fits_spec = Spec("als-bl11012-fits", version="1.0") + specs = list(specs or []) + if fits_spec not in specs: + specs.append(fits_spec) + super().__init__( + array=array, + structure=structure or ArrayStructure.from_array(array), + metadata=metadata, + specs=specs, + **kwargs, + ) + + @classmethod + def from_catalog( + cls, + data_source: DataSource, + node: Node, + /, + **kwargs: Optional[Any], + ) -> "FITSAdapter": + return init_adapter_from_catalog(cls, data_source, node, **kwargs) + + @classmethod + def from_uris( + cls, + data_uri: str, + **kwargs: Optional[Any], + ) -> "FITSAdapter": + return cls(data_uri, **kwargs) diff --git a/tiled_config/config.yaml b/tiled_config/config.yaml index b87108a..c6e8f27 100644 --- a/tiled_config/config.yaml +++ b/tiled_config/config.yaml @@ -43,6 +43,7 @@ access_control: file_extensions: edf: application/x-edf gb: application/x-gb + fits: application/fits trees: @@ -56,3 +57,4 @@ trees: adapters_by_mimetype: application/x-edf: splash_tiled.bl733.adapters.edf:EDFAdapter application/x-gb: splash_tiled.bl733.adapters.gb:GeneralBinaryPilatus2MAdapter + application/fits: splash_tiled.bl11012.adapters.fits:FITSAdapter From 052bab7b9c4f6297ac9e1a88752c164fdd7c70a6 Mon Sep 17 00:00:00 2001 From: Tanny Chavez Date: Fri, 2 Oct 2026 17:32:56 -0700 Subject: [PATCH 2/2] Add tests for BL11.0.1.2 FITS adapter Synthetic FITS files mirror the beamline layout. Covers HDU selection and fallbacks, byte order, JSON-safe metadata, table capture, and a round trip through register() and a Tiled server. Co-Authored-By: Claude Opus 5.5 --- tests/bl11012/__init__.py | 0 tests/bl11012/conftest.py | 9 +++ tests/bl11012/samples.py | 68 ++++++++++++++++ tests/bl11012/test_fits.py | 159 +++++++++++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+) create mode 100644 tests/bl11012/__init__.py create mode 100644 tests/bl11012/conftest.py create mode 100644 tests/bl11012/samples.py create mode 100644 tests/bl11012/test_fits.py diff --git a/tests/bl11012/__init__.py b/tests/bl11012/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/bl11012/conftest.py b/tests/bl11012/conftest.py new file mode 100644 index 0000000..702ad34 --- /dev/null +++ b/tests/bl11012/conftest.py @@ -0,0 +1,9 @@ +import pytest + +from tests.bl11012.samples import make_fits + + +@pytest.fixture +def bl11012_fits_path(tmp_path): + """Write a synthetic BL11.0.1.2 frame to tmp_path/scan_name.fits.""" + return make_fits(tmp_path, filename="scan_name.fits") diff --git a/tests/bl11012/samples.py b/tests/bl11012/samples.py new file mode 100644 index 0000000..77542f8 --- /dev/null +++ b/tests/bl11012/samples.py @@ -0,0 +1,68 @@ +"""Helpers that write small synthetic BL11.0.1.2 FITS files. + +They mirror the real ALS CCD layout (metadata-only primary HDU, an ``Izeros`` +BINTABLE, and the frame in an IMAGE extension named ``Images`` with BZERO +rescaling) without needing the real 33 MB files. +""" + +from pathlib import Path +from typing import Optional, Sequence + +import numpy as np +import pytest + +fits = pytest.importorskip("astropy.io.fits") + +SAMPLE_FITS_DATA = np.array([[0, 1, 65535], [10, 20, 30], [1, 2, 3]], dtype=np.uint16) + + +def base_primary_header(): + """Primary header with HIERARCH, boolean, undefined and COMMENT cards.""" + header = fits.Header() + header["HIERARCH EPU Polarization"] = (100.0, "Counts") + header["FLAG"] = True + header["UNDEF"] = fits.card.Undefined() + header["COMMENT"] = "first comment" + header["COMMENT"] = "second comment" + return header + + +def make_fits( + tmp_path: Path, + *, + filename: str = "frame.fits", + image_extname: str = "Images", + include_decoy: bool = True, + include_izeros: bool = True, + izeros_values: Sequence[float] = (1.1, 2.2, 3.3, 4.4), + image_data: Optional[np.ndarray] = None, +) -> Path: + """Write a FITS file shaped like an ALS CCD frame and return its path.""" + primary = fits.PrimaryHDU() + primary.header.update(base_primary_header()) + + hdus = [primary] + if include_decoy: + decoy = fits.ImageHDU(data=np.zeros((4, 4), dtype=np.uint8)) + decoy.header["EXTNAME"] = "Decoy" + hdus.append(decoy) + + if image_data is None: + image_data = SAMPLE_FITS_DATA + # Set EXTNAME directly rather than via name=: astropy uppercases that, but + # the real files store mixed-case 'Images'/'Izeros'. + image_hdu = fits.ImageHDU(data=image_data) + image_hdu.header["EXTNAME"] = image_extname + hdus.append(image_hdu) + + if include_izeros: + col = fits.Column( + name="0", format="E", array=np.asarray(izeros_values, dtype=np.float32) + ) + table = fits.BinTableHDU.from_columns([col]) + table.header["EXTNAME"] = "Izeros" + hdus.append(table) + + path = tmp_path / filename + fits.HDUList(hdus).writeto(path, overwrite=True) + return path diff --git a/tests/bl11012/test_fits.py b/tests/bl11012/test_fits.py new file mode 100644 index 0000000..9050b3b --- /dev/null +++ b/tests/bl11012/test_fits.py @@ -0,0 +1,159 @@ +import numpy as np +import pytest +import pytest_asyncio +from tiled.client import Context, from_context +from tiled.client.register import register +from tiled.server.app import build_app_from_config +from tiled.utils import safe_json_dump + +from splash_tiled.bl11012.adapters.fits import MAX_TABLE_ROWS, FITSAdapter +from tests.bl11012.samples import SAMPLE_FITS_DATA, base_primary_header, fits, make_fits + +FITS_ADAPTER = "splash_tiled.bl11012.adapters.fits:FITSAdapter" +FITS_MIMETYPE = "application/fits" + + +def _read(path): + return FITSAdapter(f"file://{path}") + + +# --- Direct adapter behaviour ------------------------------------------------- + + +def test_fits_returns_array_with_frame_shape(tmp_path): + adapter = _read(make_fits(tmp_path)) + assert adapter.structure_family == "array" + assert adapter.structure().shape == (3, 3) + + +def test_fits_preserves_values_and_native_byte_order(tmp_path): + adapter = _read(make_fits(tmp_path)) + arr = adapter.read() + assert arr.dtype.byteorder in ("=", "|") + np.testing.assert_array_equal(arr, SAMPLE_FITS_DATA) + + +def test_fits_metadata_is_json_safe_and_preserves_cards(tmp_path): + metadata = _read(make_fits(tmp_path)).metadata() + + safe_json_dump(metadata) # must not raise + + assert metadata["EPU Polarization"] == 100.0 + assert metadata["COMMENT"] == ["first comment", "second comment"] + assert metadata["UNDEF"] is None + assert metadata["FLAG"] is True + # The image HDU's cards are nested, not merged over the primary header. + assert metadata["image_hdu"]["EXTNAME"] == "Images" + assert "EXTNAME" not in metadata + + +def test_fits_izeros_table_is_captured_into_extensions(tmp_path): + metadata = _read(make_fits(tmp_path)).metadata() + izeros = metadata["extensions"]["Izeros"] + assert izeros["rows"] == 4 + np.testing.assert_allclose(izeros["data"]["0"], [1.1, 2.2, 3.3, 4.4], rtol=1e-5) + + +def test_fits_large_table_records_only_summary(tmp_path): + values = np.arange(MAX_TABLE_ROWS + 1, dtype=np.float32) + izeros = _read(make_fits(tmp_path, izeros_values=values)).metadata()["extensions"][ + "Izeros" + ] + assert izeros["rows"] == MAX_TABLE_ROWS + 1 + assert "data" not in izeros + + +def test_fits_images_extname_wins_over_decoy(tmp_path): + adapter = _read(make_fits(tmp_path, include_decoy=True)) + assert adapter.metadata()["image_hdu"]["EXTNAME"] == "Images" + assert adapter.structure().shape == (3, 3) # not the decoy's (4, 4) + + +def test_fits_falls_back_to_first_2d_image_without_images_extname(tmp_path): + adapter = _read(make_fits(tmp_path, include_decoy=True, image_extname="Frame")) + # The decoy precedes the renamed frame, so it wins the fallback. + assert adapter.structure().shape == (4, 4) + + +def test_fits_falls_back_to_primary_hdu_with_data(tmp_path): + primary = fits.PrimaryHDU(data=np.ones((5, 5), dtype=np.uint16)) + primary.header.update(base_primary_header()) + path = tmp_path / "primary_only.fits" + fits.HDUList([primary]).writeto(path, overwrite=True) + + assert _read(path).structure().shape == (5, 5) + + +def test_fits_raises_when_no_image_data(tmp_path): + primary = fits.PrimaryHDU() + primary.header.update(base_primary_header()) + path = tmp_path / "no_image.fits" + fits.HDUList([primary]).writeto(path, overwrite=True) + + with pytest.raises(ValueError, match="No image HDU"): + _read(path) + + +def test_fits_caller_metadata_overrides_file_cards(tmp_path): + adapter = FITSAdapter( + f"file://{make_fits(tmp_path)}", metadata={"FLAG": False, "extra": 1} + ) + metadata = adapter.metadata() + assert metadata["FLAG"] is False + assert metadata["extra"] == 1 + + +# --- Round trip through a Tiled server ---------------------------------------- + + +def _tiled_config(tmp_path): + return { + "trees": [ + { + "tree": "catalog", + "path": "/", + "args": { + "uri": str(tmp_path / "catalog.db"), + "readable_storage": [str(tmp_path)], + "init_if_not_exists": True, + "adapters_by_mimetype": {FITS_MIMETYPE: FITS_ADAPTER}, + }, + } + ], + "file_extensions": {"fits": FITS_MIMETYPE}, + } + + +@pytest_asyncio.fixture +async def fits_client(tmp_path, bl11012_fits_path): + with Context.from_app(build_app_from_config(_tiled_config(tmp_path))) as context: + client = from_context(context) + await register( + client, + tmp_path, + adapters_by_mimetype={FITS_MIMETYPE: FITS_ADAPTER}, + mimetypes_by_file_ext={".fits": FITS_MIMETYPE}, + ) + yield client + + +@pytest.mark.asyncio +async def test_fits_server_reads_array(fits_client): + """Array returned by the tiled client matches the frame written to disk.""" + result = fits_client["scan_name"].read() + np.testing.assert_array_equal(result, SAMPLE_FITS_DATA) + + +@pytest.mark.asyncio +async def test_fits_server_metadata_includes_header_fields(fits_client): + """Primary-header cards and the nested image header survive registration.""" + metadata = fits_client["scan_name"].metadata + assert metadata["EPU Polarization"] == 100.0 + assert metadata["image_hdu"]["EXTNAME"] == "Images" + assert metadata["extensions"]["Izeros"]["rows"] == 4 + + +@pytest.mark.asyncio +async def test_fits_server_node_carries_spec(fits_client): + specs = fits_client["scan_name"].specs + assert any(spec.name == "als-bl11012-fits" for spec in specs)