From cd67e5b90fb7b56912b242507215665d6d77b311 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 8 Jul 2025 10:40:15 +0200 Subject: [PATCH 1/4] add zarrh5 --- pyproject.toml | 3 + src/silx/io/meson.build | 1 + src/silx/io/zarrh5.py | 163 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 167 insertions(+) create mode 100644 src/silx/io/zarrh5.py diff --git a/pyproject.toml b/pyproject.toml index 68a645e364..67e3f9281b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,9 @@ full = [ h5pyd = [ 'h5pyd >= 0.20.0' ] +zarr = [ + 'zarr[remote] >= 3.0.0' +] doc = [ 'silx[full]', 'Sphinx', # To build the documentation in doc/ diff --git a/src/silx/io/meson.build b/src/silx/io/meson.build index e3deb3e28d..28e5c53bc0 100644 --- a/src/silx/io/meson.build +++ b/src/silx/io/meson.build @@ -18,6 +18,7 @@ py.install_sources([ 'spech5.py', 'url.py', 'utils.py', + 'zarrh5.py', ], subdir: 'silx/io', # Folder relative to site-packages to install to ) diff --git a/src/silx/io/zarrh5.py b/src/silx/io/zarrh5.py new file mode 100644 index 0000000000..de0f03666e --- /dev/null +++ b/src/silx/io/zarrh5.py @@ -0,0 +1,163 @@ +# /*########################################################################## +# Copyright (C) 2025 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ############################################################################*/ +""" +Provides a wrapper to expose `Zarr `_ +This is a preview feature. +""" + +from __future__ import annotations + +import logging +import urllib.parse +from typing import Generator +import numpy +from . import commonh5 +import zarr + + +_logger = logging.getLogger(__name__) + + +def _children(group: zarr.Group) -> Generator[ZarrDataset | ZarrGroup]: + for name, item in group.members(): + if isinstance(item, zarr.Group): + yield ZarrGroup(name, item, attrs=item.attrs) + elif isinstance(item, zarr.Array): + yield ZarrDataset(name, item, attrs=item.attrs) + else: + _logger.warning(f"Cannot map child {name}: Ignored") + + +class ZarrH5(commonh5.File): + """Zarr client wrapper""" + + def __init__( + self, + name: str, + mode: str | None = None, + attrs: dict | None = None, + ) -> None: + assert mode in ("r", None) + if name.startswith("zarr+"): + name = name[5:] + full_url = urllib.parse.urlparse(name) + if full_url.fragment: + raise ValueError("URL fragment is not supported") + + base_url = urllib.parse.urlunparse( + (full_url.scheme, full_url.netloc, full_url.path, "", "", "") + ) + + # quick&dirty storage_options parsing: it would need pydantic model + storage_options = {} + for key, values in urllib.parse.parse_qs(full_url.query).items(): + value = values[-1] + if key == "use_ssl": + value = True if value.lower() == "true" else False + storage_options[key] = value + self.__group = zarr.open_group( + base_url, storage_options=storage_options or None + ) + + if attrs is None: + attrs = {} + super().__init__( + base_url.rstrip("/"), mode, attrs={**self.__group.attrs, **attrs} + ) + + for child in _children(self.__group): + self.add_node(child) + + _logger.warning( + "Zarr support is a preview feature: This may change or be removed without notice." + ) + + def close(self) -> None: + super().close() + self.__group = None + + +class ZarrGroup(commonh5.LazyLoadableGroup): + """Zarr Group wrapper""" + + def __init__( + self, + name: str, + group: zarr.Group, + parent: ZarrH5 | ZarrGroup | None = None, + attrs: dict | None = None, + ) -> None: + super().__init__(name, parent, attrs) + self.__group = group + + def _create_child(self) -> None: + for child in _children(self.__group): + self.add_node(child) + + +class ZarrDataset(commonh5.Dataset): + """Zarr Array wrapper""" + + def __init__( + self, + name: str, + array: zarr.Array, + parent: ZarrH5 | ZarrGroup | None = None, + attrs: dict | None = None, + ) -> None: + super().__init__(name, array, parent, attrs) + + @property + def shape(self) -> tuple[int, ...]: + return self._get_data().shape + + @property + def size(self) -> int: + return self._get_data().size + + def __len__(self) -> int: + return self.shape[0] + + def __getitem__(self, item): + return self._get_data()[item] + + @property + def value(self) -> numpy.ndarray: + return self._get_data()[()] + + @property + def compression(self): + data = self._get_data() + compressors = data.compressors + if not compressors: + return None + codec = compressors[0] + if data.metadata.zarr_format == 2: + # zarr format 2 compressors are numcodecs codecs + return codec.codec_id + # zarr format 3 compressors are zarr codecs + return codec.to_dict().get("name") + + @property + def chunks(self): + return self._get_data().chunks From f40ac604310139806f2c75e279404432d1989d50 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 8 Jul 2025 10:40:39 +0200 Subject: [PATCH 2/4] add zarr support to silx.io.open --- src/silx/io/utils.py | 45 +++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/src/silx/io/utils.py b/src/silx/io/utils.py index a48d0bb398..5208936c45 100644 --- a/src/silx/io/utils.py +++ b/src/silx/io/utils.py @@ -32,6 +32,7 @@ import sys import time import logging +import urllib.parse from collections.abc import Generator import numpy @@ -49,6 +50,12 @@ except ImportError: h5pyd = None +try: + from .zarrh5 import ZarrH5 +except ImportError: + ZarrH5 = None + + logger = logging.getLogger(__name__) NEXUS_HDF5_EXT = [".h5", ".nx5", ".nxs", ".hdf", ".hdf5", ".cxi"] @@ -690,34 +697,46 @@ def open(filename): # pylint:disable=redefined-builtin :raises: IOError if the file can't be loaded or path can't be found :rtype: h5py-like node """ - url = DataUrl(filename) + url = urllib.parse.urlparse(filename) + if url.scheme.startswith("zarr+"): + if ZarrH5 is None: + raise IOError( + f"Zarr support is not available, please install zarr, cannot open: {filename}" + ) + try: + return ZarrH5(filename) + except Exception as e: + raise IOError(f"Failed to open URL with zarr: {type(e)} {e}") - if url.scheme() in [None, "file", "silx"]: + data_url = DataUrl(filename) + if data_url.scheme() in [None, "file", "silx"]: # That's a local file - if not url.is_valid(): + if not data_url.is_valid(): raise OSError("URL '%s' is not valid" % filename) - h5_file = _open_local_file(url.file_path()) - elif url.scheme() in ("http", "https"): + h5_file = _open_local_file(data_url.file_path()) + elif data_url.scheme() in ("http", "https"): return _open_url_with_h5pyd(filename) else: - raise OSError(f"Unsupported URL scheme {url.scheme}: {filename}") + raise OSError(f"Unsupported URL scheme {data_url.scheme}: {filename}") - if url.data_path() in [None, "/", ""]: # The full file is requested - if url.data_slice(): + if data_url.data_path() in [None, "/", ""]: # The full file is requested + if data_url.data_slice(): raise OSError(f"URL '{filename}' containing slicing is not supported") return h5_file else: # Only a children is requested - if url.data_path() not in h5_file: - msg = f"File '{filename}' does not contain path '{url.data_path()}'." + if data_url.data_path() not in h5_file: + msg = f"File '{filename}' does not contain path '{data_url.data_path()}'." raise OSError(msg) - node = h5_file[url.data_path()] + node = h5_file[data_url.data_path()] - if url.data_slice() is not None: + if data_url.data_slice() is not None: from . import _sliceh5 # Lazy-import to avoid circular dependency try: - return _sliceh5.DatasetSlice(node, url.data_slice(), attrs=node.attrs) + return _sliceh5.DatasetSlice( + node, data_url.data_slice(), attrs=node.attrs + ) except ValueError: raise OSError( f"URL {filename} contains slicing, but it is not a dataset" From 8ce9f018973db30aaa6409b2d8c961250cbd3ecb Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 8 Jul 2025 10:41:03 +0200 Subject: [PATCH 3/4] allow to open zarr URL from silx view --- src/silx/app/utils/parseutils.py | 7 +++++++ src/silx/app/view/main.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/silx/app/utils/parseutils.py b/src/silx/app/utils/parseutils.py index eef20e23ee..e378a412b3 100644 --- a/src/silx/app/utils/parseutils.py +++ b/src/silx/app/utils/parseutils.py @@ -29,6 +29,7 @@ from collections.abc import Sequence import glob import logging +import urllib.parse from typing import Any from collections.abc import Generator, Iterable from pathlib import Path @@ -84,6 +85,12 @@ def filenames_to_dataurls( extra_slices = tuple(slices) for filename in filenames: + if isinstance(filename, str) and urllib.parse.urlparse( + filename + ).scheme.startswith("zarr+"): + yield filename + continue + url = DataUrl(filename) for file_path in sorted(silx.utils.files.expand_filenames([url.file_path()])): diff --git a/src/silx/app/view/main.py b/src/silx/app/view/main.py index e30af2293d..d4f9f1a038 100644 --- a/src/silx/app/view/main.py +++ b/src/silx/app/view/main.py @@ -183,7 +183,7 @@ def exceptHook(type_, value, trace): for url in parseutils.filenames_to_dataurls(options.files, options.slices): # TODO: Would be nice to add a process widget and a cancel button try: - window.appendFile(url.path()) + window.appendFile(url if isinstance(url, str) else url.path()) except OSError as e: _logger.error(e.args[0]) _logger.debug("Backtrace", exc_info=True) From 5dc10bc03898eccd4b4bc80acfde24d59c8e6d57 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 22 Sep 2026 16:47:30 +0200 Subject: [PATCH 4/4] add tests for zarrh5 --- src/silx/io/test/test_zarrh5.py | 81 +++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/silx/io/test/test_zarrh5.py diff --git a/src/silx/io/test/test_zarrh5.py b/src/silx/io/test/test_zarrh5.py new file mode 100644 index 0000000000..625d5fff7a --- /dev/null +++ b/src/silx/io/test/test_zarrh5.py @@ -0,0 +1,81 @@ +"""Tests for zarrh5 wrapper""" + +import h5py +import numpy +import pytest + +zarr = pytest.importorskip("zarr") + +from .. import zarrh5 # noqa: E402 + + +@pytest.fixture +def zarr_dir(tmp_path): + root = zarr.open_group(str(tmp_path), mode="w") + root.attrs["file_attr"] = "hello" + + data = root.create_array("data", shape=(3, 4), dtype="f8") + data[...] = numpy.arange(12).reshape(3, 4) + data.attrs["units"] = "m" + + root.create_array("no_compression", shape=(5,), dtype="i4", compressors=None)[ + ... + ] = numpy.arange(5) + + group = root.create_group("group") + group.attrs["group_attr"] = 42 + group.create_array("sub_data", shape=(2, 3), dtype="i4")[...] = numpy.arange( + 6 + ).reshape(2, 3) + + return str(tmp_path) + + +def test_extra_attrs_are_merged_with_zarr_attrs(zarr_dir): + h5 = zarrh5.ZarrH5(zarr_dir, attrs={"extra": "value"}) + assert h5.attrs["file_attr"] == "hello" + assert h5.attrs["extra"] == "value" + + +def test_h5py_class(zarr_dir): + h5 = zarrh5.ZarrH5(zarr_dir) + assert h5.h5py_class == h5py.File + assert h5["group"].h5py_class == h5py.Group + assert h5["data"].h5py_class == h5py.Dataset + + +def test_group(zarr_dir): + h5 = zarrh5.ZarrH5(zarr_dir) + assert set(h5.keys()) == {"data", "no_compression", "group"} + + assert set(h5["group"].keys()) == {"sub_data"} + + assert "data" in h5 + assert "group/sub_data" in h5 + assert "not_there" not in h5 + + +def test_dataset(zarr_dir): + h5 = zarrh5.ZarrH5(zarr_dir) + assert h5["data"].shape == (3, 4) + assert h5["data"].size == 12 + assert h5["data"].dtype == numpy.dtype("f8") + assert len(h5["data"]) == 3 + assert h5["data"].chunks == (3, 4) + assert h5["data"].compression == "zstd" + + numpy.testing.assert_array_equal(h5["data"][...], numpy.arange(12).reshape(3, 4)) + numpy.testing.assert_array_equal(h5["data"].value, numpy.arange(12).reshape(3, 4)) + numpy.testing.assert_array_equal(h5["data"][0], numpy.arange(4)) + + +def test_dataset_no_compression(zarr_dir): + h5 = zarrh5.ZarrH5(zarr_dir) + assert h5["no_compression"].compression is None + + +def test_attrs(zarr_dir): + h5 = zarrh5.ZarrH5(zarr_dir) + assert dict(h5.attrs) == {"file_attr": "hello"} + assert dict(h5["group"].attrs) == {"group_attr": 42} + assert dict(h5["data"].attrs) == {"units": "m"}