silx.io.open: Added basic support for zarr URLs

pyproject.toml

@@ -72,6 +72,9 @@ full = [
 h5pyd = [
     'h5pyd >= 0.20.0'
 ]
+zarr = [
+    'zarr[remote] >= 3.0.0'
+]
 doc = [
     'silx[full]',
     'Sphinx',                    # To build the documentation in doc/

src/silx/app/utils/parseutils.py

@@ -29,6 +29,7 @@
 from collections.abc import Sequence
 import glob
 import logging
+import urllib.parse
 from typing import Any
 from collections.abc import Generator, Iterable
 from pathlib import Path
@@ -84,6 +85,12 @@ def filenames_to_dataurls(
     extra_slices = tuple(slices)
 
     for filename in filenames:
+        if isinstance(filename, str) and urllib.parse.urlparse(
+            filename
+        ).scheme.startswith("zarr+"):
+            yield filename
+            continue
+
         url = DataUrl(filename)
 
         for file_path in sorted(silx.utils.files.expand_filenames([url.file_path()])):

src/silx/app/view/main.py

@@ -183,7 +183,7 @@ def exceptHook(type_, value, trace):
     for url in parseutils.filenames_to_dataurls(options.files, options.slices):
         # TODO: Would be nice to add a process widget and a cancel button
         try:
-            window.appendFile(url.path())
+            window.appendFile(url if isinstance(url, str) else url.path())
         except OSError as e:
             _logger.error(e.args[0])
             _logger.debug("Backtrace", exc_info=True)

src/silx/io/meson.build

@@ -18,6 +18,7 @@ py.install_sources([
     'spech5.py',
     'url.py',
     'utils.py',
+    'zarrh5.py',
 ],
 subdir: 'silx/io',  # Folder relative to site-packages to install to
 )

src/silx/io/test/test_zarrh5.py

@@ -0,0 +1,81 @@
+"""Tests for zarrh5 wrapper"""
+
+import h5py
+import numpy
+import pytest
+
+zarr = pytest.importorskip("zarr")
+
+from .. import zarrh5  # noqa: E402
+
+
+@pytest.fixture
+def zarr_dir(tmp_path):
+    root = zarr.open_group(str(tmp_path), mode="w")
+    root.attrs["file_attr"] = "hello"
+
+    data = root.create_array("data", shape=(3, 4), dtype="f8")
+    data[...] = numpy.arange(12).reshape(3, 4)
+    data.attrs["units"] = "m"
+
+    root.create_array("no_compression", shape=(5,), dtype="i4", compressors=None)[
+        ...
+    ] = numpy.arange(5)
+
+    group = root.create_group("group")
+    group.attrs["group_attr"] = 42
+    group.create_array("sub_data", shape=(2, 3), dtype="i4")[...] = numpy.arange(
+        6
+    ).reshape(2, 3)
+
+    return str(tmp_path)
+
+
+def test_extra_attrs_are_merged_with_zarr_attrs(zarr_dir):
+    h5 = zarrh5.ZarrH5(zarr_dir, attrs={"extra": "value"})
+    assert h5.attrs["file_attr"] == "hello"
+    assert h5.attrs["extra"] == "value"
+
+
+def test_h5py_class(zarr_dir):
+    h5 = zarrh5.ZarrH5(zarr_dir)
+    assert h5.h5py_class == h5py.File
+    assert h5["group"].h5py_class == h5py.Group
+    assert h5["data"].h5py_class == h5py.Dataset
+
+
+def test_group(zarr_dir):
+    h5 = zarrh5.ZarrH5(zarr_dir)
+    assert set(h5.keys()) == {"data", "no_compression", "group"}
+
+    assert set(h5["group"].keys()) == {"sub_data"}
+
+    assert "data" in h5
+    assert "group/sub_data" in h5
+    assert "not_there" not in h5
+
+
+def test_dataset(zarr_dir):
+    h5 = zarrh5.ZarrH5(zarr_dir)
+    assert h5["data"].shape == (3, 4)
+    assert h5["data"].size == 12
+    assert h5["data"].dtype == numpy.dtype("f8")
+    assert len(h5["data"]) == 3
+    assert h5["data"].chunks == (3, 4)
+    assert h5["data"].compression == "zstd"
+
+    numpy.testing.assert_array_equal(h5["data"][...], numpy.arange(12).reshape(3, 4))
+    numpy.testing.assert_array_equal(h5["data"].value, numpy.arange(12).reshape(3, 4))
+    numpy.testing.assert_array_equal(h5["data"][0], numpy.arange(4))
+
+
+def test_dataset_no_compression(zarr_dir):
+    h5 = zarrh5.ZarrH5(zarr_dir)
+    assert h5["no_compression"].compression is None
+
+
+def test_attrs(zarr_dir):
+    h5 = zarrh5.ZarrH5(zarr_dir)
+    assert dict(h5.attrs) == {"file_attr": "hello"}
+    assert dict(h5["group"].attrs) == {"group_attr": 42}
+    assert dict(h5["data"].attrs) == {"units": "m"}

src/silx/io/utils.py

@@ -32,6 +32,7 @@
 import sys
 import time
 import logging
+import urllib.parse
 from collections.abc import Generator
 
 import numpy
@@ -49,6 +50,12 @@
 except ImportError:
     h5pyd = None
 
+try:
+    from .zarrh5 import ZarrH5
+except ImportError:
+    ZarrH5 = None
+
+
 logger = logging.getLogger(__name__)
 
 NEXUS_HDF5_EXT = [".h5", ".nx5", ".nxs", ".hdf", ".hdf5", ".cxi"]
@@ -690,34 +697,46 @@ def open(filename):  # pylint:disable=redefined-builtin
     :raises: IOError if the file can't be loaded or path can't be found
     :rtype: h5py-like node
     """
-    url = DataUrl(filename)
+    url = urllib.parse.urlparse(filename)
+    if url.scheme.startswith("zarr+"):
+        if ZarrH5 is None:
+            raise IOError(
+                f"Zarr support is not available, please install zarr, cannot open: {filename}"
+            )
+        try:
+            return ZarrH5(filename)
+        except Exception as e:
+            raise IOError(f"Failed to open URL with zarr: {type(e)} {e}")
 
-    if url.scheme() in [None, "file", "silx"]:
+    data_url = DataUrl(filename)
+    if data_url.scheme() in [None, "file", "silx"]:
         # That's a local file
-        if not url.is_valid():
+        if not data_url.is_valid():
             raise OSError("URL '%s' is not valid" % filename)
-        h5_file = _open_local_file(url.file_path())
-    elif url.scheme() in ("http", "https"):
+        h5_file = _open_local_file(data_url.file_path())
+    elif data_url.scheme() in ("http", "https"):
         return _open_url_with_h5pyd(filename)
     else:
-        raise OSError(f"Unsupported URL scheme {url.scheme}: {filename}")
+        raise OSError(f"Unsupported URL scheme {data_url.scheme}: {filename}")
 
-    if url.data_path() in [None, "/", ""]:  # The full file is requested
-        if url.data_slice():
+    if data_url.data_path() in [None, "/", ""]:  # The full file is requested
+        if data_url.data_slice():
             raise OSError(f"URL '{filename}' containing slicing is not supported")
         return h5_file
     else:
         # Only a children is requested
-        if url.data_path() not in h5_file:
-            msg = f"File '{filename}' does not contain path '{url.data_path()}'."
+        if data_url.data_path() not in h5_file:
+            msg = f"File '{filename}' does not contain path '{data_url.data_path()}'."
             raise OSError(msg)
-        node = h5_file[url.data_path()]
+        node = h5_file[data_url.data_path()]
 
-        if url.data_slice() is not None:
+        if data_url.data_slice() is not None:
             from . import _sliceh5  # Lazy-import to avoid circular dependency
 
             try:
-                return _sliceh5.DatasetSlice(node, url.data_slice(), attrs=node.attrs)
+                return _sliceh5.DatasetSlice(
+                    node, data_url.data_slice(), attrs=node.attrs
+                )
             except ValueError:
                 raise OSError(
                     f"URL {filename} contains slicing, but it is not a dataset"

src/silx/io/zarrh5.py

@@ -0,0 +1,163 @@
+# /*##########################################################################
+# Copyright (C) 2025 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ############################################################################*/
+"""
+Provides a wrapper to expose `Zarr <https://zarr.readthedocs.io/>`_
+This is a preview feature.
+"""
+
+from __future__ import annotations
+
+import logging
+import urllib.parse
+from typing import Generator
+import numpy
+from . import commonh5
+import zarr
+
+
+_logger = logging.getLogger(__name__)
+
+
+def _children(group: zarr.Group) -> Generator[ZarrDataset | ZarrGroup]:
+    for name, item in group.members():
+        if isinstance(item, zarr.Group):
+            yield ZarrGroup(name, item, attrs=item.attrs)
+        elif isinstance(item, zarr.Array):
+            yield ZarrDataset(name, item, attrs=item.attrs)
+        else:
+            _logger.warning(f"Cannot map child {name}: Ignored")
+
+
+class ZarrH5(commonh5.File):
+    """Zarr client wrapper"""
+
+    def __init__(
+        self,
+        name: str,
+        mode: str | None = None,
+        attrs: dict | None = None,
+    ) -> None:
+        assert mode in ("r", None)
+        if name.startswith("zarr+"):
+            name = name[5:]
+        full_url = urllib.parse.urlparse(name)
+        if full_url.fragment:
+            raise ValueError("URL fragment is not supported")
+
+        base_url = urllib.parse.urlunparse(
+            (full_url.scheme, full_url.netloc, full_url.path, "", "", "")
+        )
+
+        # quick&dirty storage_options parsing: it would need pydantic model
+        storage_options = {}
+        for key, values in urllib.parse.parse_qs(full_url.query).items():
+            value = values[-1]
+            if key == "use_ssl":
+                value = True if value.lower() == "true" else False
+            storage_options[key] = value
+        self.__group = zarr.open_group(
+            base_url, storage_options=storage_options or None
+        )
+
+        if attrs is None:
+            attrs = {}
+        super().__init__(
+            base_url.rstrip("/"), mode, attrs={**self.__group.attrs, **attrs}
+        )
+
+        for child in _children(self.__group):
+            self.add_node(child)
+
+        _logger.warning(
+            "Zarr support is a preview feature: This may change or be removed without notice."
+        )
+
+    def close(self) -> None:
+        super().close()
+        self.__group = None
+
+
+class ZarrGroup(commonh5.LazyLoadableGroup):
+    """Zarr Group wrapper"""
+
+    def __init__(
+        self,
+        name: str,
+        group: zarr.Group,
+        parent: ZarrH5 | ZarrGroup | None = None,
+        attrs: dict | None = None,
+    ) -> None:
+        super().__init__(name, parent, attrs)
+        self.__group = group
+
+    def _create_child(self) -> None:
+        for child in _children(self.__group):
+            self.add_node(child)
+
+
+class ZarrDataset(commonh5.Dataset):
+    """Zarr Array wrapper"""
+
+    def __init__(
+        self,
+        name: str,
+        array: zarr.Array,
+        parent: ZarrH5 | ZarrGroup | None = None,
+        attrs: dict | None = None,
+    ) -> None:
+        super().__init__(name, array, parent, attrs)
+
+    @property
+    def shape(self) -> tuple[int, ...]:
+        return self._get_data().shape
+
+    @property
+    def size(self) -> int:
+        return self._get_data().size
+
+    def __len__(self) -> int:
+        return self.shape[0]
+
+    def __getitem__(self, item):
+        return self._get_data()[item]
+
+    @property
+    def value(self) -> numpy.ndarray:
+        return self._get_data()[()]
+
+    @property
+    def compression(self):
+        data = self._get_data()
+        compressors = data.compressors
+        if not compressors:
+            return None
+        codec = compressors[0]
+        if data.metadata.zarr_format == 2:
+            # zarr format 2 compressors are numcodecs codecs
+            return codec.codec_id
+        # zarr format 3 compressors are zarr codecs
+        return codec.to_dict().get("name")
+
+    @property
+    def chunks(self):
+        return self._get_data().chunks