From 6d03d6859716bed357ab7c5516938d5fc7ca0f88 Mon Sep 17 00:00:00 2001 From: Olivier Hoenen Date: Fri, 27 Feb 2026 15:26:16 +0100 Subject: [PATCH 01/18] Fix training assets (#105) --- imas/assets/ITER_134173_106_equilibrium.ids | 2 -- imas/test/test_convert_core_edge_plasma.py | 2 +- imas/training.py | 17 +++++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/imas/assets/ITER_134173_106_equilibrium.ids b/imas/assets/ITER_134173_106_equilibrium.ids index 2650ff50..d429344e 100644 --- a/imas/assets/ITER_134173_106_equilibrium.ids +++ b/imas/assets/ITER_134173_106_equilibrium.ids @@ -26,8 +26,6 @@ equilibrium/vacuum_toroidal_field/b0 dim: 1 size: 3 -5.2999999999999998e+00 -5.2999999999999998e+00 -5.2999999999999998e+00 -equilibrium/grids_ggd - size: 1 equilibrium/time_slice size: 3 equilibrium/time_slice[0]/boundary/outline/r diff --git a/imas/test/test_convert_core_edge_plasma.py b/imas/test/test_convert_core_edge_plasma.py index 08d8ca91..165e85fd 100644 --- a/imas/test/test_convert_core_edge_plasma.py +++ b/imas/test/test_convert_core_edge_plasma.py @@ -12,7 +12,7 @@ def assert_equal(core_edge, plasma): def test_convert_training_core_profiles(): - with imas.training.get_training_db_entry() as entry: + with imas.training.get_training_db_entry(convert=True) as entry: cp = entry.get("core_profiles") pp = imas.convert_to_plasma_profiles(cp) diff --git a/imas/training.py b/imas/training.py index 6effcc5b..53c85c91 100644 --- a/imas/training.py +++ b/imas/training.py @@ -2,8 +2,6 @@ # You should have received the IMAS-Python LICENSE file with this project. """Functions that are useful for the IMAS-Python training courses.""" -from unittest.mock import patch - try: from importlib.resources import files except ImportError: # Python 3.8 support @@ -12,15 +10,22 @@ import imas -def get_training_db_entry() -> imas.DBEntry: - """Open and return an ``imas.DBEntry`` pointing to the training data.""" +def get_training_db_entry(convert=False) -> imas.DBEntry: + """Open and return an ``imas.DBEntry`` pointing to the training data. + + Args: + convert: if True, converts assets to default DD version + """ assets_path = files(imas) / "assets/" entry = imas.DBEntry(f"imas:ascii?path={assets_path}", "r") - output_entry = imas.DBEntry("imas:memory?path=/", "w") + version = None if convert else "3.39.0" + output_entry = imas.DBEntry("imas:memory?path=/", "w", dd_version=version) for ids_name in ["core_profiles", "equilibrium"]: ids = entry.get(ids_name, autoconvert=False) - with patch.dict("os.environ", {"IMAS_AL_DISABLE_VALIDATE": "1"}): + if convert: output_entry.put(imas.convert_ids(ids, output_entry.dd_version)) + else: + output_entry.put(ids) entry.close() return output_entry From 5a721671a5358f146bba3569589c360816829817 Mon Sep 17 00:00:00 2001 From: Maarten Sebregts <110895564+maarten-ic@users.noreply.github.com> Date: Tue, 3 Mar 2026 16:53:27 +0100 Subject: [PATCH 02/18] Feature/list filled paths (#104) --- docs/source/intro.rst | 40 +++++---- imas/backends/db_entry_impl.py | 8 ++ imas/backends/imas_core/al_context.py | 11 +++ imas/backends/imas_core/db_entry_al.py | 13 +++ imas/backends/imas_core/imas_interface.py | 5 ++ imas/backends/netcdf/db_entry_nc.py | 29 +++++-- imas/db_entry.py | 66 +++++++++++++- imas/test/test_list_filled_paths.py | 101 ++++++++++++++++++++++ 8 files changed, 249 insertions(+), 24 deletions(-) create mode 100644 imas/test/test_list_filled_paths.py diff --git a/docs/source/intro.rst b/docs/source/intro.rst index 3027a242..80b183fc 100644 --- a/docs/source/intro.rst +++ b/docs/source/intro.rst @@ -86,17 +86,6 @@ get an error message if this is not possible: Load and store an IDS to disk with IMAS-Core '''''''''''''''''''''''''''''''''''''''''''' -.. note:: - - - This functionality requires the IMAS-Core, until this library is openly available - on GitHub you may need to fetch it from `git.iter.org `_ - (requires to have an ITER account). Using IMAS-Core also enable slicing methods - :py:meth:`~imas.db_entry.DBEntry.get_slice`, - :py:meth:`~imas.db_entry.DBEntry.put_slice` and - :py:meth:`~imas.db_entry.DBEntry.get_sample` (with IMAS-Core>=5.4). - - If you can't have access to it, you can save IDS to disk with the built-in - netCDF backend :ref:`Load and store an IDS to disk with netCDF` - To store an IDS to disk, we need to indicate the following URI to the IMAS-Core: ``imas:?path=`` or using the legacy query keys ``imas:?user=;database=;version=;pulse=;run=`` @@ -115,11 +104,9 @@ In IMAS-Python you do this as follows: >>> # now store the core_profiles IDS we just populated >>> dbentry.put(core_profiles) -.. image:: imas_structure.png - To load an IDS from disk, you need to specify the same information as when storing the IDS (see above). Once the data entry is opened, you -can use ``.get()`` to load IDS data from disk: +can use ``dbentry.get()`` to load IDS data from disk: .. code-block:: python @@ -146,7 +133,7 @@ In IMAS-Python you do this as follows: To load an IDS from disk, you need to specify the same file information as when storing the IDS. Once the data entry is opened, you -can use ``.get()`` to load IDS data from disk: +can use ``dbentry.get()`` to load IDS data from disk: .. code-block:: python @@ -154,3 +141,26 @@ can use ``.get()`` to load IDS data from disk: >>> dbentry2 = imas.DBEntry("mypulsefile.nc","r") >>> core_profiles2 = dbentry2.get("core_profiles") >>> print(core_profiles2.ids_properties.comment.value) + + +Data Entry API overview +''''''''''''''''''''''' + +See the documentation of :py:class:`imas.DBEntry ` for more +details on reading and writing IDSs to disk. Useful functions include: + +- :py:meth:`~imas.db_entry.DBEntry.put` and :py:meth:`~imas.db_entry.DBEntry.put_slice` + to write a full IDS or write append a time slice to existing data. +- :py:meth:`~imas.db_entry.DBEntry.get`, :py:meth:`~imas.db_entry.DBEntry.get_slice` and + :py:meth:`~imas.db_entry.DBEntry.get_sample` to read all time slices, a single time + slice, or a sample of time slices from disk. ``get_slice()`` and ``get_sample()`` can + also interpolate data to a requested point in time. + + All three ``get()`` methods have a ``lazy`` mode, which will only load data from disk + when you need it. This can greatly speed up data access in some scenarios. See + :ref:`Lazy loading` for more details. +- :py:meth:`~imas.db_entry.DBEntry.list_all_occurrences` to query whether there are any + occurrences of a certain IDS stored on disk. +- :py:meth:`~imas.db_entry.DBEntry.list_filled_paths` to query which Data Dictionary + paths have data filled inside a specific IDS. + diff --git a/imas/backends/db_entry_impl.py b/imas/backends/db_entry_impl.py index 0c1b2cd6..f6ed4d60 100644 --- a/imas/backends/db_entry_impl.py +++ b/imas/backends/db_entry_impl.py @@ -120,3 +120,11 @@ def delete_data(self, ids_name: str, occurrence: int) -> None: @abstractmethod def list_all_occurrences(self, ids_name: str) -> List[int]: """Implement DBEntry.list_all_occurrences()""" + + @abstractmethod + def list_filled_paths(self, ids_name: str, occurrence: int) -> List[str]: + """Implement DBEntry.list_filled_paths(). + + N.B. DD conversion is handled in DBEntry.list_filled_paths(), this method + returns the data paths as stored on-disk. + """ diff --git a/imas/backends/imas_core/al_context.py b/imas/backends/imas_core/al_context.py index d3d2f620..21cf1b81 100644 --- a/imas/backends/imas_core/al_context.py +++ b/imas/backends/imas_core/al_context.py @@ -174,6 +174,17 @@ def list_all_occurrences(self, ids_name: str) -> List[int]: return list(occurrences) return [] + def list_filled_paths(self, path: str) -> List[str]: + """List all filled paths in an IDS. + + Args: + path: IDS and occurrence as a string: [/] + """ + status, result = ll_interface.list_filled_paths(self.ctx, path) + if status != 0: + raise LowlevelError(f"list filled paths for {path!r}", status) + return result + def close(self): """Close this ALContext.""" ll_interface.end_action(self.ctx) diff --git a/imas/backends/imas_core/db_entry_al.py b/imas/backends/imas_core/db_entry_al.py index fc58270d..0a24bdb8 100644 --- a/imas/backends/imas_core/db_entry_al.py +++ b/imas/backends/imas_core/db_entry_al.py @@ -364,6 +364,19 @@ def list_all_occurrences(self, ids_name: str) -> List[int]: ) from None return occurrence_list + def list_filled_paths(self, ids_name: str, occurrence: int) -> List[str]: + if self._db_ctx is None: + raise RuntimeError("Database entry is not open.") + ll_path = ids_name + if occurrence != 0: + ll_path += f"/{occurrence}" + paths = self._db_ctx.list_filled_paths(ll_path) + if not paths: + raise DataEntryException( + f"IDS {ids_name!r}, occurrence {occurrence} is empty." + ) + return paths + def _check_uda_warnings(self, lazy: bool) -> None: """Various checks / warnings for the UDA backend.""" cache_mode = self._querydict.get("cache_mode") diff --git a/imas/backends/imas_core/imas_interface.py b/imas/backends/imas_core/imas_interface.py index c9d69a02..a9fb66bb 100644 --- a/imas/backends/imas_core/imas_interface.py +++ b/imas/backends/imas_core/imas_interface.py @@ -166,6 +166,11 @@ def begin_timerange_action( ): raise self._minimal_version("5.4") + # New method in AL 5.7 + + def list_filled_paths(self, ctx, path): + raise self._minimal_version("5.7") + # Dummy documentation for interface: for funcname in dir(LowlevelInterface): diff --git a/imas/backends/netcdf/db_entry_nc.py b/imas/backends/netcdf/db_entry_nc.py index 0776c47b..e9eae9af 100644 --- a/imas/backends/netcdf/db_entry_nc.py +++ b/imas/backends/netcdf/db_entry_nc.py @@ -92,6 +92,14 @@ def close(self, *, erase: bool = False) -> None: ) self._dataset.close() + def _get_group(self, ids_name: str, occurrence: int) -> "netCDF4.Group": + try: + return self._dataset[f"{ids_name}/{occurrence}"] + except LookupError as exc: + raise DataEntryException( + f"IDS {ids_name!r}, occurrence {occurrence} is not found." + ) from exc + def get( self, ids_name: str, @@ -110,12 +118,7 @@ def get( raise NotImplementedError(f"`{func}` is not available for netCDF files.") # Check if the IDS/occurrence exists, and obtain the group it is stored in - try: - group = self._dataset[f"{ids_name}/{occurrence}"] - except KeyError: - raise DataEntryException( - f"IDS {ids_name!r}, occurrence {occurrence} is not found." - ) + group = self._get_group(ids_name, occurrence) # Load data into the destination IDS if self._ds_factory.dd_version == destination._dd_version: @@ -183,3 +186,17 @@ def list_all_occurrences(self, ids_name: str) -> List[int]: occurrence_list.sort() return occurrence_list + + def list_filled_paths(self, ids_name: str, occurrence: int) -> List[str]: + # Check if the IDS/occurrence exists, and obtain the group it is stored in + group = self._get_group(ids_name, occurrence) + + result = [] + for name, variable in group.variables.items(): + if variable.ndim == 0 and variable.dtype == "S1": + continue # (Array of) Structure metadata node, no data + if name.endswith(":shape"): + continue # Shape data, not a DD path + result.append(name.replace(".", "/")) + + return result diff --git a/imas/db_entry.py b/imas/db_entry.py index 5a470641..4343e5d9 100644 --- a/imas/db_entry.py +++ b/imas/db_entry.py @@ -7,7 +7,7 @@ import logging import os import pathlib -from typing import Any, Type, overload +from typing import Any, Type, overload, List import numpy @@ -197,14 +197,14 @@ def _select_implementation(uri: str | None) -> Type[DBEntryImpl]: from imas.backends.imas_core.db_entry_al import ALDBEntryImpl as impl return impl - def __enter__(self): + def __enter__(self) -> "DBEntry": # Context manager protocol if self._dbe_impl is None: # Open if the DBEntry was not already opened or created self.open() return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, exc_value, traceback) -> None: # Context manager protocol self.close() @@ -800,3 +800,63 @@ def list_all_occurrences(self, ids_name, node_path=None): self.get(ids_name, occ, lazy=True)[node_path] for occ in occurrence_list ] return occurrence_list, node_content_list + + def list_filled_paths( + self, ids_name, occurrence: int = 0, *, autoconvert: bool = True + ) -> List[str]: + """Get a list of filled Data Dictionary paths from the backend. + + Note that this is only supported by some backends (HDF5 and netCDF), and will + result in an error on unsupported backends. + + Args: + ids_name: Name of the IDS to request filled data for. + occurrence: Occurrence number of the IDS to request filled data for. + + Keyword Args: + autoconvert: If enabled (default), this method will take NBC renames into + account in the returned list of filled paths. This argument corresponds + to the :py:data:`~get.autoconvert` argument of :py:meth:`get`. + + Returns: + List of paths which have some data filled in the backend. For example, when + ``profiles_1d/ion/temperature`` is in this list, it means that there is at + least one ``ion`` in one ``profiles_1d`` entry for which the temperature is + filled. + + The paths in this list may be ordered arbitrarily. + + Example: + >>> with imas.DBEntry("imas:hdf5?path=./path/to/data", "r") as entry: + >>> print(entry.list_filled_paths("core_profiles")) + ['ids_properties/comment', 'ids_properties/homogeneous_time', + 'profiles_1d/grid/rho_tor_norm', 'profiles_1d/electrons/temperature', + 'profiles_1d/ion/temperature', 'time'] + """ + if self._dbe_impl is None: + raise RuntimeError("Database entry is not open.") + paths = self._dbe_impl.list_filled_paths(ids_name, occurrence) + if not autoconvert: + return paths + + # DD conversion? + dd_version = self._dbe_impl.read_dd_version(ids_name, occurrence) + if dd_version == self._ids_factory.dd_version: + return paths # No conversion required + + # Follow any NBC renames: + ddmap, source_is_older = dd_version_map_from_factories( + ids_name, IDSFactory(version=dd_version), self._ids_factory + ) + nbc_map = ddmap.old_to_new if source_is_older else ddmap.new_to_old + + converted_paths = [] + for path in paths: + if path in nbc_map: + new_name = nbc_map.path[path] + if new_name is not None: + converted_paths.append(new_name) + else: + converted_paths.append(path) + + return converted_paths diff --git a/imas/test/test_list_filled_paths.py b/imas/test/test_list_filled_paths.py new file mode 100644 index 00000000..9df74f4d --- /dev/null +++ b/imas/test/test_list_filled_paths.py @@ -0,0 +1,101 @@ +import pytest + +import imas +from imas_core import _al_lowlevel +from imas.exception import DataEntryException +from imas.ids_defs import IDS_TIME_MODE_HOMOGENEOUS, IDS_TIME_MODE_INDEPENDENT + + +if not hasattr(_al_lowlevel, "al_list_filled_paths"): + marker = pytest.mark.xfail(reason="list_filled_paths not available in imas_core") +else: + marker = [] + + +@pytest.fixture(params=["netcdf", pytest.param("hdf5", marks=marker)]) +def testuri(request, tmp_path): + if request.param == "netcdf": + return str(tmp_path / "list_filled_paths.nc") + return f"imas:{request.param}?path={tmp_path}/list_filled_paths_{request.param}" + + +def test_list_filled_paths(testuri): + with imas.DBEntry(testuri, "w", dd_version="4.0.0") as dbentry: + # No IDSs in the DBEntry yet, expect an exception + with pytest.raises(DataEntryException): + dbentry.list_filled_paths("core_profiles") + + cp = dbentry.factory.core_profiles() + cp.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS + cp.ids_properties.comment = "comment" + cp.time = [0.1, 0.2] + cp.profiles_1d.resize(2) + cp.profiles_1d[0].grid.rho_tor_norm = [1.0, 2.0] + cp.profiles_1d[0].ion.resize(2) + cp.profiles_1d[0].ion[1].temperature = [1.0, 2.0] + cp.profiles_1d[1].grid.psi = [1.0, 2.0] + cp.profiles_1d[1].q = [1.0, 2.0] + cp.profiles_1d[1].e_field.radial = [1.0, 2.0] + cp.profiles_1d[1].neutral.resize(2) + cp.global_quantities.ip = [1.0, 2.0] + + dbentry.put(cp) + + filled_paths = dbentry.list_filled_paths("core_profiles") + assert isinstance(filled_paths, list) + assert set(filled_paths) == { + "ids_properties/version_put/access_layer", + "ids_properties/version_put/access_layer_language", + "ids_properties/version_put/data_dictionary", + "ids_properties/homogeneous_time", + "ids_properties/comment", + "time", + "profiles_1d/grid/rho_tor_norm", + "profiles_1d/ion/temperature", + "profiles_1d/grid/psi", + "profiles_1d/q", + "profiles_1d/e_field/radial", + "profiles_1d/e_field/radial", + "global_quantities/ip", + } + # Other occurrence should still raise an error: + with pytest.raises(DataEntryException): + dbentry.list_filled_paths("core_profiles", 1) + # Until we write data to the occurrence: + dbentry.put(cp, 3) + assert set(filled_paths) == set(dbentry.list_filled_paths("core_profiles", 3)) + + +def test_list_filled_paths_autoconvert(testuri): + with imas.DBEntry(testuri, "w", dd_version="3.25.0") as entry: + ps = entry.factory.pulse_schedule() + ps.ids_properties.homogeneous_time = IDS_TIME_MODE_INDEPENDENT + ps.ec.antenna.resize(1) + ps.ec.antenna[0].launching_angle_pol.reference_name = "test" + entry.put(ps) + + filled_paths = entry.list_filled_paths("pulse_schedule") + assert set(filled_paths) == { + "ids_properties/version_put/access_layer", + "ids_properties/version_put/access_layer_language", + "ids_properties/version_put/data_dictionary", + "ids_properties/homogeneous_time", + "ec/antenna/launching_angle_pol/reference_name", + } + + # Check autoconvert with DD 3.28.0 + with imas.DBEntry(testuri, "r", dd_version="3.28.0") as entry: + assert set(entry.list_filled_paths("pulse_schedule", autoconvert=False)) == { + "ids_properties/version_put/access_layer", + "ids_properties/version_put/access_layer_language", + "ids_properties/version_put/data_dictionary", + "ids_properties/homogeneous_time", + "ec/antenna/launching_angle_pol/reference_name", # original name + } + assert set(entry.list_filled_paths("pulse_schedule")) == { + "ids_properties/version_put/access_layer", + "ids_properties/version_put/access_layer_language", + "ids_properties/version_put/data_dictionary", + "ids_properties/homogeneous_time", + "ec/launcher/steering_angle_pol/reference_name", # autoconverted name + } From fad72c4eb16bac72998f1620c794e24f86a4a400 Mon Sep 17 00:00:00 2001 From: Maarten Sebregts <110895564+maarten-ic@users.noreply.github.com> Date: Thu, 26 Mar 2026 14:17:09 +0100 Subject: [PATCH 03/18] Enable readthedocs `fail_on_warning` (#111) --- .github/workflows/verify_sphinx_doc.yml | 53 ------------------------- .readthedocs.yml | 2 +- docs/source/conf.py | 3 ++ 3 files changed, 4 insertions(+), 54 deletions(-) delete mode 100644 .github/workflows/verify_sphinx_doc.yml diff --git a/.github/workflows/verify_sphinx_doc.yml b/.github/workflows/verify_sphinx_doc.yml deleted file mode 100644 index 6a12690e..00000000 --- a/.github/workflows/verify_sphinx_doc.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: verify-sphinx-doc-generation - -on: - push: - pull_request: - types: [opened, synchronize, reopened] - -jobs: - build-and-test: - runs-on: ubuntu-22.04 - - steps: - - name: Checkout IMAS-Python sources - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - with: - # until saxonche is available in 3.13 - # https://saxonica.plan.io/issues/6561 - python-version: "<3.13" - - - name: Display Python version - run: python -c "import sys; print(sys.version)" - - - - name: Set up Python virtual environment - run: | - python -m venv venv - source venv/bin/activate - - - name: Install build dependencies - run: | - pip install --upgrade pip setuptools wheel build - - - name: Build package - run: | - rm -rf dist - python -m build . - - - name: Install package and dependencies - run: | - pip install "$(readlink -f dist/*.whl)[docs,netcdf]" - - - name: Debug dependencies - run: | - pip freeze - - - name: Build Sphinx documentation - run: | - export SPHINXOPTS='-W -n --keep-going' - make -C docs clean html diff --git a/.readthedocs.yml b/.readthedocs.yml index 426920c7..05ae2765 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -20,4 +20,4 @@ python: sphinx: builder: html configuration: docs/source/conf.py - fail_on_warning: false \ No newline at end of file + fail_on_warning: true diff --git a/docs/source/conf.py b/docs/source/conf.py index 06f59e76..f5a8a205 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -34,6 +34,9 @@ author = "ITER Organization" src_host = "https://github.com/iterorganization/" +# Warn about missing references +nitpicky = True + # Parse urls here for convenience, to be re-used # ITER docs iter_projects = "https://github.com/iterorganization/" From 39306068ec50903154dcd33db9a8e0ede559779d Mon Sep 17 00:00:00 2001 From: Olivier Hoenen Date: Wed, 1 Apr 2026 13:47:21 +0200 Subject: [PATCH 04/18] Fix conversion 3to4 of name/identifier when identifier is empty (#115) --- imas/ids_convert.py | 15 +++++++++++++++ imas/test/test_ids_convert.py | 23 +++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/imas/ids_convert.py b/imas/ids_convert.py index a1707c16..ce6a1296 100644 --- a/imas/ids_convert.py +++ b/imas/ids_convert.py @@ -456,6 +456,8 @@ def _apply_3to4_conversion(self, old: Element, new: Element) -> None: # Map DD3 name -> DD4 description if name_path not in self.old_to_new.path: self._add_rename(name_path, desc_path) + # GH#114: Also preserve name in DD4 name when identifier is empty + self.old_to_new.type_change[name_path] = _name_identifier_3to4 # Map DD3 identifier -> DD4 name if id_path in self.old_to_new.path: @@ -1154,6 +1156,19 @@ def _circuit_connections_4to3(node: IDSPrimitive) -> None: node.value = new_value +def _name_identifier_3to4(source_name: IDSBase, target_description: IDSBase) -> None: + """Preserve name when identifier is empty, see GH#114.""" + # Always copy DD3 name -> DD4 description + target_description.value = source_name.value + + # When DD3 identifier is empty, also preserve name in DD4 name + source_parent = source_name._parent + source_identifier = getattr(source_parent, "identifier", None) + if source_identifier is None or not source_identifier.value: + target_parent = target_description._parent + target_parent.name = source_name.value + + def _ids_properties_source(source: IDSString0D, provenance: IDSStructure) -> None: """Handle DD3to4 migration of ids_properties/source to ids_properties/provenance.""" if len(provenance.node) > 0: diff --git a/imas/test/test_ids_convert.py b/imas/test/test_ids_convert.py index b79fb1ba..1c712b75 100644 --- a/imas/test/test_ids_convert.py +++ b/imas/test/test_ids_convert.py @@ -575,6 +575,29 @@ def test_4to3_name_identifier_mapping_magnetics(): assert dst.b_field_pol_probe[0].identifier == "TEST_NAME" +def test_3to4_name_identifier_empty_identifier(): + """GH#114: name must be preserved when identifier is empty.""" + factory = IDSFactory("3.40.1") + + src = factory.pf_active() + src.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS + src.coil.resize(2) + # Case 1: name populated, identifier empty + src.coil[0].name = "TEST_NAME" + src.coil[0].identifier = "" + # Case 2: name populated, identifier not set at all + src.coil[1].name = "TEST_NAME2" + + dst = convert_ids(src, "4.0.0") + + # name must be preserved in DD4 name (not overwritten by empty identifier) + assert dst.coil[0].name == "TEST_NAME" + assert dst.coil[0].description == "TEST_NAME" + + assert dst.coil[1].name == "TEST_NAME2" + assert dst.coil[1].description == "TEST_NAME2" + + def test_3to4_cocos_hardcoded_paths(): # Check for existence in 3.42.0 factory = IDSFactory("3.42.0") From 062c2ca155418666efa79e1d54883efa62b31fb1 Mon Sep 17 00:00:00 2001 From: Olivier Hoenen Date: Wed, 1 Apr 2026 17:20:30 +0200 Subject: [PATCH 05/18] Clarify API doc (#106) --- imas/db_entry.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/imas/db_entry.py b/imas/db_entry.py index 4343e5d9..7bc02ed9 100644 --- a/imas/db_entry.py +++ b/imas/db_entry.py @@ -113,7 +113,10 @@ def __init__( object before you can use it for reading or writing data. Args: - uri: URI to the data entry, see explanation above. + uri: URI to the data entry (a path to a netCDF file ending with .nc, + or an `IMAS URI for one of IMAS-Core's backends + `__ + ). mode: Mode to open the Data Entry in: - ``"r"``: Open an existing data entry. Raises an error when the data From abd7b6cfe008ad25c259aabbfaa7c8485ece5d08 Mon Sep 17 00:00:00 2001 From: Maarten Sebregts <110895564+maarten-ic@users.noreply.github.com> Date: Thu, 2 Apr 2026 15:59:15 +0200 Subject: [PATCH 06/18] Include metadata variables for (arrays of) structures in `to_xarray` (#112) --- docs/source/netcdf.rst | 36 ++++++++++++ imas/_to_xarray.py | 33 ++++++----- imas/backends/netcdf/ids2nc.py | 58 ++++--------------- imas/backends/netcdf/ids_tensorizer.py | 79 ++++++++++++++++++++++---- imas/backends/netcdf/nc2ids.py | 27 +++++---- imas/test/test_to_xarray.py | 27 ++++++++- 6 files changed, 175 insertions(+), 85 deletions(-) diff --git a/docs/source/netcdf.rst b/docs/source/netcdf.rst index 868ae429..f772144e 100644 --- a/docs/source/netcdf.rst +++ b/docs/source/netcdf.rst @@ -183,3 +183,39 @@ specific paths inside the IDS. The latter variant can also be combined with # profiles_1d.grid.rho_tor # profiles_1d.grid.rho_tor_norm # profiles_1d.grid.psi + + +Store Xarray Datasets in IMAS-compatible netCDF file +'''''''''''''''''''''''''''''''''''''''''''''''''''' + +.. versionadded:: 2.3.0 :py:meth:`~imas.util.to_xarray` now includes the required + metadata to load the IDS from a netCDF file. + +The following snippet shows how to store an IMAS Xarray dataset in an IMAS-compatible +netCDF file. The group name in the netCDF file must correspond to ``/`` (``core_profiles/0`` in the snippet). + +.. code-block:: python + :caption: Store IMAS Xarray dataset in an IMAS-compatible netCDF file + + import imas.training + import netCDF4 + + with imas.training.get_training_db_entry() as training_entry: + core_profiles = training_entry.get("core_profiles") + xrds = imas.util.to_xarray(core_profiles) + + # Store the xarray dataset in an IMAS-compatible netCDF dataset + filename = "data.nc" + xrds.to_netcdf( + filename, + group="core_profiles/0", # Update to the correct IDS name and occurrence + # auto_complex=True, # Uncomment if the dataset contains complex data + ) + # Set global DD version metadata + with netCDF4.Dataset(filename, "a") as ds: + ds.data_dictionary_version = imas.util.get_data_dictionary_version(ids) + + # Test that we can get the IDS from the netCDF file + with imas.DBEntry(filename, "r") as entry: + ids2 = entry.get("core_profiles") diff --git a/imas/_to_xarray.py b/imas/_to_xarray.py index 13525c82..979b126f 100644 --- a/imas/_to_xarray.py +++ b/imas/_to_xarray.py @@ -8,7 +8,7 @@ from imas.ids_data_type import IDSDataType fillvals = { - IDSDataType.INT: -(2**31) + 1, + IDSDataType.INT: numpy.int32(-(2**31) + 1), IDSDataType.STR: "", IDSDataType.FLT: numpy.nan, IDSDataType.CPX: numpy.nan * (1 + 1j), @@ -50,21 +50,28 @@ def to_xarray(ids: IDSToplevel, *paths: str) -> xarray.Dataset: var_name = path.replace("/", ".") metadata = ids.metadata[path] if metadata.data_type in (IDSDataType.STRUCTURE, IDSDataType.STRUCT_ARRAY): - continue # We don't store these in xarray - - dimensions = tensorizer.ncmeta.get_dimensions(path, tensorizer.homogeneous_time) - data = tensorizer.tensorize(path, fillvals[metadata.data_type]) - - attrs = dict(documentation=metadata.documentation) - if metadata.units: - attrs["units"] = metadata.units - coordinates = tensorizer.filter_coordinates(path) - if coordinates: - coordinate_names.update(coordinates.split(" ")) - attrs["coordinates"] = coordinates + # Metadata variables for (arrays of) structures + if paths and path not in paths: + continue + dimensions = () + data = b"" + else: + dimensions = tensorizer.get_dimensions(path) + data = tensorizer.tensorize(path, fillvals[metadata.data_type]) + attrs = tensorizer.get_attributes(path, fillvals) + if "coordinates" in attrs: + coordinate_names.update(attrs["coordinates"].split(" ")) data_vars[var_name] = (dimensions, data, attrs) + # :shape array for sparse data + if path in tensorizer.shapes and metadata.ndim: + shape_name = f"{var_name}:shape" + dimensions = tensorizer.get_shape_dimensions(path) + data = tensorizer.shapes[path] + attrs = tensorizer.get_shape_attributes(var_name) + data_vars[shape_name] = (dimensions, data, attrs) + # Remove coordinates from data_vars and put in coordinates mapping: coordinates = {} for coordinate_name in coordinate_names: diff --git a/imas/backends/netcdf/ids2nc.py b/imas/backends/netcdf/ids2nc.py index 531c7ac2..a01cc6ea 100644 --- a/imas/backends/netcdf/ids2nc.py +++ b/imas/backends/netcdf/ids2nc.py @@ -47,7 +47,6 @@ def create_dimensions(self) -> None: def create_variables(self) -> None: """Create netCDF variables.""" - get_dimensions = self.ncmeta.get_dimensions for path in self.filled_data: metadata = self.ids.metadata[path] var_name = path.replace("/", ".") @@ -75,54 +74,21 @@ def create_variables(self) -> None: if dtype is not dtypes[IDSDataType.CPX]: # Set fillvalue kwargs.update(fill_value=default_fillvals[metadata.data_type]) # Create variable - dimensions = get_dimensions(path, self.homogeneous_time) + dimensions = self.get_dimensions(path) var = self.group.createVariable(var_name, dtype, dimensions, **kwargs) # Fill metadata attributes - var.documentation = metadata.documentation - if metadata.units: - var.units = metadata.units - - ancillary_variables = " ".join( - error_var - for error_var in [f"{var_name}_error_upper", f"{var_name}_error_lower"] - if error_var in self.filled_variables - ) - if ancillary_variables: - var.ancillary_variables = ancillary_variables - - if metadata.data_type is not IDSDataType.STRUCT_ARRAY: - coordinates = self.filter_coordinates(path) - if coordinates: - var.coordinates = coordinates - - # Sparsity and :shape array - if path in self.shapes: - if not metadata.ndim: - # Doesn't need a :shape array: - var.sparse = "Sparse data, missing data is filled with _FillValue" - var.sparse += f" ({default_fillvals[metadata.data_type]})" - - else: - shape_name = f"{var_name}:shape" - var.sparse = f"Sparse data, data shapes are stored in {shape_name}" - - # Create variable to store data shape - dimensions = get_dimensions( - self.ncmeta.aos.get(path), self.homogeneous_time - ) + (f"{metadata.ndim}D",) - shape_var = self.group.createVariable( - shape_name, - SHAPE_DTYPE, - dimensions, - ) - doc_indices = ",".join(chr(ord("i") + i) for i in range(3)) - shape_var.documentation = ( - f"Shape information for {var_name}.\n" - f"{shape_name}[{doc_indices},:] describes the shape of filled " - f"data of {var_name}[{doc_indices},...]. Data outside this " - "shape is unset (i.e. filled with _Fillvalue)." - ) + var.setncatts(self.get_attributes(path, default_fillvals)) + + # :shape array for sparse data + if path in self.shapes and metadata.ndim: + shape_name = f"{var_name}:shape" + # Create variable to store data shape + dimensions = self.get_shape_dimensions(path) + shape_var = self.group.createVariable( + shape_name, SHAPE_DTYPE, dimensions + ) + shape_var.setncatts(self.get_shape_attributes(var_name)) def store_data(self) -> None: """Store data in the netCDF variables""" diff --git a/imas/backends/netcdf/ids_tensorizer.py b/imas/backends/netcdf/ids_tensorizer.py index 7e9e33ec..a4019c9c 100644 --- a/imas/backends/netcdf/ids_tensorizer.py +++ b/imas/backends/netcdf/ids_tensorizer.py @@ -3,7 +3,7 @@ """Tensorization logic to convert IDSs to netCDF files and/or xarray Datasets.""" from collections import deque -from typing import List +from typing import List, Tuple, Dict import numpy @@ -47,13 +47,26 @@ def __init__(self, ids: IDSToplevel, paths_to_tensorize: List[str]) -> None: """Map of IDS paths to filled data nodes.""" self.filled_variables = set() """Set of filled IDS variables""" - self.homogeneous_time = ( + self.homogeneous_time = bool( ids.ids_properties.homogeneous_time == IDS_TIME_MODE_HOMOGENEOUS ) """True iff the IDS time mode is homogeneous.""" self.shapes = {} """Map of IDS paths to data shape arrays.""" + def get_dimensions(self, path: str) -> Tuple[str, ...]: + """Get the dimensions for a netCDF variable. + + Args: + path: Data Dictionary path to the variable, e.g. ``ids_properties/comment``. + """ + return self.ncmeta.get_dimensions(path, self.homogeneous_time) + + def get_shape_dimensions(self, path: str) -> Tuple[str, ...]: + """Get dimensions names for shape array of the tensorized variable""" + ndim = self.ids.metadata[path].ndim + return self.get_dimensions(self.ncmeta.aos.get(path, "")) + (f"{ndim}D",) + def include_coordinate_paths(self) -> None: """Append all paths that are coordinates of self.paths_to_tensorize""" # Use a queue so we can also take coordinates of coordinates into account @@ -62,7 +75,7 @@ def include_coordinate_paths(self) -> None: for path in self.paths_to_tensorize: while path: path, _, _ = path.rpartition("/") - if self.ncmeta.get_dimensions(path, self.homogeneous_time): + if self.get_dimensions(path): queue.append(path) self.paths_to_tensorize = [] @@ -82,7 +95,6 @@ def collect_filled_data(self) -> None: # Initialize dictionary with all paths that could exist in this IDS filled_data = {path: {} for path in self.ncmeta.paths} dimension_size = {} - get_dimensions = self.ncmeta.get_dimensions if self.paths_to_tensorize: # Restrict tensorization to provided paths @@ -102,7 +114,7 @@ def collect_filled_data(self) -> None: ndim = node.metadata.ndim if not ndim: continue - dimensions = get_dimensions(path, self.homogeneous_time) + dimensions = self.get_dimensions(path) # We're only interested in the non-tensorized dimensions: [-ndim:] for dim_name, size in zip(dimensions[-ndim:], node.shape): dimension_size[dim_name] = max(dimension_size.get(dim_name, 0), size) @@ -115,15 +127,13 @@ def collect_filled_data(self) -> None: def determine_data_shapes(self) -> None: """Determine tensorized data shapes and sparsity, save in :attr:`shapes`.""" - get_dimensions = self.ncmeta.get_dimensions - for path, nodes_dict in self.filled_data.items(): metadata = self.ids.metadata[path] # Structures don't have a size if metadata.data_type is IDSDataType.STRUCTURE: continue ndim = metadata.ndim - dimensions = get_dimensions(path, self.homogeneous_time) + dimensions = self.get_dimensions(path) # node shape if it is completely filled full_shape = tuple(self.dimension_size[dim] for dim in dimensions[-ndim:]) @@ -137,7 +147,7 @@ def determine_data_shapes(self) -> None: else: # Data is tensorized, determine if it is homogeneously shaped - aos_dims = get_dimensions(self.ncmeta.aos[path], self.homogeneous_time) + aos_dims = self.get_dimensions(self.ncmeta.aos[path]) shapes_shape = [self.dimension_size[dim] for dim in aos_dims] if ndim: shapes_shape.append(ndim) @@ -168,6 +178,55 @@ def filter_coordinates(self, path: str) -> str: if coordinate in self.filled_variables ) + def get_attributes(self, path: str, fillvals: dict) -> Dict[str, str]: + """Get metadata attributes of the tensorized variable""" + metadata = self.ids.metadata[path] + var_name = path.replace("/", ".") + + assert metadata.documentation is not None + attrs = {"documentation": metadata.documentation} + if metadata.units: + attrs["units"] = metadata.units + + ancillary_variables = " ".join( + error_var + for error_var in [f"{var_name}_error_upper", f"{var_name}_error_lower"] + if error_var in self.filled_variables + ) + if ancillary_variables: + attrs["ancillary_variables"] = ancillary_variables + + if metadata.data_type is not IDSDataType.STRUCT_ARRAY: + coordinates = self.filter_coordinates(path) + if coordinates: + attrs["coordinates"] = coordinates + + # Sparsity + if path in self.shapes: + if not metadata.ndim: + # Doesn't need a :shape array + attrs["sparse"] = ( + "Sparse data, missing data is filled with _FillValue" + f" ({fillvals[metadata.data_type]})" + ) + else: + attrs["sparse"] = ( + f"Sparse data, data shapes are stored in {var_name}:shape" + ) + + return attrs + + def get_shape_attributes(self, var_name: str) -> Dict[str, str]: + """Get attributes of the :shape variable corresponding to var_name""" + doc_indices = ",".join(chr(ord("i") + i) for i in range(3)) + documentation = ( + f"Shape information for {var_name}.\n" + f"{var_name}:shape[{doc_indices},:] describes the shape of filled " + f"data of {var_name}[{doc_indices},...]. Data outside this " + "shape is unset (i.e. filled with _Fillvalue)." + ) + return {"documentation": documentation} + def tensorize(self, path, fillvalue): """ Tensorizes the data at the given path with the specified fill value. @@ -180,7 +239,7 @@ def tensorize(self, path, fillvalue): Returns: A tensor filled with the data from the specified path. """ - dimensions = self.ncmeta.get_dimensions(path, self.homogeneous_time) + dimensions = self.get_dimensions(path) shape = tuple(self.dimension_size[dim] for dim in dimensions) # TODO: depending on the data, tmp_var may be HUGE, we may need a more diff --git a/imas/backends/netcdf/nc2ids.py b/imas/backends/netcdf/nc2ids.py index 564d5210..5688d5aa 100644 --- a/imas/backends/netcdf/nc2ids.py +++ b/imas/backends/netcdf/nc2ids.py @@ -1,6 +1,6 @@ import logging import os -from typing import Optional +from typing import Optional, Tuple import netCDF4 import numpy as np @@ -80,6 +80,14 @@ def __init__( ) self.homogeneous_time = var[()] == IDS_TIME_MODE_HOMOGENEOUS + def get_dimensions(self, path: str) -> Tuple[str, ...]: + """Get the dimensions for a netCDF variable. + + Args: + path: Data Dictionary path to the variable, e.g. ``ids_properties/comment``. + """ + return self.ncmeta.get_dimensions(path, self.homogeneous_time) + def run(self, lazy: bool) -> None: """Load the data from the netCDF group into the IDS.""" self.variables.sort() @@ -130,9 +138,7 @@ def run(self, lazy: bool) -> None: else: # FIXME: extract dimension name from nc file? - dim = self.ncmeta.get_dimensions( - metadata.path_string, self.homogeneous_time - )[-1] + dim = self.get_dimensions(metadata.path_string)[-1] size = self.group.dimensions[dim].size for _, node in indexed_tree_iter(self.ids, target_metadata): node.resize(size) @@ -235,9 +241,7 @@ def _validate_variable(self, var: netCDF4.Variable, metadata: IDSMetadata) -> No raise variable_error(var, "data type", var.dtype, expected_dtype) # Dimensions - expected_dims = self.ncmeta.get_dimensions( - metadata.path_string, self.homogeneous_time - ) + expected_dims = self.get_dimensions(metadata.path_string) if var.dimensions != expected_dims: raise variable_error(var, "dimensions", var.dimensions, expected_dims) @@ -298,9 +302,7 @@ def _validate_sparsity( return # Sparsity is stored with _Fillvalue, nothing to validate # Dimensions - aos_dimensions = self.ncmeta.get_dimensions( - self.ncmeta.aos.get(metadata.path_string), self.homogeneous_time - ) + aos_dimensions = self.get_dimensions(self.ncmeta.aos.get(metadata.path_string)) shape_dimensions = shape_var.dimensions if ( len(shape_dimensions) != len(aos_dimensions) + 1 @@ -331,7 +333,6 @@ def get_child(self, child): Args: child: The child IDS node which should be lazy loaded. - """ metadata = child.metadata path = metadata.path_string @@ -347,9 +348,7 @@ def get_child(self, child): size = nc2ids.group[var.name + ":shape"][self.index][0] else: # FIXME: extract dimension name from nc file? - dim = nc2ids.ncmeta.get_dimensions( - metadata.path_string, nc2ids.homogeneous_time - )[-1] + dim = nc2ids.get_dimensions(metadata.path_string)[-1] size = nc2ids.group.dimensions[dim].size child._set_lazy_context(LazyArrayStructContext(nc2ids, self.index, size)) diff --git a/imas/test/test_to_xarray.py b/imas/test/test_to_xarray.py index a5df6a1e..1ef6f184 100644 --- a/imas/test/test_to_xarray.py +++ b/imas/test/test_to_xarray.py @@ -1,11 +1,13 @@ import numpy as np +import netCDF4 import pytest import imas import imas.training +from imas.test.test_helpers import compare_children from imas.util import to_xarray -pytest.importorskip("xarray") +xarray = pytest.importorskip("xarray") @pytest.fixture @@ -87,8 +89,29 @@ def test_to_xarray(): ids.profiles_1d[0].time = 0.0 # These should all be identical: - ds1 = to_xarray(ids) + ds1 = to_xarray(ids).drop_vars( + ["profiles_1d", "profiles_1d.electrons", "profiles_1d.grid"] + ) ds2 = to_xarray(ids, "profiles_1d.electrons.temperature") ds3 = to_xarray(ids, "profiles_1d/electrons/temperature") assert ds1.equals(ds2) assert ds2.equals(ds3) + + +@pytest.mark.parametrize("idsname", ["core_profiles", "equilibrium"]) +def test_roundtrip_xarray_netcdf(tmp_path, entry, idsname): + ids = entry.get(idsname) + xrds = to_xarray(ids) + fname = f"{tmp_path}/test-{idsname}-xarray.nc" + # First write mandatory file-level metadata + with netCDF4.Dataset(fname, "x") as ds: + ds.data_dictionary_version = imas.util.get_data_dictionary_version(ids) + # Then use xarray to write the IDS + xrds.to_netcdf(fname, "a", format="NETCDF4", group=f"{idsname}/0") + # And read it back with a DBEntry + with imas.DBEntry(fname, "r") as entry: + ids2 = entry.get(idsname) + compare_children(ids, ids2) + # Reading the netCDF file with xarray should produce an identical dataset + ncxrds = xarray.load_dataset(fname, group=f"{idsname}/0") + assert xrds.equals(ncxrds) From 7659e54b877d56a8b5aee455f98b7a9edb3bb458 Mon Sep 17 00:00:00 2001 From: Maarten Sebregts <110895564+maarten-ic@users.noreply.github.com> Date: Mon, 11 May 2026 15:13:14 +0200 Subject: [PATCH 07/18] Migrate deprecated magnetics fields (#119) --- imas/ids_convert.py | 87 +++++++++++++++++++++-------------- imas/test/test_ids_convert.py | 49 ++++++++++++++++++++ 2 files changed, 102 insertions(+), 34 deletions(-) diff --git a/imas/ids_convert.py b/imas/ids_convert.py index ce6a1296..a14f1542 100644 --- a/imas/ids_convert.py +++ b/imas/ids_convert.py @@ -274,16 +274,7 @@ def get_old_path(path: str, previous_name: str) -> str: self.version_old, ) elif self._check_data_type(old_item, new_item): - # use class helper to register simple renames and - # reciprocal mappings self._add_rename(old_path, new_path) - if old_item.get("data_type") in DDVersionMap.STRUCTURE_TYPES: - # Add entries for common sub-elements - for path in old_paths: - if path.startswith(old_path): - npath = path.replace(old_path, new_path, 1) - if npath in new_path_set: - self._add_rename(path, npath) elif nbc_description == "type_changed": pass # We will handle this (if possible) in self._check_data_type elif nbc_description == "repeat_children_first_point": @@ -334,28 +325,40 @@ def get_old_path(path: str, previous_name: str) -> str: # Additional conversion rules for DDv3 to DDv4 if self.version_old.major == 3 and new_version and new_version.major == 4: self._apply_3to4_conversion(old, new) + # 3to4 rules may have introduced additional missing items in self.old_to_new + self._map_missing( + False, old_path_set.difference(new_path_set, self.old_to_new) + ) - def _add_rename(self, old_path: str, new_path: str) -> None: + def _add_rename( + self, old_path: str, new_path: str, reciprocal: bool = True + ) -> None: """Register a simple rename from old_path -> new_path using the path->Element maps stored on the instance (self.old_paths/self.new_paths). This will also add the reciprocal mapping when possible. """ old_item = self.old_paths[old_path] new_item = self.new_paths[new_path] - - # forward mapping + # Forward mapping self.old_to_new[old_path] = ( new_path, _get_tbp(new_item, self.new_paths), _get_ctxpath(new_path, self.new_paths), ) - - # reciprocal mapping - self.new_to_old[new_path] = ( - old_path, - _get_tbp(old_item, self.old_paths), - _get_ctxpath(old_path, self.old_paths), - ) + # Reciprocal mapping + if reciprocal: + self.new_to_old[new_path] = ( + old_path, + _get_tbp(old_item, self.old_paths), + _get_ctxpath(old_path, self.old_paths), + ) + # Apply to descendent nodes as well if the item is a struct or AoS + for item in old_item.findall("field"): + path = item.get("path") + assert path is not None and path.startswith(old_path) + npath = path.replace(old_path, new_path, 1) + if npath in self.new_paths: + self._add_rename(path, npath, reciprocal) def _apply_3to4_conversion(self, old: Element, new: Element) -> None: # Postprocessing for COCOS definition change: @@ -421,6 +424,13 @@ def _apply_3to4_conversion(self, old: Element, new: Element) -> None: to_update[p] = v self.old_to_new.path.update(to_update) + # Migrate additional obsolescent nodes + # TODO: define migrations in a separate variable (as with the sign flips)? + if self.ids_name == "magnetics": + self._add_rename("bpol_probe", "b_field_pol_probe", reciprocal=False) + self._add_rename("method", "ip", reciprocal=False) + self.old_to_new.type_change["method"] = _magnetics_method_to_ip + # GH#59: To improve further the conversion of DD3 to DD4, especially the # Machine Description part of the IDSs, we would like to add a 3to4 specific # rule to convert any siblings name + identifier (that are not part of an @@ -431,19 +441,20 @@ def _apply_3to4_conversion(self, old: Element, new: Element) -> None: # Only perform the mapping if the corresponding target fields exist in the # new DD and if we don't already have a mapping for the involved paths. # use self.old_paths and self.new_paths set in _build_map - for p in self.old_paths: + for name_path in self.old_paths: # look for name children - if not p.endswith("/name"): + if not name_path.endswith("/name"): continue - parent = p.rsplit("/", 1)[0] - name_path = f"{parent}/name" + parent = name_path.rsplit("/", 1)[0] id_path = f"{parent}/identifier" index_path = f"{parent}/index" - desc_path = f"{parent}/description" - new_name_path = name_path + # Follow renames of parent structure + new_parent = self.old_to_new.path.get(parent) or parent + desc_path = f"{new_parent}/description" + new_name_path = f"{new_parent}/name" - # If neither 'name' nor 'identifier' existed in the old DD, skip this parent - if name_path not in self.old_paths or id_path not in self.old_paths: + # If 'identifier' doesn't exist in the old DD, skip this parent + if id_path not in self.old_paths: continue # exclude identifier-structure (has index sibling) if index_path in self.old_paths: @@ -454,14 +465,11 @@ def _apply_3to4_conversion(self, old: Element, new: Element) -> None: continue # Map DD3 name -> DD4 description - if name_path not in self.old_to_new.path: - self._add_rename(name_path, desc_path) - # GH#114: Also preserve name in DD4 name when identifier is empty - self.old_to_new.type_change[name_path] = _name_identifier_3to4 - + self._add_rename(name_path, desc_path) + # GH#114: Also preserve name in DD4 name when identifier is empty + self.old_to_new.type_change[name_path] = _name_identifier_3to4 # Map DD3 identifier -> DD4 name - if id_path in self.old_to_new.path: - self._add_rename(id_path, new_name_path) + self._add_rename(id_path, new_name_path) def _map_missing(self, is_new: bool, missing_paths: Set[str]): rename_map = self.new_to_old if is_new else self.old_to_new @@ -1329,3 +1337,14 @@ def _equilibrium_boundary_3to4(eq3: IDSToplevel, eq4: IDSToplevel, deepcopy: boo node[2].psi = -ts3.boundary_secondary_separatrix.psi # COCOS change node[2].levelset.r = copy(ts3.boundary_secondary_separatrix.outline.r) node[2].levelset.z = copy(ts3.boundary_secondary_separatrix.outline.z) + + +def _magnetics_method_to_ip(method: IDSBase, ip: IDSBase) -> None: + """Convert obsolescent method(:) to ip(:) in the magnetics IDS.""" + if not len(method): + return + ip.resize(len(method)) + for old_item, new_item in zip(method, ip, strict=True): + new_item.method_name.value = old_item.name.value + new_item.data.value = old_item.ip.data.value + new_item.time.value = old_item.ip.time.value diff --git a/imas/test/test_ids_convert.py b/imas/test/test_ids_convert.py index 1c712b75..0568bbb8 100644 --- a/imas/test/test_ids_convert.py +++ b/imas/test/test_ids_convert.py @@ -439,6 +439,55 @@ def test_3to4_cocos_magnetics_workaround(dd4factory): compare_children(mag, mag3) +def test_3to4_deprecated_magnetics(dd4factory): + # Test migrating deprecated bpol_probe + mag = IDSFactory("3.39.0").magnetics() + mag.bpol_probe.resize(2) + mag.bpol_probe[0].name = "name1" + mag.bpol_probe[0].identifier = "identifier1" + mag.bpol_probe[0].position.r = 1 + mag.bpol_probe[0].field.data = [0.1, 0.2, 0.3] + mag.bpol_probe[1].name = "name2" + mag.bpol_probe[1].voltage.data = [0.1, 0.2, 0.3] + + mag.method.resize(2) + for i, method in enumerate(mag.method): + method.name = f"name{i}" + method.ip.data = [i, 1.0, 2.0] + method.ip.time = [i + 1, 2.0, 3.0] + + mag4 = convert_ids(mag, None, factory=dd4factory) + assert len(mag4.b_field_pol_probe) == 2 + assert mag4.b_field_pol_probe[0].name == "identifier1" + assert mag4.b_field_pol_probe[0].description == "name1" + assert mag4.b_field_pol_probe[0].position.r == 1 + assert array_equal(mag4.b_field_pol_probe[0].field.data, [0.1, 0.2, 0.3]) + assert mag4.b_field_pol_probe[1].name == "name2" + assert mag4.b_field_pol_probe[1].description == "name2" + assert array_equal(mag4.b_field_pol_probe[1].voltage.data, [0.1, 0.2, 0.3]) + + assert len(mag4.ip) == 2 + assert mag4.ip[0].method_name == "name0" + assert array_equal(mag4.ip[0].data, [0.0, 1.0, 2.0]) + assert array_equal(mag4.ip[0].time, [1.0, 2.0, 3.0]) + assert mag4.ip[1].method_name == "name1" + assert array_equal(mag4.ip[1].data, [1.0, 1.0, 2.0]) + assert array_equal(mag4.ip[1].time, [2.0, 2.0, 3.0]) + + # If both the deprecated and the "correct" quantity exist, we expect only the + # correct one to be converted to DD4: + mag.b_field_pol_probe.resize(1) + mag.b_field_pol_probe[0].name = "test" + mag.ip.resize(1) + mag.ip[0].method_name = "ip" + + mag4 = convert_ids(mag, None, factory=dd4factory) + assert len(mag4.b_field_pol_probe) == 1 + assert mag4.b_field_pol_probe[0].name == "test" + assert len(mag4.ip) == 1 + assert mag4.ip[0].method_name == "ip" + + def test_3to4_pulse_schedule(): ps = IDSFactory("3.39.0").pulse_schedule() ps.ids_properties.homogeneous_time = IDS_TIME_MODE_HETEROGENEOUS From 77d73f8bcebe163814c8b5bcbfcdf065a9ff8356 Mon Sep 17 00:00:00 2001 From: Louwrens van Dellen Date: Mon, 18 May 2026 16:31:16 +0200 Subject: [PATCH 08/18] ci: add dependabot.yml to update github-actions --- .github/dependabot.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..d52cfe9a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + # Enable version updates for Github Actions + - package-ecosystem: "github-actions" + # Look for `/.github/workflows` and `/action.yml` or `.yaml` + directory: "/" + # Check for updates once a week + schedule: + interval: "weekly" + From b7b61cda2ae3782c8449d7fd326a7c6b5b51150c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 07:11:01 +0000 Subject: [PATCH 09/18] Bump actions/upload-artifact from 4 to 7 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4 to 7. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...v7) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- .github/workflows/test_with_pytest.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a2f88020..b05cec7b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,7 +25,7 @@ jobs: - name: Build a binary wheel and a source tarball run: python3 -m build . - name: Store the distribution packages - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: python-package-distributions path: dist/ diff --git a/.github/workflows/test_with_pytest.yml b/.github/workflows/test_with_pytest.yml index 03c4716e..f0f3cbe9 100644 --- a/.github/workflows/test_with_pytest.yml +++ b/.github/workflows/test_with_pytest.yml @@ -37,13 +37,13 @@ jobs: python -m pytest -n=auto --cov=imas --cov-report=term-missing --cov-report=xml:coverage.xml --cov-report=html:htmlcov --junit-xml=junit.xml - name: Upload coverage report ${{ matrix.python-version }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: coverage-report-${{ matrix.python-version }} path: htmlcov - name: Upload test report ${{ matrix.python-version }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: test-report-${{ matrix.python-version }} path: junit.xml From 1c73f65e66e518b55752c186e270daf95c4bed20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 07:11:00 +0000 Subject: [PATCH 10/18] Bump actions/checkout from 4 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/linting.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/test_with_pytest.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index e18f1c38..65f225cc 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout IMAS-Python sources - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b05cec7b..641dbaa3 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,7 +10,7 @@ jobs: name: Build distribution runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Set up Python diff --git a/.github/workflows/test_with_pytest.yml b/.github/workflows/test_with_pytest.yml index f0f3cbe9..26cb42e0 100644 --- a/.github/workflows/test_with_pytest.yml +++ b/.github/workflows/test_with_pytest.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} From 499972257ee287c89bbd721e214ad6ca49d1d944 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 07:11:07 +0000 Subject: [PATCH 11/18] Bump actions/setup-python from 4 to 6 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/linting.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/test_with_pytest.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 65f225cc..b4a47fd7 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: # until saxonche is available in 3.13 # https://saxonica.plan.io/issues/6561 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 641dbaa3..8e2201e6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: # until saxonche is available in 3.13 # https://saxonica.plan.io/issues/6561 diff --git a/.github/workflows/test_with_pytest.yml b/.github/workflows/test_with_pytest.yml index 26cb42e0..e68d08b3 100644 --- a/.github/workflows/test_with_pytest.yml +++ b/.github/workflows/test_with_pytest.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Display Python version From b6f3ed48265de3e0cba25d2f25bc265de1141a03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 07:11:03 +0000 Subject: [PATCH 12/18] Bump actions/download-artifact from 4 to 8 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4 to 8. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v4...v8) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: '8' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e2201e6..7dc38217 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,7 +43,7 @@ jobs: id-token: write # IMPORTANT: mandatory for trusted publishing steps: - name: Download all the dists - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: python-package-distributions path: dist/ @@ -63,7 +63,7 @@ jobs: id-token: write # IMPORTANT: mandatory for trusted publishing steps: - name: Download all the dists - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: python-package-distributions path: dist/ From 16958f61ca20102432ddfa3c6e461f4eb8f9bbf1 Mon Sep 17 00:00:00 2001 From: Maarten Sebregts Date: Tue, 26 May 2026 14:36:18 +0200 Subject: [PATCH 13/18] Remove outdated "TODO" comments for imas-core dependency --- pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ed3f964e..924fc581 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,10 +78,6 @@ docs = [ "sphinx-immaterial>=0.11.0,<0.12", "sphinx-click", ] - -# TODO enable when imas-core is available on pypi -# imas-core = [ "imas-core@git+https://github.com/iterorganization/imas-core.git@main" ] - netcdf = [ "netCDF4>=1.4.1", ] @@ -106,7 +102,6 @@ test = [ # Pint is used in training snippets "pint", # Optional dependencies - # TODO add imas-core when it is available on pypi "imas-python[netcdf,h5py,xarray,saxonche]", ] From dceda68b78d1be3bff77e81fe0a14ac13a00c0f8 Mon Sep 17 00:00:00 2001 From: Olivier Hoenen Date: Tue, 26 May 2026 19:00:57 +0200 Subject: [PATCH 14/18] remove uda workaround option which fails with non-existing fields in the current default DD version --- docs/source/multi-dd.rst | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/docs/source/multi-dd.rst b/docs/source/multi-dd.rst index ea0e9f1b..9d694b16 100644 --- a/docs/source/multi-dd.rst +++ b/docs/source/multi-dd.rst @@ -290,24 +290,17 @@ you may see the following error: Dictionary version of the DBEntry (3.42.0). This is not supported when using the UDA backend. -There are three possible workarounds. The first two require passing an additional option +There are two possible workarounds. The first one requires passing an additional option in the IMAS UDA URI: please see the `imas-core documentation `__ -for more details on these URI options. +for more details on URI options. 1. Use UDA fetch to bypass the cache problem. You can do this by appending ``&fetch=1`` to the URI when you create the :py:class:`~imas.db_entry.DBEntry`. Note that this will download the entire IDS files from the remote server, this may not be desired if you only want to read a single time slice. -2. Disable the UDA cache. You can do this by appending ``&cache_mode=none`` to the URI - when you create the :py:class:`~imas.db_entry.DBEntry`. - - Note that this may make the ``get()`` (a lot) slower, since a separate request needs - to be sent to the remote UDA server for every data variable. However, this may still - be the best performing option if you are only interested in a subset of all the data - in an IDS (and use :ref:`lazy loading`). -3. Explicitly provide the data dictionary version when you create the +2. Explicitly provide the data dictionary version when you create the :py:class:`~imas.db_entry.DBEntry`, setting it to match the Data Dictionary version of the data you want to load. To obtain the version of the data on the remote server from the field `ids_properties.put_version.data_dictionary` via a _lazy_ ``get()`` @@ -316,7 +309,7 @@ for more details on these URI options. Note that you may need to call ``imas.convert_ids`` to convert the IDS to your desired Data Dictionary version. -All three possible workarounds are shown in the examples below: +The two possible workarounds are shown in the examples below: .. md-tab-set:: @@ -346,20 +339,7 @@ All three possible workarounds are shown in the examples below: with imas.DBEntry(URI, "r") as entry: cp = entry.get("core_profiles") - .. md-tab-item:: 2. Disable the UDA cache - - .. code-block:: python - - import imas - - URI = ( - "imas://uda.iter.org:56565/uda?backend=hdf5" - "&path=/work/imas/shared/imasdb/ITER/3/121013/50&cache_mode=none" - ) - with imas.DBEntry(URI, "r") as entry: - cp = entry.get("core_profiles") - - .. md-tab-item:: 3. Explicitly provide the DD version + .. md-tab-item:: 2. Explicitly provide the DD version .. code-block:: python From 8caae6d0f459e392bb0b6a050a76011aac8d203a Mon Sep 17 00:00:00 2001 From: Maarten Sebregts Date: Wed, 10 Jun 2026 15:54:53 +0200 Subject: [PATCH 15/18] Add `units` metadata to Identifiers N.B. This commit also includes a refactor to remove IDSIdentifier.__init__ and use __new__ only. See https://docs.python.org/3/howto/enum.html#when-to-use-new-vs-init --- docs/source/identifiers.rst | 33 ++++++++++++++++++++++----- imas/ids_identifiers.py | 43 +++++++++++++++++++++++------------ imas/test/test_identifiers.py | 27 ++++++++++++++-------- 3 files changed, 73 insertions(+), 30 deletions(-) diff --git a/docs/source/identifiers.rst b/docs/source/identifiers.rst index 408c7abe..8a05a4a5 100644 --- a/docs/source/identifiers.rst +++ b/docs/source/identifiers.rst @@ -14,13 +14,16 @@ enumerated list of options for defining, for example: - These may have alternative naming conventions supported through aliases (e.g., "235U" and "U_235" for Uranium 235). -Identifiers are a list of possible valid labels. Each label has up to four -representations: +Identifiers are a list of possible valid options. Each option has three representations +that are stored in an IDS: 1. An index (integer) 2. A name (short string) 3. A description (long string) -4. List of aliases (list of short strings) + +.. seealso:: + `Data Dictionary documentation for identifiers + `__ Identifiers in IMAS-Python @@ -32,6 +35,20 @@ constructed on-demand from the loaded Data Dictionary definitions. All identifier enums can be accessed through ``imas.identifiers``. A list of the available identifiers is stored as ``imas.identifiers.identifiers``. +Each identifier option provides the following attributes: + +- ``name``: the name of the option. +- ``index``: the integer index value of the option. +- ``description``: a longer string describing the option. +- ``aliases``: a list of aliases that can be used instead of the name. +- ``units``: optional information about the units of the quantities that are affected by + the identifier option. Take, for example, the `poloidal plan coordinate identifier + `__ + which affects the units of ``grid/dim1`` and ``grid/dim2``. + +.. versionadded:: 2.1.0 ``aliases`` for identifiers. +.. versionadded:: 2.3.0 ``units`` metadata. + .. code-block:: python :caption: Accessing identifiers @@ -47,6 +64,9 @@ the available identifiers is stored as ``imas.identifiers.identifiers``. print(csid.total.index) print(csid.total.description) + # Search identifier options by their index value + print(csid(1)) + # Access identifiers with aliases (when available) mid = imas.identifiers.materials_identifier print(mid["235U"].name) # Access by canonical name @@ -57,12 +77,12 @@ the available identifiers is stored as ``imas.identifiers.identifiers``. assert mid["235U"].name is mid.U_235.name # Item access is also possible - print(identifiers["edge_source_identifier"]) + print(imas.identifiers["edge_source_identifier"]) # You can use imas.util.inspect to list all options - imas.util.inspect(identifiers.ggd_identifier) + imas.util.inspect(imas.identifiers.ggd_identifier) # And also to get more details of a specific option - imas.util.inspect(identifiers.ggd_identifier.SN) + imas.util.inspect(imas.identifiers.ggd_identifier.SN) # When an IDS node is an identifier, you can use # metadata.identifier_enum to get the identifier @@ -186,6 +206,7 @@ material_identifier["235U"]. mat.names[0] = mid.U_235.name # enum value via alias mat.names[0] = mid["U_235"].name # enum value via alias + Compare identifiers ------------------- diff --git a/imas/ids_identifiers.py b/imas/ids_identifiers.py index 1525a070..8bb522ef 100644 --- a/imas/ids_identifiers.py +++ b/imas/ids_identifiers.py @@ -4,7 +4,7 @@ import logging from enum import Enum -from typing import Iterable, List, Type +from typing import Iterable, List, Type, Optional from xml.etree.ElementTree import fromstring from imas import dd_zip @@ -15,19 +15,29 @@ class IDSIdentifier(Enum): """Base class for all identifier enums.""" - def __new__(cls, value: int, description: str, aliases: list = []): + name: str + """Name of this identifier value.""" + index: int + """Unique index for this identifier value.""" + description: str + """Description for this identifier value.""" + aliases: list[str] + """Alternative names for this identifier value.""" + units: Optional[str] + """Units of the quantity/quantities altered by this identifier. May be ``None`` if + the Data Dictionary doesn't provide this metadata.""" + + def __new__( + cls, value: int, description: str, aliases: list[str], units: Optional[str] + ): obj = object.__new__(cls) obj._value_ = value + obj.index = value + obj.description = description + obj.aliases = aliases + obj.units = units return obj - def __init__(self, value: int, description: str, aliases: list = []) -> None: - self.index = value - """Unique index for this identifier value.""" - self.description = description - """Description for this identifier value.""" - self.aliases = aliases - """Alternative names for this identifier value.""" - def __eq__(self, other): if self is other: return True @@ -68,19 +78,22 @@ def __eq__(self, other): def _from_xml(cls, identifier_name, xml) -> Type["IDSIdentifier"]: element = fromstring(xml) enum_values = {} - aliases = {} for int_element in element.iterfind("int"): name = int_element.get("name") value = int_element.text + assert value is not None description = int_element.get("description") # alias attribute may contain multiple comma-separated aliases alias_attr = int_element.get("alias", "") aliases = [a.strip() for a in alias_attr.split(",") if a.strip()] - # Canonical entry: use the canonical 'name' as key - enum_values[name] = (int(value), description, aliases) - # Also add alias names as enum *aliases* (they become enum attributes) + units = int_element.get("units") + + # Identifier can be looked up by its name or any of its aliases: + enumvalue = (int(value), description, aliases, units) + enum_values[name] = enumvalue for alias in aliases: - enum_values[alias] = (int(value), description, aliases) + enum_values[alias] = enumvalue + # Create the enumeration enum = cls( identifier_name, diff --git a/imas/test/test_identifiers.py b/imas/test/test_identifiers.py index 119e0e88..f7e7b5fc 100644 --- a/imas/test/test_identifiers.py +++ b/imas/test/test_identifiers.py @@ -6,11 +6,9 @@ from imas.ids_factory import IDSFactory from imas.ids_identifiers import IDSIdentifier, identifiers -has_aliases = Version(importlib.metadata.version("imas_data_dictionaries")) >= Version( - "4.1.0" -) -requires_aliases = pytest.mark.skipif( - not has_aliases, reason="Requires DD 4.1.0 for identifier aliases" +requires_dd4_1 = pytest.mark.skipif( + Version(importlib.metadata.version("imas_data_dictionaries")) < Version("4.1.0"), + reason="Test requires DD 4.1.0 for additional identifier metadata", ) @@ -112,7 +110,7 @@ def test_identifiers_with_aliases(): assert identifier.CxHy.aliases == ["alias1", "alias2", "3alias"] -@requires_aliases +@requires_dd4_1 def test_identifier_struct_assignment_with_aliases(): """Test identifier struct assignment with aliases using materials_identifier.""" mid = identifiers.materials_identifier @@ -174,7 +172,7 @@ def test_invalid_identifier_assignment(): cs.source[0].identifier = -1 -@requires_aliases +@requires_dd4_1 def test_identifier_aliases(): """Test identifier enum aliases functionality.""" mid = identifiers.materials_identifier @@ -197,7 +195,7 @@ def test_identifier_aliases(): assert mid[alias] is mid.U_235 -@requires_aliases +@requires_dd4_1 def test_identifier_alias_equality(): """Test that identifiers with aliases are equal when comparing names and aliases.""" mid = identifiers.materials_identifier @@ -276,7 +274,7 @@ def test_identifier_alias_equality(): assert mat5.descriptions[2] == mid["U_235"].description -@requires_aliases +@requires_dd4_1 def test_identifier_alias_equality_non_ggd(): """Test identifier aliases functionality on non-ggd material""" mid = identifiers.materials_identifier @@ -293,3 +291,14 @@ def test_identifier_alias_equality_non_ggd(): summary_ids.wall.material.name = "235U" # Use canonical name assert summary_ids.wall.material == mid["235U"] assert summary_ids.wall.material == mid["U_235"] + + +@requires_dd4_1 +def test_identifier_units(): + ppcid = identifiers.poloidal_plane_coordinates_identifier + assert ppcid.rectangular.units == "m,m" + assert ppcid.inverse.units == "m,rad" + + # materials identifier doesn't have units (and I don't expect they'll ever get any) + mid = identifiers.materials_identifier + assert mid.W.units is None From 7789093e5d72f254137981216894c6f8b6111348 Mon Sep 17 00:00:00 2001 From: Olivier Hoenen Date: Mon, 15 Jun 2026 13:01:16 +0200 Subject: [PATCH 16/18] fix deprecation warning with __array__'s copy keyword for Numpy>2 (#132) --- imas/ids_primitive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/imas/ids_primitive.py b/imas/ids_primitive.py index a86faa95..4843ca02 100644 --- a/imas/ids_primitive.py +++ b/imas/ids_primitive.py @@ -334,7 +334,7 @@ class IDSNumeric0D(IDSPrimitive): __doc__ = IDSDoc(__doc__) __slots__ = () - def __array__(self, dtype=None): + def __array__(self, dtype=None, copy=None): return np.array(self.value, dtype=dtype) def __str__(self): @@ -437,8 +437,8 @@ class IDSNumericArray(IDSPrimitive, np.lib.mixins.NDArrayOperatorsMixin): # list, to support operations like np.add(array_like, list) _HANDLED_TYPES = (np.ndarray, Number) - def __array__(self, dtype=None): - return self.value.astype(dtype, copy=False) + def __array__(self, dtype=None, copy=None): + return self.value.astype(dtype, copy=bool(copy)) def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): out = kwargs.get("out", ()) From 82fdd1f3c538c048e89af199e8ca5222c4ac46c8 Mon Sep 17 00:00:00 2001 From: Olivier Hoenen Date: Wed, 17 Jun 2026 16:04:46 +0200 Subject: [PATCH 17/18] Update changelog for 2.3.0 release (and add missing one for 2.2.0) --- docs/source/changelog.rst | 49 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 0e7348c1..d5105fd0 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -3,6 +3,55 @@ Changelog ========= +What's new in IMAS-Python 2.3.0 +------------------------------- + +Features +'''''''' + +- :merge:`131`: introduce alias and units metadata in the :ref:`Identifiers` API +- :merge:`104`: add a :py:func:`imas.DBEntry.list_filled_paths` function (requires imas_core >= 5.7 for HDF5 backend) + +Improvements +'''''''''''' + +- :issue:`116`: migrate `magnetics` obsolescent fields (`method -> ip`, `bpol_probe -> b_field_pol_probe`) with a value during a 3to4 conversion +- :merge:`112`: include metadata variables when writing an IDS to a netCDF files with the function `to_xarray`, allowing to read back the IDS with a `get` (see :ref:`Store Xarray Datasets in IMAS-compatible netCDF file`) + - improve documentation w.r.t netcdf URI and UDA backend usage limitations and associated workarounds + +Bug fixes +''''''''' + +- :issue:`118`: fix deprecation warning with copy keyword for __array__ implementation (Numpy > 2) +- :issue:`117`: fix 3to4 conversion of name/identifier when identifier is empty +- :merge:`105`: fix training data assets that failed loading + + + +What's new in IMAS-Python 2.2.0 +------------------------------- + +Features +'''''''' +- :issue:`44`: add `--convert-to-plasma-ids` option to `imas convert` + +Improvements +'''''''''''' + +- :merge:`100`: remove legacy tool `extract_test_data` +- :merge:`98`: support MDSplus models configuration change introduced in imas_core >= 5.6 +- :issue:`97`: better documentation, exception message and control of DD compatibility when using UDA backend +- :merge:`95`: defer loading the default DD definitions +- :issue:`91`: remove hidden `has_imas` attribute (may break compatibility of applications that used it!) + +Bug fixes +''''''''' + +- :merge:`100`: fix incorrect type hints +- :issue:`89`: properly unpack 0D data when reading an IDS from a netCDF file + + + What's new in IMAS-Python 2.1.0 ------------------------------- From 77939fcb56a9216ca71385a739992135967db7e6 Mon Sep 17 00:00:00 2001 From: Olivier Hoenen Date: Wed, 17 Jun 2026 16:39:57 +0200 Subject: [PATCH 18/18] fixup documentation refs --- docs/source/changelog.rst | 2 +- docs/source/netcdf.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index d5105fd0..ac662c06 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -10,7 +10,7 @@ Features '''''''' - :merge:`131`: introduce alias and units metadata in the :ref:`Identifiers` API -- :merge:`104`: add a :py:func:`imas.DBEntry.list_filled_paths` function (requires imas_core >= 5.7 for HDF5 backend) +- :merge:`104`: add a :py:func:`imas.db_entry.DBEntry.list_filled_paths` function (requires imas_core >= 5.7 for HDF5 backend) Improvements '''''''''''' diff --git a/docs/source/netcdf.rst b/docs/source/netcdf.rst index f772144e..66fe29f7 100644 --- a/docs/source/netcdf.rst +++ b/docs/source/netcdf.rst @@ -185,6 +185,8 @@ specific paths inside the IDS. The latter variant can also be combined with # profiles_1d.grid.psi +.. _`Store Xarray Datasets in IMAS-compatible netCDF file`: + Store Xarray Datasets in IMAS-compatible netCDF file ''''''''''''''''''''''''''''''''''''''''''''''''''''