diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9128cbc --- /dev/null +++ b/.gitattributes @@ -0,0 +1,95 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +# +# Normalise line endings and, above all, keep the binary test fixtures binary. +# +# An EPC is a ZIP and an .h5 is an HDF5 container: a single CRLF<->LF substitution inside one +# corrupts the archive, and git would happily do it on a Windows checkout for any file it +# guesses is text. The `binary` macro below is `-text -diff`, which turns both the eol +# conversion and the textual diff off. + +# --------------------------------------------------------------------------- +# Default: let git detect, and store text with LF in the repository. +# --------------------------------------------------------------------------- +* text=auto + +# --------------------------------------------------------------------------- +# Source and data formats that are always text +# --------------------------------------------------------------------------- +*.py text diff=python +*.pyi text diff=python +*.md text diff=markdown +*.rst text +*.txt text +*.xml text +*.json text +*.yml text +*.yaml text +*.toml text +*.cfg text +*.ini text +*.csv text +*.html text diff=html +*.css text +*.js text +*.sql text +*.sh text eol=lf +*.bash text eol=lf +Dockerfile text +Makefile text +.gitattributes text +.gitignore text + +# Windows-only scripts keep CRLF or cmd.exe mis-parses them +*.bat text eol=crlf +*.cmd text eol=crlf +*.ps1 text eol=crlf + +# --------------------------------------------------------------------------- +# Energyml / geoscience binaries — never touch, never diff +# --------------------------------------------------------------------------- +*.epc binary +*.h5 binary +*.hdf5 binary +*.parquet binary +*.las binary +*.segy binary +*.sgy binary +*.resqml binary + +# --------------------------------------------------------------------------- +# Generic binaries +# --------------------------------------------------------------------------- +*.zip binary +*.gz binary +*.tgz binary +*.bz2 binary +*.xz binary +*.7z binary +*.jar binary +*.whl binary +*.pdf binary +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.bmp binary +*.tif binary +*.tiff binary +*.woff binary +*.woff2 binary +*.ttf binary +*.otf binary +*.eot binary +*.so binary +*.dll binary +*.dylib binary +*.pyd binary +*.exe binary +*.pyc binary + +# --------------------------------------------------------------------------- +# Keep generated / vendored trees out of the language stats and of PR diffs +# --------------------------------------------------------------------------- +poetry.lock linguist-generated=true -diff diff --git a/.github/workflows/ci_energyml_utils_pull_request.yml b/.github/workflows/ci_energyml_utils_pull_request.yml index 4015294..ef327f7 100644 --- a/.github/workflows/ci_energyml_utils_pull_request.yml +++ b/.github/workflows/ci_energyml_utils_pull_request.yml @@ -36,9 +36,26 @@ jobs: run: | poetry install --all-extras - - name: Run pytest + - name: Check the published test fixtures are present + # tests/ runs against real EPCs rather than mock dataclasses, and rc/**/*.epc is + # git-ignored with a per-file allow list. Only the FESAPI testing packages are cleared + # for publication; the field-data EPCs stay local and their tests skip here. Dropping + # one of these from the allow list would silently turn its tests into skips, so fail + # loudly instead. run: | - poetry run pytest -v --tb=short + missing=0 + for f in rc/epc/testingPackageCpp.epc rc/epc/testingPackageCpp.h5 \ + rc/epc/testingPackageCpp22.epc rc/epc/testingPackageCpp22.h5; do + if [ ! -f "$f" ]; then echo "::error::missing test fixture $f"; missing=1; fi + done + exit $missing + + - name: Run pytest (whole suite, slow tests included) + # pyproject.toml sets `addopts = "-m 'not slow'"` so a local run stays quick; `-m ""` + # clears that filter. Without it CI silently skips the tests that compare EpcFile with + # EpcStreamReader over the large packages. + run: | + poetry run pytest -v --tb=short -m "" --durations=15 build: name: Build distribution diff --git a/.gitignore b/.gitignore index c55fd22..c4a6412 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,14 @@ *.sublime-project *.sublime-workspace +# AI coding assistants: local instructions / settings, not part of the project +.claude/ +CLAUDE.md +AGENTS.md +.cursor/ +.aider* +.github/copilot-instructions.md + # Checkpoints .ipynb_checkpoints __pycache__/ diff --git a/energyml-utils/.gitignore b/energyml-utils/.gitignore index b0e48a8..75994aa 100644 --- a/energyml-utils/.gitignore +++ b/energyml-utils/.gitignore @@ -59,6 +59,10 @@ manip* docs/*.md # DATA +# Default output folder of `extract_3d` / export_multiple_data. The extensions below cover the +# files it writes, but not the ones a malformed name produced before sanitize_file_name existed +# (a ':' in a citation title left extension-less remnants on Windows), so ignore the folder. +exported_meshes/ *.obj *.off *.mtl @@ -72,6 +76,17 @@ rc/**/*.epc rc/**/*.h5 rc/**/*.hdf5 +# ...except the FESAPI testing packages, which are the only fixtures cleared for publication. +# `tests/` runs against real EPCs rather than mock dataclasses, so the suite needs them in CI. +# +# DO NOT add a fixture here without checking it may be published: the other EPCs of rc/epc/ are +# field data (Volve, SPASS, the 80-well surveys) and stay local. A test that needs one of them +# must skip when it is absent — see the `fixture_epc` fixture of tests/test_epc_file.py. +!rc/epc/testingPackageCpp.epc +!rc/epc/testingPackageCpp.h5 +!rc/epc/testingPackageCpp22.epc +!rc/epc/testingPackageCpp22.h5 + # WIP src/energyml/utils/wip* @@ -80,4 +95,14 @@ rc/camunda # code profiling -*.prof \ No newline at end of file +*.prof +AUDIT.md + +# AI coding assistants: local instructions / settings, not part of the project +.claude/ +CLAUDE.md +AGENTS.md + +# Local scratch: dead code kept around locally, and output folders of ad-hoc runs +_to_delete/ +results/ \ No newline at end of file diff --git a/energyml-utils/README.md b/energyml-utils/README.md index b292c01..ddded17 100644 --- a/energyml-utils/README.md +++ b/energyml-utils/README.md @@ -268,16 +268,31 @@ finally: The EpcStreamReader is perfect for applications that need to work with large EPC files efficiently, such as data processing pipelines, web applications, or analysis tools where memory usage is a concern. -# Poetry scripts : - -- extract_3d : extract a representation into an 3D file (obj/off) -- csv_to_dataset : translate csv data into h5 dataset -- generate_data : generate a random data from a qualified_type -- xml_to_json : translate an energyml xml file into json. -- json_to_xml : translate an energyml json file into an xml file +# Command line scripts : + +Installing the package (`pip install energyml-utils`) creates these executables. They live in +`energyml.utils.cli`, so they work from an installed wheel as well as from a checkout. + +- extract_3d : extract a representation into a 3D / GIS file (obj/off/stl/vtk/geojson) +- csv_to_dataset : translate csv data into h5 or parquet datasets (needs the `parquet` extra for the csv reader) +- generate_data : generate a random object from a qualified_type +- generate_multiple_data : same, for several types at once, optionally one file per object +- xml_to_json : translate an energyml xml file (or every object of an EPC) into json +- json_to_xml : translate an energyml json file into one xml file per object +- json_to_epc : package every object of an energyml json file into a single EPC +- loadNsave : read a file or a folder (json/xml/epc) and write it back as an EPC - describe_as_csv : create a csv description of an EPC content - validate : validate an energyml object or an EPC instance (or a folder containing energyml objects) +Every command accepts `--help`, and `-v` / `-vv` to raise the log level (`-q` to only report +errors). They can also be called from python, passing the arguments explicitly: + +```python +from energyml.utils.cli import extract_representation_in_3d_file + +extract_representation_in_3d_file(["--epc", "file.epc", "--output", "out", "-ff", "geojson"]) +``` + ## Installation to test poetry scripts : @@ -355,6 +370,85 @@ Extract to OFF format without CRS displacement: poetry run extract_3d --epc "path/to/file.epc" --output "output_folder" --file-format OFF --no-crs ``` +### Extract 3D Representations as GeoJSON + +Export every exportable representation of the EPC to GeoJSON (one `.geojson` file per +representation; a representation that cannot be read is logged and skipped): +```bash +poetry run extract_3d --epc "path/to/file.epc" --output "output_folder" --file-format geojson +``` + +Export only some representations: +```bash +poetry run extract_3d --epc "path/to/file.epc" --output "output_folder" --file-format geojson --uuid "uuid1" "uuid2" +``` + +**Coordinates are reprojected to WGS84 by default**, as required by [RFC 7946](https://www.rfc-editor.org/rfc/rfc7946). +This needs the `crs` extra: +```bash +poetry install --extras crs # or : pip install energyml-utils[crs] +``` + +Without it (or when no EPSG code can be found in the CRS), a warning is logged, the coordinates are +left in their source CRS, and that CRS is advertised in the output through the `crs` (GeoJSON 2008, +read by GDAL / QGIS) and `coordRefSys` (OGC JSON-FG) members. + +Keep the coordinates in the source projected CRS: +```bash +poetry run extract_3d --epc "path/to/file.epc" --output "output_folder" --file-format geojson --no-wgs84 +``` + +Allow PROJ to download the geoid grids used by the vertical datum transformation. Without them the +height conversion is silently skipped, which can be off by tens of metres: +```bash +poetry run extract_3d --epc "path/to/file.epc" --output "output_folder" --file-format geojson --proj-network +``` + +Each feature carries the identification metadata of its source object: the energyml `uuid` in the +RFC 7946 `id` member, and the `uuid`, `qualified_type`, `content_type`, ETP `uri`, `Citation` fields +(title, originator, creation, last_update, …) and EPSG codes in `properties`: + +```json +{ + "type": "FeatureCollection", + "name": "Bartonien Top", + "bbox": [2.3675, 48.9129, 30.13, 2.4057, 48.9198, 37.15], + "features": [ + { + "type": "Feature", + "id": "02cc9411-6b90-4619-a9fd-a39ac332b367", + "properties": { + "uuid": "02cc9411-6b90-4619-a9fd-a39ac332b367", + "qualified_type": "resqml22.PointSetRepresentation", + "uri": "eml:///resqml22.PointSetRepresentation(02cc9411-6b90-4619-a9fd-a39ac332b367)", + "title": "Bartonien Top", + "originator": "Geosiris", + "creation": "2025-12-17T16:11:36Z", + "last_update": "2025-12-17T16:11:36Z", + "projected_epsg_code": 3949, + "source_crs": "EPSG:3949", + "coordinates_crs": "OGC:CRS84" + }, + "geometry": { "type": "MultiPoint", "coordinates": [[2.4055336, 48.9140288, 37.15]] } + } + ] +} +``` + +The same options are available from python: +```python +from energyml.utils.data.mesh import MeshFileFormat, export_multiple_data + +export_multiple_data( + epc_path="path/to/file.epc", + uuid_list=["uuid1"], + output_folder_path="output_folder", + file_format=MeshFileFormat.GEOJSON, + to_wgs84=True, # default + use_network=False, # True to download the geoid grids +) +``` + ### CSV to Dataset Convert CSV to HDF5: @@ -399,6 +493,11 @@ Using qualified type: poetry run generate_data --type "resqml22.WellboreFeature" --file-format json ``` +Generate multiple data : +```bash +poetry run generate_multiple_data -o generated -ff xml -t eml23.AbstractObject --exclude witsml --exclude prodml +``` + ### XML to JSON Conversion Convert an XML file to JSON: diff --git a/energyml-utils/example/tools.py b/energyml-utils/example/tools.py deleted file mode 100644 index 938a058..0000000 --- a/energyml-utils/example/tools.py +++ /dev/null @@ -1,692 +0,0 @@ -# Copyright (c) 2023-2024 Geosiris. -# SPDX-License-Identifier: Apache-2.0 -import argparse -import json -import os -import pathlib -import traceback -from typing import Optional, List, Dict, Any -import sys -from pathlib import Path - -# Add src directory to path -src_path = Path(__file__).parent.parent / "src" -sys.path.insert(0, str(src_path)) - -from energyml.utils.validation import ErrorType, validate_epc - -from energyml.utils.constants import get_property_kind_dict_path_as_xml -from energyml.utils.data.datasets_io import CSVFileReader, HDF5FileWriter, ParquetFileWriter, DATFileReader -from energyml.utils.data.mesh import MeshFileFormat, export_multiple_data, export_obj, read_mesh_object -from energyml.utils.epc import Epc, gen_energyml_object_path -from energyml.utils.introspection import ( - get_class_from_simple_name, - get_enum_values, - get_module_name_and_type_from_content_or_qualified_type, - random_value_from_class, - search_class_in_module_from_partial_name, - set_attribute_from_path, - get_object_attribute, - get_qualified_type_from_class, - get_content_type_from_class, - get_object_attribute_rgx, - get_direct_dor_list, - get_obj_uuid, - get_class_from_qualified_type, - get_object_attribute_or_create, -) -from energyml.utils.serialization import ( - serialize_json, - JSON_VERSION, - serialize_xml, - read_energyml_json_bytes, - read_energyml_xml_bytes, - read_energyml_xml_str, -) - - -def dat_to_h5( - csv_in, - h5_out, - dataset_name: Optional[str] = None, - datasets_prefix: Optional[str] = None, - ignore: List[str] = None, - map_col_name_to_csv_col: Dict[str, List[str]] = None, - **csvparams, -): - """ - :param csv_in: - :param h5_out: - :param dataset_name: if None, csv headers are used - :param ignore: - :param map_col_name_to_csv_col: - :param csvparams: - :return: - """ - reader = DATFileReader() - writer = HDF5FileWriter() - - _ignore = list(map(lambda x: x.lower(), ignore or [])) - - if dataset_name is None: - csv_data = reader.read_array(csv_in, **csvparams) - - if map_col_name_to_csv_col is not None: - for k, col_list in map_col_name_to_csv_col.items(): - col_list = list(map(lambda x: x.lower(), col_list)) - - print("csv_data") - print(csv_data) - data = [] - if len(col_list) > 1: - for h in col_list: - if h.lower() not in _ignore: - try: - data = data + [csv_data[h]] - except KeyError: - pass - _ignore.append(h) - else: - h = col_list[0] if isinstance(col_list, list) else col_list - data = csv_data[h] - _ignore.append(h) - try: - writer.write_array(h5_out, list(map(list, zip(*data))), (datasets_prefix or "") + k) - except ValueError: - continue - except Exception as e: - raise e - headers = csv_data.keys() - for h in headers: - if h not in _ignore: - try: - writer.write_array(h5_out, csv_data[h], (datasets_prefix or "") + h) - except ValueError: - continue - # except Exception as e: - # raise e - - -def csv_to_h5( - csv_in, - h5_out, - dataset_name: Optional[str] = None, - datasets_prefix: Optional[str] = None, - ignore: List[str] = None, - map_col_name_to_csv_col: Dict[str, List[str]] = None, - **csvparams, -): - """ - :param csv_in: - :param h5_out: - :param dataset_name: if None, csv headers are used - :param ignore: - :param map_col_name_to_csv_col: - :param csvparams: - :return: - """ - reader = CSVFileReader() - writer = HDF5FileWriter() - - ignore = ignore or [] - - if dataset_name is None: - csv_data = reader.read_array_as_panda_dict(csv_in, **csvparams) - - if map_col_name_to_csv_col is not None: - for k, col_list in map_col_name_to_csv_col.items(): - print(csv_data) - data = [] - if len(col_list) > 1: - for h in col_list: - if h not in ignore: - try: - data = data + [csv_data[h]] - except KeyError: - pass - ignore.append(h) - else: - h = col_list[0] if isinstance(col_list, list) else col_list - data = csv_data[h] - ignore.append(h) - try: - writer.write_array(h5_out, list(map(list, zip(*data))), (datasets_prefix or "") + k) - except ValueError: - continue - except Exception as e: - raise e - headers = csv_data.keys() - for h in headers: - if h not in ignore: - try: - writer.write_array(h5_out, csv_data[h], (datasets_prefix or "") + h) - except ValueError: - continue - # except Exception as e: - # raise e - - -def csv_to_parquet( - csv_in, - parquet_out, - dataset_name: Optional[str] = None, - datasets_prefix: Optional[str] = None, - ignore: List[str] = None, - map_col_name_to_csv_col: Dict[str, List[str]] = None, - **csvparams, -): - """ - :param csv_in: - :param parquet_out: - :param dataset_name: if None, csv headers are used - :param ignore: - :param map_col_name_to_csv_col: - :param csvparams: - :return: - """ - reader = CSVFileReader() - writer = ParquetFileWriter() - - ignore = ignore or [] - - if dataset_name is None: - csv_data = reader.read_array_as_panda_dict(csv_in, **csvparams) - # print(csv_data) - datadict = {} - if map_col_name_to_csv_col is not None: - for k, col_list in map_col_name_to_csv_col.items(): - data = [] - if len(col_list) > 1: - for h in col_list: - if h not in ignore: - try: - data = data + [csv_data[h]] - except KeyError: - pass - ignore.append(h) - else: - h = col_list[0] if isinstance(col_list, list) else col_list - data = csv_data[h] - ignore.append(h) - try: - datadict[(datasets_prefix or "") + k] = list(map(list, zip(*data))) - except ValueError: - continue - except Exception as e: - raise e - - headers = csv_data.keys() - for h in headers: - if h not in ignore: - try: - datadict[(datasets_prefix or "") + h] = csv_data[h] - except ValueError: - continue - except Exception as e: - raise e - keys = list(datadict.keys()) - writer.write_array(parquet_out, [datadict[k] for k in keys], keys) - - -def csv_to_dataset(): - sample = {"FINAL_DATASET_NAME_A": ["CSV_COL_NAME_0", "CSV_COL_NAME_N"], "FINAL_DATASET_NAME_B": ["CSV_COL_NAME_X"]} - parser = argparse.ArgumentParser() - parser.add_argument("--csv", "-f", type=str, help="Csv file path") - parser.add_argument("--output", "-o", type=str, help="Output file path") - parser.add_argument("--prefix", "-p", type=str, default="", help="Dataset path prefix") - parser.add_argument("--csv-delimiter", "-d", type=str, default=",", help="CSV delimiter") - parser.add_argument( - "--mapping", - "-m", - type=str, - help=f"Json file path. The json content should look like this : {json.dumps(sample)}", - ) - parser.add_argument( - "--mapping-line", "-ml", type=str, help=f"A json dict that should look like this : {json.dumps(sample)}" - ) - parser.add_argument("--ignore", "-i", type=str, help="A csv column name to ignore", nargs="+") - - args = parser.parse_args() - - print(args.csv_delimiter) - print(args.mapping_line) - - mapping = args.mapping_line or args.mapping - if mapping is not None: - mapping = json.loads(mapping) - - print(mapping) - - output_file_path = args.output - if output_file_path.lower().endswith(".parquet") or output_file_path.lower().endswith(".pqt"): - csv_to_parquet( - csv_in=args.csv, - parquet_out=output_file_path, - datasets_prefix=args.prefix, - ignore=args.ignore, - map_col_name_to_csv_col=mapping, - delimiter=args.csv_delimiter, - ) - else: - csv_to_h5( - csv_in=args.csv, - h5_out=output_file_path, - datasets_prefix=args.prefix, - ignore=args.ignore, - map_col_name_to_csv_col=mapping, - delimiter=args.csv_delimiter, - ) - - -def generate_data(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--type", - "-t", - type=str, - default="energyml.resqml.v2_2.resqmlv2.TriangulatedSetRepresentation", - help="Object type (e.g. energyml.resqml.v2_2.resqmlv2.TriangulatedSetRepresentation)", - ) - - parser.add_argument( - "--file-format", - "-ff", - type=str, - default="json", - help="Type of the output files (one of : ['json', 'xml']). Default is 'json'", - ) - - args = parser.parse_args() - - obj_class = None - try: - obj_class = get_class_from_simple_name( - args.type[args.type.rindex(".") + 1 :], [args.type[: args.type.rindex(".")]] - ) - except NameError: - obj_class = get_class_from_qualified_type(args.type) - - if obj_class is None: - print("Class not found, please check the type name.") - print("Possible types are :") - module_name, object_type = get_module_name_and_type_from_content_or_qualified_type(args.type) - for cn in search_class_in_module_from_partial_name(module_name, object_type): - print(f" - {cn.__name__}") - return - - obj = random_value_from_class(obj_class) - if args.file_format.lower() == "xml": - print(serialize_xml(obj)) - else: - print(serialize_json(obj, JSON_VERSION.OSDU_OFFICIAL)) - - -_sample_osdu_map_ = { - "acl.owners": "osduintegration.OwnerGroup", - "acl.viewers": "osduintegration.ViewerGroup", - "legal.legaltags": "osduintegration.LegalTags", - "createTime": "Citation.Creation", - "modifyTime": "Citation.LastUpdate", - "modifyUser": "Citation.Editor", - "createUser": "Citation.Originator", - "data.Name": "Citation.Title", -} - - -def osdu_schema_to_energyml(input: str, target_obj: Any, attrib_map: Dict): - obj_in = json.loads(input) - for k, k_o in attrib_map.items(): - try: - get_object_attribute_or_create(target_obj, k_o) - print(target_obj) - new_value = get_object_attribute(obj_in, k, force_snake_case=False) - set_attribute_from_path(target_obj, k_o, new_value) - except Exception as e: - raise e - return target_obj - - -def extract_representation_in_3d_file(): - parser = argparse.ArgumentParser() - parser.add_argument("--epc", "-f", type=str, help="Epc file path") - parser.add_argument("--output", "-o", type=str, help="Output folder path") - parser.add_argument("--no-crs", action="store_false", help="Disable crs displacement") - parser.add_argument( - "--file-format", - "-ff", - type=MeshFileFormat, - default=MeshFileFormat.OBJ, - help=f"Type of the output files (one of : {[e.value for e in MeshFileFormat]}). Default is 'obj'", - ) - parser.add_argument("--uuid", "-u", type=str, help="The uuids of representations to extract", nargs="+") - - args = parser.parse_args() - - export_multiple_data( - epc_path=args.epc, - uuid_list=args.uuid, - output_folder_path=args.output, - file_format=args.file_format, - use_crs_displacement=not args.no_crs, - ) - - -def prop_kind_to_json(): - from importlib.resources import files - - try: - import energyml.utils.rc as RC - except: - import src.energyml.utils.rc as RC - with files(RC).joinpath(f"PropertyKindDictionary_v2.3.json").open("w", encoding="utf-8") as f: - f.write(serialize_json(read_energyml_xml_str(get_property_kind_dict_path_as_xml()))) - - -def xml_to_json(): - parser = argparse.ArgumentParser() - parser.add_argument("--file", "-f", type=str, help="Input File") - parser.add_argument("--out", "-o", type=str, default=None, help=f"Output file") - - args = parser.parse_args() - - output_path = args.out or args.file[:-4] + ".json" - - json_content = None - if args.file.lower().endswith(".xml"): - with open(args.file, "rb") as f: - f_content = f.read() - objs = read_energyml_xml_bytes(f_content) - json_content = serialize_json(objs, JSON_VERSION.OSDU_OFFICIAL) - elif args.file.lower().endswith(".epc"): - epc = Epc.read_file(args.file) - # print(epc.energyml_objects) - json_content = ( - "[\n" - + ",".join(list(map(lambda o: serialize_json(o, JSON_VERSION.OSDU_OFFICIAL), epc.energyml_objects))) - + "]" - ) - - with open(output_path, "w") as fout: - # print(json_content) - if json_content is not None: - fout.write(json_content) - - -def json_to_xml(): - parser = argparse.ArgumentParser() - parser.add_argument("--file", "-f", type=str, help="Input File") - parser.add_argument("--out", "-o", type=str, default=None, help="Output file") - - args = parser.parse_args() - - with open(args.file, "rb") as f: - f_content = f.read() - objs = [] - try: - objs = read_energyml_json_bytes(f_content, JSON_VERSION.OSDU_OFFICIAL) - except: - objs = read_energyml_json_bytes(f_content, JSON_VERSION.XSDATA) - - dir = pathlib.Path(args.out or args.file).parent.resolve() - for obj in objs: - fname = gen_energyml_object_path(obj) - xml_content = serialize_xml(obj) - with open(f"{dir}/{fname}", "w") as fout: - fout.write(xml_content) - - -def json_to_epc(): - parser = argparse.ArgumentParser() - parser.add_argument("--file", "-f", type=str, help="Input File") - parser.add_argument("--out", "-o", type=str, default=None, help="Output EPC file") - - args = parser.parse_args() - - epc = Epc(epc_file_path=args.out) - with open(args.file, "rb") as f: - f_content = f.read() - objs = [] - try: - objs = read_energyml_json_bytes(f_content, JSON_VERSION.OSDU_OFFICIAL) - except: - objs = read_energyml_json_bytes(f_content, JSON_VERSION.XSDATA) - - dir = pathlib.Path(args.out or args.file).parent.resolve() - for obj in objs: - epc.energyml_objects.append(obj) - - epc.export_file(args.out) - - -def describe_as_csv(): - parser = argparse.ArgumentParser() - parser.add_argument("--folder", "-f", type=str, help="Input File") - parser.add_argument( - "--columnsNames", - "-c", - type=str, - default=["Title", "QualifiedType", "Uuid", "SchemaVersion", "Path", "Dors uuids"], - nargs="*", - help=f"Columns titles", - ) - parser.add_argument( - "--columnsValues", - "-v", - type=str, - default=["citation.title", "$qualifiedtype", "Uuid|Uid", "schemaVersion", "$Path", "$Dor"], - nargs="*", - help=f"Columns values. Use $type/$qualifiedType/$contentType/$path/$dor or simpler, a regex matching an attribute", - ) - - args = parser.parse_args() - print(f"folder : {args.folder}") - objects = [] - print("Reading files") - for filename in os.listdir(args.folder): - f = os.path.join(args.folder, filename) - # checking if it is a file - if os.path.isfile(f): - if f.endswith(".json"): - with open(f, "rb") as file: - f_content = file.read() - objs = [] - try: - objs = read_energyml_json_bytes(f_content, JSON_VERSION.OSDU_OFFICIAL) - except: - objs = read_energyml_json_bytes(f_content, JSON_VERSION.XSDATA) - objects = objects + list(map(lambda _o: (_o, f), objs)) - elif f.endswith(".xml"): - with open(f, "rb") as file: - f_content = file.read() - obj = read_energyml_xml_bytes(f_content) - objects.append((obj, f)) - elif f.endswith(".epc"): - epc = Epc.read_file(f) - objects = objects + list(map(lambda _o: (_o, f), epc.energyml_objects)) - - out_name = "describe.csv" - cpt = 0 - while os.path.exists(os.path.join(args.folder, out_name)): - out_name = f"describe_{cpt}.csv" - cpt += 1 - - print("Parsing objects") - - out_path = os.path.join(args.folder, out_name) - with open(out_path, "w") as out: - for c in args.columnsNames: - out.write(c) - out.write(";") - - out.write("\n") - - for o, path in objects: - for c in args.columnsValues: - if c.startswith("$"): - clw = c.lower() - if clw == "$type": - out.write(type(o)) - elif clw == "$qualifiedtype": - out.write(get_qualified_type_from_class(o)) - elif clw == "$contenttype": - out.write(get_content_type_from_class(o)) - elif clw == "$path": - out.write(path) - elif clw == "$dor": - out.write( - str(list(set(list(map(lambda _o: get_obj_uuid(_o), get_direct_dor_list(o)))))).replace( - ";", ", " - ) - ) - else: - out.write(get_object_attribute_rgx(o, c)) - out.write(";") - out.write("\n") - - print("Finished") - - -def validate_files(): - parser = argparse.ArgumentParser() - # parser.add_argument("--folder", type=str, help="Input folder") - parser.add_argument("--file", "-f", type=str, help="Input file (json or xml or epc)") - parser.add_argument( - "--ignore-err-type", - "-i", - type=str, - help=f"Error types to ignore. Possible values {get_enum_values(ErrorType)}", - nargs="*", - ) - - parser.add_argument( - "--ignore-prodml-version-errs", - action="store_false", - dest="ignore_prodml_version_errs", - help="Disable ignoring errors related to Prodml version (by default, these errors are ignored)", - ) - - parser.add_argument( - "--group-by-err-class", - action="store_true", - help="Group errors by their class (e.g. all validation errors together, all parsing errors together, etc.)", - ) - - args = parser.parse_args() - - objects = [] - - if not os.path.exists(args.file): - print(f"File {args.file} does not exist.") - return - elif not os.path.isdir(args.file) and not args.file.lower().endswith((".json", ".xml", ".epc")): - print(f"File {args.file} is not a valid input file (should be a folder or a json/xml/epc file).") - return - elif os.path.isdir(args.file): - for filename in os.listdir(args.file): - f = os.path.join(args.file, filename) - if os.path.isfile(f): - if f.endswith(".json"): - with open(f, "rb") as file: - f_content = file.read() - try: - objs = read_energyml_json_bytes(f_content, JSON_VERSION.OSDU_OFFICIAL) - objects.extend(objs) - except Exception as e: - print(f"File {filename} is NOT a valid EnergyML JSON file: {e}") - elif f.endswith(".xml"): - with open(f, "rb") as file: - f_content = file.read() - try: - obj = read_energyml_xml_bytes(f_content) - objects.append(obj) - except Exception as e: - print(f"File {filename} is NOT a valid EnergyML XML file: {e}") - elif f.endswith(".epc"): - try: - epc = Epc.read_file(f) - if epc is not None: - objects.extend(epc.energyml_objects) - else: - print(f"File {filename} is NOT a valid EnergyML EPC file: Empty EPC") - except Exception as e: - print(f"File {filename} is NOT a valid EnergyML EPC file: {e}") - elif os.path.isfile(args.file): - f = args.file - filename = os.path.basename(f) - if f.endswith(".json"): - with open(f, "rb") as file: - f_content = file.read() - try: - objs = read_energyml_json_bytes(f_content, JSON_VERSION.OSDU_OFFICIAL) - objects.extend(objs) - except Exception as e: - print(f"File {filename} is NOT a valid EnergyML JSON file: {e}") - elif f.endswith(".xml"): - with open(f, "rb") as file: - f_content = file.read() - try: - obj = read_energyml_xml_bytes(f_content) - objects.append(obj) - except Exception as e: - print(f"File {filename} is NOT a valid EnergyML XML file: {e}") - elif f.endswith(".epc"): - try: - epc = Epc.read_file(f) - if epc is not None: - objects.extend(epc.energyml_objects) - else: - print(f"File {filename} is NOT a valid EnergyML EPC file: Empty EPC") - except Exception as e: - traceback.print_exc() - print(f"File {filename} is NOT a valid EnergyML EPC file: {e}") - - epc = Epc() - epc.energyml_objects = objects - - err_json = [ - err.toJson() - for err in validate_epc(epc) - if str(err.error_type).lower() not in (et.lower() for et in (args.ignore_err_type or [])) - ] - - err_json_sorted = sorted( - err_json, key=lambda x: (x["err_class"], x["error_type"], x["object_uuid"] if "object_uuid" in x else "") - ) - - if args.ignore_prodml_version_errs: - err_json_sorted = [err for err in err_json_sorted if not ("prodml23" in err.get("msg", ""))] - - if args.group_by_err_class: - err_json_grouped = {} - for err in err_json_sorted: - err_class = err.get("err_class", "UnknownErrorClass") - if err_class not in err_json_grouped: - err_json_grouped[err_class] = [] - err_json_grouped[err_class].append(err) - print(json.dumps(err_json_grouped, indent=4)) - else: - # print(json.dumps(err_json, indent=4)) - print(json.dumps(err_json_sorted, indent=4)) - - -# def export_wavefront(): -# parser = argparse.ArgumentParser() -# parser.add_argument("--epc", "-f", type=str, help="Epc file path") -# parser.add_argument("--output", "-o", type=str, help="Output folder path") -# parser.add_argument("--uuid", "-u", type=str, help="The uuids of representations to extract", nargs="+") - -# args = parser.parse_args() - -# epc = Epc.read_file(args.epc) -# for uuid in args.uuid: -# obj = epc.get_object_by_uuid(uuid)[0] - -# mesh = read_mesh_object( -# energyml_object=obj, -# workspace=epc, -# ) - -# if obj is not None: -# fname = gen_energyml_object_path(obj) -# with open(os.path.join(args.output, fname + ".obj"), "w") as f: -# export_obj(mesh_list=mesh, out=f) # Assuming the object can be serialized to XML diff --git a/energyml-utils/pyproject.toml b/energyml-utils/pyproject.toml index e87f632..1690e90 100644 --- a/energyml-utils/pyproject.toml +++ b/energyml-utils/pyproject.toml @@ -58,23 +58,36 @@ python_classes = [ "Test*" ] python_functions = [ "test_*" ] [tool.poetry.extras] -parquet = ["pyarrow", "numpy", "pandas"] -hdf5 = ["h5py"] +# A dependency listed here is made *optional* by poetry, whatever `optional = false` says below. +# `numpy` and `h5py` used to be listed in `parquet` / `hdf5`, so the published wheel declared them +# as `; extra == "parquet"` / `; extra == "hdf5"` — a plain `pip install energyml-utils` installed +# neither, and `import energyml.utils.epc` then died on `ModuleNotFoundError: No module named +# 'numpy'`. They are hard dependencies (numpy is imported at module level, HDF5 is the default +# external-array format of EnergyML) and are therefore declared only in `dependencies` now. +parquet = ["pyarrow", "pandas"] las = ["lasio"] segy = ["segyio"] geometry = ["scipy"] +crs = ["pyproj"] [tool.poetry.dependencies] python = "^3.9" xsdata = {version = "^24.0", extras = ["cli", "lxml"]} energyml-opc = "^1.12.0" h5py = { version = "^3.11.0", optional = false } -numpy = { version = "^1.16.6", optional = false } +# `^1.16.6` meant `<2.0.0`. NumPy 2 is the default in most new environments, and consumers ended +# up on it anyway — where `np.array(x, copy=False)` changed meaning and made every HDF5 array +# silently unreadable (see datasets_io.read_array_view). The spelling is fixed and the suite runs +# on 1.26 and 2.3 alike, so the ceiling is now the next major. +numpy = { version = ">=1.16.6,<3.0.0", optional = false } scipy = { version = ">=1.7", optional = true } pyarrow = { version = "^14.0.1", optional = true } pandas = { version = "^1.1.0", optional = true } lasio = { version = "^0.31", optional = true } segyio = { version = "^1.9", optional = true } +# CRS reprojection (WGS84). pyproj >=3.7 requires python >=3.10, so keep a loose constraint +# and let the resolver pick 3.6.x on python 3.9. +pyproj = { version = ">=3.4", optional = true } [tool.poetry.group.dev.dependencies] coverage = {extras = ["toml"], version = "^6.2"} @@ -149,11 +162,13 @@ format-jinja = """ """ [tool.poetry.scripts] -extract_3d = "example.tools:extract_representation_in_3d_file" -csv_to_dataset = "example.tools:csv_to_dataset" -generate_data = "example.tools:generate_data" -xml_to_json = "example.tools:xml_to_json" -json_to_xml = "example.tools:json_to_xml" -json_to_epc = "example.tools:json_to_epc" -describe_as_csv = "example.tools:describe_as_csv" -validate = "example.tools:validate_files" \ No newline at end of file +extract_3d = "energyml.utils.cli:extract_representation_in_3d_file" +csv_to_dataset = "energyml.utils.cli:csv_to_dataset" +generate_data = "energyml.utils.cli:generate_data" +generate_multiple_data = "energyml.utils.cli:generate_multiple_data" +xml_to_json = "energyml.utils.cli:xml_to_json" +json_to_xml = "energyml.utils.cli:json_to_xml" +json_to_epc = "energyml.utils.cli:json_to_epc" +loadNsave = "energyml.utils.cli:load_n_save" +describe_as_csv = "energyml.utils.cli:describe_as_csv" +validate = "energyml.utils.cli:validate_files" diff --git a/energyml-utils/rc/epc/README.md b/energyml-utils/rc/epc/README.md index 1411d95..868d660 100644 --- a/energyml-utils/rc/epc/README.md +++ b/energyml-utils/rc/epc/README.md @@ -1 +1,90 @@ -TestingPackage epc + h5 files comes from FESAPI library : https://fastapi.tiangolo.com/ \ No newline at end of file +# EPC test fixtures + +## What may be published + +Only the **FESAPI testing packages** are committed: + +| file | provenance | +|---|---| +| `testingPackageCpp.epc` + `.h5` | FESAPI example package, RESQML 2.0.1 — | +| `testingPackageCpp22.epc` + `.h5` | same example package, RESQML 2.2 / EML 2.3 | + +Everything else in this directory is **field or customer data and stays local**: the 80-well +surveys, `SPASS_40+80wells`, the Volve exports, `out-galaxy-*`, `output-val`, `result_pse`. +`.gitignore` ignores `rc/**/*.epc` and `rc/**/*.h5` and re-allows only the four files above — +do not add to that allow list without checking the data may be published. + +A test that needs a local-only EPC **must skip when the file is absent**, never fail: see the +`fixture_epc` fixture of `tests/test_epc_file.py` and the `epc` fixture of +`tests/test_geojson_export.py`. On a fresh clone the suite is green with 28 skips. + +> `80wells_surf_modified_val_color.epc` is misleadingly named: all 165 of its objects come from +> the testing packages (same UUIDs, same `F2I-CONSULTING:FESAPI Example` format) with colour +> maps added, and it shares nothing with `80wells_surf.epc`. It is still not committed — the +> testing packages already cover everything it was used for. + +## What the testing packages cover + +Enough for almost the whole suite, and in particular the grid work +(`tests/test_mesh_numpy_ijk_spec.py` runs entirely on `testingPackageCpp22.epc`): + +- 22 `IjkGridRepresentation`: explicit **and** parametric geometry, left- **and** right-handed, + faulted (split coordinate lines) and unfaulted, K-gaps, `CellGeometryIsDefined=false` cells, + a `ParentWindow` LGR, and a grid whose pillars mix vertical, linear and Z-linear-cubic lines + with NaN-padded knots. +- 6 `GridConnectionSetRepresentation`, with and without `ConnectionInterpretations`. +- `UnstructuredGridRepresentation`, `Grid2dRepresentation`, `TriangulatedSetRepresentation`, + `PolylineSetRepresentation`, `PointSetRepresentation`, `PlaneSetRepresentation`, + `SealedSurfaceFrameworkRepresentation`, `RepresentationSetRepresentation`, `SubRepresentation`, + the wellbore family (trajectory, frame, marker frame, seismic frame), properties, colour maps, + `GraphicalInformationSet`, `ColumnBasedTable`, `TimeSeries`, `PropertyKind`/`PropertySet`. +- Both CRS shapes: `LocalDepth3dCrs` / `LocalTime3dCrs` (2.0.1) and the 2.2 DOR chain + `LocalEngineeringCompoundCrs` → `LocalEngineering2dCrs` → `ProjectedCrs`. +- A resolvable **projected** EPSG code (23031, ED50 / UTM 31N), so local → projected → WGS84 runs. +- Deliberately malformed packaging in `testingPackageCpp.epc` (objects declared twice, wrong + path, wrong content type) — what the `EpcFile` indexing tests rely on. + +## What they do NOT cover + +This is the shopping list for a publishable replacement EPC. + +### CRS / reprojection — the biggest gap + +1. **No vertical EPSG code at all.** Every CRS resolves `vertical_epsg_code = None`, so the + compound `EPSG:h+EPSG:v` source of `reproject_to_wgs84`, the Z flip for a depth-type vertical + CRS, and the geoid-grid warning are never exercised. The local files carry 5714/5715. +2. **No standalone `ProjectedCrs`** — the shape where a representation references a `ProjectedCrs` + directly instead of going through `LocalEngineeringCompoundCrs`. That is what + `tests/test_geojson_export.py` needs, and why its 8 tests skip. +3. **No unusable vertical code**, i.e. a *datum* code written where a CRS code belongs + (Volve declares `EPSG:6230`). `_build_transformer_with_vertical_fallback` exists only for that + case and is currently covered by unit tests, not by a file. +4. **No non-zero areal rotation and no northing-first axis order**, so the full local → projected + transform is only ever exercised on synthetic data (see `TestGrid2dGetsTheFullTransform`). + +### Scale + +5. **Small packages** (165/168 objects, ~300 kB). The `EpcFile` indexing and + `EpcStreamReader`-comparison tests parametrised on `SPASS_40+80wells.epc` (1678 objects, + 2.8 MB) skip — 12 tests. The assertions still run on the small packages; what is lost is the + behaviour at scale, which is the whole point of the lazy index. + +### Object types + +6. Absent from the testing packages, present in the local files: `ProjectedCrs` (standalone), + `ReferencePointInACrs`, `CommentProperty`, `DataobjectCollection`, + `CollectionsToDataobjectsAssociationSet`, `GeologicUnitOccurrenceInterpretation`. + +### External array backends + +7. **HDF5 only.** No Parquet, CSV, LAS or SEGY external array is referenced anywhere, so those + `FileHandlerRegistry` handlers are only covered by synthetic tests. + +### Representation types with no fixture anywhere + +Not a gap of the testing packages specifically — no EPC in `rc/epc/` contains them, so these +readers are written from the specification and unvalidated against a real file: +`DeviationSurveyRepresentation`, `StreamlinesRepresentation`, `Graph2dRepresentation`, +`UnstructuredColumnLayerGridRepresentation` (and its truncated variant), +`TruncatedIjkGridRepresentation`, `Seismic2d/3dPostStackRepresentation`, +`RedefinedGeometryRepresentation`, `GpGridRepresentation`. diff --git a/energyml-utils/rc/epc/testingPackageCpp.epc b/energyml-utils/rc/epc/testingPackageCpp.epc index 0987e95..75ae965 100644 Binary files a/energyml-utils/rc/epc/testingPackageCpp.epc and b/energyml-utils/rc/epc/testingPackageCpp.epc differ diff --git a/energyml-utils/rc/epc/testingPackageCpp22.epc b/energyml-utils/rc/epc/testingPackageCpp22.epc index 8e949c8..9f55352 100644 Binary files a/energyml-utils/rc/epc/testingPackageCpp22.epc and b/energyml-utils/rc/epc/testingPackageCpp22.epc differ diff --git a/energyml-utils/src/energyml/utils/cli/__init__.py b/energyml-utils/src/energyml/utils/cli/__init__.py new file mode 100644 index 0000000..18ad83f --- /dev/null +++ b/energyml-utils/src/energyml/utils/cli/__init__.py @@ -0,0 +1,66 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +""" +Command line entry points of energyml-utils. + +Every console script declared in ``[tool.poetry.scripts]`` points into this package. They used +to live in ``example/tools.py``, which is **not** part of the distribution — the wheel only ships +``energyml/`` — so all ten executables were created by ``pip install`` and every one of them +failed at import with ``ModuleNotFoundError: No module named 'example'``. It went unnoticed in +development because the repository root happens to be on ``sys.path`` there. + +============================ ========================================================== +Console script Function +============================ ========================================================== +``extract_3d`` :func:`~energyml.utils.cli.mesh.extract_representation_in_3d_file` +``csv_to_dataset`` :func:`~energyml.utils.cli.dataset.csv_to_dataset` +``generate_data`` :func:`~energyml.utils.cli.generate.generate_data` +``generate_multiple_data`` :func:`~energyml.utils.cli.generate.generate_multiple_data` +``xml_to_json`` :func:`~energyml.utils.cli.convert.xml_to_json` +``json_to_xml`` :func:`~energyml.utils.cli.convert.json_to_xml` +``json_to_epc`` :func:`~energyml.utils.cli.convert.json_to_epc` +``loadNsave`` :func:`~energyml.utils.cli.convert.load_n_save` +``describe_as_csv`` :func:`~energyml.utils.cli.describe.describe_as_csv` +``validate`` :func:`~energyml.utils.cli.validate.validate_files` +============================ ========================================================== + +Each entry point takes an optional ``argv`` list, so it can be driven from a test or from +another program without going through ``sys.argv``. +""" + +from energyml.utils.cli.convert import ( + json_to_epc, + json_to_xml, + load_n_save, + osdu_schema_to_energyml, + prop_kind_to_json, + xml_to_json, +) +from energyml.utils.cli.dataset import csv_to_dataset, csv_to_h5, csv_to_parquet, dat_to_h5 +from energyml.utils.cli.describe import describe_as_csv +from energyml.utils.cli.generate import generate_data, generate_multiple_data +from energyml.utils.cli.mesh import extract_representation_in_3d_file +from energyml.utils.cli.validate import validate_files + +__all__ = [ + # 3-D / GIS export + "extract_representation_in_3d_file", + # datasets + "csv_to_dataset", + "csv_to_h5", + "csv_to_parquet", + "dat_to_h5", + # generation + "generate_data", + "generate_multiple_data", + # conversions + "xml_to_json", + "json_to_xml", + "json_to_epc", + "load_n_save", + "prop_kind_to_json", + "osdu_schema_to_energyml", + # description / validation + "describe_as_csv", + "validate_files", +] diff --git a/energyml-utils/src/energyml/utils/cli/_common.py b/energyml-utils/src/energyml/utils/cli/_common.py new file mode 100644 index 0000000..cbf45a2 --- /dev/null +++ b/energyml-utils/src/energyml/utils/cli/_common.py @@ -0,0 +1,328 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""Helpers shared by the command line entry points. + +Nothing here is part of the public API of the library: these are the pieces the commands of +:mod:`energyml.utils.cli` have in common (logging setup, class lookup from a type name, packaging +of an input path into an :class:`~energyml.utils.epc.Epc`). +""" + +from __future__ import annotations + +import argparse +import logging +import os +import sys +from typing import Any, Callable, List, Optional + +from energyml.utils.epc import Epc +from energyml.utils.epc_utils import get_epc_content_type_path +from energyml.utils.introspection import ( + get_class_from_qualified_type, + get_class_from_simple_name, + get_non_abstract_classes, + get_module_name_and_type_from_content_or_qualified_type, + get_qualified_type_from_class, + is_abstract, + random_value_from_class, + search_class_in_module_from_partial_name, +) +from energyml.utils.serialization import ( + JSON_VERSION, + read_energyml_json_bytes, + read_energyml_xml_bytes, + serialize_json, + serialize_xml, +) + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- + + +def _free_flags(parser: argparse.ArgumentParser, *flags: str) -> List[str]: + """Keep only the *flags* the parser does not already use. + + The short forms are conveniences, not contracts: ``describe_as_csv`` already spends ``-v`` on + ``--columnsValues``, and stealing it would break every existing command line. + """ + taken = {opt for action in parser._actions for opt in action.option_strings} + return [flag for flag in flags if flag not in taken] + + +def add_verbosity_argument(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + """Add the ``--verbose`` / ``--quiet`` flags (and their short forms when free) to *parser*.""" + group = parser.add_mutually_exclusive_group() + group.add_argument( + *_free_flags(parser, "--verbose", "-v"), + action="count", + default=0, + help="Increase verbosity (-v for INFO, -vv for DEBUG)", + ) + group.add_argument( + *_free_flags(parser, "--quiet", "-q"), + action="store_true", + help="Only report errors", + ) + return parser + + +def configure_logging(args: argparse.Namespace) -> None: + """Configure the root logger from the verbosity flags of *args*. + + Only a command line entry point may do this: a library must never call + :func:`logging.basicConfig`, since it would silently take over the configuration of the + application that imports it. ``extract_3d`` used to force ``level=logging.DEBUG`` + unconditionally, which buried its own output under thousands of lines. + """ + verbose = getattr(args, "verbose", 0) + if getattr(args, "quiet", False): + level = logging.ERROR + elif verbose >= 2: + level = logging.DEBUG + elif verbose == 1: + level = logging.INFO + else: + level = logging.WARNING + logging.basicConfig(level=level, format="%(levelname)s %(name)s: %(message)s", stream=sys.stderr) + + +def parse_args(parser: argparse.ArgumentParser, argv: Optional[List[str]] = None) -> argparse.Namespace: + """Parse *argv* with *parser* and apply the verbosity flags.""" + add_verbosity_argument(parser) + args = parser.parse_args(argv) + configure_logging(args) + return args + + +# --------------------------------------------------------------------------- +# Class lookup +# --------------------------------------------------------------------------- + + +def find_class_from_type_name(type_name: str) -> Optional[type]: + """ + Search a class from a full python path (e.g. 'energyml.resqml.v2_2.resqmlv2.TriangulatedSetRepresentation') + or from a qualified type (e.g. 'resqml22.TriangulatedSetRepresentation'). + + :param type_name: + :return: the class or None if not found + """ + try: + return get_class_from_simple_name(type_name[type_name.rindex(".") + 1 :], [type_name[: type_name.rindex(".")]]) + except (NameError, ValueError): + try: + return get_class_from_qualified_type(type_name) + except ValueError: + return None + + +def print_close_type_names(type_name: str) -> None: + """Print the types that look like *type_name*, to help the user fix their input.""" + print(f"Class not found for '{type_name}', please check the type name.") + try: + module_name, object_type = get_module_name_and_type_from_content_or_qualified_type(type_name) + except ValueError: + return + print("Possible types are :") + for cn in search_class_in_module_from_partial_name(module_name, object_type): + print(f" - {cn.__name__}") + + +# --------------------------------------------------------------------------- +# Serialization +# --------------------------------------------------------------------------- + + +def serialize_object(obj: Any, file_format: str) -> str: + """Serialize *obj* as XML or as OSDU JSON.""" + if file_format.lower() == "xml": + return serialize_xml(obj) + return serialize_json(obj, JSON_VERSION.OSDU_OFFICIAL) + + +def file_name_prefix(obj: Any) -> str: + """ + File name prefix for a generated object : its qualified type (e.g. 'resqml22.TriangulatedSetRepresentation'), + or its class name if the qualified type cannot be computed. + + :param obj: + :return: str + """ + try: + return get_qualified_type_from_class(obj) or type(obj).__name__ + except Exception: # pragma: no cover — defensive: any class without a qualified type + return type(obj).__name__ + + +def read_energyml_json_any_version(content: bytes) -> List[Any]: + """Read an energyml JSON payload, trying the OSDU flavour first then the xsdata one.""" + try: + return read_energyml_json_bytes(content, JSON_VERSION.OSDU_OFFICIAL) + except Exception as osdu_error: + logger.debug("Not an OSDU JSON payload (%s), retrying with the xsdata flavour.", osdu_error) + return read_energyml_json_bytes(content, JSON_VERSION.XSDATA) + + +# --------------------------------------------------------------------------- +# Random object generation +# --------------------------------------------------------------------------- + + +def is_excluded(cls: type, exclude: Optional[List[str]]) -> bool: + """ + Test if the class *cls* must be excluded : it is the case if one of the *exclude* values is contained (case + insensitive) in one of the following values : + + - the module of the class (e.g. 'energyml.witsml.v2_1.witsmlv2'), + - the class name (e.g. 'Trajectory'), + - the full path 'module.ClassName', + - the qualified type of the class (e.g. 'witsml21.Trajectory'). + + E.g. '-e witsml' excludes every witsml class, '-e trajectory' excludes every class named '*Trajectory*'. + + :param cls: + :param exclude: + :return: bool + """ + if not exclude: + return False + + module_name = getattr(cls, "__module__", "") or "" + class_name = getattr(cls, "__name__", "") or "" + searched_in = [module_name, class_name, f"{module_name}.{class_name}"] + try: + searched_in.append(get_qualified_type_from_class(cls) or "") + except Exception: # pragma: no cover — defensive: any class without a qualified type + pass + searched_in = [value.lower() for value in searched_in if len(value) > 0] + + return any(excluded.lower() in value for excluded in exclude for value in searched_in) + + +def generate_random_objects( + obj_class: type, + callback: Optional[Callable[[Any], None]] = None, + exclude: Optional[List[str]] = None, +) -> List[Any]: + """ + Generate a random object for *obj_class*, or, if *obj_class* is abstract, one random object per non abstract + sub class of *obj_class*. + + :param obj_class: + :param callback: if not None, it is called with each object right after its generation (e.g. to write it on the + disk without waiting for the end of the whole generation). In that case, the objects are not + kept in memory and an empty list is returned. + :param exclude: list of values : every class matching one of them (see :func:`is_excluded`) is not generated + :return: a list of generated objects, or an empty list if a *callback* was given + """ + if is_abstract(obj_class): + classes_to_generate = get_non_abstract_classes(obj_class) + if len(classes_to_generate) == 0: + print(f"No instanciable sub class found for the abstract class '{obj_class.__name__}'.") + return [] + + nb_found = len(classes_to_generate) + classes_to_generate = [cls for cls in classes_to_generate if not is_excluded(cls, exclude)] + nb_excluded = nb_found - len(classes_to_generate) + + excluded_msg = f", {nb_excluded} excluded" if nb_excluded > 0 else "" + print( + f"'{obj_class.__name__}' is abstract : generating one object per sub class " + f"({nb_found} found{excluded_msg})." + ) + elif is_excluded(obj_class, exclude): + print(f"'{obj_class.__name__}' is excluded by the filter.") + return [] + else: + classes_to_generate = [obj_class] + + objs = [] + for cls in classes_to_generate: + # a failure on one class must not stop the generation of the others + try: + obj = random_value_from_class(cls) + except Exception as e: + logger.error("Failed to generate an object for '%s': %s: %s", cls.__name__, type(e).__name__, e) + continue + + if callback is not None: + callback(obj) + else: + objs.append(obj) + + return objs + + +# --------------------------------------------------------------------------- +# Input packaging +# --------------------------------------------------------------------------- + + +def _read_objects_from_file(file_path: str) -> List[Any]: + """Read every energyml object of a single json / xml / epc file. Never raises.""" + file_name = os.path.basename(file_path) + try: + if file_path.endswith(".json"): + with open(file_path, "rb") as file: + return list(read_energyml_json_any_version(file.read())) + if file_path.endswith(".xml"): + if get_epc_content_type_path() in file_path: + return [] + with open(file_path, "rb") as file: + return [read_energyml_xml_bytes(file.read())] + if file_path.endswith(".epc"): + epc = Epc.read_file(file_path) + if epc is None: + logger.error("File %s is NOT a valid EnergyML EPC file: empty EPC", file_name) + return [] + return list(epc.energyml_objects) + except Exception as e: + logger.error("File %s is NOT a valid EnergyML file: %s: %s", file_name, type(e).__name__, e) + return [] + + +def package_file_or_folder_in_epc(input_path: str) -> Optional[Epc]: + """ + Read every energyml object found in *input_path* — a json / xml / epc file, or a folder of + them — and return them packaged in a single in-memory :class:`~energyml.utils.epc.Epc`. + + Returns ``None`` when *input_path* does not exist or is not a supported kind of input. + """ + if not os.path.exists(input_path): + logger.error("File %s does not exist.", input_path) + return None + if not os.path.isdir(input_path) and not input_path.lower().endswith((".json", ".xml", ".epc")): + logger.error("File %s is not a valid input file (should be a folder or a json/xml/epc file).", input_path) + return None + + objects: List[Any] = [] + if os.path.isdir(input_path): + for filename in sorted(os.listdir(input_path)): + candidate = os.path.join(input_path, filename) + if os.path.isfile(candidate): + objects.extend(_read_objects_from_file(candidate)) + else: + objects.extend(_read_objects_from_file(input_path)) + + epc = Epc() + epc.energyml_objects = objects + return epc + + +__all__ = [ + "add_verbosity_argument", + "configure_logging", + "parse_args", + "find_class_from_type_name", + "print_close_type_names", + "serialize_object", + "file_name_prefix", + "read_energyml_json_any_version", + "is_excluded", + "generate_random_objects", + "package_file_or_folder_in_epc", +] diff --git a/energyml-utils/src/energyml/utils/cli/convert.py b/energyml-utils/src/energyml/utils/cli/convert.py new file mode 100644 index 0000000..450d717 --- /dev/null +++ b/energyml-utils/src/energyml/utils/cli/convert.py @@ -0,0 +1,180 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""``xml_to_json``, ``json_to_xml``, ``json_to_epc`` and ``loadNsave`` — format conversions.""" + +from __future__ import annotations + +import argparse +import json +import logging +import pathlib +from typing import Any, Dict, List, Optional + +from energyml.utils.cli._common import package_file_or_folder_in_epc, parse_args, read_energyml_json_any_version +from energyml.utils.constants import EpcExportVersion, get_property_kind_dict_path_as_xml +from energyml.utils.epc import Epc, gen_energyml_object_path +from energyml.utils.introspection import ( + get_object_attribute, + get_object_attribute_or_create, + set_attribute_from_path, +) +from energyml.utils.serialization import ( + JSON_VERSION, + read_energyml_xml_bytes, + read_energyml_xml_str, + serialize_json, + serialize_xml, +) + +logger = logging.getLogger(__name__) + +#: Example mapping from the OSDU schema to the energyml attribute paths, for +#: :func:`osdu_schema_to_energyml`. +SAMPLE_OSDU_MAP = { + "acl.owners": "osduintegration.OwnerGroup", + "acl.viewers": "osduintegration.ViewerGroup", + "legal.legaltags": "osduintegration.LegalTags", + "createTime": "Citation.Creation", + "modifyTime": "Citation.LastUpdate", + "modifyUser": "Citation.Editor", + "createUser": "Citation.Originator", + "data.Name": "Citation.Title", +} + + +def osdu_schema_to_energyml(input: str, target_obj: Any, attrib_map: Dict) -> Any: + """Copy the attributes of an OSDU JSON payload into *target_obj*, following *attrib_map*.""" + obj_in = json.loads(input) + for osdu_path, energyml_path in attrib_map.items(): + get_object_attribute_or_create(target_obj, energyml_path) + new_value = get_object_attribute(obj_in, osdu_path, force_snake_case=False) + set_attribute_from_path(target_obj, energyml_path, new_value) + return target_obj + + +def prop_kind_to_json(argv: Optional[List[str]] = None) -> None: + """Regenerate the packaged ``PropertyKindDictionary_v2.3.json`` from its XML counterpart.""" + from importlib.resources import files + + import energyml.utils.rc as RC + + parser = argparse.ArgumentParser( + prog="prop_kind_to_json", + description="Regenerate the packaged PropertyKindDictionary json from the packaged xml.", + ) + parse_args(parser, argv) + + with files(RC).joinpath("PropertyKindDictionary_v2.3.json").open("w", encoding="utf-8") as f: + f.write(serialize_json(read_energyml_xml_str(get_property_kind_dict_path_as_xml()))) + print("PropertyKindDictionary_v2.3.json regenerated") + + +def xml_to_json(argv: Optional[List[str]] = None) -> None: + """Entry point of the ``xml_to_json`` command: convert an energyml xml or epc file to json.""" + parser = argparse.ArgumentParser( + prog="xml_to_json", + description="Convert an energyml XML file (or every object of an EPC) into OSDU JSON.", + ) + parser.add_argument("--file", "-f", type=str, required=True, help="Input file (xml or epc)") + parser.add_argument("--out", "-o", type=str, default=None, help="Output file") + + args = parse_args(parser, argv) + + output_path = args.out or args.file[:-4] + ".json" + + json_content = None + if args.file.lower().endswith(".xml"): + with open(args.file, "rb") as f: + json_content = serialize_json(read_energyml_xml_bytes(f.read()), JSON_VERSION.OSDU_OFFICIAL) + elif args.file.lower().endswith(".epc"): + epc = Epc.read_file(args.file) + json_content = ( + "[\n" + ",".join(serialize_json(o, JSON_VERSION.OSDU_OFFICIAL) for o in epc.energyml_objects) + "]" + ) + else: + logger.error("Unsupported input file '%s': expected a .xml or a .epc file.", args.file) + return + + with open(output_path, "w", encoding="utf-8") as fout: + fout.write(json_content) + print(f"Written in {output_path}") + + +def json_to_xml(argv: Optional[List[str]] = None) -> None: + """Entry point of the ``json_to_xml`` command: one xml file per object of a json input.""" + parser = argparse.ArgumentParser( + prog="json_to_xml", + description="Convert an energyml JSON file into one XML file per object it contains.", + ) + parser.add_argument("--file", "-f", type=str, required=True, help="Input file (json)") + parser.add_argument("--out", "-o", type=str, default=None, help="Output folder (defaults to the input folder)") + + args = parse_args(parser, argv) + + with open(args.file, "rb") as f: + objs = read_energyml_json_any_version(f.read()) + + output_folder = pathlib.Path(args.out or args.file).parent.resolve() + for obj in objs: + file_path = output_folder / gen_energyml_object_path(obj) + with open(file_path, "w", encoding="utf-8") as fout: + fout.write(serialize_xml(obj)) + print(f"Written in {file_path}") + + +def json_to_epc(argv: Optional[List[str]] = None) -> None: + """Entry point of the ``json_to_epc`` command: package the objects of a json input in an EPC.""" + parser = argparse.ArgumentParser( + prog="json_to_epc", + description="Package every energyml object of a JSON file into a single EPC.", + ) + parser.add_argument("--file", "-f", type=str, required=True, help="Input file (json)") + parser.add_argument("--out", "-o", type=str, required=True, help="Output EPC file") + + args = parse_args(parser, argv) + + epc = Epc(epc_file_path=args.out) + with open(args.file, "rb") as f: + for obj in read_energyml_json_any_version(f.read()): + epc.energyml_objects.append(obj) + + epc.export_file(args.out) + print(f"Written in {args.out}") + + +def load_n_save(argv: Optional[List[str]] = None) -> None: + """Entry point of the ``loadNsave`` command: read a file or a folder and write it back as an EPC.""" + parser = argparse.ArgumentParser( + prog="loadNsave", + description="Read every energyml object of a file or a folder (json / xml / epc) and write them " + "back into a single EPC.", + ) + parser.add_argument("--file", "-f", type=str, required=True, help="Input file (json or xml or epc) or folder") + parser.add_argument("--output", "-o", type=str, help="Output file epc path") + parser.add_argument( + "--pkg-classical", action="store_true", help="Use classical packaging (one file per object) instead of EPC" + ) + + args = parse_args(parser, argv) + + epc = package_file_or_folder_in_epc(args.file) + if epc is None: + return + epc.export_version = EpcExportVersion.CLASSIC if args.pkg_classical else EpcExportVersion.EXPANDED + + output_path = args.output or ( + args.file[:-4] + "_bis.epc" if args.file.lower().endswith(".epc") else args.file + "_bis.epc" + ) + epc.export_file(output_path) + print(f"Written in {output_path}") + + +__all__ = [ + "SAMPLE_OSDU_MAP", + "osdu_schema_to_energyml", + "prop_kind_to_json", + "xml_to_json", + "json_to_xml", + "json_to_epc", + "load_n_save", +] diff --git a/energyml-utils/src/energyml/utils/cli/dataset.py b/energyml-utils/src/energyml/utils/cli/dataset.py new file mode 100644 index 0000000..f81ecce --- /dev/null +++ b/energyml-utils/src/energyml/utils/cli/dataset.py @@ -0,0 +1,218 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""``csv_to_dataset`` — turn the columns of a CSV / DAT file into HDF5 or Parquet datasets.""" + +from __future__ import annotations + +import argparse +import json +import logging +from typing import Dict, List, Optional + +from energyml.utils.cli._common import parse_args +from energyml.utils.data.datasets_io import ( + CSVFileReader, + DATFileReader, + HDF5FileWriter, + ParquetFileWriter, +) + +logger = logging.getLogger(__name__) + +#: Shape of the ``--mapping`` payload, shown in the help of the command. +_MAPPING_SAMPLE = { + "FINAL_DATASET_NAME_A": ["CSV_COL_NAME_0", "CSV_COL_NAME_N"], + "FINAL_DATASET_NAME_B": ["CSV_COL_NAME_X"], +} + + +def _write_mapped_columns( + writer, + target, + columns: Dict[str, List], + map_col_name_to_csv_col: Optional[Dict[str, List[str]]], + datasets_prefix: Optional[str], + ignore: List[str], + case_insensitive: bool, +) -> None: + """Write the datasets described by *map_col_name_to_csv_col*, and mark their columns as used. + + *ignore* is updated in place: a column consumed by the mapping must not be written a second + time under its own name by the caller. + """ + if not map_col_name_to_csv_col: + return + + def key(name: str) -> str: + return name.lower() if case_insensitive else name + + for dataset_name, col_list in map_col_name_to_csv_col.items(): + col_list = [key(c) for c in col_list] if case_insensitive else col_list + data: List = [] + if len(col_list) > 1: + for column in col_list: + if key(column) not in ignore: + try: + data = data + [columns[column]] + except KeyError: + logger.warning("Column '%s' is not in the input file — skipped.", column) + ignore.append(key(column)) + data = list(map(list, zip(*data))) + else: + column = col_list[0] if isinstance(col_list, list) else col_list + data = columns[column] + ignore.append(key(column)) + try: + writer.write_array(target, data, (datasets_prefix or "") + dataset_name) + except ValueError as e: + logger.warning("Dataset '%s' could not be written: %s", dataset_name, e) + + +def _write_remaining_columns(writer, target, columns, datasets_prefix, ignore, case_insensitive) -> None: + """Write one dataset per column that the mapping did not already consume.""" + for header in columns.keys(): + if (header.lower() if case_insensitive else header) in ignore: + continue + try: + writer.write_array(target, columns[header], (datasets_prefix or "") + header) + except ValueError as e: + logger.warning("Column '%s' could not be written: %s", header, e) + + +def dat_to_h5( + csv_in, + h5_out, + dataset_name: Optional[str] = None, + datasets_prefix: Optional[str] = None, + ignore: Optional[List[str]] = None, + map_col_name_to_csv_col: Optional[Dict[str, List[str]]] = None, + **csvparams, +): + """ + Write every column of the DAT file *csv_in* as a dataset of the HDF5 file *h5_out*. + + :param csv_in: + :param h5_out: + :param dataset_name: if None, csv headers are used + :param datasets_prefix: prefix prepended to every dataset path + :param ignore: column names not to write + :param map_col_name_to_csv_col: ``{dataset_name: [csv_column, ...]}`` + :param csvparams: forwarded to the reader (e.g. ``delimiter``) + """ + if dataset_name is not None: + return + writer = HDF5FileWriter() + columns = DATFileReader().read_array(csv_in, **csvparams) + _ignore = [c.lower() for c in (ignore or [])] + _write_mapped_columns(writer, h5_out, columns, map_col_name_to_csv_col, datasets_prefix, _ignore, True) + _write_remaining_columns(writer, h5_out, columns, datasets_prefix, _ignore, True) + + +def csv_to_h5( + csv_in, + h5_out, + dataset_name: Optional[str] = None, + datasets_prefix: Optional[str] = None, + ignore: Optional[List[str]] = None, + map_col_name_to_csv_col: Optional[Dict[str, List[str]]] = None, + **csvparams, +): + """ + Write every column of the CSV file *csv_in* as a dataset of the HDF5 file *h5_out*. + + See :func:`dat_to_h5` for the parameters. + """ + if dataset_name is not None: + return + writer = HDF5FileWriter() + columns = CSVFileReader().read_array_as_panda_dict(csv_in, **csvparams) + _ignore = list(ignore or []) + _write_mapped_columns(writer, h5_out, columns, map_col_name_to_csv_col, datasets_prefix, _ignore, False) + _write_remaining_columns(writer, h5_out, columns, datasets_prefix, _ignore, False) + + +def csv_to_parquet( + csv_in, + parquet_out, + dataset_name: Optional[str] = None, + datasets_prefix: Optional[str] = None, + ignore: Optional[List[str]] = None, + map_col_name_to_csv_col: Optional[Dict[str, List[str]]] = None, + **csvparams, +): + """ + Write every column of the CSV file *csv_in* as a column of the Parquet file *parquet_out*. + + See :func:`dat_to_h5` for the parameters. + """ + if dataset_name is not None: + return + columns = CSVFileReader().read_array_as_panda_dict(csv_in, **csvparams) + _ignore = list(ignore or []) + datadict: Dict[str, List] = {} + + # a dict is not a writer, but it exposes what _write_mapped_columns needs + class _DictWriter: + @staticmethod + def write_array(_target, array, path): + datadict[path] = array + + _write_mapped_columns(_DictWriter, None, columns, map_col_name_to_csv_col, datasets_prefix, _ignore, False) + _write_remaining_columns(_DictWriter, None, columns, datasets_prefix, _ignore, False) + + keys = list(datadict.keys()) + ParquetFileWriter().write_array(parquet_out, [datadict[k] for k in keys], keys) + + +def csv_to_dataset(argv: Optional[List[str]] = None) -> None: + """Entry point of the ``csv_to_dataset`` command.""" + parser = argparse.ArgumentParser( + prog="csv_to_dataset", + description="Convert the columns of a CSV file into HDF5 or Parquet datasets. " + "The output format is chosen from the extension of --output ('.parquet' / '.pqt' for " + "Parquet, HDF5 otherwise).", + ) + parser.add_argument("--csv", "-f", type=str, required=True, help="Csv file path") + parser.add_argument("--output", "-o", type=str, required=True, help="Output file path") + parser.add_argument("--prefix", "-p", type=str, default="", help="Dataset path prefix") + parser.add_argument("--csv-delimiter", "-d", type=str, default=",", help="CSV delimiter") + parser.add_argument( + "--mapping", + "-m", + type=str, + help=f"Json file path. The json content should look like this : {json.dumps(_MAPPING_SAMPLE)}", + ) + parser.add_argument( + "--mapping-line", + "-ml", + type=str, + help=f"A json dict that should look like this : {json.dumps(_MAPPING_SAMPLE)}", + ) + parser.add_argument("--ignore", "-i", type=str, help="A csv column name to ignore", nargs="+") + + args = parse_args(parser, argv) + + mapping = args.mapping_line or args.mapping + if mapping is not None: + mapping = json.loads(mapping) + logger.debug("delimiter=%r mapping=%s", args.csv_delimiter, mapping) + + output_file_path = args.output + convert = csv_to_parquet if output_file_path.lower().endswith((".parquet", ".pqt")) else csv_to_h5 + convert( + args.csv, + output_file_path, + datasets_prefix=args.prefix, + ignore=args.ignore, + map_col_name_to_csv_col=mapping, + delimiter=args.csv_delimiter, + ) + print(f"Datasets written in {output_file_path}") + + +__all__ = [ + "dat_to_h5", + "csv_to_h5", + "csv_to_parquet", + "csv_to_dataset", +] diff --git a/energyml-utils/src/energyml/utils/cli/describe.py b/energyml-utils/src/energyml/utils/cli/describe.py new file mode 100644 index 0000000..e0e7818 --- /dev/null +++ b/energyml-utils/src/energyml/utils/cli/describe.py @@ -0,0 +1,121 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""``describe_as_csv`` — summarize the energyml objects of a folder in a CSV table.""" + +from __future__ import annotations + +import argparse +import logging +import os +from typing import Any, List, Optional, Tuple + +from energyml.utils.cli._common import parse_args, read_energyml_json_any_version +from energyml.utils.epc import Epc +from energyml.utils.introspection import ( + get_content_type_from_class, + get_direct_dor_list, + get_obj_uuid, + get_object_attribute_rgx, + get_qualified_type_from_class, +) +from energyml.utils.serialization import read_energyml_xml_bytes + +logger = logging.getLogger(__name__) + +_DEFAULT_COLUMN_NAMES = ["Title", "QualifiedType", "Uuid", "SchemaVersion", "Path", "Dors uuids"] +_DEFAULT_COLUMN_VALUES = ["citation.title", "$qualifiedtype", "Uuid|Uid", "schemaVersion", "$Path", "$Dor"] + + +def _read_folder(folder: str) -> List[Tuple[Any, str]]: + """Return ``[(energyml_object, source_file_path), ...]`` for every readable file of *folder*.""" + objects: List[Tuple[Any, str]] = [] + for filename in sorted(os.listdir(folder)): + path = os.path.join(folder, filename) + if not os.path.isfile(path): + continue + try: + if path.endswith(".json"): + with open(path, "rb") as file: + objects.extend((o, path) for o in read_energyml_json_any_version(file.read())) + elif path.endswith(".xml"): + with open(path, "rb") as file: + objects.append((read_energyml_xml_bytes(file.read()), path)) + elif path.endswith(".epc"): + objects.extend((o, path) for o in Epc.read_file(path).energyml_objects) + except Exception as e: + # one unreadable file must not stop the description of the others + logger.error("File %s could not be read: %s: %s", filename, type(e).__name__, e) + return objects + + +def _cell_value(obj: Any, column: str, source_path: str) -> str: + """Value of one cell : a ``$`` directive, or a regex matching an attribute of *obj*.""" + if not column.startswith("$"): + return str(get_object_attribute_rgx(obj, column) or "") + + directive = column.lower() + if directive == "$type": + return type(obj).__name__ + if directive == "$qualifiedtype": + return str(get_qualified_type_from_class(obj) or "") + if directive == "$contenttype": + return str(get_content_type_from_class(obj) or "") + if directive == "$path": + return source_path + if directive == "$dor": + return str(sorted({get_obj_uuid(dor) for dor in get_direct_dor_list(obj)})).replace(";", ", ") + logger.warning("Unknown column directive '%s' — an empty cell is written.", column) + return "" + + +def describe_as_csv(argv: Optional[List[str]] = None) -> None: + """Entry point of the ``describe_as_csv`` command.""" + parser = argparse.ArgumentParser( + prog="describe_as_csv", + description="Write a ';'-separated CSV describing every energyml object found in a folder.", + ) + parser.add_argument("--folder", "-f", type=str, required=True, help="Input folder") + parser.add_argument( + "--columnsNames", + "-c", + type=str, + default=_DEFAULT_COLUMN_NAMES, + nargs="*", + help="Columns titles", + ) + parser.add_argument( + "--columnsValues", + "-v", + type=str, + default=_DEFAULT_COLUMN_VALUES, + nargs="*", + help="Columns values. Use $type/$qualifiedType/$contentType/$path/$dor or simpler, " + "a regex matching an attribute", + ) + + args = parse_args(parser, argv) + + logger.info("Reading files of %s", args.folder) + objects = _read_folder(args.folder) + + out_name = "describe.csv" + cpt = 0 + while os.path.exists(os.path.join(args.folder, out_name)): + out_name = f"describe_{cpt}.csv" + cpt += 1 + + logger.info("Parsing %d object(s)", len(objects)) + out_path = os.path.join(args.folder, out_name) + with open(out_path, "w", encoding="utf-8") as out: + out.write(";".join(args.columnsNames)) + out.write(";\n") + for obj, source_path in objects: + for column in args.columnsValues: + out.write(_cell_value(obj, column, source_path)) + out.write(";") + out.write("\n") + + print(f"Written in {out_path}") + + +__all__ = ["describe_as_csv"] diff --git a/energyml-utils/src/energyml/utils/cli/generate.py b/energyml-utils/src/energyml/utils/cli/generate.py new file mode 100644 index 0000000..6db2051 --- /dev/null +++ b/energyml-utils/src/energyml/utils/cli/generate.py @@ -0,0 +1,148 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""``generate_data`` / ``generate_multiple_data`` — generate random energyml objects.""" + +from __future__ import annotations + +import argparse +import logging +import os +import pathlib +from typing import Any, List, Optional + +from energyml.utils.cli._common import ( + file_name_prefix, + find_class_from_type_name, + generate_random_objects, + parse_args, + print_close_type_names, + serialize_object, +) +from energyml.utils.introspection import get_obj_uuid + +logger = logging.getLogger(__name__) + +_DEFAULT_TYPE = "energyml.resqml.v2_2.resqmlv2.TriangulatedSetRepresentation" +_FILE_FORMATS = ["json", "xml"] + + +def generate_data(argv: Optional[List[str]] = None) -> None: + """Entry point of the ``generate_data`` command: print one random object of a given type.""" + parser = argparse.ArgumentParser( + prog="generate_data", + description="Generate a random energyml object of the given type and print it on the standard output. " + "When the type is abstract, one object per non abstract sub class is generated.", + ) + parser.add_argument( + "--type", + "-t", + type=str, + default=_DEFAULT_TYPE, + help=f"Object type (e.g. {_DEFAULT_TYPE})", + ) + parser.add_argument( + "--file-format", + "-ff", + type=str, + choices=_FILE_FORMATS, + default="json", + help=f"Type of the output files (one of : {_FILE_FORMATS}). Default is 'json'", + ) + + args = parse_args(parser, argv) + + obj_class = find_class_from_type_name(args.type) + if obj_class is None: + print_close_type_names(args.type) + return + + for obj in generate_random_objects(obj_class): + # a class that cannot be serialized must not stop the ones that follow it — the same + # policy `generate_multiple_data` already applies + try: + print(serialize_object(obj, args.file_format)) + except Exception as e: + logger.error("Failed to serialize an object of type '%s': %s: %s", type(obj).__name__, type(e).__name__, e) + + +def generate_multiple_data(argv: Optional[List[str]] = None) -> None: + """ + Entry point of the ``generate_multiple_data`` command. + + Same as :func:`generate_data` but for several object types at once, sharing a common file + format. If an output folder is given, one file per object is written in it (as soon as it is + generated), else all objects are printed on stdout. + """ + parser = argparse.ArgumentParser( + prog="generate_multiple_data", + description="Generate random energyml objects for several types at once.", + ) + parser.add_argument( + "--type", + "-t", + type=str, + nargs="+", + default=[_DEFAULT_TYPE], + help=f"Object types (e.g. {_DEFAULT_TYPE} energyml.resqml.v2_2.resqmlv2.PolylineSetRepresentation)", + ) + parser.add_argument( + "--file-format", + "-ff", + type=str, + choices=_FILE_FORMATS, + default="json", + help=f"Type of the output files (one of : {_FILE_FORMATS}). Default is 'json'", + ) + parser.add_argument( + "--output", + "-o", + type=str, + default=None, + help="Output folder path. If not set, the objects are printed on the standard output", + ) + parser.add_argument( + "--exclude", + "-e", + type=str, + nargs="+", + action="extend", # to support both '-e witsml prodml' and '-e witsml -e prodml' + default=[], + help="Do not generate the classes whose module, class name, 'module.ClassName' or qualified type contains " + "one of these values (case insensitive). E.g. '-e witsml prodml' skips every witsml and prodml class", + ) + + args = parse_args(parser, argv) + + file_format = args.file_format.lower() + if args.output is not None: + pathlib.Path(args.output).mkdir(parents=True, exist_ok=True) + + def export_object(obj: Any) -> None: + """Export an object as soon as it has been generated : one file per object, or the standard output.""" + try: + content = serialize_object(obj, file_format) + except Exception as e: + logger.error("Failed to serialize an object of type '%s': %s: %s", type(obj).__name__, type(e).__name__, e) + return + + if args.output is None: + print(f"# ----- {type(obj).__name__} -----") + print(content) + else: + file_path = os.path.join(args.output, f"{file_name_prefix(obj)}_{get_obj_uuid(obj)}.{file_format}") + with open(file_path, "w", encoding="utf-8") as f: + f.write(content) + print(f"Object written in {file_path}") + + for type_name in args.type: + obj_class = find_class_from_type_name(type_name) + if obj_class is None: + print_close_type_names(type_name) + continue + generate_random_objects(obj_class, callback=export_object, exclude=args.exclude) + + +__all__ = [ + "generate_data", + "generate_multiple_data", +] diff --git a/energyml-utils/src/energyml/utils/cli/mesh.py b/energyml-utils/src/energyml/utils/cli/mesh.py new file mode 100644 index 0000000..bb34d47 --- /dev/null +++ b/energyml-utils/src/energyml/utils/cli/mesh.py @@ -0,0 +1,65 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""``extract_3d`` — export the representations of an EPC as 3-D / GIS files.""" + +from __future__ import annotations + +import argparse +import logging +from typing import List, Optional + +from energyml.utils.cli._common import parse_args +from energyml.utils.data.mesh import MeshFileFormat, export_multiple_data + +logger = logging.getLogger(__name__) + + +def extract_representation_in_3d_file(argv: Optional[List[str]] = None) -> None: + """Entry point of the ``extract_3d`` command.""" + formats = [e.value for e in MeshFileFormat] + parser = argparse.ArgumentParser( + prog="extract_3d", + description="Export the representations of an EPC into 3-D or GIS files " + f"(one of : {formats}), one file per representation.", + ) + parser.add_argument("--epc", "-f", type=str, required=True, help="Epc file path") + parser.add_argument("--output", "-o", type=str, required=True, help="Output folder path") + # store_true, not store_false: with store_false the flag defaulted to True, so + # `use_crs_displacement=not args.no_crs` was False unless --no-crs was passed — the switch + # was inverted and the export was written in local coordinates by default. + parser.add_argument("--no-crs", action="store_true", help="Disable crs displacement") + parser.add_argument( + "--file-format", + "-ff", + type=MeshFileFormat, + choices=list(MeshFileFormat), + default=MeshFileFormat.OBJ, + help=f"Type of the output files (one of : {formats}). Default is 'obj'", + ) + parser.add_argument("--uuid", "-u", type=str, help="The uuids of representations to extract", nargs="+") + parser.add_argument( + "--no-wgs84", + action="store_true", + help="GeoJSON only : keep the coordinates in their source CRS instead of reprojecting them to WGS84", + ) + parser.add_argument( + "--proj-network", + action="store_true", + help="GeoJSON only : allow PROJ to download the geoid grids needed by the vertical datum transformation", + ) + + args = parse_args(parser, argv) + + export_multiple_data( + epc_path=args.epc, + uuid_list=args.uuid, + output_folder_path=args.output, + file_format=args.file_format, + use_crs_displacement=not args.no_crs, + to_wgs84=not args.no_wgs84, + use_network=args.proj_network, + ) + print(f"Representations of {args.epc} exported in {args.output}") + + +__all__ = ["extract_representation_in_3d_file"] diff --git a/energyml-utils/src/energyml/utils/cli/validate.py b/energyml-utils/src/energyml/utils/cli/validate.py new file mode 100644 index 0000000..a4d23da --- /dev/null +++ b/energyml-utils/src/energyml/utils/cli/validate.py @@ -0,0 +1,69 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""``validate`` — report the schema and consistency errors of an energyml file or folder.""" + +from __future__ import annotations + +import argparse +import json +import logging +from typing import Dict, List, Optional + +from energyml.utils.cli._common import package_file_or_folder_in_epc, parse_args +from energyml.utils.introspection import get_enum_values +from energyml.utils.validation import ErrorType, validate_epc + +logger = logging.getLogger(__name__) + + +def validate_files(argv: Optional[List[str]] = None) -> None: + """Entry point of the ``validate`` command: print the validation errors as JSON.""" + parser = argparse.ArgumentParser( + prog="validate", + description="Validate every energyml object of a file or a folder and print the errors as JSON " + "on the standard output.", + ) + parser.add_argument("--file", "-f", type=str, required=True, help="Input file (json or xml or epc) or folder") + parser.add_argument( + "--ignore-err-type", + "-i", + type=str, + help=f"Error types to ignore. Possible values {get_enum_values(ErrorType)}", + nargs="*", + ) + parser.add_argument( + "--ignore-prodml-version-errs", + action="store_false", + dest="ignore_prodml_version_errs", + help="Disable ignoring errors related to Prodml version (by default, these errors are ignored)", + ) + parser.add_argument( + "--group-by-err-class", + action="store_true", + help="Group errors by their class (e.g. all validation errors together, all parsing errors together, etc.)", + ) + + args = parse_args(parser, argv) + + epc = package_file_or_folder_in_epc(args.file) + if epc is None: + return + + ignored = {et.lower() for et in (args.ignore_err_type or [])} + err_json = [err.toJson() for err in validate_epc(epc) if str(err.error_type).lower() not in ignored] + + err_json_sorted = sorted(err_json, key=lambda x: (x["err_class"], x["error_type"], x.get("object_uuid", ""))) + + if args.ignore_prodml_version_errs: + err_json_sorted = [err for err in err_json_sorted if "prodml23" not in err.get("msg", "")] + + if args.group_by_err_class: + grouped: Dict[str, List[Dict]] = {} + for err in err_json_sorted: + grouped.setdefault(err.get("err_class", "UnknownErrorClass"), []).append(err) + print(json.dumps(grouped, indent=4)) + else: + print(json.dumps(err_json_sorted, indent=4)) + + +__all__ = ["validate_files"] diff --git a/energyml-utils/src/energyml/utils/constants.py b/energyml-utils/src/energyml/utils/constants.py index 10ec83b..7e40a8c 100644 --- a/energyml-utils/src/energyml/utils/constants.py +++ b/energyml-utils/src/energyml/utils/constants.py @@ -453,6 +453,57 @@ def pascal_case(string: str) -> str: return snake_case(string).replace("_", " ").title().replace(" ", "") +#: Characters no Windows file name may contain. ``/`` is added for POSIX, where it is the path +#: separator; the ASCII control characters are rejected by every filesystem. +_FORBIDDEN_FILE_NAME_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]') + +#: Windows device names: reserved whatever the extension, so ``CON.geojson`` is unusable too. +_RESERVED_FILE_NAMES = frozenset( + ["CON", "PRN", "AUX", "NUL"] + [f"COM{i}" for i in range(1, 10)] + [f"LPT{i}" for i in range(1, 10)] +) + + +def sanitize_file_name(name: str, replacement: str = "_", max_length: int = 150) -> str: + """Make *name* usable as a single file-name component on every platform. + + Energyml titles go into export file names, and they are free text: ``AUB-PRO-SP05512: + Trajectory`` is a perfectly legal citation title. On Windows a ``:`` in a path opens an + *alternate data stream* instead of failing — ``open("well: Traj.geojson", "w")`` silently + creates an empty file called ``well`` carrying a hidden stream named ``: Traj.geojson``. + The export looks like it worked and leaves extension-less, apparently empty files behind. + + Replaces the forbidden characters, collapses the runs they leave, strips the trailing dots + and spaces Windows drops silently, escapes the reserved device names, and truncates to + *max_length* so that a long title cannot push the whole path past the limit. + + :param name: the raw file-name component (no directory separator is preserved). + :param replacement: what to substitute for a forbidden character. + :param max_length: maximum length of the returned component. + """ + if not name: + return "unnamed" + + cleaned = _FORBIDDEN_FILE_NAME_CHARS.sub(replacement, name) + if replacement: + # "a: b" would otherwise become "a__b" — one for the colon, one for the space after it. + cleaned = re.sub(re.escape(replacement) + r"{2,}", replacement, cleaned) + # Windows drops trailing dots and spaces without telling, so "x." and "x" collide. + cleaned = cleaned.rstrip(". ").strip() + + # "///" or "..." carried no name to begin with; a bare separator is not a better answer. + if not cleaned or (replacement and cleaned.strip(replacement) == ""): + return "unnamed" + + stem, dot, extension = cleaned.partition(".") + if stem.upper() in _RESERVED_FILE_NAMES: + cleaned = f"{stem}{replacement}{dot}{extension}" if dot else f"{stem}{replacement}" + + if len(cleaned) > max_length: + cleaned = cleaned[:max_length].rstrip(". ") + + return cleaned or "unnamed" + + def flatten_concatenation(matrix) -> List: """ Flatten a matrix efficiently. @@ -748,6 +799,92 @@ def get_property_kind_dict_path_as_xml() -> str: # MAIN EXECUTION (for testing) # =================================== + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "ENERGYML_NAMESPACES", + "WELLKNOWN_NAMESPACES", + "ENERGYML_NAMESPACES_PACKAGE", + "ENERGYML_MODULES_NAMES", + "RELATED_MODULES_MAP", + "RGX_ENERGYML_MODULE_NAME", + "RGX_PROJECT_VERSION", + "RGX_UUID_NO_GRP", + "RGX_UUID", + "RGX_DOMAIN_VERSION", + "RGX_DOMAIN_VERSION_FLAT", + "RGX_MIME_TYPE_MEDIA", + "RGX_CT_ENERGYML_DOMAIN", + "RGX_CT_XML_DOMAIN", + "RGX_CT_TOKEN_VERSION", + "RGX_CT_TOKEN_TYPE", + "RGX_CONTENT_TYPE", + "RGX_QUALIFIED_TYPE", + "RGX_SCHEMA_VERSION", + "RGX_ENERGYML_FILE_NAME_OLD", + "RGX_ENERGYML_FILE_NAME_NEW", + "RGX_ENERGYML_FILE_NAME", + "RGX_XML_HEADER", + "RGX_IDENTIFIER", + "URI_RGX_GRP_DOMAIN", + "URI_RGX_GRP_DOMAIN_VERSION", + "URI_RGX_GRP_UUID", + "URI_RGX_GRP_DATASPACE", + "URI_RGX_GRP_VERSION", + "URI_RGX_GRP_OBJECT_TYPE", + "URI_RGX_GRP_UUID2", + "URI_RGX_GRP_COLLECTION_DOMAIN", + "URI_RGX_GRP_COLLECTION_DOMAIN_VERSION", + "URI_RGX_GRP_COLLECTION_TYPE", + "URI_RGX_GRP_QUERY", + "URI_RGX", + "DOT_PATH_ATTRIBUTE", + "DOT_PATH", + "OptimizedRegex", + "RELS_CONTENT_TYPE", + "RELS_FOLDER_NAME", + "CORE_PROPERTIES_FOLDER_NAME", + "primitives", + "MimeType", + "EpcExportVersion", + "EPCRelsRelationshipType", + "RawFile", + "MIME_TYPE_TO_EXTENSION", + "MIME_TYPE_ALIASES", + "EXTENSION_ALIASES", + "mime_type_to_file_extension", + "file_extension_to_mime_type", + "snake_case", + "snake_case_old", + "pascal_case", + "sanitize_file_name", + "flatten_concatenation", + "parse_content_type", + "parse_qualified_type", + "parse_content_or_qualified_type", + "content_type_to_qualified_type", + "qualified_type_to_content_type", + "get_domain_version_from_content_or_qualified_type", + "get_obj_type_from_content_or_qualified_type", + "split_identifier", + "now", + "epoch", + "date_to_epoch", + "date_to_datetime", + "epoch_to_date", + "gen_uuid", + "extract_uuid_from_string", + "path_next_attribute", + "path_last_attribute", + "path_iter", + "path_parent_attribute", + "get_property_kind_dict_path_as_json", + "get_property_kind_dict_path_as_dict", + "get_property_kind_dict_path_as_xml", +] + + if __name__ == "__main__": # Test optimized regex patterns test_cases = [ diff --git a/energyml-utils/src/energyml/utils/data/crs.py b/energyml-utils/src/energyml/utils/data/crs.py index df74355..2ebd4ec 100644 --- a/energyml-utils/src/energyml/utils/data/crs.py +++ b/energyml-utils/src/energyml/utils/data/crs.py @@ -24,6 +24,8 @@ import logging import math from dataclasses import dataclass, field +from enum import Enum +from functools import lru_cache from typing import Any, Optional import numpy as np @@ -106,11 +108,10 @@ class CrsInfo: vertical_unknown: Optional[str] = None """Free-text vertical CRS descriptor.""" - + time_uom: Optional[str] = None """Unit of measure for time coordinates (e.g. ``"s"``, ``"min"``, ``"h"``).""" - # ------------------------------------------------------------------ # Rotation / azimuth # ------------------------------------------------------------------ @@ -325,7 +326,7 @@ def _extract_vertical_crs_details(vertical_crs_obj: Any) -> dict: **must not** override a parent-level ``ZIncreasingDownward`` when this value is ``None``. """ - # logging.debug( + # logger.debug( # f"Extracting vertical CRS details from object of type {type(vertical_crs_obj).__name__} with URI {get_obj_uri(vertical_crs_obj)}" # ) result: dict = { @@ -413,7 +414,7 @@ def _from_abstract_local3dcrs( DORs when provided. """ type_name = type(crs_obj).__name__ - # logging.debug(f"@_from_abstract_local3dcrs Extracting CRS info from {type_name} with URI {get_obj_uri(crs_obj)}") + # logger.debug(f"@_from_abstract_local3dcrs Extracting CRS info from {type_name} with URI {get_obj_uri(crs_obj)}") # --- Offsets ----------------------------------------------------------- x_offset = 0.0 @@ -435,7 +436,7 @@ def _from_abstract_local3dcrs( # --- Z direction ------------------------------------------------------- z_increasing_downward: bool = False zid_raw = get_object_attribute_no_verif(crs_obj, "zincreasing_downward") - logging.debug(f"v2.0.1 ZIncreasingDownward raw value: {zid_raw}") + logger.debug(f"v2.0.1 ZIncreasingDownward raw value: {zid_raw}") if zid_raw is not None: if isinstance(zid_raw, bool): z_increasing_downward = zid_raw @@ -477,8 +478,8 @@ def _from_abstract_local3dcrs( # Direction from VerticalCrs overrides the top-level ZIncreasingDownward # only when explicitly set. - # logging.debug("z_increasing_downward before vertical CRS details: %s", z_increasing_downward) - # logging.debug( + # logger.debug("z_increasing_downward before vertical CRS details: %s", z_increasing_downward) + # logger.debug( # f"Vertical CRS details: {vertical_details} -- vertical_crs_obj type: {type(vertical_crs_obj).__name__ if vertical_crs_obj else 'None'}" # ) if vertical_crs_obj is not None and vertical_details.get("z_increasing_downward") is not None: @@ -486,7 +487,7 @@ def _from_abstract_local3dcrs( if vertical_details.get("uom"): vertical_uom = vertical_details["uom"] - # logging.debug("z_increasing_downward after vertical CRS details: %s", z_increasing_downward) + # logger.debug("z_increasing_downward after vertical CRS details: %s", z_increasing_downward) return CrsInfo( x_offset=x_offset, @@ -575,6 +576,25 @@ def _from_local_engineering2d_crs( ) +def _from_projected_crs(crs_obj: Any) -> CrsInfo: + """ + Handle a standalone ``ProjectedCrs`` document object — **EML v2.3 / RESQML v2.2**. + + Such an object carries no local offset / rotation (those live in the + ``LocalEngineering2dCrs``); only the EPSG code, the UOM and the axis order are available. + """ + type_name = type(crs_obj).__name__ + details = _extract_projected_crs_details(crs_obj) + return CrsInfo( + projected_epsg_code=details.get("epsg_code"), + projected_uom=details.get("uom"), + projected_axis_order=details.get("axis_order"), + projected_wkt=details.get("wkt"), + projected_unknown=details.get("unknown"), + source_type=type_name, + ) + + def _from_vertical_crs(crs_obj: Any) -> CrsInfo: """ Handle a standalone ``VerticalCrs`` document object — **EML v2.3 / RESQML v2.2**. @@ -627,7 +647,7 @@ def _from_local_engineering_compound_crs( vert_axis_uom_raw = get_object_attribute(crs_obj, "vertical_axis.uom") if vert_axis_uom_raw is not None: vert_axis_uom = _uom_to_str(vert_axis_uom_raw) - + is_time = get_object_attribute(crs_obj, "vertical_axis.is_time") time_uom = None if is_time is not None and str(is_time).lower() in ("true", "1", "yes"): @@ -856,6 +876,290 @@ def apply_from_crs_info( return pts +# --------------------------------------------------------------------------- +# WGS84 reprojection (requires the 'crs' extra : pyproj) +# --------------------------------------------------------------------------- + +#: EPSG code of WGS84 as a 2D geographic CRS (longitude, latitude). +WGS84_2D_EPSG_CODE = 4326 + +#: EPSG code of WGS84 as a 3D geographic CRS (longitude, latitude, ellipsoidal height). +#: This is the target to use whenever a vertical CRS is known, since EPSG:4326 carries no height. +WGS84_3D_EPSG_CODE = 4979 + +#: Number of points reprojected per block by :func:`reproject_to_wgs84`. It bounds the scratch +#: memory of the reprojection at ``3 x _REPROJECT_CHUNK x 8`` bytes (6 MiB here) whatever the size +#: of the input, while staying large enough for the per-call PROJ overhead to be negligible. +_REPROJECT_CHUNK = 1 << 18 + + +def crs_ogc_uri(epsg_code: int) -> str: + """ + Return the OGC URI of an EPSG code, e.g. ``http://www.opengis.net/def/crs/EPSG/0/32631``. + + This is the identifier form used by OGC API - Features and by JSON-FG ``coordRefSys`` + members, and is the standard way to advertise a CRS in a GeoJSON-like document. + """ + return f"http://www.opengis.net/def/crs/EPSG/0/{int(epsg_code)}" + + +def crs_urn(epsg_code: int) -> str: + """ + Return the OGC URN of an EPSG code, e.g. ``urn:ogc:def:crs:EPSG::32631``. + + This is the form used by the (deprecated but still widely supported by GDAL / QGIS) + GeoJSON 2008 ``crs`` member. + """ + return f"urn:ogc:def:crs:EPSG::{int(epsg_code)}" + + +def is_pyproj_available() -> bool: + """Return ``True`` when :mod:`pyproj` (the ``crs`` extra) can be imported.""" + try: + import pyproj # noqa: F401 + + return True + except ImportError: + return False + + +def build_source_crs_id( + projected_epsg_code: Optional[int], + vertical_epsg_code: Optional[int] = None, +) -> Optional[str]: + """ + Build the CRS identifier to hand over to pyproj. + + Returns ``"EPSG:+EPSG:"`` (compound CRS) when a vertical code is given, + ``"EPSG:"`` when only the horizontal one is known, and ``None`` when no + horizontal code is available (nothing can be reprojected in that case). + """ + if projected_epsg_code is None: + return None + if vertical_epsg_code is not None: + return f"EPSG:{int(projected_epsg_code)}+EPSG:{int(vertical_epsg_code)}" + return f"EPSG:{int(projected_epsg_code)}" + + +@lru_cache(maxsize=32) +def _get_transformer(source_crs_id: str, target_epsg_code: int, network_enabled: bool = False): + """ + Build (and cache) a pyproj ``Transformer``. Building one costs a few ms, reuse is free. + + ``network_enabled`` is part of the cache key on purpose: PROJ selects the transformation + pipeline when the transformer is built, so a transformer created while the network was + disabled would keep ignoring the geoid grids even after the network is turned on. + """ + from pyproj import Transformer + + # always_xy=True : force the (longitude, latitude) order on output, whatever the + # axis order declared by the EPSG registry (EPSG:4326 is officially lat/lon). + return Transformer.from_crs(source_crs_id, f"EPSG:{int(target_epsg_code)}", always_xy=True) + + +def _build_transformer_with_vertical_fallback( + source_crs_id: str, + projected_epsg_code: Optional[int], + vertical_epsg_code: Optional[int], + network_enabled: bool, +): + """ + Build the transformer for *source_crs_id*, dropping the vertical CRS if it is unusable. + + Returns ``(transformer, vertical_dropped)``. + + A file can declare a vertical EPSG code that PROJ cannot resolve — most often a *datum* + code where a CRS code was expected (the Volve export declares ``EPSG:6230``, the ED50 + datum, as its vertical CRS). The compound ``EPSG:h+EPSG:v`` then fails to build, and + refusing the whole transformation would leave the coordinates in their projected CRS + although the horizontal part is perfectly reprojectable. So the horizontal CRS is retried + alone and the Z column is passed through untouched, in its source vertical frame. + """ + try: + return _get_transformer(source_crs_id, WGS84_3D_EPSG_CODE, network_enabled), False + except Exception as exc: + horizontal_only = build_source_crs_id(projected_epsg_code, None) + if vertical_epsg_code is None or horizontal_only is None or horizontal_only == source_crs_id: + raise + logger.warning( + "reproject_to_wgs84: the compound CRS %s could not be built (%s) — most likely an " + "invalid vertical EPSG code in the source object. Falling back to %s alone: " + "longitude and latitude are correct, the Z column is left in its source vertical frame.", + source_crs_id, + exc, + horizontal_only, + ) + return _get_transformer(horizontal_only, WGS84_3D_EPSG_CODE, network_enabled), True + + +@lru_cache(maxsize=32) +def _vertical_axis_is_down(vertical_epsg_code: int) -> bool: + """ + Return ``True`` when the vertical CRS counts positive values *downward* (a depth CRS, + e.g. EPSG:5715 "MSL depth"), ``False`` for a height CRS (e.g. EPSG:5714 "MSL height"). + """ + try: + from pyproj import CRS + + for axis in CRS.from_epsg(int(vertical_epsg_code)).axis_info: + if (axis.direction or "").lower() == "down": + return True + except Exception as exc: + logger.debug("Cannot determine the axis direction of EPSG:%s : %s", vertical_epsg_code, exc) + return False + + +def reproject_to_wgs84( + points: np.ndarray, + crs_info: Optional["CrsInfo"] = None, + *, + projected_epsg_code: Optional[int] = None, + vertical_epsg_code: Optional[int] = None, + z_is_up: bool = True, + use_network: bool = False, + inplace: bool = False, +) -> np.ndarray: + """ + Reproject *points* from their projected CRS to WGS84 (longitude, latitude, ellipsoidal height). + + The input points are expected to be **already expressed in the projected CRS**, i.e. + :func:`apply_from_crs_info` must have been applied first (offsets, rotation, …). + + Parameters + ---------- + points: + ``(N, 3)`` array in the source projected CRS. + crs_info: + Source :class:`CrsInfo`; its EPSG codes are used unless overridden by the + *projected_epsg_code* / *vertical_epsg_code* parameters. + projected_epsg_code, vertical_epsg_code: + Explicit EPSG codes, taking precedence over *crs_info*. + z_is_up: + ``True`` (default) when the Z column holds heights counted upward — which is what + :func:`apply_from_crs_info` produces. When the vertical CRS is a *depth* CRS the Z + sign is flipped accordingly before the transformation. + use_network: + When ``True``, allow PROJ to download the geoid / datum grids it needs from the PROJ + CDN. **Without them a vertical datum transformation silently does nothing** (it can + be off by tens of metres), so a warning is emitted when a vertical CRS is requested + while the network is disabled. + inplace: + When ``True`` the result is written back into *points*. + + Returns + ------- + np.ndarray + ``(N, 3)`` array of ``[longitude, latitude, height]``. + + Raises + ------ + MissingExtraInstallation + When :mod:`pyproj` is not installed (``pip install energyml-utils[crs]``). + NotEnoughInformationError + When no horizontal EPSG code is available. + """ + from energyml.utils.exception import MissingExtraInstallation, NotEnoughInformationError + + if projected_epsg_code is None and crs_info is not None: + projected_epsg_code = crs_info.projected_epsg_code + if vertical_epsg_code is None and crs_info is not None: + vertical_epsg_code = crs_info.vertical_epsg_code + + source_crs_id = build_source_crs_id(projected_epsg_code, vertical_epsg_code) + if source_crs_id is None: + raise NotEnoughInformationError( + "Cannot reproject to WGS84: no projected (horizontal) EPSG code found in the CRS object." + ) + + if not is_pyproj_available(): + raise MissingExtraInstallation("crs") + + import pyproj + + pts = np.asarray(points, dtype=np.float64) + if pts.ndim == 1: + pts = pts.reshape(-1, 3) + + flip_z = False + if vertical_epsg_code is not None: + # The vertical CRS counts depths (positive down) but Z holds heights: flip it back. + # Folded into the per-chunk copy below, so it costs no extra full-size buffer. + flip_z = z_is_up and _vertical_axis_is_down(vertical_epsg_code) + + if crs_info is not None and crs_info.time_uom is not None: + logger.warning( + "reproject_to_wgs84: the source CRS is a time-based CRS (time_uom='%s'); " + "the Z column is a time, not a height — its reprojection is meaningless.", + crs_info.time_uom, + ) + + dest = pts if inplace else np.empty((len(pts), 3), dtype=np.float64) + + network_was_enabled = pyproj.network.is_network_enabled() + if use_network and not network_was_enabled: + pyproj.network.set_network_enabled(True) + try: + transformer, vertical_dropped = _build_transformer_with_vertical_fallback( + source_crs_id, + projected_epsg_code, + vertical_epsg_code, + use_network or network_was_enabled, + ) + if vertical_dropped: + # The Z column is passed through untouched, so it is still in the source vertical + # frame — flipping its sign for a vertical CRS that PROJ refused would be wrong. + flip_z = False + elif vertical_epsg_code is not None and not (use_network or network_was_enabled): + # Warned only once the vertical CRS is known to be usable, otherwise the message + # points at the wrong problem. + logger.warning( + "reproject_to_wgs84: vertical CRS EPSG:%s requested while the PROJ network is disabled — " + "the geoid grid may be missing and the vertical transformation silently skipped. " + "Pass use_network=True (or install the grids) for an accurate height.", + vertical_epsg_code, + ) + if len(pts) == 1: + # pyproj takes its scalar code path for a 1-element array (and numpy warns about it) + z0 = -pts[0, 2] if flip_z else pts[0, 2] + if vertical_dropped: + lon, lat = transformer.transform(float(pts[0, 0]), float(pts[0, 1])) + height = pts[0, 2] + else: + lon, lat, height = transformer.transform(float(pts[0, 0]), float(pts[0, 1]), float(z0)) + dest[0, 0], dest[0, 1], dest[0, 2] = lon, lat, height + else: + # pyproj needs one contiguous float64 buffer per axis, and the columns of a C-order + # (N, 3) array are strided — so a copy per axis is unavoidable. Doing it chunk by + # chunk makes that cost *constant* instead of proportional to N: three scratch + # buffers of _REPROJECT_CHUNK points, reused for every block, and + # ``inplace=True`` lets PROJ write its result back into them. + chunk = min(_REPROJECT_CHUNK, len(pts)) + x = np.empty(chunk, dtype=np.float64) + y = np.empty(chunk, dtype=np.float64) + z = np.empty(chunk, dtype=np.float64) + for start in range(0, len(pts), chunk): + stop = min(start + chunk, len(pts)) + n = stop - start + xv, yv, zv = x[:n], y[:n], z[:n] + np.copyto(xv, pts[start:stop, 0]) + np.copyto(yv, pts[start:stop, 1]) + np.copyto(zv, pts[start:stop, 2]) + if flip_z: + np.negative(zv, out=zv) + if vertical_dropped: + transformer.transform(xv, yv, inplace=True) + else: + transformer.transform(xv, yv, zv, inplace=True) + dest[start:stop, 0] = xv + dest[start:stop, 1] = yv + dest[start:stop, 2] = zv + finally: + if use_network and not network_was_enabled: + pyproj.network.set_network_enabled(False) + + return dest + + # --------------------------------------------------------------------------- # Public factory # --------------------------------------------------------------------------- @@ -926,6 +1230,9 @@ def extract_crs_info( if type_name_lower == "verticalcrs": return _from_vertical_crs(crs_obj) + if type_name_lower == "projectedcrs": + return _from_projected_crs(crs_obj) + # ------------------------------------------------------------------ # v2.0.1 types (LocalDepth3dCrs, LocalTime3dCrs, AbstractLocal3dCrs) # ------------------------------------------------------------------ @@ -951,6 +1258,21 @@ def extract_crs_info( ) return _from_local_engineering2d_crs(crs_obj, workspace) + # v2.2 pattern: has AbstractProjectedCrs / AbstractVerticalCrs → standalone CRS document + if get_object_attribute_rgx(crs_obj, "[Aa]bstract[Pp]rojected[Cc]rs") is not None: + logger.debug( + "extract_crs_info: unrecognised type '%s' — treating as ProjectedCrs (v2.2 pattern).", + type(crs_obj).__name__, + ) + return _from_projected_crs(crs_obj) + + if get_object_attribute_rgx(crs_obj, "[Aa]bstract[Vv]ertical[Cc]rs") is not None: + logger.debug( + "extract_crs_info: unrecognised type '%s' — treating as VerticalCrs (v2.2 pattern).", + type(crs_obj).__name__, + ) + return _from_vertical_crs(crs_obj) + # v2.2 pattern: has LocalEngineering2dCrs DOR → compound if get_object_attribute_rgx(crs_obj, "[Ll]ocal[Ee]ngineering2[dD][Cc]rs") is not None: logger.debug( @@ -966,9 +1288,251 @@ def extract_crs_info( return CrsInfo(source_type=type(crs_obj).__name__) +# --------------------------------------------------------------------------- +# Coordinate frames — the single place that knows how the stages compose +# --------------------------------------------------------------------------- + + +class PointFrame(Enum): + """ + Which coordinate frame a point array is expressed in. + + The three values are the successive **stages of one pipeline**, not alternatives: + + ``LOCAL`` --:func:`apply_from_crs_info`--> ``PROJECTED`` --:func:`reproject_to_wgs84`--> ``WGS84`` + + A frame can therefore only be reached from the one before it. In particular WGS84 is *not* + an alternative to the local transform: skipping ``LOCAL -> PROJECTED`` would hand pyproj + coordinates still offset by the local origin, and yield a wrong position rather than merely + large numbers. + """ + + LOCAL = "local" + """Raw coordinates, as stored in the energyml object (local engineering CRS).""" + + PROJECTED = "projected" + """Rotation, offsets, Z-flip and axis order applied — metres in the projected CRS.""" + + WGS84 = "wgs84" + """Longitude / latitude / ellipsoidal height (EPSG:4979).""" + + @property + def stage(self) -> int: + """Rank of the frame in the pipeline, used to order the transitions.""" + return _FRAME_ORDER[self] + + +_FRAME_ORDER = {PointFrame.LOCAL: 0, PointFrame.PROJECTED: 1, PointFrame.WGS84: 2} + + +@dataclass +class FramedPoints: + """ + Result of :func:`to_frame`: the transformed points plus what was actually achieved. + + ``frame`` is the frame the points are really in, which may be *earlier* than the requested + one: a WGS84 request degrades to :attr:`PointFrame.PROJECTED` when no EPSG code is available + or when ``pyproj`` is not installed. Callers use it to decide what to advertise (see the + ``crs`` / ``coordRefSys`` members of the GeoJSON writers) instead of silently claiming WGS84. + """ + + points: np.ndarray + """The ``(N, 3)`` float64 array, in :attr:`frame`.""" + + frame: PointFrame + """The frame actually reached.""" + + origin_shift: Optional[tuple] = None + """The ``(dx, dy, dz)`` vector subtracted from the coordinates, if any.""" + + degraded_reason: Optional[str] = None + """Why the requested frame could not be reached, when it could not.""" + + +def compute_origin_shift( + point_arrays, + *, + round_to: Optional[float] = 1.0, +) -> tuple: + """ + Compute a single recentring vector for a whole set of point arrays. + + Projected coordinates carry 6 to 7 significant digits (a UTM easting is ~5·10⁵ m), and most + mesh formats are re-read as float32 by viewers, which leaves roughly decimetre precision. + Subtracting a common origin brings the coordinates near zero and restores it. + + It must be computed **once for the whole export** and applied identically to every patch: + a per-patch shift would move the patches relative to each other. + + Parameters + ---------- + point_arrays: + Iterable of ``(N, 3)`` arrays. Empty arrays are ignored. + round_to: + Round the vector down to a multiple of this value, so the offset stays a readable + number that can be written in the output metadata. ``None`` disables the rounding. + + Returns + ------- + tuple + The ``(dx, dy, dz)`` vector to subtract; ``(0.0, 0.0, 0.0)`` when there is no point. + """ + mins = None + maxs = None + for arr in point_arrays: + a = np.asarray(arr, dtype=np.float64).reshape(-1, 3) + if len(a) == 0: + continue + # Grid2d holes are stored as NaN, so nanmin/nanmax are the right reduction — but they warn + # on an all-NaN column, which is a legitimate input here. Drop the non-finite rows first. + a = a[np.isfinite(a).all(axis=1)] + if len(a) == 0: + continue + a_min = a.min(axis=0) + a_max = a.max(axis=0) + mins = a_min if mins is None else np.minimum(mins, a_min) + maxs = a_max if maxs is None else np.maximum(maxs, a_max) + + if mins is None or maxs is None or not np.all(np.isfinite(mins)) or not np.all(np.isfinite(maxs)): + return (0.0, 0.0, 0.0) + + center = (mins + maxs) / 2.0 + if round_to: + center = np.floor(center / round_to) * round_to + return tuple(float(c) for c in center) + + +def to_frame( + points: np.ndarray, + crs_info: Optional[CrsInfo], + target: PointFrame, + current: PointFrame = PointFrame.LOCAL, + *, + origin_shift: Optional[Any] = None, + use_network: bool = False, + inplace: bool = True, +) -> FramedPoints: + """ + Bring *points* from the *current* frame to the *target* one. + + This is the only function that knows the order of the stages, so no caller has to remember + that the local transform comes first, and calling it twice on the same array is impossible: + when ``current`` already is ``target`` nothing is applied. + + Parameters + ---------- + points: + ``(N, 3)`` array. Modified in place when *inplace* is ``True`` (default) — the array + must then be owned and writeable (see ``mesh_numpy._ensure_float64_points``). + crs_info: + Source CRS. May be ``None``, in which case only ``origin_shift`` can be applied. + target: + Requested frame. + current: + Frame *points* is currently in. + origin_shift: + Explicit ``(dx, dy, dz)`` vector to subtract once the target frame is reached. Use + :func:`compute_origin_shift` to derive one for a whole export — this function + deliberately does not accept ``"auto"``, because a per-array shift would move the + arrays relative to each other. + use_network: + Allow PROJ to download the geoid grids used by the vertical transformation. + inplace: + When ``False``, *points* is left untouched and a copy is returned. + + Returns + ------- + FramedPoints + The points and the frame actually reached — which may be earlier than *target*. + + Raises + ------ + NotSupportedError + When *target* is before *current* in the pipeline: the inverse transforms are not + implemented, and silently returning unchanged coordinates would be worse. + """ + from energyml.utils.exception import ( + MissingExtraInstallation, + NotEnoughInformationError, + NotSupportedError, + ) + + pts = np.asarray(points, dtype=np.float64) + if pts.ndim == 1: + pts = pts.reshape(-1, 3) + if not inplace: + pts = pts.copy() + + if target.stage < current.stage: + raise NotSupportedError( + f"Cannot go back from {current.value!r} to {target.value!r}: the inverse CRS " + "transforms are not implemented." + ) + + reached = current + degraded_reason: Optional[str] = None + + if len(pts) == 0: + # Nothing to transform, but the frame is whatever was asked for: an empty patch must not + # make a whole export look 'degraded'. + return FramedPoints(points=pts, frame=target, origin_shift=None) + + # --- Stage 1: LOCAL -> PROJECTED --------------------------------------- + if reached is PointFrame.LOCAL and target.stage >= PointFrame.PROJECTED.stage: + if crs_info is None: + degraded_reason = "no CRS information available for the local -> projected transform" + logger.warning("to_frame: %s — coordinates are left in the local frame.", degraded_reason) + return FramedPoints(points=pts, frame=reached, origin_shift=None, degraded_reason=degraded_reason) + apply_from_crs_info(pts, crs_info, inplace=True) + reached = PointFrame.PROJECTED + + # --- Stage 2: PROJECTED -> WGS84 -------------------------------------- + if reached is PointFrame.PROJECTED and target is PointFrame.WGS84: + try: + reproject_to_wgs84(pts, crs_info, use_network=use_network, inplace=True) + reached = PointFrame.WGS84 + except NotEnoughInformationError as exc: + degraded_reason = str(exc) + logger.warning( + "to_frame: no projected EPSG code found — coordinates are left in the projected " + "frame instead of WGS84." + ) + except MissingExtraInstallation: + degraded_reason = "pyproj is not installed (pip install energyml-utils[crs])" + logger.warning( + "to_frame: %s — coordinates are left in EPSG:%s instead of WGS84.", + degraded_reason, + getattr(crs_info, "projected_epsg_code", None), + ) + except Exception as exc: + degraded_reason = f"{type(exc).__name__}: {exc}" + logger.warning("to_frame: reprojection to WGS84 failed (%s) — keeping projected coordinates.", exc) + + # --- Recentring ------------------------------------------------------- + applied_shift: Optional[tuple] = None + if origin_shift is not None: + shift = np.asarray(origin_shift, dtype=np.float64).reshape(3) + if np.any(shift): + pts -= shift + applied_shift = tuple(float(s) for s in shift) + + return FramedPoints(points=pts, frame=reached, origin_shift=applied_shift, degraded_reason=degraded_reason) + + __all__ = [ "CrsInfo", "extract_crs_info", "apply_from_crs_info", "apply_axis_order_swap", + "reproject_to_wgs84", + "build_source_crs_id", + "is_pyproj_available", + "crs_ogc_uri", + "crs_urn", + "PointFrame", + "FramedPoints", + "to_frame", + "compute_origin_shift", + "WGS84_2D_EPSG_CODE", + "WGS84_3D_EPSG_CODE", ] diff --git a/energyml-utils/src/energyml/utils/data/datasets_io.py b/energyml-utils/src/energyml/utils/data/datasets_io.py index d758ee4..41604be 100644 --- a/energyml-utils/src/energyml/utils/data/datasets_io.py +++ b/energyml-utils/src/energyml/utils/data/datasets_io.py @@ -167,7 +167,7 @@ def write_array( ): if isinstance(array, list): array = np.asarray(array) - print("writing array", target) + logger.debug("Writing array to %s", target) if dtype is not None and not isinstance(dtype, np.dtype): dtype = np.dtype(dtype) @@ -322,16 +322,16 @@ def read_array( source.seek(s_pos) - logging.debug(comments) + logger.debug(comments) items = [] if len(comments) > 0: _delim = re.search(r'Default\s+delimiter:\s*"(?P[^"])"', comments, re.IGNORECASE) if _delim is not None: - logging.debug("delim", _delim, _delim.group("delim")) + logger.debug("delim", _delim, _delim.group("delim")) _delim = _delim.group("delim") - logging.debug(_delim, "<==") + logger.debug(_delim, "<==") if len(_delim) > 0: delimiter = _delim @@ -340,14 +340,14 @@ def read_array( comments, re.IGNORECASE, ) - logging.debug("items", items) + logger.debug("items", items) items = list(map(lambda it: (it[0], int(it[1]), int(it[2])), items)) _cst = re.findall( r"Item\s*:\s*(?P[\w]+)\s+constant\s*:\s*(?P\w+)", comments, re.IGNORECASE ) - logging.debug("cst", _cst) + logger.debug("cst", _cst) max_line_number = 0 for _, n, _ in items: @@ -356,11 +356,11 @@ def read_array( for i in range(max_line_number - 1): source.readline() # on skip les values des autres items, on ne garde que le tableau de valeurs - logging.debug(max_line_number) - logging.debug(items) + logger.debug(max_line_number) + logger.debug(items) # removing items not related to the columns titles items items = list(filter(lambda it: it[1] == max_line_number, items)) - logging.debug(items) + logger.debug(items) if isinstance(source, BytesIO) or isinstance(source, BinaryIO) or isinstance(source, BufferedReader): source = TextIOWrapper(source, encoding=encoding) @@ -380,6 +380,15 @@ def read_array( def read_array_as_panda_dict( self, source: Union[BytesIO, TextIO, str], delimiter: Optional[str] = ",", has_header: bool = True, **fmtparams ) -> Optional[Any]: + """ + Read the whole file as a pandas object. + + :raise MissingExtraInstallation: pandas comes with the ``parquet`` extra. Without it this + used to fail with ``NameError: name 'pd' is not defined``, which says nothing about + what to install. + """ + if not __PARQUET_MODULE_EXISTS__: + raise MissingExtraInstallation(extra_name="parquet") if isinstance(source, str): with open(source, "r", newline="") as datFile: return self.read_array_as_panda_dict(datFile, delimiter, has_header=has_header, **fmtparams) @@ -439,6 +448,15 @@ def read_array( def read_array_as_panda_dict( self, source: Union[BytesIO, TextIO, str], delimiter: Optional[str] = ",", has_header: bool = True, **fmtparams ) -> Optional[Any]: + """ + Read the whole file as a pandas object. + + :raise MissingExtraInstallation: pandas comes with the ``parquet`` extra. Without it this + used to fail with ``NameError: name 'pd' is not defined``, which says nothing about + what to install. + """ + if not __PARQUET_MODULE_EXISTS__: + raise MissingExtraInstallation(extra_name="parquet") if isinstance(source, str): with open(source, "r", newline="") as csvFile: return self.read_array_as_panda_dict(csvFile, delimiter, has_header=has_header, **fmtparams) @@ -534,12 +552,12 @@ def get_external_file_path_from_external_path( # resqml 2.0.1 if hdf_proxy_lst is not None and len(hdf_proxy_lst) > 0: hdf_proxy = hdf_proxy_lst - # logging.debug("h5Proxy", hdf_proxy) + # logger.debug("h5Proxy", hdf_proxy) while isinstance(hdf_proxy, list): hdf_proxy = hdf_proxy[0] hdf_proxy_obj = epc.get_object_by_identifier(get_obj_identifier(hdf_proxy)) try: - logging.debug(f"hdf_proxy_obj : {hdf_proxy_obj} {hdf_proxy} : {hdf_proxy}") + logger.debug(f"hdf_proxy_obj : {hdf_proxy_obj} {hdf_proxy} : {hdf_proxy}") except: pass if hdf_proxy_obj is not None: @@ -570,7 +588,7 @@ def get_external_file_path_from_external_path( result = [epc.epc_file_path[:-4] + ".h5"] try: - logging.debug(f"{external_path_obj} {result} \n\t{hdf_proxy_lst}\n\t{ext_file_proxy_lst}") + logger.debug(f"{external_path_obj} {result} \n\t{hdf_proxy_lst}\n\t{ext_file_proxy_lst}") except: pass return result @@ -640,7 +658,7 @@ def read_external_dataset_array( except MissingExtraInstallation as mei: raise mei except Exception as e: - logging.debug(f"Failed to read external file {s} for {path_in_obj} with path {path_in_external} : {e}") + logger.debug(f"Failed to read external file {s} for {path_in_obj} with path {path_in_external} : {e}") pass if not succeed: raise Exception(f"Failed to read external file. Paths tried : {sources}") @@ -696,18 +714,18 @@ def get_proxy_uri_for_path_in_external(obj: Any, dataspace_name_or_uri: Union[st uri_path_map = {} _piefs = get_path_in_external_with_path(obj) if _piefs is not None and len(_piefs) > 0: - # logging.info(f"Found {_piefs} datasets in object {get_obj_uuid(obj)}") + # logger.info(f"Found {_piefs} datasets in object {get_obj_uuid(obj)}") # uri_path_map[uri] = _piefs for item in _piefs: uri = str(get_obj_uri(obj, dataspace=ds_name)) if isinstance(item, tuple): - logging.info( + logger.info( f"Item: {item}, type: {type(item)}, len: {len(item) if hasattr(item, '__len__') else 'N/A'}" ) # Then unpack path, pief = item - # logging.info(f"\t test : {path_last_attribute(path)}") + # logger.info(f"\t test : {path_last_attribute(path)}") if "hdf" in path_last_attribute(path).lower(): dor = get_object_attribute( obj=obj, attr_dot_path=path[: -len(path_last_attribute(path))] + "hdf_proxy" @@ -720,7 +738,7 @@ def get_proxy_uri_for_path_in_external(obj: Any, dataspace_name_or_uri: Union[st uri_path_map[uri] = [] uri_path_map[uri].append(pief) else: - logging.debug(f"No datasets found in object {str(get_obj_uri(obj))}") + logger.debug(f"No datasets found in object {str(get_obj_uri(obj))}") return uri_path_map @@ -732,6 +750,8 @@ def get_proxy_uri_for_path_in_external(obj: Any, dataspace_name_or_uri: Union[st from typing import Callable from energyml.utils.data.model import ExternalArrayHandler +logger = logging.getLogger(__name__) + class FileHandlerRegistry: """ @@ -851,13 +871,10 @@ class HDF5ArrayHandler(ExternalArrayHandler): def __init__(self, max_open_files: int = 3): super().__init__(max_open_files=max_open_files) - def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: - """Open an HDF5 file without using the cache.""" - try: - return h5py.File(file_path, mode) # type: ignore - except Exception as e: - # logging.debug(f"Failed to open HDF5 file {file_path}: {e}") - return None + format_name = "HDF5" + + def _open_file(self, file_path: str, mode: str = "r") -> Optional[Any]: + return h5py.File(file_path, mode) # type: ignore def read_array( self, @@ -878,8 +895,11 @@ def read_array( return full_array return None else: - with self.file_cache.get_or_open(source, self, "r") as f: # type: ignore - return self.read_array(f, path_in_external_file, start_indices, counts) + # The cache owns the handle's lifetime — see the note in read_array_view. + f = self.file_cache.get_or_open(source, self, "r") # type: ignore + if f is None: + return None + return self.read_array(f, path_in_external_file, start_indices, counts) def read_array_view( self, @@ -906,13 +926,26 @@ def read_array_view( # h5py reads only the required chunks/slabs from disk slices = tuple(slice(start, start + count) for start, count in zip(start_indices, counts)) return d_group[slices] # type: ignore - # np.array with copy=False returns a view for contiguous datasets - # Note: copy= kwarg on np.asarray requires numpy >=2.0; - # np.array(x, copy=False) works on all numpy versions. - return np.array(d_group, copy=False) # type: ignore + # NumPy 2.0 redefined `copy=False`: it used to mean "avoid a copy *if possible*" + # and now means "never copy — raise if you would have to". An HDF5 dataset lives + # on disk, so h5py always has to allocate, and `np.array(d_group, copy=False)` + # raises on numpy>=2 with + # "Dataset.__array__ received copy=False but memory allocation cannot be + # avoided on read". + # It made every external array unreadable on a numpy>=2 install while the + # numpy 1.26 lockfile of this repository kept the tests green. np.asarray() is + # the spelling that means the same thing under both majors. + return np.asarray(d_group) # type: ignore else: - with self.file_cache.get_or_open(source, self, "r") as f: # type: ignore - return self.read_array_view(f, path_in_external_file, start_indices, counts) + # `get_or_open` returns a *cached* handle: the cache owns it and closes it in + # `close_all`. Wrapping it in `with` closed it at the end of the first read while + # the cache kept serving it, so the next read on the same file got a dead handle + # and raised "invalid identifier type to function" — the order in which + # read_array and read_array_view happened to be called decided whether it worked. + f = self.file_cache.get_or_open(source, self, "r") # type: ignore + if f is None: + return None + return self.read_array_view(f, path_in_external_file, start_indices, counts) def write_array( self, @@ -960,7 +993,7 @@ def write_array( return True except Exception as e: - logging.error(f"Failed to write array to HDF5: {e}") + logger.error(f"Failed to write array to HDF5: {e}") return False def get_array_metadata( @@ -1000,7 +1033,7 @@ def get_array_metadata( self.file_cache.get_or_open(source, self, "r"), path_in_external_file, start_indices, counts ) except Exception as e: - logging.debug(f"Failed to get HDF5 metadata: {e}") + logger.debug(f"Failed to get HDF5 metadata: {e}") return None def list_arrays(self, source: Union[BytesIO, str, Any]) -> List[str]: @@ -1020,9 +1053,10 @@ class MockHDF5ArrayHandler(ExternalArrayHandler): def __init__(self, max_open_files: int = 3): super().__init__(max_open_files=max_open_files) - def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: - """Open an HDF5 file without using the cache.""" - return None + format_name = "HDF5" + + def _open_file(self, file_path: str, mode: str = "r") -> Optional[Any]: + return None # h5py is not installed: there is nothing to open def read_array( self, @@ -1071,13 +1105,10 @@ class ParquetArrayHandler(ExternalArrayHandler): def __init__(self, max_open_files: int = 3): super().__init__(max_open_files=max_open_files) - def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: - """Open a Parquet file without using the cache.""" - try: - return pq.ParquetFile(file_path) # type: ignore - except Exception as e: - logging.error(f"Failed to open Parquet file {file_path}: {e}") - return None + format_name = "Parquet" + + def _open_file(self, file_path: str, mode: str = "r") -> Optional[Any]: + return pq.ParquetFile(file_path) # type: ignore def read_array( self, @@ -1132,7 +1163,7 @@ def write_array( array_as_pd_df = pd.DataFrame({col_name: array}) else: # For higher dimensions, flatten or handle as needed - logging.warning(f"Parquet writer received {array.ndim}D array, flattening to 2D") + logger.warning(f"Parquet writer received {array.ndim}D array, flattening to 2D") array_2d = array.reshape(array.shape[0], -1) if column_titles is None: column_titles = [str(i) for i in range(array_2d.shape[1])] @@ -1146,7 +1177,7 @@ def write_array( ) return True except Exception as e: - logging.error(f"Failed to write array to Parquet: {e}") + logger.error(f"Failed to write array to Parquet: {e}") return False def get_array_metadata( @@ -1187,7 +1218,7 @@ def get_array_metadata( # Get all columns return [self.get_array_metadata(source, field.name, start_indices, counts) for field in schema] except Exception as e: - logging.debug(f"Failed to get Parquet metadata: {e}") + logger.debug(f"Failed to get Parquet metadata: {e}") return None def list_arrays(self, source: Union[BytesIO, str, Any]) -> List[str]: @@ -1213,9 +1244,10 @@ class MockParquetArrayHandler(ExternalArrayHandler): def __init__(self, max_open_files: int = 3): super().__init__(max_open_files=max_open_files) - def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: - """Open a Parquet file without using the cache.""" - return None + format_name = "Parquet" + + def _open_file(self, file_path: str, mode: str = "r") -> Optional[Any]: + return None # pyarrow is not installed: there is nothing to open def read_array( self, @@ -1264,13 +1296,10 @@ class CSVArrayHandler(ExternalArrayHandler): def __init__(self, max_open_files: int = 3): super().__init__(max_open_files=max_open_files) - def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: - """Open a CSV file without using the cache.""" - try: - return open(file_path, mode) - except Exception as e: - logging.error(f"Failed to open CSV file {file_path}: {e}") - return None + format_name = "CSV" + + def _open_file(self, file_path: str, mode: str = "r") -> Optional[Any]: + return open(file_path, mode) def read_array( self, @@ -1294,7 +1323,7 @@ def read_array( return data[slices] return data except Exception as e: - logging.debug(f"Failed to read CSV: {e}") + logger.debug(f"Failed to read CSV: {e}") return None def write_array( @@ -1312,7 +1341,7 @@ def write_array( np.savetxt(target, array, delimiter=",") return True except Exception as e: - logging.error(f"Failed to write CSV: {e}") + logger.error(f"Failed to write CSV: {e}") return False def get_array_metadata( @@ -1333,7 +1362,7 @@ def get_array_metadata( "size": data.size, } except Exception as e: - logging.debug(f"Failed to get CSV metadata: {e}") + logger.debug(f"Failed to get CSV metadata: {e}") return None def list_arrays(self, source: Union[BytesIO, str, Any]) -> List[str]: @@ -1355,13 +1384,10 @@ class LASArrayHandler(ExternalArrayHandler): def __init__(self, max_open_files: int = 3): super().__init__(max_open_files=max_open_files) - def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: - """Open a LAS file without using the cache.""" - try: - return lasio.read(file_path) # type: ignore - except Exception as e: - logging.error(f"Failed to open LAS file {file_path}: {e}") - return None + format_name = "LAS" + + def _open_file(self, file_path: str, mode: str = "r") -> Optional[Any]: + return lasio.read(file_path) # type: ignore def read_array( self, @@ -1399,10 +1425,10 @@ def read_array( if mnemonic in las.keys(): curves_data.append(las[mnemonic]) else: - logging.warning(f"Mnemonic '{mnemonic}' not found in LAS file") + logger.warning(f"Mnemonic '{mnemonic}' not found in LAS file") if not curves_data: - logging.error("No valid mnemonics found in LAS file") + logger.error("No valid mnemonics found in LAS file") return None # Stack curves horizontally @@ -1420,7 +1446,7 @@ def read_array( return np.array(data) except Exception as e: - logging.error(f"Failed to read LAS file: {e}") + logger.error(f"Failed to read LAS file: {e}") return None def write_array( @@ -1487,7 +1513,7 @@ def write_array( return True except Exception as e: - logging.error(f"Failed to write LAS file: {e}") + logger.error(f"Failed to write LAS file: {e}") return False def get_array_metadata( @@ -1538,7 +1564,7 @@ def get_array_metadata( return metadata except Exception as e: - logging.error(f"Failed to get LAS metadata: {e}") + logger.error(f"Failed to get LAS metadata: {e}") return None def list_arrays(self, source: Union[BytesIO, str, Any]) -> List[str]: @@ -1547,7 +1573,7 @@ def list_arrays(self, source: Union[BytesIO, str, Any]) -> List[str]: las = lasio.read(source) return [curve.mnemonic for curve in las.curves] except Exception as e: - logging.error(f"Failed to list LAS curves: {e}") + logger.error(f"Failed to list LAS curves: {e}") return [] def can_handle_file(self, file_path: str) -> bool: @@ -1563,9 +1589,10 @@ class MockLASArrayHandler(ExternalArrayHandler): def __init__(self, max_open_files: int = 3): super().__init__(max_open_files=max_open_files) - def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: - """Open a LAS file without using the cache.""" - return None + format_name = "LAS" + + def _open_file(self, file_path: str, mode: str = "r") -> Optional[Any]: + return None # lasio is not installed: there is nothing to open def read_array( self, @@ -1616,13 +1643,10 @@ class SEGYArrayHandler(ExternalArrayHandler): def __init__(self, max_open_files: int = 3): super().__init__(max_open_files=max_open_files) - def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: - """Open a SEG-Y file without using the cache.""" - try: - return segyio.open(file_path, mode, ignore_geometry=True) # type: ignore - except Exception as e: - logging.error(f"Failed to open SEG-Y file {file_path}: {e}") - return None + format_name = "SEG-Y" + + def _open_file(self, file_path: str, mode: str = "r") -> Optional[Any]: + return segyio.open(file_path, mode, ignore_geometry=True) # type: ignore def read_array( self, @@ -1646,7 +1670,7 @@ def read_array( try: # SEG-Y requires file path, not BytesIO if not isinstance(source, str): - logging.error("SEG-Y handler requires file path, not BytesIO") + logger.error("SEG-Y handler requires file path, not BytesIO") return None with segyio.open(source, "r", ignore_geometry=True) as f: @@ -1686,7 +1710,7 @@ def read_array( return np.array(header_data) except Exception as e: - logging.error(f"Failed to read SEG-Y file: {e}") + logger.error(f"Failed to read SEG-Y file: {e}") return None def write_array( @@ -1711,7 +1735,7 @@ def write_array( """ try: if not isinstance(target, str): - logging.error("SEG-Y handler requires file path for writing") + logger.error("SEG-Y handler requires file path for writing") return False if not isinstance(array, np.ndarray): @@ -1741,7 +1765,7 @@ def write_array( return True except Exception as e: - logging.error(f"Failed to write SEG-Y file: {e}") + logger.error(f"Failed to write SEG-Y file: {e}") return False def get_array_metadata( @@ -1759,7 +1783,7 @@ def get_array_metadata( """ try: if not isinstance(source, str): - logging.error("SEG-Y handler requires file path") + logger.error("SEG-Y handler requires file path") return None with segyio.open(source, "r", ignore_geometry=True) as f: @@ -1776,7 +1800,7 @@ def get_array_metadata( return metadata except Exception as e: - logging.error(f"Failed to get SEG-Y metadata: {e}") + logger.error(f"Failed to get SEG-Y metadata: {e}") return None def list_arrays(self, source: Union[BytesIO, str, Any]) -> List[str]: @@ -1796,9 +1820,10 @@ class MockSEGYArrayHandler(ExternalArrayHandler): def __init__(self, max_open_files: int = 3): super().__init__(max_open_files=max_open_files) - def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: - """Open a SEG-Y file without using the cache.""" - return None + format_name = "SEG-Y" + + def _open_file(self, file_path: str, mode: str = "r") -> Optional[Any]: + return None # segyio is not installed: there is nothing to open def read_array( self, @@ -1838,3 +1863,51 @@ def can_handle_file(self, file_path: str) -> bool: # Alias so the public name is always importable SEGYArrayHandler = MockSEGYArrayHandler + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "h5_list_datasets", + "DATFileReader", + "CSVFileReader", + "CSVFileWriter", + "get_external_file_path_possibilities", + "get_external_file_path_from_external_path", + "get_external_file_path_possibilities_from_folder", + "read_dataset", + "read_external_dataset_array", + "get_path_in_external", + "get_path_in_external_with_path", + "get_proxy_uri_for_path_in_external", + "FileHandlerRegistry", + "get_handler_registry", +] + +# The readers, the writers and the array handlers below are defined only when their optional +# dependency is installed — `HDF5ArrayHandler` when h5py is there, `MockHDF5ArrayHandler` +# otherwise. Listing them unconditionally in `__all__` would make `import *` fail with +# AttributeError on an installation without the extra, so the list is completed at import time +# with the names that really exist. +__all__ += [ + _name + for _name in ( + "h5py", + "lasio", + "segyio", + "HDF5FileReader", + "HDF5FileWriter", + "ParquetFileReader", + "ParquetFileWriter", + "HDF5ArrayHandler", + "MockHDF5ArrayHandler", + "ParquetArrayHandler", + "MockParquetArrayHandler", + "CSVArrayHandler", + "LASArrayHandler", + "MockLASArrayHandler", + "SEGYArrayHandler", + "MockSEGYArrayHandler", + ) + if _name in globals() +] diff --git a/energyml-utils/src/energyml/utils/data/export.py b/energyml-utils/src/energyml/utils/data/export.py deleted file mode 100644 index 3c58576..0000000 --- a/energyml-utils/src/energyml/utils/data/export.py +++ /dev/null @@ -1,1186 +0,0 @@ -# Copyright (c) 2023-2024 Geosiris. -# SPDX-License-Identifier: Apache-2.0 -""" -Module for exporting mesh data to various file formats. - -Supports OBJ, GeoJSON, VTK Legacy (ASCII + binary), VTK XML (.vtu / .vtp), -and STL formats. - -Both the legacy :class:`AbstractMesh` hierarchy (``mesh.py``) and the -high-performance :class:`NumpyMesh` / :class:`NumpyMultiMesh` hierarchy -(``mesh_numpy.py``) are accepted by every export function. - -CRS-displacement can be applied at export time (rather than at read time) by -passing ``use_crs_displacement=True`` (default) when a workspace is reachable -through the ``contexts`` dict. The original ``NumpyMesh.points`` arrays are -**never mutated** — a copy is made whenever CRS needs to be applied. - -Color metadata is sourced from :class:`RepresentationContext` objects keyed -by ``source_uuid``; if none are provided a default palette is used. -""" - -from __future__ import annotations - -import base64 -import json -import logging -import struct -from enum import Enum -from pathlib import Path -from typing import TYPE_CHECKING, Any, BinaryIO, Dict, List, Optional, TextIO, Union - -import numpy as np - -if TYPE_CHECKING: - from energyml.utils.data.mesh import AbstractMesh - from energyml.utils.data.mesh_numpy import ( - NumpyMesh, - NumpyMultiMesh, - NumpyPolylineMesh, - NumpyPointSetMesh, - NumpySurfaceMesh, - NumpyVolumeMesh, - ) - from energyml.utils.data.representation_context import RepresentationContext - -log = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# VTK cell-type constants (subset) -# --------------------------------------------------------------------------- -_VTK_VERTEX = 1 -_VTK_POLY_LINE = 4 -_VTK_TRIANGLE = 5 -_VTK_POLYGON = 7 -_VTK_TETRA = 10 -_VTK_HEXAHEDRON = 12 - -# --------------------------------------------------------------------------- -# Enumerations / option classes -# --------------------------------------------------------------------------- - - -class ExportFormat(Enum): - """Supported mesh export formats.""" - - OBJ = "obj" - GEOJSON = "geojson" - VTK = "vtk" - VTU = "vtu" - VTP = "vtp" - STL = "stl" - - @classmethod - def from_extension(cls, extension: str) -> "ExportFormat": - """Get format from file extension.""" - ext = extension.lower().lstrip(".") - for fmt in cls: - if fmt.value == ext: - return fmt - raise ValueError(f"Unsupported file extension: {extension}") - - @classmethod - def all_extensions(cls) -> List[str]: - """Get all supported file extensions.""" - return [fmt.value for fmt in cls] - - -class ExportOptions: - """Base class for export options.""" - - -class STLExportOptions(ExportOptions): - """Options for STL export.""" - - def __init__(self, binary: bool = True, ascii_precision: int = 6): - """ - :param binary: If True, export as binary STL; if False, export as ASCII STL. - :param ascii_precision: Number of decimal places for ASCII format. - """ - self.binary = binary - self.ascii_precision = ascii_precision - - -class VTKFormat(Enum): - """Sub-format selector for VTK export.""" - - LEGACY_ASCII = "legacy_ascii" - """VTK legacy format, ASCII encoding (version 3.0).""" - - LEGACY_BINARY = "legacy_binary" - """VTK legacy format, big-endian binary encoding (version 3.0).""" - - VTU = "vtu" - """VTK XML UnstructuredGrid (.vtu) — best for volumetric meshes.""" - - VTP = "vtp" - """VTK XML PolyData (.vtp) — best for surface / polyline meshes.""" - - -class VTKExportOptions(ExportOptions): - """Options for VTK export.""" - - def __init__( - self, - vtk_format: VTKFormat = VTKFormat.LEGACY_ASCII, - dataset_name: str = "mesh", - # Legacy compatibility: binary=True is equivalent to vtk_format=VTKFormat.LEGACY_BINARY - binary: bool = False, - ): - """ - :param vtk_format: VTK sub-format (legacy ASCII, legacy binary, VTU, VTP). - :param dataset_name: Dataset name embedded in legacy VTK header or XML title. - :param binary: Deprecated shorthand; when True, forces LEGACY_BINARY sub-format. - """ - self.dataset_name = dataset_name - if binary and vtk_format == VTKFormat.LEGACY_ASCII: - # Honour the legacy binary=True flag so old call-sites still work. - self.vtk_format = VTKFormat.LEGACY_BINARY - else: - self.vtk_format = vtk_format - - # Backward-compat property so code that reads ``options.binary`` still works. - @property - def binary(self) -> bool: - return self.vtk_format == VTKFormat.LEGACY_BINARY - - -class GeoJSONExportOptions(ExportOptions): - """Options for GeoJSON export.""" - - def __init__(self, indent: Optional[int] = 2, properties: Optional[dict] = None): - """ - :param indent: JSON indentation level (None for compact output). - :param properties: Extra properties merged into every feature. - """ - self.indent = indent - self.properties = properties or {} - - -# --------------------------------------------------------------------------- -# Private helpers -# --------------------------------------------------------------------------- - - -def _normalize_to_patches(meshes: Any) -> List[Any]: - """Flatten *meshes* into a list of individual mesh patches. - - Handles: - - :class:`NumpyMultiMesh` → calls ``flat_patches()`` - - Single :class:`NumpyMesh` → ``[mesh]`` - - ``list`` / ``tuple`` → recursive - - :class:`AbstractMesh` → passthrough as ``[mesh]`` - """ - from energyml.utils.data.mesh_numpy import NumpyMesh, NumpyMultiMesh - - if isinstance(meshes, NumpyMultiMesh): - return meshes.flat_patches() - if isinstance(meshes, NumpyMesh): - return [meshes] - if isinstance(meshes, (list, tuple)): - result: List[Any] = [] - for m in meshes: - result.extend(_normalize_to_patches(m)) - return result - # AbstractMesh or unknown — pass through as single element - return [meshes] - - -def _parse_vtk_flat_faces(flat: np.ndarray) -> List[np.ndarray]: - """Decode VTK flat face array ``[nv, v0, …, nv, v0, …]`` into a list of - per-face index arrays.""" - faces: List[np.ndarray] = [] - pos = 0 - flat = np.asarray(flat, dtype=np.int64) - while pos < len(flat): - nv = int(flat[pos]) - pos += 1 - if pos + nv > len(flat): - break - faces.append(flat[pos : pos + nv]) - pos += nv - return faces - - -def _parse_vtk_flat_lines(flat: np.ndarray) -> List[np.ndarray]: - """Decode VTK flat lines array ``[n, i0, i1, …, n, i0, …]`` into a list - of per-line index arrays.""" - lines: List[np.ndarray] = [] - pos = 0 - flat = np.asarray(flat, dtype=np.int64) - while pos < len(flat): - n = int(flat[pos]) - pos += 1 - if pos + n > len(flat): - break - lines.append(flat[pos : pos + n]) - pos += n - return lines - - -def _get_export_points( - mesh: Any, - use_crs_displacement: bool, - workspace: Any = None, -) -> np.ndarray: - """Return the point array for *mesh*, optionally applying CRS displacement. - - - For :class:`NumpyMesh`: if ``use_crs_displacement`` is True and a CRS - object is present, returns a *copy* with CRS applied (never mutates the - original ``mesh.points``). - - For :class:`AbstractMesh` (legacy): returns ``mesh.point_list`` as-is; - CRS was already applied by the reader. - """ - from energyml.utils.data.mesh_numpy import NumpyMesh - - if isinstance(mesh, NumpyMesh): - if use_crs_displacement and mesh.crs_object is not None and workspace is not None: - from energyml.utils.data.crs import apply_from_crs_info, extract_crs_info - - crs = mesh.crs_object[0] if isinstance(mesh.crs_object, list) and mesh.crs_object else mesh.crs_object - if crs is not None: - try: - crs_info = extract_crs_info(crs, workspace) - pts = mesh.points.copy() - apply_from_crs_info(pts, crs_info, inplace=True) - return pts - except Exception as exc: # pragma: no cover - log.warning("CRS displacement failed for %s: %s", mesh.source_uuid, exc) - return mesh.points - # AbstractMesh — point_list is a list-of-lists; convert to ndarray for uniform handling - return np.array(getattr(mesh, "point_list", []), dtype=np.float64) - - -def _get_context_color( - source_uuid: Optional[str], - contexts: Optional[Dict[str, Any]], -) -> Optional[tuple]: - """Return an (r, g, b, a) tuple in 0–255 range for *source_uuid*, or None.""" - if not contexts or not source_uuid: - return None - ctx = contexts.get(source_uuid) - if ctx is None: - return None - try: - return ctx.primary_color.to_uint8() - except Exception as exc: # pragma: no cover - log.debug("Failed to read color for %s: %s", source_uuid, exc) - return None - - -def _workspace_from_contexts(contexts: Optional[Dict[str, Any]]) -> Any: - """Return the workspace from the first available RepresentationContext.""" - if not contexts: - return None - for ctx in contexts.values(): - ws = getattr(ctx, "workspace", None) - if ws is not None: - return ws - return None - - -def _get_faces_or_cells(mesh: Any) -> np.ndarray: - """Return the face or cell connectivity array for a NumpyMesh. - - Uses ``mesh.faces`` when present and non-empty, then falls back to - ``mesh.cells``. Avoids the numpy-unsafe ``arr or other`` pattern which - raises ``ValueError`` for arrays with more than one element. - """ - faces = getattr(mesh, "faces", None) - if faces is not None and len(faces) > 0: - return faces - cells = getattr(mesh, "cells", None) - if cells is not None and len(cells) > 0: - return cells - return np.empty(0, dtype=np.int64) - - -# --------------------------------------------------------------------------- -# OBJ export -# --------------------------------------------------------------------------- - - -def export_obj( - mesh_list: Any, - out: BinaryIO, - obj_name: Optional[str] = None, - contexts: Optional[Dict[str, "RepresentationContext"]] = None, - mtl_out: Optional[BinaryIO] = None, - use_crs_displacement: bool = True, -) -> None: - """Export mesh data to Wavefront OBJ format. - - :param mesh_list: One or more meshes (``AbstractMesh``, ``NumpyMesh``, - ``NumpyMultiMesh``, or a list thereof). - :param out: Binary output stream for the ``.obj`` content. - :param obj_name: Optional object name written to the OBJ header. - :param contexts: Optional dict of :class:`RepresentationContext` keyed by - ``source_uuid``; used to emit companion ``.mtl`` material colours when - *mtl_out* is also provided. - :param mtl_out: Optional binary stream for the companion ``.mtl`` file. - Colour requires *contexts* to be supplied. - :param use_crs_displacement: When True (default), CRS origin offset and - axis transforms are applied to ``NumpyMesh`` points at export time. - """ - from energyml.utils.data.mesh import PolylineSetMesh - from energyml.utils.data.mesh_numpy import NumpyMesh, NumpyPointSetMesh, NumpyPolylineMesh - - patches = _normalize_to_patches(mesh_list) - workspace = _workspace_from_contexts(contexts) - - out.write(b"# Generated by energyml-utils (Geosiris)\n\n") - if obj_name is not None: - out.write(f"o {obj_name}\n\n".encode()) - - mtl_lib_name = obj_name or "materials" - if mtl_out is not None: - out.write(f"mtllib {mtl_lib_name}.mtl\n\n".encode()) - mtl_out.write(b"# MTL generated by energyml-utils\n\n") - - point_offset = 0 - - for mesh in patches: - pts = _get_export_points(mesh, use_crs_displacement, workspace) - patch_label = getattr(mesh, "patch_label", None) or getattr(mesh, "identifier", None) or "mesh" - source_uuid = getattr(mesh, "source_uuid", None) or getattr(mesh, "uuid", None) - patch_idx = getattr(mesh, "patch_index", None) - group_name = f"{source_uuid}_{patch_idx}" if source_uuid and patch_idx is not None else patch_label - - out.write(f"g {group_name}\n\n".encode()) - - # emit material reference when mtl output is available - if mtl_out is not None: - mat_name = f"mat_{group_name}" - color = _get_context_color(source_uuid, contexts) - if color is None: - color = (200, 200, 200, 255) - r, g, b, _a = color - out.write(f"usemtl {mat_name}\n".encode()) - mtl_out.write(f"newmtl {mat_name}\n".encode()) - mtl_out.write(f"Kd {r/255:.6f} {g/255:.6f} {b/255:.6f}\n\n".encode()) - - # write vertices - for pt in pts: - out.write(f"v {pt[0]} {pt[1]} {pt[2]}\n".encode()) - - # write connectivity - if isinstance(mesh, NumpyMesh): - if isinstance(mesh, NumpyPointSetMesh): - # bare vertex elements - for i in range(len(pts)): - out.write(f"p {i + point_offset + 1}\n".encode()) - elif isinstance(mesh, NumpyPolylineMesh): - for seg in _parse_vtk_flat_lines(mesh.lines): - if len(seg) > 1: - idx_str = " ".join(str(i + point_offset + 1) for i in seg) - out.write(f"l {idx_str}\n".encode()) - else: - # NumpySurfaceMesh (or NumpyVolumeMesh — export as faces) - faces_arr = _get_faces_or_cells(mesh) - for face in _parse_vtk_flat_faces(faces_arr): - if len(face) >= 3: - idx_str = " ".join(str(i + point_offset + 1) for i in face) - out.write(f"f {idx_str}\n".encode()) - else: - # AbstractMesh legacy path - indices = mesh.get_indices() - elt = "l" if isinstance(mesh, PolylineSetMesh) else "f" - for elem in indices: - if len(elem) > 1: - idx_str = " ".join(str(i + point_offset + 1) for i in elem) - out.write(f"{elt} {idx_str}\n".encode()) - - out.write(b"\n") - point_offset += len(pts) - - -# --------------------------------------------------------------------------- -# GeoJSON export -# --------------------------------------------------------------------------- - - -def export_geojson( - mesh_list: Any, - out: TextIO, - options: Optional[GeoJSONExportOptions] = None, - contexts: Optional[Dict[str, "RepresentationContext"]] = None, - use_crs_displacement: bool = True, -) -> None: - """Export mesh data to GeoJSON FeatureCollection. - - :param mesh_list: One or more meshes. - :param out: Text output stream. - :param options: GeoJSON export options. - :param contexts: Optional colour / metadata context dict. - :param use_crs_displacement: Apply CRS displacement to ``NumpyMesh`` points. - """ - from energyml.utils.data.mesh import PolylineSetMesh, SurfaceMesh - from energyml.utils.data.mesh_numpy import NumpyMesh, NumpyPointSetMesh, NumpyPolylineMesh - - if options is None: - options = GeoJSONExportOptions() - - patches = _normalize_to_patches(mesh_list) - workspace = _workspace_from_contexts(contexts) - features: List[dict] = [] - - for mesh in patches: - pts = _get_export_points(mesh, use_crs_displacement, workspace) - source_uuid = getattr(mesh, "source_uuid", None) - patch_idx = getattr(mesh, "patch_index", None) - color = _get_context_color(source_uuid, contexts) - base_props: dict = { - **options.properties, - "source_uuid": source_uuid, - "patch_index": patch_idx, - } - if color: - r, g, b, a = color - base_props["color"] = f"#{r:02x}{g:02x}{b:02x}" - base_props["opacity"] = round(a / 255.0, 4) - - if isinstance(mesh, NumpyMesh): - if isinstance(mesh, NumpyPointSetMesh): - coords = pts.tolist() - features.append( - { - "type": "Feature", - "geometry": {"type": "MultiPoint", "coordinates": coords}, - "properties": base_props, - } - ) - elif isinstance(mesh, NumpyPolylineMesh): - for seg in _parse_vtk_flat_lines(mesh.lines): - if len(seg) < 2: - continue - coords = pts[seg].tolist() - features.append( - { - "type": "Feature", - "geometry": {"type": "LineString", "coordinates": coords}, - "properties": base_props, - } - ) - else: - # NumpySurfaceMesh / NumpyVolumeMesh - for face in _parse_vtk_flat_faces(_get_faces_or_cells(mesh)): - if len(face) < 3: - continue - coords = pts[face].tolist() - coords.append(coords[0]) # close ring - features.append( - { - "type": "Feature", - "geometry": {"type": "Polygon", "coordinates": [coords]}, - "properties": base_props, - } - ) - else: - # AbstractMesh legacy path - indices = mesh.get_indices() - for elem_idx, elem in enumerate(indices): - if isinstance(mesh, PolylineSetMesh): - if len(elem) < 2: - continue - coords = [list(pts[i]) for i in elem] - features.append( - { - "type": "Feature", - "geometry": {"type": "LineString", "coordinates": coords}, - "properties": {**base_props, "element_index": elem_idx}, - } - ) - elif isinstance(mesh, SurfaceMesh): - if len(elem) < 3: - continue - coords = [list(pts[i]) for i in elem] - coords.append(coords[0]) - features.append( - { - "type": "Feature", - "geometry": {"type": "Polygon", "coordinates": [coords]}, - "properties": {**base_props, "element_index": elem_idx}, - } - ) - - json.dump({"type": "FeatureCollection", "features": features}, out, indent=options.indent) - - -# --------------------------------------------------------------------------- -# VTK export — private helpers -# --------------------------------------------------------------------------- - - -def _b64_vtk(arr: np.ndarray) -> str: - """Base64-encode a numpy array for VTK XML inline binary format. - - VTK prepends a 4-byte uint32 header with the byte count of the payload. - """ - raw = arr.tobytes() - header = struct.pack(" str: - """Return a VTK XML ```` element string (base64 inline).""" - return ( - f'' - f"{_b64_vtk(arr)}" - f"" - ) - - -def _collect_vtk_geometry( - patches: List[Any], - use_crs_displacement: bool, - workspace: Any, -) -> tuple: - """Merge all patches into flat VTK geometry arrays. - - Returns: - (all_pts, poly_conn, poly_off, line_conn, line_off, - vert_conn, vert_off, cell_types, patch_meta) - - *patch_meta* is a list of ``(source_uuid, n_cells)`` tuples used to - assign per-cell colour data. - """ - from energyml.utils.data.mesh import PolylineSetMesh, SurfaceMesh - from energyml.utils.data.mesh_numpy import NumpyMesh, NumpyPointSetMesh, NumpyPolylineMesh - - all_pts: List[np.ndarray] = [] - poly_conn: List[int] = [] - poly_off: List[int] = [] - line_conn: List[int] = [] - line_off: List[int] = [] - vert_conn: List[int] = [] - vert_off: List[int] = [] - cell_types: List[int] = [] - patch_meta: List[tuple] = [] # (source_uuid, cell_count) - - pt_offset = 0 - - for mesh in patches: - pts = _get_export_points(mesh, use_crs_displacement, workspace) - all_pts.append(np.asarray(pts, dtype=np.float64).reshape(-1, 3)) - source_uuid = getattr(mesh, "source_uuid", None) - cell_count = 0 - - if isinstance(mesh, NumpyMesh): - if isinstance(mesh, NumpyPointSetMesh): - for i in range(len(pts)): - vert_conn.append(i + pt_offset) - vert_off.append(len(vert_conn)) - cell_types.append(_VTK_VERTEX) - cell_count += 1 - elif isinstance(mesh, NumpyPolylineMesh): - for seg in _parse_vtk_flat_lines(mesh.lines): - for vi in seg: - line_conn.append(int(vi) + pt_offset) - line_off.append(len(line_conn)) - cell_types.append(_VTK_POLY_LINE) - cell_count += 1 - else: - faces_arr = _get_faces_or_cells(mesh) - for face in _parse_vtk_flat_faces(faces_arr): - nv = len(face) - for vi in face: - poly_conn.append(int(vi) + pt_offset) - poly_off.append(len(poly_conn)) - cell_types.append(_VTK_TRIANGLE if nv == 3 else _VTK_POLYGON) - cell_count += 1 - else: - # AbstractMesh legacy - indices = mesh.get_indices() - if isinstance(mesh, PolylineSetMesh): - for line in indices: - for vi in line: - line_conn.append(int(vi) + pt_offset) - line_off.append(len(line_conn)) - cell_types.append(_VTK_POLY_LINE) - cell_count += 1 - else: - for face in indices: - nv = len(face) - for vi in face: - poly_conn.append(int(vi) + pt_offset) - poly_off.append(len(poly_conn)) - cell_types.append(_VTK_TRIANGLE if nv == 3 else _VTK_POLYGON) - cell_count += 1 - - pt_offset += len(pts) - patch_meta.append((source_uuid, cell_count)) - - merged_pts = np.concatenate(all_pts) if all_pts else np.empty((0, 3), dtype=np.float64) - return ( - merged_pts, - np.array(poly_conn, dtype=np.int64), - np.array(poly_off, dtype=np.int64), - np.array(line_conn, dtype=np.int64), - np.array(line_off, dtype=np.int64), - np.array(vert_conn, dtype=np.int64), - np.array(vert_off, dtype=np.int64), - np.array(cell_types, dtype=np.uint8), - patch_meta, - ) - - -def _build_color_scalars( - patch_meta: List[tuple], - contexts: Optional[Dict[str, Any]], - total_cells: int, -) -> Optional[np.ndarray]: - """Build a ``(total_cells, 4)`` float32 RGBA array, or None when no colors found.""" - if not contexts: - return None - colors = np.full((total_cells, 4), 0.8, dtype=np.float32) - colors[:, 3] = 1.0 - any_found = False - cell_idx = 0 - for source_uuid, n_cells in patch_meta: - rgba = _get_context_color(source_uuid, contexts) - if rgba is not None: - any_found = True - r, g, b, a = rgba - colors[cell_idx : cell_idx + n_cells, 0] = r / 255.0 - colors[cell_idx : cell_idx + n_cells, 1] = g / 255.0 - colors[cell_idx : cell_idx + n_cells, 2] = b / 255.0 - colors[cell_idx : cell_idx + n_cells, 3] = a / 255.0 - cell_idx += n_cells - return colors if any_found else None - - -# --------------------------------------------------------------------------- -# VTK export — legacy (ASCII / binary) -# --------------------------------------------------------------------------- - - -def _export_vtk_legacy( - patches: List[Any], - out: BinaryIO, - options: VTKExportOptions, - contexts: Optional[Dict[str, Any]], - workspace: Any, -) -> None: - ascii_mode = options.vtk_format == VTKFormat.LEGACY_ASCII - ( - all_pts, - poly_conn, - poly_off, - line_conn, - line_off, - vert_conn, - vert_off, - cell_types, - patch_meta, - ) = _collect_vtk_geometry(patches, True, workspace) - - n_pts = len(all_pts) - n_poly = len(poly_off) - n_line = len(line_off) - n_vert = len(vert_off) - - def _unflatten(conn: np.ndarray, offs: np.ndarray) -> List[List[int]]: - result = [] - prev = 0 - for o in offs: - result.append(conn[prev:o].tolist()) - prev = o - return result - - polygons = _unflatten(poly_conn, poly_off) - lines = _unflatten(line_conn, line_off) - verts = _unflatten(vert_conn, vert_off) - - out.write(b"# vtk DataFile Version 3.0\n") - out.write(f"{options.dataset_name}\n".encode()) - out.write(b"ASCII\n" if ascii_mode else b"BINARY\n") - out.write(b"DATASET POLYDATA\n") - - if ascii_mode: - out.write(f"POINTS {n_pts} float\n".encode()) - for pt in all_pts: - out.write(f"{pt[0]} {pt[1]} {pt[2]}\n".encode()) - else: - out.write(f"POINTS {n_pts} float\n".encode()) - out.write(all_pts.astype(">f4").tobytes()) - out.write(b"\n") - - def _write_section(name: str, cells: List[List[int]]) -> None: - if not cells: - return - total = sum(len(c) + 1 for c in cells) - out.write(f"{name} {len(cells)} {total}\n".encode()) - if ascii_mode: - for c in cells: - out.write(f"{len(c)} {' '.join(str(i) for i in c)}\n".encode()) - else: - for c in cells: - row = np.array([len(c)] + c, dtype=np.int32).byteswap().astype(">i4") - out.write(row.tobytes()) - out.write(b"\n") - - _write_section("POLYGONS", polygons) - _write_section("LINES", lines) - _write_section("VERTICES", verts) - - total_cells = n_poly + n_line + n_vert - if total_cells > 0 and contexts: - colors = _build_color_scalars(patch_meta, contexts, total_cells) - if colors is not None: - out.write(f"CELL_DATA {total_cells}\n".encode()) - out.write(b"COLOR_SCALARS patch_color 4\n") - if ascii_mode: - for row in colors: - out.write(f"{row[0]:.6f} {row[1]:.6f} {row[2]:.6f} {row[3]:.6f}\n".encode()) - else: - out.write(colors.astype(">f4").tobytes()) - out.write(b"\n") - - -# --------------------------------------------------------------------------- -# VTK export — XML VTU -# --------------------------------------------------------------------------- - - -def _export_vtk_vtu( - patches: List[Any], - out: BinaryIO, - options: VTKExportOptions, - contexts: Optional[Dict[str, Any]], - workspace: Any, -) -> None: - """Write VTK XML UnstructuredGrid (.vtu).""" - ( - all_pts, - poly_conn, - poly_off, - line_conn, - line_off, - vert_conn, - vert_off, - cell_types, - patch_meta, - ) = _collect_vtk_geometry(patches, True, workspace) - - # Build a single merged connectivity / offsets / types for UnstructuredGrid. - conn_parts: List[np.ndarray] = [] - off_parts: List[int] = [] - types_list: List[int] = [] - running = 0 - - def _add_vtu_section(conn: np.ndarray, offs: np.ndarray, default_type: int) -> None: - nonlocal running - prev = 0 - for o in offs: - seg = conn[prev:o] - conn_parts.append(seg) - running += len(seg) - off_parts.append(running) - types_list.append(default_type) - prev = o - - _add_vtu_section(vert_conn, vert_off, _VTK_VERTEX) - _add_vtu_section(line_conn, line_off, _VTK_POLY_LINE) - - # Polygons: honour per-cell type from cell_types array (triangle vs polygon). - n_verts_cells = len(vert_off) - n_lines_cells = len(line_off) - prev = 0 - for poly_i, o in enumerate(poly_off): - seg = poly_conn[prev:o] - conn_parts.append(seg) - running += len(seg) - off_parts.append(running) - abs_idx = n_verts_cells + n_lines_cells + poly_i - types_list.append(int(cell_types[abs_idx]) if abs_idx < len(cell_types) else _VTK_POLYGON) - prev = o - - all_conn = ( - np.concatenate([np.asarray(p, dtype=np.int64) for p in conn_parts]) - if conn_parts - else np.empty(0, dtype=np.int64) - ) - all_off = np.array(off_parts, dtype=np.int64) - all_types = np.array(types_list, dtype=np.uint8) - n_cells = len(all_types) - n_pts = len(all_pts) - - xml_lines: List[str] = [ - '', - '', - " ", - f' ', - " ", - " " + _vtk_xml_data_array("Points", all_pts.astype(np.float32).ravel(), 3, "Float32"), - " ", - " ", - " " + _vtk_xml_data_array("connectivity", all_conn, 1, "Int64"), - " " + _vtk_xml_data_array("offsets", all_off, 1, "Int64"), - " " + _vtk_xml_data_array("types", all_types, 1, "UInt8"), - " ", - ] - - if contexts and n_cells > 0: - colors = _build_color_scalars(patch_meta, contexts, n_cells) - if colors is not None: - xml_lines.append(" ") - xml_lines.append(" " + _vtk_xml_data_array("patch_color", colors.ravel(), 4, "Float32")) - xml_lines.append(" ") - - xml_lines += [" ", " ", ""] - out.write("\n".join(xml_lines).encode("utf-8")) - - -# --------------------------------------------------------------------------- -# VTK export — XML VTP -# --------------------------------------------------------------------------- - - -def _export_vtk_vtp( - patches: List[Any], - out: BinaryIO, - options: VTKExportOptions, - contexts: Optional[Dict[str, Any]], - workspace: Any, -) -> None: - """Write VTK XML PolyData (.vtp).""" - ( - all_pts, - poly_conn, - poly_off, - line_conn, - line_off, - vert_conn, - vert_off, - cell_types, - patch_meta, - ) = _collect_vtk_geometry(patches, True, workspace) - - n_pts = len(all_pts) - n_polys = len(poly_off) - n_lines = len(line_off) - n_verts = len(vert_off) - total_cells = n_polys + n_lines + n_verts - - xml_lines: List[str] = [ - '', - '', - " ", - ( - f' ' - ), - " ", - " " + _vtk_xml_data_array("Points", all_pts.astype(np.float32).ravel(), 3, "Float32"), - " ", - ] - - def _topo_section(tag: str, conn: np.ndarray, offs: np.ndarray) -> List[str]: - return [ - f" <{tag}>", - " " + _vtk_xml_data_array("connectivity", conn, 1, "Int64"), - " " + _vtk_xml_data_array("offsets", offs, 1, "Int64"), - f" ", - ] - - if n_polys: - xml_lines.extend(_topo_section("Polys", poly_conn, poly_off)) - if n_lines: - xml_lines.extend(_topo_section("Lines", line_conn, line_off)) - if n_verts: - xml_lines.extend(_topo_section("Verts", vert_conn, vert_off)) - - if contexts and total_cells > 0: - colors = _build_color_scalars(patch_meta, contexts, total_cells) - if colors is not None: - xml_lines.append(" ") - xml_lines.append(" " + _vtk_xml_data_array("patch_color", colors.ravel(), 4, "Float32")) - xml_lines.append(" ") - - xml_lines += [" ", " ", ""] - out.write("\n".join(xml_lines).encode("utf-8")) - - -# --------------------------------------------------------------------------- -# VTK export — public entry point -# --------------------------------------------------------------------------- - - -def export_vtk( - mesh_list: Any, - out: BinaryIO, - options: Optional[VTKExportOptions] = None, - contexts: Optional[Dict[str, "RepresentationContext"]] = None, - use_crs_displacement: bool = True, -) -> None: - """Export mesh data to a VTK format. - - The sub-format is controlled by ``options.vtk_format`` (default: - ``VTKFormat.LEGACY_ASCII``). Supported variants: - - * **LEGACY_ASCII** — VTK 3.0 POLYDATA, ASCII encoding - * **LEGACY_BINARY** — VTK 3.0 POLYDATA, big-endian binary encoding - * **VTU** — VTK XML UnstructuredGrid (``.vtu``), base64 inline binary - * **VTP** — VTK XML PolyData (``.vtp``), base64 inline binary - - :param mesh_list: Meshes to export. - :param out: Binary output stream. - :param options: VTK export options. - :param contexts: Optional colour context dict keyed by ``source_uuid``. - :param use_crs_displacement: Apply CRS displacement to ``NumpyMesh`` points. - """ - if options is None: - options = VTKExportOptions() - - patches = _normalize_to_patches(mesh_list) - # Pass workspace only when CRS displacement is actually requested. - workspace = _workspace_from_contexts(contexts) if use_crs_displacement else None - - fmt = options.vtk_format - if fmt in (VTKFormat.LEGACY_ASCII, VTKFormat.LEGACY_BINARY): - _export_vtk_legacy(patches, out, options, contexts, workspace) - elif fmt == VTKFormat.VTU: - _export_vtk_vtu(patches, out, options, contexts, workspace) - elif fmt == VTKFormat.VTP: - _export_vtk_vtp(patches, out, options, contexts, workspace) - else: # pragma: no cover - raise ValueError(f"Unknown VTKFormat: {fmt}") - - -# --------------------------------------------------------------------------- -# STL export -# --------------------------------------------------------------------------- - - -def export_stl( - mesh_list: Any, - out: BinaryIO, - options: Optional[STLExportOptions] = None, - use_crs_displacement: bool = True, -) -> None: - """Export triangulated mesh data to STL format (binary or ASCII). - - Non-triangular polygons are fan-triangulated (vertex 0 + consecutive pairs). - Polylines and point sets are silently skipped. - - :param mesh_list: Meshes to export. - :param out: Binary output stream. - :param options: STL export options. - :param use_crs_displacement: Apply CRS displacement to ``NumpyMesh`` points. - """ - from energyml.utils.data.mesh import SurfaceMesh - from energyml.utils.data.mesh_numpy import NumpyMesh, NumpyPolylineMesh, NumpyPointSetMesh - - if options is None: - options = STLExportOptions(binary=True) - - patches = _normalize_to_patches(mesh_list) - # STL carries no colour / context; workspace not needed unless CRS is requested. - workspace = None # CRS requires a workspace — callers may read with CRS pre-applied. - - all_triangles: List[tuple] = [] - - for mesh in patches: - if isinstance(mesh, (NumpyPolylineMesh, NumpyPointSetMesh)): - continue # STL is surface-only - pts = _get_export_points(mesh, use_crs_displacement, workspace) - pts_np = np.asarray(pts, dtype=np.float64).reshape(-1, 3) - - if isinstance(mesh, NumpyMesh): - face_list = _parse_vtk_flat_faces(_get_faces_or_cells(mesh)) - else: - if not isinstance(mesh, SurfaceMesh): - continue - face_list = mesh.get_indices() - - for face in face_list: - face = list(face) - if len(face) < 3: - continue - if len(face) == 3: - all_triangles.append((pts_np[face[0]], pts_np[face[1]], pts_np[face[2]])) - else: - # Fan triangulation for quads and polygons - for j in range(1, len(face) - 1): - all_triangles.append((pts_np[face[0]], pts_np[face[j]], pts_np[face[j + 1]])) - - if options.binary: - _export_stl_binary(all_triangles, out) - else: - _export_stl_ascii(all_triangles, out, options.ascii_precision) - - -def _compute_normal(p0: np.ndarray, p1: np.ndarray, p2: np.ndarray) -> np.ndarray: - v1, v2 = p1 - p0, p2 - p0 - n = np.cross(v1, v2) - norm = np.linalg.norm(n) - return n / norm if norm > 0 else np.zeros(3) - - -def _export_stl_binary(triangles: List[tuple], out: BinaryIO) -> None: - header = b"Binary STL file generated by energyml-utils" + b"\0" * (80 - 44) - out.write(header) - out.write(struct.pack(" None: - out.write(b"solid mesh\n") - for p0, p1, p2 in triangles: - normal = _compute_normal(p0, p1, p2) - out.write( - f" facet normal {normal[0]:.{precision}e} {normal[1]:.{precision}e} {normal[2]:.{precision}e}\n".encode() - ) - out.write(b" outer loop\n") - for pt in (p0, p1, p2): - out.write(f" vertex {pt[0]:.{precision}e} {pt[1]:.{precision}e} {pt[2]:.{precision}e}\n".encode()) - out.write(b" endloop\n endfacet\n") - out.write(b"endsolid mesh\n") - - -# --------------------------------------------------------------------------- -# High-level dispatcher -# --------------------------------------------------------------------------- - - -def export_mesh( - mesh_list: Any, - output_path: Union[str, Path], - format: Optional[ExportFormat] = None, - options: Optional[ExportOptions] = None, - contexts: Optional[Dict[str, "RepresentationContext"]] = None, - use_crs_displacement: bool = True, -) -> None: - """Export mesh data to a file. - - Format is auto-detected from the file extension when *format* is None. - Supported extensions: ``.obj``, ``.geojson``, ``.vtk``, ``.vtu``, - ``.vtp``, ``.stl``. - - :param mesh_list: Meshes to export. - :param output_path: Destination file path. - :param format: Explicit format; auto-detected from extension when None. - :param options: Format-specific options. - :param contexts: Color / metadata context dict. - :param use_crs_displacement: Apply CRS displacement to ``NumpyMesh`` points. - """ - path = Path(output_path) - if format is None: - format = ExportFormat.from_extension(path.suffix) - - if format == ExportFormat.GEOJSON: - with path.open("w", encoding="utf-8") as f: - export_geojson(mesh_list, f, options, contexts, use_crs_displacement) - return - - # All remaining formats use binary streams - with path.open("wb") as f: - if format == ExportFormat.OBJ: - if contexts: - mtl_path = path.with_suffix(".mtl") - with mtl_path.open("wb") as mf: - export_obj(mesh_list, f, path.stem, contexts, mf, use_crs_displacement) - else: - export_obj(mesh_list, f, path.stem, None, None, use_crs_displacement) - elif format == ExportFormat.STL: - export_stl(mesh_list, f, options, use_crs_displacement) - elif format == ExportFormat.VTK: - export_vtk(mesh_list, f, options, contexts, use_crs_displacement) - elif format == ExportFormat.VTU: - vtk_opts = options if isinstance(options, VTKExportOptions) else VTKExportOptions() - vtk_opts.vtk_format = VTKFormat.VTU - export_vtk(mesh_list, f, vtk_opts, contexts, use_crs_displacement) - elif format == ExportFormat.VTP: - vtk_opts = options if isinstance(options, VTKExportOptions) else VTKExportOptions() - vtk_opts.vtk_format = VTKFormat.VTP - export_vtk(mesh_list, f, vtk_opts, contexts, use_crs_displacement) - else: - raise ValueError(f"Unsupported format: {format}") - - -# --------------------------------------------------------------------------- -# UI Helper Functions -# --------------------------------------------------------------------------- - - -def supported_formats() -> List[str]: - """Return all supported export format extensions.""" - return ExportFormat.all_extensions() - - -def format_description(format: Union[str, ExportFormat]) -> str: - """Return a human-readable description of *format*.""" - if isinstance(format, str): - format = ExportFormat.from_extension(format) - descriptions = { - ExportFormat.OBJ: "Wavefront OBJ — 3D geometry with optional .mtl colour", - ExportFormat.GEOJSON: "GeoJSON — geographic data (lines, polygons, point clouds)", - ExportFormat.VTK: "VTK Legacy (ASCII or binary) — POLYDATA format", - ExportFormat.VTU: "VTK XML UnstructuredGrid (.vtu) — volumes + mixed topologies", - ExportFormat.VTP: "VTK XML PolyData (.vtp) — surfaces and polylines", - ExportFormat.STL: "STL — stereolithography (triangles only)", - } - return descriptions.get(format, "Unknown format") - - -def format_filter_string(format: Union[str, ExportFormat]) -> str: - """Return a file-dialog filter string (e.g. ``"VTU Files (*.vtu)"``).""" - if isinstance(format, str): - format = ExportFormat.from_extension(format) - filters = { - ExportFormat.OBJ: "OBJ Files (*.obj)", - ExportFormat.GEOJSON: "GeoJSON Files (*.geojson)", - ExportFormat.VTK: "VTK Files (*.vtk)", - ExportFormat.VTU: "VTK XML UnstructuredGrid Files (*.vtu)", - ExportFormat.VTP: "VTK XML PolyData Files (*.vtp)", - ExportFormat.STL: "STL Files (*.stl)", - } - return filters.get(format, "All Files (*.*)") - - -def all_formats_filter_string() -> str: - """Return a ``;;``-joined filter string for all supported formats.""" - return ";;".join(format_filter_string(fmt) for fmt in ExportFormat) - - -def get_format_options_class(format: Union[str, ExportFormat]) -> Optional[type]: - """Return the options class for *format*, or None.""" - if isinstance(format, str): - format = ExportFormat.from_extension(format) - return { - ExportFormat.STL: STLExportOptions, - ExportFormat.VTK: VTKExportOptions, - ExportFormat.VTU: VTKExportOptions, - ExportFormat.VTP: VTKExportOptions, - ExportFormat.GEOJSON: GeoJSONExportOptions, - }.get(format) - - -def supports_lines(format: Union[str, ExportFormat]) -> bool: - """Return True when *format* can represent polyline primitives.""" - if isinstance(format, str): - format = ExportFormat.from_extension(format) - return format in {ExportFormat.OBJ, ExportFormat.GEOJSON, ExportFormat.VTK, ExportFormat.VTU, ExportFormat.VTP} - - -def supports_triangles(format: Union[str, ExportFormat]) -> bool: - """Return True when *format* can represent triangle / polygon primitives.""" - return True # All formats support triangles - - -def supports_pointsets(format: Union[str, ExportFormat]) -> bool: - """Return True when *format* can represent point-cloud primitives.""" - if isinstance(format, str): - format = ExportFormat.from_extension(format) - return format in {ExportFormat.OBJ, ExportFormat.GEOJSON, ExportFormat.VTK, ExportFormat.VTU, ExportFormat.VTP} diff --git a/energyml-utils/src/energyml/utils/data/export/__init__.py b/energyml-utils/src/energyml/utils/data/export/__init__.py new file mode 100644 index 0000000..66594e2 --- /dev/null +++ b/energyml-utils/src/energyml/utils/data/export/__init__.py @@ -0,0 +1,108 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +""" +Mesh export to various file formats. + +One module per format — :mod:`obj`, :mod:`off`, :mod:`geojson`, :mod:`vtk`, :mod:`stl` — each +declaring what it can do through a :class:`~._registry.FormatSpec`. The dispatcher +:func:`export_mesh` and the UI helpers (``format_description``, ``format_filter_string``, +``supports_lines``, …) all read that registry, so adding a format means adding a module and one +registration, not editing six places. + +Both mesh hierarchies are accepted by every export function: the legacy +:class:`~energyml.utils.data.mesh.AbstractMesh` and the numpy +:class:`~energyml.utils.data.mesh_numpy.NumpyMesh` / ``NumpyMultiMesh``. + +Coordinate frames +----------------- +Every writer takes ``frame=`` and ``origin_shift=``, not just the GeoJSON one: + +* ``frame`` — :class:`~energyml.utils.data.crs.PointFrame`: ``LOCAL``, ``PROJECTED`` (default for + the 3-D formats) or ``WGS84`` (default for GeoJSON, as RFC 7946 requires). Each mesh records the + frame it is already in, so only the missing stages are applied and no transform runs twice. +* ``origin_shift`` — ``None``, ``"auto"`` or an explicit ``(dx, dy, dz)``. Projected coordinates + carry 6-7 significant digits, which loses precision once a viewer reads the file as float32; + recentring restores it. ``"auto"`` is resolved **once** over the whole export so the patches keep + their relative positions. + +The legacy ``use_crs_displacement`` flag is still accepted and simply selects the default frame +(``PROJECTED`` when True, ``LOCAL`` when False). +""" + +from energyml.utils.data.export._base import ( + ExportFormat, + ExportOptions, + GeoJSONExportOptions, + EmptyMeshError, + drop_empty_patches, + STLExportOptions, + VTKExportOptions, + VTKFormat, + resolve_origin_shift, +) +from energyml.utils.data.export._registry import ( + FormatSpec, + all_formats_filter_string, + export_mesh, + format_description, + format_filter_string, + get_format_options_class, + get_format_spec, + register_format, + registered_formats, + supported_formats, + supports_lines, + supports_pointsets, + supports_triangles, +) + +# Importing the format modules is what populates the registry. +# The private helpers are re-exported on purpose: they were importable from the old +# `export` module and external code (and mesh.py) still reaches for them. +from energyml.utils.data.export.geojson import ( # noqa: E402,F401 + _feature_id, + _geojson_bbox, + _geojson_crs_members, + _prepare_geojson_points, + export_geojson, +) +from energyml.utils.data.export.obj import export_obj # noqa: E402 +from energyml.utils.data.export.off import export_off, export_off_part # noqa: E402 +from energyml.utils.data.export.stl import export_stl # noqa: E402 +from energyml.utils.data.export.vtk import export_vtk # noqa: E402 + +__all__ = [ + # Formats / options + "ExportFormat", + "ExportOptions", + "GeoJSONExportOptions", + "EmptyMeshError", + "drop_empty_patches", + "STLExportOptions", + "VTKExportOptions", + "VTKFormat", + # Registry + "FormatSpec", + "register_format", + "get_format_spec", + "registered_formats", + # Writers + "export_mesh", + "export_obj", + "export_off", + "export_off_part", + "export_geojson", + "export_vtk", + "export_stl", + # Frame helpers + "resolve_origin_shift", + # UI helpers + "supported_formats", + "format_description", + "format_filter_string", + "all_formats_filter_string", + "get_format_options_class", + "supports_lines", + "supports_triangles", + "supports_pointsets", +] diff --git a/energyml-utils/src/energyml/utils/data/export/_base.py b/energyml-utils/src/energyml/utils/data/export/_base.py new file mode 100644 index 0000000..ace781c --- /dev/null +++ b/energyml-utils/src/energyml/utils/data/export/_base.py @@ -0,0 +1,413 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""Shared building blocks of the export package: formats, options and geometry helpers.""" + +from __future__ import annotations + +import logging +from enum import Enum +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import numpy as np + +if TYPE_CHECKING: + from energyml.utils.data.crs import PointFrame + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# VTK cell-type constants (subset) +# --------------------------------------------------------------------------- +_VTK_VERTEX = 1 +_VTK_POLY_LINE = 4 +_VTK_TRIANGLE = 5 +_VTK_POLYGON = 7 +_VTK_TETRA = 10 +_VTK_HEXAHEDRON = 12 +# --------------------------------------------------------------------------- +# Enumerations / option classes +# --------------------------------------------------------------------------- + + +#: Header line stamped at the top of the text-based mesh formats. +_FILE_HEADER: bytes = b"# file exported by energyml-utils python module (Geosiris)\n" + + +class ExportFormat(Enum): + """Supported mesh export formats.""" + + OBJ = "obj" + OFF = "off" + GEOJSON = "geojson" + VTK = "vtk" + VTU = "vtu" + VTP = "vtp" + STL = "stl" + + @classmethod + def from_extension(cls, extension: str) -> "ExportFormat": + """Get format from file extension.""" + ext = extension.lower().lstrip(".") + for fmt in cls: + if fmt.value == ext: + return fmt + raise ValueError(f"Unsupported file extension: {extension}") + + @classmethod + def all_extensions(cls) -> List[str]: + """Get all supported file extensions.""" + return [fmt.value for fmt in cls] + + +class ExportOptions: + """Base class for export options.""" + + +class STLExportOptions(ExportOptions): + """Options for STL export.""" + + def __init__(self, binary: bool = True, ascii_precision: int = 6): + """ + :param binary: If True, export as binary STL; if False, export as ASCII STL. + :param ascii_precision: Number of decimal places for ASCII format. + """ + self.binary = binary + self.ascii_precision = ascii_precision + + +class VTKFormat(Enum): + """Sub-format selector for VTK export.""" + + LEGACY_ASCII = "legacy_ascii" + """VTK legacy format, ASCII encoding (version 3.0).""" + + LEGACY_BINARY = "legacy_binary" + """VTK legacy format, big-endian binary encoding (version 3.0).""" + + VTU = "vtu" + """VTK XML UnstructuredGrid (.vtu) — best for volumetric meshes.""" + + VTP = "vtp" + """VTK XML PolyData (.vtp) — best for surface / polyline meshes.""" + + +class VTKExportOptions(ExportOptions): + """Options for VTK export.""" + + def __init__( + self, + vtk_format: VTKFormat = VTKFormat.LEGACY_ASCII, + dataset_name: str = "mesh", + # Legacy compatibility: binary=True is equivalent to vtk_format=VTKFormat.LEGACY_BINARY + binary: bool = False, + ): + """ + :param vtk_format: VTK sub-format (legacy ASCII, legacy binary, VTU, VTP). + :param dataset_name: Dataset name embedded in legacy VTK header or XML title. + :param binary: Deprecated shorthand; when True, forces LEGACY_BINARY sub-format. + """ + self.dataset_name = dataset_name + if binary and vtk_format == VTKFormat.LEGACY_ASCII: + # Honour the legacy binary=True flag so old call-sites still work. + self.vtk_format = VTKFormat.LEGACY_BINARY + else: + self.vtk_format = vtk_format + + # Backward-compat property so code that reads ``options.binary`` still works. + @property + def binary(self) -> bool: + return self.vtk_format == VTKFormat.LEGACY_BINARY + + +class GeoJSONExportOptions(ExportOptions): + """Options for GeoJSON export.""" + + def __init__( + self, + indent: Optional[int] = 2, + properties: Optional[dict] = None, + to_wgs84: bool = True, + include_metadata: bool = True, + use_network: bool = False, + projected_epsg_code: Optional[int] = None, + vertical_epsg_code: Optional[int] = None, + explode_elements: bool = False, + ): + """ + :param indent: JSON indentation level (None for compact output). + :param properties: Extra properties merged into every feature. + :param to_wgs84: When True (default), coordinates are reprojected to WGS84 + (longitude, latitude, ellipsoidal height) as required by RFC 7946. + Silently disabled when no EPSG code can be found or when ``pyproj`` + (extra ``crs``) is not installed — the source CRS is then advertised + in the output instead. + :param include_metadata: When True (default), the ``uuid``, ``qualified_type`` and + ``Citation`` fields of the source object are written in the + properties of every feature. + :param use_network: Allow PROJ to download the geoid grids needed by vertical datum + transformations. Without them the height conversion is skipped. + :param projected_epsg_code: Force the horizontal EPSG code instead of reading it from the CRS. + :param vertical_epsg_code: Force the vertical EPSG code instead of reading it from the CRS. + :param explode_elements: Emit one feature per triangle / per line segment instead of one + feature per patch. Off by default: a patch is the unit a RESQML + representation is made of, and exploding it repeats the whole + metadata block — uuid, citation, EPSG codes — on every element, + which on a triangulated surface means one copy per triangle. + """ + self.indent = indent + self.properties = properties or {} + self.to_wgs84 = to_wgs84 + self.include_metadata = include_metadata + self.use_network = use_network + self.projected_epsg_code = projected_epsg_code + self.vertical_epsg_code = vertical_epsg_code + self.explode_elements = explode_elements + + +# --------------------------------------------------------------------------- +# Private helpers +# --------------------------------------------------------------------------- + + +def _normalize_to_patches(meshes: Any) -> List[Any]: + """Flatten *meshes* into a list of individual mesh patches. + + Handles: + - :class:`NumpyMultiMesh` → calls ``flat_patches()`` + - Single :class:`NumpyMesh` → ``[mesh]`` + - ``list`` / ``tuple`` → recursive + - :class:`AbstractMesh` → passthrough as ``[mesh]`` + """ + from energyml.utils.data.mesh_numpy import NumpyMesh, NumpyMultiMesh + + if isinstance(meshes, NumpyMultiMesh): + return meshes.flat_patches() + if isinstance(meshes, NumpyMesh): + return [meshes] + if isinstance(meshes, (list, tuple)): + result: List[Any] = [] + for m in meshes: + result.extend(_normalize_to_patches(m)) + return result + # AbstractMesh or unknown — pass through as single element + return [meshes] + + +class EmptyMeshError(ValueError): + """Raised when an export would produce a file with no geometry in it.""" + + +def drop_empty_patches(meshes: Any, raise_when_empty: bool = False) -> List[Any]: + """Return the patches of *meshes* that actually carry points. + + A representation whose external arrays could not be read still yields patches — with zero + points. Writing them produced a valid but useless file: ``{"type": "FeatureCollection", + "features": []}``, 54 bytes, with nothing to say that the data was missing rather than + absent. Worse, a partially readable object exported its readable patches next to empty ones. + + :param raise_when_empty: raise :class:`EmptyMeshError` when *nothing* survives, so the caller + fails loudly instead of writing an empty file. + """ + patches = _normalize_to_patches(meshes) + kept, dropped = [], 0 + for patch in patches: + points = getattr(patch, "point_list", None) + if points is None: + points = getattr(patch, "points", None) + if points is None or len(points) == 0: + dropped += 1 + continue + kept.append(patch) + + if dropped: + logger.warning( + f"{dropped} of {len(patches)} patch(es) hold no point and were dropped from the export " + "— their external arrays were most likely unreadable." + ) + if not kept and raise_when_empty: + raise EmptyMeshError( + f"Nothing to export: all {len(patches)} patch(es) are empty. The geometry could not be " + "read — check that the external (HDF5) arrays are reachable from the workspace." + ) + return kept + + +def _parse_vtk_flat_faces(flat: np.ndarray) -> List[np.ndarray]: + """Decode VTK flat face array ``[nv, v0, …, nv, v0, …]`` into a list of + per-face index arrays.""" + faces: List[np.ndarray] = [] + pos = 0 + flat = np.asarray(flat, dtype=np.int64) + while pos < len(flat): + nv = int(flat[pos]) + pos += 1 + if pos + nv > len(flat): + break + faces.append(flat[pos : pos + nv]) + pos += nv + return faces + + +def _parse_vtk_flat_lines(flat: np.ndarray) -> List[np.ndarray]: + """Decode VTK flat lines array ``[n, i0, i1, …, n, i0, …]`` into a list + of per-line index arrays.""" + lines: List[np.ndarray] = [] + pos = 0 + flat = np.asarray(flat, dtype=np.int64) + while pos < len(flat): + n = int(flat[pos]) + pos += 1 + if pos + n > len(flat): + break + lines.append(flat[pos : pos + n]) + pos += n + return lines + + +def _get_export_points( + mesh: Any, + use_crs_displacement: bool, + workspace: Any = None, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + use_network: bool = False, +) -> Tuple[np.ndarray, "PointFrame", Optional[tuple]]: + """Return ``(points, frame, applied_origin_shift)`` for *mesh* in the requested frame. + + The mesh carries the :class:`~energyml.utils.data.crs.PointFrame` its points are already in, + so only the missing pipeline stages are applied. That is what stops the double transform this + function used to cause: the readers apply the local → projected stage, and re-applying it here + whenever a workspace happened to be available shifted the geometry by the CRS origin twice. + + *mesh.points* is never mutated — the transform runs on a copy. + """ + from energyml.utils.data.crs import PointFrame, to_frame + from energyml.utils.data.mesh_numpy import NumpyMesh + + if isinstance(mesh, NumpyMesh): + points = mesh.points + current = mesh.frame + else: + # AbstractMesh — point_list is a list-of-lists; convert to ndarray for uniform handling + points = np.array(getattr(mesh, "point_list", []), dtype=np.float64) + current = getattr(mesh, "frame", PointFrame.LOCAL) + + target = frame if frame is not None else (PointFrame.PROJECTED if use_crs_displacement else PointFrame.LOCAL) + + if len(points) == 0 or (current is target and origin_shift is None): + return points, current, None + + crs_object = getattr(mesh, "crs_object", None) + crs = crs_object[0] if isinstance(crs_object, list) and crs_object else crs_object + crs_info = None + if crs is not None: + from energyml.utils.data.crs import extract_crs_info + + crs_info = extract_crs_info(crs, workspace) + + try: + framed = to_frame( + points, + crs_info, + target, + current, + origin_shift=origin_shift, + use_network=use_network, + inplace=False, # never mutate the mesh's own array + ) + return framed.points, framed.frame, framed.origin_shift + except Exception as exc: # pragma: no cover + logger.warning("Frame conversion to %s failed for %s: %s", target.value, getattr(mesh, "source_uuid", None), exc) + return points, current, None + + +def resolve_origin_shift( + patches: List[Any], + use_crs_displacement: bool, + workspace: Any, + frame: Optional["PointFrame"], + origin_shift: Optional[Any], + use_network: bool = False, +) -> Optional[tuple]: + """Turn an ``origin_shift`` option into an explicit ``(dx, dy, dz)`` vector. + + ``"auto"`` recentres the export on the bounding-box centre of **all** the patches together. + It has to be resolved once for the whole export: a per-patch centre would translate each + patch by a different vector and pull the model apart. + + Resolving ``"auto"`` costs one extra pass, since the bounding box has to be measured in the + target frame — that is the price of the option, and it is only paid when it is asked for. + ``None`` and an explicit vector are returned as-is, with no pass at all. + """ + if origin_shift is None: + return None + if not isinstance(origin_shift, str): + return tuple(float(v) for v in origin_shift) + if origin_shift != "auto": + raise ValueError(f"origin_shift must be None, 'auto', or a (dx, dy, dz) vector — got {origin_shift!r}") + + from energyml.utils.data.crs import compute_origin_shift + + framed = [ + _get_export_points(mesh, use_crs_displacement, workspace, frame, None, use_network)[0] for mesh in patches + ] + return compute_origin_shift(framed) + + +def _get_context_color( + source_uuid: Optional[str], + contexts: Optional[Dict[str, Any]], +) -> Optional[tuple]: + """Return an (r, g, b, a) tuple in 0–255 range for *source_uuid*, or None.""" + if not contexts or not source_uuid: + return None + ctx = contexts.get(source_uuid) + if ctx is None: + return None + try: + return ctx.primary_color.to_uint8() + except Exception as exc: # pragma: no cover + logger.debug("Failed to read color for %s: %s", source_uuid, exc) + return None + + +def _workspace_from_contexts(contexts: Optional[Dict[str, Any]]) -> Any: + """Return the workspace from the first available RepresentationContext.""" + if not contexts: + return None + for ctx in contexts.values(): + ws = getattr(ctx, "workspace", None) + if ws is not None: + return ws + return None + + +def _get_faces_or_cells(mesh: Any) -> np.ndarray: + """Return the face or cell connectivity array for a NumpyMesh. + + Uses ``mesh.faces`` when present and non-empty, then falls back to + ``mesh.cells``. Avoids the numpy-unsafe ``arr or other`` pattern which + raises ``ValueError`` for arrays with more than one element. + """ + faces = getattr(mesh, "faces", None) + if faces is not None and len(faces) > 0: + return faces + cells = getattr(mesh, "cells", None) + if cells is not None and len(cells) > 0: + return cells + return np.empty(0, dtype=np.int64) + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "ExportFormat", + "ExportOptions", + "STLExportOptions", + "VTKFormat", + "VTKExportOptions", + "GeoJSONExportOptions", + "EmptyMeshError", + "drop_empty_patches", + "resolve_origin_shift", +] diff --git a/energyml-utils/src/energyml/utils/data/export/_registry.py b/energyml-utils/src/energyml/utils/data/export/_registry.py new file mode 100644 index 0000000..fd281f8 --- /dev/null +++ b/energyml-utils/src/energyml/utils/data/export/_registry.py @@ -0,0 +1,252 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""Format registry and the :func:`export_mesh` dispatcher. + +Everything a format needs to be usable — how to write it, what options it takes, how to describe +it in a file dialog, which primitives it supports — is declared **once**, in a :class:`FormatSpec` +registered by the format's own module. Adding a format is therefore a new module plus one +:func:`register_format` call. + +Before, the same six formats were enumerated in an ``elif`` chain in ``export_mesh`` and in five +separate dictionaries (``format_description``, ``format_filter_string``, ``get_format_options_class``, +``supports_lines``, ``supports_pointsets``), so a new format meant editing six places and a missing +entry degraded silently to a default. +""" + +from __future__ import annotations + +import logging +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union + +from energyml.utils.data.export._base import ExportFormat, ExportOptions, drop_empty_patches + +if TYPE_CHECKING: + from energyml.utils.data.crs import PointFrame + from energyml.utils.data.representation_context import RepresentationContext + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class FormatSpec: + """Everything the export machinery knows about one output format.""" + + format: ExportFormat + """The enum member this spec describes.""" + + description: str + """Human-readable description, shown in UIs.""" + + filter_label: str + """File-dialog filter label, e.g. ``"OBJ Files (*.obj)"``.""" + + writer: Callable[..., None] + """ + The writing function. Called as + ``writer(mesh_list, out, options=..., contexts=..., use_crs_displacement=..., frame=..., + origin_shift=...)``. + """ + + binary: bool = True + """Whether *out* must be a binary stream. GeoJSON is the only text format.""" + + options_class: Optional[type] = None + """Options class accepted by the writer, when it takes one.""" + + supports_lines: bool = True + supports_triangles: bool = True + supports_pointsets: bool = True + + force_options: Optional[Dict[str, Any]] = None + """ + Option attributes forced before calling the writer. Used by ``.vtu`` / ``.vtp``, which share + the VTK writer but pin its sub-format. + """ + + companion_suffix: Optional[str] = None + """ + Suffix of a side-car file the format writes next to the main one, opened only when *contexts* + are provided. OBJ uses it for its ``.mtl`` material file. + """ + + +_REGISTRY: Dict[ExportFormat, FormatSpec] = {} + + +def register_format(spec: FormatSpec) -> FormatSpec: + """Register *spec*, replacing any previous entry for the same format.""" + _REGISTRY[spec.format] = spec + return spec + + +def get_format_spec(format: Union[str, ExportFormat]) -> FormatSpec: + """Return the :class:`FormatSpec` of *format*. + + :raises ValueError: when the format is unknown or its module was not imported. + """ + if isinstance(format, str): + format = ExportFormat.from_extension(format) + spec = _REGISTRY.get(format) + if spec is None: + raise ValueError( + f"No writer registered for format {format}. " f"Registered: {sorted(f.value for f in _REGISTRY)}." + ) + return spec + + +def registered_formats() -> List[ExportFormat]: + """Return the registered formats, in registration order.""" + return list(_REGISTRY) + + +# --------------------------------------------------------------------------- +# Dispatcher +# --------------------------------------------------------------------------- + + +def export_mesh( + mesh_list: Any, + output_path: Union[str, Path], + format: Optional[ExportFormat] = None, + options: Optional[ExportOptions] = None, + contexts: Optional[Dict[str, "RepresentationContext"]] = None, + use_crs_displacement: bool = True, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, +) -> None: + """Export mesh data to a file. + + Format is auto-detected from the file extension when *format* is None. + + :param mesh_list: Meshes to export. + :param output_path: Destination file path. + :param format: Explicit format; auto-detected from the extension when None. + :param options: Format-specific options. + :param contexts: Color / metadata context dict. + :param use_crs_displacement: Legacy switch selecting the default target frame + (``PointFrame.PROJECTED`` when True, ``PointFrame.LOCAL`` when False). Ignored when + *frame* is given. + :param frame: Target coordinate frame, e.g. ``PointFrame.WGS84``. Available for **every** + format, not only GeoJSON. + :param origin_shift: ``None``, ``"auto"``, or an explicit ``(dx, dy, dz)`` vector subtracted + from the coordinates. ``"auto"`` recentres on the bounding-box centre of the whole export, + computed once and applied identically to every patch. Useful for projected coordinates, + whose 6-7 significant digits lose precision when a viewer reads the file as float32. + """ + path = Path(output_path) + if format is None: + format = ExportFormat.from_extension(path.suffix) + + spec = get_format_spec(format) + + if spec.force_options: + if spec.options_class is not None and not isinstance(options, spec.options_class): + options = spec.options_class() + for attr, value in spec.force_options.items(): + setattr(options, attr, value) + + kwargs: Dict[str, Any] = { + "obj_name": path.stem, + "options": options, + "contexts": contexts, + "use_crs_displacement": use_crs_displacement, + "frame": frame, + "origin_shift": origin_shift, + } + + mesh_list = drop_empty_patches(mesh_list, raise_when_empty=True) + + with ExitStack() as stack: + if spec.binary: + out = stack.enter_context(path.open("wb")) + else: + out = stack.enter_context(path.open("w", encoding="utf-8")) + if spec.companion_suffix and contexts: + companion_path = path.with_suffix(spec.companion_suffix) + kwargs["companion"] = stack.enter_context(companion_path.open("wb")) + spec.writer(mesh_list, out, **kwargs) + + +# --------------------------------------------------------------------------- +# UI helpers — all derived from the registry +# --------------------------------------------------------------------------- + + +def supported_formats() -> List[str]: + """Return all supported export format extensions.""" + return [fmt.value for fmt in _REGISTRY] + + +def format_description(format: Union[str, ExportFormat]) -> str: + """Return a human-readable description of *format*.""" + try: + return get_format_spec(format).description + except ValueError: + return "Unknown format" + + +def format_filter_string(format: Union[str, ExportFormat]) -> str: + """Return a file-dialog filter string (e.g. ``"VTU Files (*.vtu)"``).""" + try: + return get_format_spec(format).filter_label + except ValueError: + return "All Files (*.*)" + + +def all_formats_filter_string() -> str: + """Return a ``;;``-joined filter string for all supported formats.""" + return ";;".join(spec.filter_label for spec in _REGISTRY.values()) + + +def get_format_options_class(format: Union[str, ExportFormat]) -> Optional[type]: + """Return the options class for *format*, or None.""" + try: + return get_format_spec(format).options_class + except ValueError: + return None + + +def supports_lines(format: Union[str, ExportFormat]) -> bool: + """Return True when *format* can represent polyline primitives.""" + try: + return get_format_spec(format).supports_lines + except ValueError: + return False + + +def supports_triangles(format: Union[str, ExportFormat]) -> bool: + """Return True when *format* can represent triangle / polygon primitives.""" + try: + return get_format_spec(format).supports_triangles + except ValueError: + return False + + +def supports_pointsets(format: Union[str, ExportFormat]) -> bool: + """Return True when *format* can represent point-cloud primitives.""" + try: + return get_format_spec(format).supports_pointsets + except ValueError: + return False + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "FormatSpec", + "register_format", + "get_format_spec", + "registered_formats", + "export_mesh", + "supported_formats", + "format_description", + "format_filter_string", + "all_formats_filter_string", + "get_format_options_class", + "supports_lines", + "supports_triangles", + "supports_pointsets", +] diff --git a/energyml-utils/src/energyml/utils/data/export/geojson.py b/energyml-utils/src/energyml/utils/data/export/geojson.py new file mode 100644 index 0000000..94d7277 --- /dev/null +++ b/energyml-utils/src/energyml/utils/data/export/geojson.py @@ -0,0 +1,1285 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""GeoJSON export (RFC 7946), dict-building and streaming writers.""" + +from __future__ import annotations + +import json +import logging +from enum import Enum +from io import BytesIO +from typing import TYPE_CHECKING, Any, Dict, List, Optional, TextIO, Tuple, Union + +import numpy as np + +from energyml.utils.data.export._base import ( + ExportFormat, + resolve_origin_shift, + GeoJSONExportOptions, + _get_context_color, + _get_export_points, + _get_faces_or_cells, + _normalize_to_patches, + _parse_vtk_flat_faces, + _parse_vtk_flat_lines, + _workspace_from_contexts, +) + +from energyml.utils.data.export._registry import FormatSpec, register_format + +if TYPE_CHECKING: + from energyml.utils.data.crs import PointFrame + from energyml.utils.data.mesh import AbstractMesh + from energyml.utils.storage_interface import EnergymlStorageInterface + from energyml.utils.data.representation_context import RepresentationContext + +logger = logging.getLogger(__name__) +#: Alias of the module logger, for the functions that take a caller-supplied ``logger`` +#: parameter — the parameter shadows the module-level name inside their body. +_MODULE_LOGGER = logger + +# --------------------------------------------------------------------------- +# GeoJSON export +# --------------------------------------------------------------------------- + + +def _geojson_crs_members( + projected_epsg_code: Optional[int], + vertical_epsg_code: Optional[int], +) -> dict: + """ + Build the members advertising a **non-WGS84** CRS in a GeoJSON document. + + RFC 7946 mandates WGS84 (CRS84) and removed the ``crs`` member, so when the coordinates + are left in their projected CRS the document is, strictly speaking, non conformant. + Two complementary standard-ish identifiers are then written: + + - ``crs`` — the GeoJSON 2008 named-CRS member. Deprecated, but it is what GDAL / OGR + and QGIS actually read. + - ``coordRefSys`` — the OGC JSON-FG member, given as OGC URI(s). A list is used for a + compound CRS (horizontal + vertical), as allowed by JSON-FG. + + Returns an empty dict when no EPSG code is available. + """ + if projected_epsg_code is None: + return {} + + from energyml.utils.data.crs import crs_ogc_uri, crs_urn + + members: dict = { + "crs": {"type": "name", "properties": {"name": crs_urn(projected_epsg_code)}}, + } + if vertical_epsg_code is not None: + members["coordRefSys"] = [crs_ogc_uri(projected_epsg_code), crs_ogc_uri(vertical_epsg_code)] + else: + members["coordRefSys"] = crs_ogc_uri(projected_epsg_code) + return members + + +def _feature_id( + source_uuid: Optional[str], + patch_index: Optional[int] = None, + element_index: Optional[int] = None, +) -> Optional[str]: + """ + Build the RFC 7946 ``id`` member of a feature (§3.2 : "If a Feature has a commonly used + identifier, that identifier SHOULD be included"). The energyml uuid is used, suffixed by + the patch / element indices when a single object yields several features. + """ + if not source_uuid: + return None + parts = [source_uuid] + if patch_index is not None: + parts.append(str(patch_index)) + if element_index is not None: + parts.append(str(element_index)) + return "_".join(parts) + + +def _geojson_bbox(all_points: List[np.ndarray]) -> Optional[List[float]]: + """ + Compute the RFC 7946 §5 ``bbox`` of the whole collection : + ``[min_x, min_y, min_z, max_x, max_y, max_z]``. + """ + non_empty = [p for p in all_points if p is not None and len(p) > 0] + if not non_empty: + return None + stacked = np.concatenate([np.asarray(p, dtype=np.float64).reshape(-1, 3) for p in non_empty], axis=0) + mins = stacked.min(axis=0) + maxs = stacked.max(axis=0) + return [float(mins[0]), float(mins[1]), float(mins[2]), float(maxs[0]), float(maxs[1]), float(maxs[2])] + + +def _resolve_crs( + mesh: Any, + workspace: Any = None, + projected_epsg_code: Optional[int] = None, + vertical_epsg_code: Optional[int] = None, +) -> Tuple[Any, Optional[int], Optional[int]]: + """Return ``(crs_info, projected_epsg_code, vertical_epsg_code)`` for *mesh*. + + The single place where a mesh's CRS is resolved for the GeoJSON writers. The forced EPSG + codes win over the ones read from the CRS object, and are folded back into the returned + ``CrsInfo`` so the reprojection uses them too. + """ + from dataclasses import replace + + from energyml.utils.data.crs import extract_crs_info + + crs_info = None + crs_obj = getattr(mesh, "crs_object", None) + if isinstance(crs_obj, list): + crs_obj = crs_obj[0] if crs_obj else None + if crs_obj is not None: + try: + crs_info = extract_crs_info(crs_obj, workspace) + except Exception as exc: # pragma: no cover — extract_crs_info is already defensive + logger.debug("CRS info extraction failed: %s", exc) + + projected_epsg_code = projected_epsg_code or getattr(crs_info, "projected_epsg_code", None) + vertical_epsg_code = vertical_epsg_code or getattr(crs_info, "vertical_epsg_code", None) + + if crs_info is not None and ( + projected_epsg_code != crs_info.projected_epsg_code or vertical_epsg_code != crs_info.vertical_epsg_code + ): + crs_info = replace( + crs_info, + projected_epsg_code=projected_epsg_code, + vertical_epsg_code=vertical_epsg_code, + ) + return crs_info, projected_epsg_code, vertical_epsg_code + + +def _geojson_crs_info(mesh: Any, options: "GeoJSONExportOptions", workspace: Any): + """Backward-compatible wrapper over :func:`_resolve_crs` taking the option object.""" + return _resolve_crs(mesh, workspace, options.projected_epsg_code, options.vertical_epsg_code) + + +def _collection_crs_members(crs_states: set, logger: Optional[Any] = None) -> dict: + """Members declaring the CRS of a whole FeatureCollection, or ``{}``. + + *crs_states* holds one ``(projected_epsg, vertical_epsg, is_wgs84)`` triple per feature. An + RFC 7946 document is implicitly in CRS84 and must **not** carry a ``crs`` member, so only the + non-WGS84 states are considered — and only when they all agree, since a single collection + cannot advertise two different source CRS. + """ + not_wgs84 = [state for state in crs_states if not state[2] and state[0] is not None] + if len(not_wgs84) == 1: + return _geojson_crs_members(not_wgs84[0][0], not_wgs84[0][1]) + if len(not_wgs84) > 1: + (logger or _MODULE_LOGGER).warning( + "GeoJSON export: %d different source CRS in the same FeatureCollection — " + "no collection-level CRS is declared, see the per-feature 'projected_epsg_code' property.", + len(not_wgs84), + ) + return {} + + +def _prepare_geojson_points( + mesh: Any, + pts: np.ndarray, + options: "GeoJSONExportOptions", + workspace: Any, + current_frame: Optional["PointFrame"] = None, +) -> tuple: + """ + Bring *pts* to WGS84 when possible and return + ``(points, projected_epsg_code, vertical_epsg_code, is_wgs84)``. + + The transform itself, and the decision to fall back to the projected coordinates when no EPSG + code is available / ``pyproj`` is missing / PROJ fails, all live in + :func:`~energyml.utils.data.crs.to_frame`. This function only resolves which CRS applies and + reports what was reached, so the GeoJSON writer and the 3-D writers degrade identically. + """ + from energyml.utils.data.crs import PointFrame, to_frame + + crs_info, projected_epsg_code, vertical_epsg_code = _geojson_crs_info(mesh, options, workspace) + + if not options.to_wgs84 or len(pts) == 0: + return pts, projected_epsg_code, vertical_epsg_code, False + + framed = to_frame( + pts, + crs_info, + PointFrame.WGS84, + current_frame or PointFrame.PROJECTED, + use_network=options.use_network, + inplace=False, + ) + is_wgs84 = framed.frame is PointFrame.WGS84 + if not is_wgs84: + logger.warning( + "GeoJSON export: %s stays in its source CRS (non RFC 7946 conformant) — %s", + getattr(mesh, "source_uuid", None) or getattr(mesh, "identifier", "?"), + framed.degraded_reason or "unknown reason", + ) + return framed.points, projected_epsg_code, vertical_epsg_code, is_wgs84 + + +def _with_points(mesh: Any, points: np.ndarray, frame: Optional["PointFrame"] = None) -> Any: + """Shallow copy of *mesh* carrying *points* (and *frame*), whatever mesh family it is. + + The coordinate field is named ``point_list`` in the legacy hierarchy and ``points`` in the + numpy one; both are dataclasses, so the copy shares every other field with the original and + the caller's array is never written back into the source mesh. + """ + from dataclasses import replace + + if hasattr(mesh, "point_list"): + updates: Dict[str, Any] = {"point_list": points.tolist()} + else: + updates = {"points": points} + if frame is not None: + updates["frame"] = frame + return replace(mesh, **updates) + + +def export_geojson( + mesh_list: Any, + out: TextIO, + options: Optional[GeoJSONExportOptions] = None, + contexts: Optional[Dict[str, "RepresentationContext"]] = None, + use_crs_displacement: bool = True, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + use_network: bool = False, +) -> None: + """Export mesh data to GeoJSON FeatureCollection. + + Coordinates are reprojected to WGS84 (longitude, latitude, ellipsoidal height) by default, + as required by RFC 7946 — see :class:`GeoJSONExportOptions`. When the reprojection cannot + be done, the source CRS is advertised through the ``crs`` (GeoJSON 2008) and + ``coordRefSys`` (OGC JSON-FG) members instead. + + Every feature carries the identification metadata of its source object : the RFC 7946 + ``id`` member holds the energyml uuid, and ``properties`` holds the ``uuid``, + ``qualified_type`` and ``Citation`` fields (title, creation, last_update, …). + + :param mesh_list: One or more meshes. + :param out: Text output stream. + :param options: GeoJSON export options. + :param contexts: Optional colour / metadata context dict. + :param use_crs_displacement: Apply CRS displacement to ``NumpyMesh`` points. + + .. note:: + This function is the *frame* half of the export — it resolves the coordinate frame + (``use_crs_displacement`` / ``frame`` / ``origin_shift``) and the presentation properties + (colours from *contexts*, ``source_uuid``, ``patch_index``) — and then hands the meshes to + :func:`export_geojson_io`, which owns the one implementation of the geometry, of the CRS + declaration and of the bounding boxes. It used to assemble the FeatureCollection a second + time on its own, with its own reprojection call and its own geometry rules; the two + assemblies had already drifted apart (no point-set branch on this side, ``MultiLineString`` + instead of ``LineString`` on the other). + """ + from energyml.utils.introspection import get_object_metadata + + if options is None: + options = GeoJSONExportOptions() + + patches = _normalize_to_patches(mesh_list) + workspace = _workspace_from_contexts(contexts) + _origin_shift = resolve_origin_shift(patches, use_crs_displacement, workspace, frame, origin_shift, use_network) + + prepared: List[Any] = [] + extra_properties: List[Optional[Dict]] = [] + feature_ids: List[Optional[str]] = [] + + for mesh in patches: + pts, pts_frame, _ = _get_export_points(mesh, use_crs_displacement, workspace, frame, _origin_shift, use_network) + prepared.append(_with_points(mesh, np.asarray(pts, dtype=np.float64).reshape(-1, 3), pts_frame)) + + source_uuid = getattr(mesh, "source_uuid", None) or get_object_metadata( + getattr(mesh, "energyml_object", None) + ).get("uuid") + patch_index = getattr(mesh, "patch_index", None) + + props: Dict[str, Any] = {**options.properties, "source_uuid": source_uuid, "patch_index": patch_index} + color = _get_context_color(getattr(mesh, "source_uuid", None), contexts) + if color: + r, g, b, a = color + props["color"] = f"#{r:02x}{g:02x}{b:02x}" + props["opacity"] = round(a / 255.0, 4) + + extra_properties.append(props) + feature_ids.append(_feature_id(source_uuid, patch_index)) + + buffer = BytesIO() + export_geojson_io( + out=buffer, + mesh_list=prepared, + properties=extra_properties, + workspace=workspace, + to_wgs84=options.to_wgs84, + include_metadata=options.include_metadata, + use_network=options.use_network, + indent=options.indent, + explode_elements=options.explode_elements, + feature_ids=feature_ids, + anycrs_prefix=False, + projected_epsg_code=options.projected_epsg_code, + vertical_epsg_code=options.vertical_epsg_code, + ) + out.write(buffer.getvalue().decode("utf-8")) + + +# --------------------------------------------------------------------------- +# Registration +# --------------------------------------------------------------------------- + + +def _write( + mesh_list: Any, + out: TextIO, + *, + obj_name: Optional[str] = None, + options: Any = None, + contexts: Optional[Dict[str, "RepresentationContext"]] = None, + use_crs_displacement: bool = True, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + companion: Any = None, +) -> None: + """Uniform adapter used by the registry.""" + export_geojson( + mesh_list, + out, + options, + contexts, + use_crs_displacement, + frame=frame, + origin_shift=origin_shift, + ) + + +register_format( + FormatSpec( + format=ExportFormat.GEOJSON, + description="GeoJSON — geographic data (lines, polygons, point clouds)", + filter_label="GeoJSON Files (*.geojson)", + writer=_write, + binary=False, + options_class=GeoJSONExportOptions, + ) +) + + +# --------------------------------------------------------------------------- +# Streaming writers (moved from mesh.py) +# +# These build a GeoJSON document incrementally into a byte stream, so the peak memory is +# bounded by one feature instead of the whole FeatureCollection. The dict-building helpers +# above and below are thin wrappers over them, so there is a single implementation of the +# geometry: mesh.py used to carry a second, independent one (_create_shape). +# --------------------------------------------------------------------------- + + +class GeoJsonGeometryType(Enum): + """GeoJson type enum""" + + Point = "Point" + MultiPoint = "MultiPoint" + LineString = "LineString" + MultiLineString = "MultiLineString" + Polygon = "Polygon" + MultiPolygon = "MultiPolygon" + + +def energyml_type_to_geojson_type(energyml_type: str): + if "PolylineSet" in energyml_type: + return GeoJsonGeometryType.MultiLineString + elif "Polyline" in energyml_type: + return GeoJsonGeometryType.LineString + elif "PointSet" in energyml_type: + return GeoJsonGeometryType.MultiPoint + elif "Point" in energyml_type: + return GeoJsonGeometryType.Point + elif "TriangulatedSet" in energyml_type: + return GeoJsonGeometryType.MultiPolygon + elif "Triangulated" in energyml_type: + return GeoJsonGeometryType.Polygon + elif "Grid2" in energyml_type: + return GeoJsonGeometryType.MultiPolygon + return GeoJsonGeometryType.Point + + +def _recompute_min_max( + old_min: List, # out parameter + old_max: List, # out parameter + potential_min: List, + potential_max: List, +) -> None: + """Merge one candidate bounding box into the accumulators, extending them when needed.""" + for i in range(len(potential_min)): + if i >= len(old_min): + old_min.append(potential_min[i]) + elif potential_min[i] is not None: + old_min[i] = min(old_min[i], potential_min[i]) + + for i in range(len(potential_max)): + if i >= len(old_max): + old_max.append(potential_max[i]) + elif potential_max[i] is not None: + old_max[i] = max(old_max[i], potential_max[i]) + + +def _recompute_min_max_from_points( + old_min: List, # out parameter + old_max: List, # out parameter + points: Any, +) -> None: + """Merge the bounding box of *points* into the accumulators. + + Reduced with numpy in one pass. The previous version recursed once per point and compared + each coordinate with the built-in ``min`` / ``max``, i.e. a handful of Python calls per point + on the only loop of the export that runs per point. + """ + arr = np.asarray(points, dtype=np.float64) + if arr.size == 0: + return + if arr.ndim == 1: + arr = arr.reshape(1, -1) + else: + arr = arr.reshape(-1, arr.shape[-1]) + _recompute_min_max(old_min, old_max, arr.min(axis=0).tolist(), arr.max(axis=0).tolist()) + + +class _JsonIndent: + """ + Whitespace emitter for the streaming GeoJSON writers. + + Only the *structure* is indented: the collection, the features, the geometry + and the containers of the coordinates. The innermost coordinate arrays stay + on a single line — unrolling every ``[x, y, z]`` over four lines does not make + a document more readable, it inflates it (measured x2.5 on a real export), and + it would put the indentation on the only hot path of these writers, the one + that runs once per point. + + ``_JsonIndent(None)`` is the disabled form: every method returns the exact + bytes the writers used before, so the single-line output is unchanged. + """ + + __slots__ = ("unit", "_depth", "_cache") + + def __init__(self, indent: Optional[Union[int, str, "_JsonIndent"]] = None): + if indent is None: + self.unit: Optional[str] = None + elif isinstance(indent, int): + self.unit = " " * max(0, indent) + else: + self.unit = str(indent) + self._depth = 0 + self._cache: Dict[int, bytes] = {} + + @classmethod + def coerce(cls, indent: Optional[Union[int, str, "_JsonIndent"]]) -> "_JsonIndent": + """Accept an already built indenter, so it can be threaded through the recursion.""" + return indent if isinstance(indent, cls) else cls(indent) + + @property + def enabled(self) -> bool: + return self.unit is not None + + def nl(self) -> bytes: + """Line break followed by the indentation of the current level (``b""`` when disabled).""" + if self.unit is None: + return b"" + cached = self._cache.get(self._depth) + if cached is None: + cached = ("\n" + self.unit * self._depth).encode() + self._cache[self._depth] = cached + return cached + + def open(self) -> bytes: + """Enter a nesting level, and return the break that starts its first item.""" + self._depth += 1 + return self.nl() + + def close(self) -> bytes: + """Leave a nesting level, and return the break that puts its closing bracket in place.""" + self._depth = max(0, self._depth - 1) + return self.nl() + + def sep(self) -> bytes: + """Comma between two items or two members, with the break (or space) that follows it.""" + return b"," + (self.nl() if self.unit is not None else b" ") + + +def _dumps_at_depth(value: Any, ind: _JsonIndent) -> bytes: + """ + Serialise a small value with :func:`json.dumps`, re-indenting its continuation + lines so that they line up with the current depth. + + Only used for the metadata members (``properties``, ``name``): they weigh a few + dozen bytes, so the extra string work is irrelevant — unlike on the coordinates. + """ + if not ind.enabled: + return json.dumps(value).encode() + text = json.dumps(value, indent=ind.unit) + if "\n" not in text: + return text.encode() + return text.replace("\n", ind.nl().decode()).encode() + + +def _write_geojson_shape( + out: BytesIO, + geo_type: GeoJsonGeometryType, + point_list: List[List[float]], + indices: Optional[Union[List[List[int]], List[int]]] = None, + point_offset: int = 0, + logger: Optional[Any] = None, + _print_list_boundaries: Optional[bool] = True, + ind: Optional[Union[int, str, _JsonIndent]] = None, +) -> Tuple[List[float], List[float]]: + """ + Write a shape from a point list [ [x0, y0 (, z0)? ], ..., [xn, yn (, zn)? ] ] + using indices. If indices is a simple list, result will be a line like : [p0, ..., pn]. With p0 and pn + a list of coordinate from "points" parameter (like [x0, y0 (, z0)? ]) + If the indices are a list of list, result will be polygones like : + [ + [poly0_p0, ..., poly0_pn], + ... + [polyn_p0, ..., polyn_pn], + ] + :param ind: indentation of the *containers* of the coordinates. The list of points of a + line or a ring is always written on a single line. + :return shape, minXYZ (as list), maxXYZ (as list) + """ + mins = [] + maxs = [] + ind = _JsonIndent.coerce(ind) + try: + if geo_type == GeoJsonGeometryType.LineString: + if indices is not None and len(indices) > 0: + cpt = 0 + if _print_list_boundaries: + out.write(b"[") + for idx in indices: + out.write(json.dumps(point_list[idx + point_offset]).encode("utf-8")) + if cpt < len(indices) - 1: + out.write(b", ") + cpt += 1 + if _print_list_boundaries: + out.write(b"]") + # One reduction for the whole line rather than one per point: this loop is the + # only hot path of the writer. + _recompute_min_max_from_points(mins, maxs, [point_list[i + point_offset] for i in indices]) + else: + out.write(json.dumps(point_list).encode("utf-8")) + _recompute_min_max_from_points(mins, maxs, point_list) + elif geo_type == GeoJsonGeometryType.MultiPoint or geo_type == GeoJsonGeometryType.Point: + out.write(json.dumps(point_list).encode("utf-8")) + _recompute_min_max_from_points(mins, maxs, point_list) + elif geo_type == GeoJsonGeometryType.MultiLineString: + if indices is not None and len(indices) > 0 and isinstance(indices[0], list): + if _print_list_boundaries: + out.write(b"[") + out.write(ind.open()) + cpt = 0 + for idx in indices: + _min, _max = _write_geojson_shape( + out=out, + geo_type=GeoJsonGeometryType.MultiLineString, + point_list=point_list, + indices=idx, + point_offset=point_offset, + logger=logger, + _print_list_boundaries=False, + ind=ind, + ) + if cpt < len(indices) - 1: + out.write(ind.sep()) + cpt += 1 + _recompute_min_max(mins, maxs, _min, _max) + if _print_list_boundaries: + out.write(ind.close()) + out.write(b"]") + else: + if _print_list_boundaries: + out.write(b"[") + out.write(ind.open()) + _min, _max = _write_geojson_shape( + out=out, + geo_type=GeoJsonGeometryType.LineString, + point_list=point_list, + indices=indices, + point_offset=point_offset, + logger=logger, + ind=ind, + ) + _recompute_min_max(mins, maxs, _min, _max) + if _print_list_boundaries: + out.write(ind.close()) + out.write(b"]") + elif geo_type == GeoJsonGeometryType.Polygon: + # First and last must be the same + if indices is not None and len(indices) > 0: + if indices[0] != indices[-1]: + indices.append(indices[0]) + elif point_list[0] != point_list[-1]: + point_list.append(point_list[0]) + + mins, maxs = _write_geojson_shape( + out=out, + geo_type=GeoJsonGeometryType.MultiLineString, # Here we only provide 1 line, the external one (outer-ring) + point_list=point_list, + indices=indices, + point_offset=point_offset, + logger=logger, + _print_list_boundaries=_print_list_boundaries, + ind=ind, + ) + elif geo_type == GeoJsonGeometryType.MultiPolygon: + if indices is not None and len(indices) > 0 and isinstance(indices[0], list): + if _print_list_boundaries: + out.write(b"[") + out.write(ind.open()) + cpt = 0 + for idx in indices: + _min, _max = _write_geojson_shape( + out=out, + geo_type=GeoJsonGeometryType.MultiPolygon, # Here we only provide 1 line, the external one (outer-ring) + point_list=point_list, + indices=idx, + point_offset=point_offset, + logger=logger, + _print_list_boundaries=False, + ind=ind, + ) + if cpt < len(indices) - 1: + out.write(ind.sep()) + cpt += 1 + _recompute_min_max(mins, maxs, _min, _max) + if _print_list_boundaries: + out.write(ind.close()) + out.write(b"]") + else: + if _print_list_boundaries: + out.write(b"[") + out.write(ind.open()) + _min, _max = _write_geojson_shape( + out=out, + geo_type=GeoJsonGeometryType.Polygon, # Here we only provide 1 line, the external one (outer-ring) + point_list=point_list, + indices=indices, + point_offset=point_offset, + logger=logger, + ind=ind, + ) + _recompute_min_max(mins, maxs, _min, _max) + if _print_list_boundaries: + out.write(ind.close()) + out.write(b"]") + except Exception as e: + # never swallow silently: a failure here produces a geometry without coordinates + (logger or _MODULE_LOGGER).error( + "@_write_geojson_shape failed for a %s geometry: %s: %s", geo_type.name, type(e).__name__, e + ) + # raise e + return mins, maxs + + +def _as_json_ready_list(value: Any) -> Any: + """ + Convert numpy arrays / numpy scalars into plain python lists and floats. + + The GeoJSON writers below serialize the points with :func:`json.dumps`, which does not + support numpy types : depending on the representation and on the way its points were read, + ``AbstractMesh.point_list`` may be a ``list`` *or* an ``ndarray``. Without this conversion + the serialization raises ``TypeError: Object of type ndarray is not JSON serializable``. + """ + if isinstance(value, np.ndarray): + return value.tolist() + if isinstance(value, np.generic): + return value.item() + if isinstance(value, (list, tuple)): + return [_as_json_ready_list(v) for v in value] + return value + + +def to_geojson_feature( + mesh: "AbstractMesh", + geo_type: GeoJsonGeometryType = GeoJsonGeometryType.Point, + geo_type_prefix: Optional[str] = "", + properties: Optional[dict] = None, + point_offset: int = 0, + logger=None, + feature_id: Optional[str] = None, +) -> Dict: + """ + Build a GeoJSON Feature as a dict. + + Serialises through :func:`write_geojson_feature` and parses the result back, so the geometry + has a single implementation. The previous dict-building path (``_create_shape``) was a second, + independent transcription of the same five-branch recursion — roughly 100 lines that had to be + kept in step with the streaming one by hand. + + :param geo_type_prefix: prefix of the ``type`` member. Empty (default) for a standard + RFC 7946 ``"Feature"``; ``"AnyCrs"`` marks non-WGS84 coordinates. + :param feature_id: value of the RFC 7946 ``id`` member (the energyml uuid, typically). + """ + raw_points = mesh_points(mesh) + if raw_points is None or len(raw_points) == 0: + return {} + + buffer = BytesIO() + write_geojson_feature( + out=buffer, + mesh=mesh, + geo_type=geo_type, + geo_type_prefix=geo_type_prefix, + properties=properties, + point_offset=point_offset, + logger=logger, + feature_id=feature_id, + ) + raw = buffer.getvalue() + if not raw: + return {} + return json.loads(raw.decode("utf-8")) + + +def _write_feature( + out: BytesIO, + geo_type: GeoJsonGeometryType, + points: Any, + indices: Optional[Union[List[List[int]], List[int]]] = None, + geo_type_prefix: Optional[str] = "", + properties: Optional[dict] = None, + point_offset: int = 0, + logger: Optional[Any] = None, + feature_id: Optional[str] = None, + ind: Optional[Union[int, str, _JsonIndent]] = None, + identifier: str = "?", +) -> Tuple[List[float], List[float]]: + """ + Write one GeoJSON Feature from already-resolved coordinates, and return its ``(mins, maxs)``. + + Takes *points* / *indices* rather than a mesh so that a single element of a patch can be + written on its own — that is what ``explode_elements`` needs, and what let the registry + writer stop transcribing the geometry rules a second time. + """ + ind = _JsonIndent.coerce(ind) + if points is None or len(points) == 0: + return [], [] + + # A ring must be closed. When several rings are given (a list of index lists) the closing + # is done here; when a single flat ring is given, _write_geojson_shape closes it itself. + close_rings = ( + geo_type in (GeoJsonGeometryType.Polygon, GeoJsonGeometryType.MultiPolygon) + and indices is not None + and len(indices) > 0 + and isinstance(indices[0], list) + ) + if close_rings: + for ring in indices: + ring.append(ring[0]) + + try: + out.write(b"{") # start feature + out.write(ind.open()) + out.write(f'"type": "{geo_type_prefix or ""}Feature"'.encode()) + if feature_id is not None: + out.write(ind.sep()) + out.write(f'"id": {json.dumps(feature_id)}'.encode()) + out.write(ind.sep()) + out.write(b'"properties": ') + out.write(_dumps_at_depth(properties or {}, ind)) + out.write(ind.sep()) + out.write(b'"geometry": ') + + out.write(b"{") # start geometry + out.write(ind.open()) + out.write(f'"type": "{geo_type.name}"'.encode()) + out.write(ind.sep()) + out.write(b'"coordinates": ') + coordinates_start = out.tell() + mins, maxs = _write_geojson_shape( + out=out, + geo_type=geo_type, + point_list=points, + indices=indices, + point_offset=point_offset, + logger=logger, + ind=ind, + ) + if out.tell() == coordinates_start: + # the shape could not be written (see the error logged by _write_geojson_shape) : + # write an empty coordinate list so that the document stays valid JSON + (logger or _MODULE_LOGGER).error( + "No coordinate written for the %s geometry of '%s' (%d points) — " + "an empty geometry is written instead.", + geo_type.name, + identifier, + len(points), + ) + out.write(b"[]") + + bbox_geometry = mins + maxs # TODO : see : https://www.rfc-editor.org/rfc/rfc7946#section-5 + + out.write(ind.sep()) + # the bbox is a flat list of 4 or 6 numbers: it stays on one line + out.write(f'"bbox": {json.dumps(bbox_geometry)}'.encode()) + out.write(ind.close()) + out.write(b"}") # end geometry + + out.write(ind.close()) + out.write(b"}") # End feature + finally: + # the closing point was appended to the caller's lists: undo it whatever happened + if close_rings: + for ring in indices: + ring.pop() + + # The extents are already computed to write the per-geometry bbox; returning them lets + # the caller build the collection-level one without a second pass over the coordinates. + return bbox_geometry[: len(bbox_geometry) // 2], bbox_geometry[len(bbox_geometry) // 2 :] + + +def write_geojson_feature( + out: BytesIO, + mesh: AbstractMesh, + geo_type: GeoJsonGeometryType = GeoJsonGeometryType.Point, + geo_type_prefix: Optional[str] = "", + properties: Optional[dict] = None, + point_offset: int = 0, + logger=None, + feature_id: Optional[str] = None, + indent: Optional[Union[int, str, _JsonIndent]] = None, +) -> Tuple[List[float], List[float]]: + """ + Write a single GeoJSON Feature for *mesh*, and return its ``(mins, maxs)`` extents. + + Resolves the coordinates and the connectivity of *mesh* — whatever mesh family it belongs + to — and hands them to :func:`_write_feature`. + + :param geo_type_prefix: prefix of the ``type`` member. Empty (default) for a standard + RFC 7946 ``"Feature"``; the historical ``"AnyCrs"`` value marks + coordinates that are *not* in WGS84. + :param feature_id: value of the RFC 7946 ``id`` member (the energyml uuid, typically). + :param indent: number of spaces (or indentation string) of the pretty-printed form. + None (default) keeps everything on a single line. See :class:`_JsonIndent` + for what is indented and what deliberately is not. + """ + raw_points = mesh_points(mesh) + if raw_points is None or len(raw_points) == 0: + return [], [] + + # points / indices may be numpy arrays : json.dumps only accepts plain python types + return _write_feature( + out=out, + geo_type=geo_type, + points=_as_json_ready_list(raw_points), + indices=_as_json_ready_list(mesh_indices(mesh)), + geo_type_prefix=geo_type_prefix, + properties=properties, + point_offset=point_offset, + logger=logger, + feature_id=feature_id, + ind=indent, + identifier=getattr(mesh, "identifier", "?") or "?", + ) + + +def mesh_points(mesh: Any) -> Any: + """Coordinates of *mesh*, whatever mesh family it belongs to. + + The legacy containers expose ``point_list``, the numpy ones ``points``. The streaming + writer only knew the first, so passing it a ``NumpyMultiMesh`` raised + ``TypeError: 'NumpyMultiMesh' object is not iterable`` — the two halves of the GeoJSON API + accepted different mesh families. + """ + points = getattr(mesh, "point_list", None) + return getattr(mesh, "points", None) if points is None else points + + +def mesh_indices(mesh: Any) -> List[List[int]]: + """Connectivity of *mesh* as a list of index lists, whatever mesh family it belongs to. + + Legacy meshes already store it that way; numpy ones store the VTK flat encoding, decoded + here. A point set has no connectivity in either family and yields ``[]``. + """ + from energyml.utils.data.mesh_numpy import NumpyMesh, NumpyPolylineMesh + + if not isinstance(mesh, NumpyMesh): + return mesh.get_indices() + + if isinstance(mesh, NumpyPolylineMesh): + return [idx.tolist() for idx in _parse_vtk_flat_lines(mesh.lines)] + connectivity = _get_faces_or_cells(mesh) + if connectivity is None or len(connectivity) == 0: + return [] + return [idx.tolist() for idx in _parse_vtk_flat_faces(connectivity)] + + +def mesh_to_geojson_type(obj: Any) -> GeoJsonGeometryType: + """Pick the GeoJSON geometry type matching *obj*, whatever mesh family it belongs to. + + The single place that maps a mesh class to a geometry kind. The registry writer used to + repeat the rule with its own ``isinstance`` chain and had no branch for a point set, so a + ``PointSetMesh`` — which legitimately carries points and *no* indices — went through a loop + over its (empty) index list and produced a FeatureCollection with zero features. + + Surfaces and volumes become polygons, poly-lines become lines, and anything else is a point + cloud: a mesh with no connectivity still has coordinates worth exporting. + """ + # Imported lazily: mesh.py imports this module, so a module-level import would be circular. + from energyml.utils.data.mesh import PolylineSetMesh, SurfaceMesh + from energyml.utils.data.mesh_numpy import ( + NumpyPolylineMesh, + NumpySurfaceMesh, + NumpyVolumeMesh, + ) + + if isinstance(obj, (SurfaceMesh, NumpySurfaceMesh, NumpyVolumeMesh)): + return GeoJsonGeometryType.MultiPolygon + if isinstance(obj, (PolylineSetMesh, NumpyPolylineMesh)): + return GeoJsonGeometryType.MultiLineString + return GeoJsonGeometryType.MultiPoint + + +def _geojson_mesh_metadata(mesh: "AbstractMesh", workspace: Optional["EnergymlStorageInterface"] = None) -> Dict: + """ + Build the properties of a feature from the energyml object carried by *mesh* : + uuid, qualified type, Citation fields and mesh identifier. + + The EPSG codes are *not* read here: they come from :func:`_resolve_crs`, which the caller + already runs to decide whether the coordinates can be reprojected. Extracting them twice meant + two ``extract_crs_info`` calls per mesh, and two places able to disagree on the answer. + + :param workspace: kept for backward compatibility; unused. + """ + from energyml.utils.introspection import get_object_metadata + + properties: Dict = dict(get_object_metadata(getattr(mesh, "energyml_object", None))) + if getattr(mesh, "identifier", None): + properties["identifier"] = mesh.identifier + return properties + + +def _geojson_reproject_mesh( + mesh: "AbstractMesh", + workspace: Optional["EnergymlStorageInterface"] = None, + use_network: bool = False, + logger: Optional[Any] = None, + to_wgs84: bool = True, + projected_epsg_code: Optional[int] = None, + vertical_epsg_code: Optional[int] = None, +) -> Tuple["AbstractMesh", bool, Optional[int], Optional[int]]: + """ + Return ``(mesh, is_wgs84, projected_epsg_code, vertical_epsg_code)`` where *mesh* is a shallow + copy whose points have been reprojected to WGS84, or the original mesh when the reprojection + is not asked for or is impossible (no EPSG code, pyproj missing, transformation error). + + The transform and its fallbacks live in :func:`~energyml.utils.data.crs.to_frame`; this + function only adapts the mesh containers to it. It used to re-implement the whole degradation + ladder (missing EPSG / missing pyproj / PROJ failure) a second time. + + The EPSG codes are returned even when nothing is reprojected, so the caller can advertise the + source CRS of a document it left in its projected coordinates. + """ + from energyml.utils.data.crs import PointFrame, to_frame + + crs_info, projected_epsg_code, vertical_epsg_code = _resolve_crs( + mesh, workspace, projected_epsg_code, vertical_epsg_code + ) + + raw_points = mesh_points(mesh) + if not to_wgs84 or crs_info is None or raw_points is None or len(raw_points) == 0: + return mesh, False, projected_epsg_code, vertical_epsg_code + + points = np.asarray(raw_points, dtype=np.float64).reshape(-1, 3) + framed = to_frame( + points, + crs_info, + PointFrame.WGS84, + getattr(mesh, "frame", PointFrame.PROJECTED), + use_network=use_network, + inplace=False, + ) + + if framed.frame is not PointFrame.WGS84: + (logger or _MODULE_LOGGER).warning( + "GeoJSON export: %s stays in its source CRS (non RFC 7946 conformant) — %s", + getattr(mesh, "source_uuid", None) or getattr(mesh, "identifier", "?"), + framed.degraded_reason or "unknown reason", + ) + return mesh, False, projected_epsg_code, vertical_epsg_code + + return _with_points(mesh, framed.points, framed.frame), True, projected_epsg_code, vertical_epsg_code + + +def _suffix_feature_id(feature_id: Optional[str], element_index: Optional[int]) -> Optional[str]: + """Append an element index to a feature id, so exploded elements keep distinct ``id`` members.""" + if feature_id is None or element_index is None: + return feature_id + return f"{feature_id}_{element_index}" + + +def _effective_geo_type(geo_kind: GeoJsonGeometryType, element_count: int) -> GeoJsonGeometryType: + """Collapse a ``Multi*`` kind to its singular form when the patch holds a single element. + + RFC 7946 has both forms and prefers the simplest one that fits, so a 15-station wellbore is a + ``LineString`` and a single triangle a ``Polygon``. The two collection builders used to + disagree exactly here — the registry one collapsed, the streaming one did not. + """ + if element_count != 1: + return geo_kind + if geo_kind == GeoJsonGeometryType.MultiLineString: + return GeoJsonGeometryType.LineString + if geo_kind == GeoJsonGeometryType.MultiPolygon: + return GeoJsonGeometryType.Polygon + return geo_kind + + +def export_geojson_io( + out: BytesIO, + mesh_list: List[AbstractMesh], + obj_name: Optional[str] = None, + properties: Optional[List[Optional[Dict]]] = None, + global_properties: Optional[Dict] = None, + logger: Optional[Any] = None, + workspace: Optional[EnergymlStorageInterface] = None, + to_wgs84: bool = True, + include_metadata: bool = True, + use_network: bool = False, + indent: Optional[Union[int, str]] = None, + explode_elements: bool = False, + feature_ids: Optional[List[Optional[str]]] = None, + anycrs_prefix: bool = True, + projected_epsg_code: Optional[int] = None, + vertical_epsg_code: Optional[int] = None, +): + """ + Stream a list of meshes as a GeoJSON FeatureCollection. + + This is the single implementation of the GeoJSON geometry: :func:`export_geojson` (the + registry writer), :func:`export_geojson_dict` and :func:`to_geojson_feature` all go through + it, so there is one set of rules for the geometry kinds, the CRS declaration and the bounding + boxes. + + :param out: output stream + :param mesh_list: meshes to export + :param obj_name: value of the ``name`` member of the collection + :param properties: extra per-mesh properties, aligned on *mesh_list*, merged on top of the + metadata built from the energyml object of each mesh + :param global_properties: extra members written at the collection level + :param logger: logger used for the per-feature diagnostics; defaults to this module's + :param workspace: used to resolve the CRS objects (needed for the v2.2 compound CRS) + :param to_wgs84: when True (default), coordinates are reprojected to WGS84 as required by + RFC 7946. When the reprojection is not possible, the source CRS is + advertised through the ``crs`` / ``coordRefSys`` members. + :param include_metadata: add the energyml metadata to the properties of every feature + :param use_network: allow PROJ to download the geoid grids used by vertical transformations + :param indent: number of spaces (or indentation string) for a pretty-printed document. + None (default) keeps the historical single-line output. + + The document structure is indented but the coordinates of a line or a ring + stay on one line: that is what keeps the file readable without inflating it, + and it leaves the per-point write path untouched, so the export costs about + the same as the compact one — far less than serialising, re-reading and + re-dumping the document with ``json.dumps(indent=...)``. + :param explode_elements: emit one feature per triangle / per line instead of one feature per + patch. Off by default: exploding repeats the whole metadata block — + uuid, citation, EPSG codes — on every element. + :param feature_ids: explicit RFC 7946 ``id`` per mesh, aligned on *mesh_list*. Defaults to the + uuid of the source object. + :param anycrs_prefix: when True (default), features whose coordinates are not WGS84 keep the + historical ``"AnyCrsFeature"`` type. Set to False for a plain + ``"Feature"`` in every case. + :param projected_epsg_code: force the horizontal EPSG code instead of reading it from the CRS + :param vertical_epsg_code: force the vertical EPSG code instead of reading it from the CRS + """ + # Accept both mesh families and every container shape, like the registry writer: a caller + # holding the NumpyMultiMesh returned by read_numpy_mesh_object used to get + # `TypeError: 'NumpyMultiMesh' object is not iterable` here. + mesh_list = _normalize_to_patches(mesh_list) + + # the source index is kept so that `properties` / `feature_ids` stay aligned on `mesh_list` + exported: List[Tuple[int, Any, Dict]] = [] + crs_states: set = set() + + for mesh_index, mesh in enumerate(mesh_list): + mesh_pts = mesh_points(mesh) + if mesh_pts is None or len(mesh_pts) == 0: + # write_geojson_feature() would write nothing for it; dropping it here keeps the + # separator logic below exact (an empty mesh in last position used to leave a + # trailing comma, which is not valid JSON). + continue + + feature_properties: Dict = {} + if include_metadata: + feature_properties.update(_geojson_mesh_metadata(mesh)) + + mesh, is_wgs84, mesh_projected, mesh_vertical = _geojson_reproject_mesh( + mesh, + workspace=workspace, + use_network=use_network, + logger=logger, + to_wgs84=to_wgs84, + projected_epsg_code=projected_epsg_code, + vertical_epsg_code=vertical_epsg_code, + ) + if mesh_projected is not None: + feature_properties["projected_epsg_code"] = mesh_projected + if mesh_vertical is not None: + feature_properties["vertical_epsg_code"] = mesh_vertical + if is_wgs84: + # keep the provenance of the coordinates now that they have been converted + feature_properties["source_crs"] = f"EPSG:{mesh_projected}" + feature_properties["coordinates_crs"] = "OGC:CRS84" + crs_states.add((mesh_projected, mesh_vertical, is_wgs84)) + exported.append((mesh_index, mesh, feature_properties)) + + ind = _JsonIndent(indent) + + out.write(b"{") + out.write(ind.open()) + out.write(b'"type": "FeatureCollection"') + if obj_name is not None: + out.write(ind.sep()) + # json.dumps rather than a raw concatenation: a title may contain a quote + out.write(f'"name": {json.dumps(obj_name)}'.encode()) + + for k, v in _collection_crs_members(crs_states, logger).items(): + out.write(ind.sep()) + out.write(f'"{k}": '.encode()) + out.write(_dumps_at_depth(v, ind)) + + if global_properties is not None and len(global_properties) > 0: + for k, v in global_properties.items(): + out.write(ind.sep()) + out.write(f"{json.dumps(k)}: ".encode()) + out.write(_dumps_at_depth(v, ind)) + + out.write(ind.sep()) + out.write(b'"features": [') + out.write(ind.open()) + + written = 0 + collection_mins: List[float] = [] + collection_maxs: List[float] = [] + + for mesh_index, mesh, feature_properties in exported: + explicit = properties[mesh_index] if properties is not None and len(properties) > mesh_index else None + feature_properties = {**feature_properties, **(explicit or {})} + # "AnyCrsFeature" keeps flagging the features whose coordinates are not WGS84 + prefix = "" if not anycrs_prefix or feature_properties.get("coordinates_crs") == "OGC:CRS84" else "AnyCrs" + base_id = feature_properties.get("uuid") + if feature_ids is not None and len(feature_ids) > mesh_index: + base_id = feature_ids[mesh_index] + + points = _as_json_ready_list(mesh_points(mesh)) + elements = _as_json_ready_list(mesh_indices(mesh)) + geo_kind = mesh_to_geojson_type(mesh) + identifier = getattr(mesh, "identifier", "?") or "?" + + if geo_kind != GeoJsonGeometryType.MultiPoint and not elements: + # Points but no usable connectivity: export the cloud rather than nothing at all. + (logger or _MODULE_LOGGER).warning( + "GeoJSON export: %s carries %d point(s) but no %s element — exported as a point cloud.", + type(mesh).__name__, + len(points), + "line" if geo_kind == GeoJsonGeometryType.MultiLineString else "face", + ) + geo_kind = GeoJsonGeometryType.MultiPoint + + if explode_elements and geo_kind != GeoJsonGeometryType.MultiPoint: + single = _effective_geo_type(geo_kind, 1) + features_to_write = [ + (single, list(element), i, {**feature_properties, "element_index": i}) + for i, element in enumerate(elements) + ] + else: + features_to_write = [(_effective_geo_type(geo_kind, len(elements)), elements, None, feature_properties)] + if features_to_write[0][0] in (GeoJsonGeometryType.LineString, GeoJsonGeometryType.Polygon): + # a single element: the shape writer expects the flat index list, not [[...]] + features_to_write = [(features_to_write[0][0], elements[0], None, feature_properties)] + + for geo_type, indices, element_index, props in features_to_write: + if written > 0: + out.write(ind.sep()) + mins, maxs = _write_feature( + out=out, + geo_type=geo_type, + points=points, + indices=indices, + geo_type_prefix=prefix, + properties=props, + feature_id=_suffix_feature_id(base_id, element_index), + logger=logger, + ind=ind, + identifier=identifier, + ) + _recompute_min_max(collection_mins, collection_maxs, mins, maxs) + written += 1 + + out.write(ind.close()) + out.write(b"]") # end features + + # RFC 7946 §5: a FeatureCollection may carry a bbox. It is written after the features + # because the extents are only known once they have all been streamed out — member order + # carries no meaning in JSON. + if collection_mins and collection_maxs: + out.write(ind.sep()) + out.write(f'"bbox": {json.dumps(collection_mins + collection_maxs)}'.encode()) + + out.write(ind.close()) + out.write(b"}") # end geojson + + +def export_geojson_dict( + mesh_list: List["AbstractMesh"], + obj_name: Optional[str] = None, + properties: Optional[List[Optional[Dict]]] = None, + logger: Optional[Any] = None, + workspace: Optional["EnergymlStorageInterface"] = None, + include_metadata: bool = True, + to_wgs84: bool = True, + use_network: bool = False, +) -> Dict: + """ + Same as :func:`export_geojson_io` but returns a dict instead of streaming. + + It now runs the streaming writer and parses its output, so both variants share one + implementation of the geometry and one CRS pipeline. + + .. note:: + **Behaviour change.** This function used to leave the coordinates in their source CRS and + tag every feature ``"AnyCrsFeature"``, producing a document that was not RFC 7946 + conformant without saying so. It now reprojects to WGS84 like every other exporter. Pass + ``to_wgs84=False`` to get the previous output. + + :param to_wgs84: reproject the coordinates to WGS84 (RFC 7946). When the reprojection is not + possible, the source CRS is advertised through the ``crs`` / ``coordRefSys`` + members and the features keep the ``AnyCrs`` prefix. + :param use_network: allow PROJ to download the geoid grids used by the vertical transformation. + """ + buffer = BytesIO() + export_geojson_io( + out=buffer, + mesh_list=mesh_list, + obj_name=obj_name, + properties=properties, + logger=logger, + workspace=workspace, + to_wgs84=to_wgs84, + include_metadata=include_metadata, + use_network=use_network, + ) + return json.loads(buffer.getvalue().decode("utf-8")) + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "export_geojson", + "GeoJsonGeometryType", + "energyml_type_to_geojson_type", + "to_geojson_feature", + "write_geojson_feature", + "mesh_points", + "mesh_indices", + "mesh_to_geojson_type", + "export_geojson_io", + "export_geojson_dict", +] diff --git a/energyml-utils/src/energyml/utils/data/export/obj.py b/energyml-utils/src/energyml/utils/data/export/obj.py new file mode 100644 index 0000000..124a6c7 --- /dev/null +++ b/energyml-utils/src/energyml/utils/data/export/obj.py @@ -0,0 +1,182 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""Wavefront OBJ export (geometry + optional .mtl colour).""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Dict, Optional, BinaryIO + +from energyml.utils.data.export._base import ( + ExportFormat, + resolve_origin_shift, + _get_context_color, + _get_export_points, + _get_faces_or_cells, + _normalize_to_patches, + _parse_vtk_flat_faces, + _parse_vtk_flat_lines, + _workspace_from_contexts, +) + +from energyml.utils.data.export._registry import FormatSpec, register_format + +if TYPE_CHECKING: + from energyml.utils.data.crs import PointFrame + from energyml.utils.data.representation_context import RepresentationContext + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# OBJ export +# --------------------------------------------------------------------------- + + +def export_obj( + mesh_list: Any, + out: BinaryIO, + obj_name: Optional[str] = None, + contexts: Optional[Dict[str, "RepresentationContext"]] = None, + mtl_out: Optional[BinaryIO] = None, + use_crs_displacement: bool = True, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + use_network: bool = False, +) -> None: + """Export mesh data to Wavefront OBJ format. + + :param mesh_list: One or more meshes (``AbstractMesh``, ``NumpyMesh``, + ``NumpyMultiMesh``, or a list thereof). + :param out: Binary output stream for the ``.obj`` content. + :param obj_name: Optional object name written to the OBJ header. + :param contexts: Optional dict of :class:`RepresentationContext` keyed by + ``source_uuid``; used to emit companion ``.mtl`` material colours when + *mtl_out* is also provided. + :param mtl_out: Optional binary stream for the companion ``.mtl`` file. + Colour requires *contexts* to be supplied. + :param use_crs_displacement: When True (default), CRS origin offset and + axis transforms are applied to ``NumpyMesh`` points at export time. + """ + from energyml.utils.data.mesh import PolylineSetMesh + from energyml.utils.data.mesh_numpy import NumpyMesh, NumpyPointSetMesh, NumpyPolylineMesh + + patches = _normalize_to_patches(mesh_list) + workspace = _workspace_from_contexts(contexts) + _origin_shift = resolve_origin_shift(patches, use_crs_displacement, workspace, frame, origin_shift, use_network) + + out.write(b"# Generated by energyml-utils (Geosiris)\n\n") + if obj_name is not None: + out.write(f"o {obj_name}\n\n".encode()) + + mtl_lib_name = obj_name or "materials" + if mtl_out is not None: + out.write(f"mtllib {mtl_lib_name}.mtl\n\n".encode()) + mtl_out.write(b"# MTL generated by energyml-utils\n\n") + + point_offset = 0 + + for mesh in patches: + pts, _pts_frame, _ = _get_export_points( + mesh, use_crs_displacement, workspace, frame, _origin_shift, use_network + ) + patch_label = getattr(mesh, "patch_label", None) or getattr(mesh, "identifier", None) or "mesh" + source_uuid = getattr(mesh, "source_uuid", None) or getattr(mesh, "uuid", None) + patch_idx = getattr(mesh, "patch_index", None) + group_name = f"{source_uuid}_{patch_idx}" if source_uuid and patch_idx is not None else patch_label + + out.write(f"g {group_name}\n\n".encode()) + + # emit material reference when mtl output is available + if mtl_out is not None: + mat_name = f"mat_{group_name}" + color = _get_context_color(source_uuid, contexts) + if color is None: + color = (200, 200, 200, 255) + r, g, b, _a = color + out.write(f"usemtl {mat_name}\n".encode()) + mtl_out.write(f"newmtl {mat_name}\n".encode()) + mtl_out.write(f"Kd {r / 255:.6f} {g / 255:.6f} {b / 255:.6f}\n\n".encode()) + + # write vertices + for pt in pts: + out.write(f"v {pt[0]} {pt[1]} {pt[2]}\n".encode()) + + # write connectivity + if isinstance(mesh, NumpyMesh): + if isinstance(mesh, NumpyPointSetMesh): + # bare vertex elements + for i in range(len(pts)): + out.write(f"p {i + point_offset + 1}\n".encode()) + elif isinstance(mesh, NumpyPolylineMesh): + for seg in _parse_vtk_flat_lines(mesh.lines): + if len(seg) > 1: + idx_str = " ".join(str(i + point_offset + 1) for i in seg) + out.write(f"l {idx_str}\n".encode()) + else: + # NumpySurfaceMesh (or NumpyVolumeMesh — export as faces) + faces_arr = _get_faces_or_cells(mesh) + for face in _parse_vtk_flat_faces(faces_arr): + if len(face) >= 3: + idx_str = " ".join(str(i + point_offset + 1) for i in face) + out.write(f"f {idx_str}\n".encode()) + else: + # AbstractMesh legacy path + indices = mesh.get_indices() + elt = "l" if isinstance(mesh, PolylineSetMesh) else "f" + for elem in indices: + if len(elem) > 1: + idx_str = " ".join(str(i + point_offset + 1) for i in elem) + out.write(f"{elt} {idx_str}\n".encode()) + + out.write(b"\n") + point_offset += len(pts) + + +# --------------------------------------------------------------------------- +# Registration +# --------------------------------------------------------------------------- + + +def _write( + mesh_list: Any, + out: BinaryIO, + *, + obj_name: Optional[str] = None, + options: Any = None, + contexts: Optional[Dict[str, "RepresentationContext"]] = None, + use_crs_displacement: bool = True, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + companion: Optional[BinaryIO] = None, +) -> None: + """Uniform adapter used by the registry; ``options`` is unused by OBJ.""" + export_obj( + mesh_list, + out, + obj_name, + contexts, + companion, + use_crs_displacement, + frame=frame, + origin_shift=origin_shift, + ) + + +register_format( + FormatSpec( + format=ExportFormat.OBJ, + description="Wavefront OBJ — 3D geometry with optional .mtl colour", + filter_label="OBJ Files (*.obj)", + writer=_write, + binary=True, + options_class=None, + companion_suffix=".mtl", + ) +) + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "export_obj", +] diff --git a/energyml-utils/src/energyml/utils/data/export/off.py b/energyml-utils/src/energyml/utils/data/export/off.py new file mode 100644 index 0000000..10b47e7 --- /dev/null +++ b/energyml-utils/src/energyml/utils/data/export/off.py @@ -0,0 +1,194 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""OFF (Object File Format) export. + +Moved here from :mod:`energyml.utils.data.mesh`, which re-exports both functions so existing +imports keep working. The writer now accepts the numpy mesh hierarchy as well as the legacy one, +like every other format of this package. +""" + +from __future__ import annotations + +import logging +from io import BytesIO +from typing import TYPE_CHECKING, Any, BinaryIO, Dict, List, Optional + +import numpy as np + +from energyml.utils.data.export._base import ( + _FILE_HEADER, + ExportFormat, + _get_export_points, + _get_faces_or_cells, + _normalize_to_patches, + _parse_vtk_flat_faces, + _workspace_from_contexts, + resolve_origin_shift, +) +from energyml.utils.data.export._registry import FormatSpec, register_format + +if TYPE_CHECKING: + from energyml.utils.data.crs import PointFrame + from energyml.utils.data.representation_context import RepresentationContext + +logger = logging.getLogger(__name__) + + +def _face_list(mesh: Any) -> List[np.ndarray]: + """Return the faces of *mesh* as a list of index arrays, for either hierarchy.""" + from energyml.utils.data.mesh_numpy import NumpyMesh + + if isinstance(mesh, NumpyMesh): + return _parse_vtk_flat_faces(_get_faces_or_cells(mesh)) + return [np.asarray(face, dtype=np.int64) for face in (mesh.get_indices() or [])] + + +def _edge_count(mesh: Any, faces: List[np.ndarray]) -> int: + """Number written in the third field of the OFF header. + + ``AbstractMesh.get_nb_edges`` is used when present so the legacy output stays byte-identical + (it counts ``len(face) - 1`` per face, which undercounts a closed polygon by one — OFF readers + ignore this field, so the historical value is preserved rather than corrected). + """ + if hasattr(mesh, "get_nb_edges"): + return mesh.get_nb_edges() + return sum(max(len(f) - 1, 0) for f in faces) + + +def export_off( + mesh_list: Any, + out: BinaryIO, + contexts: Optional[Dict[str, "RepresentationContext"]] = None, + use_crs_displacement: bool = True, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + use_network: bool = False, +) -> None: + """Export mesh data to OFF format. + + :param mesh_list: One or more meshes (``AbstractMesh``, ``NumpyMesh``, ``NumpyMultiMesh``, + or a list thereof). + :param out: Binary output stream. + :param contexts: Color / metadata context dict (only used to reach a workspace here). + :param use_crs_displacement: Legacy switch selecting the default target frame. + :param frame: Explicit target :class:`~energyml.utils.data.crs.PointFrame`. + :param origin_shift: ``None``, ``"auto"``, or an explicit ``(dx, dy, dz)`` vector. + :param use_network: Allow PROJ to download the geoid grids (``PointFrame.WGS84`` only). + """ + patches = _normalize_to_patches(mesh_list) + workspace = _workspace_from_contexts(contexts) + _origin_shift = resolve_origin_shift(patches, use_crs_displacement, workspace, frame, origin_shift, use_network) + + points_io = BytesIO() + faces_io = BytesIO() + + nb_points = 0 + nb_faces = 0 + nb_edges = 0 + point_offset = 0 + + for mesh in patches: + pts, _pts_frame, _ = _get_export_points( + mesh, use_crs_displacement, workspace, frame, _origin_shift, use_network + ) + pts = np.asarray(pts, dtype=np.float64).reshape(-1, 3) + faces = _face_list(mesh) + + nb_points += len(pts) + nb_faces += len(faces) + nb_edges += _edge_count(mesh, faces) + + export_off_part( + off_point_part=points_io, + off_face_part=faces_io, + points=pts, + indices=faces, + point_offset=point_offset, + colors=[], + ) + point_offset += len(pts) + + out.write(b"OFF\n") + out.write(_FILE_HEADER) + out.write(f"{nb_points} {nb_faces} {nb_edges}\n".encode("utf-8")) + out.write(points_io.getbuffer()) + out.write(faces_io.getbuffer()) + + +def export_off_part( + off_point_part: BinaryIO, + off_face_part: BinaryIO, + points: Any, + indices: Any, + point_offset: Optional[int] = 0, + colors: Optional[List[List[int]]] = None, +) -> None: + """Append one mesh to the point and face sections of an OFF document. + + The two sections are written to separate streams because OFF wants all the vertices before + any face, while the meshes are walked one at a time. + """ + for p in points: + for pi in p: + off_point_part.write(f"{pi} ".encode("utf-8")) + off_point_part.write(b"\n") + + for cpt, face in enumerate(indices): + if len(face) > 1: + off_face_part.write(f"{len(face)} ".encode("utf-8")) + for pi in face: + off_face_part.write(f"{pi + point_offset} ".encode("utf-8")) + + if colors is not None and len(colors) > cpt and colors[cpt] is not None and len(colors[cpt]) > 0: + for col in colors[cpt]: + off_face_part.write(f"{col} ".encode("utf-8")) + + off_face_part.write(b"\n") + + +# --------------------------------------------------------------------------- +# Registration +# --------------------------------------------------------------------------- + + +def _write( + mesh_list: Any, + out: BinaryIO, + *, + obj_name: Optional[str] = None, + options: Any = None, + contexts: Optional[Dict[str, "RepresentationContext"]] = None, + use_crs_displacement: bool = True, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + companion: Any = None, +) -> None: + """Uniform adapter used by the registry; OFF takes no options.""" + export_off( + mesh_list, + out, + contexts, + use_crs_displacement, + frame=frame, + origin_shift=origin_shift, + ) + + +register_format( + FormatSpec( + format=ExportFormat.OFF, + description="OFF — Object File Format (vertices + faces, plain text)", + filter_label="OFF Files (*.off)", + writer=_write, + binary=True, + options_class=None, + ) +) + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "export_off", + "export_off_part", +] diff --git a/energyml-utils/src/energyml/utils/data/export/stl.py b/energyml-utils/src/energyml/utils/data/export/stl.py new file mode 100644 index 0000000..66157a0 --- /dev/null +++ b/energyml-utils/src/energyml/utils/data/export/stl.py @@ -0,0 +1,178 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""STL export (binary and ASCII), triangles only.""" + +from __future__ import annotations + +import logging +import struct +from typing import TYPE_CHECKING, Any, List, Optional, BinaryIO + +import numpy as np + +from energyml.utils.data.export._base import ( + ExportFormat, + resolve_origin_shift, + STLExportOptions, + _get_export_points, + _get_faces_or_cells, + _normalize_to_patches, + _parse_vtk_flat_faces, +) + +from energyml.utils.data.export._registry import FormatSpec, register_format + +if TYPE_CHECKING: + from energyml.utils.data.crs import PointFrame + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# STL export +# --------------------------------------------------------------------------- + + +def export_stl( + mesh_list: Any, + out: BinaryIO, + options: Optional[STLExportOptions] = None, + use_crs_displacement: bool = True, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + use_network: bool = False, +) -> None: + """Export triangulated mesh data to STL format (binary or ASCII). + + Non-triangular polygons are fan-triangulated (vertex 0 + consecutive pairs). + Polylines and point sets are silently skipped. + + :param mesh_list: Meshes to export. + :param out: Binary output stream. + :param options: STL export options. + :param use_crs_displacement: Apply CRS displacement to ``NumpyMesh`` points. + """ + from energyml.utils.data.mesh import SurfaceMesh + from energyml.utils.data.mesh_numpy import NumpyMesh, NumpyPolylineMesh, NumpyPointSetMesh + + if options is None: + options = STLExportOptions(binary=True) + + patches = _normalize_to_patches(mesh_list) + # STL carries no colour / context; workspace not needed unless CRS is requested. + workspace = None # CRS requires a workspace — callers may read with CRS pre-applied. + _origin_shift = resolve_origin_shift(patches, use_crs_displacement, workspace, frame, origin_shift, use_network) + + all_triangles: List[tuple] = [] + + for mesh in patches: + if isinstance(mesh, (NumpyPolylineMesh, NumpyPointSetMesh)): + continue # STL is surface-only + pts, _pts_frame, _ = _get_export_points( + mesh, use_crs_displacement, workspace, frame, _origin_shift, use_network + ) + pts_np = np.asarray(pts, dtype=np.float64).reshape(-1, 3) + + if isinstance(mesh, NumpyMesh): + face_list = _parse_vtk_flat_faces(_get_faces_or_cells(mesh)) + else: + if not isinstance(mesh, SurfaceMesh): + continue + face_list = mesh.get_indices() + + for face in face_list: + face = list(face) + if len(face) < 3: + continue + if len(face) == 3: + all_triangles.append((pts_np[face[0]], pts_np[face[1]], pts_np[face[2]])) + else: + # Fan triangulation for quads and polygons + for j in range(1, len(face) - 1): + all_triangles.append((pts_np[face[0]], pts_np[face[j]], pts_np[face[j + 1]])) + + if options.binary: + _export_stl_binary(all_triangles, out) + else: + _export_stl_ascii(all_triangles, out, options.ascii_precision) + + +def _compute_normal(p0: np.ndarray, p1: np.ndarray, p2: np.ndarray) -> np.ndarray: + v1, v2 = p1 - p0, p2 - p0 + n = np.cross(v1, v2) + norm = np.linalg.norm(n) + return n / norm if norm > 0 else np.zeros(3) + + +def _export_stl_binary(triangles: List[tuple], out: BinaryIO) -> None: + header = b"Binary STL file generated by energyml-utils" + b"\0" * (80 - 44) + out.write(header) + out.write(struct.pack(" None: + out.write(b"solid mesh\n") + for p0, p1, p2 in triangles: + normal = _compute_normal(p0, p1, p2) + out.write( + f" facet normal {normal[0]:.{precision}e} {normal[1]:.{precision}e} {normal[2]:.{precision}e}\n".encode() + ) + out.write(b" outer loop\n") + for pt in (p0, p1, p2): + out.write(f" vertex {pt[0]:.{precision}e} {pt[1]:.{precision}e} {pt[2]:.{precision}e}\n".encode()) + out.write(b" endloop\n endfacet\n") + out.write(b"endsolid mesh\n") + + +# --------------------------------------------------------------------------- +# Registration +# --------------------------------------------------------------------------- + + +def _write( + mesh_list: Any, + out: BinaryIO, + *, + obj_name: Optional[str] = None, + options: Any = None, + contexts: Any = None, + use_crs_displacement: bool = True, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + companion: Any = None, +) -> None: + """Uniform adapter used by the registry; STL carries no colour context.""" + export_stl( + mesh_list, + out, + options, + use_crs_displacement, + frame=frame, + origin_shift=origin_shift, + ) + + +register_format( + FormatSpec( + format=ExportFormat.STL, + description="STL — stereolithography (triangles only)", + filter_label="STL Files (*.stl)", + writer=_write, + binary=True, + options_class=STLExportOptions, + supports_lines=False, + supports_pointsets=False, + ) +) + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "export_stl", +] diff --git a/energyml-utils/src/energyml/utils/data/export/vtk.py b/energyml-utils/src/energyml/utils/data/export/vtk.py new file mode 100644 index 0000000..82e376a --- /dev/null +++ b/energyml-utils/src/energyml/utils/data/export/vtk.py @@ -0,0 +1,571 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""VTK export: legacy POLYDATA (ASCII / binary) and XML (.vtu / .vtp).""" + +from __future__ import annotations + +import base64 +import logging +import struct +from typing import TYPE_CHECKING, Any, Dict, List, Optional, BinaryIO + +import numpy as np + +from energyml.utils.data.export._base import ( + ExportFormat, + resolve_origin_shift, + VTKExportOptions, + VTKFormat, + _VTK_POLYGON, + _VTK_POLY_LINE, + _VTK_TRIANGLE, + _VTK_VERTEX, + _get_context_color, + _get_export_points, + _get_faces_or_cells, + _normalize_to_patches, + _parse_vtk_flat_faces, + _parse_vtk_flat_lines, + _workspace_from_contexts, +) + +from energyml.utils.data.export._registry import FormatSpec, register_format + +if TYPE_CHECKING: + from energyml.utils.data.crs import PointFrame + from energyml.utils.data.representation_context import RepresentationContext + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# VTK export — private helpers +# --------------------------------------------------------------------------- + + +def _b64_vtk(arr: np.ndarray) -> str: + """Base64-encode a numpy array for VTK XML inline binary format. + + VTK prepends a 4-byte uint32 header with the byte count of the payload. + """ + raw = arr.tobytes() + header = struct.pack(" str: + """Return a VTK XML ```` element string (base64 inline).""" + return ( + f'' + f"{_b64_vtk(arr)}" + f"" + ) + + +def _collect_vtk_geometry( + patches: List[Any], + use_crs_displacement: bool, + workspace: Any, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + use_network: bool = False, +) -> tuple: + """Merge all patches into flat VTK geometry arrays. + + Returns: + (all_pts, poly_conn, poly_off, line_conn, line_off, + vert_conn, vert_off, cell_types, patch_meta) + + *patch_meta* is a list of ``(source_uuid, n_cells)`` tuples used to + assign per-cell colour data. + """ + from energyml.utils.data.mesh import PolylineSetMesh + from energyml.utils.data.mesh_numpy import NumpyMesh, NumpyPointSetMesh, NumpyPolylineMesh + + all_pts: List[np.ndarray] = [] + poly_conn: List[int] = [] + poly_off: List[int] = [] + line_conn: List[int] = [] + line_off: List[int] = [] + vert_conn: List[int] = [] + vert_off: List[int] = [] + cell_types: List[int] = [] + patch_meta: List[tuple] = [] # (source_uuid, cell_count) + + pt_offset = 0 + + for mesh in patches: + pts, _pts_frame, _ = _get_export_points(mesh, use_crs_displacement, workspace, frame, origin_shift, use_network) + all_pts.append(np.asarray(pts, dtype=np.float64).reshape(-1, 3)) + source_uuid = getattr(mesh, "source_uuid", None) + cell_count = 0 + + if isinstance(mesh, NumpyMesh): + if isinstance(mesh, NumpyPointSetMesh): + for i in range(len(pts)): + vert_conn.append(i + pt_offset) + vert_off.append(len(vert_conn)) + cell_types.append(_VTK_VERTEX) + cell_count += 1 + elif isinstance(mesh, NumpyPolylineMesh): + for seg in _parse_vtk_flat_lines(mesh.lines): + for vi in seg: + line_conn.append(int(vi) + pt_offset) + line_off.append(len(line_conn)) + cell_types.append(_VTK_POLY_LINE) + cell_count += 1 + else: + faces_arr = _get_faces_or_cells(mesh) + for face in _parse_vtk_flat_faces(faces_arr): + nv = len(face) + for vi in face: + poly_conn.append(int(vi) + pt_offset) + poly_off.append(len(poly_conn)) + cell_types.append(_VTK_TRIANGLE if nv == 3 else _VTK_POLYGON) + cell_count += 1 + else: + # AbstractMesh legacy + indices = mesh.get_indices() + if isinstance(mesh, PolylineSetMesh): + for line in indices: + for vi in line: + line_conn.append(int(vi) + pt_offset) + line_off.append(len(line_conn)) + cell_types.append(_VTK_POLY_LINE) + cell_count += 1 + else: + for face in indices: + nv = len(face) + for vi in face: + poly_conn.append(int(vi) + pt_offset) + poly_off.append(len(poly_conn)) + cell_types.append(_VTK_TRIANGLE if nv == 3 else _VTK_POLYGON) + cell_count += 1 + + pt_offset += len(pts) + patch_meta.append((source_uuid, cell_count)) + + merged_pts = np.concatenate(all_pts) if all_pts else np.empty((0, 3), dtype=np.float64) + return ( + merged_pts, + np.array(poly_conn, dtype=np.int64), + np.array(poly_off, dtype=np.int64), + np.array(line_conn, dtype=np.int64), + np.array(line_off, dtype=np.int64), + np.array(vert_conn, dtype=np.int64), + np.array(vert_off, dtype=np.int64), + np.array(cell_types, dtype=np.uint8), + patch_meta, + ) + + +def _build_color_scalars( + patch_meta: List[tuple], + contexts: Optional[Dict[str, Any]], + total_cells: int, +) -> Optional[np.ndarray]: + """Build a ``(total_cells, 4)`` float32 RGBA array, or None when no colors found.""" + if not contexts: + return None + colors = np.full((total_cells, 4), 0.8, dtype=np.float32) + colors[:, 3] = 1.0 + any_found = False + cell_idx = 0 + for source_uuid, n_cells in patch_meta: + rgba = _get_context_color(source_uuid, contexts) + if rgba is not None: + any_found = True + r, g, b, a = rgba + colors[cell_idx : cell_idx + n_cells, 0] = r / 255.0 + colors[cell_idx : cell_idx + n_cells, 1] = g / 255.0 + colors[cell_idx : cell_idx + n_cells, 2] = b / 255.0 + colors[cell_idx : cell_idx + n_cells, 3] = a / 255.0 + cell_idx += n_cells + return colors if any_found else None + + +# --------------------------------------------------------------------------- +# VTK export — legacy (ASCII / binary) +# --------------------------------------------------------------------------- + + +def _export_vtk_legacy( + patches: List[Any], + out: BinaryIO, + options: VTKExportOptions, + contexts: Optional[Dict[str, Any]], + workspace: Any, + frame: Optional["PointFrame"] = None, + _origin_shift: Optional[Any] = None, + use_network: bool = False, +) -> None: + ascii_mode = options.vtk_format == VTKFormat.LEGACY_ASCII + ( + all_pts, + poly_conn, + poly_off, + line_conn, + line_off, + vert_conn, + vert_off, + cell_types, + patch_meta, + ) = _collect_vtk_geometry(patches, True, workspace, frame, _origin_shift, use_network) + + n_pts = len(all_pts) + n_poly = len(poly_off) + n_line = len(line_off) + n_vert = len(vert_off) + + def _unflatten(conn: np.ndarray, offs: np.ndarray) -> List[List[int]]: + result = [] + prev = 0 + for o in offs: + result.append(conn[prev:o].tolist()) + prev = o + return result + + polygons = _unflatten(poly_conn, poly_off) + lines = _unflatten(line_conn, line_off) + verts = _unflatten(vert_conn, vert_off) + + out.write(b"# vtk DataFile Version 3.0\n") + out.write(f"{options.dataset_name}\n".encode()) + out.write(b"ASCII\n" if ascii_mode else b"BINARY\n") + out.write(b"DATASET POLYDATA\n") + + if ascii_mode: + out.write(f"POINTS {n_pts} float\n".encode()) + for pt in all_pts: + out.write(f"{pt[0]} {pt[1]} {pt[2]}\n".encode()) + else: + out.write(f"POINTS {n_pts} float\n".encode()) + out.write(all_pts.astype(">f4").tobytes()) + out.write(b"\n") + + def _write_section(name: str, cells: List[List[int]]) -> None: + if not cells: + return + total = sum(len(c) + 1 for c in cells) + out.write(f"{name} {len(cells)} {total}\n".encode()) + if ascii_mode: + for c in cells: + out.write(f"{len(c)} {' '.join(str(i) for i in c)}\n".encode()) + else: + for c in cells: + row = np.array([len(c)] + c, dtype=np.int32).byteswap().astype(">i4") + out.write(row.tobytes()) + out.write(b"\n") + + _write_section("POLYGONS", polygons) + _write_section("LINES", lines) + _write_section("VERTICES", verts) + + total_cells = n_poly + n_line + n_vert + if total_cells > 0 and contexts: + colors = _build_color_scalars(patch_meta, contexts, total_cells) + if colors is not None: + out.write(f"CELL_DATA {total_cells}\n".encode()) + out.write(b"COLOR_SCALARS patch_color 4\n") + if ascii_mode: + for row in colors: + out.write(f"{row[0]:.6f} {row[1]:.6f} {row[2]:.6f} {row[3]:.6f}\n".encode()) + else: + out.write(colors.astype(">f4").tobytes()) + out.write(b"\n") + + +# --------------------------------------------------------------------------- +# VTK export — XML VTU +# --------------------------------------------------------------------------- + + +def _export_vtk_vtu( + patches: List[Any], + out: BinaryIO, + options: VTKExportOptions, + contexts: Optional[Dict[str, Any]], + workspace: Any, + frame: Optional["PointFrame"] = None, + _origin_shift: Optional[Any] = None, + use_network: bool = False, +) -> None: + """Write VTK XML UnstructuredGrid (.vtu).""" + ( + all_pts, + poly_conn, + poly_off, + line_conn, + line_off, + vert_conn, + vert_off, + cell_types, + patch_meta, + ) = _collect_vtk_geometry(patches, True, workspace, frame, _origin_shift, use_network) + + # Build a single merged connectivity / offsets / types for UnstructuredGrid. + conn_parts: List[np.ndarray] = [] + off_parts: List[int] = [] + types_list: List[int] = [] + running = 0 + + def _add_vtu_section(conn: np.ndarray, offs: np.ndarray, default_type: int) -> None: + nonlocal running + prev = 0 + for o in offs: + seg = conn[prev:o] + conn_parts.append(seg) + running += len(seg) + off_parts.append(running) + types_list.append(default_type) + prev = o + + _add_vtu_section(vert_conn, vert_off, _VTK_VERTEX) + _add_vtu_section(line_conn, line_off, _VTK_POLY_LINE) + + # Polygons: honour per-cell type from cell_types array (triangle vs polygon). + n_verts_cells = len(vert_off) + n_lines_cells = len(line_off) + prev = 0 + for poly_i, o in enumerate(poly_off): + seg = poly_conn[prev:o] + conn_parts.append(seg) + running += len(seg) + off_parts.append(running) + abs_idx = n_verts_cells + n_lines_cells + poly_i + types_list.append(int(cell_types[abs_idx]) if abs_idx < len(cell_types) else _VTK_POLYGON) + prev = o + + all_conn = ( + np.concatenate([np.asarray(p, dtype=np.int64) for p in conn_parts]) + if conn_parts + else np.empty(0, dtype=np.int64) + ) + all_off = np.array(off_parts, dtype=np.int64) + all_types = np.array(types_list, dtype=np.uint8) + n_cells = len(all_types) + n_pts = len(all_pts) + + xml_lines: List[str] = [ + '', + '', + " ", + f' ', + " ", + " " + _vtk_xml_data_array("Points", all_pts.astype(np.float32).ravel(), 3, "Float32"), + " ", + " ", + " " + _vtk_xml_data_array("connectivity", all_conn, 1, "Int64"), + " " + _vtk_xml_data_array("offsets", all_off, 1, "Int64"), + " " + _vtk_xml_data_array("types", all_types, 1, "UInt8"), + " ", + ] + + if contexts and n_cells > 0: + colors = _build_color_scalars(patch_meta, contexts, n_cells) + if colors is not None: + xml_lines.append(" ") + xml_lines.append(" " + _vtk_xml_data_array("patch_color", colors.ravel(), 4, "Float32")) + xml_lines.append(" ") + + xml_lines += [" ", " ", ""] + out.write("\n".join(xml_lines).encode("utf-8")) + + +# --------------------------------------------------------------------------- +# VTK export — XML VTP +# --------------------------------------------------------------------------- + + +def _export_vtk_vtp( + patches: List[Any], + out: BinaryIO, + options: VTKExportOptions, + contexts: Optional[Dict[str, Any]], + workspace: Any, + frame: Optional["PointFrame"] = None, + _origin_shift: Optional[Any] = None, + use_network: bool = False, +) -> None: + """Write VTK XML PolyData (.vtp).""" + ( + all_pts, + poly_conn, + poly_off, + line_conn, + line_off, + vert_conn, + vert_off, + cell_types, + patch_meta, + ) = _collect_vtk_geometry(patches, True, workspace, frame, _origin_shift, use_network) + + n_pts = len(all_pts) + n_polys = len(poly_off) + n_lines = len(line_off) + n_verts = len(vert_off) + total_cells = n_polys + n_lines + n_verts + + xml_lines: List[str] = [ + '', + '', + " ", + ( + f' ' + ), + " ", + " " + _vtk_xml_data_array("Points", all_pts.astype(np.float32).ravel(), 3, "Float32"), + " ", + ] + + def _topo_section(tag: str, conn: np.ndarray, offs: np.ndarray) -> List[str]: + return [ + f" <{tag}>", + " " + _vtk_xml_data_array("connectivity", conn, 1, "Int64"), + " " + _vtk_xml_data_array("offsets", offs, 1, "Int64"), + f" ", + ] + + if n_polys: + xml_lines.extend(_topo_section("Polys", poly_conn, poly_off)) + if n_lines: + xml_lines.extend(_topo_section("Lines", line_conn, line_off)) + if n_verts: + xml_lines.extend(_topo_section("Verts", vert_conn, vert_off)) + + if contexts and total_cells > 0: + colors = _build_color_scalars(patch_meta, contexts, total_cells) + if colors is not None: + xml_lines.append(" ") + xml_lines.append(" " + _vtk_xml_data_array("patch_color", colors.ravel(), 4, "Float32")) + xml_lines.append(" ") + + xml_lines += [" ", " ", ""] + out.write("\n".join(xml_lines).encode("utf-8")) + + +# --------------------------------------------------------------------------- +# VTK export — public entry point +# --------------------------------------------------------------------------- + + +def export_vtk( + mesh_list: Any, + out: BinaryIO, + options: Optional[VTKExportOptions] = None, + contexts: Optional[Dict[str, "RepresentationContext"]] = None, + use_crs_displacement: bool = True, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + use_network: bool = False, +) -> None: + """Export mesh data to a VTK format. + + The sub-format is controlled by ``options.vtk_format`` (default: + ``VTKFormat.LEGACY_ASCII``). Supported variants: + + * **LEGACY_ASCII** — VTK 3.0 POLYDATA, ASCII encoding + * **LEGACY_BINARY** — VTK 3.0 POLYDATA, big-endian binary encoding + * **VTU** — VTK XML UnstructuredGrid (``.vtu``), base64 inline binary + * **VTP** — VTK XML PolyData (``.vtp``), base64 inline binary + + :param mesh_list: Meshes to export. + :param out: Binary output stream. + :param options: VTK export options. + :param contexts: Optional colour context dict keyed by ``source_uuid``. + :param use_crs_displacement: Apply CRS displacement to ``NumpyMesh`` points. + """ + if options is None: + options = VTKExportOptions() + + patches = _normalize_to_patches(mesh_list) + # Pass workspace only when CRS displacement is actually requested. + workspace = _workspace_from_contexts(contexts) if use_crs_displacement else None + _origin_shift = resolve_origin_shift(patches, use_crs_displacement, workspace, frame, origin_shift, use_network) + + fmt = options.vtk_format + if fmt in (VTKFormat.LEGACY_ASCII, VTKFormat.LEGACY_BINARY): + _export_vtk_legacy(patches, out, options, contexts, workspace, frame, _origin_shift, use_network) + elif fmt == VTKFormat.VTU: + _export_vtk_vtu(patches, out, options, contexts, workspace, frame, _origin_shift, use_network) + elif fmt == VTKFormat.VTP: + _export_vtk_vtp(patches, out, options, contexts, workspace, frame, _origin_shift, use_network) + else: # pragma: no cover + raise ValueError(f"Unknown VTKFormat: {fmt}") + + +# --------------------------------------------------------------------------- +# Registration +# --------------------------------------------------------------------------- + + +def _write( + mesh_list: Any, + out: BinaryIO, + *, + obj_name: Optional[str] = None, + options: Any = None, + contexts: Optional[Dict[str, "RepresentationContext"]] = None, + use_crs_displacement: bool = True, + frame: Optional["PointFrame"] = None, + origin_shift: Optional[Any] = None, + companion: Any = None, +) -> None: + """Uniform adapter used by the registry.""" + export_vtk( + mesh_list, + out, + options, + contexts, + use_crs_displacement, + frame=frame, + origin_shift=origin_shift, + ) + + +for _fmt, _label, _desc, _sub in ( + ( + ExportFormat.VTK, + "VTK Files (*.vtk)", + "VTK Legacy (ASCII or binary) — POLYDATA format", + None, + ), + ( + ExportFormat.VTU, + "VTK XML UnstructuredGrid Files (*.vtu)", + "VTK XML UnstructuredGrid (.vtu) — volumes + mixed topologies", + VTKFormat.VTU, + ), + ( + ExportFormat.VTP, + "VTK XML PolyData Files (*.vtp)", + "VTK XML PolyData (.vtp) — surfaces and polylines", + VTKFormat.VTP, + ), +): + # .vtu / .vtp share this writer and only pin its sub-format, which the registry applies + # through force_options instead of an extra branch in the dispatcher. + register_format( + FormatSpec( + format=_fmt, + description=_desc, + filter_label=_label, + writer=_write, + binary=True, + options_class=VTKExportOptions, + force_options={"vtk_format": _sub} if _sub is not None else None, + ) + ) + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "export_vtk", +] diff --git a/energyml-utils/src/energyml/utils/data/helper.py b/energyml-utils/src/energyml/utils/data/helper.py index afe28f6..0962b4f 100644 --- a/energyml-utils/src/energyml/utils/data/helper.py +++ b/energyml-utils/src/energyml/utils/data/helper.py @@ -94,7 +94,7 @@ def is_z_reversed(crs: Optional[Any], workspace: Optional[EnergymlStorageInterfa :return: By default, ``False`` is returned when *crs* is ``None``. """ result = extract_crs_info(crs, workspace).z_increasing_downward - # logging.debug(f"is_z_reversed: {result}") + # logger.debug(f"is_z_reversed: {result}") return result @@ -222,7 +222,7 @@ def get_crs_origin_offset(crs_obj: Any) -> np.ndarray: dtype=np.float64, ) except Exception as e: - logging.info(f"ERR reading crs offset {e}") + logger.info(f"ERR reading crs offset {e}") return np.zeros(3, dtype=np.float64) @@ -311,18 +311,18 @@ def prod_n_tab(val: Union[float, int, str], tab: List[Union[float, int, str]]): """ if val is None: return [None] * len(tab) - # logging.debug(f"Multiplying list by {val}: {tab}") + # logger.debug(f"Multiplying list by {val}: {tab}") # Convert to numpy array for vectorized operations, handling None values arr = np.array(tab, dtype=object) - # logging.debug(f"arr: {arr}") + # logger.debug(f"arr: {arr}") # Create mask for non-None values mask = arr != None # noqa: E711 # Create result array filled with None result = np.full(len(tab), None, dtype=object) - # logging.debug(f"result before multiplication: {result}") + # logger.debug(f"result before multiplication: {result}") # Multiply only non-None values result[mask] = arr[mask].astype(float) * val - # logging.debug(f"result after multiplication: {result}") + # logger.debug(f"result after multiplication: {result}") return result.tolist() @@ -379,25 +379,25 @@ def get_crs_obj( :return: """ if workspace is None: - logging.error("@get_crs_obj no Epc file given") + logger.error("@get_crs_obj no Epc file given") else: crs_list = search_attribute_matching_name(context_obj, r"\.*Crs", search_in_sub_obj=True, deep_search=False) if crs_list is not None and len(crs_list) > 0 and crs_list[0] is not None: - # logging.debug(crs_list[0]) + # logger.debug(crs_list[0]) crs = workspace.get_object(get_obj_uri(crs_list[0])) - # logging.debug(f"CRS found for {get_obj_title(context_obj)} ({type(context_obj).__name__}): {crs}") + # logger.debug(f"CRS found for {get_obj_title(context_obj)} ({type(context_obj).__name__}): {crs}") if crs is None: # if a wrong version is written in DOR - # logging.debug(f"CRS {crs_list[0]} not found (or not read correctly)") + # logger.debug(f"CRS {crs_list[0]} not found (or not read correctly)") _crs_list = workspace.get_object_by_uuid(get_obj_uuid(crs_list[0])) crs = _crs_list[0] if _crs_list is not None and len(_crs_list) > 0 else None if crs is None: - logging.error(f"CRS {crs_list[0]} not found (or not read correctly)") + logger.error(f"CRS {crs_list[0]} not found (or not read correctly)") raise ObjectNotFoundNotError(get_obj_uri(crs_list[0])) if crs is not None: return crs else: - logging.debug(f"No CRS found for {get_obj_title(context_obj)} with type {type(context_obj).__name__}") + logger.debug(f"No CRS found for {get_obj_title(context_obj)} with type {type(context_obj).__name__}") if context_obj != root_obj: upper_path = path_parent_attribute(path_in_root) @@ -410,9 +410,85 @@ def get_crs_obj( workspace=workspace, ) + # Nothing named a CRS anywhere up the object. `PointGeometry.LocalCrs` is optional in + # RESQML 2.2, and a package may simply declare its CRS once and let every representation + # inherit it — the geometry then carries no reference at all. Fall back to the package's + # CRS when it is unambiguous, rather than returning None and leaving the coordinates + # unprojected and unlabelled. + return get_package_default_crs(workspace) + return None +#: Sentinel telling "not looked up yet" apart from "looked up, found nothing". +_NO_DEFAULT_CRS = object() + + +def _crs_rank(type_name: str) -> Optional[int]: + """Rank a CRS class name by how completely it describes a representation's frame.""" + lowered = type_name.lower() + if any(k in lowered for k in ("localengineeringcompoundcrs", "localdepth3dcrs", "localtime3dcrs", "local3dcrs")): + return 0 # a full local frame: offsets, rotation and the projected/vertical chain + if "localengineering2dcrs" in lowered: + return 1 + if lowered.endswith("projectedcrs"): + return 2 # no local frame, but the horizontal EPSG code is what the reprojection needs + return None + + +def get_package_default_crs(workspace: Optional[EnergymlStorageInterface]) -> Optional[Any]: + """Return the CRS of a package that declares one unambiguously, or ``None``. + + Used only when a representation names no CRS. The best-described kind wins — a full local + 3-D / compound CRS over a bare ``ProjectedCrs`` — and the answer is only accepted when a + single object of that kind exists: picking one of several would silently place the geometry + in the wrong frame, which is worse than not projecting it. + + A standalone ``VerticalCrs`` is deliberately never returned on its own; when several exist + (a package may carry both an MSL-height and an MSL-depth CRS) there is no way to tell which + a given representation meant. + """ + if workspace is None: + return None + + cached = getattr(workspace, "_energyml_default_crs", _NO_DEFAULT_CRS) + if cached is not _NO_DEFAULT_CRS: + return cached + + result = None + try: + candidates: Dict[int, List[Any]] = {} + for metadata in workspace.list_objects(resolve_titles=False): + rank = _crs_rank(metadata.object_type or "") + if rank is not None: + candidates.setdefault(rank, []).append(metadata) + + for rank in sorted(candidates): + found = candidates[rank] + if len(found) == 1: + result = workspace.get_object(str(found[0].uri)) + if result is not None: + logger.info( + f"No CRS referenced by the representation; falling back to the only " + f"{type(result).__name__} of the package ({found[0].uuid})." + ) + break + logger.warning( + f"No CRS referenced by the representation and the package declares " + f"{len(found)} {found[0].object_type} — none can be chosen, the coordinates " + "stay in their source frame." + ) + break + except Exception as exc: + logger.debug(f"Cannot look for a package default CRS: {type(exc).__name__}: {exc}") + + try: + workspace._energyml_default_crs = result + except Exception: + pass # a workspace that refuses attributes just pays the lookup again + return result + + def linear_interpolation(md_target, md_start, md_end, p_start, p_end): """ Calcule la position 3D par interpolation linéaire simple. @@ -546,10 +622,50 @@ def _natural_cubic_spline_eval( "Install it with: pip install scipy or pip install energyml-utils[geometry]" ) from exc + if len(ctrl_params) < 3: + # RESQML special case (1): "Natural cubic splines with only two control + # points reduce to linear interpolation." CubicSpline also rejects n < 3 + # for bc_type="natural". + return _interp1d_vectorized(ctrl_params, ctrl_pts, query) + cs = CubicSpline(ctrl_params, ctrl_pts, bc_type="natural") return cs(query) # shape (Q, d) +def _trim_nan_knots( + ctrl_params: Optional[np.ndarray], # (K,) or None + ctrl_pts: np.ndarray, # (K, 3) + tangents: Optional[np.ndarray], # (K, 3) or None +) -> Tuple[Optional[np.ndarray], np.ndarray, Optional[np.ndarray]]: + """Drop the NaN padding of a single parametric line. + + ``KnotCount`` is the *maximum* number of control points over the whole array of lines, and + the RESQML documentation of ``ParametricLineArray`` states for both ``ControlPoints`` and + ``ControlPointParameters``: "If you cannot provide enough control points for a parametric + line, then pad with NaN values." A straight pillar next to a cubic one therefore carries + one real knot and ``KnotCount - 1`` NaN knots. + + Nothing trimmed them, so a single NaN reached ``np.interp`` / ``CubicSpline`` / + ``np.searchsorted`` and turned the whole pillar into NaN — silently, since NaN coordinates + do not raise. ``rc/epc/80wells_surf_modified_val_color.epc`` has such a grid ("Four faulted + sugar cubes with one cubic pillar": 3 knots, but four of its six pillars are vertical). + + A knot is kept when its point *and* its parameter are finite; the retained knots keep their + original order, which preserves the "strictly monotonically increasing" business rule. + """ + valid = np.isfinite(ctrl_pts).all(axis=1) + if ctrl_params is not None: + valid &= np.isfinite(ctrl_params) + if valid.all(): + return ctrl_params, ctrl_pts, tangents + + return ( + ctrl_params[valid] if ctrl_params is not None else None, + ctrl_pts[valid], + tangents[valid] if tangents is not None else None, + ) + + def _minimum_curvature_eval( ctrl_params: np.ndarray, # (K,) P-values (e.g. depth) at knots ctrl_pts: np.ndarray, # (K, 3) XYZ at knots @@ -628,16 +744,36 @@ def _evaluate_one_pillar( return np.full((Q, 3), np.nan, dtype=np.float64) if kind == _PARAMETRIC_KIND_VERTICAL: - # Only X, Y stored; Z coordinate = P-value. - # ctrl_pts shape (K, 2) or (K, 3) — take only first two coords regardless. - x = float(ctrl_pts[0, 0]) - y = float(ctrl_pts[0, 1]) + # RESQML: "Vertical: (1) Control points are (X,Y,-). (2) Parameter values are + # interpreted as depth => (X,Y,Z)". Only knot 0 carries the X/Y of the line. + if len(ctrl_pts) == 0 or not np.isfinite(ctrl_pts[0, :2]).all(): + return np.full((Q, 3), np.nan, dtype=np.float64) out = np.empty((Q, 3), dtype=np.float64) - out[:, 0] = x - out[:, 1] = y + out[:, 0] = float(ctrl_pts[0, 0]) + out[:, 1] = float(ctrl_pts[0, 1]) out[:, 2] = query_params return out + # Every remaining kind interpolates over (P, X, Y, Z), so the NaN padding that + # ``KnotCount`` forces onto the shorter lines has to go before anything touches it. + ctrl_params, ctrl_pts, tangents = _trim_nan_knots(ctrl_params, ctrl_pts, tangents) + + if len(ctrl_pts) == 0: + return np.full((Q, 3), np.nan, dtype=np.float64) + if len(ctrl_pts) == 1: + # A single knot defines a point, not a line: the interpolant is constant. + return np.repeat(ctrl_pts[np.newaxis, 0, :3], Q, axis=0).astype(np.float64) + + if ctrl_params is None: + # ControlPointParameters is optional in the schema but required to interpolate a + # non-vertical line. Z is the conventional parameter, and is what a writer omitting + # the array implies; warn rather than return NaN for the whole pillar. + logger.warning( + f"Parametric line kind={kind} has no control_point_parameters; " + "using the Z coordinate of the control points as the parameter." + ) + ctrl_params = np.ascontiguousarray(ctrl_pts[:, 2], dtype=np.float64) + if kind == _PARAMETRIC_KIND_LINEAR: return _interp1d_vectorized(ctrl_params, ctrl_pts[:, :3], query_params) @@ -646,7 +782,7 @@ def _evaluate_one_pillar( if kind == _PARAMETRIC_KIND_HERMITE: if tangents is None: - logging.warning( + logger.warning( "Pillar kind=3 (tangential cubic Hermite) requested but no tangent_vectors " "found — falling back to linear interpolation." ) @@ -680,7 +816,7 @@ def _evaluate_one_pillar( if kind == _PARAMETRIC_KIND_MIN_CURVATURE: if tangents is None: - logging.warning( + logger.warning( "Pillar kind=5 (minimum-curvature) requested but no tangent_vectors " "found — falling back to linear interpolation." ) @@ -688,7 +824,7 @@ def _evaluate_one_pillar( return _minimum_curvature_eval(ctrl_params, ctrl_pts[:, :3], tangents, query_params) # Unknown kind: warn and fall back to linear. - logging.warning(f"Unknown parametric line kind={kind}; falling back to linear interpolation.") + logger.warning(f"Unknown parametric line kind={kind}; falling back to linear interpolation.") return _interp1d_vectorized(ctrl_params, ctrl_pts[:, :3], query_params) @@ -780,10 +916,11 @@ def evaluate_parametric_line_array( pla: Any, root_obj: Any, workspace: Optional[EnergymlStorageInterface], - query_parameters: np.ndarray, # shape (NKL, n_pillars) + query_parameters: np.ndarray, # shape (NKL, n_columns) ni: int, nj: int, -) -> np.ndarray: # shape (NKL, n_pillars, 3) float64 + line_indices: Optional[np.ndarray] = None, # (n_columns,) column → parametric line +) -> np.ndarray: # shape (NKL, n_columns, 3) float64 """ Evaluate a ``ParametricLineArray`` at the given query P-values and return 3-D Cartesian coordinates for every grid node. @@ -791,26 +928,62 @@ def evaluate_parametric_line_array( This is the core of the ``Point3dParametricArray`` reader for :func:`read_numpy_ijk_grid_representation`. + **The number of parametric lines is not the number of node columns.** A faulted + column-layer grid has ``(NI+1)(NJ+1) + splitCount`` coordinate lines but the + ``ParametricLineArray`` only stores the ``(NI+1)(NJ+1)`` pillars: a split coordinate line + reuses the parametric line of the pillar it was split from and differs only by its + P-values, which is exactly how a fault throw is expressed on a parametric geometry. + ``ColumnLayerSplitCoordinateLines.PillarIndices`` carries that mapping — the RESQML + documentation of ``Point3dParametricArray.ParametricLineIndices`` names it as the reason + the explicit index array may be omitted ("If the mapping has already been specified, as + with the pillar Index from the column-layer geometry of a grid"). + + Sizing the control-point array from ``query_parameters.shape[1]`` therefore over-counted + the lines by ``splitCount``; the leftover factor was absorbed into a "coordinate + dimension" that came out as 2 and the reshape raised + ``cannot reshape array of size 18 into shape (1,8,2)``. ``ControlPoints`` is an + ``AbstractPoint3dArray``, so the coordinate dimension is always 3 and the line count is + what has to be derived. + :param pla: A ``ParametricLineArray`` instance (or duck-typed ``SimpleNamespace`` from :func:`resolve_parametric_line_array`). :param root_obj: Root RESQML object — passed to :func:`read_array` for external-dataset resolution. :param workspace: Workspace used for HDF5 reads. - :param query_parameters: ``(NKL, n_pillars)`` array of parametric - P-values (usually depth) at which to evaluate each pillar. + :param query_parameters: ``(NKL, n_columns)`` array of parametric + P-values (usually depth) at which to evaluate each node column. :param ni: Grid cell count in the I direction (``NI``). :param nj: Grid cell count in the J direction (``NJ``). - :return: ``(NKL, n_pillars, 3)`` float64 array of evaluated XYZ positions. + :param line_indices: Optional ``(n_columns,)`` mapping from node column to parametric + line index. Defaults to the identity when the counts already match. + :return: ``(NKL, n_columns, 3)`` float64 array of evaluated XYZ positions. :raises ValueError: If mandatory arrays (control_points, line_kind_indices) cannot be read. :raises ImportError: Propagated from :func:`_natural_cubic_spline_eval` when scipy is missing and kind-2 / kind-4 pillars are present. """ - nkl, n_pillars = query_parameters.shape + nkl, n_columns = query_parameters.shape + + knot_count: int = int(getattr(pla, "knot_count", None) or 1) - knot_count: int = getattr(pla, "knot_count", None) + # --- 1. Read line_kind_indices — it is what states how many lines there are --- + # "line_kind_indices: An array of integers indicating the parametric line kind. [...] + # Size = #Lines". The line count cannot be inferred from control_points alone: a + # (KnotCount, #Lines, 3) array and a (KnotCount, 1.5·#Lines, 2) one have the same number of + # values, and back-solving it from the *expected* pillar count is what raised + # "cannot reshape array of size 18 into shape (1,8,2)" on every faulted parametric grid. + lki_obj = getattr(pla, "line_kind_indices", None) + if lki_obj is None: + raise ValueError("ParametricLineArray.line_kind_indices is required but absent.") + raw_lki = read_array(energyml_array=lki_obj, root_obj=root_obj, workspace=workspace) + if not isinstance(raw_lki, np.ndarray): + raw_lki = np.array(raw_lki, dtype=np.int32) + kinds: np.ndarray = raw_lki.astype(np.int32).flatten() + n_lines = len(kinds) + if n_lines == 0: + raise ValueError("ParametricLineArray.line_kind_indices is empty.") - # --- 1. Read control_points --- + # --- 2. Read control_points --- cp_obj = getattr(pla, "control_points", None) if cp_obj is None: raise ValueError("ParametricLineArray.control_points is required but absent.") @@ -819,26 +992,57 @@ def evaluate_parametric_line_array( raw_cp = np.array(raw_cp, dtype=np.float64) raw_cp = raw_cp.astype(np.float64) - # Determine coordinate dimension (2 for vertical-only, 3 otherwise). - # The flat array has K*P*d values; we disambiguate using knot_count and n_pillars. - n_pillars_base = (ni + 1) * (nj + 1) - coord_dim = raw_cp.size // (knot_count * n_pillars) if knot_count and knot_count * n_pillars > 0 else 3 - if coord_dim not in (2, 3): - # Fallback: try 4-D layout (knot, NJ+1, NI+1, d) - if raw_cp.size == knot_count * (nj + 1) * (ni + 1) * 3: - raw_cp = raw_cp.reshape(knot_count, nj + 1, ni + 1, 3) - raw_cp = raw_cp.reshape(knot_count, n_pillars_base, 3) - coord_dim = 3 - else: - coord_dim = 3 # safe default - ctrl_pts = raw_cp.reshape(knot_count, n_pillars, coord_dim) - - # Optional column selection for ParametricLineFromRepresentationLatticeArray. + # "Control points are ordered by lines going fastest, then by knots going slowest" + # → (KnotCount, #Lines, coord_dim), whatever shape the writer gave the HDF5 dataset. + coord_dim = raw_cp.size // (knot_count * n_lines) if knot_count * n_lines else 0 + if coord_dim * knot_count * n_lines != raw_cp.size or coord_dim not in (2, 3): + raise ValueError( + f"ParametricLineArray.control_points holds {raw_cp.size} values, which is not " + f"knot_count({knot_count}) × #Lines({n_lines}) × 2 or 3." + ) + if coord_dim == 3: + ctrl_pts = raw_cp.reshape(knot_count, n_lines, 3) + else: + # ControlPoints is an AbstractPoint3dArray, so a conformant writer stores three + # coordinates even for vertical lines, whose Z the doc marks unused ("(X,Y,-)"). + ctrl_pts = np.zeros((knot_count, n_lines, 3), dtype=np.float64) + ctrl_pts[:, :, :2] = raw_cp.reshape(knot_count, n_lines, 2) + + # Optional line selection for ParametricLineFromRepresentationLatticeArray: it picks the + # subset of the supporting representation's lines that this grid uses, so it re-defines + # the line numbering that `line_indices` below is expressed in. col_indices: Optional[np.ndarray] = getattr(pla, "_column_indices", None) if col_indices is not None: ctrl_pts = ctrl_pts[:, col_indices, :] + kinds = kinds[col_indices] + n_lines = ctrl_pts.shape[1] + + # --- 3. Map each node column to its parametric line --- + if line_indices is not None: + eff_lines = np.asarray(line_indices, dtype=np.int64).flatten() + if len(eff_lines) != n_columns: + logger.warning( + f"line_indices length {len(eff_lines)} ≠ node column count {n_columns}; " + "falling back to the identity mapping." + ) + eff_lines = np.arange(n_columns, dtype=np.int64) + else: + eff_lines = np.arange(n_columns, dtype=np.int64) + + if n_columns != n_lines and line_indices is None: + logger.warning( + f"ParametricLineArray holds {n_lines} lines for {n_columns} node columns and no " + "column→line mapping was supplied; the extra columns cannot be evaluated." + ) + out_of_range = (eff_lines < 0) | (eff_lines >= n_lines) + if out_of_range.any(): + logger.warning( + f"{int(out_of_range.sum())} node column(s) reference a parametric line outside " + f"[0, {n_lines}); those nodes are returned as NaN." + ) + eff_lines = np.where(out_of_range, 0, eff_lines) - # --- 2. Read control_point_parameters (may be None for all-vertical) --- + # --- 4. Read control_point_parameters (may be None for all-vertical) --- cpp_obj = getattr(pla, "control_point_parameters", None) ctrl_params: Optional[np.ndarray] = None if cpp_obj is not None: @@ -846,40 +1050,26 @@ def evaluate_parametric_line_array( if not isinstance(raw_cpp, np.ndarray): raw_cpp = np.array(raw_cpp, dtype=np.float64) raw_cpp = raw_cpp.astype(np.float64).flatten() - # Layout: (K * P,) ordered knot-major → reshape to (K, P). - if raw_cpp.size == knot_count * n_pillars: - ctrl_params = raw_cpp.reshape(knot_count, n_pillars) + # Layout: (K * #Lines,) ordered knot-major → reshape to (K, #Lines). + n_lines_raw = n_lines if col_indices is None else int(np.max(col_indices)) + 1 + if raw_cpp.size == knot_count * n_lines_raw: + ctrl_params = raw_cpp.reshape(knot_count, n_lines_raw) elif raw_cpp.size == knot_count: - # Same parameters for all pillars (broadcast). - ctrl_params = np.tile(raw_cpp[:, np.newaxis], (1, n_pillars)) + # Same parameters for all lines (broadcast). + ctrl_params = np.tile(raw_cpp[:, np.newaxis], (1, n_lines_raw)) else: - logging.warning( + logger.warning( f"control_point_parameters size {raw_cpp.size} does not match " - f"knot_count={knot_count} × n_pillars={n_pillars}. " + f"knot_count={knot_count} × line count={n_lines_raw}. " "Attempting best-effort reshape." ) - ctrl_params = raw_cpp[: knot_count * n_pillars].reshape(knot_count, n_pillars) + padded = np.full(knot_count * n_lines_raw, np.nan, dtype=np.float64) + padded[: min(raw_cpp.size, padded.size)] = raw_cpp[: padded.size] + ctrl_params = padded.reshape(knot_count, n_lines_raw) if col_indices is not None: ctrl_params = ctrl_params[:, col_indices] - # --- 3. Read line_kind_indices --- - lki_obj = getattr(pla, "line_kind_indices", None) - if lki_obj is None: - raise ValueError("ParametricLineArray.line_kind_indices is required but absent.") - raw_lki = read_array(energyml_array=lki_obj, root_obj=root_obj, workspace=workspace) - if not isinstance(raw_lki, np.ndarray): - raw_lki = np.array(raw_lki, dtype=np.int32) - kinds: np.ndarray = raw_lki.astype(np.int32).flatten() - if col_indices is not None: - kinds = kinds[col_indices] - if len(kinds) != n_pillars: - logging.warning( - f"line_kind_indices length {len(kinds)} ≠ n_pillars {n_pillars}. " - "Broadcasting first kind value to all pillars." - ) - kinds = np.full(n_pillars, kinds[0] if len(kinds) > 0 else _PARAMETRIC_KIND_LINEAR, dtype=np.int32) - - # --- 4. Read tangent_vectors (optional, only for kinds 3 and 5) --- + # --- 5. Read tangent_vectors (optional, only for kinds 3 and 5) --- tv_obj = getattr(pla, "tangent_vectors", None) tangent_vecs: Optional[np.ndarray] = None unique_kinds = np.unique(kinds) @@ -888,33 +1078,25 @@ def evaluate_parametric_line_array( raw_tv = read_array(energyml_array=tv_obj, root_obj=root_obj, workspace=workspace) if not isinstance(raw_tv, np.ndarray): raw_tv = np.array(raw_tv, dtype=np.float64) - tangent_vecs = raw_tv.astype(np.float64).reshape(knot_count, n_pillars, 3) + n_lines_raw = n_lines if col_indices is None else int(np.max(col_indices)) + 1 + tangent_vecs = raw_tv.astype(np.float64).reshape(knot_count, n_lines_raw, 3) if col_indices is not None: tangent_vecs = tangent_vecs[:, col_indices, :] - # --- 5. Evaluate each pillar --- - result = np.empty((nkl, n_pillars, 3), dtype=np.float64) + # --- 6. Evaluate each node column on its parametric line --- + result = np.empty((nkl, n_columns, 3), dtype=np.float64) - for p_idx in range(n_pillars): - kind = int(kinds[p_idx]) - q_p = query_parameters[:, p_idx] # (NKL,) P-values for this pillar - cp_p = ctrl_pts[:, p_idx, :] # (K, d) - - # ctrl_params_p: (K,) — derived from global or pillar-specific params. - # For kind=0, ctrl_params is None (vertical) and we pass None. - if ctrl_params is not None: - cpp_p = ctrl_params[:, p_idx] # (K,) - else: - cpp_p = None - - tv_p = tangent_vecs[:, p_idx, :] if tangent_vecs is not None else None # (K, 3) or None - - result[:, p_idx, :] = _evaluate_one_pillar( - kind=kind, - ctrl_params=cpp_p, - ctrl_pts=cp_p, - tangents=tv_p, - query_params=q_p, + for c_idx in range(n_columns): + if out_of_range[c_idx]: + result[:, c_idx, :] = np.nan + continue + line = int(eff_lines[c_idx]) + result[:, c_idx, :] = _evaluate_one_pillar( + kind=int(kinds[line]), + ctrl_params=ctrl_params[:, line] if ctrl_params is not None else None, + ctrl_pts=ctrl_pts[:, line, :], + tangents=tangent_vecs[:, line, :] if tangent_vecs is not None else None, + query_params=query_parameters[:, c_idx], ) return result @@ -1092,7 +1274,7 @@ def read_parametric_geometry( workspace=workspace, ) except Exception as e: - logging.debug(f"No tangent vectors found for {geometry}, fallback to linear interpolation: {e}") + logger.debug(f"No tangent vectors found for {geometry}, fallback to linear interpolation: {e}") if traj_tangents is not None: if not isinstance(traj_tangents, np.ndarray): @@ -1106,7 +1288,7 @@ def read_parametric_geometry( or len(traj_points) != knot_count or (traj_tangents is not None and len(traj_tangents) != knot_count) ): - logging.warning( + logger.warning( f"Mismatch between knot_count ({knot_count}) and actual control points count (mds: {len(traj_mds)}, points: {len(traj_points)}, tangents: {len(traj_tangents) if traj_tangents is not None else 'N/A'})" ) @@ -1242,7 +1424,7 @@ def read_external_array( workspace=workspace, ) except ObjectNotFoundNotError as e: - logging.debug(f"CRS not found for {get_obj_title(root_obj)}: {e}") + logger.debug(f"CRS not found for {get_obj_title(root_obj)}: {e}") # Search for ExternalDataArrayPart type objects (RESQML v2.2) external_parts = search_attribute_matching_type( @@ -1255,7 +1437,7 @@ def read_external_array( for ext_part in external_parts: start_indices, counts, external_uri = _extract_external_data_array_part_params(ext_part) pief_list = get_path_in_external_with_path(obj=ext_part) - # logging.debug(f"Pief : {pief_list}") + # logger.debug(f"Pief : {pief_list}") for pief_path_in_obj, pief in pief_list: arr = workspace.read_array( proxy=crs or root_obj, @@ -1266,7 +1448,7 @@ def read_external_array( ) if arr is not None: array = arr if array is None else np.concatenate((array, arr)) - # logging.debug(f"\t ExternalDataArrayPart read successfully. arr : {arr} : array : {array}") + # logger.debug(f"\t ExternalDataArrayPart read successfully. arr : {arr} : array : {array}") else: # RESQML v2.0.1: Extract count from parent object, no StartIndex or URI counts = None @@ -1281,7 +1463,7 @@ def read_external_array( # Extract count from parent using simplified function _, counts, _ = _extract_external_data_array_part_params(parent_obj) except Exception as e: - logging.debug(f"Failed to extract count from parent: {e}") + logger.debug(f"Failed to extract count from parent: {e}") # Read array using path_in_external from the array object itself pief_list = get_path_in_external_with_path(obj=energyml_array) @@ -1310,7 +1492,7 @@ def read_external_array( # Fallback for non-numpy arrays array = [array[idx] for idx in sub_indices] - # logging.debug(f"External array read successfully. => {array}") + # logger.debug(f"External array read successfully. => {array}") return array @@ -1346,8 +1528,8 @@ def read_array( # if isinstance(energyml_array, list): return energyml_array elif isinstance(energyml_array, list): - # logging.debug("Warning: the array is a list, not a numpy array, be careful with the performance !") - # logging.debug(energyml_array) + # logger.debug("Warning: the array is a list, not a numpy array, be careful with the performance !") + # logger.debug(energyml_array) if len(energyml_array) > 0 and is_primitive(energyml_array[0]): return energyml_array else: @@ -1374,7 +1556,7 @@ def read_array( sub_indices=sub_indices, ) else: - logging.error(f"Type {array_type_name} is not supported: function read_{snake_case(array_type_name)} not found") + logger.error(f"Type {array_type_name} is not supported: function read_{snake_case(array_type_name)} not found") raise Exception( f"Type {array_type_name} is not supported\n\t{energyml_array}: \n\tfunction read_{snake_case(array_type_name)} not found" ) @@ -1434,7 +1616,7 @@ def read_xml_array( values = get_object_attribute_no_verif(energyml_array, "values") # count = get_object_attribute_no_verif(energyml_array, "count_per_value") - # logging.debug("values: ", values) + # logger.debug("values: ", values) if sub_indices is not None and len(sub_indices) > 0: if isinstance(values, np.ndarray): @@ -1590,7 +1772,7 @@ def read_point3d_zvalue_array( # Vectorized assignment for NumPy arrays min_len = min(len(sup_geom_array), len(zvalues_array)) if min_len < len(sup_geom_array): - logging.warning( + logger.warning( f"Z-values array ({len(zvalues_array)}) is shorter than geometry array ({len(sup_geom_array)}), only updating first {min_len} values" ) sup_geom_array[:min_len, 2] = zvalues_array[:min_len] @@ -1601,7 +1783,7 @@ def read_point3d_zvalue_array( sup_geom_array[i][2] = zvalues_array[i] except (IndexError, TypeError) as e: if not error_logged: - logging.error(f"{type(e).__name__}: index {i} is out of bound of {len(zvalues_array)}") + logger.error(f"{type(e).__name__}: index {i} is out of bound of {len(zvalues_array)}") error_logged = True return sup_geom_array @@ -1704,7 +1886,7 @@ def read_point3d_from_representation_lattice_array( result = all_sup_points[node_indices] else: # No index array: use all points in order (identity mapping) - logging.debug( + logger.debug( "Point3DFromRepresentationLatticeArray: no NodeIndices found, " "using all supporting rep points in order" ) result = all_sup_points @@ -1788,15 +1970,15 @@ def read_point3d_lattice_array( slowest_size = len(slowest_table) fastest_size = len(fastest_table) - # logging.debug(f"slowest vector: {slowest_vec}, spacing: {slowest_spacing}, size: {slowest_size}") - # logging.debug(f"fastest vector: {fastest_vec}, spacing: {fastest_spacing}, size: {fastest_size}") - # logging.debug(f"origin: {origin}") + # logger.debug(f"slowest vector: {slowest_vec}, spacing: {slowest_spacing}, size: {slowest_size}") + # logger.debug(f"fastest vector: {fastest_vec}, spacing: {fastest_spacing}, size: {fastest_size}") + # logger.debug(f"origin: {origin}") if crs_sa_count is not None and len(crs_sa_count) > 0 and crs_fa_count is not None and len(crs_fa_count) > 0: if (crs_sa_count[0] == fastest_size and crs_fa_count[0] == slowest_size) or ( crs_sa_count[0] == fastest_size - 1 and crs_fa_count[0] == slowest_size - 1 ): - logging.debug("reversing order") + logger.debug("reversing order") # if offset were given in the wrong order tmp_table = slowest_table slowest_table = fastest_table @@ -1845,7 +2027,7 @@ def read_point3d_lattice_array( except (ValueError, TypeError) as e: # Fallback to original implementation if NumPy conversion fails. - logging.warning(f"NumPy vectorization failed ({e}), falling back to iterative approach") + logger.warning(f"NumPy vectorization failed ({e}), falling back to iterative approach") fallback: List = [] for i in range(slowest_size): for j in range(fastest_size): @@ -1883,7 +2065,7 @@ def read_point3d_lattice_array( # path_in_root: Optional[str] = None, # workspace: Optional[EnergymlStorageInterface] = None # ): -# logging.debug(energyml_array) +# logger.debug(energyml_array) # ______ __ _ __ __ @@ -1929,6 +2111,8 @@ def read_point3d_lattice_array( import colorsys from dataclasses import dataclass, field as dc_field +logger = logging.getLogger(__name__) + # ───────────────────────────────────────────────────────────────────────────── # Unified output data structures @@ -2256,7 +2440,7 @@ def read_color_map(color_map_obj: Any) -> Optional[ColorMapInfo]: return read_continuous_color_map(color_map_obj) if "discrete" in type_name: return read_discrete_color_map(color_map_obj) - logging.warning(f"read_color_map: unsupported color-map type '{type(color_map_obj).__name__}'") + logger.warning(f"read_color_map: unsupported color-map type '{type(color_map_obj).__name__}'") return None @@ -2403,7 +2587,7 @@ def read_graphical_rendering_info( try: pts.append((float(idx), float(a))) except (TypeError, ValueError) as exc: - logging.warning( + logger.warning( f"read_graphical_rendering_info: skipping invalid AlphaInformation" f" control point ({idx!r}, {a!r}): {exc}" ) @@ -2481,4 +2665,53 @@ def read_graphical_rendering_info( # :return: A RESQML array object containing the data from the NumPy array. # """ # dtype = np_array.dtype - + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "CrsInfo", + "extract_crs_info", + "is_z_reversed", + "get_vertical_epsg_code", + "get_projected_epsg_code", + "get_projected_uom", + "get_crs_offsets_and_angle", + "apply_crs_transform", + "get_crs_origin_offset", + "get_datum_information", + "prod_n_tab", + "sum_lists", + "get_crs_obj", + "get_package_default_crs", + "linear_interpolation", + "hermite_interpolation", + "resolve_parametric_line_array", + "evaluate_parametric_line_array", + "get_wellbore_points", + "generate_smooth_trajectory", + "generate_vertical_well_points", + "read_parametric_geometry", + "get_supported_array", + "get_not_supported_array", + "read_external_array", + "get_array_reader_function", + "read_array", + "read_constant_array", + "read_xml_array", + "read_jagged_array", + "read_int_double_lattice_array", + "read_point3d_zvalue_array", + "read_point3d_from_representation_lattice_array", + "read_grid2d_patch", + "read_point3d_lattice_array", + "RgbaColor", + "ColorMapEntry", + "ColorMapInfo", + "IndexableElementRenderingInfo", + "ScalarRenderingInfo", + "read_continuous_color_map", + "read_discrete_color_map", + "read_color_map", + "read_graphical_rendering_info", +] diff --git a/energyml-utils/src/energyml/utils/data/mesh.py b/energyml-utils/src/energyml/utils/data/mesh.py index 6e3ebc7..0f4269b 100644 --- a/energyml-utils/src/energyml/utils/data/mesh.py +++ b/energyml-utils/src/energyml/utils/data/mesh.py @@ -1,48 +1,76 @@ # Copyright (c) 2023-2024 Geosiris. # SPDX-License-Identifier: Apache-2.0 -import inspect -import json import logging import os import re import sys -import traceback +from energyml.utils.epc_file import EpcAccessMode, EpcFile import numpy as np from dataclasses import dataclass, field from enum import Enum +from functools import lru_cache from io import BytesIO -from typing import List, Optional, Any, Callable, Dict, Union, Tuple - - -from energyml.utils.data.helper import ( - apply_crs_transform, - generate_vertical_well_points, - get_crs_offsets_and_angle, - get_datum_information, - get_wellbore_points, - hermite_interpolation, - read_array, - read_grid2d_patch, - get_crs_obj, - read_parametric_geometry, +from typing import List, Optional, Any, Callable, Union + + +from energyml.utils.data.helper import read_grid2d_patch +from energyml.utils.data.crs import PointFrame, extract_crs_info, to_frame +from energyml.utils.data.mesh_numpy import ( + _fit_grid_dimensions, + read_numpy_grid2d_representation, + read_numpy_point_representation, + read_numpy_polyline_representation, + read_numpy_representation_set_representation, + read_numpy_sub_representation, + read_numpy_triangulated_set_representation, + read_numpy_wellbore_frame_representation, + read_numpy_wellbore_trajectory_representation, ) -from energyml.utils.data.crs import extract_crs_info, apply_from_crs_info +from energyml.utils.constants import sanitize_file_name from energyml.utils.epc_utils import gen_energyml_object_path -from energyml.utils.epc_stream import EpcStreamReader -from energyml.utils.exception import NotSupportedError, ObjectNotFoundNotError +from energyml.utils.exception import NotSupportedError from energyml.utils.introspection import ( get_obj_uri, search_attribute_matching_name, - search_attribute_matching_name_with_path, snake_case, get_object_attribute, - get_object_attribute_rgx, ) from energyml.utils.storage_interface import EnergymlStorageInterface # Import export functions from new export module for backward compatibility +# The writers now live in the export package; re-exported here so existing imports +# (`from energyml.utils.data.mesh import export_off`) keep working. +# Property / table / time-series readers moved to properties.py; re-exported so that +# `from energyml.utils.data.mesh import read_property` keeps working. +from energyml.utils.data.properties import ( # noqa: F401 + get_property_reader_function, + read_abstract_values_property, + read_categorical_property, + read_column_based_table, + read_comment_property, + read_continuous_property, + read_discrete_property, + read_property, + read_property_interpreted_with_cbt, + read_time_series, +) from energyml.utils.data.export import export_obj as _export_obj_new +from energyml.utils.data.export import export_off, export_off_part # noqa: F401 +from energyml.utils.data.export.geojson import ( # noqa: F401 + GeoJsonGeometryType, + energyml_type_to_geojson_type, + export_geojson_dict, + export_geojson_io, + mesh_to_geojson_type, + to_geojson_feature, + write_geojson_feature, +) + +logger = logging.getLogger(__name__) +#: Alias of the module logger, for the functions that take a caller-supplied +#: ``logger`` parameter — the parameter shadows the module-level name in their body. +_MODULE_LOGGER = logger _FILE_HEADER: bytes = b"# file exported by energyml-utils python module (Geosiris)\n" @@ -71,35 +99,6 @@ class MeshFileFormat(Enum): GEOJSON = "geojson" -class GeoJsonGeometryType(Enum): - """GeoJson type enum""" - - Point = "Point" - MultiPoint = "MultiPoint" - LineString = "LineString" - MultiLineString = "MultiLineString" - Polygon = "Polygon" - MultiPolygon = "MultiPolygon" - - -def energyml_type_to_geojson_type(energyml_type: str): - if "PolylineSet" in energyml_type: - return GeoJsonGeometryType.MultiLineString - elif "Polyline" in energyml_type: - return GeoJsonGeometryType.LineString - elif "PointSet" in energyml_type: - return GeoJsonGeometryType.MultiPoint - elif "Point" in energyml_type: - return GeoJsonGeometryType.Point - elif "TriangulatedSet" in energyml_type: - return GeoJsonGeometryType.MultiPolygon - elif "Triangulated" in energyml_type: - return GeoJsonGeometryType.Polygon - elif "Grid2" in energyml_type: - return GeoJsonGeometryType.MultiPolygon - return GeoJsonGeometryType.Point - - @dataclass class AbstractMesh: energyml_object: Any = field(default=None) @@ -114,6 +113,14 @@ class AbstractMesh: default="", ) + frame: PointFrame = field(default=PointFrame.LOCAL) + """ + Coordinate frame :attr:`point_list` is expressed in. + + Readers set it to what they produced; :func:`read_mesh_object` then applies only the missing + pipeline stages, so a CRS transform cannot be applied twice. + """ + def get_nb_edges(self) -> int: return 0 @@ -161,16 +168,27 @@ def get_indices(self) -> Union[List[List[int]], np.ndarray]: return self.faces_indices +@lru_cache(maxsize=None) def get_object_reader_function(mesh_type_name: str) -> Optional[Callable]: """ - Returns the name of the potential appropriate function to read an object with type is named mesh_type_name + Returns the potential appropriate function to read an object whose type is named mesh_type_name. + + The lookup is a cached ``getattr`` on this module rather than a scan of every module member: + the dispatcher is called once per object, and :func:`inspect.getmembers` sorts and reads + *all* the attributes of the module on each call. + + Only functions **defined in this module** are eligible. The ``read_`` prefix is otherwise + shared with helpers imported here (``read_array``, ``read_grid2d_patch``, + ``read_parametric_geometry``), so a type named ``Array`` or ``Grid2dPatch`` used to resolve + to one of them and then fail on its signature. + :param mesh_type_name: the initial type name - :return: + :return: the reader function, or None when no ``read_`` function exists """ - for name, obj in inspect.getmembers(sys.modules[__name__]): - if name == f"read_{snake_case(mesh_type_name)}": - return obj - return None + reader = getattr(sys.modules[__name__], f"read_{snake_case(mesh_type_name)}", None) + if not callable(reader) or getattr(reader, "__module__", None) != __name__: + return None + return reader def get_mesh_reader_function(mesh_type_name: str) -> Optional[Callable]: @@ -180,12 +198,19 @@ def get_mesh_reader_function(mesh_type_name: str) -> Optional[Callable]: def _mesh_name_mapping(array_type_name: str) -> str: """ - Transform the type name to match existing reader function + Transform the type name to match existing reader function. + + Accepts the three spellings the same type takes across the code base: the python class name + (``ObjTriangulatedSetRepresentation``), the schema type carried by a content type or a + :class:`ResourceMetadata` (``obj_TriangulatedSetRepresentation``, RESQML 2.0.1 keeps the + ``obj_`` prefix), and a qualified type (``resqml20.obj_TriangulatedSetRepresentation``). + :param array_type_name: :return: """ + array_type_name = array_type_name.rsplit(".", 1)[-1] array_type_name = array_type_name.replace("3D", "3d").replace("2D", "2d") - array_type_name = re.sub(r"^[Oo]bj([A-Z])", r"\1", array_type_name) + array_type_name = re.sub(r"^[Oo]bj_?([A-Z])", r"\1", array_type_name) array_type_name = re.sub(r"(Polyline|Point)Set", r"\1", array_type_name) return array_type_name @@ -195,62 +220,164 @@ def read_mesh_object( workspace: Optional[EnergymlStorageInterface] = None, use_crs_displacement: bool = True, sub_indices: Optional[Union[List[int], np.ndarray]] = None, + frame: Optional[PointFrame] = None, + use_network: bool = False, ) -> List[AbstractMesh]: """ Read and "meshable" object. If :param:`energyml_object` is not supported, an exception will be raised. - :param energyml_object: + :param energyml_object: a single energyml object, or a list of them (each one is read in turn + and the resulting meshes are concatenated) :param workspace: - :param use_crs_displacement: If true :func:`apply_from_crs_info` is used to apply the full CRS - transform (rotation, offsets, Z-flip, axis-order swap) to the mesh points + :param use_crs_displacement: legacy switch, kept for compatibility. It selects the default + target frame: :attr:`PointFrame.PROJECTED` when True (rotation, offsets, Z-flip, axis-order + swap), :attr:`PointFrame.LOCAL` when False. Ignored when :param:`frame` is given. + :param frame: explicit target frame, e.g. :attr:`PointFrame.WGS84`. + :param use_network: allow PROJ to download the geoid grids used by the vertical datum + transformation. Only relevant for :attr:`PointFrame.WGS84`. :return: """ if isinstance(energyml_object, list): - return energyml_object + # a list of objects is read recursively: returning it as-is would hand back the energyml + # objects themselves instead of the List[AbstractMesh] this function is declared to return. + return [ + mesh + for obj in energyml_object + for mesh in read_mesh_object( + energyml_object=obj, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + frame=frame, + use_network=use_network, + ) + ] array_type_name = _mesh_name_mapping(type(energyml_object).__name__) reader_func = get_object_reader_function(array_type_name) if reader_func is not None: - # logging.info(f"using function {reader_func} to read type {array_type_name}") surfaces: List[AbstractMesh] = reader_func( energyml_object=energyml_object, workspace=workspace, sub_indices=sub_indices, use_crs_displacement=use_crs_displacement, ) - _tn = array_type_name.lower() - if ( - use_crs_displacement - and "wellbore" not in _tn - and "triangulated" not in _tn # per-patch CRS applied inside reader - and "point" not in _tn # per-patch CRS applied inside reader - and "polyline" not in _tn # per-patch CRS applied inside reader - and "representationset" not in _tn # each sub-mesh already had CRS applied by its own reader - and "subrepresentation" not in _tn # delegates entirely to inner read_mesh_object call - ): - for s in surfaces: - crs = s.crs_object[0] if isinstance(s.crs_object, list) and s.crs_object else s.crs_object - if crs is None: - continue - logging.debug(f"Applying CRS transform to surface {s.identifier}") - pts_arr = np.asarray(s.point_list, dtype=np.float64).reshape(-1, 3) - apply_from_crs_info(pts_arr, extract_crs_info(crs, workspace), inplace=True) - s.point_list = pts_arr.tolist() + # Each mesh reports the frame its reader produced, so only the missing stages are applied. + # This replaces the previous list of type-name substrings, where a missing entry silently + # transformed the points twice and an extra one left them untransformed. + target = frame if frame is not None else (PointFrame.PROJECTED if use_crs_displacement else PointFrame.LOCAL) + for s in surfaces: + if s.frame is target or s.point_list is None or len(s.point_list) == 0: + continue + crs = s.crs_object[0] if isinstance(s.crs_object, list) and s.crs_object else s.crs_object + logger.debug(f"Bringing surface {s.identifier} from {s.frame.value} to {target.value}") + pts_arr = np.asarray(s.point_list, dtype=np.float64).reshape(-1, 3) + framed = to_frame( + pts_arr, + extract_crs_info(crs, workspace) if crs is not None else None, + target, + s.frame, + use_network=use_network, + inplace=True, + ) + s.point_list = framed.points.tolist() + s.frame = framed.frame return surfaces else: - # logging.error(f"Type {array_type_name} is not supported: function read_{snake_case(array_type_name)} not found") + # logger.error(f"Type {array_type_name} is not supported: function read_{snake_case(array_type_name)} not found") raise NotSupportedError( f"Type {array_type_name} is not supported\n\tfunction read_{snake_case(array_type_name)} not found" ) +def _legacy_identifier(patch: Any, patch_index: int) -> str: + """Rebuild the identifier the legacy readers used to produce. + + The numpy readers label their patches ``"{TypeName}_patch_{n}"``; the legacy ones used the + object URI (and a different wording for point sets). The strings are regenerated from the + patch metadata rather than translated, so they stay exactly what they were. + """ + from energyml.utils.data.mesh_numpy import NumpyPointSetMesh + + if isinstance(patch, NumpyPointSetMesh): + return f"Patch num {patch_index}" + # Wellbore representations were named after the object alone, with no patch suffix — and that + # must hold whichever entry point was used, including when they are reached through a + # RepresentationSetRepresentation. + if "wellbore" in (patch.source_type or "").lower(): + return f"{get_obj_uri(patch.energyml_object)}" + return f"{get_obj_uri(patch.energyml_object)}_patch{patch_index}" + + +def _to_legacy_meshes( + multi: Any, + identifier: Optional[Callable[[Any, int], str]] = None, +) -> List[AbstractMesh]: + """Convert a :class:`~energyml.utils.data.mesh_numpy.NumpyMultiMesh` to the legacy containers. + + ``point_list`` keeps the ``(N, 3)`` float64 array produced by the numpy reader — the field is + annotated ``Union[List[Point], np.ndarray]`` and every writer in this package already handles + both — and the VTK flat connectivity is expanded back into the lists of indices the legacy + containers expose through ``get_indices()``. + + :param identifier: overrides the identifier rule; called as ``identifier(patch, index)``. + :raises NotSupportedError: for volumetric patches, which have no legacy container. + """ + from energyml.utils.data.export._base import _parse_vtk_flat_faces, _parse_vtk_flat_lines + from energyml.utils.data.mesh_numpy import ( + NumpyPointSetMesh, + NumpyPolylineMesh, + NumpySurfaceMesh, + NumpyVolumeMesh, + ) + + meshes: List[AbstractMesh] = [] + for position, patch in enumerate(multi.flat_patches()): + patch_index = patch.patch_index if patch.patch_index is not None else position + name = identifier(patch, patch_index) if identifier is not None else _legacy_identifier(patch, patch_index) + common = dict( + identifier=name, + energyml_object=patch.energyml_object, + crs_object=patch.crs_object, + point_list=patch.points, + frame=patch.frame, + ) + + if isinstance(patch, NumpyVolumeMesh): + raise NotSupportedError( + f"{patch.source_type} produces a volumetric mesh, which AbstractMesh cannot hold. " + "Use energyml.utils.data.mesh_numpy.read_numpy_mesh_object instead." + ) + if isinstance(patch, NumpySurfaceMesh): + meshes.append(SurfaceMesh(faces_indices=[f.tolist() for f in _parse_vtk_flat_faces(patch.faces)], **common)) + elif isinstance(patch, NumpyPolylineMesh): + meshes.append( + PolylineSetMesh(line_indices=[line.tolist() for line in _parse_vtk_flat_lines(patch.lines)], **common) + ) + elif isinstance(patch, NumpyPointSetMesh): + meshes.append(PointSetMesh(**common)) + else: + meshes.append(AbstractMesh(**common)) + + return meshes + + def read_ijk_grid_representation( energyml_object: Any, workspace: EnergymlStorageInterface, use_crs_displacement: bool = True, sub_indices: Optional[Union[List[int], np.ndarray]] = None, -) -> List[Any]: - raise NotSupportedError("IJKGrid representation reading is not supported yet.") +) -> List[AbstractMesh]: + """ + Not available through the legacy containers: an IJK grid is volumetric and :class:`AbstractMesh` + only models points, polylines and surfaces. + + :func:`energyml.utils.data.mesh_numpy.read_numpy_ijk_grid_representation` does support it. + """ + raise NotSupportedError( + "IjkGridRepresentation is volumetric and has no legacy AbstractMesh container. " + "Use energyml.utils.data.mesh_numpy.read_numpy_mesh_object instead." + ) def read_point_representation( @@ -259,72 +386,15 @@ def read_point_representation( use_crs_displacement: bool = True, sub_indices: Optional[Union[List[int], np.ndarray]] = None, ) -> List[PointSetMesh]: - # pt_geoms = search_attribute_matching_type(point_set, "AbstractGeometry") - - meshes = [] - - patch_idx = 0 - total_size = 0 - - patches_geom = search_attribute_matching_name_with_path( - energyml_object, r"NodePatch.[\d]+.Geometry.Points" - ) + search_attribute_matching_name_with_path( # resqml 2.0.1 - energyml_object, r"NodePatchGeometry.[\d]+.Points" - ) - # logging.debug(f"Found {len(patches_geom)} patches for point representation") - # logging.debug(f"\t=> {patches_geom}") - - for points_path_in_obj, points_obj in patches_geom: - points = read_array( - energyml_array=points_obj, - root_obj=energyml_object, - path_in_root=points_path_in_obj, + """Read a ``PointRepresentation`` / ``PointSetRepresentation`` into legacy containers.""" + return _to_legacy_meshes( + read_numpy_point_representation( + energyml_object=energyml_object, workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, ) - - crs = None - try: - crs = get_crs_obj( - context_obj=points_obj, - path_in_root=points_path_in_obj, - root_obj=energyml_object, - workspace=workspace, - ) - except ObjectNotFoundNotError as e: - logging.error(e) - pass - - if sub_indices is not None and len(sub_indices) > 0: - new_points = [] - for idx in sub_indices: - t_idx = idx - total_size - if 0 <= t_idx < len(points): - new_points.append(points[t_idx]) - total_size = total_size + len(points) - points = new_points - # else: - # total_size = total_size + len(points) - - # Apply full CRS transform per patch; crs_object kept on mesh for reference - # but the outer dispatcher is guarded to skip crs_displacement for this type. - if use_crs_displacement and crs is not None and points is not None and len(points) > 0: - pts_arr = np.asarray(points, dtype=np.float64).reshape(-1, 3) - apply_from_crs_info(pts_arr, extract_crs_info(crs, workspace), inplace=True) - points = pts_arr.tolist() - - if points is not None: - meshes.append( - PointSetMesh( - identifier=f"Patch num {patch_idx}", - energyml_object=energyml_object, - crs_object=crs, - point_list=points, - ) - ) - - patch_idx = patch_idx + 1 - - return meshes + ) def read_polyline_representation( @@ -333,120 +403,152 @@ def read_polyline_representation( use_crs_displacement: bool = True, sub_indices: Optional[Union[List[int], np.ndarray]] = None, ) -> List[PolylineSetMesh]: - # pt_geoms = search_attribute_matching_type(point_set, "AbstractGeometry") + """Read a ``PolylineRepresentation`` / ``PolylineSetRepresentation`` into legacy containers.""" + return _to_legacy_meshes( + read_numpy_polyline_representation( + energyml_object=energyml_object, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + ) + ) - meshes = [] - patch_idx = 0 - total_size = 0 - for patch_path_in_obj, patch in search_attribute_matching_name_with_path( - energyml_object, "NodePatch" - ) + search_attribute_matching_name_with_path(energyml_object, r"LinePatch.[\d]+"): +def read_grid2d_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + keep_holes: bool = False, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> List[SurfaceMesh]: + """Read a ``Grid2dRepresentation`` into legacy containers.""" + return _to_legacy_meshes( + read_numpy_grid2d_representation( + energyml_object=energyml_object, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + keep_holes=keep_holes, + sub_indices=sub_indices, + ) + ) - pts = search_attribute_matching_name_with_path(patch, "Geometry.Points") - if pts is None or len(pts) == 0: - pts = search_attribute_matching_name_with_path(patch, "Points") - try: - points_path, points_obj = pts[0] - except Exception as e: - logging.error(f"Cannot find points for patch {patch_path_in_obj} : {e}") - logging.error(patch) - raise e - - points = read_array( - energyml_array=points_obj, - root_obj=energyml_object, - path_in_root=patch_path_in_obj + "." + points_path, +def read_triangulated_set_representation( + energyml_object: Any, + workspace: EnergymlStorageInterface, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> List[SurfaceMesh]: + """Read a ``TriangulatedSetRepresentation`` into legacy containers.""" + return _to_legacy_meshes( + read_numpy_triangulated_set_representation( + energyml_object=energyml_object, workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, ) + ) - crs = None - try: - crs = get_crs_obj( - context_obj=points_obj, - path_in_root=patch_path_in_obj + "." + points_path, - root_obj=energyml_object, - workspace=workspace, - ) - except ObjectNotFoundNotError as e: - logging.error(e) - close_poly = None - try: - (close_poly_path, close_poly_obj,) = search_attribute_matching_name_with_path( - patch, "ClosedPolylines" - )[0] - close_poly = read_array( - energyml_array=close_poly_obj, - root_obj=energyml_object, - path_in_root=patch_path_in_obj + "." + close_poly_path, - workspace=workspace, - ) - except IndexError: - pass +def read_wellbore_frame_representation( + energyml_object: Any, + workspace: EnergymlStorageInterface, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> List[PolylineSetMesh]: + """ + Read a WellboreFrameRepresentation and construct a polyline mesh from the trajectory. - point_indices = [] - try: - (node_count_per_poly_path_in_obj, node_count_per_poly,) = search_attribute_matching_name_with_path( - patch, "NodeCountPerPolyline" - )[0] - node_counts_list = read_array( - energyml_array=node_count_per_poly, - root_obj=energyml_object, - path_in_root=patch_path_in_obj + node_count_per_poly_path_in_obj, - workspace=workspace, - ) - idx = 0 - poly_idx = 0 - for nb_node in node_counts_list: - point_indices.append([x for x in range(idx, idx + nb_node)]) - if close_poly is not None and len(close_poly) > poly_idx and close_poly[poly_idx]: - point_indices[len(point_indices) - 1].append(idx) - idx = idx + nb_node - poly_idx = poly_idx + 1 - except IndexError: - # No NodeCountPerPolyline for Polyline but only in PolylineSet - pass - - if point_indices is None or len(point_indices) == 0: - # No indices ==> all point in the polyline - point_indices = [list(range(len(points)))] - - if sub_indices is not None and len(sub_indices) > 0: - new_indices = [] - for idx in sub_indices: - t_idx = idx - total_size - if 0 <= t_idx < len(point_indices): - new_indices.append(point_indices[t_idx]) - total_size = total_size + len(point_indices) - point_indices = new_indices - else: - total_size = total_size + len(point_indices) + :param energyml_object: The WellboreFrameRepresentation object + :param workspace: The EnergymlStorageInterface to access related objects + :param sub_indices: Optional list of indices to filter specific nodes + :return: List containing a single PolylineSetMesh representing the wellbore + """ + frame_uri = f"{get_obj_uri(energyml_object)}" + return _to_legacy_meshes( + read_numpy_wellbore_frame_representation( + energyml_object=energyml_object, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + ), + identifier=lambda _patch, _index: frame_uri, + ) - # Apply full CRS transform per patch; crs_object kept on mesh for reference - # but the outer dispatcher is guarded to skip crs_displacement for this type. - if use_crs_displacement and crs is not None and len(points) > 0: - pts_arr = np.asarray(points, dtype=np.float64).reshape(-1, 3) - apply_from_crs_info(pts_arr, extract_crs_info(crs, workspace), inplace=True) - points = pts_arr.tolist() - if len(points) > 0: - meshes.append( - PolylineSetMesh( - identifier=f"{get_obj_uri(energyml_object)}_patch{patch_idx}", - energyml_object=energyml_object, - crs_object=crs, - point_list=points, - line_indices=point_indices, - ) +def read_wellbore_trajectory_representation( + energyml_object: Any, + workspace: EnergymlStorageInterface, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, + wellbore_frame_mds: Optional[Union[List[float], np.ndarray]] = None, + step_meter: float = 5.0, +) -> List[PolylineSetMesh]: + """Read a ``WellboreTrajectoryRepresentation`` into legacy containers. + + A list of trajectories is accepted and read in turn, as before. + """ + if energyml_object is None: + return [] + if isinstance(energyml_object, list): + return [ + mesh + for obj in energyml_object + for mesh in read_wellbore_trajectory_representation( + obj, workspace, use_crs_displacement, sub_indices, wellbore_frame_mds, step_meter ) + ] + + return _to_legacy_meshes( + read_numpy_wellbore_trajectory_representation( + energyml_object=energyml_object, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + wellbore_frame_mds=wellbore_frame_mds, + step_meter=step_meter, + ), + identifier=lambda patch, _index: f"{get_obj_uri(patch.energyml_object)}", + ) - patch_idx = patch_idx + 1 +def read_sub_representation( + energyml_object: Any, + workspace: EnergymlStorageInterface, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> List[AbstractMesh]: + """Read a ``SubRepresentation`` by delegating to its supporting representation.""" + meshes = _to_legacy_meshes( + read_numpy_sub_representation( + energyml_object=energyml_object, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + ) + ) + for m in meshes: + m.identifier = f"sub representation {get_obj_uri(energyml_object)} of {m.identifier}" return meshes +def read_representation_set_representation( + energyml_object: Any, + workspace: EnergymlStorageInterface, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> List[AbstractMesh]: + """Read every member representation of a ``RepresentationSetRepresentation``.""" + return _to_legacy_meshes( + read_numpy_representation_set_representation( + energyml_object=energyml_object, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + ) + ) + + def gen_surface_grid_geometry( energyml_object: Any, patch: Any, @@ -456,14 +558,23 @@ def gen_surface_grid_geometry( sub_indices: Optional[Union[List[int], np.ndarray]] = None, offset: int = 0, ): + """ + Build the points and quad indices of one Grid2d patch. + + .. deprecated:: + No longer used internally — :func:`read_grid2d_representation` delegates to + :func:`~energyml.utils.data.mesh_numpy.read_numpy_grid2d_representation`, which builds the + same connectivity by broadcasting instead of a Python double loop (about 16x faster on a + 1000x1000 grid). Kept because it is public. Prefer the numpy reader. + """ points = read_grid2d_patch( patch=patch, grid2d=energyml_object, path_in_root=patch_path, workspace=workspace, ) - logging.debug(f"Total points read: {len(points)}") - logging.debug(f"Sample points: {points[0:5]}") + logger.debug(f"Total points read: {len(points)}") + logger.debug(f"Sample points: {points[0:5]}") fa_count = search_attribute_matching_name(patch, "FastestAxisCount") if fa_count is None: @@ -476,7 +587,7 @@ def gen_surface_grid_geometry( fa_count = fa_count[0] sa_count = sa_count[0] - # logging.debug(f"sa_count {sa_count} fa_count {fa_count}") + # logger.debug(f"sa_count {sa_count} fa_count {fa_count}") points_no_nan = [] @@ -494,21 +605,15 @@ def gen_surface_grid_geometry( points_no_nan.append(p) indices = [] - while sa_count * fa_count > len(points): - sa_count = sa_count - 1 - fa_count = fa_count - 1 - - while sa_count * fa_count < len(points): - sa_count = sa_count + 1 - fa_count = fa_count + 1 + sa_count, fa_count = _fit_grid_dimensions(sa_count, fa_count, len(points)) - logging.debug(f"sa_count {sa_count} fa_count {fa_count} : {sa_count * fa_count} - {len(points)} ") + logger.debug(f"sa_count {sa_count} fa_count {fa_count} : {sa_count * fa_count} - {len(points)} ") for sa in range(sa_count - 1): for fa in range(fa_count - 1): line = sa * fa_count # if sa+1 == int(sa_count / 2) and fa == int(fa_count / 2): - # logging.debug( + # logger.debug( # "\n\t", (line + fa), " : ", (line + fa) in indice_to_final_indice, # "\n\t", (line + fa + 1), " : ", (line + fa + 1) in indice_to_final_indice, # "\n\t", (line + fa_count + fa + 1), " : ", (line + fa_count + fa + 1) in indice_to_final_indice, @@ -544,1439 +649,81 @@ def gen_surface_grid_geometry( if 0 <= t_idx < len(indices): new_indices.append(indices[t_idx]) indices = new_indices - # logging.debug(indices) + # logger.debug(indices) return points if keep_holes else points_no_nan, indices -def read_grid2d_representation( - energyml_object: Any, - workspace: Optional[EnergymlStorageInterface] = None, - use_crs_displacement: bool = True, - keep_holes: bool = False, - sub_indices: Optional[Union[List[int], np.ndarray]] = None, -) -> List[SurfaceMesh]: - # h5_reader = HDF5FileReader() - meshes = [] - - if sub_indices is not None: - sub_indices = list(sorted(sub_indices)) - - patch_idx = 0 - total_size = 0 +# __ ______________ __ __ _____ __ ____ __ +# / |/ / ____/ ___// / / / / __(_) /__ _____ / __/___ _________ ___ ____ _/ /_ +# / /|_/ / __/ \__ \/ /_/ / / /_/ / / _ \/ ___/ / /_/ __ \/ ___/ __ `__ \/ __ `/ __/ +# / / / / /___ ___/ / __ / / __/ / / __(__ ) / __/ /_/ / / / / / / / / /_/ / /_ +# /_/ /_/_____//____/_/ /_/ /_/ /_/_/\___/____/ /_/ \____/_/ /_/ /_/ /_/\__,_/\__/ - # Resqml 201 - for patch_path, patch in search_attribute_matching_name_with_path(energyml_object, "Grid2dPatch"): - logging.debug("Trying to read Grid2d representation with Resqml 2.0.1 schema (Grid2dPatch)") - logging.debug(f" > {get_obj_uri(energyml_object)}Found patch at path {patch_path} with object {patch}") - crs = None - try: - crs = get_crs_obj( - context_obj=patch, - path_in_root=patch_path, - root_obj=energyml_object, - workspace=workspace, - ) - except ObjectNotFoundNotError: - pass - points, indices = gen_surface_grid_geometry( - energyml_object=energyml_object, - patch=patch, - patch_path=patch_path, - workspace=workspace, - keep_holes=keep_holes, - sub_indices=sub_indices, - offset=total_size, - ) +def export_obj(mesh_list: List[AbstractMesh], out: BytesIO, obj_name: Optional[str] = None): + """ + Export an :class:`AbstractMesh` into obj format. - total_size = total_size + len(indices) + This function is maintained for backward compatibility and delegates to the + export module. For new code, consider importing from energyml.utils.data.export. - meshes.append( - SurfaceMesh( - identifier=f"{get_obj_uri(energyml_object)}_patch{patch_idx}", - energyml_object=energyml_object, - crs_object=crs, - point_list=points, - faces_indices=indices, - ) - ) - patch_idx = patch_idx + 1 + Each AbstractMesh from the list :param:`mesh_list` will be placed into its own group. + :param mesh_list: + :param out: + :param obj_name: + :return: + """ + # Delegate to the new export module + _export_obj_new(mesh_list, out, obj_name) - # Resqml 22 - if hasattr(energyml_object, "geometry"): - logging.debug( - "Trying to read Grid2d representation with Resqml 2.2 schema (geometry attribute on the representation)" - ) - crs = None - try: - crs = get_crs_obj( - context_obj=energyml_object, - path_in_root=".", - root_obj=energyml_object, - workspace=workspace, - ) - except ObjectNotFoundNotError as e: - logging.error(e) - # geometry = energyml_object.geometry - # points = read_grid2d_patch( - # patch=energyml_object, - # grid2d=energyml_object, - # path_in_root="", - # workspace=workspace, - # ) - points, indices = gen_surface_grid_geometry( - energyml_object=energyml_object, - patch=energyml_object, - patch_path="", - workspace=workspace, - keep_holes=keep_holes, - sub_indices=sub_indices, - offset=total_size, - ) - meshes.append( - SurfaceMesh( - identifier=f"{get_obj_uri(energyml_object)}_patch{patch_idx}", - energyml_object=energyml_object, - crs_object=crs, - point_list=points, - faces_indices=indices, - ) - ) - return meshes +def _list_exportable_uuids(epc: EnergymlStorageInterface, logger: Optional[Any] = None) -> List[str]: + """ + Return the uuids of every object of the EPC that has a mesh reader (i.e. that can be + exported as a 3D / GeoJSON file). + """ + uuids: List[str] = [] + for metadata in epc.list_objects(): + object_type = getattr(metadata, "object_type", None) + uuid = getattr(metadata, "uuid", None) + if not object_type or not uuid or uuid in uuids: + continue + if get_object_reader_function(_mesh_name_mapping(object_type)) is not None: + uuids.append(uuid) + (logger or _MODULE_LOGGER).debug(f"{len(uuids)} exportable representations found") + return uuids -def read_triangulated_set_representation( - energyml_object: Any, - workspace: EnergymlStorageInterface, +def export_multiple_data( + epc_path: str, + uuid_list: Optional[List[str]] = None, + output_folder_path: str = ".", + output_file_path_suffix: str = "", + file_format: MeshFileFormat = MeshFileFormat.OBJ, use_crs_displacement: bool = True, - sub_indices: Optional[Union[List[int], np.ndarray]] = None, -) -> List[SurfaceMesh]: - meshes = [] + logger: Optional[Any] = None, + to_wgs84: bool = True, + use_network: bool = False, +): + """ + :param uuid_list: uuids of the representations to export. When None or empty, every + exportable representation of the EPC is exported. + :param to_wgs84: GeoJSON only — reproject the coordinates to WGS84 (RFC 7946) when the EPSG + codes are available and ``pyproj`` is installed. + :param use_network: GeoJSON only — allow PROJ to download the geoid grids needed by the + vertical datum transformation. + """ + _MODULE_LOGGER.debug(f"Opening epc : {epc_path}") + epc = EpcFile(epc_file_path=epc_path, mode=EpcAccessMode.MANUAL, compact_on_close=False) + _MODULE_LOGGER.debug("Opened") - point_offset = 0 - patch_idx = 0 - total_size = 0 + if not uuid_list: + uuid_list = _list_exportable_uuids(epc, logger) - patches = search_attribute_matching_name_with_path( - energyml_object, - "\\w*Patch.\\d+", - deep_search=False, - search_in_sub_obj=False, - ) - # logging.debug(f"Found {len(patches)} patches for triangulated set representation") - - for patch_path, patch in patches: - crs = None - try: - crs = get_crs_obj( - context_obj=patch, - path_in_root=patch_path, - root_obj=energyml_object, - workspace=workspace, - ) - except ObjectNotFoundNotError: - pass - - point_list: List[Point] = [] - for point_path, point_obj in search_attribute_matching_name_with_path(patch, "Geometry.Points"): - _array = read_array( - energyml_array=point_obj, - root_obj=energyml_object, - path_in_root=patch_path + "." + point_path, - workspace=workspace, - ) - if isinstance(_array, np.ndarray): - _array = _array.tolist() - - point_list = point_list + _array - - # Apply full CRS transform (rotation + offsets + z-flip + axis-swap) per patch. - # Setting crs_object=None on the resulting mesh prevents the outer - # read_mesh_object dispatcher from calling crs_displacement() a second time. - logging.debug( - f"Applying use_crs_displacement {use_crs_displacement} with crs {crs} on patch {patch_path} with {len(point_list)} points for triangulated set representation {get_obj_uri(energyml_object)}" - ) - if use_crs_displacement and crs is not None and point_list: - logging.debug(f"Original points sample: {point_list[0:5]}") - pts_arr = np.asarray(point_list, dtype=np.float64).reshape(-1, 3) - crs_info = extract_crs_info(crs, workspace) - apply_from_crs_info(pts_arr, crs_info, inplace=True) - logging.debug(f"Transformed points sample: {pts_arr[0:5]}") - point_list = pts_arr.tolist() - - triangles_list: List[List[int]] = [] - for ( - triangles_path, - triangles_obj, - ) in search_attribute_matching_name_with_path(patch, "Triangles"): - _array = read_array( - energyml_array=triangles_obj, - root_obj=energyml_object, - path_in_root=patch_path + "." + triangles_path, - workspace=workspace, - ) - if isinstance(_array, np.ndarray): - _array = _array.tolist() - triangles_list = triangles_list + _array - - triangles_list = list(map(lambda tr: [ti - point_offset for ti in tr], triangles_list)) - if sub_indices is not None and len(sub_indices) > 0: - new_triangles_list = [] - for idx in sub_indices: - t_idx = idx - total_size - if 0 <= t_idx < len(triangles_list): - new_triangles_list.append(triangles_list[t_idx]) - total_size = total_size + len(triangles_list) - triangles_list = new_triangles_list - else: - total_size = total_size + len(triangles_list) - meshes.append( - SurfaceMesh( - identifier=f"{get_obj_uri(energyml_object)}_patch{patch_idx}", - energyml_object=energyml_object, - crs_object=crs, - point_list=point_list, - faces_indices=triangles_list, - ) - ) - point_offset = point_offset + len(point_list) - patch_idx += 1 - - return meshes - - -def read_wellbore_frame_representation( - energyml_object: Any, - workspace: EnergymlStorageInterface, - use_crs_displacement: bool = True, - sub_indices: Optional[Union[List[int], np.ndarray]] = None, -) -> List[PolylineSetMesh]: - """ - Read a WellboreFrameRepresentation and construct a polyline mesh from the trajectory. - - :param energyml_object: The WellboreFrameRepresentation object - :param workspace: The EnergymlStorageInterface to access related objects - :param sub_indices: Optional list of indices to filter specific nodes - :return: List containing a single PolylineSetMesh representing the wellbore - """ - - meshes = [] - - try: - # Read measured depths (NodeMd) - wellbore_frame_mds = None - try: - node_md_path, node_md_obj = search_attribute_matching_name_with_path(energyml_object, "NodeMd")[0] - wellbore_frame_mds = read_array( - energyml_array=node_md_obj, - root_obj=energyml_object, - path_in_root=node_md_path, - workspace=workspace, - ) - # Ensure wellbore_frame_mds is a numpy array for filtering operations - if not isinstance(wellbore_frame_mds, np.ndarray): - wellbore_frame_mds = np.array(wellbore_frame_mds) - except (IndexError, AttributeError) as e: - logging.warning(f"Could not read NodeMd from wellbore frame: {e}") - return meshes - - # Get reference point (wellhead location) - try different attribute paths for different versions - md_min = np.min(wellbore_frame_mds) if len(wellbore_frame_mds) > 0 else 0.0 - md_max = np.max(wellbore_frame_mds) if len(wellbore_frame_mds) > 0 else 0.0 - - try: - # Only works for RESQML 2.2+ - _md_min = get_object_attribute(energyml_object, "md_interval.md_min") - if _md_min is not None: - md_min = _md_min - _md_max = get_object_attribute(energyml_object, "md_interval.md_max") - if _md_max is not None: - md_max = _md_max - except AttributeError: - # logging.debug( - # "Could not get md_interval.md_min or md_interval.md_max, using NodeMd min/max instead" - # ) - pass - - # remove md values from array if outside of md_min/md_max range (can happen if md_interval is used and NodeMd contains values outside of the interval) - wellbore_frame_mds = wellbore_frame_mds[(wellbore_frame_mds >= md_min) & (wellbore_frame_mds <= md_max)] - - # Get trajectory reference - trajectory_dor = search_attribute_matching_name(obj=energyml_object, name_rgx="Trajectory")[0] - trajectory_obj = workspace.get_object(get_obj_uri(trajectory_dor)) - - # print(f"Mds {wellbore_frame_mds}") - - meshes = read_wellbore_trajectory_representation( - energyml_object=trajectory_obj, - workspace=workspace, - use_crs_displacement=use_crs_displacement, - sub_indices=sub_indices, - wellbore_frame_mds=wellbore_frame_mds, - ) - for mesh in meshes: - mesh.identifier = f"{get_obj_uri(energyml_object)}" - return meshes - except Exception as e: - logging.error(f"Failed to read wellbore frame representation: {e}") - import traceback - - traceback.print_exc() - - return meshes - - -def read_wellbore_trajectory_representation( - energyml_object: Any, - workspace: EnergymlStorageInterface, - use_crs_displacement: bool = True, - sub_indices: Optional[Union[List[int], np.ndarray]] = None, - wellbore_frame_mds: Optional[Union[List[float], np.ndarray]] = None, - step_meter: float = 5.0, -) -> List[PolylineSetMesh]: - if energyml_object is None: - return [] - - if isinstance(energyml_object, list): - return [ - mesh - for obj in energyml_object - for mesh in read_wellbore_trajectory_representation( - obj, workspace, use_crs_displacement, sub_indices, wellbore_frame_mds, step_meter - ) - ] - - # CRS - crs = None - head_x, head_y, head_z, z_increasing_downward, projected_epsg_code, vertical_epsg_code = ( - 0.0, - 0.0, - 0.0, - False, - None, - None, - ) - - # Get CRS from trajectory geometry if available - try: - crs_attr = get_object_attribute(energyml_object, "geometry.LocalCrs") - if crs_attr is not None: - crs = workspace.get_object(get_obj_uri(crs_attr)) - else: - raise ObjectNotFoundNotError("LocalCrs attribute not found in trajectory geometry") - except Exception: - logging.debug("Could not get CRS from trajectory geometry") - - # ========== - # MD Datum - # ========== - try: - # Try to get MdDatum (RESQML 2.0.1) or MdInterval.Datum (RESQML 2.2+) - md_datum_dor = None - try: - md_datum_dor = search_attribute_matching_name(obj=energyml_object, name_rgx=r"MdDatum")[0] - except IndexError: - try: - md_datum_dor = search_attribute_matching_name(obj=energyml_object, name_rgx=r"MdInterval.Datum")[0] - except IndexError: - pass - - if md_datum_dor is not None: - md_datum_identifier = get_obj_uri(md_datum_dor) - md_datum_obj = workspace.get_object(md_datum_identifier) - - if md_datum_obj is not None: - ( - head_x, - head_y, - head_z, - z_increasing_downward, - projected_epsg_code, - vertical_epsg_code, - crs, - ) = get_datum_information(md_datum_obj, workspace) - # if crs is None: - # crs = get_crs_obj( - # context_obj=md_datum_obj, - # path_in_root=".", - # root_obj=energyml_object, - # workspace=workspace, - # ) - except Exception as e: - logging.debug(f"Could not get reference point / Datum from trajectory: {e}") - - # ========== - well_points = None - logging.debug( - f"wellbore mds : {wellbore_frame_mds}\n\tCRs : {crs}\n\thead x,y,z : {head_x}, {head_y}, {head_z}\n\tz increasing downward : {z_increasing_downward}" - ) - try: - crs_info = extract_crs_info(crs, workspace) - # Try to read parametric Geometry from the trajectory. - traj_mds, traj_points, traj_tangents = read_parametric_geometry( - getattr(energyml_object, "geometry", None), workspace - ) - well_points = get_wellbore_points(wellbore_frame_mds, traj_mds, traj_points, traj_tangents, step_meter) - if use_crs_displacement: - well_points = apply_from_crs_info( - np.asarray(well_points, dtype=np.float64), - crs_info, - ) - except Exception as e: - if wellbore_frame_mds is not None: - logging.debug(f"Could not read parametric geometry from trajectory. Well is interpreted as vertical: {e}") - well_points = generate_vertical_well_points( - head_x=head_x, - head_y=head_y, - head_z=head_z, - wellbore_mds=wellbore_frame_mds, - z_increasing_downward=z_increasing_downward, - ) - else: - traceback.print_exc() - raise ValueError( - "Cannot read wellbore trajectory representation: no parametric geometry and no measured depth information available to generate points" - ) - - meshes = [] - if well_points is not None and len(well_points) > 0: - - meshes.append( - PolylineSetMesh( - identifier=f"{get_obj_uri(energyml_object)}", - energyml_object=energyml_object, - crs_object=crs, - point_list=well_points, - line_indices=[[i, i + 1] for i in range(len(well_points) - 1)], - ) - ) - return meshes - - -def read_sub_representation( - energyml_object: Any, - workspace: EnergymlStorageInterface, - use_crs_displacement: bool = True, - sub_indices: Optional[Union[List[int], np.ndarray]] = None, -) -> List[AbstractMesh]: - supporting_rep_dor = search_attribute_matching_name( - obj=energyml_object, name_rgx=r"(SupportingRepresentation|RepresentedObject)" - )[0] - supporting_rep_identifier = get_obj_uri(supporting_rep_dor) - supporting_rep = workspace.get_object(supporting_rep_identifier) - - total_size = 0 - all_indices = None - for patch_path, patch_indices in search_attribute_matching_name_with_path( - obj=energyml_object, - name_rgx="SubRepresentationPatch.\\d+.ElementIndices.\\d+.Indices", - deep_search=False, - search_in_sub_obj=False, - ) + search_attribute_matching_name_with_path( - obj=energyml_object, - name_rgx="SubRepresentationPatch.\\d+.Indices", - deep_search=False, - search_in_sub_obj=False, - ): - array = read_array( - energyml_array=patch_indices, - root_obj=energyml_object, - path_in_root=patch_path, - workspace=workspace, - sub_indices=sub_indices, - ) - - if sub_indices is not None and len(sub_indices) > 0: - new_array = [] - for idx in sub_indices: - t_idx = idx - total_size - if 0 <= t_idx < len(array): - new_array.append(array[t_idx]) - total_size = total_size + len(array) - array = new_array - else: - total_size = total_size + len(array) - - all_indices = all_indices + array if all_indices is not None else array - meshes = read_mesh_object( - energyml_object=supporting_rep, - workspace=workspace, - use_crs_displacement=use_crs_displacement, - sub_indices=all_indices, - ) - - for m in meshes: - m.identifier = f"sub representation {get_obj_uri(energyml_object)} of {m.identifier}" - - return meshes - - -def read_representation_set_representation( - energyml_object: Any, - workspace: EnergymlStorageInterface, - use_crs_displacement: bool = True, - sub_indices: Optional[Union[List[int], np.ndarray]] = None, -) -> List[AbstractMesh]: - - repr_list = get_object_attribute(energyml_object, "representation") - if repr_list is None or not isinstance(repr_list, list): - logging.error( - f"RepresentationSetRepresentation {get_obj_uri(energyml_object)} has no 'representation' list attribute" - ) - return [] - - meshes = [] - for repr_dor in repr_list: - rpr_uri = get_obj_uri(repr_dor) - repr_obj = workspace.get_object(rpr_uri) - if repr_obj is None: - logging.error(f"Representation {rpr_uri} in RepresentationSetRepresentation not found") - continue - meshes.extend( - read_mesh_object(energyml_object=repr_obj, workspace=workspace, use_crs_displacement=use_crs_displacement) - ) - - return meshes - - -def read_property( - energyml_object: Any, - workspace: EnergymlStorageInterface, -) -> np.ndarray: - """ - Read a property or column-based table from an Energyml object. - - Dispatches to the appropriate reader function based on the object's type name. - If no specific reader is found, raises a NotSupportedError. - - Args: - energyml_object: The Energyml object to read from. - workspace: The storage interface for accessing related objects. - - Returns: - np.ndarray: The read property or table data. - - Raises: - NotSupportedError: If the object type is not supported. - """ - property_type = type(energyml_object).__name__ - reader_func = get_object_reader_function(property_type) - if reader_func is not None: - return reader_func(energyml_object=energyml_object, workspace=workspace) - else: - # logging.error(f"Type {array_type_name} is not supported: function read_{snake_case(array_type_name)} not found") - raise NotSupportedError( - f"Type {property_type} is not supported\n\tfunction read_{snake_case(property_type)} not found" - ) - - -def read_property_interpreted_with_cbt( - energyml_object: Any, - workspace: EnergymlStorageInterface, - _cache_property_arrays: Optional[np.ndarray] = None, - _return_none_if_no_category_lookup: bool = False, -) -> Optional[np.ndarray]: - """ - Read a property with category lookup interpretation. - - Reads property arrays and applies category lookup mapping if available. - Supports both array and dictionary-based category lookups. - - Args: - energyml_object: The Energyml property object. - workspace: The storage interface for accessing related objects. - _cache_property_arrays: Optional cached property arrays to avoid re-reading. - _return_none_if_no_category_lookup: If True, return None when no category lookup is found. - - Returns: - Optional[np.ndarray]: The interpreted property values, or None if no lookup and flag is set. - """ - - result = None - - prop_arrays = ( - read_property(energyml_object, workspace) if _cache_property_arrays is None else _cache_property_arrays - ) - - category_lookup_dor = get_object_attribute(energyml_object, "category_lookup") - if category_lookup_dor is not None: - category_lookup_obj = workspace.get_object(get_obj_uri(category_lookup_dor)) - if category_lookup_obj is not None: - category_lookup_data = read_column_based_table(category_lookup_obj, workspace) - - # print(f"category_lookup_array : {category_lookup_data}") - if isinstance(category_lookup_data, list): - category_lookup_data = np.array(category_lookup_data) - if isinstance(category_lookup_data, np.ndarray): - # map props values to category lookup values using prop value as index in category lookup array - result = ( - np.array( - [ - ( - category_lookup_data[prop] - if prop is not None and prop < len(category_lookup_data) - else None - ) - for prop in prop_arrays - ] - ) - if prop_arrays is not None - else None - ) - elif isinstance(category_lookup_data, dict): - # Transpose so that each index corresponds to a category (column), not a row. - # logging.debug(f"category_lookup_data dict : {category_lookup_data}") - - # Guard against inhomogeneous column lengths (e.g. one column is - # empty while another is not). Pad all columns with None up to - # the maximum column length so that np.array() can build a - # rectangular (n_columns, max_rows) matrix before transposing. - col_values = [list(v) if not isinstance(v, list) else v for v in category_lookup_data.values()] - max_len = max((len(c) for c in col_values), default=0) - if max_len == 0: - # All columns empty — nothing to look up. - return prop_arrays if not _return_none_if_no_category_lookup else None - - padded = [c + [None] * (max_len - len(c)) for c in col_values] - category_lookup_matrice = np.array(padded, dtype=object).T - # logging.debug(f"category_lookup_matrice : {category_lookup_matrice}") - # return a matrice with the same shape as prop_arrays but with the values from the category lookup array using the prop value as key in the category lookup array - result = ( - np.array( - [ - [ - ( - category_lookup_matrice[prop].tolist() - if prop is not None and 0 <= prop < len(category_lookup_matrice) - else None - ) - for prop in prop_row - ] - for prop_row in prop_arrays - ] - ) - if prop_arrays is not None - else None - ) - else: - raise NotSupportedError( - f"Category lookup array type {type(category_lookup_matrice)} is not supported, expected list or dict" - ) - - return prop_arrays if result is None and not _return_none_if_no_category_lookup else result - - -def read_abstract_values_property( - energyml_object: Any, - workspace: EnergymlStorageInterface, -) -> np.ndarray: - """ - Read abstract values property from patches. - - Extracts and concatenates arrays from all 'values_for_patch' attributes. - - Args: - energyml_object: The Energyml object containing the property. - workspace: The storage interface for accessing arrays. - - Returns: - np.ndarray: The concatenated array of property values. - """ - arrays = [] - for values_for_patch in search_attribute_matching_name_with_path(energyml_object, "values_for_patch"): - array = read_array( - energyml_array=values_for_patch[1], - root_obj=energyml_object, - path_in_root=".", - workspace=workspace, - ) - if isinstance(array, list): - array = np.array(array) - arrays.append(array) - if len(arrays) == 1: - return arrays[0] - else: - return np.concatenate(arrays) - - -def read_discrete_property( - energyml_object: Any, - workspace: EnergymlStorageInterface, -) -> np.ndarray: - """ - Read a discrete property. - - Delegates to read_abstract_values_property for implementation. - - Args: - energyml_object: The discrete property object. - workspace: The storage interface. - - Returns: - np.ndarray: The property values. - """ - - return read_abstract_values_property(energyml_object, workspace) - - -def read_continuous_property( - energyml_object: Any, - workspace: EnergymlStorageInterface, -) -> np.ndarray: - """ - Read a continuous property. - - Delegates to read_abstract_values_property for implementation. - - Args: - energyml_object: The continuous property object. - workspace: The storage interface. - - Returns: - np.ndarray: The property values. - """ - - return read_abstract_values_property(energyml_object, workspace) - - -def read_categorical_property( - energyml_object: Any, - workspace: EnergymlStorageInterface, -) -> np.ndarray: - """ - Read a categorical property. - - Note: Categorical values are returned as integers. Use the property's - 'code_list' attribute to map to string values. - - Args: - energyml_object: The categorical property object. - workspace: The storage interface. - - Returns: - np.ndarray: The integer-coded property values. - """ - # TODO: the categorical values should be converted to strings using the code list of the property, but for now we keep the integer values and let the user manage the conversion if needed. - logging.warning( - "CategoricalProperty is read as a continuous property, the categorical values are not converted to strings but kept as integers. Use the 'code_list' attribute of the property to get the list of possible string values corresponding to the integer values in the array" - ) - return read_abstract_values_property(energyml_object, workspace) - - -def read_comment_property( - energyml_object: Any, - workspace: EnergymlStorageInterface, -) -> np.ndarray: - """ - Read a comment property. - - Delegates to read_abstract_values_property for implementation. - - Args: - energyml_object: The comment property object. - workspace: The storage interface. - - Returns: - np.ndarray: The comment values. - """ - return read_abstract_values_property(energyml_object, workspace) - - -def read_column_based_table( - energyml_object: Any, - workspace: EnergymlStorageInterface, -) -> Dict[str, np.ndarray]: - """ - Read a column-based table. - - Extracts column data into a dictionary keyed by column titles. - - Args: - energyml_object: The table object with 'column' attributes. - workspace: The storage interface for accessing arrays. - - Returns: - Dict[str, np.ndarray]: Dictionary of column names to arrays. - """ - columns = {} - for column in get_object_attribute(energyml_object, "column"): - column_name = getattr(column, "title", "_") - # print(f"Reading column: {column_name} : {column}") - # print(f"getattr(column_array, 'values', None): {getattr(column, 'values', None)}") - array = read_array( - energyml_array=getattr(column, "values", None), - root_obj=energyml_object, - path_in_root=".", - workspace=workspace, - ) - if isinstance(array, list): - array = np.array(array) - columns[column_name] = array - return columns - - -def read_time_series( - energyml_object: Any, - workspace: EnergymlStorageInterface, -) -> List[Tuple[str, int]]: - """ - Read a time series from an Energyml object. - - Extracts date-time values and time step indices, constructing a normalized - list of (step_index, datetime) tuples for each time step. - - Args: - energyml_object: The Energyml time series object. - workspace: The storage interface for accessing related objects. - - Returns: - List[Tuple[str, int]]: List of tuples containing (step_index, datetime_string). - """ - - # 1. Extraction des DateTime - times_iso = search_attribute_matching_name(energyml_object, "date_time") - - # 2. Extraction des TimeSteps (v2.2+) - steps_indices = [] - time_step_obj = get_object_attribute(energyml_object, "time_step") - if time_step_obj is not None: - steps_indices = read_array(time_step_obj, energyml_object, ".", workspace, sub_indices=None) - else: - # Fallback : on utilise l'index de la liste - steps_indices = list(range(len(times_iso))) - - # 3. Construction de la structure normalisée - steps_data = [] - for i in range(len(times_iso)): - steps_data.append( - (steps_indices[i], times_iso[i]) - # {"index": i, "datetime": times_iso[i], "step_val": steps_indices[i]} # L'index utilisé par les propriétés - ) - - return steps_data - - -# __ ______________ __ __ _____ __ ____ __ -# / |/ / ____/ ___// / / / / __(_) /__ _____ / __/___ _________ ___ ____ _/ /_ -# / /|_/ / __/ \__ \/ /_/ / / /_/ / / _ \/ ___/ / /_/ __ \/ ___/ __ `__ \/ __ `/ __/ -# / / / / /___ ___/ / __ / / __/ / / __(__ ) / __/ /_/ / / / / / / / / /_/ / /_ -# /_/ /_/_____//____/_/ /_/ /_/ /_/_/\___/____/ /_/ \____/_/ /_/ /_/ /_/\__,_/\__/ - - -def _recompute_min_max( - old_min: List, # out parameters - old_max: List, # out parameters - potential_min: List, - potential_max: List, -) -> None: - for i in range(len(potential_min)): - if i >= len(old_min): - old_min.append(potential_min[i]) - elif potential_min[i] is not None: - old_min[i] = min(old_min[i], potential_min[i]) - - for i in range(len(potential_max)): - if i >= len(old_max): - old_max.append(potential_max[i]) - elif potential_max[i] is not None: - old_max[i] = max(old_max[i], potential_max[i]) - - -def _recompute_min_max_from_points( - old_min: List, # out parameters - old_max: List, # out parameters - points: Union[List[Point], Point], -) -> None: - if len(points) > 0: - if isinstance(points[0], list): - for p in points: - _recompute_min_max_from_points(old_min, old_max, p) - else: - _recompute_min_max(old_min, old_max, points, points) - - -def _create_shape( - geo_type: GeoJsonGeometryType, - point_list: List[List[float]], - indices: Optional[Union[List[List[int]], List[int]]] = None, - point_offset: int = 0, - logger: Optional[Any] = None, -) -> Tuple[List, List[float], List[float]]: - """ - Creates a shape from a point list [ [x0, y0 (, z0)? ], ..., [xn, yn (, zn)? ] ] - using indices. If indices is a simple list, result will be a line like : [p0, ..., pn]. With p0 and pn - a list of coordinate from "points" parameter (like [x0, y0 (, z0)? ]) - If the indices are a list of list, result will be polygones like : - [ - [poly0_p0, ..., poly0_pn], - ... - [polyn_p0, ..., polyn_pn], - ] - :return shape, minXYZ (as list), maxXYZ (as list) - """ - mins = [] - maxs = [] - result = None - try: - if geo_type == GeoJsonGeometryType.LineString: - result = [] - if indices is not None and len(indices) > 0: - for idx in indices: - result.append(point_list[idx + point_offset]) - _recompute_min_max_from_points(mins, maxs, point_list[idx + point_offset]) - else: - result = point_list - _recompute_min_max_from_points(mins, maxs, result) - elif geo_type == GeoJsonGeometryType.MultiPoint or geo_type == GeoJsonGeometryType.Point: - result = point_list - _recompute_min_max_from_points(mins, maxs, result) - elif geo_type == GeoJsonGeometryType.MultiLineString: - if indices is not None and len(indices) > 0 and isinstance(indices[0], list): - result = [] - for idx in indices: - _res, _min, _max = _create_shape( - geo_type=GeoJsonGeometryType.MultiLineString, - point_list=point_list, - indices=idx, - point_offset=point_offset, - logger=logger, - ) - result = result + _res - _recompute_min_max(mins, maxs, _min, _max) - else: - _res, _min, _max = _create_shape( - geo_type=GeoJsonGeometryType.LineString, - point_list=point_list, - indices=indices, - point_offset=point_offset, - logger=logger, - ) - result = [_res] - _recompute_min_max(mins, maxs, _min, _max) - elif geo_type == GeoJsonGeometryType.Polygon: - result, mins, maxs = _create_shape( - geo_type=GeoJsonGeometryType.MultiLineString, # Here we only provide 1 line, the external one (outer-ring) - point_list=point_list, - indices=indices, - point_offset=point_offset, - logger=logger, - ) - # First and last must be the same - if len(result) > 0 and result[0] != result[-1]: - result.append(result[0]) - elif geo_type == GeoJsonGeometryType.MultiPolygon: - if indices is not None and len(indices) > 0 and isinstance(indices[0], list): - result = [] - for idx in indices: - _res, _min, _max = _create_shape( - geo_type=GeoJsonGeometryType.MultiPolygon, # Here we only provide 1 line, the external one (outer-ring) - point_list=point_list, - indices=idx, - point_offset=point_offset, - logger=logger, - ) - result = result + _res - _recompute_min_max(mins, maxs, _min, _max) - else: - _res, _min, _max = _create_shape( - geo_type=GeoJsonGeometryType.Polygon, # Here we only provide 1 line, the external one (outer-ring) - point_list=point_list, - indices=indices, - point_offset=point_offset, - logger=logger, - ) - result = [_res] - _recompute_min_max(mins, maxs, _min, _max) - except Exception as e: - if logger is not None: - logger.error(e) - # raise e - return result, mins, maxs - - -def _write_geojson_shape( - out: BytesIO, - geo_type: GeoJsonGeometryType, - point_list: List[List[float]], - indices: Optional[Union[List[List[int]], List[int]]] = None, - point_offset: int = 0, - logger: Optional[Any] = None, - _print_list_boundaries: Optional[bool] = True, -) -> Tuple[List[float], List[float]]: - """ - Write a shape from a point list [ [x0, y0 (, z0)? ], ..., [xn, yn (, zn)? ] ] - using indices. If indices is a simple list, result will be a line like : [p0, ..., pn]. With p0 and pn - a list of coordinate from "points" parameter (like [x0, y0 (, z0)? ]) - If the indices are a list of list, result will be polygones like : - [ - [poly0_p0, ..., poly0_pn], - ... - [polyn_p0, ..., polyn_pn], - ] - :return shape, minXYZ (as list), maxXYZ (as list) - """ - mins = [] - maxs = [] - try: - if geo_type == GeoJsonGeometryType.LineString: - if indices is not None and len(indices) > 0: - cpt = 0 - if _print_list_boundaries: - out.write(b"[") - for idx in indices: - out.write(json.dumps(point_list[idx + point_offset]).encode("utf-8")) - if cpt < len(indices) - 1: - out.write(b", ") - cpt += 1 - _recompute_min_max_from_points(mins, maxs, point_list[idx + point_offset]) - if _print_list_boundaries: - out.write(b"]") - else: - out.write(json.dumps(point_list).encode("utf-8")) - _recompute_min_max_from_points(mins, maxs, point_list) - elif geo_type == GeoJsonGeometryType.MultiPoint or geo_type == GeoJsonGeometryType.Point: - out.write(json.dumps(point_list).encode("utf-8")) - _recompute_min_max_from_points(mins, maxs, point_list) - elif geo_type == GeoJsonGeometryType.MultiLineString: - if indices is not None and len(indices) > 0 and isinstance(indices[0], list): - if _print_list_boundaries: - out.write(b"[") - cpt = 0 - for idx in indices: - _min, _max = _write_geojson_shape( - out=out, - geo_type=GeoJsonGeometryType.MultiLineString, - point_list=point_list, - indices=idx, - point_offset=point_offset, - logger=logger, - _print_list_boundaries=False, - ) - if cpt < len(indices) - 1: - out.write(b", ") - cpt += 1 - _recompute_min_max(mins, maxs, _min, _max) - if _print_list_boundaries: - out.write(b"]") - else: - if _print_list_boundaries: - out.write(b"[") - _min, _max = _write_geojson_shape( - out=out, - geo_type=GeoJsonGeometryType.LineString, - point_list=point_list, - indices=indices, - point_offset=point_offset, - logger=logger, - ) - _recompute_min_max(mins, maxs, _min, _max) - if _print_list_boundaries: - out.write(b"]") - elif geo_type == GeoJsonGeometryType.Polygon: - # First and last must be the same - if indices is not None and len(indices) > 0: - if indices[0] != indices[-1]: - indices.append(indices[0]) - elif point_list[0] != point_list[-1]: - point_list.append(point_list[0]) - - mins, maxs = _write_geojson_shape( - out=out, - geo_type=GeoJsonGeometryType.MultiLineString, # Here we only provide 1 line, the external one (outer-ring) - point_list=point_list, - indices=indices, - point_offset=point_offset, - logger=logger, - _print_list_boundaries=_print_list_boundaries, - ) - elif geo_type == GeoJsonGeometryType.MultiPolygon: - if indices is not None and len(indices) > 0 and isinstance(indices[0], list): - if _print_list_boundaries: - out.write(b"[") - cpt = 0 - for idx in indices: - _min, _max = _write_geojson_shape( - out=out, - geo_type=GeoJsonGeometryType.MultiPolygon, # Here we only provide 1 line, the external one (outer-ring) - point_list=point_list, - indices=idx, - point_offset=point_offset, - logger=logger, - _print_list_boundaries=False, - ) - if cpt < len(indices) - 1: - out.write(b", ") - cpt += 1 - _recompute_min_max(mins, maxs, _min, _max) - if _print_list_boundaries: - out.write(b"]") - else: - if _print_list_boundaries: - out.write(b"[") - _min, _max = _write_geojson_shape( - out=out, - geo_type=GeoJsonGeometryType.Polygon, # Here we only provide 1 line, the external one (outer-ring) - point_list=point_list, - indices=indices, - point_offset=point_offset, - logger=logger, - ) - _recompute_min_max(mins, maxs, _min, _max) - if _print_list_boundaries: - out.write(b"]") - except Exception as e: - if logger is not None: - logger.error(e) - # raise e - return mins, maxs - - -def to_geojson_feature( - mesh: AbstractMesh, - geo_type: GeoJsonGeometryType = GeoJsonGeometryType.Point, - geo_type_prefix: Optional[str] = "AnyCrs", - properties: Optional[dict] = None, - point_offset: int = 0, - logger=None, -) -> Dict: - feature = {} - - if mesh.point_list is not None and len(mesh.point_list) > 0: - points = mesh.point_list - - # TODO: remove : - # points = list(map( - # lambda p: list(map(lambda x: round(x/10000., 4), p)), - # mesh.point_list - # )) - - indices = mesh.get_indices() - # polygon must have the first and last point as the same - if geo_type == GeoJsonGeometryType.Polygon or geo_type == GeoJsonGeometryType.MultiPolygon: - if logger is not None: - logger.debug("# to_geojson_feature > Reshaping indices for polygons") - if indices is not None: - for indices_i in indices: - indices_i.append(indices_i[0]) - if logger is not None: - logger.debug("\t# to_geojson_feature > Indices reshaped") - - if logger is not None: - logger.debug("# to_geojson_feature > Computing shape") - - coordinates, mins, maxs = _create_shape( - geo_type=geo_type, - point_list=points, - indices=indices, - point_offset=point_offset, - logger=logger, - ) - - # Pop previously added last : - if geo_type == GeoJsonGeometryType.Polygon or geo_type == GeoJsonGeometryType.MultiPolygon: - if indices is not None: - for indices_i in indices: - indices_i.pop() - - if logger is not None: - logger.debug("\t# to_geojson_feature > shaped") - - bbox_geometry = [] # TODO : see : https://www.rfc-editor.org/rfc/rfc7946#section-5 - - bbox_geometry = mins + maxs - - geometry = { - # "type": f"{geo_type_prefix}{geo_type.name}", - "type": f"{geo_type.name}", - "coordinates": coordinates, - "bbox": bbox_geometry, - } - - feature = { - "type": f"{geo_type_prefix}Feature", - "properties": properties or {}, - "geometry": geometry, - } - - return feature - - -def write_geojson_feature( - out: BytesIO, - mesh: AbstractMesh, - geo_type: GeoJsonGeometryType = GeoJsonGeometryType.Point, - geo_type_prefix: Optional[str] = "AnyCrs", - properties: Optional[dict] = None, - point_offset: int = 0, - logger=None, -) -> None: - if mesh.point_list is not None and len(mesh.point_list) > 0: - points = mesh.point_list - - indices = mesh.get_indices() - # polygon must have the first and last point as the same - if geo_type == GeoJsonGeometryType.Polygon or geo_type == GeoJsonGeometryType.MultiPolygon: - if logger is not None: - logger.debug("# to_geojson_feature > Reshaping indices for polygons") - if indices is not None: - for indices_i in indices: - indices_i.append(indices_i[0]) - if logger is not None: - logger.debug("\t# to_geojson_feature > Indices reshaped") - - if logger is not None: - logger.debug("# to_geojson_feature > Computing shape") - - out.write(b"{") # start feature - out.write(f'"type": "{geo_type_prefix}Feature", '.encode()) - out.write(f'"properties": {json.dumps(properties or {}) }, '.encode()) - out.write(b'"geometry": ') - - out.write(b"{") # start geometry - # "type": f"{geo_type_prefix}{geo_type.name}", - out.write(f'"type": "{geo_type.name}", '.encode()) - out.write('"coordinates": '.encode()) - mins, maxs = _write_geojson_shape( - out=out, - geo_type=geo_type, - point_list=points, - indices=indices, - point_offset=point_offset, - logger=logger, - ) - bbox_geometry = mins + maxs # TODO : see : https://www.rfc-editor.org/rfc/rfc7946#section-5 - - out.write(f', "bbox": {json.dumps(bbox_geometry)}'.encode()) - out.write(b"}") # end geometry - - # Pop previously added last : - if geo_type == GeoJsonGeometryType.Polygon or geo_type == GeoJsonGeometryType.MultiPolygon: - if indices is not None: - for indices_i in indices: - indices_i.pop() - - if logger is not None: - logger.debug("\t# to_geojson_feature > shaped") - - out.write(b"}") # End feature - - -def mesh_to_geojson_type(obj: AbstractMesh) -> GeoJsonGeometryType: - if isinstance(obj, SurfaceMesh): - return GeoJsonGeometryType.MultiPolygon - elif isinstance(obj, PolylineSetMesh): - return GeoJsonGeometryType.MultiLineString - else: - return GeoJsonGeometryType.MultiPoint - - -def export_geojson_io( - out: BytesIO, - mesh_list: List[AbstractMesh], - obj_name: Optional[str] = None, - properties: Optional[List[Optional[Dict]]] = None, - global_properties: Optional[Dict] = None, - logger: Optional[Any] = None, -): - out.write(b"{") - out.write(b'"type": "FeatureCollection",') - if obj_name is not None: - out.write(b'"name": "') - out.write(obj_name.encode()) - out.write(b'",') - - if global_properties is not None and len(global_properties) > 0: - for k, v in global_properties.items(): - out.write(b'"') - out.write(k.encode()) - out.write(b'": ') - out.write(json.dumps(v).encode()) - out.write(b",") - - out.write(b'"features": [') - - cpt = 0 - point_offset = 0 - - for mesh in mesh_list: - pos = out.tell() - write_geojson_feature( - out=out, - mesh=mesh, - geo_type=mesh_to_geojson_type(mesh), - properties=properties[cpt] if properties is not None and len(properties) > cpt else None, - point_offset=0, # point_offset, - logger=logger, - ) - if out.tell() != pos and cpt < len(mesh_list) - 1: - out.write(b",") - cpt += 1 - point_offset = point_offset + len(mesh.point_list) - out.write(b"]") # end features - out.write(b"}") # end geojson - - -def export_geojson_dict( - mesh_list: List[AbstractMesh], - obj_name: Optional[str] = None, - properties: Optional[List[Optional[Dict]]] = None, - logger: Optional[Any] = None, -): - res = {"type": "FeatureCollection", "features": []} - cpt = 0 - point_offset = 0 - for mesh in mesh_list: - feature = to_geojson_feature( - mesh=mesh, - geo_type=mesh_to_geojson_type(mesh), - properties=properties[cpt] if properties is not None and len(properties) > cpt else None, - point_offset=0, # point_offset, - logger=logger, - ) - if feature is not None: - res["features"].append(feature) - cpt += 1 - point_offset = point_offset + len(mesh.point_list) - - return res - - -def export_off(mesh_list: List[AbstractMesh], out: BytesIO): - """ - Export an :class:`AbstractMesh` into off format. - :param mesh_list: - :param out: - :return: - """ - nb_points = sum(list(map(lambda m: len(m.point_list), mesh_list))) - nb_edges = sum(list(map(lambda m: m.get_nb_edges(), mesh_list))) - nb_faces = sum(list(map(lambda m: m.get_nb_faces(), mesh_list))) - - out.write(b"OFF\n") - out.write(_FILE_HEADER) - out.write(f"{nb_points} {nb_faces} {nb_edges}\n".encode("utf-8")) - - points_io = BytesIO() - faces_io = BytesIO() - - point_offset = 0 - for m in mesh_list: - export_off_part( - off_point_part=points_io, - off_face_part=faces_io, - points=m.point_list, - indices=m.get_indices(), - point_offset=point_offset, - colors=[], - ) - point_offset = point_offset + len(m.point_list) - - out.write(points_io.getbuffer()) - out.write(faces_io.getbuffer()) - - -def export_off_part( - off_point_part: BytesIO, - off_face_part: BytesIO, - points: List[List[float]], - indices: List[List[int]], - point_offset: Optional[int] = 0, - colors: Optional[List[List[int]]] = None, -) -> None: - for p in points: - for pi in p: - off_point_part.write(f"{pi} ".encode("utf-8")) - off_point_part.write(b"\n") - - cpt = 0 - for face in indices: - if len(face) > 1: - off_face_part.write(f"{len(face)} ".encode("utf-8")) - for pi in face: - off_face_part.write(f"{pi + point_offset} ".encode("utf-8")) - - if colors is not None and len(colors) > cpt and colors[cpt] is not None and len(colors[cpt]) > 0: - for col in colors[cpt]: - off_face_part.write(f"{col} ".encode("utf-8")) - - off_face_part.write(b"\n") - cpt += 1 - - -def export_obj(mesh_list: List[AbstractMesh], out: BytesIO, obj_name: Optional[str] = None): - """ - Export an :class:`AbstractMesh` into obj format. - - This function is maintained for backward compatibility and delegates to the - export module. For new code, consider importing from energyml.utils.data.export. - - Each AbstractMesh from the list :param:`mesh_list` will be placed into its own group. - :param mesh_list: - :param out: - :param obj_name: - :return: - """ - # Delegate to the new export module - _export_obj_new(mesh_list, out, obj_name) - - -def _export_obj_elt( - off_point_part: BytesIO, - off_face_part: BytesIO, - points: List[List[float]], - indices: List[List[int]], - point_offset: Optional[int] = 0, - colors: Optional[List[List[int]]] = None, - elt_letter: str = "f", -) -> None: - """ - - :param off_point_part: - :param off_face_part: - :param points: - :param indices: - :param point_offset: - :param colors: currently not supported - :param elt_letter: "l" for line and "f" for faces - :return: - """ - offset_obj = 1 # OBJ point indices starts at 1 not 0 - for p in points: - if len(p) > 0: - off_point_part.write(f"v {' '.join(list(map(lambda xyz: str(xyz), p)))}\n".encode("utf-8")) - - # cpt = 0 - for face in indices: - if len(face) > 1: - off_face_part.write( - f"{elt_letter} {' '.join(list(map(lambda x: str(x + point_offset + offset_obj), face)))}\n".encode( - "utf-8" - ) - ) - - # if colors is not None and len(colors) > cpt and colors[cpt] is not None and len(colors[cpt]) > 0: - # for col in colors[cpt]: - # off_face_part.write(f"{col} ".encode('utf-8')) - - # off_face_part.write(b"\n") - - -def export_multiple_data( - epc_path: str, - uuid_list: List[str], - output_folder_path: str, - output_file_path_suffix: str = "", - file_format: MeshFileFormat = MeshFileFormat.OBJ, - use_crs_displacement: bool = True, - logger: Optional[Any] = None, -): - epc = EpcStreamReader(epc_path) - - # with open(epc_path.replace(".epc", ".h5"), "rb") as fh: - # buf = BytesIO(fh.read()) - # epc.h5_io_files.append(buf) + # with open(epc_path.replace(".epc", ".h5"), "rb") as fh: + # buf = BytesIO(fh.read()) + # epc.h5_io_files.append(buf) try: os.makedirs(output_folder_path, exist_ok=True) @@ -1987,44 +734,103 @@ def export_multiple_data( energyml_obj = None try: energyml_obj = epc.get_object_by_uuid(uuid)[0] - except: - if logger is not None: - logger.error(f"Object with uuid {uuid} not found") - else: - logging.error(f"Object with uuid {uuid} not found") + except Exception as e: + # a bare `except` here also swallowed KeyboardInterrupt / SystemExit + (logger or _MODULE_LOGGER).error(f"Object with uuid {uuid} not found : {type(e).__name__}: {e}") continue - file_name = ( + # A citation title is free text and lands in the file name: sanitize it, or a title + # containing ':' (e.g. "AUB-PRO-SP05512: Trajectory") silently writes into an NTFS + # alternate data stream on Windows, leaving an empty extension-less file behind. + # The extension is appended after sanitizing so it can never be truncated away. + file_name = sanitize_file_name( f"{gen_energyml_object_path(energyml_obj)}_" f"[{get_object_attribute(energyml_obj, 'citation.title')}]" f"{output_file_path_suffix}" - f".{file_format.value}" ) - file_path = f"{output_folder_path}/{file_name}" - logging.debug(f"Exporting : {file_path}") - mesh_list = read_mesh_object( - energyml_object=energyml_obj, - workspace=epc, - use_crs_displacement=use_crs_displacement, - ) - if file_format == MeshFileFormat.OBJ: - with open(file_path, "wb") as f: - export_obj( - mesh_list=mesh_list, - out=f, - ) - elif file_format == MeshFileFormat.OFF: - with open(file_path, "wb") as f: - export_off( - mesh_list=mesh_list, - out=f, - ) - elif file_format == MeshFileFormat.GEOJSON: - with open(file_path, "wb") as f: - export_geojson_io( - out=f, - mesh_list=mesh_list, - logger=logger, - global_properties={"epc_path": epc_path}, - ) - else: - logging.error(f"Code is not written for format {file_format}") + file_path = os.path.join(output_folder_path, f"{file_name}.{file_format.value}") + _MODULE_LOGGER.debug(f"Exporting : {file_path}") + + # a representation that cannot be read (e.g. a trajectory without geometry) must not + # stop the export of the others + try: + mesh_list = read_mesh_object( + energyml_object=energyml_obj, + workspace=epc, + use_crs_displacement=use_crs_displacement, + ) + + if file_format == MeshFileFormat.OBJ: + with open(file_path, "wb") as f: + export_obj( + mesh_list=mesh_list, + out=f, + ) + elif file_format == MeshFileFormat.OFF: + with open(file_path, "wb") as f: + export_off( + mesh_list=mesh_list, + out=f, + ) + elif file_format == MeshFileFormat.GEOJSON: + with open(file_path, "wb") as f: + export_geojson_io( + out=f, + mesh_list=mesh_list, + obj_name=get_object_attribute(energyml_obj, "citation.title"), + logger=logger, + workspace=epc, + to_wgs84=to_wgs84, + use_network=use_network, + global_properties={"epc_path": epc_path}, + indent=2, + ) + else: + _MODULE_LOGGER.error(f"Code is not written for format {file_format}") + except Exception as e: + (logger or _MODULE_LOGGER).error(f"Failed to export the object {uuid} : {type(e).__name__}: {e}") + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "get_property_reader_function", + "read_abstract_values_property", + "read_categorical_property", + "read_column_based_table", + "read_comment_property", + "read_continuous_property", + "read_discrete_property", + "read_property", + "read_property_interpreted_with_cbt", + "read_time_series", + "export_off", + "export_off_part", + "GeoJsonGeometryType", + "energyml_type_to_geojson_type", + "export_geojson_dict", + "export_geojson_io", + "mesh_to_geojson_type", + "to_geojson_feature", + "write_geojson_feature", + "Point", + "MeshFileFormat", + "AbstractMesh", + "PointSetMesh", + "PolylineSetMesh", + "SurfaceMesh", + "get_object_reader_function", + "get_mesh_reader_function", + "read_mesh_object", + "read_ijk_grid_representation", + "read_point_representation", + "read_polyline_representation", + "read_grid2d_representation", + "read_triangulated_set_representation", + "read_wellbore_frame_representation", + "read_wellbore_trajectory_representation", + "read_sub_representation", + "read_representation_set_representation", + "gen_surface_grid_geometry", + "export_obj", + "export_multiple_data", +] diff --git a/energyml-utils/src/energyml/utils/data/mesh_numpy.py b/energyml-utils/src/energyml/utils/data/mesh_numpy.py index 053a082..cd810d6 100644 --- a/energyml-utils/src/energyml/utils/data/mesh_numpy.py +++ b/energyml-utils/src/energyml/utils/data/mesh_numpy.py @@ -11,11 +11,23 @@ ------------ * **No list conversion** - no ``.tolist()`` calls anywhere. Arrays stay as numpy throughout. -* **Best-effort zero-copy** - geometry is read via +* **Best-effort zero-copy read** - geometry is read via :meth:`EnergymlStorageInterface.read_array_view`. For contiguous, uncompressed HDF5 datasets this returns a numpy view backed directly by the memory-mapped file buffer (no RAM copy). Chunked / compressed datasets fall back silently to a copy. + + That view must **not** be mutated (it is the reader's own buffer, possibly the + mapped file), so ``_ensure_float64_points`` takes ownership of the *points* + before any CRS transform is applied in place: exactly one full-size copy per + patch, and none at all when no transform is requested + (``frame=PointFrame.LOCAL``). Connectivity arrays keep the zero-copy path — + they are only ever read. + +* **Explicit coordinate frame** - every patch carries the + :class:`~energyml.utils.data.crs.PointFrame` its points are in, so + ``read_numpy_mesh_object`` applies only the missing pipeline stages and a + transform can never be applied twice. * **PyVista-ready connectivity** - ``faces`` / ``lines`` / ``cells`` arrays use the VTK flat-count-prefixed format consumed directly by ``pyvista.PolyData`` and ``pyvista.UnstructuredGrid`` without additional @@ -42,21 +54,18 @@ """ from __future__ import annotations -import inspect import logging import re import sys -import traceback from dataclasses import dataclass, field +from functools import lru_cache from typing import Any, Callable, Dict, List, Optional, Tuple, Union import numpy as np from energyml.utils.data.helper import ( - apply_crs_transform, evaluate_parametric_line_array, generate_vertical_well_points, - get_crs_offsets_and_angle, get_crs_obj, get_crs_origin_offset, get_datum_information, @@ -67,7 +76,12 @@ resolve_parametric_line_array, get_wellbore_points, ) -from energyml.utils.data.crs import extract_crs_info, apply_from_crs_info +from energyml.utils.data.crs import ( + PointFrame, + apply_from_crs_info, + extract_crs_info, + to_frame, +) from energyml.utils.exception import NotSupportedError, ObjectNotFoundNotError from energyml.utils.introspection import ( get_obj_uri, @@ -79,6 +93,8 @@ ) from energyml.utils.storage_interface import EnergymlStorageInterface +logger = logging.getLogger(__name__) + # --------------------------------------------------------------------------- # Internal helper: thin proxy that makes read_array_view look like read_array # so that helper.read_array benefits from zero-copy semantics transparently. @@ -149,6 +165,10 @@ class NumpyMesh: source_uuid: Optional[str] = field(default=None) #: Python class name of the source RESQML object. source_type: Optional[str] = field(default=None) + #: Coordinate frame ``points`` is expressed in. Readers set it to what they produced, and + #: :func:`read_numpy_mesh_object` only applies the stages still missing — so a CRS transform + #: cannot be applied twice, whatever the representation type. + frame: PointFrame = field(default=PointFrame.LOCAL) #: Optional named arrays attached to this mesh (e.g. ``node_time_values``). extra_arrays: Dict[str, np.ndarray] = field(default_factory=dict) @@ -268,9 +288,16 @@ def crs_displacement_np( ) -> np.ndarray: """Apply CRS origin offset and optional Z-axis inversion to *points*. + .. deprecated:: + Use :func:`~energyml.utils.data.crs.to_frame` instead. This function only applies the + offsets and the Z flip — **not** the areal rotation nor the axis-order swap — so it does + not produce coordinates in the projected CRS. It used to be the dispatcher's fallback, + which is why a ``Grid2dRepresentation`` came out un-rotated from the numpy stack while the + same object came out rotated from :mod:`mesh`. It is kept because it is part of the public + API, but no reader calls it any more. + Operates on an ``(N, 3)`` numpy array using broadcast arithmetic — no - Python-level loops. Prefer :func:`apply_from_crs_info` for full CRS - transforms (rotation, axis-order swap, etc.). + Python-level loops. Args: points: Shape ``(N, 3)``, dtype ``float64``. Modified in-place when @@ -308,24 +335,70 @@ def crs_displacement_np( # --------------------------------------------------------------------------- -def _ensure_float64_points(arr: Any) -> np.ndarray: - """Convert *arr* to ``(N, 3) float64``. +def _ensure_float64_points(arr: Any, *, own: bool = True) -> np.ndarray: + """Convert *arr* to ``(N, 3) float64``, owning the buffer by default. - Accepts numpy arrays (any shape that contains N*3 elements) or nested - Python lists. Returns a 2-D view/cast when possible, copy only when - dtype conversion is required. + Accepts numpy arrays (any shape that contains N*3 elements) or nested Python lists. + + ``own=True`` (default) guarantees the result is a writeable array backed by memory this + module allocated. That matters because the geometry may arrive from + :meth:`EnergymlStorageInterface.read_array_view`, whose contract is explicit — *"the caller + must not mutate the returned array"*: for a contiguous uncompressed HDF5 dataset it is a view + on the memory-mapped file, and the CRS transform is applied in place. Mutating it would either + raise (read-only buffer) or corrupt the reader's cache, so that a second read of the same + array would come back already transformed. + + Pass ``own=False`` only when the caller immediately copies anyway (e.g. feeding + :func:`numpy.concatenate`), to save one full-size buffer. """ a = np.asarray(arr, dtype=np.float64) + + # `base is not None` means we are looking at a view of someone else's buffer; a non-writeable + # array cannot be transformed in place either. In both cases we must own a copy first. + if own and (a.base is not None or not a.flags.writeable): + a = a.copy() + if a.ndim == 1: a = a.reshape(-1, 3) elif a.ndim == 2 and a.shape[1] == 2: - # 2-D points (e.g. seismic / plan view) — pad Z column with zeros - a = np.column_stack([a, np.zeros(len(a), dtype=np.float64)]) + # A 2-column array is ambiguous. RESQML point arrays are 3-component, and some datasets + # store them with a shape that does not reflect that — ``80wells_surf_modified_val_color`` + # holds 4 XYZ points in a (6, 2) dataset. So a size divisible by 3 is read as XYZ, which + # is what the legacy reader did with its plain reshape(-1, 3); only a size that cannot be + # XYZ is treated as 2-D points (seismic / plan view) and padded with a zero Z. + if a.size % 3 == 0: + a = a.reshape(-1, 3) + else: + a = np.column_stack([a, np.zeros(len(a), dtype=np.float64)]) elif a.ndim == 2 and a.shape[1] != 3: raise ValueError(f"Expected (N, 2) or (N, 3) points array, got shape {a.shape}") return a +def _local_to_projected( + points: np.ndarray, + crs: Any, + workspace: Optional[EnergymlStorageInterface], + use_crs_displacement: bool, +) -> PointFrame: + """Apply the local → projected transform to *points* in place and report the frame reached. + + Readers call this instead of :func:`apply_from_crs_info` so that the frame they produced is + recorded on the mesh. :func:`read_numpy_mesh_object` then tops the points up to the requested + frame, and a transform can never be applied twice — which is what the hard-coded list of type + names used to guard, one entry per reader. + """ + if not use_crs_displacement or crs is None or len(points) == 0: + return PointFrame.LOCAL + return to_frame( + points, + extract_crs_info(crs, workspace), + PointFrame.PROJECTED, + PointFrame.LOCAL, + inplace=True, + ).frame + + def _ensure_int64(arr: Any) -> np.ndarray: """Return *arr* as a flat ``int64`` numpy array.""" a = np.asarray(arr, dtype=np.int64) @@ -352,6 +425,24 @@ def _build_vtk_faces_from_quads(quad: np.ndarray) -> np.ndarray: return np.concatenate([counts, quad], axis=1).ravel() +def _build_vtk_single_polyline(n_points: int) -> np.ndarray: + """Build a VTK flat lines array holding *one* polyline through all *n_points* nodes. + + Result: ``[n, 0, 1, …, n-1]``. + + This is what a ``PolylineRepresentation`` without ``NodeCountPerPolyline`` means: a single + polyline, not a bag of independent segments. :func:`_build_vtk_lines_from_segments` encodes the + same geometry as ``n-1`` two-point cells, which renders identically but is a different + topology — and produces one OBJ/OFF element per segment instead of one per line. + """ + if n_points < 2: + return np.empty(0, dtype=np.int64) + part = np.empty(n_points + 1, dtype=np.int64) + part[0] = n_points + part[1:] = np.arange(n_points, dtype=np.int64) + return part + + def _build_vtk_lines_from_segments(n_points: int) -> np.ndarray: """Build VTK flat lines array for a single poly-line of *n_points* nodes. @@ -366,27 +457,39 @@ def _build_vtk_lines_from_segments(n_points: int) -> np.ndarray: return np.concatenate([counts, pairs], axis=1).ravel() -def _build_vtk_lines_from_node_counts(node_counts: np.ndarray) -> np.ndarray: - """Build VTK flat lines array from per-polyline node counts. +def _fit_grid_dimensions(sa_count: int, fa_count: int, nb_points: int) -> Tuple[int, int]: + """Reconcile the declared axis counts of a Grid2d patch with the points actually read. - For each polyline of length *n* we emit ``[n, 0, 1, …, n-1]`` with - indices local to the global point array (starting at the correct offset). + ``SlowestAxisCount`` / ``FastestAxisCount`` sometimes disagree with the length of the + points array (truncated dataset, count declared on the representation rather than on the + patch, …). The fastest-axis count defines the row stride of the connectivity, so it is + kept and the slowest-axis count is derived from it. The result always satisfies + ``sa * fa <= nb_points``, which is what keeps the generated indices in range. - Returns ``(total_entries,)`` int64 array. + Both readers previously decremented (then re-incremented) *both* counts until the product + fitted, which changed the grid shape and, with ``keep_holes=True``, could emit indices past + the end of the points array. + + Returns: + The (possibly adjusted) ``(sa_count, fa_count)``; ``(0, 0)`` when no face can be built. """ - result_parts = [] - offset = 0 - for n in node_counts: - n = int(n) - local = np.arange(offset, offset + n, dtype=np.int64) - part = np.empty(n + 1, dtype=np.int64) - part[0] = n - part[1:] = local - result_parts.append(part) - offset += n - if not result_parts: - return np.empty(0, dtype=np.int64) - return np.concatenate(result_parts) + if fa_count <= 0 or sa_count <= 0 or nb_points <= 0: + logger.warning( + f"Grid2d patch: unusable dimensions (slowest={sa_count}, fastest={fa_count}, " + f"{nb_points} points) — no face is generated." + ) + return 0, 0 + + if sa_count * fa_count == nb_points: + return sa_count, fa_count + + fitted_sa = nb_points // fa_count + logger.warning( + f"Grid2d patch: {sa_count} x {fa_count} = {sa_count * fa_count} nodes declared but " + f"{nb_points} points read — keeping the fastest axis ({fa_count}) and using " + f"{fitted_sa} for the slowest one." + ) + return fitted_sa, fa_count def _read_array_np( @@ -447,20 +550,34 @@ def _decode_jagged_array( def _numpy_mesh_name_mapping(arr_type_name: str) -> str: - """Normalise the energyml type name to match a ``read_numpy_`` function.""" + """Normalise the energyml type name to match a ``read_numpy_`` function. + + Accepts a python class name (``ObjTriangulatedSetRepresentation``), a schema type + (``obj_TriangulatedSetRepresentation``, the RESQML 2.0.1 spelling) and a qualified type + (``resqml20.obj_TriangulatedSetRepresentation``) alike. + """ + arr_type_name = arr_type_name.rsplit(".", 1)[-1] arr_type_name = arr_type_name.replace("3D", "3d").replace("2D", "2d") - arr_type_name = re.sub(r"^[Oo]bj([A-Z])", r"\1", arr_type_name) + arr_type_name = re.sub(r"^[Oo]bj_?([A-Z])", r"\1", arr_type_name) arr_type_name = re.sub(r"(Polyline|Point)Set", r"\1", arr_type_name) return arr_type_name +@lru_cache(maxsize=None) def get_numpy_reader_function(mesh_type_name: str) -> Optional[Callable]: - """Return the ``read_numpy_`` function for *mesh_type_name*, or ``None``.""" - target = f"read_numpy_{snake_case(mesh_type_name)}" - for name, obj in inspect.getmembers(sys.modules[__name__]): - if name == target: - return obj - return None + """Return the ``read_numpy_`` function for *mesh_type_name*, or ``None``. + + A cached ``getattr`` rather than a scan of the module members: the dispatcher runs once per + object, and ``inspect.getmembers`` sorts and reads *every* attribute of the module on each + call — measurable when listing the exportable objects of a large EPC. + + Only functions defined in this module are eligible, so an imported helper can never be + mistaken for a reader (see :func:`mesh.get_object_reader_function`). + """ + reader = getattr(sys.modules[__name__], f"read_numpy_{snake_case(mesh_type_name)}", None) + if not callable(reader) or getattr(reader, "__module__", None) != __name__: + return None + return reader # --------------------------------------------------------------------------- @@ -507,15 +624,16 @@ def read_numpy_point_representation( pass if sub_indices is not None and len(sub_indices) > 0: + # total_size must advance by the number of points this patch *contributes to the + # global numbering*, not by the number that survived the filter — otherwise every + # subsequent patch shifts its sub_indices window. + patch_size = len(points) t_idx = np.asarray(sub_indices, dtype=np.int64) - total_size - mask = (t_idx >= 0) & (t_idx < len(points)) + mask = (t_idx >= 0) & (t_idx < patch_size) points = points[t_idx[mask]] - total_size += len(points) + total_size += patch_size - # Apply full CRS transform per patch; crs_object kept for reference, - # outer dispatcher is guarded to skip crs_displacement_np for this type. - if use_crs_displacement and crs is not None and len(points) > 0: - apply_from_crs_info(points, extract_crs_info(crs, workspace), inplace=True) + frame = _local_to_projected(points, crs, workspace, use_crs_displacement) label = f"{src_type}_patch_{patch_idx}" multi.patches.append( @@ -524,6 +642,7 @@ def read_numpy_point_representation( energyml_object=energyml_object, crs_object=crs, points=points, + frame=frame, patch_index=patch_idx, patch_label=label, source_uuid=src_uuid, @@ -562,7 +681,7 @@ def read_numpy_polyline_representation( if not pts_list: pts_list = search_attribute_matching_name_with_path(patch, "Points") if not pts_list: - logging.error(f"Cannot find points for patch {patch_path_in_obj}") + logger.error(f"Cannot find points for patch {patch_path_in_obj}") continue points_path, points_obj = pts_list[0] @@ -613,8 +732,8 @@ def read_numpy_polyline_representation( offset += n if close_poly is None or poly_idx >= len(close_poly) or not close_poly[poly_idx] else n - 1 lines = np.concatenate(parts) if parts else np.empty(0, dtype=np.int64) except IndexError: - # Single polyline — all points in sequence - lines = _build_vtk_lines_from_segments(len(points)) + # No NodeCountPerPolyline: the patch is a single polyline through all its points. + lines = _build_vtk_single_polyline(len(points)) # --- sub_indices filtering --- # sub_indices select individual *polylines* (by index within this patch). @@ -661,10 +780,7 @@ def read_numpy_polyline_representation( else: total_size += 1 # at least one polyline - # Apply full CRS transform per patch; crs_object kept for reference, - # outer dispatcher is guarded to skip crs_displacement_np for this type. - if use_crs_displacement and crs is not None and len(points) > 0: - apply_from_crs_info(points, extract_crs_info(crs, workspace), inplace=True) + frame = _local_to_projected(points, crs, workspace, use_crs_displacement) if len(points) > 0: label = f"{src_type}_patch_{patch_idx}" @@ -675,6 +791,7 @@ def read_numpy_polyline_representation( crs_object=crs, points=points, lines=lines, + frame=frame, patch_index=patch_idx, patch_label=label, source_uuid=src_uuid, @@ -737,19 +854,16 @@ def read_numpy_triangulated_set_representation( pts_parts: List[np.ndarray] = [] for point_path, point_obj in search_attribute_matching_name_with_path(patch, "Geometry.Points"): raw = _read_array_np(point_obj, energyml_object, patch_path + "." + point_path, ws) - pts_parts.append(_ensure_float64_points(raw)) + # own=False: np.concatenate below allocates the owned buffer anyway, even for a + # single part, so taking ownership here would cost one extra full-size copy. + pts_parts.append(_ensure_float64_points(raw, own=False)) if not pts_parts: patch_idx += 1 continue - points = np.concatenate(pts_parts, axis=0) # (N, 3) + points = np.concatenate(pts_parts, axis=0) # (N, 3), owned - # Apply full CRS transform (rotation + offsets + z-flip + axis-swap) per patch. - # Setting crs_object=None on the resulting mesh prevents the outer - # read_numpy_mesh_object dispatcher from calling crs_displacement_np() again. - if use_crs_displacement and crs is not None and len(points) > 0: - crs_info = extract_crs_info(crs, workspace) - apply_from_crs_info(points, crs_info, inplace=True) + frame = _local_to_projected(points, crs, workspace, use_crs_displacement) # --- Triangles --- tri_parts: List[np.ndarray] = [] @@ -766,12 +880,14 @@ def read_numpy_triangulated_set_representation( if point_offset != 0: triangles -= point_offset # local 0-based indices - # sub_indices face filtering + # sub_indices face filtering — total_size advances by the patch's own face count, not by + # the number of faces that survived the filter (see read_numpy_point_representation). + patch_face_count = len(triangles) if sub_indices is not None and len(sub_indices) > 0: t_idx = np.asarray(sub_indices, dtype=np.int64) - total_size - mask = (t_idx >= 0) & (t_idx < len(triangles)) + mask = (t_idx >= 0) & (t_idx < patch_face_count) triangles = triangles[t_idx[mask]] - total_size += len(triangles) + total_size += patch_face_count # Build VTK flat faces array: [3, v0, v1, v2, 3, v0, v1, v2, …] faces = _build_vtk_faces_from_triangles(triangles) @@ -784,6 +900,7 @@ def read_numpy_triangulated_set_representation( crs_object=crs, points=points, faces=faces, + frame=frame, patch_index=patch_idx, patch_label=label, source_uuid=src_uuid, @@ -848,14 +965,11 @@ def _process_patch(patch: Any, patch_path: str, crs: Any) -> Optional[NumpySurfa fa = int(fa_count[0]) sa = int(sa_count[0]) - # Clamp dimensions to actual number of points + # Reconcile the declared dimensions with the points actually read total_pts = len(pts) - while sa * fa > total_pts and sa > 0 and fa > 0: - sa -= 1 - fa -= 1 - while sa * fa < total_pts: - sa += 1 - fa += 1 + sa, fa = _fit_grid_dimensions(sa, fa, total_pts) + if sa == 0 or fa == 0: + return None z_col = pts[:, 2] nan_mask = np.isnan(z_col) # True where Z is NaN (hole) @@ -863,9 +977,7 @@ def _process_patch(patch: Any, patch_path: str, crs: Any) -> Optional[NumpySurfa if keep_holes: pts[nan_mask, 2] = 0.0 final_pts = pts - # All original indices are valid - local_idx = np.arange(total_pts, dtype=np.int64) - remap = local_idx # identity + remap = None # every original index stays valid: no remapping needed else: valid_mask = ~nan_mask final_pts = pts[valid_mask] @@ -873,32 +985,36 @@ def _process_patch(patch: Any, patch_path: str, crs: Any) -> Optional[NumpySurfa remap = np.full(total_pts, -1, dtype=np.int64) remap[valid_mask] = np.arange(valid_mask.sum(), dtype=np.int64) - # Build quad face list (vectorised) - quad_rows = [] - for sa_i in range(sa - 1): - for fa_i in range(fa - 1): - line = sa_i * fa - a = line + fa_i - b = line + fa_i + 1 - c = line + fa + fa_i + 1 - d = line + fa + fa_i - if keep_holes: - quad_rows.append([a, b, c, d]) - else: - ra, rb, rc, rd = remap[a], remap[b], remap[c], remap[d] - if ra >= 0 and rb >= 0 and rc >= 0 and rd >= 0: - quad_rows.append([ra, rb, rc, rd]) - - if not quad_rows: + # Build the quad connectivity. The corner indices of every cell are an affine function of + # (sa_i, fa_i), so the whole (sa-1) x (fa-1) grid is one broadcast instead of a Python + # double loop with a list append per cell. + if sa < 2 or fa < 2: + return None + sa_i = np.arange(sa - 1, dtype=np.int64).reshape(-1, 1) # (sa-1, 1) + fa_i = np.arange(fa - 1, dtype=np.int64).reshape(1, -1) # (1, fa-1) + corner_a = (sa_i * fa + fa_i).ravel() + quads = np.empty((corner_a.size, 4), dtype=np.int64) + quads[:, 0] = corner_a + quads[:, 1] = corner_a + 1 + quads[:, 2] = corner_a + fa + 1 + quads[:, 3] = corner_a + fa + + if not keep_holes: + # remap sends a NaN node to -1; a cell survives only when its four corners do. + quads = remap[quads] + quads = quads[(quads >= 0).all(axis=1)] + + if len(quads) == 0: return None - quads = np.asarray(quad_rows, dtype=np.int64) # (M, 4) - # sub_indices filtering + # sub_indices filtering — total_size advances by the patch's own quad count (see + # read_numpy_point_representation). + patch_quad_count = len(quads) if sub_indices is not None and len(sub_indices) > 0: t_idx = np.asarray(sub_indices, dtype=np.int64) - total_size - mask = (t_idx >= 0) & (t_idx < len(quads)) + mask = (t_idx >= 0) & (t_idx < patch_quad_count) quads = quads[t_idx[mask]] - total_size += len(quads) + total_size += patch_quad_count faces = _build_vtk_faces_from_quads(quads) label = f"{src_type}_patch_{patch_idx}" @@ -943,7 +1059,7 @@ def _process_patch(patch: Any, patch_path: str, crs: Any) -> Optional[NumpySurfa workspace=workspace, ) except ObjectNotFoundNotError as e: - logging.error(e) + logger.error(e) m = _process_patch(energyml_object, "", crs) if m is not None: multi.patches.append(m) @@ -984,7 +1100,7 @@ def read_numpy_wellbore_trajectory_representation( else: raise ObjectNotFoundNotError("LocalCrs not found") except Exception: - logging.debug("Could not get CRS from trajectory geometry") + logger.debug("Could not get CRS from trajectory geometry") # MD datum / reference point (fixes always-at-origin bug) try: @@ -1005,8 +1121,13 @@ def read_numpy_wellbore_trajectory_representation( md_datum_obj, workspace ) except Exception as e: - logging.debug(f"Could not resolve MdDatum from trajectory: {e}") + logger.debug(f"Could not resolve MdDatum from trajectory: {e}") + # The two paths below do not produce the same frame, which is why this reader could never be + # handled by the generic CRS pass: the parametric geometry is local and gets transformed, + # whereas the vertical fallback is built from the MD datum, whose coordinates + # (get_datum_information) are *already* expressed in the projected CRS. + frame = PointFrame.LOCAL try: crs_info = extract_crs_info(crs, workspace) traj_mds, traj_points, traj_tangents = read_parametric_geometry( @@ -1018,24 +1139,39 @@ def read_numpy_wellbore_trajectory_representation( np.asarray(well_points_list, dtype=np.float64), crs_info, ) + frame = PointFrame.PROJECTED except Exception as e: - if wellbore_frame_mds is not None: - logging.debug(f"Trajectory parametric geometry unavailable, treating as vertical: {e}") + mds = wellbore_frame_mds + if mds is None: + # A trajectory may carry no geometry at all and only declare the interval it spans + # — `MdInterval` plus the `Datum` it is measured from. That is enough to place a + # vertical well, and it is how every wellbore of a "MD interval only" package is + # written; raising here dropped all of them. + md_min = get_object_attribute(energyml_object, "md_interval.md_min") + md_max = get_object_attribute(energyml_object, "md_interval.md_max") + if md_min is not None and md_max is not None: + logger.info( + f"WellboreTrajectoryRepresentation {get_obj_uuid(energyml_object)} has no geometry; " + f"building a vertical well from MdInterval [{md_min}, {md_max}]." + ) + mds = np.array([float(md_min), float(md_max)], dtype=np.float64) + + if mds is not None: + logger.debug(f"Trajectory parametric geometry unavailable, treating as vertical: {e}") well_points_list = generate_vertical_well_points( head_x=head_x, head_y=head_y, head_z=head_z, - wellbore_mds=wellbore_frame_mds - if isinstance(wellbore_frame_mds, np.ndarray) - else np.asarray(wellbore_frame_mds), + wellbore_mds=mds if isinstance(mds, np.ndarray) else np.asarray(mds), z_increasing_downward=z_increasing_downward, ) + # Built from the datum: already projected, whatever use_crs_displacement says. + frame = PointFrame.PROJECTED else: - traceback.print_exc() raise ValueError( - "Cannot read WellboreTrajectoryRepresentation: " - "no parametric geometry and no measured depth information available." - ) + "Cannot read WellboreTrajectoryRepresentation: no parametric geometry, no measured " + "depth information and no MdInterval available." + ) from e if well_points_list is None or len(well_points_list) == 0: return NumpyMultiMesh( @@ -1046,7 +1182,10 @@ def read_numpy_wellbore_trajectory_representation( ) pts = _ensure_float64_points(np.asarray(well_points_list, dtype=np.float64)) - lines = _build_vtk_lines_from_segments(len(pts)) + # A trajectory is *one* polyline through its stations, not a bag of independent two-point + # cells. Both encodings render the same, but the segment one makes every consumer that + # iterates the cells — the GeoJSON writer, OBJ, OFF — produce N-1 elements for one well. + lines = _build_vtk_single_polyline(len(pts)) src_uuid = get_obj_uuid(energyml_object) src_type = type(energyml_object).__name__ label = f"{src_type}_patch_0" @@ -1062,6 +1201,7 @@ def read_numpy_wellbore_trajectory_representation( crs_object=crs, points=pts, lines=lines, + frame=frame, patch_index=0, patch_label=label, source_uuid=src_uuid, @@ -1092,7 +1232,7 @@ def read_numpy_wellbore_frame_representation( if not isinstance(wellbore_frame_mds, np.ndarray): wellbore_frame_mds = np.asarray(wellbore_frame_mds, dtype=np.float64) except (IndexError, AttributeError) as e: - logging.warning(f"Could not read NodeMd from wellbore frame: {e}") + logger.warning(f"Could not read NodeMd from wellbore frame: {e}") return empty md_min = float(wellbore_frame_mds.min()) if len(wellbore_frame_mds) > 0 else 0.0 @@ -1121,8 +1261,15 @@ def read_numpy_wellbore_frame_representation( wellbore_frame_mds=wellbore_frame_mds, ) frame_uri = str(get_obj_uri(energyml_object)) + # The geometry comes from the trajectory, but the patches were produced by reading *this* + # frame, and that is what they must report — like every other reader. Leaving the trajectory + # on them made a frame reached through a RepresentationSetRepresentation indistinguishable + # from the trajectory itself. for m in result.flat_patches(): m.identifier = frame_uri + m.energyml_object = energyml_object + m.source_uuid = get_obj_uuid(energyml_object) + m.source_type = type(energyml_object).__name__ result.identifier = frame_uri result.source_uuid = get_obj_uuid(energyml_object) result.source_type = type(energyml_object).__name__ @@ -1157,11 +1304,13 @@ def read_numpy_sub_representation( search_in_sub_obj=False, ): arr = _read_array_np(patch_indices, energyml_object, patch_path, ws).astype(np.int64).ravel() + # total_size advances by the patch's own index count (see read_numpy_point_representation). + patch_index_count = len(arr) if sub_indices is not None and len(sub_indices) > 0: t_idx = np.asarray(sub_indices, dtype=np.int64) - total_size - mask = (t_idx >= 0) & (t_idx < len(arr)) + mask = (t_idx >= 0) & (t_idx < patch_index_count) arr = arr[t_idx[mask]] - total_size += len(arr) + total_size += patch_index_count all_indices = np.concatenate([all_indices, arr]) if all_indices is not None else arr inner = read_numpy_mesh_object( @@ -1203,7 +1352,7 @@ def read_numpy_representation_set_representation( rpr_uri = get_obj_uri(repr_dor) repr_obj = workspace.get_object(rpr_uri) if repr_obj is None: - logging.error(f"Representation {rpr_uri} not found in RepresentationSetRepresentation") + logger.error(f"Representation {rpr_uri} not found in RepresentationSetRepresentation") continue child = read_numpy_mesh_object( energyml_object=repr_obj, @@ -1218,10 +1367,8 @@ def read_numpy_representation_set_representation( # VTK cell-type codes (subset used by RESQML readers) # --------------------------------------------------------------------------- -_VTK_TETRA = 10 +_VTK_EMPTY_CELL = 0 _VTK_HEXAHEDRON = 12 -_VTK_WEDGE = 13 -_VTK_PYRAMID = 14 _VTK_POLYHEDRON = 42 @@ -1265,8 +1412,10 @@ def read_numpy_plane_set_representation( root_obj=energyml_object, workspace=workspace, ) - except (ObjectNotFoundNotError, Exception): - pass + except Exception as exc: + # `(ObjectNotFoundNotError, Exception)` was just `Exception` with misleading intent. + # get_crs_obj can fail in several ways and a missing CRS is not fatal here. + logger.debug(f"No CRS resolved: {type(exc).__name__}: {exc}") planes_list = search_attribute_matching_name_with_path(energyml_object, "Planes") patch_idx = 0 @@ -1306,12 +1455,11 @@ def read_numpy_plane_set_representation( faces = _build_vtk_faces_from_triangles(tris) else: - logging.warning(f"PlaneSetRepresentation: unknown geometry type {geom_type!r} — skipping patch {patch_idx}") + logger.warning(f"PlaneSetRepresentation: unknown geometry type {geom_type!r} — skipping patch {patch_idx}") patch_idx += 1 continue - if use_crs_displacement and crs is not None and len(points) > 0: - apply_from_crs_info(points, extract_crs_info(crs, workspace), inplace=True) + frame = _local_to_projected(points, crs, workspace, use_crs_displacement) label = f"{src_type}_patch_{patch_idx}" multi.patches.append( @@ -1321,6 +1469,7 @@ def read_numpy_plane_set_representation( crs_object=crs, points=points, faces=faces, + frame=frame, patch_index=patch_idx, patch_label=label, source_uuid=src_uuid, @@ -1359,8 +1508,8 @@ def read_numpy_seismic_wellbore_frame_representation( node_time_values = _read_array_np(ntv_obj, energyml_object, ntv_path, ws) for patch in result.flat_patches(): patch.extra_arrays["node_time_values"] = node_time_values - except (IndexError, Exception) as exc: - logging.warning(f"SeismicWellboreFrameRepresentation: could not read NodeTimeValues: {exc}") + except Exception as exc: # IndexError from [0] on an empty match, or any read failure + logger.warning(f"SeismicWellboreFrameRepresentation: could not read NodeTimeValues: {exc}") result.source_type = type(energyml_object).__name__ return result @@ -1489,6 +1638,95 @@ def _build_split_pillar_map( return pillar_map +def _blank_undefined_pillars( + points: np.ndarray, # (NKL * n_pillars_total, 3), modified in place + geom: Any, + energyml_object: Any, + ws: Any, + nkl: int, + n_pillars_base: int, + n_pillars_total: int, + pillar_indices_arr: Optional[np.ndarray], +) -> None: + """Set the nodes of pillars flagged ``PillarGeometryIsDefined=false`` to NaN. + + RESQML makes the flag authoritative — "If the indicator does not indicate that the pillar + geometry is defined, then this over-rides any other node geometry specification" — so the + coordinates stored for such a pillar are meaningless and must not be drawn. The array is + indexed by pillar (``#Pillars`` = ``(NI+1)(NJ+1)``, 1-D or 2-D), so a split coordinate line + takes the flag of the pillar it was split from. + + A no-op when the flag is absent or every pillar is defined. + """ + flag_results = search_attribute_matching_name_with_path(geom, "PillarGeometryIsDefined") + if not flag_results: + return + flag_path, flag_obj = flag_results[0] + if flag_obj is None: + return + try: + defined = _read_array_np(flag_obj, energyml_object, f"geometry.{flag_path}", ws).astype(bool).ravel() + except Exception as exc: + logger.debug(f"Cannot read PillarGeometryIsDefined: {type(exc).__name__}: {exc}") + return + + if defined.size != n_pillars_base: + logger.warning( + f"PillarGeometryIsDefined holds {defined.size} entries for {n_pillars_base} pillars; ignoring it." + ) + return + if defined.all(): + return + + # Map every coordinate line to its pillar, then to the flag. + line_defined = np.ones(n_pillars_total, dtype=bool) + line_defined[:n_pillars_base] = defined + n_splits = n_pillars_total - n_pillars_base + if n_splits > 0 and pillar_indices_arr is not None: + pi = np.asarray(pillar_indices_arr, dtype=np.int64).ravel()[:n_splits] + valid = (pi >= 0) & (pi < n_pillars_base) + line_defined[n_pillars_base : n_pillars_base + len(pi)] = np.where(valid, defined[np.where(valid, pi, 0)], True) + + logger.info( + f"IjkGridRepresentation: {int((~line_defined).sum())}/{n_pillars_total} coordinate lines " + "flagged PillarGeometryIsDefined=false; their nodes are set to NaN." + ) + points.reshape(nkl, n_pillars_total, 3)[:, ~line_defined, :] = np.nan + + +def _read_cell_geometry_undefined( + geom: Any, + energyml_object: Any, + ws: Any, + ni: int, + nj: int, + nk: int, +) -> Optional[np.ndarray]: + """Return a ``(ni*nj*nk,)`` boolean mask of cells flagged ``CellGeometryIsDefined=false``. + + The array is cell-indexed, so it follows the grid's own ordering (I fastest, then J, then + K) and lines up with the cells built by :func:`read_numpy_ijk_grid_representation` without + any permutation. ``None`` when the flag is absent or unreadable. + """ + flag_results = search_attribute_matching_name_with_path(geom, "CellGeometryIsDefined") + if not flag_results: + return None + flag_path, flag_obj = flag_results[0] + if flag_obj is None: + return None + try: + defined = _read_array_np(flag_obj, energyml_object, f"geometry.{flag_path}", ws).astype(bool).ravel() + except Exception as exc: + logger.debug(f"Cannot read CellGeometryIsDefined: {type(exc).__name__}: {exc}") + return None + + n_cells = ni * nj * nk + if defined.size != n_cells: + logger.warning(f"CellGeometryIsDefined holds {defined.size} entries for {n_cells} cells; ignoring it.") + return None + return ~defined + + def _read_direct_points( pts_obj: Any, pts_path: str, @@ -1527,7 +1765,7 @@ def _read_direct_points( raise ValueError( f"IjkGridRepresentation: unexpected points array size {raw_pts.size}. " f"Expected {expected_3d} (3-D layout, nkl={nkl}, n_pillars={n_pillars_total}) " - f"or {expected_4d} (4-D layout, nkl={nkl}, nj+1={nj+1}, ni+1={ni+1})." + f"or {expected_4d} (4-D layout, nkl={nkl}, nj+1={nj + 1}, ni+1={ni + 1})." ) @@ -1540,6 +1778,7 @@ def _read_point3d_parametric_array( n_pillars_base: int, ni: int, nj: int, + pillar_indices_arr: Optional[np.ndarray] = None, ) -> np.ndarray: """ Evaluate a ``Point3dParametricArray`` and return a @@ -1593,7 +1832,7 @@ def _read_point3d_parametric_array( pad = np.full((nkl, n_pillars_total - n_pillars_base), np.nan, dtype=np.float64) query_params = np.concatenate([query_params, pad], axis=1) else: - logging.warning( + logger.warning( f"Point3dParametricArray.parameters size {raw_params.size} does not match " f"expected {expected_3d} (3-D) or {expected_4d} (4-D). Attempting flat reshape." ) @@ -1603,31 +1842,45 @@ def _read_point3d_parametric_array( # When present, each column index in query_params maps to a pillar index # in the ParametricLineArray (needed for grids with truncated or # non-contiguous pillar numbering). + # ``ParametricLineIndices`` maps *array index → parametric line index*. It is optional + # precisely because a column-layer grid already carries that mapping in + # ``ColumnLayerSplitCoordinateLines.PillarIndices``: coordinate line c < nPillars is + # pillar c, and split line nPillars+s reuses the line of pillar PillarIndices[s]. + # + # The previous code permuted the *query* columns (``query_params[:, raw_pli]``) instead + # of selecting lines, which reorders the grid nodes themselves, and it never derived the + # implicit mapping at all — so a faulted parametric grid asked the evaluator for more + # lines than the ParametricLineArray contains. + line_indices: Optional[np.ndarray] = None pli_obj = getattr(pts_obj, "parametric_line_indices", None) if pli_obj is not None: - logging.debug( - "Point3dParametricArray.parametric_line_indices is present. " - "This re-indexing is applied inside evaluate_parametric_line_array " - "via the column-selection mechanism of resolve_parametric_line_array." - ) - # The indices are handled by passing the re-ordered query_params. - # Build a column-permuted view so pillar p of query_params maps to - # pillar pli[p] of the ParametricLineArray. raw_pli = _read_array_np(pli_obj, energyml_object, "geometry.Points.parametric_line_indices", ws) - raw_pli = raw_pli.astype(np.int64).flatten() - # Reorder query_params columns to match the PLA pillar ordering. - # (Each position i in query_params[:,i] uses PLA pillar raw_pli[i].) - # We pass this as-is; evaluate_parametric_line_array iterates by - # query_params column index, which now aligns with pli-selected pillars. - # NOTE: If pli introduces a non-injective mapping (two query columns → - # same PLA pillar), the evaluation is repeated — this is correct per spec. - query_params_reordered = query_params[:, raw_pli] if len(raw_pli) > 0 else query_params - query_params = query_params_reordered + line_indices = raw_pli.astype(np.int64).flatten() + if len(line_indices) != n_pillars_total: + logger.warning( + f"Point3dParametricArray.parametric_line_indices holds {len(line_indices)} " + f"entries for {n_pillars_total} coordinate lines; ignoring it." + ) + line_indices = None + + if line_indices is None: + line_indices = np.arange(n_pillars_total, dtype=np.int64) + n_splits = n_pillars_total - n_pillars_base + if n_splits > 0: + if pillar_indices_arr is None: + logger.warning( + f"{n_splits} split coordinate line(s) but no " + "ColumnLayerSplitCoordinateLines.PillarIndices: their parametric lines " + "cannot be resolved." + ) + else: + pi = np.asarray(pillar_indices_arr, dtype=np.int64).flatten() + line_indices[n_pillars_base : n_pillars_base + len(pi)] = pi[:n_splits] # --- 3. Handle optional truncated_line_indices --- tli_obj = getattr(pts_obj, "truncated_line_indices", None) if tli_obj is not None: - logging.warning( + logger.warning( "Point3dParametricArray.truncated_line_indices is present. " "Full truncated-pillar support is not yet implemented — " "truncation metadata will be ignored and results may be geometrically " @@ -1638,7 +1891,7 @@ def _read_point3d_parametric_array( pla_raw = getattr(pts_obj, "parametric_lines", None) if pla_raw is None: raise ValueError("Point3dParametricArray.parametric_lines is required but absent.") - pla = resolve_parametric_line_array(pla_raw, energyml_object, ws, n_pillars_total) + pla = resolve_parametric_line_array(pla_raw, energyml_object, ws, n_pillars_base) # --- 5. Evaluate pillar splines --- pts_3d = evaluate_parametric_line_array( @@ -1648,6 +1901,7 @@ def _read_point3d_parametric_array( query_parameters=query_params, ni=ni, nj=nj, + line_indices=line_indices, ) # (NKL, n_pillars_total, 3) return pts_3d @@ -1676,13 +1930,22 @@ def read_numpy_ijk_grid_representation( unfaulted vectorised path when possible. * **Degenerate cells** — pillars with co-located nodes (e.g. wedge columns) are preserved; PyVista tolerates degenerate hex nodes. + * **Parametric pillars** — ``Point3dParametricArray`` is evaluated through + :func:`~energyml.utils.data.helper.evaluate_parametric_line_array` (all six RESQML line + kinds; kinds 2 and 4 need scipy). ``Point3dExternalArray`` — direct XYZ — is read as is. + * **Handedness** — ``GridIsRighthanded`` decides the corner winding so the emitted + hexahedra always have a positive Jacobian. + * **Undefined geometry** — ``PillarGeometryIsDefined`` blanks the nodes of the flagged + coordinate lines, ``CellGeometryIsDefined`` turns the flagged cells into VTK empty cells + (kept in place, so cell-indexed properties still line up). + + Cells are emitted in the RESQML order — I fastest, then J, then K — which is the order the + grid's cell-indexed properties use. Known limitation ---------------- - ``Point3DParametricArray`` pillar geometry is not yet supported (only - ``Point3DExternalArray`` — direct HDF5 XYZ coordinates — is handled). A - :exc:`~energyml.utils.exception.NotSupportedError` is raised for parametric - grids. + A grid whose geometry comes from a ``ParentWindow`` (LGR) instead of its own ``Geometry`` + is returned empty: the regridding of the parent's pillars is not implemented. """ ws = _view_workspace(workspace) src_uuid = get_obj_uuid(energyml_object) @@ -1692,7 +1955,7 @@ def read_numpy_ijk_grid_representation( nj = getattr(energyml_object, "nj", None) nk = getattr(energyml_object, "nk", None) if ni is None or nj is None or nk is None: - logging.warning("IjkGridRepresentation: ni/nj/nk not set — returning empty mesh") + logger.warning("IjkGridRepresentation: ni/nj/nk not set — returning empty mesh") return NumpyMultiMesh( energyml_object=energyml_object, identifier=str(src_uuid), @@ -1703,7 +1966,14 @@ def read_numpy_ijk_grid_representation( geom = getattr(energyml_object, "geometry", None) if geom is None: - logging.warning("IjkGridRepresentation has no geometry — returning empty mesh") + if getattr(energyml_object, "parent_window", None) is not None: + logger.warning( + f"IjkGridRepresentation {src_uuid} is a local grid refinement: its geometry is " + "inherited from the parent grid through ParentWindow, which is not implemented — " + "returning an empty mesh." + ) + else: + logger.warning("IjkGridRepresentation has no geometry — returning empty mesh") return NumpyMultiMesh( energyml_object=energyml_object, identifier=str(src_uuid), @@ -1768,7 +2038,7 @@ def read_numpy_ijk_grid_representation( # --- POINTS --- pts_results = search_attribute_matching_name_with_path(geom, "Points") if not pts_results: - logging.warning("IjkGridRepresentation: cannot find Points in geometry") + logger.warning("IjkGridRepresentation: cannot find Points in geometry") return empty pts_path, pts_obj = pts_results[0] @@ -1784,6 +2054,7 @@ def read_numpy_ijk_grid_representation( n_pillars_base=n_pillars_base, ni=ni, nj=nj, + pillar_indices_arr=pillar_indices_arr, ) else: pts_3d = _read_direct_points( @@ -1799,7 +2070,9 @@ def read_numpy_ijk_grid_representation( nj=nj, ) - points = pts_3d.reshape(-1, 3).astype(np.float64, copy=False) + # pts_3d may be a reshaped *view* of the workspace array (see _read_direct_points), and + # astype(copy=False) would keep it that way — the CRS transform below writes in place. + points = _ensure_float64_points(pts_3d.reshape(-1, 3)) # --- CRS --- crs = None @@ -1810,8 +2083,10 @@ def read_numpy_ijk_grid_representation( root_obj=energyml_object, workspace=workspace, ) - except (ObjectNotFoundNotError, Exception): - pass + except Exception as exc: + # `(ObjectNotFoundNotError, Exception)` was just `Exception` with misleading intent. + # get_crs_obj can fail in several ways and a missing CRS is not fatal here. + logger.debug(f"No CRS resolved: {type(exc).__name__}: {exc}") # --- PILLAR MAP for faulted grids --- use_pillar_map = n_splits > 0 and pillar_indices_arr is not None @@ -1819,68 +2094,97 @@ def read_numpy_ijk_grid_representation( if use_pillar_map: pillar_map = _build_split_pillar_map(ni, nj, pillar_indices_arr, columns_per_split, n_splits) + # --- PILLARS WITHOUT GEOMETRY --- + # "Indicator that a pillar has at least one node with a defined cell geometry. [...] If the + # indicator does not indicate that the pillar geometry is defined, then this over-rides any + # other node geometry specification." The flag is indexed by *pillar*, so a split coordinate + # line inherits the flag of the pillar it was split from. + _blank_undefined_pillars( + points=points, + geom=geom, + energyml_object=energyml_object, + ws=ws, + nkl=nkl, + n_pillars_base=n_pillars_base, + n_pillars_total=n_pillars_total, + pillar_indices_arr=pillar_indices_arr, + ) + # --- BUILD HEXAHEDRAL CELL CONNECTIVITY --- - if pillar_map is None: - # Fully vectorised path for unfaulted grids - ii_arr, ij_arr, ik_arr = np.meshgrid( - np.arange(ni, dtype=np.int64), - np.arange(nj, dtype=np.int64), - np.arange(nk, dtype=np.int64), - indexing="ij", - ) # each shape (ni, nj, nk) - - kl_b = kl_bottom[ik_arr] # (ni, nj, nk) - kl_t = kl_top[ik_arr] - p_tl = ij_arr * (ni + 1) + ii_arr # pillar TL - p_tr = ij_arr * (ni + 1) + (ii_arr + 1) # pillar TR - p_bl = (ij_arr + 1) * (ni + 1) + ii_arr # pillar BL - p_br = (ij_arr + 1) * (ni + 1) + (ii_arr + 1) # pillar BR - - def _nidx(kl, pl): - return kl * n_pillars_total + pl - - # VTK_HEXAHEDRON node ordering (bottom face ccw, top face aligned) - n0 = _nidx(kl_b, p_tl).ravel() - n1 = _nidx(kl_b, p_tr).ravel() - n2 = _nidx(kl_b, p_br).ravel() - n3 = _nidx(kl_b, p_bl).ravel() - n4 = _nidx(kl_t, p_tl).ravel() - n5 = _nidx(kl_t, p_tr).ravel() - n6 = _nidx(kl_t, p_br).ravel() - n7 = _nidx(kl_t, p_bl).ravel() - - n_cells = ni * nj * nk - count_col = np.full(n_cells, 8, dtype=np.int64) - cells = np.column_stack([count_col, n0, n1, n2, n3, n4, n5, n6, n7]).ravel() - cell_types = np.full(n_cells, _VTK_HEXAHEDRON, dtype=np.uint8) + # Cell ordering is the RESQML one — I fastest, then J, then K — which is what every + # cell-indexed array of the grid uses (properties, CellGeometryIsDefined, ...). The node + # arrays observed in the files make the convention explicit: PillarGeometryIsDefined is + # stored as (NJ+1, NI+1) and the point parameters as (NKL, NJ+1, NI+1). + # + # Neither previous path produced that order, and the two disagreed with each other: the + # unfaulted branch enumerated K fastest then J then I, the faulted branch K fastest then I + # then J. Any property read back onto the cells was therefore permuted, differently + # depending on whether the grid happened to be faulted. + ik_arr, ij_arr, ii_arr = np.meshgrid( + np.arange(nk, dtype=np.int64), + np.arange(nj, dtype=np.int64), + np.arange(ni, dtype=np.int64), + indexing="ij", + ) # each shape (nk, nj, ni) → ravel() in C order gives I fastest, K slowest + if pillar_map is None: + p_tl = ij_arr * (ni + 1) + ii_arr + p_tr = ij_arr * (ni + 1) + (ii_arr + 1) + p_bl = (ij_arr + 1) * (ni + 1) + ii_arr + p_br = (ij_arr + 1) * (ni + 1) + (ii_arr + 1) + else: + # The faulted case is a gather on the pre-built (nj, ni, 4) map — no Python loop needed. + p_tl = pillar_map[ij_arr, ii_arr, 0] + p_tr = pillar_map[ij_arr, ii_arr, 1] + p_bl = pillar_map[ij_arr, ii_arr, 2] + p_br = pillar_map[ij_arr, ii_arr, 3] + + kl_b = kl_bottom[ik_arr] + kl_t = kl_top[ik_arr] + + # VTK requires the first four nodes to wind so that the right-hand-rule normal points at the + # opposite face; otherwise the hexahedron has a negative Jacobian and its faces are inverted. + # (I, J, K) is that orientation only when the grid is right-handed — which is exactly what + # GridIsRighthanded reports, and the flag was ignored. rc/epc/80wells_surf_modified_val_color.epc + # ships the pair "Four by Three by Two Left Handed" / "... Right Handed" for this: every cell + # of the left-handed one came out inside-out. + # + # The flag describes the grid in the real-world sense, i.e. once the CRS has been applied. + # These fixtures measure Z as a depth, so (X, Y, Z) is left-handed in the *local* frame and a + # right-handed grid still has a negative Jacobian there; it comes out positive after the Z + # flip of apply_from_crs_info. Orientation is therefore correct in the PROJECTED frame, which + # is the default and the one a viewer renders. + righthanded = getattr(geom, "grid_is_righthanded", None) + if righthanded is None: + logger.debug("IjkGridRepresentation: GridIsRighthanded absent, assuming right-handed.") + righthanded = True + base_corners = (p_tl, p_tr, p_br, p_bl) if righthanded else (p_tl, p_bl, p_br, p_tr) + + n_cells = ni * nj * nk + node_cols = [(kl_b * n_pillars_total + p).ravel() for p in base_corners] + node_cols += [(kl_t * n_pillars_total + p).ravel() for p in base_corners] + rows = np.column_stack([np.full(n_cells, 8, dtype=np.int64), *node_cols]) # (n_cells, 9) + + cell_types = np.full(n_cells, _VTK_HEXAHEDRON, dtype=np.uint8) + + # --- CELLS WITHOUT GEOMETRY --- + undefined = _read_cell_geometry_undefined(geom, energyml_object, ws, ni, nj, nk) + if undefined is not None and undefined.any(): + logger.info( + f"IjkGridRepresentation: {int(undefined.sum())}/{n_cells} cells flagged " + "CellGeometryIsDefined=false; emitted as empty cells." + ) + cell_types[undefined] = _VTK_EMPTY_CELL + # An empty cell carries no node, so its row shrinks to the lone count prefix. Keeping the + # cell *present* is what preserves the 1:1 match with the grid's cell-indexed properties. + rows[undefined, 0] = 0 + keep = np.ones((n_cells, 9), dtype=bool) + keep[undefined, 1:] = False + cells = rows[keep] else: - # Per-column loop for faulted grids (pillar_map resolved) - cells_parts: List[int] = [] - for ij_idx in range(nj): - for ii_idx in range(ni): - p_tl = int(pillar_map[ij_idx, ii_idx, 0]) - p_tr = int(pillar_map[ij_idx, ii_idx, 1]) - p_bl = int(pillar_map[ij_idx, ii_idx, 2]) - p_br = int(pillar_map[ij_idx, ii_idx, 3]) - for ik_idx in range(nk): - kl_b = int(kl_bottom[ik_idx]) - kl_t = int(kl_top[ik_idx]) - n0 = kl_b * n_pillars_total + p_tl - n1 = kl_b * n_pillars_total + p_tr - n2 = kl_b * n_pillars_total + p_br - n3 = kl_b * n_pillars_total + p_bl - n4 = kl_t * n_pillars_total + p_tl - n5 = kl_t * n_pillars_total + p_tr - n6 = kl_t * n_pillars_total + p_br - n7 = kl_t * n_pillars_total + p_bl - cells_parts.extend([8, n0, n1, n2, n3, n4, n5, n6, n7]) - cells = np.array(cells_parts, dtype=np.int64) - n_cells = ni * nj * nk - cell_types = np.full(n_cells, _VTK_HEXAHEDRON, dtype=np.uint8) - - if use_crs_displacement and crs is not None and len(points) > 0: - apply_from_crs_info(points, extract_crs_info(crs, workspace), inplace=True) + cells = rows.ravel() + + frame = _local_to_projected(points, crs, workspace, use_crs_displacement) label = f"{src_type}_patch_0" multi = NumpyMultiMesh( @@ -1897,6 +2201,7 @@ def _nidx(kl, pl): points=points, cells=cells, cell_types=cell_types, + frame=frame, patch_index=0, patch_label=label, source_uuid=src_uuid, @@ -1930,7 +2235,7 @@ def read_numpy_unstructured_grid_representation( geom = getattr(energyml_object, "geometry", None) if geom is None: - logging.warning("UnstructuredGridRepresentation has no geometry — returning empty mesh") + logger.warning("UnstructuredGridRepresentation has no geometry — returning empty mesh") return NumpyMultiMesh( energyml_object=energyml_object, identifier=str(src_uuid), @@ -1952,7 +2257,7 @@ def read_numpy_unstructured_grid_representation( # --- POINTS --- pts_results = search_attribute_matching_name_with_path(geom, "Points") if not pts_results: - logging.warning("UnstructuredGridRepresentation: cannot find Points in geometry") + logger.warning("UnstructuredGridRepresentation: cannot find Points in geometry") return empty pts_path, pts_obj = pts_results[0] raw_pts = _read_array_np(pts_obj, energyml_object, pts_path, ws) @@ -1967,14 +2272,16 @@ def read_numpy_unstructured_grid_representation( root_obj=energyml_object, workspace=workspace, ) - except (ObjectNotFoundNotError, Exception): - pass + except Exception as exc: + # `(ObjectNotFoundNotError, Exception)` was just `Exception` with misleading intent. + # get_crs_obj can fail in several ways and a missing CRS is not fatal here. + logger.debug(f"No CRS resolved: {type(exc).__name__}: {exc}") # --- JAGGED ARRAYS --- npf_obj = getattr(geom, "nodes_per_face", None) fpc_obj = getattr(geom, "faces_per_cell", None) if npf_obj is None or fpc_obj is None: - logging.warning( + logger.warning( "UnstructuredGridRepresentation: missing nodes_per_face or faces_per_cell " "— returning point-set mesh" ) label = f"{src_type}_patch_0" @@ -2009,8 +2316,8 @@ def read_numpy_unstructured_grid_representation( try: rh_path, rh_obj = search_attribute_matching_name_with_path(geom, "CellFaceIsRightHanded")[0] rh_arr = _read_array_np(rh_obj, energyml_object, f"geometry.{rh_path}", ws).astype(bool) - except (IndexError, Exception) as exc: - logging.debug(f"UnstructuredGridRepresentation: CellFaceIsRightHanded not readable: {exc}") + except Exception as exc: # IndexError from [0] on an empty match, or any read failure + logger.debug(f"UnstructuredGridRepresentation: CellFaceIsRightHanded not readable: {exc}") # --- BUILD VTK_POLYHEDRON CELL ARRAY --- # VTK polyhedron flat format per cell: @@ -2039,8 +2346,7 @@ def read_numpy_unstructured_grid_representation( cells = np.array(cells_flat, dtype=np.int64) cell_types = np.full(cell_count, _VTK_POLYHEDRON, dtype=np.uint8) - if use_crs_displacement and crs is not None and len(points) > 0: - apply_from_crs_info(points, extract_crs_info(crs, workspace), inplace=True) + frame = _local_to_projected(points, crs, workspace, use_crs_displacement) label = f"{src_type}_patch_0" multi = NumpyMultiMesh( @@ -2057,6 +2363,7 @@ def read_numpy_unstructured_grid_representation( points=points, cells=cells, cell_types=cell_types, + frame=frame, patch_index=0, patch_label=label, source_uuid=src_uuid, @@ -2067,89 +2374,1348 @@ def read_numpy_unstructured_grid_representation( # --------------------------------------------------------------------------- -# Main dispatcher +# Delegating readers +# +# These representations add semantics on top of a geometry another reader already produces. +# Dispatch is by function name, so each needs its own entry point even when the body is a +# single call — that is the registration. # --------------------------------------------------------------------------- -def read_numpy_mesh_object( +def read_numpy_wellbore_marker_frame_representation( energyml_object: Any, workspace: Optional[EnergymlStorageInterface] = None, use_crs_displacement: bool = True, sub_indices: Optional[Union[List[int], np.ndarray]] = None, ) -> "NumpyMultiMesh": - """Dispatcher — equivalent to :func:`mesh.read_mesh_object` but returns - a :class:`NumpyMultiMesh` container. + """Read a ``WellboreMarkerFrameRepresentation`` — the markers positioned on their trajectory. - Args: - energyml_object: Any supported RESQML/EnergyML geometry/representation object. - workspace: Storage interface (``Epc`` or ``EpcStreamReader``). - use_crs_displacement: When ``True`` (default), applies - :func:`crs_displacement_np` to the points of every - returned mesh (excluding wellbore representations - which apply the transform internally). - sub_indices: Optional list of face/line/point indices to include. + "A well log frame where each entry represents a well marker": the geometry is a + ``NodeMd`` list plus a ``Trajectory`` reference, exactly like + :class:`WellboreFrameRepresentation`, so the frame reader handles it as is. The points of + the returned polyline are the marker positions, in ``NodeMd`` order — index *i* is the + position of ``wellbore_marker[i]``. + """ + return read_numpy_wellbore_frame_representation( + energyml_object=energyml_object, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + ) - Returns: - :class:`NumpyMultiMesh` containing one or more :class:`NumpyMesh` patches - (and/or nested children for ``RepresentationSetRepresentation``). - Raises: - :exc:`energyml.utils.exception.NotSupportedError`: if the object type - has no registered reader. +def read_numpy_blocked_wellbore_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``BlockedWellboreRepresentation`` as its trajectory sampled at the node MDs. + + A blocked wellbore is a ``WellboreFrameRepresentation`` whose intervals are annotated with + the grid cells they cross (``IntervalGridCells``). The added information is topological, not + geometric: the geometry is still ``NodeMd`` along ``Trajectory``. """ - if isinstance(energyml_object, list): - # Synthetic container aggregating multiple top-level objects. - synthetic = NumpyMultiMesh(identifier="multi_object_list") - for obj in energyml_object: - synthetic.children.append( - read_numpy_mesh_object( - energyml_object=obj, - workspace=workspace, - use_crs_displacement=use_crs_displacement, - sub_indices=sub_indices, - ) - ) - return synthetic + return read_numpy_wellbore_frame_representation( + energyml_object=energyml_object, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + ) - type_name = _numpy_mesh_name_mapping(type(energyml_object).__name__) - reader_func = get_numpy_reader_function(type_name) - if reader_func is None: - from energyml.utils.exception import NotSupportedError as _NSE +def read_numpy_non_sealed_surface_framework_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``NonSealedSurfaceFrameworkRepresentation`` — its member representations. - raise _NSE( - f"No numpy mesh reader found for type '{type_name}'. " - f"Expected function 'read_numpy_{snake_case(type_name)}' in {__name__}." - ) + Like its sealed counterpart it is a ``RepresentationSetRepresentation`` subtype; the + ``contacts`` it adds describe how the surfaces meet and carry no geometry of their own. + """ + result = read_numpy_representation_set_representation( + energyml_object=energyml_object, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + ) + result.source_type = type(energyml_object).__name__ + return result - result: NumpyMultiMesh = reader_func( + +def read_numpy_sealed_volume_framework_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``SealedVolumeFrameworkRepresentation`` — the surfaces bounding its regions. + + The object is a BREP: its ``regions`` assemble shells out of the surfaces of a sealed + surface framework. Only the member representations are returned, i.e. the bounding + surfaces; the region-to-shell assembly is not turned into closed volumes. + """ + result = read_numpy_representation_set_representation( energyml_object=energyml_object, workspace=workspace, + use_crs_displacement=use_crs_displacement, sub_indices=sub_indices, + ) + result.source_type = type(energyml_object).__name__ + if getattr(energyml_object, "regions", None): + logger.debug( + "SealedVolumeFrameworkRepresentation: returning the bounding surfaces only; " + "the volume regions are not assembled into closed shells." + ) + return result + + +def read_numpy_grid2d_set_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``Grid2dSetRepresentation`` (RESQML 2.0.1) — one patch per member 2-D grid. + + "Set of representations based on a 2D grid. Each 2D grid representation corresponds to one + patch of the set." :func:`read_numpy_grid2d_representation` already loops over every + ``Grid2dPatch`` it finds, which is exactly the set's content. + """ + result = read_numpy_grid2d_representation( + energyml_object=energyml_object, + workspace=workspace, use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, ) + result.source_type = type(energyml_object).__name__ + return result - # Apply fallback CRS displacement for readers that do NOT handle it - # internally (e.g. Grid2d which has no per-patch CRS apply call yet). - _tn = type_name.lower() - if ( - use_crs_displacement - and "wellbore" not in _tn - and "triangulated" not in _tn # per-patch CRS applied inside reader - and "point" not in _tn # per-patch CRS applied inside reader - and "polyline" not in _tn # per-patch CRS applied inside reader - and "representationset" not in _tn # each child already had CRS applied - and "subrepresentation" not in _tn # delegates entirely to inner call - and "planeset" not in _tn # per-patch CRS applied inside reader - and "seismicwellbore" not in _tn # delegates to wellbore reader - and "sealedsurface" not in _tn # delegates to representation-set reader - and "unstructuredgrid" not in _tn # per-patch CRS applied inside reader - and "ijkgrid" not in _tn # per-patch CRS applied inside reader - ): - for m in result.flat_patches(): - crs = m.crs_object[0] if isinstance(m.crs_object, list) and m.crs_object else m.crs_object - if crs is not None and len(m.points) > 0: - crs_displacement_np(m.points, crs, inplace=True) + +def read_numpy_unstructured_column_layer_grid_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read an ``UnstructuredColumnLayerGridRepresentation`` as ``VTK_POLYHEDRON`` cells. + + "Grid whose topology is characterized by an unstructured column index and a layer index, K. + Cell geometry is characterized by nodes on coordinate lines, where each column of the model + may have an arbitrary number of sides." + + It is the IJK reader with the implicit ``(NI+1)(NJ+1)`` pillar lattice replaced by an + explicit ``PillarsPerColumn`` list of lists, so everything else carries over: coordinate + line nodes with NKL nodes per line, K-gaps, split coordinate lines, and the + ``PillarGeometryIsDefined`` / ``CellGeometryIsDefined`` overrides. + + Each cell is emitted as a polyhedron — bottom face, top face and one quad per column edge — + rather than a shape-specific VTK type, because ``ColumnShape`` may be ``polygonal``. Cells + are ordered column fastest, then layer, which is the grid's own cell ordering. + """ + ws = _view_workspace(workspace) + src_uuid = get_obj_uuid(energyml_object) + src_type = type(energyml_object).__name__ + try: + identifier = str(get_obj_uri(energyml_object)) + except Exception: + identifier = str(src_uuid) + multi = NumpyMultiMesh( + energyml_object=energyml_object, + identifier=identifier, + source_uuid=src_uuid, + source_type=src_type, + ) + + nk = getattr(energyml_object, "nk", None) + column_count = getattr(energyml_object, "column_count", None) + geom = getattr(energyml_object, "geometry", None) + if nk is None or column_count is None or geom is None: + if geom is None and getattr(energyml_object, "parent_window", None) is not None: + logger.warning( + f"{src_type} {src_uuid} is a local grid refinement: its geometry is inherited " + "through ParentWindow, which is not implemented — returning an empty mesh." + ) + else: + logger.warning(f"{src_type} {src_uuid}: nk / column_count / geometry missing — returning empty mesh.") + return multi + nk, column_count = int(nk), int(column_count) + + pillar_count = int(getattr(geom, "pillar_count", 0) or 0) + ppc_obj = getattr(geom, "pillars_per_column", None) + if ppc_obj is None: + logger.warning(f"{src_type} {src_uuid}: PillarsPerColumn is required but absent.") + return multi + pillars_per_column = _decode_jagged_array(ppc_obj, energyml_object, "geometry.pillars_per_column", ws) + if len(pillars_per_column) < column_count: + logger.warning( + f"{src_type} {src_uuid}: PillarsPerColumn describes {len(pillars_per_column)} columns " + f"for ColumnCount={column_count}." + ) + column_count = len(pillars_per_column) + + # --- K-GAPS (identical to the IJK case) --- + kgaps_obj = getattr(energyml_object, "kgaps", None) + gap_after: Optional[np.ndarray] = None + n_kgaps = 0 + if kgaps_obj is not None: + n_kgaps = int(getattr(kgaps_obj, "count", 0) or 0) + gap_attr_list = search_attribute_matching_name_with_path(kgaps_obj, "GapAfterLayer") + if gap_attr_list: + gap_path, gap_obj = gap_attr_list[0] + if gap_obj is not None: + gap_after = _read_array_np(gap_obj, energyml_object, f"kgaps.{gap_path}", ws).astype(bool) + nkl = nk + n_kgaps + 1 + kl_bottom, kl_top = _build_kl_mapping(nk, gap_after) + + # --- SPLIT COORDINATE LINES --- + split_cl = getattr(geom, "column_layer_split_coordinate_lines", None) + n_splits = 0 + pillar_indices_arr: Optional[np.ndarray] = None + columns_per_split: List[np.ndarray] = [] + if split_cl is not None: + n_splits = int(getattr(split_cl, "count", 0) or 0) + if n_splits > 0: + pi_list = [(p, o) for p, o in search_attribute_matching_name_with_path(split_cl, "PillarIndices") if o] + if pi_list: + pi_path, pi_obj = pi_list[0] + pillar_indices_arr = _read_array_np( + pi_obj, energyml_object, f"geometry.column_layer_split_coordinate_lines.{pi_path}", ws + ) + cps_obj = getattr(split_cl, "columns_per_split_coordinate_line", None) + if cps_obj is not None: + columns_per_split = _decode_jagged_array( + cps_obj, + energyml_object, + "geometry.column_layer_split_coordinate_lines.columns_per_split_coordinate_line", + ws, + ) + + n_lines = pillar_count + n_splits + + # --- POINTS --- + pts_results = [(p, o) for p, o in search_attribute_matching_name_with_path(geom, "Points") if o is not None] + if not pts_results: + logger.warning(f"{src_type} {src_uuid}: cannot find Points in geometry.") + return multi + pts_path, pts_obj = pts_results[0] + raw_pts = _read_array_np(pts_obj, energyml_object, f"geometry.{pts_path}", ws) + if raw_pts.size != nkl * n_lines * 3: + logger.warning( + f"{src_type} {src_uuid}: points array holds {raw_pts.size} values, expected " + f"NKL({nkl}) × lines({n_lines}) × 3 = {nkl * n_lines * 3}." + ) + return multi + points = _ensure_float64_points(raw_pts.reshape(-1, 3)) + + _blank_undefined_pillars( + points=points, + geom=geom, + energyml_object=energyml_object, + ws=ws, + nkl=nkl, + n_pillars_base=pillar_count, + n_pillars_total=n_lines, + pillar_indices_arr=pillar_indices_arr, + ) + + crs = None + try: + crs = get_crs_obj(context_obj=geom, path_in_root="geometry", root_obj=energyml_object, workspace=workspace) + except Exception as exc: + logger.debug(f"No CRS resolved: {type(exc).__name__}: {exc}") + + # --- Corner coordinate line of every column, split lines substituted in --- + corner_lines: List[np.ndarray] = [np.asarray(pillars_per_column[c], dtype=np.int64) for c in range(column_count)] + if n_splits > 0 and pillar_indices_arr is not None: + pi = np.asarray(pillar_indices_arr, dtype=np.int64).ravel() + for s in range(min(n_splits, len(pi), len(columns_per_split))): + replaced, new_line = int(pi[s]), pillar_count + s + for col in np.asarray(columns_per_split[s], dtype=np.int64).ravel(): + col = int(col) + if 0 <= col < column_count: + corner_lines[col] = np.where(corner_lines[col] == replaced, new_line, corner_lines[col]) + + right_handed: Optional[np.ndarray] = None + rh_obj = getattr(geom, "column_is_right_handed", None) + if rh_obj is not None: + try: + right_handed = ( + _read_array_np(rh_obj, energyml_object, "geometry.column_is_right_handed", ws).astype(bool).ravel() + ) + except Exception as exc: + logger.debug(f"Cannot read ColumnIsRightHanded: {type(exc).__name__}: {exc}") + + undefined = _read_cell_geometry_undefined(geom, energyml_object, ws, column_count, 1, nk) + + # --- Cells: column fastest, then layer --- + cells_flat: List[int] = [] + cell_types: List[int] = [] + for k in range(nk): + kb, kt = int(kl_bottom[k]) * n_lines, int(kl_top[k]) * n_lines + for col in range(column_count): + cell_idx = k * column_count + col + lines_of_col = corner_lines[col] + n_side = len(lines_of_col) + if (undefined is not None and cell_idx < len(undefined) and undefined[cell_idx]) or n_side < 3: + cells_flat.append(0) + cell_types.append(_VTK_EMPTY_CELL) + continue + bottom = [kb + int(p) for p in lines_of_col] + top = [kt + int(p) for p in lines_of_col] + # The bottom face is wound the other way round so both K faces point out of the cell. + faces: List[List[int]] = [list(reversed(bottom)), list(top)] + for i in range(n_side): + j = (i + 1) % n_side + faces.append([bottom[i], bottom[j], top[j], top[i]]) + # "List of columns that are right handed" — the flag is per column, not per cell. + if right_handed is not None and col < len(right_handed) and not right_handed[col]: + faces = [list(reversed(f)) for f in faces] + body: List[int] = [len(faces)] + for f in faces: + body.append(len(f)) + body.extend(f) + cells_flat.append(len(body)) + cells_flat.extend(body) + cell_types.append(_VTK_POLYHEDRON) + + frame = _local_to_projected(points, crs, workspace, use_crs_displacement) + label = f"{src_type}_patch_0" + multi.patches.append( + NumpyVolumeMesh( + identifier=label, + energyml_object=energyml_object, + crs_object=crs, + points=points, + cells=np.array(cells_flat, dtype=np.int64), + cell_types=np.array(cell_types, dtype=np.uint8), + frame=frame, + patch_index=0, + patch_label=label, + source_uuid=src_uuid, + source_type=src_type, + ) + ) + return multi + + +def read_numpy_truncated_unstructured_column_layer_grid_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``TruncatedUnstructuredColumnLayerGridRepresentation`` — its untruncated geometry. + + Same relation as ``TruncatedIjkGridRepresentation`` to ``IjkGridRepresentation``: the base + ``UnstructuredColumnLayerGridGeometry`` is read in full, the ``TruncationCellPatch`` is not + applied. + """ + if getattr(energyml_object, "truncation_cell_patch", None) is not None: + logger.warning( + f"{type(energyml_object).__name__} {get_obj_uuid(energyml_object)}: the TruncationCellPatch " + "is not applied — the truncated cells are returned in their untruncated form." + ) + result = read_numpy_unstructured_column_layer_grid_representation( + energyml_object=energyml_object, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + ) + result.source_type = type(energyml_object).__name__ + return result + + +def read_numpy_truncated_ijk_grid_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``TruncatedIjkGridRepresentation`` — its untruncated IJK geometry. + + The type is "a grid class with an underlying IJK topology, together with a 1D split-cell + list", and it carries the same ``ni``/``nj``/``nk`` and ``IjkGridGeometry`` as a plain IJK + grid. That base geometry is read here in full. + + ``TruncationCellPatch`` is **not** applied: it replaces some hexahedra with arbitrary + polyhedra ("the truncated IJK cells have more than the usual 6 faces"). The truncated cells + are therefore returned in their untruncated form, which is reported once per object. + """ + if getattr(energyml_object, "truncation_cell_patch", None) is not None: + logger.warning( + f"TruncatedIjkGridRepresentation {get_obj_uuid(energyml_object)}: the TruncationCellPatch " + "is not applied — the truncated cells are returned as full hexahedra." + ) + result = read_numpy_ijk_grid_representation( + energyml_object=energyml_object, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + ) + result.source_type = type(energyml_object).__name__ + return result + + +def _read_numpy_via_supporting_representation( + energyml_object: Any, + attribute: str, + workspace: Optional[EnergymlStorageInterface], + use_crs_displacement: bool, + sub_indices: Optional[Union[List[int], np.ndarray]], +) -> "NumpyMultiMesh": + """Read the representation referenced by *attribute* and re-stamp it as *energyml_object*. + + Used by the representations that hold no geometry at all and simply point at the one that + does. The patches must report the referencing object, not the referenced one, so that a + caller can tell them apart — the same rule the wellbore-frame reader follows. + """ + src_uuid = get_obj_uuid(energyml_object) + src_type = type(energyml_object).__name__ + empty = NumpyMultiMesh( + energyml_object=energyml_object, + identifier=str(get_obj_uri(energyml_object)), + source_uuid=src_uuid, + source_type=src_type, + ) + + dor = getattr(energyml_object, attribute, None) + if dor is None: + found = search_attribute_matching_name(obj=energyml_object, name_rgx=attribute) + dor = found[0] if found else None + if dor is None or workspace is None: + logger.warning(f"{src_type} {src_uuid}: no '{attribute}' to take the geometry from.") + return empty + + target = workspace.get_object(get_obj_uri(dor)) + if target is None: + logger.warning(f"{src_type} {src_uuid}: {get_obj_uri(dor)} not found in the workspace.") + return empty + + result = read_numpy_mesh_object( + energyml_object=target, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + ) + uri = str(get_obj_uri(energyml_object)) + for m in result.flat_patches(): + m.identifier = uri + m.energyml_object = energyml_object + m.source_uuid = src_uuid + m.source_type = src_type + result.identifier = uri + result.energyml_object = energyml_object + result.source_uuid = src_uuid + result.source_type = src_type + return result + + +def read_numpy_seismic3d_post_stack_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``Seismic3dPostStackRepresentation`` — the 2-D lattice it is defined on. + + The object holds no geometry: it references the ``SeismicLatticeRepresentation`` + (a ``Grid2dRepresentation``) whose feature it shares, and adds the trace sampling. The + lattice surface is returned; the trace samples themselves are properties, not geometry. + """ + return _read_numpy_via_supporting_representation( + energyml_object, "seismic_lattice_representation", workspace, use_crs_displacement, sub_indices + ) + + +def read_numpy_seismic2d_post_stack_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``Seismic2dPostStackRepresentation`` — the seismic line it is defined on. + + As for its 3-D counterpart the geometry is entirely in the referenced + ``SeismicLineRepresentation`` (a ``PolylineRepresentation``). + """ + return _read_numpy_via_supporting_representation( + energyml_object, "seismic_line_representation", workspace, use_crs_displacement, sub_indices + ) + + +def read_numpy_redefined_geometry_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``RedefinedGeometryRepresentation`` — the supporting representation with the + redefined points substituted in. + + "A representation derived from an existing representation by redefining its geometry. + Example use cases include deformation of the geometry of an object, change of coordinate + system, and change of time <=> depth." Topology comes from ``SupportingRepresentation``; + each ``PatchOfGeometry`` overrides the points of one of its patches. + + A patch whose point count does not match the one it redefines is skipped with a warning + rather than silently corrupting the connectivity. + """ + ws = _view_workspace(workspace) + src_uuid = get_obj_uuid(energyml_object) + src_type = type(energyml_object).__name__ + + result = _read_numpy_via_supporting_representation( + energyml_object, "supporting_representation", workspace, use_crs_displacement, sub_indices + ) + patches = result.flat_patches() + if not patches: + return result + + pog_list = getattr(energyml_object, "patch_of_geometry", None) or [] + for pog in pog_list: + target_idx = getattr(pog, "representation_patch_index", None) + target_idx = 0 if target_idx is None else int(target_idx) + if target_idx >= len(patches): + logger.warning(f"{src_type} {src_uuid}: PatchOfGeometry targets patch {target_idx}, which does not exist.") + continue + pts_list = [(p, o) for p, o in search_attribute_matching_name_with_path(pog, "Points") if o is not None] + if not pts_list: + continue + pts_path, pts_obj = pts_list[0] + try: + new_pts = _ensure_float64_points( + _read_array_np(pts_obj, energyml_object, f"patch_of_geometry.{pts_path}", ws) + ) + except Exception as exc: + logger.warning(f"{src_type} {src_uuid}: cannot read the redefined points: {type(exc).__name__}: {exc}") + continue + patch = patches[target_idx] + if len(new_pts) != len(patch.points): + logger.warning( + f"{src_type} {src_uuid}: PatchOfGeometry {target_idx} holds {len(new_pts)} points but the " + f"supporting patch has {len(patch.points)}; keeping the original geometry." + ) + continue + # The redefined points are expressed in this object's own CRS, i.e. back at the LOCAL + # stage, so the frame has to be reset for read_numpy_mesh_object to transform them. + patch.points = new_pts + patch.frame = PointFrame.LOCAL + return result + + +# --------------------------------------------------------------------------- +# Streamlines, graphs and deviation surveys +# --------------------------------------------------------------------------- + + +def _build_vtk_lines_from_counts( + node_counts: Optional[np.ndarray], + n_points: int, + closed: Optional[np.ndarray] = None, +) -> np.ndarray: + """Build a VTK flat line array from per-polyline node counts.""" + if node_counts is None or len(node_counts) == 0: + return _build_vtk_single_polyline(n_points) + parts: List[np.ndarray] = [] + offset = 0 + for poly_idx, raw_n in enumerate(node_counts): + n = int(raw_n) + if n <= 0: + continue + indices = np.arange(offset, offset + n, dtype=np.int64) + if closed is not None and poly_idx < len(closed) and closed[poly_idx]: + indices = np.append(indices, offset) + part = np.empty(len(indices) + 1, dtype=np.int64) + part[0] = len(indices) + part[1:] = indices + parts.append(part) + offset += n + return np.concatenate(parts) if parts else np.empty(0, dtype=np.int64) + + +def read_numpy_streamlines_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``StreamlinesRepresentation`` as one polyline per streamline. + + ``geometry`` is a single ``PolylineSetPatch``: all the streamline nodes concatenated, split + by ``NodeCountPerPolyline``. ``LineCount`` states how many streamlines to expect. + """ + ws = _view_workspace(workspace) + src_uuid = get_obj_uuid(energyml_object) + src_type = type(energyml_object).__name__ + multi = NumpyMultiMesh( + energyml_object=energyml_object, + identifier=str(get_obj_uri(energyml_object)), + source_uuid=src_uuid, + source_type=src_type, + ) + + geom = getattr(energyml_object, "geometry", None) + if geom is None: + logger.warning(f"StreamlinesRepresentation {src_uuid} has no geometry.") + return multi + + pts_list = search_attribute_matching_name_with_path(geom, "Points") + if not pts_list: + logger.warning(f"StreamlinesRepresentation {src_uuid}: no points in geometry.") + return multi + pts_path, pts_obj = pts_list[0] + points = _ensure_float64_points(_read_array_np(pts_obj, energyml_object, f"geometry.{pts_path}", ws)) + + node_counts = None + nc_list = [(p, o) for p, o in search_attribute_matching_name_with_path(geom, "NodeCountPerPolyline") if o] + if nc_list: + nc_path, nc_obj = nc_list[0] + node_counts = _read_array_np(nc_obj, energyml_object, f"geometry.{nc_path}", ws).astype(np.int64).ravel() + + line_count = int(getattr(energyml_object, "line_count", 0) or 0) + if node_counts is not None and line_count and len(node_counts) != line_count: + logger.warning( + f"StreamlinesRepresentation {src_uuid}: NodeCountPerPolyline holds " + f"{len(node_counts)} entries for LineCount={line_count}." + ) + + lines = _build_vtk_lines_from_counts(node_counts, len(points)) + crs = None + try: + crs = get_crs_obj(context_obj=geom, path_in_root="geometry", root_obj=energyml_object, workspace=workspace) + except Exception as exc: + logger.debug(f"No CRS resolved: {type(exc).__name__}: {exc}") + frame = _local_to_projected(points, crs, workspace, use_crs_displacement) + + label = f"{src_type}_patch_0" + multi.patches.append( + NumpyPolylineMesh( + identifier=label, + energyml_object=energyml_object, + crs_object=crs, + points=points, + lines=lines, + frame=frame, + patch_index=0, + patch_label=label, + source_uuid=src_uuid, + source_type=src_type, + ) + ) + return multi + + +def read_numpy_graph2d_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``Graph2dRepresentation`` — its nodes joined by its edges. + + ``edges`` is a ``2 x #Edges`` array of node indices; each edge becomes a two-point VTK line. + A graph with no edges comes back as a point set. + """ + ws = _view_workspace(workspace) + src_uuid = get_obj_uuid(energyml_object) + src_type = type(energyml_object).__name__ + multi = NumpyMultiMesh( + energyml_object=energyml_object, + identifier=str(get_obj_uri(energyml_object)), + source_uuid=src_uuid, + source_type=src_type, + ) + + geom = getattr(energyml_object, "geometry", None) + pts_list = search_attribute_matching_name_with_path(geom, "Points") if geom is not None else [] + if not pts_list: + logger.warning(f"Graph2dRepresentation {src_uuid} has no geometry.") + return multi + pts_path, pts_obj = pts_list[0] + points = _ensure_float64_points(_read_array_np(pts_obj, energyml_object, f"geometry.{pts_path}", ws)) + + edges_obj = getattr(energyml_object, "edges", None) + edges: Optional[np.ndarray] = None + if edges_obj is not None: + try: + raw = _read_array_np(edges_obj, energyml_object, "edges", ws).astype(np.int64).ravel() + if raw.size % 2 == 0: + edges = raw.reshape(-1, 2) + else: + logger.warning(f"Graph2dRepresentation {src_uuid}: Edges holds an odd number of values.") + except Exception as exc: + logger.warning(f"Graph2dRepresentation {src_uuid}: cannot read Edges: {type(exc).__name__}: {exc}") + + crs = None + try: + crs = get_crs_obj(context_obj=geom, path_in_root="geometry", root_obj=energyml_object, workspace=workspace) + except Exception as exc: + logger.debug(f"No CRS resolved: {type(exc).__name__}: {exc}") + frame = _local_to_projected(points, crs, workspace, use_crs_displacement) + + label = f"{src_type}_patch_0" + common = dict( + identifier=label, + energyml_object=energyml_object, + crs_object=crs, + points=points, + frame=frame, + patch_index=0, + patch_label=label, + source_uuid=src_uuid, + source_type=src_type, + ) + if edges is not None and len(edges) > 0: + valid = (edges >= 0).all(axis=1) & (edges < len(points)).all(axis=1) + if not valid.all(): + logger.warning( + f"Graph2dRepresentation {src_uuid}: {int((~valid).sum())} edge(s) reference a " + f"node outside [0, {len(points)}); dropped." + ) + edges = edges[valid] + if edges is not None and len(edges) > 0: + lines = np.column_stack([np.full(len(edges), 2, dtype=np.int64), edges]).ravel() + multi.patches.append(NumpyPolylineMesh(lines=lines, **common)) + else: + multi.patches.append(NumpyPointSetMesh(**common)) + return multi + + +#: Conversion to radians of the ``PlaneAngleUom`` values a deviation survey realistically uses. +_ANGLE_TO_RAD: Dict[str, float] = { + "dega": np.pi / 180.0, + "rad": 1.0, + "gon": np.pi / 200.0, + "grad": np.pi / 200.0, + "mrad": 1e-3, + "urad": 1e-6, + "krad": 1e3, + "mila": np.pi / 3200.0, + "mina": np.pi / (180.0 * 60.0), + "seca": np.pi / (180.0 * 3600.0), +} + + +def read_numpy_deviation_survey_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``DeviationSurveyRepresentation`` as the polyline through its stations. + + The survey stores station ``Mds`` with an ``Inclinations`` / ``Azimuths`` pair rather than + coordinates, so the positions have to be integrated. RESQML is explicit that this is not a + lossless geometry: "The deviation survey does not provide a complete specification of the + geometry of a wellbore trajectory. Although a minimum-curvature algorithm is used in most + cases, the implementation varies sufficiently that no single algorithmic specification is + available as a data transfer standard." The standard minimum-curvature integration is used + here; where a matching ``WellboreTrajectoryRepresentation`` exists it is the authoritative + geometry and should be preferred. + + Azimuths are measured clockwise from North, inclinations from vertical. The station chain + starts at ``FirstStationLocation`` when present, otherwise at the ``MdDatum``. + """ + ws = _view_workspace(workspace) + src_uuid = get_obj_uuid(energyml_object) + src_type = type(energyml_object).__name__ + multi = NumpyMultiMesh( + energyml_object=energyml_object, + identifier=str(get_obj_uri(energyml_object)), + source_uuid=src_uuid, + source_type=src_type, + ) + + def _read(name: str) -> Optional[np.ndarray]: + found = [(p, o) for p, o in search_attribute_matching_name_with_path(energyml_object, name) if o is not None] + if not found: + return None + path, obj = found[0] + try: + return _read_array_np(obj, energyml_object, path, ws).astype(np.float64).ravel() + except Exception as exc: + logger.warning(f"DeviationSurveyRepresentation {src_uuid}: cannot read {name}: {exc}") + return None + + mds = _read("Mds") + incs = _read("Inclinations") + azis = _read("Azimuths") + if mds is None or incs is None or azis is None: + logger.warning(f"DeviationSurveyRepresentation {src_uuid}: Mds/Inclinations/Azimuths missing.") + return multi + n = min(len(mds), len(incs), len(azis)) + if n < 1: + return multi + mds, incs, azis = mds[:n], incs[:n], azis[:n] + + angle_uom = getattr(energyml_object, "angle_uom", None) + uom_name = str(getattr(angle_uom, "value", angle_uom) or "dega") + if uom_name not in _ANGLE_TO_RAD: + logger.warning(f"DeviationSurveyRepresentation {src_uuid}: unknown AngleUom '{uom_name}'; assuming degrees.") + k = _ANGLE_TO_RAD.get(uom_name, _ANGLE_TO_RAD["dega"]) + inc = incs * k + azi = azis * k + + # --- Origin --- + origin = np.zeros(3, dtype=np.float64) + z_increasing_downward = True + crs = None + first = getattr(energyml_object, "first_station_location", None) + if first is not None: + coords = getattr(first, "coordinate1", None), getattr(first, "coordinate2", None), getattr( + first, "coordinate3", None + ) + if all(c is not None for c in coords): + origin = np.array([float(c) for c in coords], dtype=np.float64) + md_datum_dor = getattr(energyml_object, "md_datum", None) + if md_datum_dor is not None and workspace is not None: + try: + datum_obj = workspace.get_object(get_obj_uri(md_datum_dor)) + if datum_obj is not None: + dx, dy, dz, z_increasing_downward, _, _, crs = get_datum_information(datum_obj, workspace) + if first is None: + origin = np.array([dx, dy, dz], dtype=np.float64) + except Exception as exc: + logger.debug(f"Cannot resolve MdDatum of {src_uuid}: {type(exc).__name__}: {exc}") + + # --- Minimum-curvature integration --- + points = np.empty((n, 3), dtype=np.float64) + points[0] = origin + for i in range(1, n): + d_md = float(mds[i] - mds[i - 1]) + i1, i2, a1, a2 = float(inc[i - 1]), float(inc[i]), float(azi[i - 1]), float(azi[i]) + cos_dl = np.cos(i2 - i1) - np.sin(i1) * np.sin(i2) * (1.0 - np.cos(a2 - a1)) + dl = float(np.arccos(np.clip(cos_dl, -1.0, 1.0))) + rf = (2.0 / dl) * np.tan(dl / 2.0) if dl > 1e-9 else 1.0 + half = d_md / 2.0 * rf + d_north = half * (np.sin(i1) * np.cos(a1) + np.sin(i2) * np.cos(a2)) + d_east = half * (np.sin(i1) * np.sin(a1) + np.sin(i2) * np.sin(a2)) + d_tvd = half * (np.cos(i1) + np.cos(i2)) + points[i, 0] = points[i - 1, 0] + d_east + points[i, 1] = points[i - 1, 1] + d_north + points[i, 2] = points[i - 1, 2] + (d_tvd if z_increasing_downward else -d_tvd) + + # get_datum_information reports coordinates already in the projected CRS, like the + # wellbore-trajectory reader's datum path. + frame = PointFrame.PROJECTED if crs is not None else PointFrame.LOCAL + label = f"{src_type}_patch_0" + multi.patches.append( + NumpyPolylineMesh( + identifier=label, + energyml_object=energyml_object, + crs_object=crs, + points=points, + lines=_build_vtk_single_polyline(n), + frame=frame, + patch_index=0, + patch_label=label, + source_uuid=src_uuid, + source_type=src_type, + ) + ) + multi.patches[0].extra_arrays["node_md"] = mds + return multi + + +# --------------------------------------------------------------------------- +# Grid connection sets +# --------------------------------------------------------------------------- + +# Local face-per-cell index of an IJK cell, expressed as the pair of column corners the face +# spans. Corner names follow `_build_split_pillar_map`: TL=(j,i) TR=(j,i+1) BL=(j+1,i) BR=(j+1,i+1), +# so "L"/"R" is the I direction and "T"/"B" the J direction. +# +# The RESQML documentation states the ordering rule — "the top and bottom faces always come +# first, followed by the side faces" (11.5.3, Local Faces per Cell indexing for an IJK Grid +# Cell) — but publishes the index-to-direction assignment only as a figure. Faces 3 and 5 are +# pinned by the fixtures: in rc/epc/80wells_surf_modified_val_color.epc every connection of the +# fault sets uses the pair (3, 5) between a cell at I and its neighbour at I+1, and the nodes +# those two faces resolve to are the two walls of the fault plane at X=375 (a 50 m throw apart). +# The four side faces therefore cycle J-, I+, J+, I- around the column, which fixes 2 and 4. +_IJK_LOCAL_FACE_CORNERS: Dict[int, Tuple[str, ...]] = { + 0: ("TL", "TR", "BR", "BL"), # K- : the whole bottom quad + 1: ("TL", "TR", "BR", "BL"), # K+ : the whole top quad + 2: ("TL", "TR"), # J- + 3: ("TR", "BR"), # I+ + 4: ("BL", "BR"), # J+ + 5: ("TL", "BL"), # I- +} +_IJK_K_FACES = (0, 1) + + +def _ijk_corner_slots(grid_obj: Any) -> Dict[str, int]: + """Map a column corner name to its slot in the 8-node VTK hexahedron of that grid. + + :func:`read_numpy_ijk_grid_representation` reverses the base-quad winding on a left-handed + grid so the emitted cell has a positive Jacobian, so the slot of a given corner depends on + ``GridIsRighthanded``. + """ + geom = getattr(grid_obj, "geometry", None) + righthanded = getattr(geom, "grid_is_righthanded", None) if geom is not None else None + if righthanded is None: + righthanded = True + order = ("TL", "TR", "BR", "BL") if righthanded else ("TL", "BL", "BR", "TR") + return {name: slot for slot, name in enumerate(order)} + + +def _split_vtk_cells(cells: np.ndarray, cell_types: np.ndarray) -> List[np.ndarray]: + """Split a VTK flat cell array into one node array per cell.""" + out: List[np.ndarray] = [] + off = 0 + for _ in range(len(cell_types)): + if off >= len(cells): + break + n = int(cells[off]) + out.append(np.asarray(cells[off + 1 : off + 1 + n], dtype=np.int64)) + off += 1 + n + return out + + +def _polyhedron_faces(cell_entry: np.ndarray) -> List[np.ndarray]: + """Decode a VTK_POLYHEDRON cell body ``[n_faces, npts, p…, npts, p…]`` into face node lists. + + The faces come back in the order :func:`read_numpy_unstructured_grid_representation` wrote + them, which is the order of ``FacesPerCell`` — i.e. the local face index of the grid. + """ + faces: List[np.ndarray] = [] + if len(cell_entry) == 0: + return faces + n_faces = int(cell_entry[0]) + off = 1 + for _ in range(n_faces): + if off >= len(cell_entry): + break + npts = int(cell_entry[off]) + faces.append(np.asarray(cell_entry[off + 1 : off + 1 + npts], dtype=np.int64)) + off += 1 + npts + return faces + + +def _connection_face_nodes( + grid_obj: Any, + cell_nodes: List[np.ndarray], + cell_types: np.ndarray, + corner_slots: Dict[str, int], + cell_index: int, + local_face: int, +) -> Optional[np.ndarray]: + """Return the node indices of *local_face* of *cell_index*, or ``None``. + + Handles the two cell shapes a grid reader emits: the hexahedron of a column-layer grid, + whose local faces follow :data:`_IJK_LOCAL_FACE_CORNERS`, and the polyhedron of an + unstructured grid, whose local face index is a position in its own face list. + """ + if cell_index < 0 or cell_index >= len(cell_nodes): + return None + nodes = cell_nodes[cell_index] + ctype = int(cell_types[cell_index]) if cell_index < len(cell_types) else _VTK_EMPTY_CELL + + if ctype == _VTK_POLYHEDRON: + faces = _polyhedron_faces(nodes) + return faces[local_face] if 0 <= local_face < len(faces) else None + + if ctype != _VTK_HEXAHEDRON or len(nodes) != 8: + return None # empty cell (CellGeometryIsDefined=false) or an unexpected shape + + corners = _IJK_LOCAL_FACE_CORNERS.get(local_face) + if corners is None: + return None + if local_face in _IJK_K_FACES: + base = 0 if local_face == 0 else 4 + return np.array([nodes[corner_slots[c] + base] for c in corners], dtype=np.int64) + + # A side face is the quad swept by two column corners between the bottom and top layers. + c0, c1 = corners + s0, s1 = corner_slots[c0], corner_slots[c1] + return np.array([nodes[s0], nodes[s1], nodes[s1 + 4], nodes[s0 + 4]], dtype=np.int64) + + +def read_numpy_grid_connection_set_representation( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, +) -> "NumpyMultiMesh": + """Read a ``GridConnectionSetRepresentation`` as the surface of its cell faces. + + A grid connection set is "a list of connections between grid cells [...] in the form of + (Grid,Cell,Face)1<=>(Grid,Cell,Face)2" and is "the preferred means of representing faults on + a grid". It carries no geometry of its own: every face is looked up on the grid(s) it + references, which are read through :func:`read_numpy_mesh_object`. + + Output + ------ + * With ``LocalFacePerCellIndexPairs`` — a :class:`NumpySurfaceMesh` of quads. **Both** sides + of a connection are emitted when both are defined: across a fault the two faces are the + two walls and do not coincide, so drawing only one hides the throw. A side whose cell or + face index is null (the array's ``NullValue``) is skipped, which is how the boundary + connections of a fault are stored. + * Without it — the array is optional, "e.g., for a block-centered grid" — a + :class:`NumpyPolylineMesh` of one segment per connection, joining the two cell centroids. + + ``extra_arrays`` carries ``connection_index`` (the connection each face/segment came from) + and, when ``ConnectionInterpretations`` is present, ``interpretation_index`` — the first + interpretation of that connection, which is what lets a viewer colour the set by fault. + + Grids that are missing from the workspace, or that yield no cells, are skipped with a + warning rather than failing the whole set. + """ + ws = _view_workspace(workspace) + src_uuid = get_obj_uuid(energyml_object) + src_type = type(energyml_object).__name__ + multi = NumpyMultiMesh( + energyml_object=energyml_object, + identifier=str(get_obj_uri(energyml_object)), + source_uuid=src_uuid, + source_type=src_type, + ) + + count = int(getattr(energyml_object, "count", 0) or 0) + if count <= 0: + logger.warning(f"GridConnectionSetRepresentation {src_uuid} declares no connection.") + return multi + + cell_pairs, cip_obj = _read_index_pairs(energyml_object, "CellIndexPairs", ws) + if cell_pairs is None: + logger.warning(f"GridConnectionSetRepresentation {src_uuid} has no CellIndexPairs.") + return multi + cell_null = _array_null_value(cip_obj) + + face_pairs, lfp_obj = _read_index_pairs(energyml_object, "LocalFacePerCellIndexPairs", ws) + face_null = _array_null_value(lfp_obj) if face_pairs is not None else None + + grid_pairs, _ = _read_index_pairs(energyml_object, "GridIndexPairs", ws) + + # --- Resolve and read the referenced grids --- + grid_dors = get_object_attribute(energyml_object, "grid") + if grid_dors is None: + grid_dors = [] + elif not isinstance(grid_dors, list): + grid_dors = [grid_dors] + if not grid_dors: + logger.warning(f"GridConnectionSetRepresentation {src_uuid} references no grid.") + return multi + + grid_objs: List[Any] = [] + grid_points: List[np.ndarray] = [] + grid_cells: List[List[np.ndarray]] = [] + grid_cell_types: List[np.ndarray] = [] + grid_slots: List[Dict[str, int]] = [] + point_offsets: List[int] = [] + all_points: List[np.ndarray] = [] + grid_frame: Optional[PointFrame] = None + grid_crs: Any = None + n_points = 0 + + for dor in grid_dors: + grid_obj = workspace.get_object(get_obj_uri(dor)) if workspace is not None else None + if grid_obj is None: + logger.warning(f"GridConnectionSetRepresentation {src_uuid}: grid {get_obj_uri(dor)} not found.") + grid_objs.append(None) + grid_points.append(np.empty((0, 3), dtype=np.float64)) + grid_cells.append([]) + grid_cell_types.append(np.empty(0, dtype=np.uint8)) + grid_slots.append({}) + point_offsets.append(n_points) + continue + # Read the grid through the public dispatcher so its points come back already in the + # frame this call targets; the patches we build below then report that same frame and + # read_numpy_mesh_object leaves them alone instead of transforming them a second time. + grid_mesh = read_numpy_mesh_object( + energyml_object=grid_obj, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + ) + patches = [p for p in grid_mesh.flat_patches() if isinstance(p, NumpyVolumeMesh)] + if not patches: + logger.warning( + f"GridConnectionSetRepresentation {src_uuid}: grid {get_obj_uuid(grid_obj)} " + f"({type(grid_obj).__name__}) produced no volume cells." + ) + patch = patches[0] if patches else None + pts = patch.points if patch is not None else np.empty((0, 3), dtype=np.float64) + grid_objs.append(grid_obj) + grid_points.append(pts) + grid_cells.append(_split_vtk_cells(patch.cells, patch.cell_types) if patch is not None else []) + grid_cell_types.append(patch.cell_types if patch is not None else np.empty(0, dtype=np.uint8)) + grid_slots.append(_ijk_corner_slots(grid_obj)) + point_offsets.append(n_points) + all_points.append(pts) + n_points += len(pts) + if patch is not None and grid_frame is None: + grid_frame = patch.frame + grid_crs = patch.crs_object + + if n_points == 0: + logger.warning(f"GridConnectionSetRepresentation {src_uuid}: no grid geometry available.") + return multi + points = np.concatenate(all_points, axis=0) if len(all_points) > 1 else all_points[0] + + interp_of_connection = _first_interpretation_per_connection(energyml_object, ws, count) + + def _grid_of(conn: int, side: int) -> int: + if grid_pairs is None or conn >= len(grid_pairs): + return 0 + g = int(grid_pairs[conn, side]) + return g if 0 <= g < len(grid_objs) else 0 + + n_conn = min(count, len(cell_pairs)) + if sub_indices is not None: + wanted = {int(i) for i in sub_indices} + else: + wanted = None + + faces_flat: List[int] = [] + face_conn: List[int] = [] + lines_flat: List[int] = [] + line_conn: List[int] = [] + extra_pts: List[np.ndarray] = [] # centroids, appended after the grid points + + for conn in range(n_conn): + if wanted is not None and conn not in wanted: + continue + sides = [] + for side in (0, 1): + cell = int(cell_pairs[conn, side]) + if cell_null is not None and cell == cell_null: + continue + g = _grid_of(conn, side) + if not grid_cells[g]: + continue + sides.append((g, cell, side)) + + if face_pairs is not None: + for g, cell, side in sides: + lf = int(face_pairs[conn, side]) if conn < len(face_pairs) else -1 + if lf < 0 or (face_null is not None and lf == face_null): + continue + nodes = _connection_face_nodes( + grid_objs[g], grid_cells[g], grid_cell_types[g], grid_slots[g], cell, lf + ) + if nodes is None or len(nodes) < 3: + continue + faces_flat.append(len(nodes)) + faces_flat.extend(int(x) + point_offsets[g] for x in nodes) + face_conn.append(conn) + else: + # No face information: join the cell centroids, which is the only geometry the + # connection still defines. + centroids = [] + for g, cell, _side in sides: + nodes = grid_cells[g][cell] + if len(nodes) == 0: + continue + centroids.append(grid_points[g][nodes].mean(axis=0)) + if len(centroids) == 2: + base = n_points + len(extra_pts) + extra_pts.extend(centroids) + lines_flat.extend([2, base, base + 1]) + line_conn.append(conn) + + if faces_flat: + mesh: NumpyMesh = NumpySurfaceMesh( + identifier=f"{src_type}_patch_0", + energyml_object=energyml_object, + crs_object=grid_crs, + points=points, + faces=np.array(faces_flat, dtype=np.int64), + frame=grid_frame if grid_frame is not None else PointFrame.LOCAL, + patch_index=0, + patch_label=f"{src_type}_patch_0", + source_uuid=src_uuid, + source_type=src_type, + ) + conn_idx = np.array(face_conn, dtype=np.int64) + elif lines_flat: + mesh = NumpyPolylineMesh( + identifier=f"{src_type}_patch_0", + energyml_object=energyml_object, + crs_object=grid_crs, + points=np.concatenate([points, np.asarray(extra_pts, dtype=np.float64)], axis=0), + lines=np.array(lines_flat, dtype=np.int64), + frame=grid_frame if grid_frame is not None else PointFrame.LOCAL, + patch_index=0, + patch_label=f"{src_type}_patch_0", + source_uuid=src_uuid, + source_type=src_type, + ) + conn_idx = np.array(line_conn, dtype=np.int64) + else: + logger.warning( + f"GridConnectionSetRepresentation {src_uuid}: none of the {n_conn} connections " + "resolved to a face or a cell pair." + ) + return multi + + mesh.extra_arrays["connection_index"] = conn_idx + if interp_of_connection is not None: + mesh.extra_arrays["interpretation_index"] = interp_of_connection[conn_idx] + multi.patches.append(mesh) + return multi + + +def _read_index_pairs( + energyml_object: Any, + name: str, + ws: Any, +) -> Tuple[Optional[np.ndarray], Any]: + """Read a ``2 x #Connections`` integer array as ``(N, 2)``, or ``(None, None)``. + + ``search_attribute_matching_name_with_path`` reports an attribute that exists on the class + even when the document left it empty, so the ``None`` has to be filtered here — the three + index-pair arrays of a connection set are all optional but one. + """ + results = [(p, o) for p, o in search_attribute_matching_name_with_path(energyml_object, name) if o is not None] + if not results: + return None, None + path, obj = results[0] + try: + arr = _read_array_np(obj, energyml_object, path, ws) + except Exception as exc: + logger.warning(f"Cannot read {name}: {type(exc).__name__}: {exc}") + return None, None + arr = np.asarray(arr).astype(np.int64).ravel() + if arr.size % 2 != 0: + logger.warning(f"{name} holds an odd number of values ({arr.size}); ignoring it.") + return None, None + return arr.reshape(-1, 2), obj + + +def _array_null_value(array_obj: Any) -> Optional[int]: + """Return the ``NullValue`` declared on an integer array, or ``None``.""" + null = getattr(array_obj, "null_value", None) + try: + return int(null) if null is not None else None + except (TypeError, ValueError): + return None + + +def _first_interpretation_per_connection( + energyml_object: Any, + ws: Any, + count: int, +) -> Optional[np.ndarray]: + """Return ``(count,)`` of the first interpretation index of each connection, or ``None``. + + ``ConnectionInterpretations.InterpretationIndices`` is a list-of-lists — a connection may + belong to several interpretations — so only the first is kept, which is enough to colour a + fault set by fault. ``-1`` marks a connection with no interpretation. + """ + ci = getattr(energyml_object, "connection_interpretations", None) + if ci is None: + return None + idx_obj = getattr(ci, "interpretation_indices", None) + if idx_obj is None: + return None + try: + per_conn = _decode_jagged_array( + idx_obj, energyml_object, "connection_interpretations.interpretation_indices", ws + ) + except Exception as exc: + logger.debug(f"Cannot read ConnectionInterpretations.InterpretationIndices: {type(exc).__name__}: {exc}") + return None + out = np.full(count, -1, dtype=np.int64) + for i, entry in enumerate(per_conn[:count]): + if len(entry) > 0: + out[i] = int(entry[0]) + return out + + +# --------------------------------------------------------------------------- +# Main dispatcher +# --------------------------------------------------------------------------- + + +def read_numpy_mesh_object( + energyml_object: Any, + workspace: Optional[EnergymlStorageInterface] = None, + use_crs_displacement: bool = True, + sub_indices: Optional[Union[List[int], np.ndarray]] = None, + frame: Optional[PointFrame] = None, + use_network: bool = False, +) -> "NumpyMultiMesh": + """Dispatcher — equivalent to :func:`mesh.read_mesh_object` but returns + a :class:`NumpyMultiMesh` container. + + Every returned patch carries the :class:`~energyml.utils.data.crs.PointFrame` its points are + expressed in, and this function only applies the pipeline stages a reader has not already + applied. That replaces the previous list of type names — one substring per reader — where a + missing entry silently transformed the same points twice, and an extra one left them + untransformed. + + Args: + energyml_object: Any supported RESQML/EnergyML geometry/representation object. + workspace: Storage interface (``Epc``, ``EpcStreamReader`` or ``EpcFile``). + use_crs_displacement: Legacy switch kept for compatibility. It selects the default + target frame: ``PointFrame.PROJECTED`` when ``True`` (default), + ``PointFrame.LOCAL`` when ``False``. Ignored when *frame* is given. + sub_indices: Optional list of face/line/point indices to include. + frame: Explicit target frame. ``PointFrame.WGS84`` reads the geometry directly + in longitude / latitude / ellipsoidal height — convenient for mapping + output, but note that X/Y are then degrees while Z stays metres, a ratio + no 3-D viewer handles sensibly. + use_network: Allow PROJ to download the geoid grids used by the vertical datum + transformation. Only relevant for ``PointFrame.WGS84``. + + Returns: + :class:`NumpyMultiMesh` containing one or more :class:`NumpyMesh` patches + (and/or nested children for ``RepresentationSetRepresentation``). + + Raises: + :exc:`energyml.utils.exception.NotSupportedError`: if the object type + has no registered reader. + """ + if isinstance(energyml_object, list): + # Synthetic container aggregating multiple top-level objects. + synthetic = NumpyMultiMesh(identifier="multi_object_list") + for obj in energyml_object: + synthetic.children.append( + read_numpy_mesh_object( + energyml_object=obj, + workspace=workspace, + use_crs_displacement=use_crs_displacement, + sub_indices=sub_indices, + frame=frame, + use_network=use_network, + ) + ) + return synthetic + + type_name = _numpy_mesh_name_mapping(type(energyml_object).__name__) + reader_func = get_numpy_reader_function(type_name) + + if reader_func is None: + raise NotSupportedError( + f"No numpy mesh reader found for type '{type_name}'. " + f"Expected function 'read_numpy_{snake_case(type_name)}' in {__name__}." + ) + + result: NumpyMultiMesh = reader_func( + energyml_object=energyml_object, + workspace=workspace, + sub_indices=sub_indices, + use_crs_displacement=use_crs_displacement, + ) + + target = frame if frame is not None else (PointFrame.PROJECTED if use_crs_displacement else PointFrame.LOCAL) + + for m in result.flat_patches(): + if m.frame is target or len(m.points) == 0: + continue + crs = m.crs_object[0] if isinstance(m.crs_object, list) and m.crs_object else m.crs_object + framed = to_frame( + m.points, + extract_crs_info(crs, workspace) if crs is not None else None, + target, + m.frame, + use_network=use_network, + inplace=True, + ) + m.points = framed.points + m.frame = framed.frame return result @@ -2213,7 +3779,7 @@ def numpy_mesh_to_pyvista(mesh: NumpyMesh) -> Any: return pv.PolyData(pts) # Generic fallback: just export points - logging.warning(f"numpy_mesh_to_pyvista: unknown mesh type {type(mesh).__name__}, exporting points only.") + logger.warning(f"numpy_mesh_to_pyvista: unknown mesh type {type(mesh).__name__}, exporting points only.") return pv.PolyData(pts) @@ -2248,6 +3814,8 @@ def numpy_multi_mesh_to_pyvista(multi: "NumpyMultiMesh") -> Any: # --------------------------------------------------------------------------- __all__ = [ + # Coordinate frames (re-exported from crs.py for convenience) + "PointFrame", # Dataclasses "NumpyMesh", "NumpyPointSetMesh", diff --git a/energyml-utils/src/energyml/utils/data/model.py b/energyml-utils/src/energyml/utils/data/model.py index cbfdfff..cacc226 100644 --- a/energyml-utils/src/energyml/utils/data/model.py +++ b/energyml-utils/src/energyml/utils/data/model.py @@ -10,6 +10,8 @@ import numpy as np +logger = logging.getLogger(__name__) + @dataclass class DatasetReader: @@ -70,12 +72,12 @@ def get_or_open(self, file_path: str, handler: "ExternalArrayHandler", mode: str self._cache.move_to_end(file_path) return cached_handle # Otherwise, close and reopen with new mode - # logging.debug(f"Mode change for cached file {file_path}: {cached_mode} -> {mode}. Reopening.") + # logger.debug(f"Mode change for cached file {file_path}: {cached_mode} -> {mode}. Reopening.") try: if hasattr(cached_handle, "close"): cached_handle.close() except Exception as e: - logging.debug(f"Error closing cached file {file_path}: {e}") + logger.debug(f"Error closing cached file {file_path}: {e}") del self._cache[file_path] if file_path in self._handlers: del self._handlers[file_path] @@ -98,7 +100,7 @@ def get_or_open(self, file_path: str, handler: "ExternalArrayHandler", mode: str return file_handle except Exception as e: - logging.debug(f"Failed to open file {file_path}: {e}") + logger.debug(f"Failed to open file {file_path}: {e}") return None def _evict_oldest(self) -> None: @@ -114,7 +116,7 @@ def _evict_oldest(self) -> None: if hasattr(oldest_handle, "close"): oldest_handle.close() except Exception as e: - logging.debug(f"Error closing cached file {oldest_path}: {e}") + logger.debug(f"Error closing cached file {oldest_path}: {e}") # Remove handler reference if oldest_path in self._handlers: @@ -127,7 +129,7 @@ def close_all(self) -> None: if hasattr(file_handle, "close"): file_handle.close() except Exception as e: - logging.debug(f"Error closing file {file_path}: {e}") + logger.debug(f"Error closing file {file_path}: {e}") self._cache.clear() self._handlers.clear() @@ -147,7 +149,7 @@ def remove(self, file_path: str) -> None: if hasattr(file_handle, "close"): file_handle.close() except Exception as e: - logging.debug(f"Error closing file {file_path}: {e}") + logger.debug(f"Error closing file {file_path}: {e}") if file_path in self._handlers: del self._handlers[file_path] @@ -173,7 +175,7 @@ def _is_mode_compatible(cached_mode: str, requested_mode: str) -> bool: rw_modes = {"r+", "a"} destructive_modes = {"w", "w+", "x"} - # logging.debug(f"Checking mode compatibility: cached_mode={cached_mode}, requested_mode={requested_mode}") + # logger.debug(f"Checking mode compatibility: cached_mode={cached_mode}, requested_mode={requested_mode}") result = False @@ -184,7 +186,7 @@ def _is_mode_compatible(cached_mode: str, requested_mode: str) -> bool: if cached_mode in rw_modes and (requested_mode in rw_modes or requested_mode in readonly_modes): result = True - # logging.debug(f"\tMode compatibility result: {result}") + # logger.debug(f"\tMode compatibility result: {result}") return result @@ -207,6 +209,30 @@ class ExternalArrayHandler(ABC): def __init__(self, max_open_files: int = 3): self.file_cache = FileCacheManager(max_open_files=max_open_files) + def close(self) -> None: + """Close every file handle this handler still holds open.""" + cache = getattr(self, "file_cache", None) + if cache is not None: + cache.close_all() + + def __del__(self): + # The cache is per-handler and owns its handles, so releasing the handler must release + # them. Nothing used to do it: the read methods wrapped the *cached* handle in `with`, + # which closed a handle the cache went on serving — the next read on the same file then + # failed with "invalid identifier type to function". Closing here instead keeps the + # handles valid for as long as the handler lives, and still frees them afterwards + # (on Windows an open HDF5 handle keeps the file locked). + try: + self.close() + except Exception: # interpreter shutdown can pull the rug from under us + pass + + def __enter__(self) -> "ExternalArrayHandler": + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + @abstractmethod def read_array( self, @@ -305,7 +331,27 @@ def can_handle_file(self, file_path: str) -> bool: """ pass - @abstractmethod + #: Human-readable name of the format, used in the diagnostics of :meth:`open_file_no_cache`. + format_name: str = "file" + + def _open_file(self, file_path: str, mode: str = "r") -> Optional[Any]: + """ + Open *file_path* with the underlying library, letting any failure propagate. + + This is the **only** thing a concrete handler has to provide: the error handling and the + diagnostic message are shared, in :meth:`open_file_no_cache`. Each of the five real + handlers used to carry its own copy of the same four-line try / log / return-None block, + two of which were byte-for-byte identical, and one of which had had its log commented out + — so an unreadable HDF5 file failed with no message at all. + + Args: + file_path: Path to the file + mode: File open mode + Returns: + Open file handle, or None when this handler cannot open files at all + """ + raise NotImplementedError + def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: """ Open a file without using the cache. This is for handlers that manage their own file handles. @@ -316,7 +362,11 @@ def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: Returns: Open file handle, or None if opening failed """ - pass + try: + return self._open_file(file_path, mode) + except Exception as e: + logger.error("Failed to open %s file %s: %s", self.format_name, file_path, e) + return None # @dataclass @@ -326,3 +376,12 @@ def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]: # def get_array_dimension(self, source: str, path_in_external_file: str) -> Optional[np.ndarray]: # return None + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "DatasetReader", + "FileCacheManager", + "ExternalArrayHandler", +] diff --git a/energyml-utils/src/energyml/utils/data/properties.py b/energyml-utils/src/energyml/utils/data/properties.py new file mode 100644 index 0000000..f85b70b --- /dev/null +++ b/energyml-utils/src/energyml/utils/data/properties.py @@ -0,0 +1,380 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +""" +Readers for energyml *properties* and tables: continuous / discrete / categorical / comment +properties, column-based tables and time series. + +Split out of :mod:`energyml.utils.data.mesh`, which re-exports every public name here so existing +imports keep working. They never had anything to do with meshes, and sharing that module had a +concrete consequence: :func:`read_property` dispatches on ``read_`` in its own +module namespace, which used to contain the geometry readers too — so calling it on, say, a +``PointRepresentation`` silently returned meshes instead of raising. +""" + +import logging +import sys +from functools import lru_cache +from typing import Any, Callable, Dict, List, Optional, Tuple + +import numpy as np + +from energyml.utils.data.helper import read_array +from energyml.utils.exception import NotSupportedError +from energyml.utils.introspection import ( + get_obj_uri, + get_object_attribute, + search_attribute_matching_name, + search_attribute_matching_name_with_path, + snake_case, +) +from energyml.utils.storage_interface import EnergymlStorageInterface + +logger = logging.getLogger(__name__) + + +@lru_cache(maxsize=None) +def get_property_reader_function(property_type_name: str) -> Optional[Callable]: + """Return the ``read_`` function of this module, or None. + + Only functions defined here are eligible, so an imported helper (``read_array``) can never be + mistaken for a property reader. + """ + reader = getattr(sys.modules[__name__], f"read_{snake_case(property_type_name)}", None) + if not callable(reader) or getattr(reader, "__module__", None) != __name__: + return None + return reader + + +def read_property( + energyml_object: Any, + workspace: EnergymlStorageInterface, +) -> np.ndarray: + """ + Read a property or column-based table from an Energyml object. + + Dispatches to the appropriate reader function based on the object's type name. + If no specific reader is found, raises a NotSupportedError. + + Args: + energyml_object: The Energyml object to read from. + workspace: The storage interface for accessing related objects. + + Returns: + np.ndarray: The read property or table data. + + Raises: + NotSupportedError: If the object type is not supported. + """ + property_type = type(energyml_object).__name__ + reader_func = get_property_reader_function(property_type) + if reader_func is not None: + return reader_func(energyml_object=energyml_object, workspace=workspace) + else: + # logger.error(f"Type {array_type_name} is not supported: function read_{snake_case(array_type_name)} not found") + raise NotSupportedError( + f"Type {property_type} is not supported\n\tfunction read_{snake_case(property_type)} not found" + ) + + +def read_property_interpreted_with_cbt( + energyml_object: Any, + workspace: EnergymlStorageInterface, + _cache_property_arrays: Optional[np.ndarray] = None, + _return_none_if_no_category_lookup: bool = False, +) -> Optional[np.ndarray]: + """ + Read a property with category lookup interpretation. + + Reads property arrays and applies category lookup mapping if available. + Supports both array and dictionary-based category lookups. + + Args: + energyml_object: The Energyml property object. + workspace: The storage interface for accessing related objects. + _cache_property_arrays: Optional cached property arrays to avoid re-reading. + _return_none_if_no_category_lookup: If True, return None when no category lookup is found. + + Returns: + Optional[np.ndarray]: The interpreted property values, or None if no lookup and flag is set. + """ + + result = None + + prop_arrays = ( + read_property(energyml_object, workspace) if _cache_property_arrays is None else _cache_property_arrays + ) + + category_lookup_dor = get_object_attribute(energyml_object, "category_lookup") + if category_lookup_dor is not None: + category_lookup_obj = workspace.get_object(get_obj_uri(category_lookup_dor)) + if category_lookup_obj is not None: + category_lookup_data = read_column_based_table(category_lookup_obj, workspace) + + # print(f"category_lookup_array : {category_lookup_data}") + if isinstance(category_lookup_data, list): + category_lookup_data = np.array(category_lookup_data) + if isinstance(category_lookup_data, np.ndarray): + # map props values to category lookup values using prop value as index in category lookup array + result = ( + np.array( + [ + ( + category_lookup_data[prop] + if prop is not None and prop < len(category_lookup_data) + else None + ) + for prop in prop_arrays + ] + ) + if prop_arrays is not None + else None + ) + elif isinstance(category_lookup_data, dict): + # Transpose so that each index corresponds to a category (column), not a row. + # logger.debug(f"category_lookup_data dict : {category_lookup_data}") + + # Guard against inhomogeneous column lengths (e.g. one column is + # empty while another is not). Pad all columns with None up to + # the maximum column length so that np.array() can build a + # rectangular (n_columns, max_rows) matrix before transposing. + col_values = [list(v) if not isinstance(v, list) else v for v in category_lookup_data.values()] + max_len = max((len(c) for c in col_values), default=0) + if max_len == 0: + # All columns empty — nothing to look up. + return prop_arrays if not _return_none_if_no_category_lookup else None + + padded = [c + [None] * (max_len - len(c)) for c in col_values] + category_lookup_matrice = np.array(padded, dtype=object).T + # logger.debug(f"category_lookup_matrice : {category_lookup_matrice}") + # return a matrice with the same shape as prop_arrays but with the values from the category lookup array using the prop value as key in the category lookup array + result = ( + np.array( + [ + [ + ( + category_lookup_matrice[prop].tolist() + if prop is not None and 0 <= prop < len(category_lookup_matrice) + else None + ) + for prop in prop_row + ] + for prop_row in prop_arrays + ] + ) + if prop_arrays is not None + else None + ) + else: + # category_lookup_data is what was actually read; the previous message referred to + # a variable only bound in the dict branch above, which raised NameError instead. + raise NotSupportedError( + f"Category lookup array type {type(category_lookup_data)} is not supported, expected list or dict" + ) + + return prop_arrays if result is None and not _return_none_if_no_category_lookup else result + + +def read_abstract_values_property( + energyml_object: Any, + workspace: EnergymlStorageInterface, +) -> np.ndarray: + """ + Read abstract values property from patches. + + Extracts and concatenates arrays from all 'values_for_patch' attributes. + + Args: + energyml_object: The Energyml object containing the property. + workspace: The storage interface for accessing arrays. + + Returns: + np.ndarray: The concatenated array of property values. + """ + arrays = [] + for values_for_patch in search_attribute_matching_name_with_path(energyml_object, "values_for_patch"): + array = read_array( + energyml_array=values_for_patch[1], + root_obj=energyml_object, + path_in_root=".", + workspace=workspace, + ) + if isinstance(array, list): + array = np.array(array) + arrays.append(array) + if len(arrays) == 1: + return arrays[0] + else: + return np.concatenate(arrays) + + +def read_discrete_property( + energyml_object: Any, + workspace: EnergymlStorageInterface, +) -> np.ndarray: + """ + Read a discrete property. + + Delegates to read_abstract_values_property for implementation. + + Args: + energyml_object: The discrete property object. + workspace: The storage interface. + + Returns: + np.ndarray: The property values. + """ + + return read_abstract_values_property(energyml_object, workspace) + + +def read_continuous_property( + energyml_object: Any, + workspace: EnergymlStorageInterface, +) -> np.ndarray: + """ + Read a continuous property. + + Delegates to read_abstract_values_property for implementation. + + Args: + energyml_object: The continuous property object. + workspace: The storage interface. + + Returns: + np.ndarray: The property values. + """ + + return read_abstract_values_property(energyml_object, workspace) + + +def read_categorical_property( + energyml_object: Any, + workspace: EnergymlStorageInterface, +) -> np.ndarray: + """ + Read a categorical property. + + Note: Categorical values are returned as integers. Use the property's + 'code_list' attribute to map to string values. + + Args: + energyml_object: The categorical property object. + workspace: The storage interface. + + Returns: + np.ndarray: The integer-coded property values. + """ + # TODO: the categorical values should be converted to strings using the code list of the property, but for now we keep the integer values and let the user manage the conversion if needed. + logger.warning( + "CategoricalProperty is read as a continuous property, the categorical values are not converted to strings but kept as integers. Use the 'code_list' attribute of the property to get the list of possible string values corresponding to the integer values in the array" + ) + return read_abstract_values_property(energyml_object, workspace) + + +def read_comment_property( + energyml_object: Any, + workspace: EnergymlStorageInterface, +) -> np.ndarray: + """ + Read a comment property. + + Delegates to read_abstract_values_property for implementation. + + Args: + energyml_object: The comment property object. + workspace: The storage interface. + + Returns: + np.ndarray: The comment values. + """ + return read_abstract_values_property(energyml_object, workspace) + + +def read_column_based_table( + energyml_object: Any, + workspace: EnergymlStorageInterface, +) -> Dict[str, np.ndarray]: + """ + Read a column-based table. + + Extracts column data into a dictionary keyed by column titles. + + Args: + energyml_object: The table object with 'column' attributes. + workspace: The storage interface for accessing arrays. + + Returns: + Dict[str, np.ndarray]: Dictionary of column names to arrays. + """ + columns = {} + for column in get_object_attribute(energyml_object, "column"): + column_name = getattr(column, "title", "_") + # print(f"Reading column: {column_name} : {column}") + # print(f"getattr(column_array, 'values', None): {getattr(column, 'values', None)}") + array = read_array( + energyml_array=getattr(column, "values", None), + root_obj=energyml_object, + path_in_root=".", + workspace=workspace, + ) + if isinstance(array, list): + array = np.array(array) + columns[column_name] = array + return columns + + +def read_time_series( + energyml_object: Any, + workspace: EnergymlStorageInterface, +) -> List[Tuple[str, int]]: + """ + Read a time series from an Energyml object. + + Extracts date-time values and time step indices, constructing a normalized + list of (step_index, datetime) tuples for each time step. + + Args: + energyml_object: The Energyml time series object. + workspace: The storage interface for accessing related objects. + + Returns: + List[Tuple[str, int]]: List of tuples containing (step_index, datetime_string). + """ + + # 1. Extraction des DateTime + times_iso = search_attribute_matching_name(energyml_object, "date_time") + + # 2. Extraction des TimeSteps (v2.2+) + steps_indices = [] + time_step_obj = get_object_attribute(energyml_object, "time_step") + if time_step_obj is not None: + steps_indices = read_array(time_step_obj, energyml_object, ".", workspace, sub_indices=None) + else: + # Fallback : on utilise l'index de la liste + steps_indices = list(range(len(times_iso))) + + # 3. Construction de la structure normalisée + steps_data = [] + for i in range(len(times_iso)): + steps_data.append( + (steps_indices[i], times_iso[i]) + # {"index": i, "datetime": times_iso[i], "step_val": steps_indices[i]} # L'index utilisé par les propriétés + ) + + return steps_data + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "get_property_reader_function", + "read_property", + "read_property_interpreted_with_cbt", + "read_abstract_values_property", + "read_discrete_property", + "read_continuous_property", + "read_categorical_property", + "read_comment_property", + "read_column_based_table", + "read_time_series", +] diff --git a/energyml-utils/src/energyml/utils/data/representation_context.py b/energyml-utils/src/energyml/utils/data/representation_context.py index b7c5c51..795bc93 100644 --- a/energyml-utils/src/energyml/utils/data/representation_context.py +++ b/energyml-utils/src/energyml/utils/data/representation_context.py @@ -15,6 +15,8 @@ from energyml.utils.data.helper import RgbaColor, ScalarRenderingInfo, IndexableElementRenderingInfo, read_color_map, read_graphical_rendering_info from energyml.utils.data.crs import extract_crs_info +logger = logging.getLogger(__name__) + NO_KIND = "NO_KIND" # ───────────────────────────────────────────────────────────────────────────── @@ -99,7 +101,7 @@ def collect_graphical_info_from_rels( uuid, version = extract_uuid_and_version_from_obj_path(r.target) graphical_info_set = workspace.get_object_by_uuid_versioned(uuid, version) if graphical_info_set is None: - logging.warning(f"GraphicalInformationSet {r.target} not found in workspace") + logger.warning(f"GraphicalInformationSet {r.target} not found in workspace") continue graphical_info_set_uri = get_obj_uri(graphical_info_set) for graphical_info in getattr(graphical_info_set, "graphical_information", []): @@ -140,7 +142,7 @@ def get_color(cell_type: Union[str, CellType], object_type: str, graphical_info: color_map = read_color_map(cmap_obj) return color_map.entries[value_vector_index].color except Exception as exc: - logging.debug(f"Error reading ColorInformation: {exc}") + logger.debug(f"Error reading ColorInformation: {exc}") # Search for DefaultGraphicalInformation with matching cell_type info_whole_color = [] @@ -156,7 +158,7 @@ def get_color(cell_type: Union[str, CellType], object_type: str, graphical_info: if elem_info.constant_color is not None: return RgbaColor.from_hsv(elem_info.constant_color) except Exception as exc: - logging.debug(f"Error reading DefaultGraphicalInformation: {exc}") + logger.debug(f"Error reading DefaultGraphicalInformation: {exc}") if info_whole_color: return info_whole_color[0].to_rgb() @@ -230,7 +232,7 @@ def collect_time_series(self): if ts_obj is not None: self.time_series.append(ts_obj) else: - logging.warning(f"TimeSeries {get_obj_uri(ts_dor)} not found in workspace") + logger.warning(f"TimeSeries {get_obj_uri(ts_dor)} not found in workspace") def _collect_properties(self, rels: List[Relationship]): # Collect related properties keyed by property uuid @@ -241,7 +243,7 @@ def _collect_properties(self, rels: List[Relationship]): uuid, version = extract_uuid_and_version_from_obj_path(r.target) prop = self.workspace.get_object_by_uuid_versioned(uuid, version) if prop is None: - logging.warning(f"Property {r.target} not found in workspace") + logger.warning(f"Property {r.target} not found in workspace") continue prop_uuid = getattr(prop, "uuid", NO_KIND) self._props[prop_uuid] = prop @@ -282,7 +284,7 @@ def _collect_crs(self): crs_uuids.add(crs_uuid) self.crs_infos.append(extract_crs_info(crs, self.workspace)) else: - logging.warning(f"CRS {get_obj_uri(crs_ref)} not found in workspace") + logger.warning(f"CRS {get_obj_uri(crs_ref)} not found in workspace") def _collect_graphical_info(self, rels: List[Relationship]): # Collect graphical information entries whose target matches this representation @@ -313,7 +315,7 @@ def rendering_info(self) -> Optional[ScalarRenderingInfo]: if ri.contour_minor_line_info is not None: accumulated.contour_minor_line_info = ri.contour_minor_line_info except Exception as exc: - logging.debug(f"Error reading graphical rendering info: {exc}") + logger.debug(f"Error reading graphical rendering info: {exc}") self._rendering_info = accumulated return self._rendering_info @@ -331,7 +333,9 @@ def get_related_color(self, cell_type: Union[str, CellType]) -> Optional[RgbaCol for r in self.rels + (self.interpretation_as_context.rels if self.interpretation_as_context is not None else []): if "Unit" in r.target: uuid, version = extract_uuid_and_version_from_obj_path(r.target) - print(f"Found Unit reference in relationship: {r.target} (uuid={uuid}, version={version})") + logger.debug( + "Found Unit reference in relationship: %s (uuid=%s, version=%s)", r.target, uuid, version + ) if uuid not in cached_uuids: cached_uuids.add(uuid) unit_obj = self.workspace.get_object_by_uuid_versioned(uuid, version) @@ -402,7 +406,7 @@ def domain(self) -> Optional[str]: try: return interp.domain.value except Exception as e: - print(f"Error accessing interpretation domain: {e}") + logger.warning("Error accessing interpretation domain: %s", e) pass return None @@ -478,11 +482,11 @@ def get_properties_time_series(self, property_uuid: str) -> Dict[str, List[Any]] the given property uuid. Returns an empty dict when the property has no time series reference. """ - from energyml.utils.data.mesh import read_time_series, read_property + from energyml.utils.data.properties import read_time_series, read_property prop = self.get_property(property_uuid) if prop is None: - logging.warning(f"Property {property_uuid} not found in context") + logger.warning(f"Property {property_uuid} not found in context") return {} time_series_dor = search_attribute_matching_name(prop, r"TimeSeries") @@ -504,7 +508,7 @@ def get_properties_time_series(self, property_uuid: str) -> Dict[str, List[Any]] def seach_same_representation_in_other_time_step(self) -> List[Uri]: """Search for another representation that has the same interpretation, and same TimeSeries reference (if any), but different time step.""" if self.time_series is None or len(self.time_series) == 0: - logging.debug( + logger.debug( f"Representation {self.uri} has no TimeSeries reference, skipping search for same representation in other time step" ) return [] @@ -579,6 +583,19 @@ def dump(self) -> str: return "\n".join(lines) +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "NO_KIND", + "CellType", + "collect_graphical_info", + "collect_graphical_info_from_rels", + "get_color", + "get_color_from_object", + "RepresentationContext", +] + + if __name__ == "__main__2": import sys @@ -619,7 +636,7 @@ def dump(self) -> str: # Detail: property arrays (truncated) if repr_ctx._props: - from energyml.utils.data.mesh import read_property + from energyml.utils.data.properties import read_property print("\nProperty arrays (first 10 values):") for uuid, prop in repr_ctx._props.items(): diff --git a/energyml-utils/src/energyml/utils/epc.py b/energyml-utils/src/energyml/utils/epc.py index e1e9782..5fcbc7f 100644 --- a/energyml-utils/src/energyml/utils/epc.py +++ b/energyml-utils/src/energyml/utils/epc.py @@ -604,9 +604,8 @@ def get_reverse_index_stats(self) -> Dict[str, Any]: def _handle_error(self, msg: str) -> None: if self._error_policy == EpcRelsCacheErrorPolicy.LOG: - import logging - logging.error(msg) + logger.error(msg) elif self._error_policy == EpcRelsCacheErrorPolicy.RAISE: raise RuntimeError(msg) # else: SKIP @@ -690,18 +689,18 @@ def wrapper(*args, **kwargs): file_path = kwargs["epc_file_path"] path_info = f" [{file_path}]" if file_path else "" - print(f"⏱️ [{timestamp_start}] Starting {func_name}{path_info}") + logger.debug(f"⏱️ [{timestamp_start}] Starting {func_name}{path_info}") try: result = func(*args, **kwargs) elapsed = time.perf_counter() - start_time timestamp_end = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] - print(f"✅ [{timestamp_end}] Completed {func_name} in {elapsed:.3f}s{path_info}") + logger.debug(f"✅ [{timestamp_end}] Completed {func_name} in {elapsed:.3f}s{path_info}") return result except Exception as e: elapsed = time.perf_counter() - start_time timestamp_end = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] - print(f"❌ [{timestamp_end}] Failed {func_name} after {elapsed:.3f}s{path_info}: {e}") + logger.debug(f"❌ [{timestamp_end}] Failed {func_name} after {elapsed:.3f}s{path_info}: {e}") raise return wrapper @@ -849,7 +848,7 @@ def add_file(self, obj: Union[List, bytes, BytesIO, str, RawFile]): elif not f_ext.lower().endswith(".rels"): self.add_file(RawFile(f_name, BytesIO(file_content))) else: - logging.error(f"Not supported file extension {f_name}") + logger.error(f"Not supported file extension {f_name}") else: try: xml_obj = read_energyml_xml_str(obj) @@ -869,7 +868,7 @@ def add_file(self, obj: Union[List, bytes, BytesIO, str, RawFile]): # another specific package comes in the future self.energyml_objects.append(obj) else: - logging.error(f"unsupported type {str(type(obj))}") + logger.error(f"unsupported type {str(type(obj))}") # === Relationships management functions === @@ -1067,7 +1066,7 @@ def read_array( file_paths = self.external_files_path if not file_paths: - logging.warning(f"No external file paths found for proxy: {proxy}") + logger.warning(f"No external file paths found for proxy: {proxy}") return None # Get the file handler registry @@ -1077,7 +1076,7 @@ def read_array( # Get the appropriate handler for this file type handler = handler_registry.get_handler_for_file(file_path) if handler is None: - logging.debug(f"No handler found for file: {file_path}") + logger.debug(f"No handler found for file: {file_path}") continue try: @@ -1086,10 +1085,10 @@ def read_array( if array is not None: return array except Exception as e: - # logging.debug(f"Failed to read dataset from {file_path}: {e}") + # logger.debug(f"Failed to read dataset from {file_path}: {e}") pass - logging.error(f"Failed to read array from any available file paths: {file_paths}") + logger.error(f"Failed to read array from any available file paths: {file_paths}") return None def read_array_view( @@ -1124,16 +1123,17 @@ def read_array_view( handler = handler_registry.get_handler_for_file(file_path) if handler is None: continue - try: - read_view_fn = getattr(handler, "read_array_view", None) - if read_view_fn is not None: - array = read_view_fn(file_path, path_in_external, start_indices, counts) - else: - array = handler.read_array(file_path, path_in_external, start_indices, counts) - if array is not None: - return array - except Exception as e: - logging.debug(f"Failed to read_array_view from {file_path}: {e}") + # The zero-copy view is an optimisation: when it fails, retry the same file with a + # plain read instead of giving up on it (see _read_array_from_handler). + from energyml.utils.epc_file import _read_array_from_handler + + array = _read_array_from_handler(handler, file_path, path_in_external, start_indices, counts) + if array is not None: + return array + logger.warning( + f"No external array could be read for '{path_in_external}' — tried {len(file_paths)} file(s). " + "The object will come back without geometry." + ) return None def write_array( @@ -1169,7 +1169,7 @@ def write_array( file_paths = self.external_files_path if not file_paths: - logging.warning(f"No external file paths found for proxy: {proxy}") + logger.warning(f"No external file paths found for proxy: {proxy}") return False # Get the file handler registry @@ -1180,7 +1180,7 @@ def write_array( # Get the appropriate handler for this file type handler = handler_registry.get_handler_for_file(file_path) if handler is None: - logging.debug(f"No handler found for file: {file_path}") + logger.debug(f"No handler found for file: {file_path}") continue try: @@ -1189,9 +1189,9 @@ def write_array( if success: return True except Exception as e: - logging.error(f"Failed to write dataset to {file_path}: {e}") + logger.error(f"Failed to write dataset to {file_path}: {e}") - logging.error(f"Failed to write array to any available file paths: {file_paths}") + logger.error(f"Failed to write array to any available file paths: {file_paths}") return False def get_array_metadata( @@ -1221,7 +1221,7 @@ def get_array_metadata( file_paths = self.external_files_path if not file_paths: - logging.warning(f"No external file paths found for proxy: {proxy}") + logger.warning(f"No external file paths found for proxy: {proxy}") return None # Get the file handler registry @@ -1231,7 +1231,7 @@ def get_array_metadata( # Get the appropriate handler for this file type handler = handler_registry.get_handler_for_file(file_path) if handler is None: - logging.debug(f"No handler found for file: {file_path}") + logger.debug(f"No handler found for file: {file_path}") continue try: @@ -1262,7 +1262,7 @@ def get_array_metadata( custom_data={"size": metadata_dict.get("size", 0)}, ) except Exception as e: - logging.debug(f"Failed to get metadata from file {file_path}: {e}") + logger.debug(f"Failed to get metadata from file {file_path}: {e}") return None @@ -1564,7 +1564,7 @@ def _export_io_ultra_fast(self, zip_file: zipfile.ZipFile, force_recompute_objec xml_bytes = future.result() if isinstance(xml_bytes, Exception): - logging.error(f"Erreur sérialisation sur {path}: {xml_bytes}") + logger.error(f"Erreur sérialisation sur {path}: {xml_bytes}") else: zip_file.writestr(path, xml_bytes) @@ -1626,7 +1626,7 @@ def read_stream( :param recompute_rels: If True, recompute all relationships after loading :return: an :class:`EPC` instance """ - print("Reading EPC file seq...") + logger.debug("Reading EPC file seq...") try: _read_files = [] obj_list = [] @@ -1651,13 +1651,13 @@ def read_stream( _read_files.append(content_type_file_name) if content_type_info is None: - logging.error(f"No {content_type_file_name} file found") + logger.error(f"No {content_type_file_name} file found") else: content_type_obj: Types = read_energyml_xml_bytes(epc_file.read(content_type_file_name)) for ov in content_type_obj.override: ov_ct = ov.content_type ov_path = ov.part_name - # logging.debug(ov_ct) + # logger.debug(ov_ct) while ov_path.startswith("/") or ov_path.startswith("\\"): ov_path = ov_path[1:] if is_energyml_content_type(ov_ct): @@ -1672,12 +1672,12 @@ def read_stream( path_to_obj[ov_path] = ov_obj obj_list.append(ov_obj) except Exception: - logging.error(traceback.format_exc()) - logging.error( + logger.error(traceback.format_exc()) + logger.error( f"Epc.@read_stream failed to parse file {ov_path} for content-type: {ov_ct} => {str(get_class_from_content_type(ov_ct))}\n\n", ) try: - logging.debug(epc_file.read(ov_path)) + logger.debug(epc_file.read(ov_path)) except: pass # raise e @@ -1698,11 +1698,11 @@ def read_stream( ) ) except IOError: - logging.error(traceback.format_exc()) + logger.error(traceback.format_exc()) elif f_info.filename != "_rels/.rels": # CoreProperties rels file # RELS FILES READING START - # logging.debug(f"reading rels {f_info.filename}") + # logger.debug(f"reading rels {f_info.filename}") rels_path = Path(f_info.filename) obj_folder = ( str(rels_path.parent.parent) + "/" if str(rels_path.parent.parent) != "." else "" @@ -1723,7 +1723,7 @@ def read_stream( # additional_rels_key = get_obj_identifier(path_to_obj[obj_path]) # # Keep only non-computable rels in additional_rels (legacy support) # for rel in rels_file.relationship: - # # logging.debug(f"\t\t{rel.type_value}") + # # logger.debug(f"\t\t{rel.type_value}") # if ( # rel.type_value != EPCRelsRelationshipType.DESTINATION_OBJECT.get_type() # and rel.type_value != EPCRelsRelationshipType.SOURCE_OBJECT.get_type() @@ -1734,13 +1734,13 @@ def read_stream( # additional_rels[additional_rels_key] = [] # additional_rels[additional_rels_key].append(rel) except AttributeError: - logging.error(traceback.format_exc()) + logger.error(traceback.format_exc()) pass # 'CoreProperties' object has no attribute 'object_version' except Exception as e: - logging.error(f"Error with obj path {obj_path} {path_to_obj[obj_path]}") + logger.error(f"Error with obj path {obj_path} {path_to_obj[obj_path]}") raise e else: - logging.error( + logger.error( f"xml file '{f_info.filename}' is not associate to any readable object " f"(or the object type is not supported because" f" of a lack of a dependency module) " @@ -1768,7 +1768,7 @@ def read_stream( return epc except zipfile.BadZipFile as error: - logging.error(error) + logger.error(error) return None @@ -1779,7 +1779,7 @@ def read_stream_ultra_fast( from concurrent.futures import ProcessPoolExecutor, as_completed import multiprocessing - print("Reading EPC file parrallel v1...") + logger.debug("Reading EPC file parrallel v1...") obj_to_process = {} rels_to_process = {} @@ -1840,7 +1840,7 @@ def read_stream_ultra_fast( path_to_obj[path] = res obj_list.append(res) else: - logging.error(f"Erreur objet {path}: {res}") + logger.error(f"Erreur objet {path}: {res}") # D. Récupération des rels for future in as_completed(rel_futures): @@ -1851,7 +1851,7 @@ def read_stream_ultra_fast( o_path = str(Path(r_path).parent.parent / Path(r_path).stem).replace("\\", "/") rels_content_map[o_path] = res else: - logging.error(f"Erreur rels {r_path}: {res}") + logger.error(f"Erreur rels {r_path}: {res}") # 3. Assemblage final dans le processus parent epc = Epc(energyml_objects=EnergymlObjectCollection(obj_list), raw_files=raw_files, core_props=core_props) @@ -1873,7 +1873,7 @@ def read_stream_ultra_fast_v2( ) -> Optional["Epc"]: from concurrent.futures import ThreadPoolExecutor # Passage au ThreadPool - print("Reading EPC file parrallel v2...") + logger.debug("Reading EPC file parrallel v2...") obj_list = [] path_to_obj = {} @@ -1982,6 +1982,8 @@ def read_stream_ultra_fast_v2( # Also export the cache dict for backward compatibility from energyml.utils.epc_utils import __CACHE_PROP_KIND_DICT__ +logger = logging.getLogger(__name__) + __all__ = [ "Epc", "update_prop_kind_dict_cache", diff --git a/energyml-utils/src/energyml/utils/epc_file.py b/energyml-utils/src/energyml/utils/epc_file.py new file mode 100644 index 0000000..aebd312 --- /dev/null +++ b/energyml-utils/src/energyml/utils/epc_file.py @@ -0,0 +1,1488 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +""" +Lazy, write-buffered EPC handler. + +:class:`EpcFile` sits between the two existing implementations of +:class:`~energyml.utils.storage_interface.EnergymlStorageInterface`: + +- :class:`~energyml.utils.epc.Epc` deserialises every part when reading and + re-serialises every part when writing; +- :class:`~energyml.utils.epc_stream.EpcStreamReader` loads objects lazily but + reads the full XML of every part while indexing, and rebuilds the whole ZIP + (decompressing and recompressing every entry) on each single modification. + +:class:`EpcFile` indexes the archive from its central directory and +``[Content_Types].xml`` only, deserialises a part when it is actually asked for, +buffers modifications in memory, and writes the archive at most once — copying +the compressed payload of untouched parts verbatim (see +:mod:`energyml.utils.zip_raw`). + +When ``[Content_Types].xml`` is missing, truncated, or disagrees with the actual +content of the archive, the index falls back to listing the ZIP entries and +sniffing the root element of the undeclared XML parts, so a damaged package +still opens. + +Persistence is driven by :class:`EpcAccessMode`:: + + with EpcFile("f.epc", mode=EpcAccessMode.READ_ONLY) as epc: # no write allowed + with EpcFile("f.epc", mode=EpcAccessMode.IN_MEMORY) as epc: # edit, never persisted + with EpcFile("f.epc", mode=EpcAccessMode.MANUAL) as epc: # persisted on save() + with EpcFile("f.epc") as epc: # persisted on close() + with EpcFile("f.epc", mode=EpcAccessMode.IMMEDIATE) as epc: # persisted on each write + +Instances are not thread-safe. +""" + +import logging +import os +import re +import zipfile +from collections import OrderedDict +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, Union + +import numpy as np +from lxml import etree as ETREE + +from energyml.opc.opc import ( + CoreProperties, + Default, + Override, + Relationship, + Relationships, + Types, +) +from energyml.utils.constants import ( + EPCRelsRelationshipType, + EpcExportVersion, + MimeType, + OptimizedRegex, + date_to_datetime, +) +from energyml.utils.data.datasets_io import get_handler_registry +from energyml.utils.epc_stream import RelsUpdateMode +from energyml.utils.epc_utils import ( + EXPANDED_EXPORT_FOLDER_PREFIX, + create_default_core_properties, + create_default_types, + create_external_relationship, + gen_core_props_path, + gen_core_props_rels_path, + gen_energyml_object_path, + gen_rels_path_from_obj_path, + get_dor_or_external_uris_from_obj, + get_epc_content_type_path, + get_epc_content_type_rels_path, + get_file_folder, + get_rels_dor_type, + in_epc_file_path_to_mime_type, + is_core_prop_or_extension_path, + make_path_relative_to_filepath_list, + make_path_relative_to_other_file, + relationships_equal, +) +from energyml.utils.introspection import ( + gen_uuid, + get_class_from_content_type, + get_content_type_from_class, + get_obj_title, + get_obj_uri, + get_object_attribute_advanced, +) +from energyml.utils.manager import reshape_version +from energyml.utils.serialization import read_energyml_xml_bytes, serialize_xml +from energyml.utils.storage_interface import ( + DataArrayMetadata, + EnergymlStorageInterface, + ResourceMetadata, + create_resource_metadata_from_uri, +) +from energyml.utils.uri import Uri, create_uri_from_content_type_or_qualified_type, parse_uri +from energyml.utils.xml_utils import ( + find_schema_version_in_element, + get_pkg_from_namespace, + get_root_namespace, + is_energyml_content_type, +) +from energyml.utils.zip_raw import append_to_zip, count_shadowed_entries, rewrite_zip + +logger = logging.getLogger(__name__) + +__all__ = [ + "EpcAccessMode", + "EpcFile", + "EpcFileStats", + "ReadOnlyEpcError", +] + +_DEFAULT_HEAD_SIZE = 8192 +"""Bytes read from the head of a part when resolving its citation lazily.""" + +# Deliberately tolerant: the namespace prefix differs between versions (`eml:`, +# `eml20:`, `eml23:`, none at all) and the tags carry attributes in 2.0.1 +# (``). +_RE_OBJECT_VERSION = re.compile(rb'\bobjectVersion\s*=\s*"([^"]*)"') +_RE_UUID_ATTR = re.compile(rb'\buuid\s*=\s*"([^"]*)"', re.IGNORECASE) +_RE_TITLE = re.compile(rb"<(?:[\w.\-]+:)?Title\b[^>]*>(.*?)]*>(.*?) bool: + """True when the mode accepts modifications at all.""" + return self is not EpcAccessMode.READ_ONLY + + @property + def persists(self) -> bool: + """True when modifications can reach the source file.""" + return self in (EpcAccessMode.MANUAL, EpcAccessMode.ON_CLOSE, EpcAccessMode.IMMEDIATE) + + +@dataclass +class EpcFileStats: + """Counters describing what an :class:`EpcFile` actually had to do.""" + + parts_indexed: int = 0 + objects_indexed: int = 0 + parts_sniffed: int = 0 + """Parts whose type had to be guessed because [Content_Types].xml did not declare them.""" + head_reads: int = 0 + """Bounded reads done to resolve a citation (title / version / last update).""" + objects_deserialized: int = 0 + cache_hits: int = 0 + cache_misses: int = 0 + bytes_read: int = 0 + flushes: int = 0 + parts_raw_copied: int = 0 + parts_recompressed: int = 0 + + @property + def cache_hit_rate(self) -> float: + total = self.cache_hits + self.cache_misses + return self.cache_hits / total if total else 0.0 + + +@dataclass +class _ObjectEntry: + """ + Index entry for one energyml part. Everything past the path and the content + type is resolved on demand. + """ + + path: str + content_type: str + uuid: str + declared: bool = True + """False when the part was not declared in [Content_Types].xml.""" + + version: Optional[str] = None + title: Optional[str] = None + last_changed: Optional[datetime] = None + head_resolved: bool = False + """True once the citation has been read (or filled in from a put_object).""" + + _uri: Optional[Uri] = field(default=None, repr=False) + + @property + def uri(self) -> Uri: + if self._uri is None or self._uri.version != self.version: + self._uri = create_uri_from_content_type_or_qualified_type(self.content_type, self.uuid, self.version) + return self._uri + + @property + def identifier(self) -> str: + return self.uri.as_identifier() + + @property + def qualified_type(self) -> str: + return self.uri.get_qualified_type() + + def to_resource_metadata(self) -> ResourceMetadata: + return create_resource_metadata_from_uri(self.uri, title=self.title, last_changed=self.last_changed) + + +def _read_array_from_handler( + handler: Any, + file_path: Any, + path_in_external: Optional[str], + start_indices: Optional[List[int]] = None, + counts: Optional[List[int]] = None, +) -> Optional[np.ndarray]: + """Read an external array through *handler*, preferring the zero-copy view. + + The view is an **optimisation**, so a failure to obtain one must fall back to the ordinary + read *of the same file* — not skip the file. It used to move straight on to the next + candidate, so when ``read_array_view`` raised for every candidate the array came back as + ``None`` and the representation was silently exported with no geometry. That is what an + ``h5py``/``numpy>=2`` pair did to every HDF5 array (see the ``copy=False`` note in + ``datasets_io``): empty GeoJSON files, and only a DEBUG line to say why. + + Returns ``None`` only when this file genuinely has nothing to offer. + """ + read_view = getattr(handler, "read_array_view", None) + if read_view is not None: + try: + array = read_view(file_path, path_in_external, start_indices, counts) + if array is not None: + return array + except Exception as exc: + logger.debug( + f"read_array_view failed on {file_path} for '{path_in_external}' " + f"({type(exc).__name__}: {exc}) — retrying with a plain read." + ) + + try: + return handler.read_array(file_path, path_in_external, start_indices, counts) + except Exception as exc: + logger.debug(f"read_array failed on {file_path} for '{path_in_external}': {type(exc).__name__}: {exc}") + return None + + +def _sniff_root_element(head: bytes) -> Optional[Any]: + """ + Parse just enough of ``head`` to get the root element with its attributes. + + Works on a truncated document, which is the point: identifying a part must + not require reading it whole. + """ + try: + parser = ETREE.XMLPullParser(events=("start",), recover=True) + parser.feed(head) + for _, element in parser.read_events(): + return element + except Exception as e: + logger.debug(f"Failed to sniff XML head: {e}") + return None + + +def _content_type_from_head(head: bytes) -> Optional[str]: + """ + Derive the content type of an energyml part from the first bytes of its XML. + + Built from the XML alone — root namespace, ``schemaVersion`` and root type — + rather than by resolving the python class: indexing a package must not + require its data model package to be installed, exactly like indexing from + ``[Content_Types].xml`` does not. + + The object type is taken from the ``xsi:type`` attribute when present, since + that is where 2.0.1 carries the ``obj_`` prefixed form the content type uses + (````). + """ + root = _sniff_root_element(head) + if root is None: + return None + try: + package = get_pkg_from_namespace(get_root_namespace(root)) + if package is None or package == "opc": + return None + schema_version = find_schema_version_in_element(root) + if not schema_version: + return None + version = reshape_version(schema_version, 2) + + xsi_type = root.get("{http://www.w3.org/2001/XMLSchema-instance}type") + object_type = xsi_type.split(":")[-1] if xsi_type else ETREE.QName(root).localname + if not object_type: + return None + return f"application/x-{package}+xml;version={version};type={object_type}" + except Exception as e: + logger.debug(f"Failed to derive the content type from the XML head: {e}") + return None + + +def _decode(value: Optional[bytes]) -> Optional[str]: + if value is None: + return None + return value.decode("utf-8", errors="ignore").strip() or None + + +def _serialize(obj: Any) -> bytes: + """ + Serialise to bytes. + + ``serialize_xml`` returns ``str``; the overlay stores bytes so that a part + read back from it behaves exactly like one read from the archive. + """ + data = serialize_xml(obj) + return data.encode("utf-8") if isinstance(data, str) else data + + +class EpcFile(EnergymlStorageInterface): + """ + Lazy EPC reader/writer with buffered writes. + + :param epc_file_path: path of the EPC. Created empty when missing and the mode + allows writing. + :param mode: persistence policy, see :class:`EpcAccessMode`. + :param export_version: packaging used for *new* parts. Detected from the + archive when it already contains parts. + :param rels_update_mode: when relationships are recomputed for modified objects. + :param cache_size: number of deserialised objects kept in the LRU cache. + :param scan_undeclared_parts: sniff the XML parts that ``[Content_Types].xml`` + does not declare. Disable to trust it blindly. + :param compact_on_close: in IMMEDIATE mode, rewrite the archive on close to + drop the entries shadowed by appends. + :param force_h5_path: bypass relationship resolution for external arrays. + """ + + def __init__( + self, + epc_file_path: Union[str, Path], + mode: EpcAccessMode = EpcAccessMode.ON_CLOSE, + export_version: EpcExportVersion = EpcExportVersion.CLASSIC, + rels_update_mode: RelsUpdateMode = RelsUpdateMode.UPDATE_AT_MODIFICATION, + cache_size: int = 128, + scan_undeclared_parts: bool = True, + compact_on_close: bool = True, + force_h5_path: Optional[str] = None, + compression: int = zipfile.ZIP_DEFLATED, + head_size: int = _DEFAULT_HEAD_SIZE, + ): + self.epc_file_path = Path(epc_file_path) + self.mode = mode + self.rels_update_mode = rels_update_mode + self.cache_size = max(1, cache_size) + self.scan_undeclared_parts = scan_undeclared_parts + self.compact_on_close = compact_on_close + self.force_h5_path = force_h5_path + self.compression = compression + self.head_size = head_size + self.stats = EpcFileStats() + + self.export_version = export_version + + # --- index (built at open, never holds a part body) --- + self._zip_entries: Dict[str, zipfile.ZipInfo] = {} + self._by_path: Dict[str, _ObjectEntry] = {} + self._by_uuid: Dict[str, List[_ObjectEntry]] = {} + self._core_props_path: Optional[str] = None + + # --- write overlay --- + self._pending: Dict[str, bytes] = {} + self._deleted: Set[str] = set() + self._content_types_dirty = False + self._pending_rels_rebuild: Set[str] = set() + + # --- caches --- + self._object_cache: "OrderedDict[str, Any]" = OrderedDict() + self._zip: Optional[zipfile.ZipFile] = None + self._closed = False + + if not self.epc_file_path.exists(): + if not mode.allows_write: + raise FileNotFoundError(f"EPC file not found: {self.epc_file_path}") + self._create_empty_epc() + elif not zipfile.is_zipfile(self.epc_file_path): + raise ValueError(f"File is not a valid ZIP/EPC file: {self.epc_file_path}") + + self._build_index() + + # ------------------------------------------------------------------ + # Index + # ------------------------------------------------------------------ + + def _create_empty_epc(self) -> None: + """Create the minimal valid EPC structure for a file that does not exist yet.""" + core_props = create_default_core_properties() + parts = { + get_epc_content_type_path(): _serialize(create_default_types()), + gen_core_props_path(): _serialize(core_props), + gen_core_props_rels_path(): _serialize(Relationships()), + get_epc_content_type_rels_path(): _serialize( + Relationships( + relationship=[ + Relationship( + id="CoreProperties", + type_value=str(EPCRelsRelationshipType.CORE_PROPERTIES), + target=gen_core_props_path(), + ) + ] + ) + ), + } + self.epc_file_path.parent.mkdir(parents=True, exist_ok=True) + rewrite_zip(None, self.epc_file_path, updates=parts, compression=self.compression) + + def _open_zip(self) -> zipfile.ZipFile: + if self._zip is None: + self._zip = zipfile.ZipFile(self.epc_file_path, "r") + return self._zip + + def _close_zip(self) -> None: + if self._zip is not None: + try: + self._zip.close() + except Exception as e: # pragma: no cover - defensive + logger.debug(f"Error closing ZIP handle: {e}") + self._zip = None + + def _build_index(self) -> None: + """ + Build the object index from the central directory and ``[Content_Types].xml``. + + No part body is read, except for the XML parts the content types fail to + describe correctly (and only when ``scan_undeclared_parts`` is set). + """ + self._by_path.clear() + self._by_uuid.clear() + + zf = self._open_zip() + self._zip_entries = {info.filename: info for info in zf.infolist()} + self.stats.parts_indexed = len(self._zip_entries) + + # Packaging is readable from the paths alone. + if any(name.lstrip("/").startswith(EXPANDED_EXPORT_FOLDER_PREFIX) for name in self._zip_entries): + self.export_version = EpcExportVersion.EXPANDED + + for part_name, content_type in self._read_declared_content_types().items(): + if content_type == MimeType.CORE_PROPERTIES.value: + # Only believe it when the path agrees: real packages exist where + # object parts are declared with the core properties content type. + if is_core_prop_or_extension_path(part_name): + self._core_props_path = part_name + continue + try: + if not is_energyml_content_type(content_type): + continue + except Exception: + continue + self._register_part(part_name, content_type, declared=True) + + if self.scan_undeclared_parts: + # Anything that looks like an object part and did not come out of the + # content types is identified from its own root element. That covers a + # missing [Content_Types].xml as well as one declaring a part with the + # wrong content type. + for part_name in self._zip_entries: + if part_name in self._by_path or not self._is_candidate_object_part(part_name): + continue + self._sniff_and_register(part_name) + + if self._core_props_path is None and gen_core_props_path() in self._zip_entries: + self._core_props_path = gen_core_props_path() + + if self.stats.parts_sniffed: + logger.info( + f"{self.epc_file_path}: {self.stats.parts_sniffed} part(s) identified by reading their root element " + f"because {get_epc_content_type_path()} does not describe them usably" + ) + + self.stats.objects_indexed = len(self._by_path) + + def _read_declared_content_types(self) -> Dict[str, str]: + """ + Read ``[Content_Types].xml`` and keep only the overrides that match a part + actually present in the archive. + + An override pointing at a missing part is dropped rather than trusted: it + is the usual symptom of a package edited by a tool that forgot to update + the content types, and keeping it would surface objects that cannot be read. + """ + data = self._read_part(get_epc_content_type_path()) + if data is None: + for name in self._zip_entries: + if name.lower() == get_epc_content_type_path().lower(): + data = self._read_part(name) + break + if data is None: + logger.warning(f"No {get_epc_content_type_path()} in {self.epc_file_path}, indexing from the ZIP listing") + return {} + + try: + types = read_energyml_xml_bytes(data, Types) + except Exception as e: + logger.warning(f"Unreadable {get_epc_content_type_path()} ({e}), indexing from the ZIP listing") + return {} + + result: Dict[str, str] = {} + for override in types.override or []: + if not override.part_name or not override.content_type: + continue + part_name = override.part_name.lstrip("/\\") + if part_name not in self._zip_entries: + logger.debug(f"Content type declares a missing part, ignored: {part_name}") + continue + result[part_name] = override.content_type + return result + + @staticmethod + def _is_candidate_object_part(part_name: str) -> bool: + """True for the parts that could hold an energyml object.""" + lowered = part_name.lower() + if not lowered.endswith(".xml"): + return False + if lowered.startswith("[content_types]") or "/_rels/" in lowered or lowered.startswith("_rels/"): + return False + if part_name in (gen_core_props_path(), f"/{gen_core_props_path()}"): + return False + return True + + def _sniff_and_register(self, part_name: str) -> None: + """Identify an undeclared XML part from its root element.""" + head = self._read_part_head(part_name, self.head_size) + if not head: + return + content_type = _content_type_from_head(head) + if content_type is None: + logger.debug(f"Undeclared part not recognised as energyml, ignored: {part_name}") + return + entry = self._register_part(part_name, content_type, declared=False, head=head) + if entry is not None: + self.stats.parts_sniffed += 1 + logger.debug(f"Part not usable from the content types, recovered by sniffing: {part_name}") + + def _register_part( + self, part_name: str, content_type: str, declared: bool, head: Optional[bytes] = None + ) -> Optional[_ObjectEntry]: + """Add an energyml part to the index, deriving its uuid from the path or the head.""" + uuid = self._uuid_from_path(part_name) + if uuid is None: + head = head if head is not None else self._read_part_head(part_name, self.head_size) + match = _RE_UUID_ATTR.search(head or b"") + uuid = _decode(match.group(1)) if match else None + if not uuid: + logger.warning(f"Cannot determine the uuid of part {part_name}, ignored") + return None + + entry = _ObjectEntry(path=part_name, content_type=content_type, uuid=uuid, declared=declared) + if head is not None: + self._fill_from_head(entry, head) + self._by_path[part_name] = entry + self._by_uuid.setdefault(uuid, []).append(entry) + return entry + + @staticmethod + def _uuid_from_path(part_name: str) -> Optional[str]: + match = OptimizedRegex.UUID_NO_GRP.search(part_name) + return match.group(0) if match is not None else None + + # ------------------------------------------------------------------ + # Lazy citation resolution + # ------------------------------------------------------------------ + + def _read_part_head(self, part_name: str, size: int) -> Optional[bytes]: + """Read at most ``size`` bytes of a part, honouring the write overlay.""" + if part_name in self._deleted: + return None + if part_name in self._pending: + return self._pending[part_name][:size] + if part_name not in self._zip_entries: + return None + try: + with self._open_zip().open(part_name) as f: + data = f.read(size) + self.stats.bytes_read += len(data) + self.stats.head_reads += 1 + return data + except Exception as e: + logger.debug(f"Failed to read the head of {part_name}: {e}") + return None + + def _fill_from_head(self, entry: _ObjectEntry, head: bytes) -> None: + version = _RE_OBJECT_VERSION.search(head) + if version is not None: + entry.version = _decode(version.group(1)) + title = _RE_TITLE.search(head) + if title is not None: + entry.title = _decode(title.group(1)) + last_update = _RE_LAST_UPDATE.search(head) + if last_update is not None: + raw = _decode(last_update.group(1)) + try: + entry.last_changed = date_to_datetime(raw) if raw else None + except Exception: + entry.last_changed = None + entry.head_resolved = title is not None + + def _resolve_entry(self, entry: _ObjectEntry) -> _ObjectEntry: + """ + Resolve the citation of an entry (object version, title, last update). + + Reads a bounded head first and only falls back to the whole part when the + citation is further in. Idempotent. + """ + if entry.head_resolved: + return entry + head = self._read_part_head(entry.path, self.head_size) + if head: + self._fill_from_head(entry, head) + if not entry.head_resolved: + info = self._zip_entries.get(entry.path) + bigger = entry.path in self._pending or (info is not None and info.file_size > self.head_size) + if bigger: + whole = self._read_part(entry.path) + if whole: + self._fill_from_head(entry, whole) + entry.head_resolved = True + return entry + + def resolve_all(self) -> None: + """ + Resolve the citation of every indexed object. + + Only useful when titles are needed for the whole package; this is the one + operation whose cost is proportional to the size of the EPC. + """ + for entry in list(self._by_path.values()): + self._resolve_entry(entry) + + # ------------------------------------------------------------------ + # Parts + # ------------------------------------------------------------------ + + def _read_part(self, part_name: str) -> Optional[bytes]: + """Read a whole part, honouring the write overlay.""" + if part_name in self._deleted: + return None + if part_name in self._pending: + return self._pending[part_name] + if part_name not in self._zip_entries: + return None + try: + data = self._open_zip().read(part_name) + except KeyError: + return None + self.stats.bytes_read += len(data) + return data + + def _part_exists(self, part_name: str) -> bool: + if part_name in self._deleted: + return False + return part_name in self._pending or part_name in self._zip_entries + + def list_parts(self) -> List[str]: + """All part paths currently in the package, overlay included.""" + names = set(self._zip_entries) - self._deleted + names.update(self._pending) + return sorted(names) + + def get_part(self, part_name: str) -> Optional[bytes]: + """Raw content of any part (energyml or not).""" + return self._read_part(part_name) + + def put_part(self, part_name: str, data: bytes) -> None: + """ + Add or replace a non-energyml part (a PDF, an image, ...). + + Use :meth:`put_object` for energyml objects. + """ + self._check_writable() + self._pending[part_name] = data + self._deleted.discard(part_name) + self._content_types_dirty = True + self._after_write() + + def delete_part(self, part_name: str) -> bool: + """Remove any part from the package.""" + self._check_writable() + if not self._part_exists(part_name): + return False + self._pending.pop(part_name, None) + if part_name in self._zip_entries: + self._deleted.add(part_name) + self._content_types_dirty = True + self._after_write() + return True + + # ------------------------------------------------------------------ + # Object lookup + # ------------------------------------------------------------------ + + @staticmethod + def _split_identifier(identifier: Union[str, Uri, Any]) -> Tuple[Optional[str], Optional[str]]: + """ + Split an identifier into (uuid, version). + + A version of ``None`` means "any", which is what a bare uuid or a trailing + dot (``uuid.``) asks for. + """ + if identifier is None: + return None, None + if isinstance(identifier, Uri): + return identifier.uuid, identifier.version or None + if isinstance(identifier, str): + text = identifier.strip() + if text.startswith("eml:///"): + try: + uri = parse_uri(text) + return uri.uuid, uri.version or None + except Exception: + return None, None + match = OptimizedRegex.UUID_NO_GRP.search(text) + if match is None: + return None, None + uuid = match.group(0) + rest = text[match.end() :] + if rest.startswith("."): + rest = rest[1:] + return uuid, rest or None + uri = get_obj_uri(obj=identifier, dataspace=None) + if uri is not None: + return uri.uuid, uri.version or None + return None, None + + def _find_entries(self, identifier: Union[str, Uri, Any]) -> List[_ObjectEntry]: + """Entries matching an identifier, resolving versions only when needed.""" + uuid, version = self._split_identifier(identifier) + if uuid is None: + return [] + candidates = list(self._by_uuid.get(uuid, ())) + if not candidates or version is None: + return candidates + # A version was requested: resolving it costs one bounded head read per + # candidate, and there is normally exactly one. + return [entry for entry in candidates if self._resolve_entry(entry).version == version] + + def get_object(self, identifier: Union[str, Uri]) -> Optional[Any]: + entries = self._find_entries(identifier) + if not entries: + logger.debug(f"No object found for identifier {identifier}") + return None + if len(entries) > 1: + logger.debug(f"{len(entries)} objects share the uuid of {identifier}, returning the first one") + return self._load(entries[0]) + + def get_object_by_uuid(self, uuid: str) -> List[Any]: + objects = [self._load(entry) for entry in self._by_uuid.get(uuid, ())] + return [obj for obj in objects if obj is not None] + + def _load(self, entry: _ObjectEntry) -> Optional[Any]: + """Deserialise a part, through the LRU cache.""" + cached = self._object_cache.get(entry.path) + if cached is not None: + self._object_cache.move_to_end(entry.path) + self.stats.cache_hits += 1 + return cached + self.stats.cache_misses += 1 + + data = self._read_part(entry.path) + if data is None: + logger.warning(f"Part {entry.path} is indexed but unreadable") + return None + try: + cls = get_class_from_content_type(entry.content_type) + obj = read_energyml_xml_bytes(data, cls) + except Exception as e: + logger.error(f"Failed to deserialise {entry.path}: {e}") + return None + + self.stats.objects_deserialized += 1 + self._cache_object(entry.path, obj) + return obj + + def _cache_object(self, part_name: str, obj: Any) -> None: + self._object_cache[part_name] = obj + self._object_cache.move_to_end(part_name) + while len(self._object_cache) > self.cache_size: + self._object_cache.popitem(last=False) + + def clear_cache(self) -> None: + """Drop the deserialised objects. Pending modifications are unaffected.""" + self._object_cache.clear() + + def list_objects( + self, + dataspace: Optional[str] = None, + object_type: Optional[str] = None, + resolve_titles: bool = True, + ) -> List[ResourceMetadata]: + """ + Metadata of the indexed objects. + + ``object_type`` accepts a qualified type (``resqml22.TriangulatedSetRepresentation``) + as well as a bare type (``TriangulatedSetRepresentation``), the latter + being what :attr:`ResourceMetadata.object_type` carries. The filter is + applied on the index, so it never triggers a read. Pass + ``resolve_titles=False`` to skip the citation resolution entirely. + """ + entries = list(self._by_path.values()) + if object_type: + entries = [entry for entry in entries if object_type in (entry.qualified_type, entry.uri.object_type)] + if resolve_titles: + entries = [self._resolve_entry(entry) for entry in entries] + return [entry.to_resource_metadata() for entry in entries] + + def get_object_path(self, identifier: Union[str, Uri, Any]) -> Optional[str]: + """ + In-package path of an object. + + Always the path found in the archive, never one regenerated from the + metadata, so a package whose naming does not match what this library would + produce stays readable. + """ + entries = self._find_entries(identifier) + return entries[0].path if entries else None + + @property + def core_properties(self) -> Optional[CoreProperties]: + if self._core_props_path is None: + return None + data = self._read_part(self._core_props_path) + if data is None: + return None + try: + return read_energyml_xml_bytes(data, CoreProperties) + except Exception as e: + logger.warning(f"Failed to read the core properties: {e}") + return None + + @core_properties.setter + def core_properties(self, core_props: CoreProperties) -> None: + self._check_writable() + path = self._core_props_path or gen_core_props_path() + self._core_props_path = path + self._pending[path] = _serialize(core_props) + self._deleted.discard(path) + self._content_types_dirty = True + self._after_write() + + # ------------------------------------------------------------------ + # Object modification + # ------------------------------------------------------------------ + + def _check_writable(self) -> None: + if self._closed: + raise RuntimeError("This EpcFile is closed") + if not self.mode.allows_write: + raise ReadOnlyEpcError( + f"{self.epc_file_path} is opened in {self.mode.name} mode; reopen it with a writable EpcAccessMode" + ) + + def _after_write(self) -> None: + """Persist right away in IMMEDIATE mode, otherwise let the overlay grow.""" + if self.mode is EpcAccessMode.IMMEDIATE: + self.flush() + + @property + def has_pending_changes(self) -> bool: + """True when modifications are buffered and not yet written.""" + return bool(self._pending or self._deleted or self._content_types_dirty) + + def put_object(self, obj: Any, dataspace: Optional[str] = None) -> Optional[str]: + self._check_writable() + + uri = get_obj_uri(obj=obj, dataspace=None) + if uri is None: + raise ValueError("Failed to build a URI for the object, cannot store it in the EPC") + + existing = self._find_entries(uri) + # Reuse the path already in the package when updating, so the naming of a + # foreign package is preserved. + path = existing[0].path if existing else gen_energyml_object_path(obj, self.export_version) + content_type = get_content_type_from_class(obj) + + self._pending[path] = _serialize(obj) + self._deleted.discard(path) + + entry = self._by_path.get(path) + if entry is None: + entry = _ObjectEntry(path=path, content_type=content_type, uuid=uri.uuid) + self._by_path[path] = entry + self._by_uuid.setdefault(uri.uuid, []).append(entry) + self._content_types_dirty = True + entry.content_type = content_type + entry.version = uri.version or None + entry.title = get_obj_title(obj) + entry.last_changed = self._object_last_update(obj) + entry.head_resolved = True + + self._cache_object(path, obj) + + if self.rels_update_mode is RelsUpdateMode.UPDATE_AT_MODIFICATION: + self._stage_rels_for(obj, entry) + elif self.rels_update_mode is RelsUpdateMode.UPDATE_ON_CLOSE: + self._pending_rels_rebuild.add(path) + + self._after_write() + return entry.identifier + + def add_object(self, obj: Any, replace_if_exists: bool = True) -> Optional[str]: + """Store an object, optionally refusing to overwrite an existing one.""" + if not replace_if_exists: + uri = get_obj_uri(obj=obj, dataspace=None) + if uri is not None and self._find_entries(uri): + raise ValueError(f"Object {uri.as_identifier()} already exists and replace_if_exists is False") + return self.put_object(obj) + + def delete_object(self, identifier: Union[str, Uri, Any]) -> bool: + self._check_writable() + + entries = self._find_entries(identifier) + if not entries: + logger.warning(f"No object to delete for identifier {identifier}") + return False + + for entry in entries: + rels_path = gen_rels_path_from_obj_path(entry.path) + # Read the .rels before dropping it: it is what tells us which other + # objects have to be fixed up. + own_rels = self._read_rels(rels_path) + for path in (entry.path, rels_path): + self._pending.pop(path, None) + if path in self._zip_entries: + self._deleted.add(path) + self._object_cache.pop(entry.path, None) + self._by_path.pop(entry.path, None) + siblings = self._by_uuid.get(entry.uuid, []) + if entry in siblings: + siblings.remove(entry) + if not siblings: + self._by_uuid.pop(entry.uuid, None) + self._pending_rels_rebuild.discard(entry.path) + self._drop_incoming_rels(entry.path, own_rels) + + self._content_types_dirty = True + self._after_write() + return True + + def remove_object(self, identifier: Union[str, Uri, Any]) -> bool: + """Alias of :meth:`delete_object`.""" + return self.delete_object(identifier) + + @staticmethod + def _object_last_update(obj: Any) -> Optional[datetime]: + raw = get_object_attribute_advanced(obj, "citation.lastUpdate") + if isinstance(raw, datetime): + return raw + if isinstance(raw, str): + try: + return date_to_datetime(raw) + except Exception: + return None + return None + + # ------------------------------------------------------------------ + # Relationships + # ------------------------------------------------------------------ + + def _read_rels(self, rels_path: str) -> List[Relationship]: + data = self._read_part(rels_path) + if not data: + return [] + try: + return list(read_energyml_xml_bytes(data, Relationships).relationship or []) + except Exception as e: + logger.warning(f"Failed to read {rels_path}: {e}") + return [] + + def _write_rels(self, rels_path: str, rels: List[Relationship]) -> None: + self._pending[rels_path] = _serialize(Relationships(relationship=rels)) + self._deleted.discard(rels_path) + + @staticmethod + def _merge_rels(existing: List[Relationship], additions: List[Relationship]) -> List[Relationship]: + merged = list(existing) + for addition in additions: + if not any(relationships_equal(addition, current) for current in merged): + merged.append(addition) + return merged + + def get_obj_rels(self, obj: Union[str, Uri, Any]) -> List[Relationship]: + path = self.get_object_path(obj) + if path is None: + return [] + return self._read_rels(gen_rels_path_from_obj_path(path)) + + def _stage_rels_for(self, obj: Any, entry: _ObjectEntry) -> None: + """ + Recompute the relationships touched by storing ``obj``. + + Only the ``.rels`` of the object and of the objects it points at are read + and rewritten; everything else in the package is left alone. + """ + try: + dor_uris, external_uris = get_dor_or_external_uris_from_obj(obj) + except Exception as e: + logger.warning(f"Failed to extract the references of {entry.path}: {e}") + return + + own_path = entry.path + own_rels_path = gen_rels_path_from_obj_path(own_path) + additions: List[Relationship] = [] + + for dor_uri in dor_uris: + target_entries = self._find_entries(dor_uri) + if not target_entries: + logger.debug(f"{own_path} references {dor_uri}, absent from the package") + continue + target_path = target_entries[0].path + additions.append( + Relationship( + target=target_path, + type_value=get_rels_dor_type(dor_uri, in_dor_owner_rels_file=True), + id=f"_{gen_uuid()}", + ) + ) + # Mirror the SOURCE relationship in the target's own .rels. + target_rels_path = gen_rels_path_from_obj_path(target_path) + back = Relationship( + target=own_path, + type_value=get_rels_dor_type(dor_uri, in_dor_owner_rels_file=False), + id=f"_{gen_uuid()}", + ) + self._write_rels(target_rels_path, self._merge_rels(self._read_rels(target_rels_path), [back])) + + for external_uri, _mime_type in external_uris: + if external_uri: + additions.append(create_external_relationship(external_uri)) + + existing = [rel for rel in self._read_rels(own_rels_path) if rel.target not in (None, own_path)] + self._write_rels(own_rels_path, self._merge_rels(existing, additions)) + + def _drop_incoming_rels(self, removed_path: str, own_rels: List[Relationship]) -> None: + """ + Remove the relationships pointing at a part that has just been removed. + + Only the ``.rels`` of the neighbours are touched: an object's own ``.rels`` + lists both the objects it points at and the ones pointing at it, so the + number of files to fix is its degree in the reference graph, not the size + of the package. + """ + for rel in own_rels: + target = rel.target + if not target or target not in self._by_path: + continue + neighbour_rels_path = gen_rels_path_from_obj_path(target) + existing = self._read_rels(neighbour_rels_path) + kept = [candidate for candidate in existing if candidate.target != removed_path] + if len(kept) != len(existing): + self._write_rels(neighbour_rels_path, kept) + + def rebuild_all_rels(self) -> int: + """ + Recompute every ``.rels`` from the references found in the objects. + + Unlike the rest of this class, this loads every object. Returns the number + of ``.rels`` parts written. + """ + self._check_writable() + for rels_path in [name for name in self.list_parts() if name.endswith(".rels")]: + if rels_path not in (get_epc_content_type_rels_path(), gen_core_props_rels_path()): + self._pending[rels_path] = _serialize(Relationships()) + self._deleted.discard(rels_path) + + written_before = len(self._pending) + for entry in list(self._by_path.values()): + obj = self._load(entry) + if obj is not None: + self._stage_rels_for(obj, entry) + self._pending_rels_rebuild.clear() + self._after_write() + return len(self._pending) - written_before + + # ------------------------------------------------------------------ + # Content types + # ------------------------------------------------------------------ + + def _gen_content_types(self) -> Types: + """Regenerate ``[Content_Types].xml`` from the index and the part listing.""" + types = Types(default=[Default(extension="rels", content_type=str(MimeType.RELS))], override=[]) + + core_path = self._core_props_path or gen_core_props_path() + if self._part_exists(core_path): + types.override.append(Override(content_type=str(MimeType.CORE_PROPERTIES), part_name=f"/{core_path}")) + + for entry in self._by_path.values(): + types.override.append(Override(content_type=entry.content_type, part_name=f"/{entry.path}")) + + known = set(self._by_path) | {core_path, get_epc_content_type_path()} + for part_name in self.list_parts(): + if part_name in known or part_name.endswith(".rels"): + continue + mime_type = in_epc_file_path_to_mime_type(part_name) + if mime_type: + types.override.append(Override(content_type=mime_type, part_name=f"/{part_name}")) + + return types + + # ------------------------------------------------------------------ + # Persistence + # ------------------------------------------------------------------ + + def _build_flush_payload(self) -> Dict[str, bytes]: + """ + Parts to write out. + + ``[Content_Types].xml`` is only regenerated when the set of parts changed: + updating an object in place leaves the content types untouched, and + rebuilding them means re-serialising one override per object in the + package, which dominates the cost of an otherwise tiny append. + """ + payload = dict(self._pending) + if self._content_types_dirty: + payload[get_epc_content_type_path()] = _serialize(self._gen_content_types()) + return payload + + def flush(self) -> bool: + """ + Write the buffered modifications to the source file. + + Appends when the mode is IMMEDIATE and nothing has to be removed; + rewrites the archive otherwise, copying the compressed payload of the + untouched parts verbatim. + + :return: True when something was written. + """ + if not self.has_pending_changes: + return False + if self.mode is EpcAccessMode.IN_MEMORY: + logger.warning( + f"{self.epc_file_path} is opened in IN_MEMORY mode: modifications are not written. Use save_as()." + ) + return False + if not self.mode.allows_write: + raise ReadOnlyEpcError(f"{self.epc_file_path} is opened read-only") + + if self.rels_update_mode is RelsUpdateMode.UPDATE_ON_CLOSE and self._pending_rels_rebuild: + for path in list(self._pending_rels_rebuild): + entry = self._by_path.get(path) + if entry is not None: + obj = self._load(entry) + if obj is not None: + self._stage_rels_for(obj, entry) + self._pending_rels_rebuild.clear() + + payload = self._build_flush_payload() + can_append = self.mode is EpcAccessMode.IMMEDIATE and not self._deleted + + self._close_zip() + if can_append: + append_to_zip(self.epc_file_path, payload, compression=self.compression) + else: + raw_copied, recompressed = rewrite_zip( + self.epc_file_path, + self.epc_file_path, + updates=payload, + deleted=self._deleted, + compression=self.compression, + ) + self.stats.parts_raw_copied += raw_copied + self.stats.parts_recompressed += recompressed + + self.stats.flushes += 1 + self._pending.clear() + self._deleted.clear() + self._content_types_dirty = False + + self._zip_entries = {info.filename: info for info in self._open_zip().infolist()} + return True + + def save(self) -> bool: + """Alias of :meth:`flush`, for symmetry with :meth:`save_as`.""" + return self.flush() + + def save_as(self, target: Union[str, Path]) -> Path: + """ + Write the package, modifications included, to another file. + + The current instance keeps pointing at its original file; this is the way + to materialise an EPC edited in IN_MEMORY mode. + """ + target_path = Path(target) + if target_path.exists() and os.path.samefile(target_path, self.epc_file_path): + self.flush() + return self.epc_file_path + + target_path.parent.mkdir(parents=True, exist_ok=True) + raw_copied, recompressed = rewrite_zip( + self.epc_file_path, + target_path, + updates=self._build_flush_payload(), + deleted=self._deleted, + compression=self.compression, + ) + self.stats.parts_raw_copied += raw_copied + self.stats.parts_recompressed += recompressed + return target_path + + def discard_changes(self) -> None: + """Drop the buffered modifications and rebuild the index from the file.""" + self._pending.clear() + self._deleted.clear() + self._content_types_dirty = False + self._pending_rels_rebuild.clear() + self._object_cache.clear() + self._build_index() + + def compact(self) -> bool: + """ + Rewrite the archive to drop the entries shadowed by IMMEDIATE-mode appends. + + No-op when there is nothing to reclaim. + """ + if not self.mode.persists: + return False + self.flush() + if count_shadowed_entries(self._open_zip()) == 0: + return False + self._close_zip() + raw_copied, recompressed = rewrite_zip(self.epc_file_path, self.epc_file_path, compression=self.compression) + self.stats.parts_raw_copied += raw_copied + self.stats.parts_recompressed += recompressed + self._zip_entries = {info.filename: info for info in self._open_zip().infolist()} + return True + + def close(self) -> None: + if self._closed: + return + try: + if self.mode is EpcAccessMode.ON_CLOSE and self.has_pending_changes: + self.flush() + elif self.mode is EpcAccessMode.MANUAL and self.has_pending_changes: + logger.warning( + f"{self.epc_file_path} is closed with unsaved modifications " + f"({len(self._pending)} parts written, {len(self._deleted)} removed): " + f"they are discarded. Call save() to keep them." + ) + elif self.mode is EpcAccessMode.IMMEDIATE and self.compact_on_close: + self.compact() + finally: + self._close_zip() + self._object_cache.clear() + self._closed = True + + # ------------------------------------------------------------------ + # External arrays + # ------------------------------------------------------------------ + + def get_h5_file_paths( + self, obj_or_id: Union[str, Uri, Any] = None, make_path_absolute_from_epc_path: bool = True + ) -> List[str]: + """External file paths reachable from an object's relationships, plus the EPC folder content.""" + if self.force_h5_path is not None: + return [self.force_h5_path] + + paths: Set[str] = set() + part_path = self.get_object_path(obj_or_id) if obj_or_id is not None else None + if part_path is not None: + for rel in self._read_rels(gen_rels_path_from_obj_path(part_path)): + if rel.type_value == str(EPCRelsRelationshipType.EXTERNAL_RESOURCE) and rel.target: + paths.add(rel.target) + + if make_path_absolute_from_epc_path: + paths = set(make_path_relative_to_filepath_list(list(paths), str(self.epc_file_path))) + + folder = get_file_folder(str(self.epc_file_path)) + if folder is not None and os.path.isdir(folder): + for name in os.listdir(folder): + if name.lower().endswith(".h5"): + paths.add(os.path.join(folder, name)) + + return list(paths) + + def _candidate_array_files(self, proxy: Any, external_uri: Optional[str]) -> List[str]: + paths = self.get_h5_file_paths(proxy) + if external_uri: + paths.insert(0, make_path_relative_to_other_file(external_uri, str(self.epc_file_path))) + return paths + + def read_array( + self, + proxy: Union[str, Uri, Any], + path_in_external: str, + start_indices: Optional[List[int]] = None, + counts: Optional[List[int]] = None, + external_uri: Optional[str] = None, + ) -> Optional[np.ndarray]: + file_paths = self._candidate_array_files(proxy, external_uri) + if not file_paths: + logger.warning(f"No external file found for proxy: {proxy}") + return None + + registry = get_handler_registry() + for file_path in file_paths: + handler = registry.get_handler_for_file(file_path) + if handler is None: + continue + try: + array = handler.read_array(file_path, path_in_external, start_indices, counts) + if array is not None: + return array + except Exception as e: + logger.debug(f"Failed to read {path_in_external} from {file_path}: {e}") + logger.error(f"Failed to read {path_in_external} from any of: {file_paths}") + return None + + def read_array_view( + self, + proxy: Union[str, Uri, Any], + path_in_external: str, + start_indices: Optional[List[int]] = None, + counts: Optional[List[int]] = None, + external_uri: Optional[str] = None, + ) -> Optional[np.ndarray]: + file_paths = self._candidate_array_files(proxy, external_uri) + if not file_paths: + return None + + registry = get_handler_registry() + for file_path in file_paths: + handler = registry.get_handler_for_file(file_path) + if handler is None: + continue + array = _read_array_from_handler(handler, file_path, path_in_external, start_indices, counts) + if array is not None: + return array + logger.warning( + f"No external array could be read for '{path_in_external}' — tried {len(file_paths)} file(s): " + f"{[str(p) for p in file_paths]}. The object will come back without geometry." + ) + return None + + def write_array( + self, + proxy: Union[str, Uri, Any], + path_in_external: str, + array: np.ndarray, + start_indices: Optional[List[int]] = None, + external_uri: Optional[str] = None, + **kwargs, + ) -> bool: + if external_uri is not None: + folder = os.path.dirname(str(self.epc_file_path)) or "." + file_paths = ( + [external_uri] if os.path.isabs(external_uri) else [os.path.join(folder, external_uri), external_uri] + ) + elif self.force_h5_path is not None: + file_paths = [self.force_h5_path] + else: + file_paths = self.get_h5_file_paths(proxy) + + if not file_paths: + logger.warning(f"No external file found for proxy: {proxy}") + return False + + registry = get_handler_registry() + for file_path in file_paths: + handler = registry.get_handler_for_file(file_path) + if handler is None: + continue + try: + if handler.write_array(file_path, array, path_in_external, start_indices, **kwargs): + return True + except Exception as e: + logger.error(f"Failed to write {path_in_external} to {file_path}: {e}") + return False + + def get_array_metadata( + self, + proxy: Union[str, Uri, Any], + path_in_external: Optional[str] = None, + start_indices: Optional[List[int]] = None, + counts: Optional[List[int]] = None, + ) -> Union[DataArrayMetadata, List[DataArrayMetadata], None]: + file_paths = [self.force_h5_path] if self.force_h5_path is not None else self.get_h5_file_paths(proxy) + if not file_paths: + logger.warning(f"No external file found for proxy: {proxy}") + return None + + registry = get_handler_registry() + for file_path in file_paths: + handler = registry.get_handler_for_file(file_path) + if handler is None: + continue + try: + raw = handler.get_array_metadata(file_path, path_in_external, start_indices, counts) + except Exception as e: + logger.debug(f"Failed to read the array metadata of {file_path}: {e}") + continue + if raw is None: + continue + if isinstance(raw, list): + return [self._to_array_metadata(item, start_indices) for item in raw] + return self._to_array_metadata(raw, start_indices) + return None + + @staticmethod + def _to_array_metadata(raw: Dict[str, Any], start_indices: Optional[List[int]]) -> DataArrayMetadata: + return DataArrayMetadata( + path_in_resource=raw.get("path"), + array_type=raw.get("dtype", "unknown"), + dimensions=raw.get("shape", []), + start_indices=start_indices, + custom_data={"size": raw.get("size", 0)}, + ) + + # ------------------------------------------------------------------ + # Dunder / misc + # ------------------------------------------------------------------ + + def get_object_dependencies(self, identifier: Union[str, Uri]) -> List[str]: + """Identifiers of the objects referenced by this one.""" + obj = self.get_object(identifier) + if obj is None: + return [] + dor_uris, _ = get_dor_or_external_uris_from_obj(obj) + return [uri.as_identifier() for uri in dor_uris] + + def __len__(self) -> int: + return len(self._by_path) + + def __iter__(self) -> Iterator[str]: + return iter(entry.identifier for entry in self._by_path.values()) + + def __contains__(self, identifier: Union[str, Uri, Any]) -> bool: + return bool(self._find_entries(identifier)) + + def __enter__(self) -> "EpcFile": + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: + if exc_type is not None and self.mode is EpcAccessMode.ON_CLOSE and self.has_pending_changes: + logger.warning( + f"Leaving the context of {self.epc_file_path} on {exc_type.__name__}: " + f"the pending modifications are discarded rather than written." + ) + self._pending.clear() + self._deleted.clear() + self._content_types_dirty = False + self.close() + return False + + def __del__(self): + try: + if not self._closed: + self._close_zip() + except Exception: # pragma: no cover - interpreter shutdown + pass + + def __str__(self) -> str: + return ( + f"EpcFile({self.epc_file_path}, {self.export_version.name}, mode={self.mode.name}) " + f"{len(self._by_path)} objects / {len(self._zip_entries)} parts" + + (f" [+{len(self._pending)} pending, -{len(self._deleted)} removed]" if self.has_pending_changes else "") + ) diff --git a/energyml-utils/src/energyml/utils/epc_stream.py b/energyml-utils/src/energyml/utils/epc_stream.py index 7d00648..d906acb 100644 --- a/energyml-utils/src/energyml/utils/epc_stream.py +++ b/energyml-utils/src/energyml/utils/epc_stream.py @@ -90,6 +90,8 @@ from energyml.utils.xml_utils import is_energyml_content_type +logger = logging.getLogger(__name__) + def get_dor_identifiers_from_obj(obj: Any) -> Set[str]: """Get identifiers of all Data Object References (DORs) directly referenced by the given object.""" @@ -102,9 +104,9 @@ def get_dor_identifiers_from_obj(obj: Any) -> Set[str]: if identifier: identifiers.add(identifier) except Exception as e: - logging.warning(f"Failed to extract identifier from DOR: {e}") + logger.warning(f"Failed to extract identifier from DOR: {e}") except Exception as e: - logging.warning(f"Failed to get DOR list from object: {e}") + logger.warning(f"Failed to get DOR list from object: {e}") return identifiers @@ -284,7 +286,7 @@ def process_object_for_rels_worker( referenced_objects.append((target_identifier, target_type)) except Exception as e: # Don't fail entire object for one bad DOR - logging.debug(f"Skipping invalid DOR URI in {identifier}: {e}") + logger.debug(f"Skipping invalid DOR URI in {identifier}: {e}") return { "identifier": identifier, @@ -294,7 +296,7 @@ def process_object_for_rels_worker( } except Exception as e: - logging.warning(f"Worker failed to process {identifier}: {e}") + logger.warning(f"Worker failed to process {identifier}: {e}") return None @@ -366,7 +368,7 @@ def close(self) -> None: try: self._persistent_zip.close() except Exception as e: - logging.debug(f"Error closing persistent ZIP file: {e}") + logger.debug(f"Error closing persistent ZIP file: {e}") finally: self._persistent_zip = None @@ -432,7 +434,7 @@ def load_metadata(self, detect_export_version: bool = True) -> None: elif self._is_core_properties(override.content_type): self._process_core_properties_metadata(override) else: - logging.debug( + logger.debug( f"Epc_StreamReader @load_metadata Skipping non-EnergyML content type: {override.content_type}" ) @@ -444,13 +446,13 @@ def load_metadata(self, detect_export_version: bool = True) -> None: (EXPANDED_EXPORT_FOLDER_PREFIX, f"/{EXPANDED_EXPORT_FOLDER_PREFIX}") ) ): - logging.debug(f"Detected EXPANDED EPC version based on path: {override.part_name}") + logger.debug(f"Detected EXPANDED EPC version based on path: {override.part_name}") self._export_version = EpcExportVersion.EXPANDED self.stats.total_objects = len(self._metadata) except Exception as e: - logging.error(f"Failed to load metadata from EPC file: {e}") + logger.error(f"Failed to load metadata from EPC file: {e}") raise def get_metadata(self, identifier: str) -> Optional[EpcObjectMetadata]: @@ -539,7 +541,7 @@ def core_properties(self) -> Optional[CoreProperties]: self.stats.bytes_read += len(core_data) self._core_props = read_energyml_xml_bytes(core_data, CoreProperties) except Exception as e: - logging.error(f"Failed to load core properties, creating a default one: {e}") + logger.error(f"Failed to load core properties, creating a default one: {e}") self._core_props = create_default_core_properties() return self._core_props @@ -583,7 +585,7 @@ def _write_core_properties_to_zip(self, core_props: CoreProperties) -> None: # Reopen the zip file to reflect changes self.zip_accessor.reopen_persistent_zip() - logging.info(f"Successfully updated core properties in {self.zip_accessor.epc_file_path}") + logger.info(f"Successfully updated core properties in {self.zip_accessor.epc_file_path}") except Exception as e: # Clean up temp file if it exists @@ -614,15 +616,15 @@ def detect_epc_version(self) -> EpcExportVersion: if file_path.startswith("namespace_"): path_parts = file_path.split("/") if len(path_parts) >= 2: - logging.info(f"Detected EXPANDED EPC version based on path: {file_path}") + logger.info(f"Detected EXPANDED EPC version based on path: {file_path}") return EpcExportVersion.EXPANDED # If no EXPANDED patterns found, assume CLASSIC - logging.info("Detected CLASSIC EPC version") + logger.info("Detected CLASSIC EPC version") return EpcExportVersion.CLASSIC except Exception as e: - logging.warning(f"Failed to detect EPC version, defaulting to CLASSIC: {e}") + logger.warning(f"Failed to detect EPC version, defaulting to CLASSIC: {e}") return EpcExportVersion.CLASSIC def get_content_type(self, zf: zipfile.ZipFile) -> Types: @@ -726,7 +728,7 @@ def _process_energyml_object_metadata(self, zf: zipfile.ZipFile, override: Overr pass except Exception as e: - logging.debug(f"Failed to extract version/title/last_update from XML content for {file_path}: {e}") + logger.debug(f"Failed to extract version/title/last_update from XML content for {file_path}: {e}") if uuid: # Only process if we successfully extracted UUID uri = create_uri_from_content_type_or_qualified_type(ct_or_qt=content_type, uuid=uuid, version=version) @@ -746,7 +748,7 @@ def _process_energyml_object_metadata(self, zf: zipfile.ZipFile, override: Overr except Exception as e: traceback.print_exc() - logging.warning(f"Failed to process metadata for {file_path}: {e}") + logger.warning(f"Failed to process metadata for {file_path}: {e}") def _is_core_properties(self, content_type: str) -> bool: """Check if content type is CoreProperties.""" @@ -834,12 +836,12 @@ def update_rels_for_new_object(self, obj: Any, obj_identifier: str) -> None: """Update relationships when a new object is added (UPDATE_AT_MODIFICATION mode).""" metadata = self.metadata_manager.get_metadata(obj_identifier) if not metadata: - logging.warning(f"Metadata not found for {obj_identifier}") + logger.warning(f"Metadata not found for {obj_identifier}") return # Get all objects this new object references dest_target_uris = get_dor_uris_from_obj(obj) - # logging.debug(f"Updating relationships for new object {obj_identifier}, found DOR targets: {dest_target_uris}") + # logger.debug(f"Updating relationships for new object {obj_identifier}, found DOR targets: {dest_target_uris}") obj_file_path = metadata.file_path(export_version=self.metadata_manager._export_version) @@ -872,7 +874,7 @@ def update_rels_for_modified_object(self, obj: Any, obj_identifier: str) -> None """Update relationships when an object is modified (UPDATE_AT_MODIFICATION mode).""" metadata = self.metadata_manager.get_metadata(obj_identifier) if not metadata: - logging.warning(f"Metadata not found for {obj_identifier}") + logger.warning(f"Metadata not found for {obj_identifier}") return obj_path = metadata.file_path(export_version=self.metadata_manager._export_version) @@ -884,7 +886,7 @@ def update_rels_for_modified_object(self, obj: Any, obj_identifier: str) -> None } # Latest DORs from the modified object dest_target_uris = get_dor_uris_from_obj(obj) - # logging.debug(f"Updating relationships for new object {obj_identifier}, found DOR targets: {dest_target_uris}") + # logger.debug(f"Updating relationships for new object {obj_identifier}, found DOR targets: {dest_target_uris}") # Build new SOURCE relationships current_rels_additions: List[Relationship] = [] @@ -1000,13 +1002,13 @@ def _write_rels_updates( # - Handling different update modes (immediate vs on close) # 1st : debug log the inputs - # logging.debug( + # logger.debug( # f"Writing rels updates for current_object_id={current_object_id}, current_rels_additions={current_rels_additions}, current_rels_removals={current_rels_removals}, target_path_rels_additions={target_path_rels_additions}, target_path_rels_removals={target_path_rels_removals}, delete_current_obj_rels_file_and_file={delete_current_obj_rels_file_and_file}\n\n" # ) current_obj_meta = self.metadata_manager.get_metadata(current_object_id) if not current_obj_meta: - logging.warning(f"Metadata not found for {current_object_id}, cannot write rels updates") + logger.warning(f"Metadata not found for {current_object_id}, cannot write rels updates") return current_object_path = current_obj_meta.file_path(export_version=self.metadata_manager._export_version) current_rels_path = self.metadata_manager.gen_rels_path_from_metadata(current_obj_meta) @@ -1051,7 +1053,7 @@ def _write_rels_updates( else: target_meta = self.metadata_manager.get_metadata(target_id) if not target_meta: - logging.warning( + logger.warning( f"Metadata not found for target {target_id}, skipping rels updates for this target" ) continue @@ -1088,7 +1090,7 @@ def _write_rels_updates( files_to_skip = set(files_to_delete).union(set(rels_updates.keys())) - # logging.debug( + # logger.debug( # f"====\nFiles to delete: {files_to_delete}, rels updates to write: {list(rels_updates.keys())}, files to skip in copy: {files_to_skip}\n\n" # ) @@ -1111,7 +1113,7 @@ def _write_rels_updates( # Write updated rels files for rels_path, rels_xml in rels_updates.items(): target_zf.writestr(rels_path, rels_xml) - # logging.debug(f"Wrote updated rels file: {rels_path} -> {rels_xml}") + # logger.debug(f"Wrote updated rels file: {rels_path} -> {rels_xml}") if delete_current_obj_rels_file_and_file: ct_object: Optional[Types] = None @@ -1139,7 +1141,7 @@ def _write_rels_updates( except Exception as e: if os.path.exists(temp_path): os.unlink(temp_path) - logging.error(f"Failed to write rels updates: {e}") + logger.error(f"Failed to write rels updates: {e}") raise @@ -1184,7 +1186,7 @@ def __init__( # Validate file exists and is readable # ===================================== if not self.epc_file_path.exists(): - logging.info(f"EPC file not found: {self.epc_file_path}. Creating a new empty EPC file.") + logger.info(f"EPC file not found: {self.epc_file_path}. Creating a new empty EPC file.") create_mandatory_structure_epc(self.epc_file_path) is_new_file = True @@ -1254,11 +1256,11 @@ def add_object(self, obj: Any, replace_if_exists: bool = True) -> Optional[str]: if not replace_if_exists: obj_uri: Uri = get_obj_uri(obj=obj, dataspace=None) if obj_uri is None: - logging.error("Failed to get URI for the object, cannot add to EPC") + logger.error("Failed to get URI for the object, cannot add to EPC") return None obj_identifier = obj_uri.as_identifier() if self._metadata_mgr.get_metadata(obj_identifier) is not None: - logging.warning( + logger.warning( f"Object with identifier {obj_identifier} already exists and replace_if_exists is False, skipping add" ) raise ValueError( @@ -1313,7 +1315,7 @@ def add_rels_for_object( _id = self._id_from_uri_or_identifier(identifier=identifier, get_first_if_simple_uuid=True) if _id is None: - logging.warning(f"Invalid identifier provided for adding relationships: {identifier}") + logger.warning(f"Invalid identifier provided for adding relationships: {identifier}") return if not isinstance(relationships, list): @@ -1425,12 +1427,12 @@ def get_object(self, identifier: Union[str, Uri]) -> Optional[Any]: """ _id = self._id_from_uri_or_identifier(identifier=identifier, get_first_if_simple_uuid=True) if _id is None: - logging.warning(f"Invalid identifier provided: {identifier}") + logger.warning(f"Invalid identifier provided: {identifier}") return None metadata = self._metadata_mgr.get_metadata(_id) if metadata is None: - logging.warning(f"Object with identifier {_id} not found in metadata") + logger.warning(f"Object with identifier {_id} not found in metadata") return None # Check cache first @@ -1456,7 +1458,7 @@ def get_object(self, identifier: Union[str, Uri]) -> Optional[Any]: return obj except Exception as e: - logging.error(f"Failed to load object {identifier}: {e}") + logger.error(f"Failed to load object {identifier}: {e}") return None def get_object_by_uuid(self, uuid: str) -> List[Any]: @@ -1493,17 +1495,17 @@ def get_object_by_uuid(self, uuid: str) -> List[Any]: """ # Type guard: ensure uuid is a string if not isinstance(uuid, str): - logging.warning(f"get_object_by_uuid called with non-string uuid: {type(uuid)}") + logger.warning(f"get_object_by_uuid called with non-string uuid: {type(uuid)}") return [] # Type guard: ensure uuid is not empty if not uuid or not uuid.strip(): - logging.warning("get_object_by_uuid called with empty UUID") + logger.warning("get_object_by_uuid called with empty UUID") return [] # Type guard: validate UUID format if OptimizedRegex.UUID.fullmatch(uuid) is None: - logging.warning(f"get_object_by_uuid called with invalid UUID format: {uuid}") + logger.warning(f"get_object_by_uuid called with invalid UUID format: {uuid}") return [] # Get all identifiers for this UUID @@ -1511,11 +1513,11 @@ def get_object_by_uuid(self, uuid: str) -> List[Any]: # Guard: check if identifiers list is valid if identifiers is None or not isinstance(identifiers, list): - logging.debug(f"No identifiers found for UUID: {uuid}") + logger.debug(f"No identifiers found for UUID: {uuid}") return [] if len(identifiers) == 0: - # logging.debug(f"No objects found with UUID: {uuid}") + # logger.debug(f"No objects found with UUID: {uuid}") return [] # Phase 1: Collect cached objects and prepare list of non-cached identifiers @@ -1525,13 +1527,13 @@ def get_object_by_uuid(self, uuid: str) -> List[Any]: for identifier in identifiers: # Type guard: ensure identifier is valid if not identifier or not isinstance(identifier, str): - logging.warning(f"Skipping invalid identifier in UUID lookup: {identifier}") + logger.warning(f"Skipping invalid identifier in UUID lookup: {identifier}") continue # Get metadata first to validate object exists metadata = self._metadata_mgr.get_metadata(identifier) if metadata is None: - logging.warning(f"Metadata not found for identifier {identifier}, skipping") + logger.warning(f"Metadata not found for identifier {identifier}, skipping") continue # Check cache first for consistency @@ -1543,7 +1545,7 @@ def get_object_by_uuid(self, uuid: str) -> List[Any]: objects.append(obj) else: # Remove invalid cached entry and mark for re-loading - logging.warning(f"Removing invalid cached object for {identifier}") + logger.warning(f"Removing invalid cached object for {identifier}") del self._object_cache[identifier] non_cached_metadata.append((identifier, metadata)) self.stats.cache_misses += 1 @@ -1568,7 +1570,7 @@ def get_object_by_uuid(self, uuid: str) -> List[Any]: # Guard: validate deserialized object if obj is None: - logging.warning(f"Deserialization returned None for {identifier}") + logger.warning(f"Deserialization returned None for {identifier}") continue # Add to cache with consistency check @@ -1577,12 +1579,12 @@ def get_object_by_uuid(self, uuid: str) -> List[Any]: objects.append(obj) except KeyError: - logging.error(f"File not found in ZIP for identifier {identifier}: {file_path}") + logger.error(f"File not found in ZIP for identifier {identifier}: {file_path}") except Exception as e: - logging.error(f"Failed to deserialize object {identifier}: {e}") + logger.error(f"Failed to deserialize object {identifier}: {e}") except Exception as e: - logging.error(f"Failed to open ZIP file for batch loading: {e}") + logger.error(f"Failed to open ZIP file for batch loading: {e}") return objects @@ -1612,7 +1614,7 @@ def put_object(self, obj: Any, dataspace: Optional[str] = None) -> Optional[str] # Copy all existing files except the one being updated (if update) and its .rels file with self._zip_accessor.get_zip_file() as source_zf: for item in source_zf.infolist(): - # logging.debug( + # logger.debug( # f"Test {get_epc_content_type_path() in item.filename} with {item.filename} and {get_epc_content_type_path()} " # ) if get_epc_content_type_path() in item.filename: @@ -1630,19 +1632,19 @@ def put_object(self, obj: Any, dataspace: Optional[str] = None) -> Optional[str] if not file_allready_exists: ct_object = None if epc_content_type is not None: - # logging.debug("Existing content type found, adding new object to it") + # logger.debug("Existing content type found, adding new object to it") # add the new object to the existing content type and write it ct_object = read_energyml_xml_bytes(epc_content_type, Types) - # logging.debug("Existing content type before adding object: " + str(ct_object)) + # logger.debug("Existing content type before adding object: " + str(ct_object)) ct_object.override.append( Override(part_name=file_path, content_type=get_content_type_from_class(obj)) ) if ct_object is None: - # logging.debug("No existing content type found, generating new one from metadata manager") + # logger.debug("No existing content type found, generating new one from metadata manager") ct_object = self._metadata_mgr.get_content_type(zf) - # logging.debug("New content type after adding object: " + str(ct_object)) + # logger.debug("New content type after adding object: " + str(ct_object)) zf.writestr(get_epc_content_type_path(), serialize_xml(ct_object)) - # logging.debug("Written content type to EPC with new object : " + serialize_xml(ct_object)) + # logger.debug("Written content type to EPC with new object : " + serialize_xml(ct_object)) elif epc_content_type is not None: zf.writestr(get_epc_content_type_path(), epc_content_type) # Replace original @@ -1673,11 +1675,11 @@ def delete_object(self, identifier: Union[str, Uri, Any]) -> bool: # 4. Return True if deletion was successful, False otherwise _id = self._id_from_uri_or_identifier(identifier=identifier) if _id is None: - logging.warning(f"Invalid identifier provided for deletion: {identifier}") + logger.warning(f"Invalid identifier provided for deletion: {identifier}") return False metadata = self._metadata_mgr.get_metadata(_id) if metadata is None: - logging.warning(f"Object with identifier {_id} not found in metadata, cannot delete") + logger.warning(f"Object with identifier {_id} not found in metadata, cannot delete") return False if self.rels_update_mode == RelsUpdateMode.UPDATE_AT_MODIFICATION: @@ -1724,7 +1726,7 @@ def read_array( file_paths.insert(0, make_path_relative_to_other_file(external_uri, self.epc_file_path)) if not file_paths: - logging.warning(f"No external file paths found for proxy: {proxy}") + logger.warning(f"No external file paths found for proxy: {proxy}") return None # Get the file handler registry @@ -1734,7 +1736,7 @@ def read_array( # Get the appropriate handler for this file type handler = handler_registry.get_handler_for_file(file_path) if handler is None: - logging.debug(f"No handler found for file: {file_path}") + logger.debug(f"No handler found for file: {file_path}") continue try: @@ -1743,10 +1745,10 @@ def read_array( if array is not None: return array except Exception as e: - # logging.debug(f"Failed to read dataset from {file_path}: {e}") + # logger.debug(f"Failed to read dataset from {file_path}: {e}") pass - logging.error(f"Failed to read array from any available file paths: {file_paths}") + logger.error(f"Failed to read array from any available file paths: {file_paths}") return None def read_array_view( @@ -1776,15 +1778,19 @@ def read_array_view( if handler is None: continue try: - read_view_fn = getattr(handler, "read_array_view", None) - if read_view_fn is not None: - array = read_view_fn(file_path, path_in_external, start_indices, counts) - else: - array = handler.read_array(file_path, path_in_external, start_indices, counts) + # The zero-copy view is an optimisation: when it fails, retry the same file with + # a plain read instead of giving up on it (see _read_array_from_handler). + from energyml.utils.epc_file import _read_array_from_handler + + array = _read_array_from_handler(handler, file_path, path_in_external, start_indices, counts) if array is not None: return array except Exception as e: - logging.debug(f"Failed to read_array_view from {file_path}: {e}") + logger.debug(f"Failed to read the array from {file_path}: {e}") + logger.warning( + f"No external array could be read for '{path_in_external}' — tried {len(file_paths)} file(s). " + "The object will come back without geometry." + ) return None def write_array( @@ -1831,7 +1837,7 @@ def write_array( file_paths = self.get_h5_file_paths(proxy) if not file_paths: - logging.warning(f"No external file paths found for proxy: {proxy}") + logger.warning(f"No external file paths found for proxy: {proxy}") return False # Get the file handler registry @@ -1842,7 +1848,7 @@ def write_array( # Get the appropriate handler for this file type handler = handler_registry.get_handler_for_file(file_path) if handler is None: - logging.debug(f"No handler found for file: {file_path}") + logger.debug(f"No handler found for file: {file_path}") continue try: @@ -1851,9 +1857,9 @@ def write_array( if success: return True except Exception as e: - logging.error(f"Failed to write dataset to {file_path}: {e}") + logger.error(f"Failed to write dataset to {file_path}: {e}") - logging.error(f"Failed to write array to any available file paths: {file_paths}") + logger.error(f"Failed to write array to any available file paths: {file_paths}") return False def get_array_metadata( @@ -1887,7 +1893,7 @@ def get_array_metadata( file_paths = self.get_h5_file_paths(proxy) if not file_paths: - logging.warning(f"No external file paths found for proxy: {proxy}") + logger.warning(f"No external file paths found for proxy: {proxy}") return None # Get the file handler registry handler_registry = get_handler_registry() @@ -1896,7 +1902,7 @@ def get_array_metadata( # Get the appropriate handler for this file type handler = handler_registry.get_handler_for_file(file_path) if handler is None: - logging.debug(f"No handler found for file: {file_path}") + logger.debug(f"No handler found for file: {file_path}") continue try: @@ -1927,7 +1933,7 @@ def get_array_metadata( custom_data={"size": metadata_dict.get("size", 0)}, ) except Exception as e: - logging.debug(f"Failed to get metadata from file {file_path}: {e}") + logger.debug(f"Failed to get metadata from file {file_path}: {e}") return None @@ -1940,12 +1946,12 @@ def get_obj_rels(self, obj: Union[str, Uri, Any]) -> List[Relationship]: _id = self._id_from_uri_or_identifier(obj) if _id is None: - logging.warning(f"Could not resolve identifier for object {obj}, cannot get relationships") + logger.warning(f"Could not resolve identifier for object {obj}, cannot get relationships") return [] metadata = self._metadata_mgr.get_metadata(_id) if metadata is None: - logging.warning(f"Object with identifier {_id} not found in metadata, cannot get relationships") + logger.warning(f"Object with identifier {_id} not found in metadata, cannot get relationships") return [] return self._rels_mgr.get_obj_rels(_id) @@ -1961,9 +1967,9 @@ def close(self) -> None: if self.rels_update_mode == RelsUpdateMode.UPDATE_ON_CLOSE: try: self.rebuild_all_rels(clean_first=True) - logging.info("Rebuilt all relationships on close (UPDATE_ON_CLOSE mode)") + logger.info("Rebuilt all relationships on close (UPDATE_ON_CLOSE mode)") except Exception as e: - logging.warning(f"Error rebuilding rels on close: {e}") + logger.warning(f"Error rebuilding rels on close: {e}") # Close file cache if hasattr(self, "_file_cache"): @@ -1974,7 +1980,7 @@ def close(self) -> None: try: self.cache_opened_h5.close() except Exception as e: - logging.debug(f"Error closing cache_opened_h5: {e}") + logger.debug(f"Error closing cache_opened_h5: {e}") self.cache_opened_h5 = None # Delegate to ZIP accessor @@ -2049,7 +2055,7 @@ def _id_from_uri_or_identifier( return as_identifier(identifier) except Exception: if not get_first_if_simple_uuid: - logging.warning( + logger.warning( f"Identifier {identifier} is a simple UUID, but get_first_if_simple_uuid is False, cannot resolve to full identifier" ) return None @@ -2060,7 +2066,7 @@ def _id_from_uri_or_identifier( 0 ] # If multiple metadata entries for the same UUID, we take the first one (this should not happen in a well-formed EPC file) else: - logging.warning(f"No metadata found for UUID {identifier}, cannot get relationships") + logger.warning(f"No metadata found for UUID {identifier}, cannot get relationships") return None def _rebuild_all_rels_sequential(self, clean_first: bool = True) -> Dict[str, int]: @@ -2090,7 +2096,7 @@ def _rebuild_all_rels_sequential(self, clean_first: bool = True) -> Dict[str, in "destination_relationships": 0, } - logging.info(f"Starting rebuild of all .rels files for {len(self._metadata)} objects...") + logger.info(f"Starting rebuild of all .rels files for {len(self._metadata)} objects...") # Build a map of which objects are referenced by which objects # Key: target identifier, Value: list of (source_identifier, source_obj) @@ -2123,7 +2129,7 @@ def _rebuild_all_rels_sequential(self, clean_first: bool = True) -> Dict[str, in pass except Exception as e: - logging.warning(f"Failed to analyze object {identifier}: {e}") + logger.warning(f"Failed to analyze object {identifier}: {e}") # Second pass: create the .rels files # Map of rels_file_path -> Relationships object @@ -2163,7 +2169,7 @@ def _rebuild_all_rels_sequential(self, clean_first: bool = True) -> Dict[str, in stats["destination_relationships"] += 1 except Exception as e: - logging.debug(f"Failed to create DESTINATION relationship: {e}") + logger.debug(f"Failed to create DESTINATION relationship: {e}") if relationships and obj_rels_path: if obj_rels_path not in rels_files: @@ -2171,7 +2177,7 @@ def _rebuild_all_rels_sequential(self, clean_first: bool = True) -> Dict[str, in rels_files[obj_rels_path].relationship.extend(relationships) except Exception as e: - logging.warning(f"Failed to create DESTINATION rels for {identifier}: {e}") + logger.warning(f"Failed to create DESTINATION rels for {identifier}: {e}") # Add SOURCE relationships (in target's .rels file, pointing back to sources) for target_identifier, source_list in reverse_references.items(): @@ -2202,10 +2208,10 @@ def _rebuild_all_rels_sequential(self, clean_first: bool = True) -> Dict[str, in stats["source_relationships"] += 1 except Exception as e: - logging.debug(f"Failed to create SOURCE relationship: {e}") + logger.debug(f"Failed to create SOURCE relationship: {e}") except Exception as e: - logging.warning(f"Failed to create SOURCE rels for {target_identifier}: {e}") + logger.warning(f"Failed to create SOURCE rels for {target_identifier}: {e}") stats["rels_files_created"] = len(rels_files) @@ -2249,7 +2255,7 @@ def _rebuild_all_rels_sequential(self, clean_first: bool = True) -> Dict[str, in # Create new entry with only preserved relationships rels_files[filename] = Relationships(relationship=preserved_rels) except Exception as e: - logging.debug(f"Could not preserve existing rels from {filename}: {e}") + logger.debug(f"Could not preserve existing rels from {filename}: {e}") # Update core_prop_rels with extended props if needed new_core_prop_rels = Relationships( @@ -2305,7 +2311,7 @@ def _rebuild_all_rels_sequential(self, clean_first: bool = True) -> Dict[str, in shutil.move(temp_path, self.epc_file_path) self._zip_accessor.reopen_persistent_zip() - logging.info( + logger.info( f"Rebuilt .rels files: processed {stats['objects_processed']} objects, " f"created {stats['rels_files_created']} .rels files, " f"added {stats['source_relationships']} SOURCE and " @@ -2347,7 +2353,7 @@ def _rebuild_all_rels_parallel(self, clean_first: bool = True) -> Dict[str, int] } num_objects = len(self._metadata) - logging.info(f"Starting PARALLEL rebuild of all .rels files for {num_objects} objects...") + logger.info(f"Starting PARALLEL rebuild of all .rels files for {num_objects} objects...") # Prepare work items for parallel processing # Pass metadata as dict (serializable) instead of keeping references @@ -2361,7 +2367,7 @@ def _rebuild_all_rels_parallel(self, clean_first: bool = True) -> Dict[str, int] # Don't spawn more workers than CPUs; use user-configurable ratio for workload per worker worker_ratio = self.parallel_worker_ratio if hasattr(self, "parallel_worker_ratio") else _WORKER_POOL_SIZE_RATIO num_workers = min(cpu_count(), max(1, num_objects // worker_ratio)) - logging.info(f"Using {num_workers} worker processes for {num_objects} objects (ratio: {worker_ratio})") + logger.info(f"Using {num_workers} worker processes for {num_objects} objects (ratio: {worker_ratio})") # ============================================================================ # PHASE 1: PARALLEL - Compute SOURCE relationships across worker processes @@ -2447,10 +2453,10 @@ def _rebuild_all_rels_parallel(self, clean_first: bool = True) -> Dict[str, int] stats["source_relationships"] += 1 except Exception as e: - logging.debug(f"Failed to create SOURCE relationship: {e}") + logger.debug(f"Failed to create SOURCE relationship: {e}") except Exception as e: - logging.warning(f"Failed to create SOURCE rels for {target_identifier}: {e}") + logger.warning(f"Failed to create SOURCE rels for {target_identifier}: {e}") stats["rels_files_created"] = len(rels_files) @@ -2496,7 +2502,7 @@ def _rebuild_all_rels_parallel(self, clean_first: bool = True) -> Dict[str, int] else: rels_files[filename] = Relationships(relationship=preserved_rels) except Exception as e: - logging.debug(f"Could not preserve existing rels from {filename}: {e}") + logger.debug(f"Could not preserve existing rels from {filename}: {e}") # update core_prop_rels with extended props if needed new_core_prop_rels = Relationships( @@ -2522,7 +2528,7 @@ def _rebuild_all_rels_parallel(self, clean_first: bool = True) -> Dict[str, int] core_prop_rels.relationship.append(new_rel) rels_files[gen_core_props_rels_path()] = core_prop_rels - print(f"Coreprops : {core_prop_rels}") + logger.debug("Core properties relationships: %s", core_prop_rels) # ============================================================================ # PHASE 5: SEQUENTIAL - Write all relationships to ZIP file @@ -2560,7 +2566,7 @@ def _rebuild_all_rels_parallel(self, clean_first: bool = True) -> Dict[str, int] execution_time = time.time() - start_time stats["execution_time"] = execution_time - logging.info( + logger.info( f"Rebuilt .rels files (PARALLEL): processed {stats['objects_processed']} objects, " f"created {stats['rels_files_created']} .rels files, " f"added {stats['source_relationships']} SOURCE and " @@ -2589,3 +2595,15 @@ def update_object(self, obj: Any) -> Optional[str]: def get_object_by_identifier(self, identifier: Union[str, Uri]) -> Optional[Any]: """Alias for get_object for backward compatibility.""" return self.get_object(identifier) + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "get_dor_identifiers_from_obj", + "RelsUpdateMode", + "EpcObjectMetadata", + "EpcStreamingStats", + "process_object_for_rels_worker", + "EpcStreamReader", +] diff --git a/energyml-utils/src/energyml/utils/epc_utils.py b/energyml-utils/src/energyml/utils/epc_utils.py index c61bab9..33dab8d 100644 --- a/energyml-utils/src/energyml/utils/epc_utils.py +++ b/energyml-utils/src/energyml/utils/epc_utils.py @@ -68,6 +68,8 @@ from energyml.utils.uri import Uri, parse_uri from energyml.utils.storage_interface import ResourceMetadata +logger = logging.getLogger(__name__) + # ____ ___ ________ __ # / __ \/ |/_ __/ / / / # / /_/ / /| | / / / /_/ / @@ -144,7 +146,7 @@ def gen_energyml_object_path( return get_epc_content_type_path() else: obj_type = get_object_type_for_file_path_from_class(energyml_object.__class__) - # logging.debug("is_dor: ", str(is_dor(energyml_object)), "object type : " + str(obj_type)) + # logger.debug("is_dor: ", str(is_dor(energyml_object)), "object type : " + str(obj_type)) pkg = get_class_pkg(energyml_object) pkg_version = get_class_pkg_version(energyml_object) object_version = get_obj_version(energyml_object) @@ -455,7 +457,7 @@ def valdiate_basic_epc_structure(epc: Union[str, Path, zipfile.ZipFile, BytesIO] epc_files = set(epc_io.namelist()) missing_files = required_files - epc_files if missing_files: - logging.warning(f"The EPC file is missing the following required files: {missing_files}") + logger.warning(f"The EPC file is missing the following required files: {missing_files}") return False finally: if should_close: @@ -511,7 +513,7 @@ def create_mandatory_structure_epc(epc: Union[str, Path, zipfile.ZipFile, BytesI def repair_epc_structure_if_not_valid(epc: Union[str, Path, zipfile.ZipFile, BytesIO]) -> None: if not valdiate_basic_epc_structure(epc): - logging.warning("EPC structure validation failed. Attempting auto-repair.") + logger.warning("EPC structure validation failed. Attempting auto-repair.") create_mandatory_structure_epc(epc) @@ -547,7 +549,7 @@ def get_property_kind_by_uuid(uuid: str) -> Optional[Any]: try: update_prop_kind_dict_cache() except FileNotFoundError as e: - logging.error(f"Failed to parse propertykind dict {e}") + logger.error(f"Failed to parse propertykind dict {e}") return __CACHE_PROP_KIND_DICT__.get(uuid, None) def get_property_kind_by_title(title: str) -> Optional[Any]: @@ -565,7 +567,7 @@ def get_property_kind_by_title(title: str) -> Optional[Any]: try: update_prop_kind_dict_cache() except FileNotFoundError as e: - logging.error(f"Failed to parse propertykind dict {e}") + logger.error(f"Failed to parse propertykind dict {e}") title_reshaped = title.replace(" ", "_").lower() for prop in __CACHE_PROP_KIND_DICT__.values(): pk_title_reshaped = prop.citation.title.replace(" ", "_").lower() if prop.citation and prop.citation.title else "" @@ -593,7 +595,7 @@ def get_property_kind_and_parents(uuids: list) -> Dict[str, Any]: if parent_uuid is not None and parent_uuid not in dict_props: dict_props = get_property_kind_and_parents([parent_uuid]) | dict_props else: - logging.warning(f"PropertyKind with UUID {prop_uuid} not found.") + logger.warning(f"PropertyKind with UUID {prop_uuid} not found.") continue return dict_props @@ -637,7 +639,7 @@ def as_dor(obj_or_identifier: Union[str, Uri, Any], dor_qualified_type: str = "e parsed_uri = obj_or_identifier if isinstance(obj_or_identifier, Uri) else parse_uri(obj_or_identifier) if parsed_uri is not None: # From URI - logging.debug(f"====> parsed uri {parsed_uri} : uuid is {parsed_uri.uuid}") + logger.debug(f"====> parsed uri {parsed_uri} : uuid is {parsed_uri.uuid}") dor_uuid = parsed_uri.uuid dor_version = parsed_uri.version dor_qualified_type_str = parsed_uri.get_qualified_type() @@ -648,7 +650,7 @@ def as_dor(obj_or_identifier: Union[str, Uri, Any], dor_qualified_type: str = "e try: update_prop_kind_dict_cache() except FileNotFoundError as e: - logging.error(f"Failed to parse propertykind dict {e}") + logger.error(f"Failed to parse propertykind dict {e}") try: uuid, version = split_identifier(obj_or_identifier) if uuid in __CACHE_PROP_KIND_DICT__: @@ -657,7 +659,7 @@ def as_dor(obj_or_identifier: Union[str, Uri, Any], dor_qualified_type: str = "e dor_uuid = uuid dor_version = version except AttributeError: - logging.error(f"Failed to parse identifier {obj_or_identifier}. DOR will be empty") + logger.error(f"Failed to parse identifier {obj_or_identifier}. DOR will be empty") else: if is_dor(obj_or_identifier): # DOR conversion @@ -690,12 +692,12 @@ def as_dor(obj_or_identifier: Union[str, Uri, Any], dor_qualified_type: str = "e try: dor_qualified_type_str = get_qualified_type_from_class(obj_or_identifier) except Exception as e: - logging.error(f"Failed to set qualified_type for DOR {e}") + logger.error(f"Failed to set qualified_type for DOR {e}") try: dor_content_type_str = get_content_type_from_class(obj_or_identifier) except Exception as e: - logging.error(f"Failed to set content_type for DOR {e}") + logger.error(f"Failed to set content_type for DOR {e}") dor_title = get_object_attribute(obj_or_identifier, "Citation.Title") dor_uuid = get_obj_uuid(obj_or_identifier) @@ -841,9 +843,9 @@ def get_dor_uris_from_obj(obj: Any) -> Set[Uri]: if uri and uri.is_object_uri(): uri_set.add(uri) except Exception as e: - logging.warning(f"Failed to extract uri from DOR: {e}") + logger.warning(f"Failed to extract uri from DOR: {e}") except Exception as e: - logging.warning(f"Failed to get DOR list from object: {e}") + logger.warning(f"Failed to get DOR list from object: {e}") return uri_set @@ -906,7 +908,7 @@ def get_dor_or_external_uris_from_obj(obj: Any) -> Tuple[Set[Uri], Set[Tuple[str if uri and uri.is_object_uri(): dor_uris.add(uri) except Exception as e: - logging.warning(f"Failed to extract uri from DOR: {e}") + logger.warning(f"Failed to extract uri from DOR: {e}") else: # External reference case (e.g. ExternalDataArrayPart) try: @@ -915,9 +917,9 @@ def get_dor_or_external_uris_from_obj(obj: Any) -> Tuple[Set[Uri], Set[Tuple[str if ext_uri: external_uris.add((ext_uri, ext_mime_type)) except Exception as e: - logging.warning(f"Failed to extract uri from external reference: {e}") + logger.warning(f"Failed to extract uri from external reference: {e}") except Exception as e: - logging.warning(f"Failed to get DOR list from object: {e}") + logger.warning(f"Failed to get DOR list from object: {e}") return dor_uris, external_uris @@ -937,3 +939,47 @@ def get_file_folder_and_name_from_path(path: str) -> Tuple[str, str]: obj_folder = path[: path.rindex("/") + 1] if "/" in path else "" obj_file_name = path[path.rindex("/") + 1 :] if "/" in path else path return obj_folder, obj_file_name + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "EXPANDED_EXPORT_FOLDER_PREFIX", + "PATH_VERSION_PREFIX", + "gen_core_props_rels_path", + "is_core_prop_or_extension_path", + "gen_core_props_path", + "gen_energyml_object_path", + "gen_rels_path", + "gen_rels_path_from_obj_path", + "get_epc_content_type_path", + "get_epc_content_type_rels_path", + "extract_uuid_and_version_from_obj_path", + "in_epc_file_path_to_mime_type", + "get_file_folder", + "make_path_relative_to_other_file", + "make_path_relative_to_filepath_list", + "as_identifier", + "create_external_relationship", + "create_h5_external_relationship", + "relationships_equal", + "create_default_core_properties", + "create_default_types", + "match_external_proxy_type", + "get_rels_dor_type", + "valdiate_basic_epc_structure", + "create_mandatory_structure_epc", + "repair_epc_structure_if_not_valid", + "update_prop_kind_dict_cache", + "get_property_kind_by_uuid", + "get_property_kind_by_title", + "get_property_kind_and_parents", + "get_property_kind_uuid_from_property_object", + "as_dor", + "create_energyml_object", + "create_external_part_reference", + "get_reverse_dor_list", + "get_dor_uris_from_obj", + "get_dor_or_external_uris_from_obj", + "get_file_folder_and_name_from_path", +] diff --git a/energyml-utils/src/energyml/utils/epc_validator.py b/energyml-utils/src/energyml/utils/epc_validator.py index cae06d3..3d5e3c5 100644 --- a/energyml-utils/src/energyml/utils/epc_validator.py +++ b/energyml-utils/src/energyml/utils/epc_validator.py @@ -616,3 +616,13 @@ def validate_epc_file( """ validator = EpcValidator(epc_path, strict=strict, check_relationships=check_relationships) return validator.validate() + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "ValidationResult", + "EpcParser", + "EpcValidator", + "validate_epc_file", +] diff --git a/energyml-utils/src/energyml/utils/exception.py b/energyml-utils/src/energyml/utils/exception.py index a3cfe72..7dcee16 100644 --- a/energyml-utils/src/energyml/utils/exception.py +++ b/energyml-utils/src/energyml/utils/exception.py @@ -125,3 +125,27 @@ class CorePropertiesValidationError(EpcValidationError): class NotUriError(Exception): def __init__(self, uri: Optional[str] = None): super().__init__(f"Not a valid URI: {uri}") + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "DetailedNotImplementedError", + "MissingExtraInstallation", + "NoCrsError", + "ObjectNotFoundNotError", + "UnknownTypeFromQualifiedType", + "NotParsableType", + "UnparsableFile", + "NotSupportedError", + "NotEnoughInformationError", + "EpcValidationError", + "ZipIntegrityError", + "MissingRequiredFileError", + "InvalidXmlStructureError", + "RelationshipValidationError", + "NamingConventionError", + "ContentTypeValidationError", + "CorePropertiesValidationError", + "NotUriError", +] diff --git a/energyml-utils/src/energyml/utils/introspection.py b/energyml-utils/src/energyml/utils/introspection.py index e18d5a6..a3e05f4 100644 --- a/energyml-utils/src/energyml/utils/introspection.py +++ b/energyml-utils/src/energyml/utils/introspection.py @@ -14,6 +14,11 @@ from enum import Enum from importlib import import_module from types import ModuleType + +try: + from types import UnionType # 'A | B' syntax, python >= 3.10 +except ImportError: + UnionType = () # isinstance(x, ()) is always False from typing import Any, List, Optional, Union, Dict, Tuple from energyml.utils.constants import ( @@ -41,6 +46,20 @@ from energyml.utils.uri import Uri, parse_uri from energyml.utils.constants import parse_content_type, ENERGYML_NAMESPACES, parse_qualified_type +logger = logging.getLogger(__name__) + + +def is_union_type(cls: Any) -> bool: + """ + Returns True if :param:`cls` is a union type : 'typing.Union[A, B]', 'typing.Optional[A]' or 'A | B'. + + Note: do not use 'isinstance(cls, typing.Union.__class__)' for that : since python 3.14, 'typing.Union' is + 'types.UnionType', thus 'typing.Union.__class__' is 'type' and the test is True for every class. + :param cls: + :return: bool + """ + return typing.get_origin(cls) is Union or isinstance(cls, UnionType) + def is_enum(cls: Union[type, Any]): """ @@ -151,7 +170,7 @@ def find_class_in_module(module_name: str, class_name: str): if cls: return cls - logging.error(f"Not Found : {module_name}; {class_name}") + logger.error(f"Not Found : {module_name}; {class_name}") return None @@ -198,7 +217,7 @@ def search_class_in_module_from_partial_name(module_name: str, class_partial_nam return matching_classes except Exception as e: - logging.error(f"Error searching in module '{module_name}': {e}") + logger.error(f"Error searching in module '{module_name}': {e}") return None @@ -257,7 +276,7 @@ def get_class_from_name(class_name_and_module: str) -> Optional[type]: return find_class_in_module(module_name, last_ns_part) except AttributeError as e: # if "2d" in last_ns_part: - # logging.debug("replace 2D") + # logger.debug("replace 2D") # return get_class_from_name( # class_name_and_module.replace("2d", "2D") # ) @@ -271,7 +290,7 @@ def get_class_from_name(class_name_and_module: str) -> Optional[type]: # ) # elif "2D" in last_ns_part or "3D" in last_ns_part: # idx = -1 - # logging.debug(class_name_and_module) + # logger.debug(class_name_and_module) # try: # idx = class_name_and_module.rindex("2D") + 2 # except: @@ -282,13 +301,13 @@ def get_class_from_name(class_name_and_module: str) -> Optional[type]: # + class_name_and_module[idx].lower() # + class_name_and_module[idx + 1:] # ) - # logging.debug(f"reformated {reformated}") + # logger.debug(f"reformated {reformated}") # return get_class_from_name(reformated) # else: - # logging.debug(e) - logging.error(e) + # logger.debug(e) + logger.error(e) except KeyError: - logging.error(f"[ERR] module not found : '{module_name}'") + logger.error(f"[ERR] module not found : '{module_name}'") return None @@ -304,8 +323,8 @@ def get_energyml_class_in_related_dev_pkg(cls: type): try: res.append(get_class_from_name(f"{dev_module_name}.{class_name}")) except Exception as e: - logging.error(f"FAILED {dev_module_name}.{class_name}") - logging.error(e) + logger.error(f"FAILED {dev_module_name}.{class_name}") + logger.error(e) pass return res @@ -319,10 +338,10 @@ def get_energyml_module_dev_version(pkg: str, current_version: str): current_version = current_version.replace("-", "_").replace(".", "_") res = [] if pkg in accessible_modules: - # logging.debug("\t", pkg, current_version) + # logger.debug("\t", pkg, current_version) for am_pkg_version in accessible_modules[pkg]: if am_pkg_version != current_version and am_pkg_version.startswith(current_version): - # logging.debug("\t\t", am_pkg_version) + # logger.debug("\t\t", am_pkg_version) res.append(get_module_name(pkg, am_pkg_version)) return res @@ -347,7 +366,7 @@ def get_module_name_and_type_from_content_or_qualified_type(cqt: str) -> Tuple[s domain = ct.group("domain") if domain is None: - # logging.debug(f"\tdomain {domain} xmlDomain {ct.group('xmlDomain')} ") + # logger.debug(f"\tdomain {domain} xmlDomain {ct.group('xmlDomain')} ") domain = "opc" if domain == "opc": @@ -413,8 +432,8 @@ def import_related_module(energyml_module_name: str) -> None: # Only log once per unique module if m not in _FAILED_IMPORT_MODULES: _FAILED_IMPORT_MODULES.add(m) - logging.debug(f"Could not import related module {m}: {e}") - # logging.error(e) + logger.debug(f"Could not import related module {m}: {e}") + # logger.error(e) def list_function_parameters_with_types(func, is_class_function: bool = False) -> Dict[str, Any]: @@ -526,14 +545,14 @@ def get_all_matching_class_attribute_name( # search regex after to avoid shadowing perfect match pattern = re.compile(attribute_name, flags=re_flags) for name, cf in class_fields.items(): - # logging.error(f"\t->{name} : {attribute_name} {pattern.match(name)} {('name' in cf.metadata and pattern.match(cf.metadata['name']))}") + # logger.error(f"\t->{name} : {attribute_name} {pattern.match(name)} {('name' in cf.metadata and pattern.match(cf.metadata['name']))}") if pattern.match(name) or ( hasattr(cf, "metadata") and "name" in cf.metadata and pattern.match(cf.metadata["name"]) ): matching_names.add(name) except Exception as e: - logging.error(f"Failed to get attribute {attribute_name} from class {cls}") - logging.error(e) + logger.error(f"Failed to get attribute {attribute_name} from class {cls}") + logger.error(e) return list(matching_names) @@ -567,7 +586,7 @@ def get_object_attribute(obj: Any, attr_dot_path: str, force_snake_case=True) -> current_attrib_name, path_next = path_next_attribute(attr_dot_path) if current_attrib_name is None: - logging.error(f"Attribute path '{attr_dot_path}' is invalid.") + logger.error(f"Attribute path '{attr_dot_path}' is invalid.") return None value = None @@ -603,7 +622,7 @@ def create_default_value_for_type(cls: Any): return False elif is_enum(cls): return cls[cls._member_names_[random.randint(0, len(cls._member_names_) - 1)]] - elif isinstance(cls, typing.Union.__class__): + elif is_union_type(cls): type_list = list(cls.__args__) if type(None) in type_list: type_list.remove(type(None)) # we don't want to generate none value @@ -662,7 +681,7 @@ def get_object_attribute_or_create( current_attrib_name, path_next = path_next_attribute(attr_dot_path) if current_attrib_name is None: - logging.error(f"Attribute path '{attr_dot_path}' is invalid.") + logger.error(f"Attribute path '{attr_dot_path}' is invalid.") return None if force_snake_case: @@ -692,30 +711,49 @@ def get_object_attribute_or_create( def get_object_attribute_advanced(obj: Any, attr_dot_path: str) -> Any: """ see @get_matching_class_attribute_name and @get_object_attribute - """ - current_attrib_name = attr_dot_path - if "." in attr_dot_path: - current_attrib_name = attr_dot_path.split(".")[0] + Unlike :func:`get_object_attribute`, the class attributes are matched loosely + (``LinePatch`` finds ``line_patch``), which is what makes it usable with the paths + :func:`search_attribute_matching_name_with_path` returns. - current_attrib_name = get_matching_class_attribute_name(obj, current_attrib_name) + Two things it used to get wrong, both silent: + + - a **list index** was handed to :func:`get_matching_class_attribute_name`, which of course + never matches a digit, so the whole path was declared invalid: ``line_patch.0.geometry`` + logged ``Attribute path '0.geometry' is invalid`` and returned ``None``. Every RESQML + 2.0.1 external array therefore lost the element count read from its parent patch + (``read_external_array``), and the HDF5 dataset was read whole. + - the remaining path was cut with the length of the **matched** attribute name rather than + the one written in the path, so any component whose spelling differs in length from the + python attribute (``LinePatch`` → ``line_patch``) shifted the rest of the path. + """ + current_attrib_name, path_next = path_next_attribute(attr_dot_path) if current_attrib_name is None: - logging.error(f"Attribute path '{attr_dot_path}' is invalid.") + logger.error(f"Attribute path '{attr_dot_path}' is invalid.") return None - value = None if isinstance(obj, list): - value = obj[int(current_attrib_name)] + try: + value = obj[int(current_attrib_name)] + except (ValueError, IndexError): + logger.error(f"Attribute path '{attr_dot_path}' is invalid (not an index of the list).") + return None elif isinstance(obj, dict): + if current_attrib_name not in obj: + logger.error(f"Attribute path '{attr_dot_path}' is invalid (not a key of the dict).") + return None value = obj[current_attrib_name] else: - value = getattr(obj, current_attrib_name) + matched_name = get_matching_class_attribute_name(obj, current_attrib_name) + if matched_name is None: + logger.error(f"Attribute path '{attr_dot_path}' is invalid.") + return None + value = getattr(obj, matched_name) - if "." in attr_dot_path: - return get_object_attribute_advanced(value, attr_dot_path[len(current_attrib_name) + 1 :]) - else: - return value + if path_next is not None: + return get_object_attribute_advanced(value, path_next) + return value def get_object_attribute_no_verif(obj: Any, attr_name: str, default: Optional[Any] = None) -> Any: @@ -1051,7 +1089,7 @@ def search_attribute_matching_name_with_path( # next_match = ".".join(attrib_list[1:]) current_match, next_match = path_next_attribute(name_rgx) if current_match is None: - # logging.error(f"Attribute name regex '{name_rgx}' is invalid.") + # logger.error(f"Attribute name regex '{name_rgx}' is invalid.") return [] res = [] @@ -1082,17 +1120,17 @@ def search_attribute_matching_name_with_path( not_match_path_and_obj.append((f"{current_path}{k}", s_o)) elif not is_primitive(obj): current_match = current_match.replace("\\.", ".") - # logging.debug(f"searching {current_match} in {type(obj)} with path {current_path} and next match {next_match}") + # logger.debug(f"searching {current_match} in {type(obj)} with path {current_path} and next match {next_match}") match_values = get_all_matching_class_attribute_name(obj, current_match, re_flags) for match_value in match_values: - # logging.debug(f"\tmatch found : {match_value}") + # logger.debug(f"\tmatch found : {match_value}") match_path_and_obj.append( ( f"{current_path}{match_value}", get_object_attribute_no_verif(obj, match_value), ) ) - # logging.debug("f------") + # logger.debug("f------") for att_name in get_class_attributes(obj): if att_name not in match_values: not_match_path_and_obj.append( @@ -1101,7 +1139,7 @@ def search_attribute_matching_name_with_path( get_object_attribute_no_verif(obj, att_name), ) ) - # logging.debug(f"\tmatch_path_and_obj: {match_path_and_obj}") + # logger.debug(f"\tmatch_path_and_obj: {match_path_and_obj}") for matched_path, matched in match_path_and_obj: if next_match is not None: # next_match is different, match is not final @@ -1213,7 +1251,7 @@ def set_attribute_from_path(obj: Any, attribute_path: str, value: Any) -> None: current_attrib_name, path_next = path_next_attribute(attribute_path) if current_attrib_name is None: - logging.error(f"Attribute path '{attribute_path}' is invalid.") + logger.error(f"Attribute path '{attribute_path}' is invalid.") return if path_next is not None: @@ -1323,7 +1361,7 @@ def get_obj_version(obj: Any) -> Optional[str]: ) except AttributeError: # Log with full call stack to see WHO called this function - # logging.error( + # logger.error( # f"Error getting version for {type(obj)} -- {obj}", # exc_info=True, # stack_info=True, # This shows the full call stack including caller @@ -1377,6 +1415,67 @@ def get_obj_title(obj: Any) -> Optional[str]: return None +#: Citation sub-attributes exported by :func:`get_object_metadata`, mapped to the metadata key. +_CITATION_ATTRIBUTES_ = { + "title": "title", + "originator": "originator", + "creation": "creation", + "last_update": "last_update", + "editor": "editor", + "format": "format", + "description": "description", +} + + +def get_object_metadata(obj: Any) -> Dict[str, Any]: + """ + Extract the identification metadata of an energyml object as a plain ``dict``. + + Collected values (absent / empty ones are omitted) : + - ``uuid``, ``object_version`` + - ``qualified_type`` (e.g. ``resqml22.TriangulatedSetRepresentation``), ``content_type`` + - ``uri`` (ETP URI) + - the ``Citation`` fields : ``title``, ``originator``, ``creation``, ``last_update``, + ``editor``, ``format``, ``description``. Dates are returned as ISO 8601 strings. + + Never raises : an unreadable attribute is simply skipped. + + :param obj: an energyml data object + :return: a JSON-serializable dict + """ + metadata: Dict[str, Any] = {} + if obj is None: + return metadata + + for key, getter in ( + ("uuid", lambda: get_obj_uuid(obj)), + ("object_version", lambda: get_obj_version(obj)), + ("qualified_type", lambda: get_qualified_type_from_class(obj)), + ("content_type", lambda: get_content_type_from_class(obj)), + ("uri", lambda: str(get_object_uri(obj))), + ): + try: + value = getter() + except Exception: # a partially filled object must not break the export + continue + if value is not None and str(value) != "": + metadata[key] = value if isinstance(value, (int, float, bool)) else str(value) + + citation = get_object_attribute_no_verif(obj, "citation", default=None) + if citation is not None: + for attribute_name, key in _CITATION_ATTRIBUTES_.items(): + try: + value = get_object_attribute_no_verif(citation, attribute_name, default=None) + except Exception: + continue + if value is None or str(value) == "": + continue + # XmlDateTime / XmlDate values serialize to ISO 8601 through str() + metadata[key] = value if isinstance(value, (int, float, bool)) else str(value) + + return metadata + + def get_obj_pkg_pkgv_type_uuid_version( obj: Any, ) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str], Optional[str]]: @@ -1409,7 +1508,7 @@ def get_obj_pkg_pkgv_type_uuid_version( ct = get_object_attribute_no_verif(obj, "ContentType") except: pass - + if ct is not None: ct_match = parse_content_type(ct) if ct_match is not None: @@ -1425,7 +1524,7 @@ def get_obj_pkg_pkgv_type_uuid_version( qt = get_object_attribute_no_verif(obj, "QualifiedType") except: pass - + if qt is not None: qt_match = parse_qualified_type(qt) if qt_match is not None: @@ -1533,7 +1632,7 @@ def as_obj_prefixed_class_if_possible(o: Any) -> Any: if o is not None: if not isinstance(o, type): o_type = type(o) - # logging.info( + # logger.info( # f"Trying to convert object of type {o_type.__module__} -- {o_type.__name__} to obj prefixed class : {o_type.__name__.lower().startswith('obj')}" # ) if o_type.__name__.lower().startswith("obj"): @@ -1542,21 +1641,21 @@ def as_obj_prefixed_class_if_possible(o: Any) -> Any: try: sub_name = str(o_type.__name__).replace(o_type.__name__, o_type.__name__[3:]) sub_class_name = f"{o_type.__module__}.{sub_name}" - # logging.info(f"\n\nSearching subclass {sub_class_name} for {o_type}") + # logger.info(f"\n\nSearching subclass {sub_class_name} for {o_type}") sub = get_class_from_name(sub_class_name) - # logging.info(f"Found subclass {sub} for {sub}") + # logger.info(f"Found subclass {sub} for {sub}") if sub is not None and issubclass(sub, o_type): try: try: if sub.Meta is not None: o_type.Meta.namespace = sub.Meta.namespace # keep the same namespace except Exception: - logging.debug(f"Failed to set namespace for {sub}") + logger.debug(f"Failed to set namespace for {sub}") except Exception as e: - # logging.debug(f"Failed to convert {o} to {sub}") - logging.debug(e) + # logger.debug(f"Failed to convert {o} to {sub}") + logger.debug(e) except Exception: - logging.debug(f"Error using Meta class for {o_type}") + logger.debug(f"Error using Meta class for {o_type}") return o if o_type.__bases__ is not None: for bc in o_type.__bases__: @@ -1567,11 +1666,11 @@ def as_obj_prefixed_class_if_possible(o: Any) -> Any: if bc.Meta is not None: bc.Meta.namespace = o_type.Meta.namespace # keep the same namespace except Exception: - logging.error(f"Failed to set namespace for {bc}") + logger.error(f"Failed to set namespace for {bc}") return bc(**o.__dict__) except Exception as e: - logging.error(f"Failed to convert {o} to {bc}") - logging.error(e) + logger.error(f"Failed to convert {o} to {bc}") + logger.error(e) return o return o return None @@ -1605,12 +1704,12 @@ def dor_to_uris(dor: Any, dataspace: Optional[str] = None) -> Optional[Uri]: value = get_object_attribute_no_verif(dor, "qualified_type") result = parse_qualified_type(value) except Exception as e: - logging.error(e) + logger.error(e) try: value = get_object_attribute_no_verif(dor, "content_type") result = parse_content_type(value) except Exception as e2: - logging.error(e2) + logger.error(e2) if result is None: return None @@ -1642,7 +1741,7 @@ def get_content_type_from_class(cls: Union[type, Any], print_dev_version=True, n + get_object_type_for_file_path_from_class(cls) ) - logging.error(f"@get_content_type_from_class not supported type : {cls}") + logger.error(f"@get_content_type_from_class not supported type : {cls}") return None @@ -1725,11 +1824,11 @@ def get_obj_attribute_class( return get_obj_attribute_class(chosen_type, None, random_for_typing) elif cls is not None: - if isinstance(cls, typing.Union.__class__): + if is_union_type(cls): type_list = list(cls.__args__) if type(None) in type_list: type_list.remove(type(None)) # we don't want to generate none value - chosen_type = type_list[random.randint(0, len(type_list))] + chosen_type = type_list[random.randint(0, len(type_list) - 1)] elif cls.__module__ == "typing": type_list = list(cls.__args__) if type(None) in type_list: @@ -1761,14 +1860,25 @@ def get_class_from_simple_name(simple_name: str, energyml_module_context=None) - try: return eval(simple_name) except NameError: + # Note: the imported names must be stored in an explicit namespace shared by the 'exec' and the 'eval' : + # since python 3.13 (PEP 667), 'locals()' returns an independent snapshot inside a function, so what + # 'exec' defines is not visible from the following 'eval'. + namespace = { + "List": List, + "Optional": Optional, + "Union": Union, + "Dict": Dict, + "Tuple": Tuple, + "Any": Any, + } for mod in energyml_module_context: try: - exec(f"from {mod} import *") + exec(f"from {mod} import *", namespace) # required to be able to access to type in # typing values like "List[ObjectAlias]" except ModuleNotFoundError: pass - return eval(simple_name) + return eval(simple_name, namespace) def _gen_str_from_attribute_name(attribute_name: Optional[str], _parent_class: Optional[type] = None) -> str: @@ -1837,6 +1947,21 @@ def random_value_from_class(cls: type): return None +def get_non_abstract_classes(cls: type, include_self: bool = True) -> List[type]: + """ + List all non abstract classes that can be instanciated for the type :param:`cls` : the class itself (if it is + not abstract and :param:`include_self` is True) and all its sub classes (recursively). + + Example : get_non_abstract_classes(energyml.eml.v2_3.commonv2.AbstractObject) + + :param cls: the (potentially abstract) class + :param include_self: if False, :param:`cls` is never returned, even if it is not abstract + :return: a list of non abstract classes (may be empty) + """ + potential_classes = ([cls] if include_self else []) + get_sub_classes(cls) + return list(dict.fromkeys(filter(lambda _c: not is_abstract(_c), potential_classes))) + + def get_all_possible_instanciable_classes( classes: Union[type, List[Any]], energyml_module_context: List[str] ) -> List[type]: @@ -1884,8 +2009,8 @@ def get_all_possible_instanciable_classes_for_attribute(parent_obj: Any, attribu else: if attribute_name is not None and len(attribute_name) > 0: ctx = get_related_energyml_modules_name(parent_obj) - # logging.debug(get_class_fields(cls)[attribute_name]) - # logging.debug(get_class_fields(cls)[attribute_name].type) + # logger.debug(get_class_fields(cls)[attribute_name]) + # logger.debug(get_class_fields(cls)[attribute_name].type) sub_cls = get_class_from_simple_name( simple_name=get_class_fields(cls)[attribute_name].type, energyml_module_context=ctx, @@ -1915,6 +2040,13 @@ def _random_value_from_class( """ try: + if cls is object or cls is Any: + # An `xs:any` / `Any` field carries no schema, so there is no meaningful value to + # invent for it. Instantiating a bare `object()` — what this used to do — produced an + # object with no `__module__` matching an energyml package, and every serializer then + # died on it: `generate_data` crashed with + # `AttributeError: 'object' object has no attribute '__module__'` for *every* type. + return None if isinstance(cls, str) or cls == str: return _gen_str_from_attribute_name(attribute_name, _parent_class) elif isinstance(cls, int) or cls == int: @@ -1925,12 +2057,14 @@ def _random_value_from_class( return random.randint(0, 1) == 1 elif is_enum(cls): return cls[cls._member_names_[random.randint(0, len(cls._member_names_) - 1)]] - elif isinstance(cls, typing.Union.__class__): + elif is_union_type(cls): type_list = list(cls.__args__) if type(None) in type_list: type_list.remove(type(None)) # we don't want to generate none value - chosen_type = type_list[random.randint(0, len(type_list))] - return _random_value_from_class(chosen_type, energyml_module_context, attribute_name, cls) + chosen_type = type_list[random.randint(0, len(type_list) - 1)] + # '_parent_class' must stay the real parent class (not the union alias) : it is used to generate + # coherent values for attributes like 'title', 'qualified_type' or 'schema_version' + return _random_value_from_class(chosen_type, energyml_module_context, attribute_name, _parent_class) elif cls.__module__ == "typing": type_list = list(cls.__args__) if type(None) in type_list: @@ -1969,7 +2103,7 @@ def _random_value_from_class( chosen_type = potential_classes[random.randint(0, len(potential_classes) - 1)] args = {} for k, v in get_class_fields(chosen_type).items(): - # logging.debug(f"get_class_fields {k} : {v}, { isinstance(v, type)}, {v}") + # logger.debug(f"get_class_fields {k} : {v}, { isinstance(v, type)}, {v}") args[k] = _random_value_from_class( cls=( get_class_from_simple_name( @@ -1990,8 +2124,84 @@ def _random_value_from_class( return chosen_type(**args) except Exception as e: - logging.error(f"exception on attribute '{attribute_name}' for class {cls} :") + logger.error(f"exception on attribute '{attribute_name}' for class {cls} :") raise e - logging.error(f"@_random_value_from_class Not supported object type generation {cls}") + logger.error(f"@_random_value_from_class Not supported object type generation {cls}") return None + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "UnionType", + "is_union_type", + "is_enum", + "is_primitive", + "is_abstract", + "get_module_classes_from_name", + "get_module_metadata_map", + "find_class_in_module", + "get_module_classes", + "search_class_in_module_from_partial_name", + "get_class_methods", + "get_class_from_name", + "get_energyml_class_in_related_dev_pkg", + "get_energyml_module_dev_version", + "get_module_name_and_type_from_content_or_qualified_type", + "get_class_from_qualified_type", + "get_class_from_content_type", + "get_module_name", + "import_related_module", + "list_function_parameters_with_types", + "get_class_fields", + "get_class_attributes", + "get_class_attribute_type", + "get_all_matching_class_attribute_name", + "get_matching_class_attribute_name", + "get_object_attribute", + "create_default_value_for_type", + "get_object_attribute_or_create", + "get_object_attribute_advanced", + "get_object_attribute_no_verif", + "get_object_attribute_rgx", + "get_obj_type", + "class_match_rgx", + "get_dor_obj_info", + "is_dor", + "search_attribute_matching_type_with_path", + "search_attribute_in_upper_matching_name", + "search_attribute_matching_type", + "search_attribute_matching_name_with_path", + "search_attribute_matching_name", + "set_attribute_from_json_str", + "set_attribute_from_dict", + "set_attribute_from_path", + "set_attribute_value", + "copy_attributes", + "get_obj_uuid", + "get_obj_version", + "get_obj_title", + "get_object_metadata", + "get_obj_pkg_pkgv_type_uuid_version", + "get_obj_qualified_type", + "get_obj_content_type", + "get_obj_identifier", + "get_obj_uri", + "get_direct_dor_list", + "get_obj_usable_class", + "as_obj_prefixed_class_if_possible", + "get_data_object_type", + "get_qualified_type_from_class", + "get_object_uri", + "dor_to_uris", + "get_content_type_from_class", + "get_object_type_for_file_path_from_class", + "get_obj_attribute_class", + "get_class_from_simple_name", + "random_value_from_class", + "get_non_abstract_classes", + "get_all_possible_instanciable_classes", + "get_all_possible_instanciable_classes_for_attribute", + "get_enum_values", +] diff --git a/energyml-utils/src/energyml/utils/manager.py b/energyml-utils/src/energyml/utils/manager.py index 34a18cb..33cec2b 100644 --- a/energyml-utils/src/energyml/utils/manager.py +++ b/energyml-utils/src/energyml/utils/manager.py @@ -15,6 +15,8 @@ RGX_PROJECT_VERSION, ) +logger = logging.getLogger(__name__) + def get_related_energyml_modules_name(cls: Union[type, Any]) -> List[str]: """ @@ -41,10 +43,10 @@ def dict_energyml_modules() -> Dict: modules = {} energyml_module = importlib.import_module("energyml") - # logging.debug("> energyml") + # logger.debug("> energyml") for mod in pkgutil.iter_modules(energyml_module.__path__): - # logging.debug(f"{mod.name}") + # logger.debug(f"{mod.name}") if mod.name in ENERGYML_MODULES_NAMES: energyml_sub_module = importlib.import_module(f"energyml.{mod.name}") if mod.name not in modules: @@ -61,7 +63,7 @@ def list_energyml_modules() -> List: energyml_module = importlib.import_module("energyml") modules = [] for obj in pkgutil.iter_modules(energyml_module.__path__): - # logging.debug(f"{obj.name}") + # logger.debug(f"{obj.name}") if obj.name in ENERGYML_MODULES_NAMES: modules.append(obj.name) return modules @@ -83,7 +85,7 @@ def list_classes(module_path: str) -> List: class_list.append(obj) return class_list except ModuleNotFoundError: - logging.error(f"Err : module {module_path} not found") + logger.error(f"Err : module {module_path} not found") return [] @@ -183,8 +185,8 @@ def get_class_pkg(cls): match = p.search(cls.__module__) return match.group("pkg") # type: ignore except AttributeError as e: - logging.debug(f"Exception to get class package for '{cls}'") - logging.debug( + logger.debug(f"Exception to get class package for '{cls}'") + logger.debug( f"Error getting package for {type(cls)} -- {cls}", exc_info=True, stack_info=True, # This shows the full call stack including caller @@ -265,3 +267,22 @@ def get_class_pkg_version(cls, print_dev_version: bool = True, nb_max_version_di # except Exception: # pass # return protocolDict + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "get_related_energyml_modules_name", + "dict_energyml_modules", + "list_energyml_modules", + "list_classes", + "get_sub_classes", + "class_has_parent_with_name", + "get_classes_matching_name", + "get_all_energyml_classes", + "get_all_classes", + "get_class_pkg", + "reshape_version", + "reshape_version_from_regex_match", + "get_class_pkg_version", +] diff --git a/energyml-utils/src/energyml/utils/rc/__init__.py b/energyml-utils/src/energyml/utils/rc/__init__.py index e69de29..3ed0881 100644 --- a/energyml-utils/src/energyml/utils/rc/__init__.py +++ b/energyml-utils/src/energyml/utils/rc/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 diff --git a/energyml-utils/src/energyml/utils/serialization.py b/energyml-utils/src/energyml/utils/serialization.py index 816bb00..1c0a505 100644 --- a/energyml-utils/src/energyml/utils/serialization.py +++ b/energyml-utils/src/energyml/utils/serialization.py @@ -44,6 +44,8 @@ from xsdata.formats.dataclass.parsers.handlers import LxmlEventHandler +logger = logging.getLogger(__name__) + GLOBAL_XML_CONTEXT = XmlContext( # element_name_generator=text.camel_case, # attribute_name_generator=text.kebab_case @@ -56,9 +58,23 @@ class JSON_VERSION(Enum): class FallbackNamespaceXmlParser(XmlParser): """ - Custom XML parser that injects fallback namespaces + Custom XML parser that injects fallback namespaces before xsdata attempts to resolve xsi:type types. + + The fallback lets a prefixed ``xsi:type`` resolve even when the document forgot to declare + the prefix (``xsi:type="eml:VerticalCrsEpsgCode"`` without ``xmlns:eml``). + + The *unprefixed* case must keep working too, and it is the common one: an + ``xsi:type="VerticalCrsEpsgCode"`` resolves against the **default** namespace of the + document, which energyml files usually set to ``commonv2`` — exactly where that type is + declared. :meth:`xsdata.formats.converter.QNameConverter.resolve` reads the default + namespace as ``ns_map[None]`` (``text.split`` returns a ``None`` prefix when the value holds + no colon), so the ``None`` key must be preserved. Rewriting it to ``""`` silently dropped + the type: the element was then built as its abstract base and every child was reported as an + unknown property (a v2.0.1 ``LocalDepth3dCrs`` lost both of its EPSG codes, which is enough + to make the WGS84 reprojection impossible). """ + def __init__(self, fallback_namespaces: dict[str, str], *args, **kwargs): super().__init__(*args, **kwargs) self.fallback_namespaces = fallback_namespaces @@ -66,15 +82,18 @@ def __init__(self, fallback_namespaces: dict[str, str], *args, **kwargs): def start(self, clazz: Any, queue: list, objects: list, qname: str, attrs: dict, ns_map: dict): # 1. Prepare a new dictionary including our fallback namespaces merged_ns = dict(self.fallback_namespaces) - + # 2. Update it with the namespaces actually found in the document # (Document namespaces always take precedence) if ns_map: for prefix, uri in ns_map.items(): - # lxml uses 'None' for the default namespace, xsdata prefers an empty string "" - clean_prefix = "" if prefix is None else prefix - merged_ns[clean_prefix] = uri - + merged_ns[prefix] = uri + if prefix is None or prefix == "": + # both spellings of "the default namespace" are kept: xsdata resolves an + # unprefixed qname through None, other code paths use "" + merged_ns[None] = uri + merged_ns[""] = uri + # 3. Pass exactly the 6 expected arguments to the standard xsdata logic super().start(clazz, queue, objects, qname, attrs, merged_ns) @@ -104,11 +123,11 @@ def _read_energyml_xml_bytes_as_class( try: return parser.from_bytes(file, obj_class) except ParserError as e: - logging.error(f"Failed to parse file {file} as class {obj_class}") + logger.error(f"Failed to parse file {file} as class {obj_class}") if len(e.args) > 0: if "unknown property" in e.args[0].lower(): - logging.error(e) - logging.error( + logger.error(e) + logger.error( "A property has not been found, please check if your 'xsi::type' values contains " "the xml namespace (e.g. 'xsi:type=\"eml:VerticalCrsEpsgCode\"')." ) @@ -133,18 +152,18 @@ def read_energyml_xml_bytes(file: bytes, obj_type: Optional[type] = None) -> Any except xsdata.exceptions.ParserError as e: if len(e.args) > 0: if "unknown property" in e.args[0].lower(): - logging.error("Trying reading without fail on unknown attribute/property") + logger.error("Trying reading without fail on unknown attribute/property") try: return _read_energyml_xml_bytes_as_class(file, obj_type, False, False) except Exception: - logging.error(traceback.print_stack()) + logger.error(traceback.print_stack()) pass # Otherwise for obj_type_dev in get_energyml_class_in_related_dev_pkg(obj_type): try: - logging.debug(f"Trying with class : {obj_type_dev}") + logger.debug(f"Trying with class : {obj_type_dev}") obj = _read_energyml_xml_bytes_as_class(file, obj_type_dev) - logging.debug(f" ==> succeed read with {obj_type_dev}") + logger.debug(f" ==> succeed read with {obj_type_dev}") return obj except Exception: pass @@ -189,7 +208,7 @@ def _read_energyml_json_bytes_as_class(file: bytes, json_version: JSON_VERSION, try: return parser.from_bytes(file, obj_class, ns_map=WELLKNOWN_NAMESPACES) except ParserError as e: - logging.error(f"Failed to parse file {file} as class {obj_class}") + logger.error(f"Failed to parse file {file} as class {obj_class}") raise e elif json_version == JSON_VERSION.OSDU_OFFICIAL: return read_json_dict(json.loads(file)) @@ -226,14 +245,14 @@ def read_energyml_json_bytes( try: result = result + _read_energyml_json_bytes_as_class(obj, obj_type) except xsdata.exceptions.ParserError as e: - logging.error( + logger.error( f"Failed to read file with type {obj_type}: {get_energyml_class_in_related_dev_pkg(obj_type)}" ) for obj_type_dev in get_energyml_class_in_related_dev_pkg(obj_type): try: - logging.debug(f"Trying with class : {obj_type_dev}") + logger.debug(f"Trying with class : {obj_type_dev}") obj = _read_energyml_json_bytes_as_class(obj, obj_type_dev) - logging.debug(f" ==> succeed read with {obj_type_dev}") + logger.debug(f" ==> succeed read with {obj_type_dev}") result = result + obj except Exception: pass @@ -241,8 +260,8 @@ def read_energyml_json_bytes( elif json_version == JSON_VERSION.OSDU_OFFICIAL: result = result + read_json_dict(obj) except Exception as e: - logging.error(e) - logging.error(obj) + logger.error(e) + logger.error(obj) raise e obj_type = None @@ -296,14 +315,14 @@ def read_energyml_obj(data: Union[str, bytes], format_: str = "xml") -> Any: def serialize_xml(obj, check_obj_prefixed_classes: bool = True) -> str: - # logging.debug(f"[1] Serializing object of type {type(obj)}") + # logger.debug(f"[1] Serializing object of type {type(obj)}") obj = as_obj_prefixed_class_if_possible(obj) if check_obj_prefixed_classes else obj - # logging.debug(f"[2] Serializing object of type {type(obj)}") + # logger.debug(f"[2] Serializing object of type {type(obj)}") serializer_config = SerializerConfig(indent=" ") serializer = XmlSerializer(context=GLOBAL_XML_CONTEXT, config=serializer_config) # res = serializer.render(obj) res = serializer.render(obj, ns_map=WELLKNOWN_NAMESPACES) - # logging.debug(f"[3] Serialized XML with meta namespace : {obj.Meta.namespace}: {serialize_json(obj)}") + # logger.debug(f"[3] Serialized XML with meta namespace : {obj.Meta.namespace}: {serialize_json(obj)}") return res @@ -394,11 +413,11 @@ def _read_json_dict(obj_json: Any, sub_obj: List) -> Any: _read_json_dict(val, sub_obj), ) else: - logging.error(f"No matching attribute for attribute {att} in {obj}") + logger.error(f"No matching attribute for attribute {att} in {obj}") except Exception: - logging.error(f"Error assign attribute value for attribute {att} in {obj}") + logger.error(f"Error assign attribute value for attribute {att} in {obj}") except Exception as e: - logging.error( + logger.error( f"Err on {att}", search_attribute_matching_name( obj=obj, @@ -413,7 +432,7 @@ def _read_json_dict(obj_json: Any, sub_obj: List) -> Any: elif isinstance(obj_json, list): return [_read_json_dict(o, sub_obj) for o in obj_json] elif is_primitive(obj_json): - # logging.debug(f"PRIM : {obj_json}") + # logger.debug(f"PRIM : {obj_json}") return obj_json else: raise NotParsableType(type(obj_json) + " " + obj_json) @@ -460,7 +479,7 @@ def _fill_dict_with_attribs( value = getattr(obj, att_name) if "Any_element" in str(field_name): - logging.debug(f"\t> {field_name}, {att_name} : {value}, {type(obj)}") + logger.debug(f"\t> {field_name}, {att_name} : {value}, {type(obj)}") if (value is not None or mandatory) and (not isinstance(value, list) or len(value) > 0): res[field_name] = _to_json_dict_fn(value, f_identifier_to_obj, obj) @@ -473,7 +492,7 @@ def _fill_dict_with_attribs( if ref_value is not None: res["_data"] = to_json_dict_fn(ref_value, f_identifier_to_obj) else: - # logging.debug(f"NotFound : {ref_identifier}") + # logger.debug(f"NotFound : {ref_identifier}") pass @@ -494,7 +513,7 @@ def _to_json_dict_fn( if obj is None: return None elif isinstance(obj, float) and np.isnan(obj): - print("NaN found") + logger.warning("NaN value found while serializing: it is written as null.") return None elif is_enum(obj): return obj.value @@ -518,5 +537,30 @@ def _to_json_dict_fn( _fill_dict_with_attribs(res, obj, f_identifier_to_obj, _parent) return res except Exception as e: - logging.error(f"Except on qt: {obj} - {type(obj)}") + logger.error(f"Except on qt: {obj} - {type(obj)}") raise e + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "GLOBAL_XML_CONTEXT", + "JSON_VERSION", + "FallbackNamespaceXmlParser", + "read_energyml_xml_tree", + "read_energyml_xml_bytes", + "read_energyml_xml_io", + "read_energyml_xml_str", + "read_energyml_xml_file", + "read_energyml_json_bytes", + "read_energyml_json_io", + "read_energyml_json_str", + "read_energyml_json_file", + "read_energyml_obj", + "serialize_xml", + "serialize_json", + "get_class_from_json_dict", + "read_json_dict", + "to_json_dict", + "to_json_dict_fn", +] diff --git a/energyml-utils/src/energyml/utils/uri.py b/energyml-utils/src/energyml/utils/uri.py index ffa8689..b7eed51 100644 --- a/energyml-utils/src/energyml/utils/uri.py +++ b/energyml-utils/src/energyml/utils/uri.py @@ -168,3 +168,13 @@ def create_uri_from_content_type_or_qualified_type(ct_or_qt: str, uuid: str, ver f"Failed to parse content type or qualified type: {ct_or_qt} -- {m}" ) from e raise NotUriError(f"Unable to parse content type: {ct_or_qt}") + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "Uri", + "parse_uri_raise_if_failed", + "parse_uri", + "create_uri_from_content_type_or_qualified_type", +] diff --git a/energyml-utils/src/energyml/utils/validation.py b/energyml-utils/src/energyml/utils/validation.py index 4ea509c..f3350a8 100644 --- a/energyml-utils/src/energyml/utils/validation.py +++ b/energyml-utils/src/energyml/utils/validation.py @@ -30,6 +30,8 @@ get_object_uri, ) +logger = logging.getLogger(__name__) + class ErrorType(Enum): CRITICAL = "critical" @@ -236,12 +238,12 @@ def dor_validation_object( # debug if isinstance(target, list): - # logging.error( + # logger.error( # f"Multiple objects found with uuid '{dor_uuid}' for DOR in object '{get_obj_identifier(obj)}' at path '{dor_path}'. This should not happen and can lead to wrong validation results.", # exc_info=True, # stack_info=True, # This shows the full call stack including caller # ) - # logging.error( + # logger.error( # f'\t{target} => Object ct and qt {get_object_attribute_rgx(dor, "content_type")} : {get_object_attribute_rgx(dor, "qualified_type")}' # ) if len(target) == 0: @@ -370,9 +372,9 @@ def _patterns_validation(obj: Any, root_obj: Any, current_attribute_dot_path: st for k, val in obj.items(): error_list = error_list + _patterns_validation(val, root_obj, f"{current_attribute_dot_path}.{k}") else: - # logging.debug(get_class_fields(obj)) + # logger.debug(get_class_fields(obj)) for att_name, att_field in get_class_fields(obj).items(): - # logging.debug(f"att_name : {att_field.metadata}") + # logger.debug(f"att_name : {att_field.metadata}") error_list = error_list + validate_attribute( get_object_attribute(obj, att_name, False), root_obj, @@ -492,8 +494,14 @@ def validate_attribute(value: Any, root_obj: Any, att_field: Field, path: str) - ) ) except Exception as e: - print(f"Error while validating attribute '{att_field}' with value '{value}': {str(e)} for {path}") - print(f"att_field : {att_field}, is primitive : {is_primitive(att_field)}") + logger.warning( + "Error while validating attribute '%s' with value '%s' at '%s' (primitive: %s): %s", + att_field, + value, + path, + is_primitive(att_field), + e, + ) errs.append( ValidationObjectError( _msg=f"Error while validating attribute '{att_field}' with value '{value}': {str(e)}", @@ -556,3 +564,23 @@ def correct_dor(energyml_objects: List[Any]) -> None: dor_qualified_type = get_object_attribute_no_verif(dor, "qualified_type") if dor_qualified_type != target_qualified_type: dor.qualified_type = target_qualified_type + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "ErrorType", + "ValidationError", + "ValidationObjectError", + "ValidationObjectInfo", + "MandatoryError", + "MissingEntityError", + "validate_epc", + "validate_objects", + "validate_obj", + "dor_validation_object", + "dor_validation", + "patterns_validation", + "validate_attribute", + "correct_dor", +] diff --git a/energyml-utils/src/energyml/utils/xml_utils.py b/energyml-utils/src/energyml/utils/xml_utils.py index 05e88a1..ee36e1a 100644 --- a/energyml-utils/src/energyml/utils/xml_utils.py +++ b/energyml-utils/src/energyml/utils/xml_utils.py @@ -9,6 +9,8 @@ from energyml.utils.constants import ENERGYML_NAMESPACES, ENERGYML_NAMESPACES_PACKAGE, OptimizedRegex, parse_content_type +logger = logging.getLogger(__name__) + def get_pkg_from_namespace(namespace: str) -> Optional[str]: for k, v in ENERGYML_NAMESPACES_PACKAGE.items(): @@ -31,7 +33,7 @@ def get_class_name_from_xml(tree: ETREE.Element) -> Optional[str]: root_namespace = get_root_namespace(tree) pkg = get_pkg_from_namespace(root_namespace) if pkg is None: - logging.error(f"No pkg found for elt {tree}") + logger.error(f"No pkg found for elt {tree}") return None else: if pkg == "opc": @@ -120,3 +122,20 @@ def find_schema_version_in_element(tree: ETREE.ElementTree) -> str: if match_version is not None: return match_version.group(0).replace("dev", "-dev") return None + + +#: Public API of this module. Declared explicitly so that renaming or removing anything +#: else is not a breaking change, and so `from ... import *` does not leak the imports. +__all__ = [ + "get_pkg_from_namespace", + "is_energyml_content_type", + "get_root_namespace", + "get_class_name_from_xml", + "get_xml_encoding", + "get_tree", + "energyml_xpath", + "search_element_has_child_xpath", + "get_uuid", + "get_root_type", + "find_schema_version_in_element", +] diff --git a/energyml-utils/src/energyml/utils/zip_raw.py b/energyml-utils/src/energyml/utils/zip_raw.py new file mode 100644 index 0000000..7c466d6 --- /dev/null +++ b/energyml-utils/src/energyml/utils/zip_raw.py @@ -0,0 +1,236 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +""" +Low level ZIP helpers used to rewrite an archive without paying for the +decompression/recompression of the entries that did not change. + +The :mod:`zipfile` module cannot modify an existing archive in place: updating a +single part means rebuilding the whole file. The naive way to do that +(``dst.writestr(info, src.read(name))``) inflates and re-deflates *every* entry, +which dominates the cost by an order of magnitude on a real EPC: + + rewrite of a 2.8 MB / 3360 parts EPC, decompress + recompress : 0.485 s + same rewrite, raw stream copy : 0.030 s + +:func:`rewrite_zip` copies the already-deflated bytes of the untouched entries +straight from the source file, and only compresses what actually changed. + +Implementation note: this reaches into a few ``zipfile`` attributes that are not +part of the documented API (``ZipInfo.header_offset``, ``ZipFile.fp`` and +``ZipFile.start_dir``). Every raw copy is guarded and falls back to the plain +decompress/recompress path when anything looks unexpected, so a change in the +standard library degrades performance but never correctness. + +The extra field of copied entries is dropped: it only carries optional metadata +(extended timestamps, unix uid/gid) that has no meaning in an EPC, and dropping +it avoids having to reconcile a stale Zip64 extra with the one ``zipfile`` +regenerates for large entries. +""" + +import logging +import os +import shutil +import struct +import tempfile +import warnings +import zipfile +from pathlib import Path +from typing import Dict, Iterable, Iterator, Optional, Set, Tuple, Union + +logger = logging.getLogger(__name__) + +_LOCAL_HEADER_SIGNATURE = b"PK\x03\x04" +_LOCAL_HEADER_SIZE = 30 + +__all__ = [ + "iter_effective_infos", + "rewrite_zip", + "append_to_zip", + "count_shadowed_entries", +] + + +def iter_effective_infos(zf: zipfile.ZipFile) -> Iterator[zipfile.ZipInfo]: + """ + Iterate over the entries of ``zf`` that are actually reachable by name. + + A ZIP archive may contain several entries sharing the same name (that is what + makes append-based updates possible). ``zipfile`` resolves a name to the + **last** matching entry of the central directory; this generator yields + exactly those, in file order, so that a rewrite drops the shadowed ones. + """ + for info in zf.infolist(): + if zf.NameToInfo.get(info.filename) is info: + yield info + + +def count_shadowed_entries(zf: zipfile.ZipFile) -> int: + """Number of entries kept in the archive but shadowed by a later one of the same name.""" + return len(zf.infolist()) - len(zf.NameToInfo) + + +def _read_raw_entry(src_fp, info: zipfile.ZipInfo) -> Optional[bytes]: + """ + Read the compressed payload of ``info`` straight from the source file object. + + Returns ``None`` when the local header cannot be trusted, which tells the + caller to fall back to the decompress/recompress path. + """ + try: + src_fp.seek(info.header_offset) + header = src_fp.read(_LOCAL_HEADER_SIZE) + if len(header) != _LOCAL_HEADER_SIZE or not header.startswith(_LOCAL_HEADER_SIGNATURE): + return None + name_len, extra_len = struct.unpack(" bool: + """ + Append ``info`` to ``dst`` without touching its compressed payload. + + Returns True on success, False when the caller must fall back to a regular + (decompress + recompress) copy. + """ + data = _read_raw_entry(src_fp, info) + if data is None: + return False + + try: + copy = zipfile.ZipInfo(info.filename, info.date_time) + copy.compress_type = info.compress_type + copy.comment = info.comment + copy.extra = b"" + copy.create_system = info.create_system + copy.create_version = info.create_version + copy.extract_version = info.extract_version + # bit 3 marks a trailing data descriptor: we write the real sizes in the + # local header, so it must be cleared. + copy.flag_bits = info.flag_bits & ~0x08 + copy.internal_attr = info.internal_attr + copy.external_attr = info.external_attr + copy.CRC = info.CRC + copy.compress_size = info.compress_size + copy.file_size = info.file_size + + fp = dst.fp + copy.header_offset = fp.tell() + fp.write(copy.FileHeader(zip64=None)) + fp.write(data) + + dst.filelist.append(copy) + dst.NameToInfo[copy.filename] = copy + dst.start_dir = fp.tell() + dst._didModify = True + return True + except Exception as e: # pragma: no cover - defensive + logger.debug(f"Raw copy failed for {info.filename}, falling back: {e}") + return False + + +def rewrite_zip( + source: Optional[Union[str, Path]], + target: Union[str, Path], + updates: Optional[Dict[str, bytes]] = None, + deleted: Optional[Union[Set[str], Iterable[str]]] = None, + compression: int = zipfile.ZIP_DEFLATED, + allow_raw_copy: bool = True, +) -> Tuple[int, int]: + """ + Write ``target`` from ``source`` applying ``updates`` and ``deleted``. + + Entries of ``source`` that are neither updated nor deleted are copied with + their compressed payload untouched when possible. + + :param source: archive to copy from, or None to create ``target`` from scratch + :param target: path of the archive to write; may be the same file as ``source`` + (the write then goes through a temporary file) + :param updates: part path -> new content, written (and compressed) as-is + :param deleted: part paths to drop + :param compression: compression used for the entries of ``updates`` + :param allow_raw_copy: set to False to force the decompress/recompress path + :return: (number of entries raw-copied, number of entries re-compressed) + """ + updates = updates or {} + deleted = set(deleted or ()) + source_path = Path(source) if source is not None else None + target_path = Path(target) + + in_place = ( + source_path is not None + and source_path.exists() + and target_path.exists() + and os.path.samefile(source_path, target_path) + ) + + if in_place: + fd, tmp_name = tempfile.mkstemp(suffix=".epc", dir=str(target_path.parent)) + os.close(fd) + write_to = Path(tmp_name) + else: + write_to = target_path + + raw_copied = 0 + recompressed = 0 + skipped = deleted | set(updates.keys()) + + try: + with zipfile.ZipFile(write_to, "w", compression, allowZip64=True) as dst: + if source_path is not None and source_path.exists(): + with zipfile.ZipFile(source_path, "r") as src, open(source_path, "rb") as src_fp: + for info in iter_effective_infos(src): + if info.filename in skipped: + continue + if allow_raw_copy and _copy_entry_raw(src_fp, info, dst): + raw_copied += 1 + else: + dst.writestr(info, src.read(info.filename)) + recompressed += 1 + + for path, data in updates.items(): + dst.writestr(path, data) + recompressed += 1 + + if in_place: + shutil.move(str(write_to), str(target_path)) + except Exception: + if in_place and write_to.exists(): + try: + os.unlink(write_to) + except OSError: # pragma: no cover - defensive + pass + raise + + return raw_copied, recompressed + + +def append_to_zip( + target: Union[str, Path], + updates: Dict[str, bytes], + compression: int = zipfile.ZIP_DEFLATED, +) -> None: + """ + Append parts to an existing archive without rewriting it. + + An appended entry whose name already exists shadows the previous one: readers + resolve a name through the central directory, where the appended entry comes + last. The shadowed bytes stay in the file until it is rewritten, which is what + :func:`rewrite_zip` does when compacting. + + Note that this cannot express a deletion. + """ + if not updates: + return + with warnings.catch_warnings(): + # zipfile warns on duplicate names; here it is the intended mechanism. + warnings.filterwarnings("ignore", message="Duplicate name", category=UserWarning) + with zipfile.ZipFile(target, "a", compression, allowZip64=True) as zf: + for path, data in updates.items(): + zf.writestr(path, data) diff --git a/energyml-utils/tests/test_cli.py b/energyml-utils/tests/test_cli.py new file mode 100644 index 0000000..8674f9c --- /dev/null +++ b/energyml-utils/tests/test_cli.py @@ -0,0 +1,197 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +""" +The console scripts declared in ``[tool.poetry.scripts]``. + +The ten scripts used to point at ``example.tools``, a module the wheel does not ship: every one +of them was installed and every one of them died at import with ``ModuleNotFoundError``. Nothing +caught it because the repository root is on ``sys.path`` in development. + +These tests read the declarations out of ``pyproject.toml`` and resolve each of them the way +``pip`` does — import the module, get the attribute — then run ``--help`` on it. A CLI moved, +renamed or mistyped fails here rather than at the user's first ``pip install``. +""" + +from __future__ import annotations + +import importlib +import json +import re +from pathlib import Path + +import pytest + +_PYPROJECT = Path(__file__).parent.parent / "pyproject.toml" + + +def _declared_scripts() -> dict: + """``{script_name: "module:function"}`` read from ``[tool.poetry.scripts]``.""" + try: + import tomllib # python >= 3.11 + except ModuleNotFoundError: # pragma: no cover — python 3.9 / 3.10 + tomllib = None + + if tomllib is not None: + with open(_PYPROJECT, "rb") as f: + return tomllib.load(f).get("tool", {}).get("poetry", {}).get("scripts", {}) + + # minimal fallback parser, enough for the flat `name = "value"` table we declare + text = _PYPROJECT.read_text(encoding="utf-8") + section = text.split("[tool.poetry.scripts]", 1)[-1].split("\n[", 1)[0] + return dict(re.findall(r'^\s*([\w-]+)\s*=\s*"([^"]+)"\s*$', section, re.M)) + + +SCRIPTS = _declared_scripts() + + +def test_scripts_are_declared(): + assert SCRIPTS, "no console script declared in [tool.poetry.scripts]" + + +@pytest.mark.parametrize("name", sorted(SCRIPTS)) +def test_entry_point_target_is_importable(name): + """Resolve `module:function` exactly like the generated console script does.""" + module_path, _, attribute = SCRIPTS[name].partition(":") + assert module_path.startswith("energyml."), ( + f"'{name}' points at '{module_path}', which is outside the distributed package: " + "the wheel only ships 'energyml/'" + ) + module = importlib.import_module(module_path) + target = getattr(module, attribute, None) + assert callable(target), f"'{name}' -> {SCRIPTS[name]} is not callable" + + +@pytest.mark.parametrize("name", sorted(SCRIPTS)) +def test_entry_point_help(name, capsys): + """``--help`` must build the parser and exit with code 0.""" + module_path, _, attribute = SCRIPTS[name].partition(":") + target = getattr(importlib.import_module(module_path), attribute) + with pytest.raises(SystemExit) as exit_info: + target(["--help"]) + assert exit_info.value.code == 0 + assert "usage:" in capsys.readouterr().out + + +class TestRoundTrip: + """One end-to-end run of the conversion commands, on a fixture that is published.""" + + @staticmethod + def _fixture() -> Path: + path = Path(__file__).parent.parent / "rc" / "epc" / "testingPackageCpp.epc" + if not path.is_file(): + pytest.skip(f"fixture {path.name} not present in rc/epc/") + return path + + def test_xml_to_json(self, tmp_path): + from energyml.utils.cli import xml_to_json + + json_path = tmp_path / "objects.json" + xml_to_json(["-f", str(self._fixture()), "-o", str(json_path)]) + assert json_path.is_file() + objects = json.loads(json_path.read_text(encoding="utf-8")) + assert isinstance(objects, list) and len(objects) > 0 + assert all("$type" in o for o in objects) + + def test_json_to_xml_then_json_to_epc(self, tmp_path): + """JSON in, one XML per object out, then the same JSON packaged in an EPC. + + Built from a single hand-made object rather than from the EPC fixture: the JSON + round trip of that fixture is not stable (a handful of its objects fail to serialize + back to XML, and *which* ones changes from run to run — a pre-existing defect of the + serializer, unrelated to the CLI). + """ + from energyml.utils.cli import json_to_epc, json_to_xml + from energyml.utils.epc import Epc + from energyml.utils.serialization import JSON_VERSION, serialize_json + + from energyml.resqml.v2_2.resqmlv2 import BoundaryFeature + from energyml.eml.v2_3.commonv2 import Citation + + obj = BoundaryFeature( + uuid="0c1b2f30-4e5a-4a1b-9b6d-2f0d5a7c8e91", + schema_version="2.2", + citation=Citation( + title="a boundary", + originator="test", + creation="2024-01-01T00:00:00Z", + format="energyml-utils", + last_update="2024-01-01T00:00:00Z", + ), + ) + json_path = tmp_path / "objects.json" + json_path.write_text("[" + serialize_json(obj, JSON_VERSION.OSDU_OFFICIAL) + "]", encoding="utf-8") + + xml_out = tmp_path / "xml" + xml_out.mkdir() + json_to_xml(["-f", str(json_path), "-o", str(xml_out / "ignored")]) + assert list(xml_out.glob("*.xml")), "json_to_xml wrote no file" + + epc_path = tmp_path / "rebuilt.epc" + json_to_epc(["-f", str(json_path), "-o", str(epc_path)]) + assert epc_path.is_file() + assert len(Epc.read_file(str(epc_path)).energyml_objects) == 1 + + def test_load_n_save(self, tmp_path): + from energyml.utils.cli import load_n_save + from energyml.utils.epc import Epc + + out = tmp_path / "out.epc" + load_n_save(["-f", str(self._fixture()), "-o", str(out)]) + assert out.is_file() + assert len(Epc.read_file(str(out)).energyml_objects) > 0 + + def test_describe_as_csv(self, tmp_path): + from energyml.utils.cli import describe_as_csv + + # describe_as_csv writes its output next to the objects it read + folder = tmp_path / "objects" + folder.mkdir() + (folder / self._fixture().name).write_bytes(self._fixture().read_bytes()) + + describe_as_csv(["-f", str(folder)]) + csv_path = folder / "describe.csv" + assert csv_path.is_file() + lines = csv_path.read_text(encoding="utf-8").strip().split("\n") + assert lines[0].startswith("Title;QualifiedType;") + assert len(lines) > 1, "no object described" + + def test_validate(self, tmp_path, capsys): + from energyml.utils.cli import validate_files + + validate_files(["-f", str(self._fixture())]) + # the command prints a JSON list (or a dict when grouped) on stdout + assert isinstance(json.loads(capsys.readouterr().out), list) + + def test_extract_3d_geojson(self, tmp_path): + from energyml.utils.cli import extract_representation_in_3d_file + + out = tmp_path / "meshes" + out.mkdir() + extract_representation_in_3d_file(["-f", str(self._fixture()), "-o", str(out), "-ff", "geojson"]) + written = list(out.glob("*.geojson")) + assert written, "extract_3d wrote no geojson file" + for path in written: + document = json.loads(path.read_text(encoding="utf-8")) + assert document["type"] == "FeatureCollection" + + +class TestGenerate: + def test_generate_data_prints_an_object(self, capsys): + from energyml.utils.cli import generate_data + + generate_data(["-t", "resqml22.TriangulatedSetRepresentation", "-ff", "json"]) + assert "TriangulatedSetRepresentation" in capsys.readouterr().out + + def test_unknown_type_is_reported(self, capsys): + from energyml.utils.cli import generate_data + + generate_data(["-t", "resqml22.NotAType"]) + assert "Class not found" in capsys.readouterr().out + + def test_generate_multiple_data_writes_files(self, tmp_path): + from energyml.utils.cli import generate_multiple_data + + generate_multiple_data( + ["-t", "resqml22.TriangulatedSetRepresentation", "resqml22.PointSetRepresentation", "-o", str(tmp_path)] + ) + assert len(list(tmp_path.glob("*.json"))) == 2 diff --git a/energyml-utils/tests/test_constants.py b/energyml-utils/tests/test_constants.py index e0a795c..90e6f38 100644 --- a/energyml-utils/tests/test_constants.py +++ b/energyml-utils/tests/test_constants.py @@ -1,4 +1,4 @@ -from src.energyml.utils.constants import content_type_to_qualified_type, qualified_type_to_content_type +from src.energyml.utils.constants import content_type_to_qualified_type, qualified_type_to_content_type, sanitize_file_name def test_content_type_to_qualified_type(): @@ -13,3 +13,61 @@ def test_qualified_type_to_content_type(): qualified_type_to_content_type("resqml20.obj_FaultInterpretation") == "application/x-resqml+xml;version=2.0;type=obj_FaultInterpretation" ) + + +# --------------------------------------------------------------------------- +# sanitize_file_name +# --------------------------------------------------------------------------- + + +def test_sanitize_file_name_replaces_the_colon(): + """The bug this exists for. + + A citation title such as "AUB-PRO-SP05512: Trajectory" goes into the export file name. On + Windows ``open("well: Traj.geojson", "w")`` does not fail: ``:`` opens an NTFS alternate + data stream, so the content lands in a hidden stream and an empty, extension-less file + called ``well`` is left on disk. + """ + assert ":" not in sanitize_file_name("AUB-PRO-SP05512: Trajectory") + assert sanitize_file_name("AUB-PRO-SP05512: Trajectory") == "AUB-PRO-SP05512_ Trajectory" + + +def test_sanitize_file_name_replaces_every_forbidden_char(): + for char in r'<>:"/\|?*': + assert char not in sanitize_file_name(f"a{char}b"), char + assert sanitize_file_name("a\x00b\x1fc") == "a_b_c" + + +def test_sanitize_file_name_collapses_replacement_runs(): + # "a: b" -> one for the colon, one for the space: a single separator is enough. + assert sanitize_file_name("a:/b") == "a_b" + + +def test_sanitize_file_name_strips_trailing_dots_and_spaces(): + # Windows drops them silently, so "x." and "x" would collide. + assert sanitize_file_name("name. ") == "name" + assert sanitize_file_name(" name ") == "name" + + +def test_sanitize_file_name_escapes_reserved_device_names(): + assert sanitize_file_name("CON") != "CON" + assert sanitize_file_name("con.geojson").startswith("con_") + assert sanitize_file_name("COM1") != "COM1" + # A name that merely starts with a reserved word is fine. + assert sanitize_file_name("CONTOUR") == "CONTOUR" + + +def test_sanitize_file_name_truncates(): + assert len(sanitize_file_name("x" * 400)) == 150 + assert len(sanitize_file_name("x" * 400, max_length=20)) == 20 + + +def test_sanitize_file_name_never_returns_empty(): + assert sanitize_file_name("") == "unnamed" + assert sanitize_file_name("///") == "unnamed" + assert sanitize_file_name("...") == "unnamed" + + +def test_sanitize_file_name_keeps_a_normal_title_untouched(): + assert sanitize_file_name("Bartonien Bottom") == "Bartonien Bottom" + assert sanitize_file_name("Generated Triangulation 2") == "Generated Triangulation 2" diff --git a/energyml-utils/tests/test_epc_file.py b/energyml-utils/tests/test_epc_file.py new file mode 100644 index 0000000..b760e5c --- /dev/null +++ b/energyml-utils/tests/test_epc_file.py @@ -0,0 +1,678 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +""" +Tests for :class:`EpcFile`, the lazy write-buffered EPC handler. + +They run against the real EPC fixtures of ``rc/epc`` (2.0.1 and 2.2 packaging, +list-of-lists and ndarray points, EPSG resolvable or not) rather than against +mock dataclasses, so the behaviour is checked against the actual xsdata classes. +""" +import os +import shutil +import tempfile +import zipfile + +import pytest + +from energyml.eml.v2_3.commonv2 import Citation +from energyml.resqml.v2_2.resqmlv2 import BoundaryFeature, BoundaryFeatureInterpretation +from energyml.utils.epc_file import EpcAccessMode, EpcFile, ReadOnlyEpcError +from energyml.utils.epc_stream import EpcStreamReader, RelsUpdateMode +from energyml.utils.epc_utils import as_dor, gen_rels_path_from_obj_path, get_epc_content_type_path +from energyml.utils.introspection import epoch, epoch_to_date + +RC_EPC = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "rc", "epc") + +EPC_201 = os.path.join(RC_EPC, "testingPackageCpp.epc") +EPC_22 = os.path.join(RC_EPC, "testingPackageCpp22.epc") +EPC_BIG = os.path.join(RC_EPC, "SPASS_40+80wells.epc") + +ALL_FIXTURES = [EPC_201, EPC_22, EPC_BIG] + + +@pytest.fixture(params=ALL_FIXTURES, ids=lambda p: os.path.basename(p)) +def fixture_epc(request): + """Read-only access to each real EPC fixture. + + ``rc/**/*.epc`` is git-ignored and only the fixtures the tests need are force-added, so a + working copy may legitimately be missing one. Skip rather than error: an absent fixture is + not a failing behaviour. + """ + if not os.path.isfile(request.param): + pytest.skip(f"fixture {os.path.basename(request.param)} not present in rc/epc/") + return request.param + + +@pytest.fixture +def writable_copy(): + """A throwaway copy of a fixture, so tests may modify it. + + The copy keeps its original basename inside a temporary directory, and any sibling ``.h5`` + goes with it: an EPC references its external arrays by relative path, so a copy renamed to + ``tmpXXXX.epc`` or dropped next to no HDF5 can be indexed but not read. + """ + created = [] + + def _copy(source=EPC_22): + directory = tempfile.mkdtemp() + created.append(directory) + path = os.path.join(directory, os.path.basename(source)) + shutil.copy(source, path) + h5 = os.path.splitext(source)[0] + ".h5" + if os.path.isfile(h5): + shutil.copy(h5, os.path.join(directory, os.path.basename(h5))) + return path + + yield _copy + + for directory in created: + shutil.rmtree(directory, ignore_errors=True) + + +@pytest.fixture +def new_epc_path(): + fd, path = tempfile.mkstemp(suffix=".epc") + os.close(fd) + os.unlink(path) + yield path + if os.path.exists(path): + os.unlink(path) + + +@pytest.fixture +def sample_objects(): + feature = BoundaryFeature( + citation=Citation(title="Feature under test", originator="test", creation=epoch_to_date(epoch())), + uuid="6a1f0000-0000-4000-8000-00000000f001", + object_version="1.0", + ) + interpretation = BoundaryFeatureInterpretation( + citation=Citation(title="Interpretation under test", originator="test", creation=epoch_to_date(epoch())), + uuid="6a1f0000-0000-4000-8000-00000000f002", + object_version="1.0", + interpreted_feature=as_dor(feature), + ) + return feature, interpretation + + +class TestIndexing: + @pytest.mark.slow + def test_opening_does_not_read_the_parts(self, fixture_epc, writable_copy): + """ + The index costs the central directory plus the content types. The only + parts read are the ones the content types fail to describe, and then only + their first bytes. + + Marked slow: the comparison builds an :class:`EpcStreamReader`, which reads the full XML + of every part — that is the very cost this test exists to show `EpcFile` avoids, and it + alone accounts for ~50 s on ``SPASS_40+80wells.epc``. Run it with ``pytest -m ""``. + """ + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + assert len(epc) > 0 + assert epc.stats.objects_deserialized == 0 + # Whatever was read is a bounded head, never a whole part. + content_types_size = len(epc.get_part(get_epc_content_type_path()) or b"") + assert epc.stats.bytes_read <= 2 * content_types_size + epc.stats.head_reads * epc.head_size + + # On a throwaway copy: EpcStreamReader rewrites the archive when it closes, even after a + # read-only session, and the fixtures of rc/epc/ are committed. + reader = EpcStreamReader(writable_copy(fixture_epc)) + try: + assert epc.stats.bytes_read < reader.stats.bytes_read + finally: + reader.close() + + def test_every_indexed_object_is_loadable(self, fixture_epc): + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + for metadata in epc.list_objects(resolve_titles=False): + assert epc.get_object(metadata.uuid) is not None, f"{metadata.uuid} indexed but not loadable" + + def test_titles_are_resolved_on_every_version(self, fixture_epc): + """ + Citation tags carry a version-dependent prefix and, in 2.0.1, attributes + (````): the extraction must cope with both. + """ + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + titles = [metadata.title for metadata in epc.list_objects()] + assert titles, "no object indexed" + assert any(title for title in titles), "no title resolved at all" + + def test_titles_are_not_resolved_when_not_asked_for(self, fixture_epc): + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + before = epc.stats.head_reads + epc.list_objects(resolve_titles=False) + assert epc.stats.head_reads == before + epc.list_objects(resolve_titles=True) + assert epc.stats.head_reads > before + + @pytest.mark.slow + def test_index_agrees_with_epc_stream_reader(self, fixture_epc, writable_copy): + """ + Same object set as the existing implementation, on packages whose content + types are sound. + + Marked slow for the same reason as + :meth:`test_opening_does_not_read_the_parts`: ``EpcStreamReader.list_objects`` inflates + every part of the archive. Run it with ``pytest -m ""``. + """ + reader = EpcStreamReader(writable_copy(fixture_epc)) # see above: it rewrites on close + try: + reference = {metadata.uuid for metadata in reader.list_objects()} + finally: + reader.close() + + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + found = {metadata.uuid for metadata in epc.list_objects(resolve_titles=False)} + assert found == reference + + def test_every_object_part_of_the_archive_is_indexed(self, fixture_epc): + """ + Nothing that is an energyml part in the ZIP may be left out, whatever the + content types say about it. + """ + with zipfile.ZipFile(fixture_epc) as zf: + object_parts = { + name + for name in zf.namelist() + if EpcFile._is_candidate_object_part(name) and b"uuid=" in zf.open(name).read(4096) + } + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + indexed = {epc.get_object_path(metadata.uuid) for metadata in epc.list_objects(resolve_titles=False)} + assert object_parts - indexed == set() + + def test_filtering_by_type_needs_no_read(self, fixture_epc): + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + some_type = epc.list_objects(resolve_titles=False)[0].object_type + before = epc.stats.head_reads + filtered = epc.list_objects(object_type=some_type, resolve_titles=False) + assert epc.stats.head_reads == before + assert filtered + assert all(metadata.object_type == some_type for metadata in filtered) + + def test_lookup_by_uuid_and_by_identifier(self, fixture_epc): + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + metadata = epc.list_objects()[0] + assert epc.get_object(metadata.uuid) is not None + assert epc.get_object(metadata.identifier) is not None + assert epc.get_object(metadata.uri) is not None + assert len(epc.get_object_by_uuid(metadata.uuid)) >= 1 + assert metadata.uuid in epc + + def test_unknown_object(self, fixture_epc): + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_object("00000000-0000-0000-0000-000000000000") is None + assert epc.get_object("not an identifier") is None + assert epc.get_object_by_uuid("00000000-0000-0000-0000-000000000000") == [] + + def test_object_paths_come_from_the_archive(self, fixture_epc): + """A package whose naming differs from ours must stay readable.""" + with zipfile.ZipFile(fixture_epc) as zf: + names = set(zf.namelist()) + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + for metadata in epc.list_objects(resolve_titles=False): + assert epc.get_object_path(metadata.uuid) in names + + def test_caching(self, fixture_epc): + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + uuid = epc.list_objects(resolve_titles=False)[0].uuid + first = epc.get_object(uuid) + second = epc.get_object(uuid) + assert first is second + assert epc.stats.cache_hits >= 1 + + epc.clear_cache() + assert epc.get_object(uuid) is not first + + +class TestDegradedPackage: + def test_index_without_content_types(self, new_epc_path): + """A package whose [Content_Types].xml is gone must still open.""" + with zipfile.ZipFile(EPC_22) as src, zipfile.ZipFile(new_epc_path, "w", zipfile.ZIP_DEFLATED) as dst: + for info in src.infolist(): + if get_epc_content_type_path() not in info.filename: + dst.writestr(info, src.read(info.filename)) + + with EpcFile(EPC_22, mode=EpcAccessMode.READ_ONLY) as reference: + expected = {metadata.uuid for metadata in reference.list_objects(resolve_titles=False)} + + with EpcFile(new_epc_path, mode=EpcAccessMode.READ_ONLY) as epc: + assert {metadata.uuid for metadata in epc.list_objects(resolve_titles=False)} == expected + assert epc.stats.parts_sniffed == len(expected) + assert epc.get_object(next(iter(expected))) is not None + + def test_content_types_declaring_a_missing_part(self, writable_copy): + """An override pointing nowhere must be dropped, not surfaced as an object.""" + path = writable_copy(EPC_201) + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + indexed = {metadata.uuid for metadata in epc.list_objects(resolve_titles=False)} + with zipfile.ZipFile(path) as zf: + names = zf.namelist() + for uuid in indexed: + assert any(uuid in name for name in names) + + def test_sniffing_can_be_disabled(self, new_epc_path): + with zipfile.ZipFile(EPC_22) as src, zipfile.ZipFile(new_epc_path, "w", zipfile.ZIP_DEFLATED) as dst: + for info in src.infolist(): + if get_epc_content_type_path() not in info.filename: + dst.writestr(info, src.read(info.filename)) + + with EpcFile(new_epc_path, mode=EpcAccessMode.READ_ONLY, scan_undeclared_parts=False) as epc: + assert len(epc) == 0 + + +class TestAccessModes: + def test_read_only_refuses_every_modification(self, fixture_epc, sample_objects): + feature, _ = sample_objects + with EpcFile(fixture_epc, mode=EpcAccessMode.READ_ONLY) as epc: + with pytest.raises(ReadOnlyEpcError): + epc.put_object(feature) + with pytest.raises(ReadOnlyEpcError): + epc.delete_object(epc.list_objects(resolve_titles=False)[0].uuid) + with pytest.raises(ReadOnlyEpcError): + epc.put_part("junk.txt", b"junk") + + def test_read_only_on_a_missing_file(self, new_epc_path): + with pytest.raises(FileNotFoundError): + EpcFile(new_epc_path, mode=EpcAccessMode.READ_ONLY) + + def test_in_memory_never_touches_the_file(self, writable_copy, sample_objects): + path = writable_copy() + feature, _ = sample_objects + before = os.path.getsize(path) + + with EpcFile(path, mode=EpcAccessMode.IN_MEMORY) as epc: + count_before = len(epc) + epc.put_object(feature) + assert len(epc) == count_before + 1 + assert epc.get_object(feature.uuid) is not None + + assert os.path.getsize(path) == before + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_object(feature.uuid) is None + + def test_in_memory_can_be_materialised_with_save_as(self, writable_copy, new_epc_path, sample_objects): + path = writable_copy() + feature, _ = sample_objects + before = os.path.getsize(path) + + with EpcFile(path, mode=EpcAccessMode.IN_MEMORY) as epc: + epc.put_object(feature) + epc.save_as(new_epc_path) + + assert os.path.getsize(path) == before + with EpcFile(new_epc_path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_object(feature.uuid) is not None + + def test_manual_discards_unsaved_changes(self, writable_copy, sample_objects): + path = writable_copy() + feature, _ = sample_objects + + with EpcFile(path, mode=EpcAccessMode.MANUAL) as epc: + epc.put_object(feature) + assert epc.has_pending_changes + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_object(feature.uuid) is None + + def test_manual_writes_on_save(self, writable_copy, sample_objects): + path = writable_copy() + feature, _ = sample_objects + + with EpcFile(path, mode=EpcAccessMode.MANUAL) as epc: + epc.put_object(feature) + assert epc.save() is True + assert not epc.has_pending_changes + assert epc.save() is False # nothing left to write + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_object(feature.uuid) is not None + + def test_on_close_writes_once(self, writable_copy, sample_objects): + path = writable_copy() + feature, interpretation = sample_objects + + with EpcFile(path, mode=EpcAccessMode.ON_CLOSE) as epc: + epc.put_object(feature) + epc.put_object(interpretation) + assert epc.stats.flushes == 0 + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_object(feature.uuid) is not None + assert epc.get_object(interpretation.uuid) is not None + + def test_immediate_writes_on_each_modification(self, writable_copy, sample_objects): + path = writable_copy() + feature, interpretation = sample_objects + + epc = EpcFile(path, mode=EpcAccessMode.IMMEDIATE) + epc.put_object(feature) + assert epc.stats.flushes == 1 + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as other: + assert other.get_object(feature.uuid) is not None + epc.put_object(interpretation) + assert epc.stats.flushes == 2 + epc.close() + + with zipfile.ZipFile(path) as zf: + assert zf.testzip() is None + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_object(interpretation.uuid) is not None + + def test_immediate_compacts_on_close(self, writable_copy, sample_objects): + from energyml.utils.zip_raw import count_shadowed_entries + + path = writable_copy() + feature, _ = sample_objects + + epc = EpcFile(path, mode=EpcAccessMode.IMMEDIATE) + for _ in range(3): + epc.put_object(feature) + with zipfile.ZipFile(path) as zf: + assert count_shadowed_entries(zf) > 0 + epc.close() + + with zipfile.ZipFile(path) as zf: + assert count_shadowed_entries(zf) == 0 + assert zf.testzip() is None + + def test_discard_changes(self, writable_copy, sample_objects): + path = writable_copy() + feature, _ = sample_objects + + with EpcFile(path, mode=EpcAccessMode.ON_CLOSE) as epc: + count = len(epc) + epc.put_object(feature) + epc.discard_changes() + assert not epc.has_pending_changes + assert len(epc) == count + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_object(feature.uuid) is None + + def test_pending_changes_are_dropped_on_exception(self, writable_copy, sample_objects): + path = writable_copy() + feature, _ = sample_objects + + with pytest.raises(ValueError): + with EpcFile(path, mode=EpcAccessMode.ON_CLOSE) as epc: + epc.put_object(feature) + raise ValueError("something went wrong") + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_object(feature.uuid) is None + + +class TestModification: + def test_put_then_read_back(self, writable_copy, sample_objects): + path = writable_copy() + feature, _ = sample_objects + + with EpcFile(path) as epc: + identifier = epc.put_object(feature) + assert identifier is not None + assert epc.get_object(feature.uuid) is feature + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + reloaded = epc.get_object(feature.uuid) + assert reloaded is not None + assert reloaded.citation.title == feature.citation.title + assert epc.get_object(identifier) is not None + + def test_update_reuses_the_existing_path(self, writable_copy): + path = writable_copy() + with EpcFile(path) as epc: + uuid = epc.list_objects(resolve_titles=False)[0].uuid + original_path = epc.get_object_path(uuid) + obj = epc.get_object(uuid) + obj.citation.title = "renamed by the test" + epc.put_object(obj) + assert epc.get_object_path(uuid) == original_path + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_object_path(uuid) == original_path + assert epc.get_object(uuid).citation.title == "renamed by the test" + + def test_update_does_not_duplicate_the_object(self, writable_copy): + path = writable_copy() + with EpcFile(path) as epc: + count = len(epc) + uuid = epc.list_objects(resolve_titles=False)[0].uuid + epc.put_object(epc.get_object(uuid)) + assert len(epc) == count + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert len(epc) == count + + def test_add_object_refusing_to_replace(self, writable_copy, sample_objects): + path = writable_copy() + feature, _ = sample_objects + with EpcFile(path) as epc: + epc.add_object(feature) + with pytest.raises(ValueError): + epc.add_object(feature, replace_if_exists=False) + + def test_delete(self, writable_copy): + path = writable_copy() + with EpcFile(path) as epc: + count = len(epc) + uuid = epc.list_objects(resolve_titles=False)[0].uuid + part_path = epc.get_object_path(uuid) + assert epc.delete_object(uuid) is True + assert len(epc) == count - 1 + assert epc.get_object(uuid) is None + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert len(epc) == count - 1 + assert epc.get_object(uuid) is None + assert part_path not in epc.list_parts() + with zipfile.ZipFile(path) as zf: + assert zf.testzip() is None + + def test_delete_unknown_object(self, writable_copy): + path = writable_copy() + with EpcFile(path) as epc: + assert epc.delete_object("00000000-0000-0000-0000-000000000000") is False + + def test_delete_then_put_again(self, writable_copy, sample_objects): + path = writable_copy() + feature, _ = sample_objects + with EpcFile(path) as epc: + epc.put_object(feature) + epc.delete_object(feature.uuid) + epc.put_object(feature) + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_object(feature.uuid) is not None + + def test_raw_parts(self, writable_copy): + path = writable_copy() + with EpcFile(path) as epc: + epc.put_part("docs/readme.txt", b"hello") + assert epc.get_part("docs/readme.txt") == b"hello" + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_part("docs/readme.txt") == b"hello" + + with EpcFile(path) as epc: + assert epc.delete_part("docs/readme.txt") is True + assert epc.get_part("docs/readme.txt") is None + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_part("docs/readme.txt") is None + + def test_content_types_stay_consistent(self, writable_copy, sample_objects): + path = writable_copy() + feature, _ = sample_objects + with EpcFile(path) as epc: + epc.put_object(feature) + + with zipfile.ZipFile(path) as zf: + content_types = zf.read(get_epc_content_type_path()).decode("utf-8") + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + for metadata in epc.list_objects(resolve_titles=False): + assert epc.get_object_path(metadata.uuid) in content_types + + def test_archive_stays_readable_by_epc_stream_reader(self, writable_copy, sample_objects): + """The written package must be consumable by the other implementation.""" + path = writable_copy() + feature, _ = sample_objects + with EpcFile(path) as epc: + epc.put_object(feature) + + reader = EpcStreamReader(path) + try: + assert feature.uuid in {metadata.uuid for metadata in reader.list_objects()} + finally: + reader.close() + + +class TestRelationships: + def test_rels_are_written_for_a_new_object(self, writable_copy, sample_objects): + path = writable_copy() + feature, interpretation = sample_objects + + with EpcFile(path, rels_update_mode=RelsUpdateMode.UPDATE_AT_MODIFICATION) as epc: + epc.put_object(feature) + epc.put_object(interpretation) + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + interpretation_path = epc.get_object_path(interpretation.uuid) + feature_path = epc.get_object_path(feature.uuid) + + outgoing = epc.get_obj_rels(interpretation.uuid) + assert any(rel.target == feature_path for rel in outgoing), "missing DESTINATION relationship" + + incoming = epc.get_obj_rels(feature.uuid) + assert any(rel.target == interpretation_path for rel in incoming), "missing SOURCE relationship" + + def test_rels_files_are_valid_xml(self, writable_copy, sample_objects): + path = writable_copy() + feature, interpretation = sample_objects + with EpcFile(path) as epc: + epc.put_object(feature) + epc.put_object(interpretation) + interpretation_rels = gen_rels_path_from_obj_path(epc.get_object_path(interpretation.uuid)) + + with zipfile.ZipFile(path) as zf: + from lxml import etree + + etree.fromstring(zf.read(interpretation_rels)) + + def test_deleting_an_object_cleans_the_back_references(self, writable_copy, sample_objects): + path = writable_copy() + feature, interpretation = sample_objects + + with EpcFile(path) as epc: + epc.put_object(feature) + epc.put_object(interpretation) + + with EpcFile(path) as epc: + interpretation_path = epc.get_object_path(interpretation.uuid) + epc.delete_object(interpretation.uuid) + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + remaining = epc.get_obj_rels(feature.uuid) + assert all(rel.target != interpretation_path for rel in remaining) + + def test_manual_rels_mode_writes_nothing(self, writable_copy, sample_objects): + path = writable_copy() + feature, interpretation = sample_objects + + with EpcFile(path, rels_update_mode=RelsUpdateMode.MANUAL) as epc: + epc.put_object(feature) + epc.put_object(interpretation) + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + assert epc.get_obj_rels(interpretation.uuid) == [] + + def test_rels_update_on_close(self, writable_copy, sample_objects): + path = writable_copy() + feature, interpretation = sample_objects + + with EpcFile(path, rels_update_mode=RelsUpdateMode.UPDATE_ON_CLOSE) as epc: + epc.put_object(feature) + epc.put_object(interpretation) + + with EpcFile(path, mode=EpcAccessMode.READ_ONLY) as epc: + feature_path = epc.get_object_path(feature.uuid) + assert any(rel.target == feature_path for rel in epc.get_obj_rels(interpretation.uuid)) + + def test_object_dependencies(self, writable_copy, sample_objects): + path = writable_copy() + feature, interpretation = sample_objects + with EpcFile(path) as epc: + epc.put_object(feature) + epc.put_object(interpretation) + dependencies = epc.get_object_dependencies(interpretation.uuid) + assert any(feature.uuid in dependency for dependency in dependencies) + + +class TestCreation: + def test_new_file(self, new_epc_path, sample_objects): + feature, interpretation = sample_objects + + with EpcFile(new_epc_path) as epc: + assert len(epc) == 0 + epc.put_object(feature) + epc.put_object(interpretation) + + assert os.path.exists(new_epc_path) + with zipfile.ZipFile(new_epc_path) as zf: + assert zf.testzip() is None + assert get_epc_content_type_path() in zf.namelist() + + with EpcFile(new_epc_path, mode=EpcAccessMode.READ_ONLY) as epc: + assert len(epc) == 2 + assert epc.get_object(feature.uuid) is not None + + def test_new_file_is_readable_by_epc_stream_reader(self, new_epc_path, sample_objects): + feature, _ = sample_objects + with EpcFile(new_epc_path) as epc: + epc.put_object(feature) + + reader = EpcStreamReader(new_epc_path) + try: + assert reader.get_object(feature.uuid) is not None + finally: + reader.close() + + +class TestExternalArrays: + def test_h5_paths_are_resolved(self): + with EpcFile(EPC_22, mode=EpcAccessMode.READ_ONLY) as epc: + metadata = epc.list_objects(resolve_titles=False)[0] + paths = epc.get_h5_file_paths(metadata.uuid) + assert isinstance(paths, list) + + def test_read_array_matches_epc_stream_reader(self, writable_copy): + """Both implementations must return the same arrays for the same object.""" + import numpy as np + + reader = EpcStreamReader(writable_copy(EPC_22)) # it rewrites the archive on close + try: + candidates = [ + metadata for metadata in reader.list_objects() if "Representation" in (metadata.object_type or "") + ] + compared = 0 + with EpcFile(EPC_22, mode=EpcAccessMode.READ_ONLY) as epc: + for metadata in candidates: + obj = reader.get_object(metadata.uuid) + arrays = reader.get_array_metadata(obj) + if not isinstance(arrays, list): + continue + for array_metadata in arrays: + path = array_metadata.path_in_resource + if not path: + continue + reference = reader.read_array(obj, path) + candidate = epc.read_array(metadata.uuid, path) + if reference is None: + continue + assert candidate is not None, f"{path} unreadable through EpcFile" + assert np.array_equal(np.asarray(reference), np.asarray(candidate)) + compared += 1 + if compared >= 5: + return + assert compared > 0, "no array compared" + finally: + reader.close() diff --git a/energyml-utils/tests/test_external_array_fallback.py b/energyml-utils/tests/test_external_array_fallback.py new file mode 100644 index 0000000..999a2b0 --- /dev/null +++ b/energyml-utils/tests/test_external_array_fallback.py @@ -0,0 +1,175 @@ +"""External-array reading must never fail silently, and an empty export must not be written. + +A representation whose external arrays cannot be read still produces patches — with zero points. +Nothing said so: the failure was logged at DEBUG, the patches came back empty, and the exporter +wrote a valid but useless ``{"type": "FeatureCollection", "features": []}``. + +That is what an ``h5py`` + ``numpy>=2`` pair did to *every* HDF5 array, because +``np.array(dataset, copy=False)`` changed meaning in NumPy 2.0: it used to mean "avoid a copy if +possible" and now means "never copy — raise if you would have to". An HDF5 dataset lives on disk, +so the read always has to allocate. +""" + +import os +import tempfile + +import numpy as np +import pytest + +from energyml.utils.data.export import ExportFormat, EmptyMeshError, drop_empty_patches, export_mesh +from energyml.utils.data.mesh_numpy import NumpyMultiMesh, NumpyPointSetMesh, NumpySurfaceMesh +from energyml.utils.data.crs import PointFrame +from energyml.utils.epc_file import _read_array_from_handler + + +class _Handler: + """Duck-typed array handler recording which of its two entry points were used.""" + + def __init__(self, view_result=None, view_raises=False, read_result=None, read_raises=False): + self.view_result, self.view_raises = view_result, view_raises + self.read_result, self.read_raises = read_result, read_raises + self.calls = [] + + def read_array_view(self, file_path, path, start_indices=None, counts=None): + self.calls.append("view") + if self.view_raises: + raise ValueError("Dataset.__array__ received copy=False but memory allocation cannot be avoided") + return self.view_result + + def read_array(self, file_path, path, start_indices=None, counts=None): + self.calls.append("read") + if self.read_raises: + raise OSError("unreadable") + return self.read_result + + +class TestViewFailureFallsBackToAPlainRead: + def test_a_raising_view_falls_back_to_read_array(self): + """The zero-copy view is an optimisation — losing it must not lose the data. + + This is the exact numpy>=2 failure: the view raises for every candidate file, and the + array used to come back as None. + """ + data = np.arange(6.0).reshape(2, 3) + handler = _Handler(view_raises=True, read_result=data) + result = _read_array_from_handler(handler, "f.h5", "/points") + np.testing.assert_array_equal(result, data) + assert handler.calls == ["view", "read"], "the same file must be retried, not skipped" + + def test_a_view_returning_none_falls_back_too(self): + data = np.arange(3.0) + handler = _Handler(view_result=None, read_result=data) + np.testing.assert_array_equal(_read_array_from_handler(handler, "f.h5", "/p"), data) + assert handler.calls == ["view", "read"] + + def test_a_working_view_is_used_as_is(self): + data = np.arange(3.0) + handler = _Handler(view_result=data, read_result=np.zeros(3)) + np.testing.assert_array_equal(_read_array_from_handler(handler, "f.h5", "/p"), data) + assert handler.calls == ["view"], "no need to read twice when the view worked" + + def test_both_failing_returns_none_without_raising(self): + handler = _Handler(view_raises=True, read_raises=True) + assert _read_array_from_handler(handler, "f.h5", "/p") is None + + +def _hdf5_sample(tmp_name: str = "sample.h5"): + """Write a small HDF5 file and return ``(path, handler, expected_array)``.""" + h5py = pytest.importorskip("h5py") + from energyml.utils.data.datasets_io import get_handler_registry + + expected = np.arange(12.0).reshape(4, 3) + path = os.path.join(tempfile.mkdtemp(), tmp_name) + with h5py.File(path, "w") as f: + f.create_dataset("/grp/points", data=expected) + return path, get_handler_registry().get_handler_for_file(path), expected + + +class TestReadArrayViewIsNumpy2Safe: + def test_the_view_does_not_ask_numpy_never_to_copy(self, monkeypatch): + """Reproduce the NumPy 2 contract on any NumPy: ``copy=False`` must never be used. + + Under NumPy 2, ``np.array(x, copy=False)`` raises instead of copying when a copy is + unavoidable — which it always is for an HDF5 dataset. Making the stub raise pins the + behaviour without needing a NumPy 2 interpreter. + """ + path, handler, expected = _hdf5_sample("nocopy.h5") + real_array = np.array + + def strict_array(obj, *args, **kwargs): + if kwargs.get("copy", True) is False: + raise ValueError( + "Dataset.__array__ received copy=False but memory allocation cannot be avoided on read" + ) + return real_array(obj, *args, **kwargs) + + monkeypatch.setattr(np, "array", strict_array) + np.testing.assert_array_equal(handler.read_array_view(path, "/grp/points"), expected) + + def test_reading_a_real_hdf5_array_returns_the_values(self): + path, handler, expected = _hdf5_sample() + np.testing.assert_array_equal(handler.read_array_view(path, "/grp/points"), expected) + np.testing.assert_array_equal(handler.read_array(path, "/grp/points"), expected) + + +class TestTheFileCacheKeepsItsHandlesUsable: + """The cache owns the handle; a consumer must not be able to close it.""" + + def test_reading_twice_works_in_either_order(self): + path, handler, expected = _hdf5_sample("twice.h5") + # read_array first used to close the cached handle, so the following view raised + # "invalid identifier type to function". + np.testing.assert_array_equal(handler.read_array(path, "/grp/points"), expected) + np.testing.assert_array_equal(handler.read_array_view(path, "/grp/points"), expected) + + path2, handler2, expected2 = _hdf5_sample("twice2.h5") + np.testing.assert_array_equal(handler2.read_array_view(path2, "/grp/points"), expected2) + np.testing.assert_array_equal(handler2.read_array(path2, "/grp/points"), expected2) + + +class TestEmptyExportsAreRefused: + @staticmethod + def _empty_surface(): + return NumpySurfaceMesh( + identifier="empty", + points=np.empty((0, 3), dtype=np.float64), + faces=np.empty(0, dtype=np.int64), + frame=PointFrame.PROJECTED, + ) + + @staticmethod + def _filled_points(): + return NumpyPointSetMesh( + identifier="filled", + points=np.arange(9.0).reshape(3, 3), + frame=PointFrame.PROJECTED, + ) + + def test_empty_patches_are_dropped(self): + kept = drop_empty_patches(NumpyMultiMesh(patches=[self._empty_surface(), self._filled_points()])) + assert [m.identifier for m in kept] == ["filled"] + + def test_all_empty_raises_instead_of_returning_nothing(self): + with pytest.raises(EmptyMeshError, match="Nothing to export"): + drop_empty_patches(NumpyMultiMesh(patches=[self._empty_surface()]), raise_when_empty=True) + + def test_export_mesh_refuses_to_write_an_empty_file(self): + """The reported symptom: a 54-byte GeoJSON with an empty feature list.""" + path = os.path.join(tempfile.mkdtemp(), "out.geojson") + with pytest.raises(EmptyMeshError): + export_mesh(NumpyMultiMesh(patches=[self._empty_surface()]), path, format=ExportFormat.GEOJSON) + assert not os.path.exists(path), "no file must be left behind when there is nothing to write" + + def test_export_mesh_keeps_the_readable_patches(self): + """A partially readable object must export what it has, without the empty patches.""" + import json + + path = os.path.join(tempfile.mkdtemp(), "out.geojson") + export_mesh( + NumpyMultiMesh(patches=[self._empty_surface(), self._filled_points()]), + path, + format=ExportFormat.GEOJSON, + ) + doc = json.load(open(path, encoding="utf-8")) + assert len(doc["features"]) == 1 + assert doc["features"][0]["geometry"]["type"] == "MultiPoint" diff --git a/energyml-utils/tests/test_geojson_export.py b/energyml-utils/tests/test_geojson_export.py new file mode 100644 index 0000000..b610041 --- /dev/null +++ b/energyml-utils/tests/test_geojson_export.py @@ -0,0 +1,222 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +""" +Tests for the GeoJSON export : energyml metadata, CRS declaration and WGS84 reprojection. + +The tests run against the real EPC fixture ``rc/epc/80wells_surf.epc`` (RESQML v2.2, whose +representations reference a standalone ``ProjectedCrs`` with EPSG:3949 — Lambert-93 CC49). + +The reprojection tests are skipped when the ``crs`` extra (pyproj) is not installed. +""" +from __future__ import annotations + +import io +import json +from pathlib import Path + +import numpy as np +import pytest + +from energyml.utils.data import crs as crs_module +from energyml.utils.data.crs import ( + build_source_crs_id, + crs_ogc_uri, + crs_urn, + extract_crs_info, + is_pyproj_available, +) +from energyml.utils.data.export import GeoJSONExportOptions, _geojson_crs_members, _feature_id, export_geojson +from energyml.utils.epc import Epc +from energyml.utils.introspection import get_object_metadata + +_EPC_PATH = Path(__file__).parent.parent / "rc" / "epc" / "80wells_surf.epc" + +requires_pyproj = pytest.mark.skipif(not is_pyproj_available(), reason="requires the 'crs' extra (pyproj)") + + +@pytest.fixture(scope="module") +def epc() -> Epc: + # rc/**/*.epc is git-ignored and only the needed fixtures are force-added, so a working copy + # may legitimately lack this one — skip instead of erroring out of the fixture setup. + if not _EPC_PATH.is_file(): + pytest.skip(f"fixture {_EPC_PATH.name} not present in rc/epc/") + return Epc.read_file(str(_EPC_PATH)) + + +@pytest.fixture(scope="module") +def point_set_object(epc: Epc): + obj = next((o for o in epc.energyml_objects if "PointSet" in type(o).__name__), None) + if obj is None: + pytest.skip("no PointSetRepresentation in the fixture") + return obj + + +def _read_multi_mesh(obj, epc): + from energyml.utils.data.mesh_numpy import read_numpy_mesh_object + + return read_numpy_mesh_object(obj, workspace=epc, use_crs_displacement=True) + + +def _export(obj, epc, options: GeoJSONExportOptions) -> dict: + out = io.StringIO() + export_geojson(_read_multi_mesh(obj, epc), out, options) + return json.loads(out.getvalue()) + + +# --------------------------------------------------------------------------- +# CRS identifiers +# --------------------------------------------------------------------------- + + +class TestCrsIdentifiers: + def test_ogc_uri_and_urn(self): + assert crs_ogc_uri(32631) == "http://www.opengis.net/def/crs/EPSG/0/32631" + assert crs_urn(32631) == "urn:ogc:def:crs:EPSG::32631" + + def test_build_source_crs_id(self): + assert build_source_crs_id(32631) == "EPSG:32631" + assert build_source_crs_id(32631, 5773) == "EPSG:32631+EPSG:5773" + assert build_source_crs_id(None) is None + assert build_source_crs_id(None, 5773) is None + + def test_crs_members_horizontal_only(self): + members = _geojson_crs_members(3949, None) + assert members["crs"] == {"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::3949"}} + assert members["coordRefSys"] == crs_ogc_uri(3949) + + def test_crs_members_compound(self): + members = _geojson_crs_members(32631, 5773) + assert members["coordRefSys"] == [crs_ogc_uri(32631), crs_ogc_uri(5773)] + + def test_no_crs_member_without_epsg(self): + assert _geojson_crs_members(None, 5773) == {} + + +class TestFeatureId: + def test_uuid_only(self): + assert _feature_id("abc") == "abc" + + def test_with_patch_and_element(self): + assert _feature_id("abc", 1) == "abc_1" + assert _feature_id("abc", 1, 7) == "abc_1_7" + + def test_without_uuid(self): + assert _feature_id(None, 1) is None + + +# --------------------------------------------------------------------------- +# Metadata +# --------------------------------------------------------------------------- + + +class TestObjectMetadata: + def test_metadata_of_a_real_object(self, point_set_object): + metadata = get_object_metadata(point_set_object) + assert metadata["uuid"] == point_set_object.uuid + assert metadata["qualified_type"].endswith(".PointSetRepresentation") + assert metadata["title"] + # dates are exported as ISO 8601 strings + assert "T" in metadata["creation"] + + def test_metadata_of_none(self): + assert get_object_metadata(None) == {} + + +# --------------------------------------------------------------------------- +# GeoJSON output +# --------------------------------------------------------------------------- + + +class TestGeoJsonExport: + def test_features_carry_id_and_metadata(self, point_set_object, epc): + doc = _export(point_set_object, epc, GeoJSONExportOptions(indent=None)) + assert doc["type"] == "FeatureCollection" + feature = doc["features"][0] + assert feature["type"] == "Feature" + assert feature["id"].startswith(point_set_object.uuid) + properties = feature["properties"] + assert properties["uuid"] == point_set_object.uuid + assert properties["qualified_type"].endswith(".PointSetRepresentation") + assert properties["title"] + assert properties["projected_epsg_code"] == 3949 + + def test_collection_bbox(self, point_set_object, epc): + doc = _export(point_set_object, epc, GeoJSONExportOptions(indent=None)) + assert len(doc["bbox"]) == 6 + assert doc["bbox"][0] <= doc["bbox"][3] + assert doc["bbox"][1] <= doc["bbox"][4] + + def test_metadata_can_be_disabled(self, point_set_object, epc): + doc = _export(point_set_object, epc, GeoJSONExportOptions(indent=None, include_metadata=False)) + assert "title" not in doc["features"][0]["properties"] + + def test_source_crs_is_declared_when_not_reprojected(self, point_set_object, epc): + doc = _export(point_set_object, epc, GeoJSONExportOptions(indent=None, to_wgs84=False)) + assert doc["crs"]["properties"]["name"] == crs_urn(3949) + assert doc["coordRefSys"] == crs_ogc_uri(3949) + # coordinates stay in the projected CRS (metric values, far outside the lon/lat range) + assert abs(doc["features"][0]["geometry"]["coordinates"][0][0]) > 180 + + def test_fallback_when_pyproj_is_missing(self, point_set_object, epc, monkeypatch): + monkeypatch.setattr(crs_module, "is_pyproj_available", lambda: False) + doc = _export(point_set_object, epc, GeoJSONExportOptions(indent=None, to_wgs84=True)) + # no reprojection, but the source CRS must be advertised + assert doc["crs"]["properties"]["name"] == crs_urn(3949) + assert abs(doc["features"][0]["geometry"]["coordinates"][0][0]) > 180 + + @requires_pyproj + def test_wgs84_is_the_default(self, point_set_object, epc): + doc = _export(point_set_object, epc, GeoJSONExportOptions(indent=None)) + # an RFC 7946 document is implicitly CRS84 and must NOT carry a 'crs' member + assert "crs" not in doc + lon, lat = doc["features"][0]["geometry"]["coordinates"][0][:2] + assert -180.0 <= lon <= 180.0 + assert -90.0 <= lat <= 90.0 + properties = doc["features"][0]["properties"] + assert properties["source_crs"] == "EPSG:3949" + assert properties["coordinates_crs"] == "OGC:CRS84" + + +# --------------------------------------------------------------------------- +# Reprojection +# --------------------------------------------------------------------------- + + +@requires_pyproj +class TestReprojection: + def test_utm31n_to_wgs84(self): + from energyml.utils.data.crs import reproject_to_wgs84 + + points = np.array([[463000.0, 6570000.0, -1500.0]]) + result = reproject_to_wgs84(points, projected_epsg_code=32631) + assert result.shape == (1, 3) + assert result[0][0] == pytest.approx(2.350943, abs=1e-5) + assert result[0][1] == pytest.approx(59.267329, abs=1e-5) + # without a vertical CRS the Z column is left untouched + assert result[0][2] == pytest.approx(-1500.0) + + def test_input_is_not_modified(self): + from energyml.utils.data.crs import reproject_to_wgs84 + + points = np.array([[463000.0, 6570000.0, -1500.0]]) + original = points.copy() + reproject_to_wgs84(points, projected_epsg_code=32631) + assert np.array_equal(points, original) + + def test_missing_epsg_raises(self): + from energyml.utils.data.crs import reproject_to_wgs84 + from energyml.utils.exception import NotEnoughInformationError + + with pytest.raises(NotEnoughInformationError): + reproject_to_wgs84(np.zeros((1, 3))) + + def test_crs_info_codes_are_used(self, point_set_object, epc): + from energyml.utils.data.crs import reproject_to_wgs84 + from energyml.utils.data.helper import get_crs_obj + + crs_obj = get_crs_obj(context_obj=point_set_object, root_obj=point_set_object, workspace=epc) + crs_info = extract_crs_info(crs_obj, epc) + assert crs_info.projected_epsg_code == 3949 + result = reproject_to_wgs84(np.array([[1656431.13, 8190610.64, 37.15]]), crs_info) + assert result[0][0] == pytest.approx(2.4055, abs=1e-3) + assert result[0][1] == pytest.approx(48.9140, abs=1e-3) diff --git a/energyml-utils/tests/test_geojson_features.py b/energyml-utils/tests/test_geojson_features.py new file mode 100644 index 0000000..babdee6 --- /dev/null +++ b/energyml-utils/tests/test_geojson_features.py @@ -0,0 +1,275 @@ +"""GeoJSON feature granularity, and the CRS fallback for packages that name none. + +A RESQML patch is one GeoJSON feature. Exploding a patch into one feature per triangle or per +line segment repeats the whole metadata block — uuid, citation, EPSG codes — on every element: +a 882-triangle surface produced 882 features, and a 15-station wellbore 14 two-point LineStrings. + +These tests use synthetic meshes so they need no EPC fixture; the CRS-fallback tests use a +minimal fake workspace. +""" + +import io +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from energyml.utils.data.export._base import GeoJSONExportOptions +from energyml.utils.data.export.geojson import export_geojson +from energyml.utils.data.mesh_numpy import ( + NumpyMultiMesh, + NumpyPointSetMesh, + NumpyPolylineMesh, + NumpySurfaceMesh, +) +from energyml.utils.data.crs import PointFrame + + +def _export(mesh, **opt_kwargs) -> dict: + """Run the registry writer on a single patch and parse the result.""" + multi = NumpyMultiMesh(identifier="test", patches=[mesh]) + buffer = io.StringIO() + options = GeoJSONExportOptions(to_wgs84=False, include_metadata=False, **opt_kwargs) + export_geojson(multi, buffer, options) + return json.loads(buffer.getvalue()) + + +def _square_grid_surface(n_tri: int) -> NumpySurfaceMesh: + """A fan of *n_tri* triangles sharing point 0.""" + pts = np.array([[float(i), float(i % 3), 0.0] for i in range(n_tri + 2)], dtype=np.float64) + faces = [] + for t in range(n_tri): + faces.extend([3, 0, t + 1, t + 2]) + return NumpySurfaceMesh( + identifier="surface", + points=pts, + faces=np.array(faces, dtype=np.int64), + frame=PointFrame.PROJECTED, + ) + + +def _polyline(n_points: int, n_lines: int = 1) -> NumpyPolylineMesh: + pts = np.array([[float(i), 0.0, float(i)] for i in range(n_points * n_lines)], dtype=np.float64) + lines = [] + for line in range(n_lines): + base = line * n_points + lines.append(n_points) + lines.extend(range(base, base + n_points)) + return NumpyPolylineMesh( + identifier="polyline", + points=pts, + lines=np.array(lines, dtype=np.int64), + frame=PointFrame.PROJECTED, + ) + + +class TestOneFeaturePerPatch: + def test_triangulated_patch_is_a_single_multipolygon(self): + doc = _export(_square_grid_surface(10)) + assert len(doc["features"]) == 1 + geometry = doc["features"][0]["geometry"] + assert geometry["type"] == "MultiPolygon" + assert len(geometry["coordinates"]) == 10, "one polygon per triangle, inside one feature" + for polygon in geometry["coordinates"]: + ring = polygon[0] + assert ring[0] == ring[-1], "a GeoJSON ring must be closed" + + def test_single_triangle_is_a_polygon_not_a_multipolygon(self): + doc = _export(_square_grid_surface(1)) + assert doc["features"][0]["geometry"]["type"] == "Polygon" + + def test_wellbore_polyline_is_a_single_linestring(self): + doc = _export(_polyline(15)) + assert len(doc["features"]) == 1 + geometry = doc["features"][0]["geometry"] + assert geometry["type"] == "LineString" + assert len(geometry["coordinates"]) == 15, "every station in one line" + + def test_several_lines_become_one_multilinestring(self): + doc = _export(_polyline(4, n_lines=3)) + assert len(doc["features"]) == 1 + geometry = doc["features"][0]["geometry"] + assert geometry["type"] == "MultiLineString" + assert [len(line) for line in geometry["coordinates"]] == [4, 4, 4] + + def test_point_set_is_a_single_multipoint(self): + mesh = NumpyPointSetMesh( + identifier="points", + points=np.arange(30, dtype=np.float64).reshape(10, 3), + frame=PointFrame.PROJECTED, + ) + doc = _export(mesh) + assert len(doc["features"]) == 1 + assert doc["features"][0]["geometry"]["type"] == "MultiPoint" + assert len(doc["features"][0]["geometry"]["coordinates"]) == 10 + + def test_metadata_is_written_once_per_patch(self): + """The point of the change: 10 triangles must not carry 10 copies of the citation.""" + multi = NumpyMultiMesh(identifier="test", patches=[_square_grid_surface(10)]) + buffer = io.StringIO() + export_geojson(multi, buffer, GeoJSONExportOptions(to_wgs84=False, properties={"marker": "x"})) + doc = json.loads(buffer.getvalue()) + assert sum(1 for f in doc["features"] if f["properties"].get("marker") == "x") == 1 + + +class TestExplodeElementsOption: + def test_explode_restores_one_feature_per_element(self): + doc = _export(_square_grid_surface(10), explode_elements=True) + assert len(doc["features"]) == 10 + assert {f["geometry"]["type"] for f in doc["features"]} == {"Polygon"} + assert [f["properties"]["element_index"] for f in doc["features"]] == list(range(10)) + + def test_explode_splits_a_polyline_per_line(self): + doc = _export(_polyline(4, n_lines=3), explode_elements=True) + assert len(doc["features"]) == 3 + assert {f["geometry"]["type"] for f in doc["features"]} == {"LineString"} + + +class TestPackageDefaultCrs: + """`PointGeometry.LocalCrs` is optional; a package may declare its CRS once for all.""" + + @staticmethod + def _workspace(objects): + """Minimal duck-typed workspace: list_objects() + get_object().""" + metadata = [ + SimpleNamespace(uuid=f"uuid-{i}", uri=f"eml:///{type(o).__name__}(uuid-{i})", object_type=type(o).__name__) + for i, o in enumerate(objects) + ] + by_uri = {m.uri: o for m, o in zip(metadata, objects)} + return SimpleNamespace( + list_objects=lambda resolve_titles=True: metadata, + get_object=lambda uri: by_uri.get(uri), + ) + + def test_the_only_projected_crs_is_used(self): + from energyml.utils.data.helper import get_package_default_crs + + class ProjectedCrs: + pass + + crs = ProjectedCrs() + assert get_package_default_crs(self._workspace([crs])) is crs + + def test_a_full_local_crs_wins_over_a_projected_one(self): + from energyml.utils.data.helper import get_package_default_crs + + class ProjectedCrs: + pass + + class LocalEngineeringCompoundCrs: + pass + + local = LocalEngineeringCompoundCrs() + found = get_package_default_crs(self._workspace([ProjectedCrs(), local])) + assert found is local, "the local frame describes offsets and rotation, the projected one does not" + + def test_an_ambiguous_package_returns_nothing(self): + from energyml.utils.data.helper import get_package_default_crs + + class ProjectedCrs: + pass + + # Two candidates of the same kind: choosing one would silently place the geometry + # in the wrong frame. + assert get_package_default_crs(self._workspace([ProjectedCrs(), ProjectedCrs()])) is None + + def test_a_lone_vertical_crs_is_never_chosen(self): + from energyml.utils.data.helper import get_package_default_crs + + class VerticalCrs: + pass + + assert get_package_default_crs(self._workspace([VerticalCrs()])) is None + + def test_no_workspace_is_not_an_error(self): + from energyml.utils.data.helper import get_package_default_crs + + assert get_package_default_crs(None) is None + + +class TestEveryEntryPointAcceptsBothMeshFamilies: + """The GeoJSON API has three public collection builders; they must agree. + + ``export_geojson`` (the registry writer, so ``export_mesh``) and ``export_geojson_io`` / + ``export_geojson_dict`` (the streaming pair, used by ``export_multiple_data``) used to + accept different mesh families and different geometry kinds: + + * the registry writer classified meshes with its own ``isinstance`` chain that had no + point-set branch, so a legacy ``PointSetMesh`` exported as **zero** features; + * the streaming pair reached for ``mesh.point_list``, so any numpy mesh raised + ``TypeError: 'NumpyMultiMesh' object is not iterable``. + """ + + @staticmethod + def _legacy(kind: str): + from energyml.utils.data.mesh import PointSetMesh, PolylineSetMesh, SurfaceMesh + + points = np.array([[0.0, 0.0, 0.0], [1.0, 0.0, 1.0], [1.0, 1.0, 2.0], [0.0, 1.0, 3.0]]) + if kind == "points": + return PointSetMesh(identifier="m", point_list=points) + if kind == "lines": + return PolylineSetMesh(identifier="m", point_list=points, line_indices=[[0, 1, 2, 3]]) + return SurfaceMesh(identifier="m", point_list=points, faces_indices=[[0, 1, 2], [0, 2, 3]]) + + @staticmethod + def _numpy(kind: str): + points = np.array([[0.0, 0.0, 0.0], [1.0, 0.0, 1.0], [1.0, 1.0, 2.0], [0.0, 1.0, 3.0]]) + if kind == "points": + return NumpyPointSetMesh(identifier="m", points=points, frame=PointFrame.PROJECTED) + if kind == "lines": + return NumpyPolylineMesh( + identifier="m", + points=points, + lines=np.array([4, 0, 1, 2, 3], dtype=np.int64), + frame=PointFrame.PROJECTED, + ) + return NumpySurfaceMesh( + identifier="m", + points=points, + faces=np.array([3, 0, 1, 2, 3, 0, 2, 3], dtype=np.int64), + frame=PointFrame.PROJECTED, + ) + + @staticmethod + def _all_entry_points(mesh): + """Return the feature list produced by each of the three builders.""" + from energyml.utils.data.export.geojson import export_geojson_dict, export_geojson_io + + registry_buffer = io.StringIO() + export_geojson(mesh, registry_buffer, GeoJSONExportOptions(to_wgs84=False, include_metadata=False)) + registry = json.loads(registry_buffer.getvalue()) + + io_buffer = io.BytesIO() + export_geojson_io(out=io_buffer, mesh_list=mesh, to_wgs84=False, include_metadata=False) + streamed = json.loads(io_buffer.getvalue().decode("utf-8")) + + as_dict = export_geojson_dict(mesh, to_wgs84=False, include_metadata=False) + return {"export_geojson": registry, "export_geojson_io": streamed, "export_geojson_dict": as_dict} + + @pytest.mark.parametrize("kind", ["points", "lines", "polygons"]) + @pytest.mark.parametrize("family", ["legacy", "numpy"]) + def test_every_builder_emits_one_feature(self, kind, family): + mesh = self._legacy(kind) if family == "legacy" else self._numpy(kind) + for name, doc in self._all_entry_points(mesh).items(): + features = doc.get("features", []) + assert len(features) == 1, f"{name} produced {len(features)} feature(s) for {family}/{kind}" + geometry_type = features[0]["geometry"]["type"] + expected = {"points": "MultiPoint", "lines": ("LineString", "MultiLineString"), + "polygons": ("Polygon", "MultiPolygon")}[kind] + assert geometry_type in (expected if isinstance(expected, tuple) else (expected,)), ( + f"{name} gave {geometry_type} for {family}/{kind}" + ) + + @pytest.mark.parametrize("kind", ["points", "lines", "polygons"]) + def test_every_builder_declares_a_bbox(self, kind): + for name, doc in self._all_entry_points(self._numpy(kind)).items(): + assert doc.get("bbox"), f"{name} declared no bbox for {kind}" + + @pytest.mark.parametrize("kind", ["points", "lines", "polygons"]) + def test_the_two_families_give_the_same_coordinates(self, kind): + """A legacy mesh and the numpy mesh it came from must export identically.""" + legacy = self._all_entry_points(self._legacy(kind)) + numpy_meshes = self._all_entry_points(self._numpy(kind)) + for name in legacy: + assert legacy[name]["features"][0]["geometry"] == numpy_meshes[name]["features"][0]["geometry"], name diff --git a/energyml-utils/tests/test_introspection.py b/energyml-utils/tests/test_introspection.py index f910d33..e92935f 100644 --- a/energyml-utils/tests/test_introspection.py +++ b/energyml-utils/tests/test_introspection.py @@ -464,6 +464,30 @@ def test_get_object_attribute_advanced(triangulated_set_versioned): assert get_object_attribute_advanced(triangulated_set_versioned, "citation.originator") == "Valentin" +def test_get_object_attribute_advanced_walks_through_lists(triangulated_set_versioned): + """A list index is a path component like any other. + + ``search_attribute_matching_name_with_path`` produces paths such as + ``line_patch.0.geometry.points``, and the index used to be handed to + ``get_matching_class_attribute_name`` — which never matches a digit — so the path was + declared invalid and the caller silently got ``None`` (that is how the RESQML 2.0.1 + external arrays lost the element count read from their parent patch). + """ + patches = get_object_attribute_advanced(triangulated_set_versioned, "TrianglePatch") + assert isinstance(patches, list) and patches, "fixture is expected to hold at least one patch" + + assert get_object_attribute_advanced(triangulated_set_versioned, "TrianglePatch.0") is patches[0] + # the name written in the path (CamelCase) is longer than the python attribute it matches, + # which used to shift the slicing of the remaining path + assert get_object_attribute_advanced( + triangulated_set_versioned, "TrianglePatch.0.Count" + ) == get_object_attribute_advanced(patches[0], "Count") + + # out-of-range and non-numeric components degrade to None instead of raising + assert get_object_attribute_advanced(triangulated_set_versioned, "TrianglePatch.99") is None + assert get_object_attribute_advanced(triangulated_set_versioned, "TrianglePatch.nope") is None + + # ============================================================================= # OBJECT ATTRIBUTE MODIFICATION TESTS # ============================================================================= @@ -763,9 +787,9 @@ def test_get_obj_uri(triangulated_set_no_version, fault_interpretation): uri_str_fi_dataspace == f"eml:///dataspace('/MyDataspace/')/resqml20.obj_FaultInterpretation(uuid={fault_interpretation.uuid},version='{fault_interpretation.object_version}')" ) - + uri_dict_dor = str(get_obj_uri(json.loads(serialize_json(as_dor(fault_interpretation))))) - + assert ( uri_dict_dor == f"eml:///resqml20.obj_FaultInterpretation(uuid={fault_interpretation.uuid},version='{fault_interpretation.object_version}')" diff --git a/energyml-utils/tests/test_mesh.py b/energyml-utils/tests/test_mesh.py new file mode 100644 index 0000000..0fc940c --- /dev/null +++ b/energyml-utils/tests/test_mesh.py @@ -0,0 +1,397 @@ +"""Regression tests for the fixes applied to mesh.py / mesh_numpy.py. + +Each test here pins down a defect that was previously silent: + +* ``read_mesh_object`` handed back the energyml objects themselves when given a list. +* ``read_property_interpreted_with_cbt`` raised ``NameError`` on its own error path. +* the numpy readers advanced their ``sub_indices`` window by the *filtered* count, which + misaligned every patch after the first one. +* the Grid2d axis-count reconciliation could emit indices past the end of the points array. +* the reader dispatchers scanned every module member on each call. +""" +import os + +import numpy as np +import pytest + +from energyml.utils.data.mesh import ( + AbstractMesh, + _list_exportable_uuids, + _mesh_name_mapping, + get_object_reader_function, + read_mesh_object, +) +from energyml.utils.data.properties import read_property_interpreted_with_cbt +from energyml.utils.data.mesh_numpy import ( + _fit_grid_dimensions, + get_numpy_reader_function, + read_numpy_mesh_object, +) +from energyml.utils.epc_file import EpcAccessMode, EpcFile +from energyml.utils.exception import NotSupportedError + +_WORKSPACE_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_EPC_DIR = os.path.join(_WORKSPACE_ROOT, "rc", "epc") +_EPC22 = os.path.join(_EPC_DIR, "testingPackageCpp22.epc") +_EPC201 = os.path.join(_EPC_DIR, "testingPackageCpp.epc") + +#: TriangulatedSetRepresentation of ``testingPackageCpp22.epc``: 5 patches x 4 triangles. +_MULTI_PATCH_UUID = "1a4112fa-c4ef-4c8d-aed0-47d9273bebc5" + + +requires_epc22 = pytest.mark.skipif(not os.path.isfile(_EPC22), reason="testingPackageCpp22.epc fixture is missing") +requires_epc201 = pytest.mark.skipif(not os.path.isfile(_EPC201), reason="testingPackageCpp.epc fixture is missing") + + +@pytest.fixture +def epc22(): + epc = EpcFile(epc_file_path=_EPC22, mode=EpcAccessMode.READ_ONLY) + yield epc + + +def _vtk_flat_to_triangles(faces: np.ndarray) -> np.ndarray: + """``[3, a, b, c, 3, a, b, c, …]`` → ``(M, 3)``.""" + if len(faces) == 0: + return np.empty((0, 3), dtype=np.int64) + return np.asarray(faces, dtype=np.int64).reshape(-1, 4)[:, 1:] + + +def _first_of_type(epc, type_fragment: str): + for meta in epc.list_objects(resolve_titles=False): + if type_fragment in (getattr(meta, "object_type", "") or ""): + return epc.get_object_by_uuid(meta.uuid)[0] + return None + + +def _all_triangles(multi) -> np.ndarray: + parts = [_vtk_flat_to_triangles(getattr(p, "faces", np.empty(0))) for p in multi.flat_patches()] + parts = [p for p in parts if len(p) > 0] + return np.concatenate(parts, axis=0) if parts else np.empty((0, 3), dtype=np.int64) + + +# --------------------------------------------------------------------------- +# Grid2d axis-count reconciliation +# --------------------------------------------------------------------------- + + +class TestFitGridDimensions: + def test_exact_match_is_untouched(self): + assert _fit_grid_dimensions(5, 4, 20) == (5, 4) + + def test_mismatch_keeps_the_fastest_axis(self): + # The fastest axis defines the row stride of the connectivity, so it is the one kept. + assert _fit_grid_dimensions(5, 4, 17) == (4, 4) + + def test_degenerate_dimensions_generate_no_face(self): + assert _fit_grid_dimensions(0, 4, 20) == (0, 0) + assert _fit_grid_dimensions(5, 0, 20) == (0, 0) + assert _fit_grid_dimensions(5, 4, 0) == (0, 0) + + @pytest.mark.parametrize("nb_points", [1, 2, 3, 7, 12, 19, 20, 21, 100]) + @pytest.mark.parametrize("declared", [(5, 4), (4, 5), (1, 1), (10, 10)]) + def test_product_never_exceeds_the_points_read(self, declared, nb_points): + # This is the invariant that keeps the generated indices in range: the previous + # decrement-both loop could leave sa * fa > nb_points (and even fa == 0). + sa, fa = _fit_grid_dimensions(declared[0], declared[1], nb_points) + assert sa >= 0 and fa >= 0 + assert sa * fa <= nb_points + + +# --------------------------------------------------------------------------- +# Dispatchers +# --------------------------------------------------------------------------- + + +class TestReaderDispatch: + def test_known_type_resolves_to_its_reader(self): + assert get_object_reader_function("TriangulatedSetRepresentation").__name__ == ( + "read_triangulated_set_representation" + ) + assert get_numpy_reader_function("TriangulatedSetRepresentation").__name__ == ( + "read_numpy_triangulated_set_representation" + ) + + def test_unknown_type_resolves_to_none(self): + assert get_object_reader_function("NoSuchRepresentation") is None + assert get_numpy_reader_function("NoSuchRepresentation") is None + + def test_lookup_is_stable_across_calls(self): + # The lookup is memoised; repeated calls must keep returning the same object. + first = get_object_reader_function("PointRepresentation") + assert first is get_object_reader_function("PointRepresentation") + assert get_numpy_reader_function("PointRepresentation") is get_numpy_reader_function("PointRepresentation") + + def test_property_dispatch_cannot_reach_a_geometry_reader(self): + # read_property resolves in its own module namespace; while it lived in mesh.py that + # namespace also held the geometry readers, so a PointRepresentation silently returned + # meshes instead of raising NotSupportedError. + from energyml.utils.data.properties import get_property_reader_function + + assert get_property_reader_function("TriangulatedSetRepresentation") is None + assert get_property_reader_function("PointRepresentation") is None + assert get_property_reader_function("ContinuousProperty").__name__ == "read_continuous_property" + + def test_schema_type_names_resolve_like_class_names(self): + # RESQML 2.0.1 keeps the `obj_` prefix in its schema type, which is what a content type + # and ResourceMetadata.object_type carry — the python class name drops the underscore. + # Only the latter used to be normalised, so _list_exportable_uuids found nothing at all + # in a 2.0.1 EPC and `extract_3d` exported no file. + for spelling in ( + "TriangulatedSetRepresentation", + "ObjTriangulatedSetRepresentation", + "obj_TriangulatedSetRepresentation", + "resqml20.obj_TriangulatedSetRepresentation", + ): + assert _mesh_name_mapping(spelling) == "TriangulatedSetRepresentation", spelling + assert get_object_reader_function(_mesh_name_mapping(spelling)) is not None, spelling + assert get_numpy_reader_function(_mesh_name_mapping(spelling)) is not None, spelling + + assert _mesh_name_mapping("obj_PolylineSetRepresentation") == "PolylineRepresentation" + assert _mesh_name_mapping("obj_Grid2dRepresentation") == "Grid2dRepresentation" + assert _mesh_name_mapping("obj_LocalDepth3dCrs") == "LocalDepth3dCrs" + + def test_imported_helper_is_not_mistaken_for_a_reader(self): + # mesh.py imports read_array / read_grid2d_patch / read_parametric_geometry from helper.py, + # which the `read_` convention would otherwise match on a type named + # Array or Grid2dPatch — and then call with a reader signature. + assert get_object_reader_function("Array") is None + assert get_object_reader_function("Grid2dPatch") is None + assert get_object_reader_function("ParametricGeometry") is None + + +@requires_epc201 +class TestExportableUuidListing: + """``_list_exportable_uuids`` drives ``extract_3d`` when no ``--uuid`` is given.""" + + def test_v2_0_1_representations_are_listed(self): + epc = EpcFile(epc_file_path=_EPC201, mode=EpcAccessMode.READ_ONLY) + listed = _list_exportable_uuids(epc) + + assert listed, "no exportable representation found in a v2.0.1 EPC" + + # every listed object must actually be readable as a mesh, and every representation + # that has a reader must be listed + by_uuid = {meta.uuid: meta.object_type for meta in epc.list_objects(resolve_titles=False) if meta.object_type} + expected = { + uuid + for uuid, object_type in by_uuid.items() + if get_object_reader_function(_mesh_name_mapping(object_type)) is not None + } + assert set(listed) == expected + assert any("Triangulated" in by_uuid[uuid] for uuid in listed) + + +# --------------------------------------------------------------------------- +# read_mesh_object on a list of objects +# --------------------------------------------------------------------------- + + +@requires_epc22 +class TestReadMeshObjectList: + def test_list_input_returns_meshes_not_the_input_objects(self, epc22): + obj = epc22.get_object_by_uuid(_MULTI_PATCH_UUID)[0] + + single = read_mesh_object(energyml_object=obj, workspace=epc22) + from_list = read_mesh_object(energyml_object=[obj, obj], workspace=epc22) + + assert all( + isinstance(m, AbstractMesh) for m in from_list + ), "a list input used to be returned unchanged, i.e. the energyml objects themselves" + assert len(from_list) == 2 * len(single) + + +# --------------------------------------------------------------------------- +# read_property_interpreted_with_cbt error path +# --------------------------------------------------------------------------- + + +class TestCategoryLookupErrorPath: + def test_unsupported_lookup_type_raises_not_supported(self, monkeypatch): + # The property readers live in properties.py now; mesh.py only re-exports them. + import energyml.utils.data.properties as mesh_module + + class _Dor: + pass + + class _Prop: + category_lookup = _Dor() + + class _Workspace: + def get_object(self, _uri): + return object() + + monkeypatch.setattr(mesh_module, "get_obj_uri", lambda _o: "uri") + # Anything that is neither a list/ndarray nor a dict reaches the error branch, which + # used to reference a variable bound only in the dict branch -> NameError. + monkeypatch.setattr(mesh_module, "read_column_based_table", lambda *_a, **_k: 42) + + with pytest.raises(NotSupportedError) as exc_info: + read_property_interpreted_with_cbt( + _Prop(), + _Workspace(), + _cache_property_arrays=np.array([0, 1]), + ) + assert "int" in str(exc_info.value) + + +# --------------------------------------------------------------------------- +# sub_indices window alignment across patches +# --------------------------------------------------------------------------- + + +class TestGrid2dQuadConnectivity: + """The quad connectivity is built by broadcasting instead of a Python double loop. + + The expected arrays below are written out explicitly rather than derived from a second + implementation, so the test stays a specification of the connectivity. + """ + + @staticmethod + def _read_grid(monkeypatch, points, sa, fa, keep_holes): + import energyml.utils.data.mesh_numpy as mesh_numpy + + class _Grid: + # A plain class, not a MagicMock: `hasattr(obj, "geometry")` selects the RESQML 2.2 + # branch, and a mock would answer True to everything and produce a second patch. + geometry = object() + + monkeypatch.setattr(mesh_numpy, "read_grid2d_patch", lambda **_k: points) + monkeypatch.setattr( + mesh_numpy, + "search_attribute_matching_name", + lambda _obj, name: [fa] if "Fastest" in name else [sa], + ) + monkeypatch.setattr(mesh_numpy, "search_attribute_matching_name_with_path", lambda *_a, **_k: []) + monkeypatch.setattr(mesh_numpy, "get_crs_obj", lambda **_k: None) + monkeypatch.setattr(mesh_numpy, "get_obj_uuid", lambda _o: "uuid") + monkeypatch.setattr(mesh_numpy, "get_obj_uri", lambda _o: "uri") + + result = mesh_numpy.read_numpy_grid2d_representation(_Grid(), workspace=None, keep_holes=keep_holes) + return result.flat_patches() + + def test_full_2x3_grid(self, monkeypatch): + # 2 rows x 3 columns of nodes -> 1 x 2 = 2 quads. + points = [[float(i), float(j), 0.0] for j in range(2) for i in range(3)] + patches = self._read_grid(monkeypatch, points, sa=2, fa=3, keep_holes=True) + + assert len(patches) == 1 + # VTK flat format: [4, a, b, c, d, 4, a, b, c, d] + np.testing.assert_array_equal( + patches[0].faces, + [4, 0, 1, 4, 3, 4, 1, 2, 5, 4], + ) + + def test_hole_drops_only_the_cells_that_touch_it(self, monkeypatch): + # 3x3 nodes -> 2x2 = 4 quads. Node 4 (the centre) is a hole, and it is a corner of all + # four cells, so every cell disappears while the 8 remaining nodes are kept. + points = [[float(i), float(j), 0.0] for j in range(3) for i in range(3)] + points[4][2] = float("nan") + patches = self._read_grid(monkeypatch, points, sa=3, fa=3, keep_holes=False) + assert patches == [] or len(patches[0].faces) == 0 + + def test_hole_in_a_corner_keeps_the_other_cells(self, monkeypatch): + # 3x3 nodes, node 0 is a hole: it belongs to the first cell only. + points = [[float(i), float(j), 0.0] for j in range(3) for i in range(3)] + points[0][2] = float("nan") + patches = self._read_grid(monkeypatch, points, sa=3, fa=3, keep_holes=False) + + assert len(patches) == 1 + assert len(patches[0].points) == 8, "only the NaN node is dropped" + # 3 of the 4 cells survive; indices are renumbered over the 8 surviving nodes. + faces = patches[0].faces + assert len(faces) == 3 * 5 + quads = faces.reshape(-1, 5) + assert (quads[:, 0] == 4).all() + np.testing.assert_array_equal( + quads[:, 1:], + [[0, 1, 4, 3], [2, 3, 6, 5], [3, 4, 7, 6]], + ) + + def test_degenerate_dimensions_produce_no_patch(self, monkeypatch): + points = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0]] + assert self._read_grid(monkeypatch, points, sa=1, fa=2, keep_holes=True) == [] + + +@requires_epc22 +class TestLegacyAdapter: + """``mesh.py``'s geometry readers delegate to ``mesh_numpy`` and convert the result back. + + The full before/after comparison was done over every representation of every fixture in + ``rc/epc/`` (1155 objects, 942 meshes): identifiers, coordinates, index structures and + edge/face counts all matched. These tests pin the properties that comparison relied on. + """ + + def test_point_list_is_a_numpy_array(self, epc22): + # Documented change: the legacy containers now hold the (N, 3) float64 array produced by + # the numpy reader instead of a list of lists. The field was already annotated + # Union[List[Point], np.ndarray]. + obj = epc22.get_object_by_uuid(_MULTI_PATCH_UUID)[0] + meshes = read_mesh_object(obj, workspace=epc22) + + assert meshes + for m in meshes: + assert isinstance(m.point_list, np.ndarray) + assert m.point_list.dtype == np.float64 + assert m.point_list.ndim == 2 and m.point_list.shape[1] == 3 + + def test_indices_are_plain_lists_of_int(self, epc22): + # get_indices() feeds the OBJ / OFF / GeoJSON writers, which index and len() it. + obj = epc22.get_object_by_uuid(_MULTI_PATCH_UUID)[0] + mesh = read_mesh_object(obj, workspace=epc22)[0] + + indices = mesh.get_indices() + assert isinstance(indices, list) + assert all(isinstance(face, list) for face in indices) + assert all(isinstance(i, int) for i in indices[0]) + + def test_legacy_identifiers_are_preserved(self, epc22): + obj = epc22.get_object_by_uuid(_MULTI_PATCH_UUID)[0] + meshes = read_mesh_object(obj, workspace=epc22) + + from energyml.utils.introspection import get_obj_uri + + uri = get_obj_uri(obj) + for index, mesh in enumerate(meshes): + assert mesh.identifier == f"{uri}_patch{index}" + + def test_point_sets_keep_their_own_naming(self, epc22): + obj = _first_of_type(epc22, "PointSet") + if obj is None: + pytest.skip("no PointSetRepresentation in the fixture") + meshes = read_mesh_object(obj, workspace=epc22) + assert meshes[0].identifier == "Patch num 0" + + def test_volumetric_types_are_refused_with_a_pointer_to_the_numpy_stack(self, epc22): + # AbstractMesh models points, polylines and surfaces only. + from energyml.utils.data.mesh import read_ijk_grid_representation + + with pytest.raises(NotSupportedError) as exc_info: + read_ijk_grid_representation(object(), workspace=epc22) + assert "mesh_numpy" in str(exc_info.value) + + +@requires_epc22 +class TestSubIndicesPatchAlignment: + def test_partial_selection_spanning_two_patches(self, epc22): + obj = epc22.get_object_by_uuid(_MULTI_PATCH_UUID)[0] + + every_triangle = _all_triangles(read_numpy_mesh_object(obj, workspace=epc22)) + assert len(every_triangle) == 20, "fixture changed: expected 5 patches x 4 triangles" + + # 0 and 1 live in patch 0, 5 and 6 in patch 1. Because the first patch drops half of + # its faces, an offset advanced by the filtered count desynchronises every later patch. + selection = [0, 1, 5, 6] + selected = _all_triangles(read_numpy_mesh_object(obj, workspace=epc22, sub_indices=selection)) + + assert len(selected) == len(selection) + np.testing.assert_array_equal(selected, every_triangle[selection]) + + def test_full_selection_is_identity(self, epc22): + obj = epc22.get_object_by_uuid(_MULTI_PATCH_UUID)[0] + + every_triangle = _all_triangles(read_numpy_mesh_object(obj, workspace=epc22)) + selected = _all_triangles( + read_numpy_mesh_object(obj, workspace=epc22, sub_indices=list(range(len(every_triangle)))) + ) + + np.testing.assert_array_equal(selected, every_triangle) diff --git a/energyml-utils/tests/test_mesh_numpy.py b/energyml-utils/tests/test_mesh_numpy.py index 236e6b2..09704d5 100644 --- a/energyml-utils/tests/test_mesh_numpy.py +++ b/energyml-utils/tests/test_mesh_numpy.py @@ -421,15 +421,23 @@ def test_wellbore_frame_returns_polyline(self, epc22): assert m.points.shape[1] == 3 def test_wellbore_frame_lines_vtk_format(self, epc22): + """A wellbore is *one* polyline through its stations, not N-1 two-point cells. + + Both encodings draw the same picture, but every consumer that iterates the cells — + the GeoJSON writer, OBJ, OFF — turns the segment one into N-1 elements for a single + well, each repeating the object's whole metadata block. + """ obj = epc22.get_object_by_uuid("d873e243-d893-41ab-9a3e-d20b851c099f") if not obj: pytest.skip("WellboreFrame UUID not found in fixture EPC") multi = read_numpy_mesh_object(obj[0], workspace=epc22) for m in multi.flat_patches(): assert isinstance(m, NumpyPolylineMesh) - if len(m.lines) > 0: - # First element is count (number of points in first line segment) - assert m.lines[0] == 2, "VTK segment should start with count=2" + if len(m.lines) == 0: + continue + # VTK flat format: [n, i0, i1, ..., i(n-1)] — a single cell spanning every point. + assert m.lines[0] == len(m.points), "the frame should be one polyline through all its points" + assert len(m.lines) == len(m.points) + 1, "expected exactly one VTK cell" # --- Grid2dRepresentation --- def test_grid2d_returns_surface_mesh(self, epc22): diff --git a/energyml-utils/tests/test_mesh_numpy_ijk_spec.py b/energyml-utils/tests/test_mesh_numpy_ijk_spec.py new file mode 100644 index 0000000..31c54a2 --- /dev/null +++ b/energyml-utils/tests/test_mesh_numpy_ijk_spec.py @@ -0,0 +1,263 @@ +"""Conformance of the IJK / connection-set readers to the RESQML specification. + +Everything here runs against ``rc/epc/testingPackageCpp22.epc`` — the FESAPI example package, +which is the only fixture family cleared for publication, and which happens to be exactly what +these tests need: the same grid shipped left- *and* right-handed, explicit *and* parametric +geometry, faulted and unfaulted, K-gaps, undefined cells, an LGR, and three grid connection +sets. Expected values are read out of ``testingPackageCpp22.h5``, not produced by the reader. + +Run from the workspace root: + poetry run pytest tests/test_mesh_numpy_ijk_spec.py -v +""" + +import os + +import numpy as np +import pytest + +from energyml.utils.data.mesh_numpy import ( + NumpySurfaceMesh, + read_numpy_mesh_object, +) + +_WORKSPACE_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_EPC22 = os.path.join(_WORKSPACE_ROOT, "rc", "epc", "testingPackageCpp22.epc") + +pytestmark = pytest.mark.skipif( + not os.path.isfile(_EPC22), + reason="testingPackageCpp22.epc not found in rc/epc/", +) + + +@pytest.fixture(scope="module") +def ws(): + from energyml.utils.epc_file import EpcFile + + return EpcFile(_EPC22) + + +def _unpack_vtk_cells(mesh) -> list: + """Split a VTK flat cell array into one node array per cell.""" + out, off = [], 0 + for _ in mesh.cell_types: + n = int(mesh.cells[off]) + out.append(np.asarray(mesh.cells[off + 1 : off + 1 + n], dtype=np.int64)) + off += 1 + n + assert off == len(mesh.cells), "cell array length does not match cell_types" + return out + + +def _hex_signed_volume(p: np.ndarray) -> float: + """Signed volume of a VTK hexahedron given its 8 corner points, shape (8, 3).""" + tets = [(0, 1, 3, 4), (1, 2, 3, 6), (1, 4, 5, 6), (3, 4, 6, 7), (1, 3, 4, 6)] + return float(sum(np.dot(np.cross(p[b] - p[a], p[c] - p[a]), p[d] - p[a]) / 6.0 for a, b, c, d in tets)) + + +def _grid(ws, uuid): + obj = ws.get_object(f"eml:///resqml22.IjkGridRepresentation({uuid})") + if obj is None: + pytest.skip(f"IjkGridRepresentation {uuid} not in fixture") + return obj + + +def _mesh_of(ws, uuid, local: bool = False): + """Read a grid, by default in the PROJECTED frame — the one a viewer renders. + + Handedness is only meaningful there: the local CRS of these fixtures measures Z as a depth, + which makes (X, Y, Z) left-handed on its own, so the sign of a cell's Jacobian in the local + frame says nothing about ``GridIsRighthanded``. + """ + multi = read_numpy_mesh_object(_grid(ws, uuid), workspace=ws, use_crs_displacement=not local) + return multi.flat_patches()[0] + + +def _points_of(ws, uuid): + multi = read_numpy_mesh_object(_grid(ws, uuid), workspace=ws, use_crs_displacement=False) + return np.concatenate([p.points for p in multi.flat_patches()]) + + +class TestParametricGeometry: + def test_split_coordinate_line_reuses_its_parent_pillar_line(self, ws): + """A split coordinate line takes the parametric line of ``PillarIndices``. + + "Four faulted sugar cubes (parametric geometry)": 6 pillars, 2 split lines, every line + vertical, so a node is (X, Y of the line, P). ``PillarIndices`` = [1, 4] and the split + columns are parameterised 50 m below their parent — the fault throw. + """ + xy = np.array([(0, 0), (375, 0), (700, 0), (0, 150), (375, 150), (700, 150)], dtype=float) + params = np.array( + [ + [300, 300, 350, 300, 300, 350, 350, 350], + [400, 400, 450, 400, 400, 450, 450, 450], + [500, 500, 550, 500, 500, 550, 550, 550], + ], + dtype=float, + ) + line_of_col = [0, 1, 2, 3, 4, 5, 1, 4] + expected = np.array( + [[xy[line_of_col[c]][0], xy[line_of_col[c]][1], params[k, c]] for k in range(3) for c in range(8)] + ) + np.testing.assert_allclose(_points_of(ws, "37c45c00-fa3e-11e5-a21e-0002a5d5c51b"), expected, atol=1e-9) + + def test_vertical_lines_place_z_at_the_parameter(self, ws): + """RESQML vertical line: "Control points are (X,Y,-) [...] parameter values are depth".""" + xy = np.array([(0, 0), (700, 0), (0, 150), (700, 150)], dtype=float) + expected = np.array([[xy[c][0], xy[c][1], z] for z in (300.0, 500.0) for c in range(4)]) + np.testing.assert_allclose(_points_of(ws, "53bb70fe-2eef-4691-b4fe-14541e3a57eb"), expected, atol=1e-9) + + def test_nan_padded_knots_do_not_poison_the_lines(self, ws): + """``KnotCount`` pads the shorter lines with NaN — those knots must be trimmed. + + "Four faulted sugar cubes with one cubic pillar" declares KnotCount=3, yet four of its + six pillars are vertical with one real knot and pillar 0 is a 2-knot linear spline. + """ + pts = _points_of(ws, "3ce91933-4f6f-4f35-b0ac-4ba4672f0a87") + assert not np.isnan(pts).any(), "NaN padding leaked into the evaluated geometry" + + # Pillar 0 is linear between (0,0,300)@P=300 and (50,30,1000)@P=1000. + for k, p in enumerate([300.0, 400.0, 500.0]): + t = (p - 300.0) / 700.0 + np.testing.assert_allclose(pts[k * 8], [t * 50.0, t * 30.0, 300.0 + t * 700.0], atol=1e-9) + + +class TestCellConstruction: + @pytest.mark.parametrize( + "uuid", + [ + "e96c2bde-e3ae-4d51-b078-a8e57fb1e667", # Four by Three by Two Left Handed + "4fc004e1-0f7d-46a8-935e-588f790a6f84", # Four by Three by Two Right Handed + ], + ) + def test_hexahedra_are_positively_oriented(self, ws, uuid): + """``GridIsRighthanded`` sets the winding; VTK always wants a positive Jacobian. + + The fixture ships the same grid twice, left- and right-handed, for exactly this case: + before the flag was honoured, every cell of the left-handed one came out inside-out. + Measured in the projected frame — see :func:`_mesh_of`. + """ + mesh = _mesh_of(ws, uuid) + for nodes in _unpack_vtk_cells(mesh): + if len(nodes) != 8: + continue # a cell without geometry + assert _hex_signed_volume(mesh.points[nodes]) > 0, f"inverted hexahedron in {uuid}" + + def test_cells_follow_the_resqml_ordering(self, ws): + """Cells come out I fastest, then J, then K — the order the grid's properties use.""" + uuid = "4fc004e1-0f7d-46a8-935e-588f790a6f84" # 4 x 3 x 2 + grid = _grid(ws, uuid) + ni, nj, nk = int(grid.ni), int(grid.nj), int(grid.nk) + mesh = _mesh_of(ws, uuid) + cells = _unpack_vtk_cells(mesh) + assert len(cells) == ni * nj * nk + + def pillar_xy(j, i): + return mesh.points[j * (ni + 1) + i][:2] + + for c, nodes in enumerate(cells): + if len(nodes) != 8: + continue + k, j, i = c // (ni * nj), (c // ni) % nj, c % ni + centroid = mesh.points[nodes].mean(axis=0) + xs = sorted([pillar_xy(j, i)[0], pillar_xy(j, i + 1)[0]]) + ys = sorted([pillar_xy(j, i)[1], pillar_xy(j + 1, i)[1]]) + assert xs[0] - 1e-6 <= centroid[0] <= xs[1] + 1e-6, f"cell {c} is not at I={i} (k={k}, j={j})" + assert ys[0] - 1e-6 <= centroid[1] <= ys[1] + 1e-6, f"cell {c} is not at J={j} (k={k}, i={i})" + + def test_undefined_cells_stay_in_place_as_empty_cells(self, ws): + """``CellGeometryIsDefined``=false keeps its slot so cell-indexed properties still align. + + The HDF5 of "Four by Three by Two Right Handed" stores the flag as (NK, NJ, NI) with + zeros at flat indices 0, 11 and 23. + """ + mesh = _mesh_of(ws, "4fc004e1-0f7d-46a8-935e-588f790a6f84") + empty = [c for c, t in enumerate(mesh.cell_types) if int(t) == 0] + assert empty == [0, 11, 23] + cells = _unpack_vtk_cells(mesh) + assert all(len(cells[c]) == 0 for c in empty), "an empty cell must list no node" + + def test_k_gaps_do_not_change_the_cell_count(self, ws): + """A K-gap adds a node layer, not a cell: NKL = NK + gapCount + 1.""" + grid = _grid(ws, "c14755a5-e3b3-4272-99e5-fc20993b79a0") # ... with gap layer + mesh = _mesh_of(ws, "c14755a5-e3b3-4272-99e5-fc20993b79a0") + assert len(mesh.cell_types) == int(grid.ni) * int(grid.nj) * int(grid.nk) + + def test_lgr_without_geometry_returns_empty_rather_than_raising(self, ws): + """A grid whose geometry is inherited through ``ParentWindow`` is reported, not crashed on.""" + grid = _grid(ws, "2aec1720-fa3e-11e5-a116-0002a5d5c51b") + assert getattr(grid, "geometry", None) is None + multi = read_numpy_mesh_object(grid, workspace=ws, use_crs_displacement=False) + assert multi.flat_patches() == [] + + +class TestGridConnectionSet: + @pytest.mark.parametrize( + "uuid", + [ + "03bb6fc0-fa3e-11e5-8c09-0002a5d5c51b", + "20b480a8-5e3b-4336-8f6e-1b3099c2c60f", + "a3d1462a-04e3-4374-921b-a4a1e9ba3ea3", + ], + ) + def test_faces_land_on_the_fault_plane(self, ws, uuid): + """Every connection set of the fixture faults its grid at X=375. + + This is what pins the local face-per-cell indices the files use (3 = I+, 5 = I-): a wrong + mapping would return the J or K faces, which are not planar in X. + """ + obj = ws.get_object(f"eml:///resqml22.GridConnectionSetRepresentation({uuid})") + if obj is None: + pytest.skip(f"GridConnectionSet {uuid} not in fixture") + patches = read_numpy_mesh_object(obj, workspace=ws, use_crs_displacement=False).flat_patches() + assert patches, "expected the connection faces" + mesh = patches[0] + assert isinstance(mesh, NumpySurfaceMesh) + + off, n_faces = 0, 0 + while off < len(mesh.faces): + n = int(mesh.faces[off]) + assert n == 4, "the face of an IJK cell is a quad" + nodes = mesh.faces[off + 1 : off + 1 + n] + np.testing.assert_allclose(mesh.points[nodes][:, 0], 375.0, atol=1e-9) + off += 1 + n + n_faces += 1 + assert n_faces > 0 + assert len(mesh.extra_arrays["connection_index"]) == n_faces + + def test_interpretation_index_is_exposed(self, ws): + """``ConnectionInterpretations`` is what lets a viewer colour the set by fault.""" + obj = ws.get_object("eml:///resqml22.GridConnectionSetRepresentation(03bb6fc0-fa3e-11e5-8c09-0002a5d5c51b)") + if obj is None: + pytest.skip("GridConnectionSet not in fixture") + mesh = read_numpy_mesh_object(obj, workspace=ws, use_crs_displacement=False).flat_patches()[0] + interp = mesh.extra_arrays["interpretation_index"] + assert set(np.unique(interp)) <= {-1, 0} + assert (interp == 0).any(), "the interpreted connection should be flagged" + + +class TestNewlySupportedRepresentations: + def test_wellbore_marker_frame_gives_one_point_per_marker(self, ws): + obj = ws.get_object( + "eml:///resqml20.obj_WellboreMarkerFrameRepresentation(657d5e6b-1752-425d-b3e7-237037fa11eb)" + ) + if obj is None: + pytest.skip("WellboreMarkerFrame not in fixture") + patches = read_numpy_mesh_object(obj, workspace=ws).flat_patches() + assert patches, "expected the marker positions" + assert patches[0].points.shape[1] == 3 + assert len(patches[0].points) == int(obj.node_count) + # The patch reports the marker frame, not the trajectory it took the geometry from. + assert patches[0].source_type == type(obj).__name__ + + def test_no_representation_of_the_fixture_fails(self, ws): + """Regression net for the dispatcher: the file must read end to end.""" + failures = [] + for ref in ws.list_objects(resolve_titles=False): + uri = str(ref.uri) + obj = ws.get_object(uri) + if "Representation" not in type(obj).__name__: + continue + try: + read_numpy_mesh_object(obj, workspace=ws) + except Exception as exc: # noqa: BLE001 — the point is to report, not to classify + failures.append(f"{uri}: {type(exc).__name__}: {exc}") + assert not failures, "\n".join(failures) diff --git a/energyml-utils/tests/test_point_frame.py b/energyml-utils/tests/test_point_frame.py new file mode 100644 index 0000000..4afff42 --- /dev/null +++ b/energyml-utils/tests/test_point_frame.py @@ -0,0 +1,456 @@ +"""Tests for the unified CRS pipeline: PointFrame / to_frame / compute_origin_shift. + +The pipeline replaces two mechanisms that used to coexist: + +* ``apply_from_crs_info`` — the full local -> projected transform, and +* ``crs_displacement_np`` — offsets and Z flip only, + +selected by a list of type-name substrings in each dispatcher. A reader missing from the list +had its points transformed twice; a reader wrongly listed kept raw coordinates. ``Grid2d`` was in +neither list, so the same object came out rotated from :mod:`mesh` and un-rotated from +:mod:`mesh_numpy`. + +Note: every Grid2d fixture in ``rc/epc/`` has a zero areal rotation and an easting-first axis +order, so that divergence is invisible on real files — hence the synthetic rotation in +:class:`TestGrid2dGetsTheFullTransform`. +""" +import math +import os + +import numpy as np +import pytest + +from energyml.utils.data.crs import ( + CrsInfo, + FramedPoints, + PointFrame, + apply_from_crs_info, + compute_origin_shift, + is_pyproj_available, + reproject_to_wgs84, + to_frame, +) +from energyml.utils.data.mesh_numpy import _ensure_float64_points, read_numpy_mesh_object +from energyml.utils.epc_file import EpcAccessMode, EpcFile +from energyml.utils.exception import NotSupportedError + +_WORKSPACE_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_EPC22 = os.path.join(_WORKSPACE_ROOT, "rc", "epc", "testingPackageCpp22.epc") + +requires_epc22 = pytest.mark.skipif(not os.path.isfile(_EPC22), reason="testingPackageCpp22.epc fixture is missing") +requires_pyproj = pytest.mark.skipif(not is_pyproj_available(), reason="requires the 'crs' extra (pyproj)") + + +@pytest.fixture +def epc22(): + yield EpcFile(epc_file_path=_EPC22, mode=EpcAccessMode.READ_ONLY) + + +def _first_object_of_type(epc, type_fragment: str): + for meta in epc.list_objects(resolve_titles=False): + if type_fragment in (getattr(meta, "object_type", "") or ""): + return epc.get_object_by_uuid(meta.uuid)[0] + return None + + +def _rotated_crs_info(angle_rad: float) -> CrsInfo: + """A CrsInfo whose *only* effect is an areal rotation.""" + return CrsInfo( + x_offset=0.0, + y_offset=0.0, + z_offset=0.0, + areal_rotation_value=angle_rad, + areal_rotation_uom="rad", + z_increasing_downward=False, + projected_axis_order="easting northing", + ) + + +# --------------------------------------------------------------------------- +# Frame ordering +# --------------------------------------------------------------------------- + + +class TestFrameOrdering: + def test_stages_are_ordered(self): + assert PointFrame.LOCAL.stage < PointFrame.PROJECTED.stage < PointFrame.WGS84.stage + + def test_going_backwards_is_refused(self): + pts = np.zeros((2, 3)) + with pytest.raises(NotSupportedError): + to_frame(pts, CrsInfo(), PointFrame.LOCAL, PointFrame.PROJECTED) + with pytest.raises(NotSupportedError): + to_frame(pts, CrsInfo(), PointFrame.PROJECTED, PointFrame.WGS84) + + def test_same_frame_is_a_no_op(self): + pts = np.array([[1.0, 2.0, 3.0]]) + result = to_frame(pts, _rotated_crs_info(math.pi / 3), PointFrame.LOCAL, PointFrame.LOCAL) + assert isinstance(result, FramedPoints) + assert result.frame is PointFrame.LOCAL + np.testing.assert_array_equal(result.points, [[1.0, 2.0, 3.0]]) + + def test_projected_target_on_already_projected_points_changes_nothing(self): + # This is what makes the double transform impossible: the frame is carried, so a second + # pass through the dispatcher cannot re-apply the offsets. + pts = np.array([[420000.0, 6470000.0, -100.0]]) + crs_info = CrsInfo(x_offset=420000.0, y_offset=6470000.0) + result = to_frame(pts, crs_info, PointFrame.PROJECTED, PointFrame.PROJECTED) + np.testing.assert_array_equal(result.points, [[420000.0, 6470000.0, -100.0]]) + + +# --------------------------------------------------------------------------- +# Stage 1: LOCAL -> PROJECTED +# --------------------------------------------------------------------------- + + +class TestLocalToProjected: + def test_matches_apply_from_crs_info(self): + crs_info = CrsInfo( + x_offset=1000.0, + y_offset=2000.0, + z_offset=15.0, + areal_rotation_value=0.3, + areal_rotation_uom="rad", + z_increasing_downward=True, + projected_axis_order="easting northing", + ) + pts = np.array([[10.0, 20.0, 30.0], [-5.0, 7.5, 0.0]]) + + expected = apply_from_crs_info(pts.copy(), crs_info, inplace=False) + result = to_frame(pts.copy(), crs_info, PointFrame.PROJECTED, PointFrame.LOCAL) + + assert result.frame is PointFrame.PROJECTED + np.testing.assert_allclose(result.points, expected) + + def test_rotation_is_applied(self): + angle = math.pi / 2 + result = to_frame( + np.array([[1.0, 0.0, 0.0]]), + _rotated_crs_info(angle), + PointFrame.PROJECTED, + PointFrame.LOCAL, + ) + # RESQML rotation is clockwise: x' = x·cos + y·sin, y' = -x·sin + y·cos + np.testing.assert_allclose(result.points[0], [0.0, -1.0, 0.0], atol=1e-12) + + def test_without_crs_info_the_frame_stays_local(self): + pts = np.array([[1.0, 2.0, 3.0]]) + result = to_frame(pts, None, PointFrame.PROJECTED, PointFrame.LOCAL) + assert result.frame is PointFrame.LOCAL + assert result.degraded_reason is not None + np.testing.assert_array_equal(result.points, [[1.0, 2.0, 3.0]]) + + def test_inplace_false_leaves_the_source_untouched(self): + pts = np.array([[10.0, 20.0, 30.0]]) + original = pts.copy() + result = to_frame(pts, CrsInfo(x_offset=5.0), PointFrame.PROJECTED, PointFrame.LOCAL, inplace=False) + np.testing.assert_array_equal(pts, original) + assert result.points[0, 0] == 15.0 + + +# --------------------------------------------------------------------------- +# Stage 2: PROJECTED -> WGS84 degradation +# --------------------------------------------------------------------------- + + +class TestWgs84Degradation: + def test_missing_epsg_degrades_to_projected(self): + # No projected_epsg_code -> reproject_to_wgs84 cannot run. The points must stay usable + # and the caller must be able to tell that WGS84 was not reached. + pts = np.array([[420000.0, 6470000.0, -100.0]]) + result = to_frame(pts.copy(), CrsInfo(), PointFrame.WGS84, PointFrame.PROJECTED) + + assert result.frame is PointFrame.PROJECTED + assert result.degraded_reason is not None + np.testing.assert_allclose(result.points, pts) + + def test_empty_array_reports_the_requested_frame(self): + # An empty patch must not make a whole FeatureCollection look degraded. + result = to_frame(np.empty((0, 3)), CrsInfo(), PointFrame.WGS84, PointFrame.LOCAL) + assert result.frame is PointFrame.WGS84 + assert result.degraded_reason is None + + +# --------------------------------------------------------------------------- +# Origin shift +# --------------------------------------------------------------------------- + + +class TestOriginShift: + def test_shift_is_shared_across_arrays(self): + # The whole point: one vector for every patch, so patches do not move relative to + # each other. A per-patch centroid would give each array a different shift. + a = np.array([[0.0, 0.0, 0.0], [100.0, 100.0, 10.0]]) + b = np.array([[900.0, 500.0, 0.0], [1000.0, 600.0, 20.0]]) + + shift = compute_origin_shift([a, b]) + assert shift == (500.0, 300.0, 10.0) + + moved_a = to_frame(a.copy(), None, PointFrame.LOCAL, PointFrame.LOCAL, origin_shift=shift).points + moved_b = to_frame(b.copy(), None, PointFrame.LOCAL, PointFrame.LOCAL, origin_shift=shift).points + + # Relative geometry preserved + np.testing.assert_allclose(moved_b - moved_a, b - a) + # And the coordinates are now small + assert np.max(np.abs(np.concatenate([moved_a, moved_b]))) < np.max(np.abs(np.concatenate([a, b]))) + + def test_applied_shift_is_reported(self): + result = to_frame( + np.array([[10.0, 10.0, 10.0]]), + None, + PointFrame.LOCAL, + PointFrame.LOCAL, + origin_shift=(1.0, 2.0, 3.0), + ) + assert result.origin_shift == (1.0, 2.0, 3.0) + np.testing.assert_allclose(result.points, [[9.0, 8.0, 7.0]]) + + def test_no_points_gives_a_zero_shift(self): + assert compute_origin_shift([]) == (0.0, 0.0, 0.0) + assert compute_origin_shift([np.empty((0, 3))]) == (0.0, 0.0, 0.0) + + def test_nan_only_input_does_not_produce_a_nan_shift(self): + # Grid2d holes are stored as NaN; a NaN shift would wipe out every coordinate. + shift = compute_origin_shift([np.full((3, 3), np.nan)]) + assert shift == (0.0, 0.0, 0.0) + + +# --------------------------------------------------------------------------- +# Chunked reprojection +# --------------------------------------------------------------------------- + + +@requires_pyproj +class TestReprojectionChunking: + """The reprojection copies each axis into a reusable scratch buffer, block by block, so its + scratch memory is constant instead of proportional to the point count. These tests pin the + behaviour that the blocking must not change.""" + + CRS = CrsInfo(projected_epsg_code=32631) + + def _points(self, n): + pts = np.empty((n, 3), dtype=np.float64) + pts[:, 0] = np.linspace(400000.0, 440000.0, n) + pts[:, 1] = np.linspace(6470000.0, 6510000.0, n) + pts[:, 2] = np.linspace(-3000.0, 0.0, n) + return pts + + def test_result_is_independent_of_the_chunk_size(self, monkeypatch): + import energyml.utils.data.crs as crs_module + + pts = self._points(1000) + single_block = reproject_to_wgs84(pts.copy(), self.CRS, inplace=False) + + # A chunk size that does not divide the point count exercises the ragged last block. + monkeypatch.setattr(crs_module, "_REPROJECT_CHUNK", 137) + many_blocks = reproject_to_wgs84(pts.copy(), self.CRS, inplace=False) + + np.testing.assert_allclose(many_blocks, single_block, rtol=0, atol=0) + + def test_inplace_and_copy_agree(self): + pts = self._points(500) + copied = reproject_to_wgs84(pts.copy(), self.CRS, inplace=False) + target = pts.copy() + returned = reproject_to_wgs84(target, self.CRS, inplace=True) + + assert returned is target, "inplace=True must write into the array it was given" + np.testing.assert_allclose(target, copied) + + def test_inplace_false_leaves_the_source_untouched(self): + pts = self._points(300) + original = pts.copy() + reproject_to_wgs84(pts, self.CRS, inplace=False) + np.testing.assert_array_equal(pts, original) + + def test_single_point_takes_the_scalar_path(self): + one = self._points(1) + result = reproject_to_wgs84(one.copy(), self.CRS, inplace=False) + assert result.shape == (1, 3) + # Same answer as when that point is part of a longer array. + pair = np.vstack([one, one]) + np.testing.assert_allclose(result[0], reproject_to_wgs84(pair, self.CRS, inplace=False)[0]) + + +@requires_pyproj +class TestUnusableVerticalCrs: + """A file may declare a vertical EPSG code PROJ cannot resolve — typically a *datum* code + where a CRS code was expected (``EPSG:6230``, the ED50 datum, in the Volve export). The + compound ``EPSG:h+EPSG:v`` then fails to build; refusing the whole reprojection would leave + the coordinates in their projected CRS although the horizontal part is perfectly usable.""" + + #: ED50 / UTM zone 31N, and the ED50 *datum* code the file declares as its vertical CRS. + BROKEN = CrsInfo(projected_epsg_code=23031, vertical_epsg_code=6230) + HORIZONTAL_ONLY = CrsInfo(projected_epsg_code=23031) + + def _points(self, n=64): + pts = np.empty((n, 3), dtype=np.float64) + pts[:, 0] = np.linspace(435000.0, 436000.0, n) + pts[:, 1] = np.linspace(6477000.0, 6478000.0, n) + pts[:, 2] = np.linspace(-3000.0, -2500.0, n) + return pts + + def test_horizontal_reprojection_still_happens(self): + pts = self._points() + result = reproject_to_wgs84(pts.copy(), self.BROKEN, inplace=False) + + # Volve, North Sea: ~1.9 E, ~58.4 N. + assert 1.0 < result[0, 0] < 3.0 + assert 57.0 < result[0, 1] < 60.0 + + # Same answer as a CRS that never declared a vertical code, up to the ellipsoidal height + # the ED50 -> WGS84 datum shift is fed: that path hands Z to PROJ, this one does not + # (Z is not a height here), which moves the result by well under a metre. + horizontal = reproject_to_wgs84(pts.copy(), self.HORIZONTAL_ONLY, inplace=False) + np.testing.assert_allclose(result[:, :2], horizontal[:, :2], rtol=0, atol=1e-5) + + def test_z_is_passed_through_untouched(self): + pts = self._points() + result = reproject_to_wgs84(pts.copy(), self.BROKEN, inplace=False) + # The vertical CRS was dropped, so Z is still in its source frame — not silently + # datum-shifted, and not sign-flipped either. + np.testing.assert_array_equal(result[:, 2], pts[:, 2]) + + def test_single_point_path_agrees(self): + pts = self._points() + full = reproject_to_wgs84(pts.copy(), self.BROKEN, inplace=False) + one = reproject_to_wgs84(pts[:1].copy(), self.BROKEN, inplace=False) + np.testing.assert_allclose(one[0], full[0]) + + def test_frame_reaches_wgs84(self): + framed = to_frame(self._points(), self.BROKEN, PointFrame.WGS84, PointFrame.PROJECTED) + assert framed.frame is PointFrame.WGS84 + assert framed.degraded_reason is None + + def test_a_missing_horizontal_code_still_raises(self): + from energyml.utils.exception import NotEnoughInformationError + + with pytest.raises(NotEnoughInformationError): + reproject_to_wgs84(self._points(), CrsInfo(vertical_epsg_code=6230), inplace=False) + + def test_depth_vertical_crs_flips_z_without_touching_the_source(self): + # EPSG:5715 (MSL depth) is a *depth* CRS. Two negations are in play and they must not be confused: + # - ours, because the Z column holds heights (z_is_up) while the CRS expects depths; + # - PROJ's own, when it converts that depth axis to the ellipsoidal height of EPSG:4979. + # With z_is_up=True the two cancel out, so the height comes back roughly unchanged; with + # z_is_up=False only PROJ's negation applies. Comparing the two isolates our flip, which + # is now folded into the per-block scratch fill instead of a full-size copy. + crs = CrsInfo(projected_epsg_code=32631, vertical_epsg_code=5715) + pts = self._points(64) + original = pts.copy() + + as_height = reproject_to_wgs84(pts, crs, inplace=False, z_is_up=True) + np.testing.assert_array_equal(pts, original, err_msg="inplace=False must not touch the source") + as_depth = reproject_to_wgs84(original.copy(), crs, inplace=False, z_is_up=False) + + np.testing.assert_allclose(as_height[:, 2], -as_depth[:, 2], atol=1e-6) + # And the flip is only about Z: longitude / latitude are untouched by it. + np.testing.assert_allclose(as_height[:, :2], as_depth[:, :2]) + + def test_chunking_holds_with_a_depth_crs_too(self, monkeypatch): + # The Z flip happens per block now, so a ragged last block must not skip it. + import energyml.utils.data.crs as crs_module + + crs = CrsInfo(projected_epsg_code=32631, vertical_epsg_code=5715) + pts = self._points(1000) + single_block = reproject_to_wgs84(pts.copy(), crs, inplace=False) + + monkeypatch.setattr(crs_module, "_REPROJECT_CHUNK", 137) + many_blocks = reproject_to_wgs84(pts.copy(), crs, inplace=False) + + np.testing.assert_allclose(many_blocks, single_block, rtol=0, atol=0) + + +# --------------------------------------------------------------------------- +# Buffer ownership (read_array_view must not be mutated) +# --------------------------------------------------------------------------- + + +class TestPointBufferOwnership: + def test_view_input_is_copied(self): + source = np.arange(12, dtype=np.float64).reshape(4, 3) + view = source[:] # a view: base is not None + assert view.base is not None + + points = _ensure_float64_points(view) + points[0, 0] = -999.0 + assert source[0, 0] == 0.0, "the workspace array must not be mutated through the mesh" + + def test_read_only_input_is_copied(self): + source = np.arange(12, dtype=np.float64).reshape(4, 3) + source.setflags(write=False) + + points = _ensure_float64_points(source) + assert points.flags.writeable + points[0, 0] = -999.0 + assert source[0, 0] == 0.0 + + def test_own_false_keeps_the_borrowed_buffer(self): + # Used where the caller immediately concatenates, which allocates anyway. + source = np.arange(12, dtype=np.float64).reshape(4, 3) + borrowed = _ensure_float64_points(source, own=False) + assert borrowed is source + + def test_list_input_needs_no_extra_copy(self): + points = _ensure_float64_points([[1.0, 2.0, 3.0]]) + assert points.flags.writeable + np.testing.assert_array_equal(points, [[1.0, 2.0, 3.0]]) + + +# --------------------------------------------------------------------------- +# Grid2d: the divergence the frame field fixes +# --------------------------------------------------------------------------- + + +@requires_epc22 +class TestGrid2dGetsTheFullTransform: + def test_rotation_reaches_grid2d_points(self, epc22, monkeypatch): + import energyml.utils.data.mesh_numpy as mesh_numpy + + obj = _first_object_of_type(epc22, "Grid2d") + assert obj is not None, "fixture changed: no Grid2dRepresentation found" + + local = read_numpy_mesh_object(obj, workspace=epc22, frame=PointFrame.LOCAL) + local_pts = local.flat_patches()[0].points.copy() + + angle = math.pi / 2 + monkeypatch.setattr(mesh_numpy, "extract_crs_info", lambda *_a, **_k: _rotated_crs_info(angle)) + + projected = read_numpy_mesh_object(obj, workspace=epc22, frame=PointFrame.PROJECTED) + patch = projected.flat_patches()[0] + assert patch.frame is PointFrame.PROJECTED + + # Clockwise rotation of 90 deg: (x, y) -> (y, -x). crs_displacement_np, the previous + # fallback for Grid2d in the numpy stack, applied no rotation at all. + expected_x = local_pts[:, 1] + expected_y = -local_pts[:, 0] + np.testing.assert_allclose(patch.points[:, 0], expected_x, atol=1e-9) + np.testing.assert_allclose(patch.points[:, 1], expected_y, atol=1e-9) + + def test_reading_twice_gives_the_same_coordinates(self, epc22): + # The blocklist was there to avoid transforming twice; the frame field must give the + # same guarantee for every type, Grid2d included. + obj = _first_object_of_type(epc22, "Grid2d") + first = read_numpy_mesh_object(obj, workspace=epc22).flat_patches()[0].points.copy() + second = read_numpy_mesh_object(obj, workspace=epc22).flat_patches()[0].points + np.testing.assert_allclose(first, second) + + +# --------------------------------------------------------------------------- +# frame= on the reader +# --------------------------------------------------------------------------- + + +@requires_epc22 +class TestReaderFrameParameter: + def test_local_differs_from_projected_when_the_crs_has_an_offset(self, epc22): + obj = _first_object_of_type(epc22, "Grid2d") + local = read_numpy_mesh_object(obj, workspace=epc22, frame=PointFrame.LOCAL) + projected = read_numpy_mesh_object(obj, workspace=epc22, frame=PointFrame.PROJECTED) + + assert local.flat_patches()[0].frame is PointFrame.LOCAL + assert projected.flat_patches()[0].frame is PointFrame.PROJECTED + + def test_use_crs_displacement_false_maps_to_local(self, epc22): + obj = _first_object_of_type(epc22, "Grid2d") + result = read_numpy_mesh_object(obj, workspace=epc22, use_crs_displacement=False) + assert result.flat_patches()[0].frame is PointFrame.LOCAL diff --git a/energyml-utils/tests/test_xml.py b/energyml-utils/tests/test_xml.py index 769fc50..6764358 100644 --- a/energyml-utils/tests/test_xml.py +++ b/energyml-utils/tests/test_xml.py @@ -3,8 +3,12 @@ import logging -from energyml.utils.constants import parse_qualified_type -from src.energyml.utils.xml_utils import * +from energyml.utils.constants import ENERGYML_NAMESPACES_PACKAGE, parse_content_type, parse_qualified_type + +# `xml_utils` now declares an `__all__`, so `import *` no longer leaks the names it imports +# itself (`parse_content_type`, `ENERGYML_NAMESPACES_PACKAGE`, ...) — they are imported above, +# from the module that actually defines them. +from src.energyml.utils.xml_utils import * # noqa: F403 CT_20 = "application/x-resqml+xml;version=2.0;type=obj_TriangulatedSetRepresentation" CT_22 = "application/x-resqml+xml;version=2.2;type=TriangulatedSetRepresentation" diff --git a/energyml-utils/tests/test_xsi_type_resolution.py b/energyml-utils/tests/test_xsi_type_resolution.py new file mode 100644 index 0000000..c6a62d3 --- /dev/null +++ b/energyml-utils/tests/test_xsi_type_resolution.py @@ -0,0 +1,88 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +"""An ``xsi:type`` with no prefix must resolve against the default namespace of the document. + +That is the common spelling in the wild: energyml files usually declare ``commonv2`` as their +default namespace, and the polymorphic CRS types (``VerticalCrsEpsgCode``, ``ProjectedCrsEpsgCode``) +are declared there — so ``xsi:type="VerticalCrsEpsgCode"`` is valid and unambiguous. + +``FallbackNamespaceXmlParser`` used to rewrite the default-namespace key from ``None`` to ``""`` +while merging its fallback namespaces. :meth:`xsdata.formats.converter.QNameConverter.resolve` +reads it as ``ns_map[None]`` (an unprefixed value splits to a ``None`` prefix), so the type was +lost, the element was built as its abstract base and every child became an unknown property. +Concretely: a RESQML 2.0.1 ``LocalDepth3dCrs`` lost both of its EPSG codes, which is enough to +make any WGS84 reprojection impossible. +""" + +import pytest + +from energyml.utils.data.crs import extract_crs_info +from energyml.utils.serialization import read_energyml_xml_bytes + +_CRS_TEMPLATE = """ + + Default + tests + 2019-03-22T10:29:55Z + tests + + 6470000.0 + -0.0 + 0.0 + easting northing + m + m + 420000.0 + true + + 5715 + + + 23031 + + +""" + +_EML_NS = 'xmlns:eml="http://www.energistics.org/energyml/data/commonv2" ' + + +def _crs_document(vertical: str, projected: str, extra_ns: str = "") -> bytes: + return _CRS_TEMPLATE.format(vertical=vertical, projected=projected, extra_ns=extra_ns).encode("utf-8") + + +@pytest.mark.parametrize( + "vertical,projected,extra_ns", + [ + # what SKUA-GOCAD and most exporters write: no prefix, default namespace applies + ("VerticalCrsEpsgCode", "ProjectedCrsEpsgCode", ""), + # explicitly prefixed, the spelling that already worked + ("eml:VerticalCrsEpsgCode", "eml:ProjectedCrsEpsgCode", _EML_NS), + ], + ids=["unprefixed", "prefixed"], +) +def test_polymorphic_crs_type_is_resolved(vertical, projected, extra_ns): + crs = read_energyml_xml_bytes(_crs_document(vertical, projected, extra_ns)) + + assert type(crs.vertical_crs).__name__ == "VerticalCrsEpsgCode" + assert type(crs.projected_crs).__name__ == "ProjectedCrsEpsgCode" + assert crs.vertical_crs.epsg_code == 5715 + assert crs.projected_crs.epsg_code == 23031 + + +def test_undeclared_prefix_still_falls_back(): + # The fallback namespaces are the point of the custom parser: a prefix the document never + # declares must still resolve when it is a well-known energyml one. + crs = read_energyml_xml_bytes(_crs_document("eml:VerticalCrsEpsgCode", "eml:ProjectedCrsEpsgCode")) + assert crs.projected_crs.epsg_code == 23031 + + +def test_epsg_codes_reach_the_crs_info(): + # The reason it matters: without them `to_frame` cannot reach PointFrame.WGS84. + info = extract_crs_info(read_energyml_xml_bytes(_crs_document("VerticalCrsEpsgCode", "ProjectedCrsEpsgCode"))) + + assert info.projected_epsg_code == 23031 + assert info.vertical_epsg_code == 5715 + assert info.x_offset == 420000.0 + assert info.y_offset == 6470000.0 diff --git a/energyml-utils/tests/test_zip_raw.py b/energyml-utils/tests/test_zip_raw.py new file mode 100644 index 0000000..1bf100d --- /dev/null +++ b/energyml-utils/tests/test_zip_raw.py @@ -0,0 +1,155 @@ +# Copyright (c) 2023-2024 Geosiris. +# SPDX-License-Identifier: Apache-2.0 +""" +Tests for the raw ZIP stream copy used to rewrite EPC files cheaply. + +The point of these tests is that a raw-copied archive must be indistinguishable +from a decompress/recompress one for any reader: same names, same content, same +CRC, and ``testzip()`` clean. +""" +import os +import tempfile +import zipfile + +import pytest + +from energyml.utils.zip_raw import ( + append_to_zip, + count_shadowed_entries, + iter_effective_infos, + rewrite_zip, +) + +SAMPLE_PARTS = { + "[Content_Types].xml": b"", + "a.xml": b"" + b"x" * 5000 + b"", + "folder/b.xml": "accentué € ☃".encode("utf-8"), + "_rels/.rels": b"", + "binary.bin": bytes(range(256)) * 40, + "empty.txt": b"", +} + + +@pytest.fixture +def sample_zip(): + fd, path = tempfile.mkstemp(suffix=".zip") + os.close(fd) + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as zf: + for name, data in SAMPLE_PARTS.items(): + zf.writestr(name, data) + yield path + if os.path.exists(path): + os.unlink(path) + + +@pytest.fixture +def out_path(): + fd, path = tempfile.mkstemp(suffix=".zip") + os.close(fd) + os.unlink(path) + yield path + if os.path.exists(path): + os.unlink(path) + + +def read_all(path): + with zipfile.ZipFile(path) as zf: + assert zf.testzip() is None + return {name: zf.read(name) for name in zf.namelist()} + + +class TestRewriteZip: + def test_pure_copy_preserves_everything(self, sample_zip, out_path): + raw_copied, recompressed = rewrite_zip(sample_zip, out_path) + + assert raw_copied == len(SAMPLE_PARTS) + assert recompressed == 0 + assert read_all(out_path) == SAMPLE_PARTS + + def test_raw_copy_matches_the_recompressed_result(self, sample_zip, out_path): + fd, slow_path = tempfile.mkstemp(suffix=".zip") + os.close(fd) + os.unlink(slow_path) + try: + rewrite_zip(sample_zip, out_path, allow_raw_copy=True) + rewrite_zip(sample_zip, slow_path, allow_raw_copy=False) + assert read_all(out_path) == read_all(slow_path) + + with zipfile.ZipFile(out_path) as fast, zipfile.ZipFile(slow_path) as slow: + assert {i.filename: i.CRC for i in fast.infolist()} == {i.filename: i.CRC for i in slow.infolist()} + finally: + if os.path.exists(slow_path): + os.unlink(slow_path) + + def test_update_and_delete(self, sample_zip, out_path): + rewrite_zip( + sample_zip, + out_path, + updates={"a.xml": b"new", "added.xml": b""}, + deleted={"binary.bin"}, + ) + + content = read_all(out_path) + assert content["a.xml"] == b"new" + assert content["added.xml"] == b"" + assert "binary.bin" not in content + assert content["folder/b.xml"] == SAMPLE_PARTS["folder/b.xml"] + + def test_in_place_rewrite(self, sample_zip): + rewrite_zip(sample_zip, sample_zip, updates={"a.xml": b"in place"}) + + content = read_all(sample_zip) + assert content["a.xml"] == b"in place" + assert len(content) == len(SAMPLE_PARTS) + + def test_creation_without_source(self, out_path): + rewrite_zip(None, out_path, updates={"only.xml": b""}) + assert read_all(out_path) == {"only.xml": b""} + + def test_deleting_everything_yields_a_valid_empty_archive(self, sample_zip, out_path): + rewrite_zip(sample_zip, out_path, deleted=set(SAMPLE_PARTS)) + assert read_all(out_path) == {} + + def test_shadowed_entries_are_dropped(self, sample_zip, out_path): + append_to_zip(sample_zip, {"a.xml": b"shadowing"}) + with zipfile.ZipFile(sample_zip) as zf: + assert count_shadowed_entries(zf) == 1 + + rewrite_zip(sample_zip, out_path) + + content = read_all(out_path) + assert content["a.xml"] == b"shadowing" + with zipfile.ZipFile(out_path) as zf: + assert count_shadowed_entries(zf) == 0 + assert len(zf.infolist()) == len(SAMPLE_PARTS) + + def test_source_is_left_untouched_when_the_write_fails(self, sample_zip, out_path): + before = read_all(sample_zip) + + class Unwritable: + def __len__(self): + raise RuntimeError("boom") + + with pytest.raises(Exception): + rewrite_zip(sample_zip, sample_zip, updates={"a.xml": Unwritable()}) + + assert read_all(sample_zip) == before + + +class TestAppendToZip: + def test_appended_entry_is_readable(self, sample_zip): + append_to_zip(sample_zip, {"new.xml": b""}) + assert read_all(sample_zip)["new.xml"] == b"" + + def test_appending_an_existing_name_shadows_it(self, sample_zip): + append_to_zip(sample_zip, {"a.xml": b"v2"}) + + with zipfile.ZipFile(sample_zip) as zf: + assert zf.read("a.xml") == b"v2" + assert zf.namelist().count("a.xml") == 2 + assert [i.filename for i in iter_effective_infos(zf)].count("a.xml") == 1 + + def test_empty_update_is_a_noop(self, sample_zip): + before = os.path.getsize(sample_zip) + append_to_zip(sample_zip, {}) + assert os.path.getsize(sample_zip) == before