From 11b39c921ef58fc49ed28395c481bb8b74af035d Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Thu, 29 Oct 2026 18:18:45 -0600 Subject: [PATCH 01/13] [#1663] ignore all environment folders starting with `.venv` --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3be2acdd813..e2c1f116eac 100755 --- a/.gitignore +++ b/.gitignore @@ -56,7 +56,7 @@ supportData/EphemerisData/*.bsp **/tests/report/ **/__pycache__/ venv/ -.venv/ +.venv*/ # VSCode / VS .devcontainer From 04231c3d6280319d2f229c55a73110b59591a423 Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Thu, 29 Oct 2026 18:45:13 -0600 Subject: [PATCH 02/13] [#1663] Enforce supported Python versions in Conan and CMake builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read project.requires-python from pyproject.toml through a shared validator. Reject unsupported interpreters before build-script side effects and validate CMake's selected Python interpreter. Preserve the Python 3.9 deprecation warning, add tomli for Python 3.9–3.10, and include regression tests and documentation updates. --- .github/workflows/requirements_dev.txt | 1 + conanfile.py | 13 +- docs/source/Build/installBuild.rst | 6 + .../python-version-validation.rst | 1 + .../python-version-validation.rst | 1 + pyproject.toml | 1 + requirements_dev.txt | 1 + src/CMakeLists.txt | 8 +- src/buildSupport/checkPythonVersion.py | 80 ++++++++ src/cmake/bskCheckPythonVersion.cmake | 36 ++++ src/tests/test_conan_command.py | 2 + src/tests/test_python_version.py | 173 ++++++++++++++++++ 12 files changed, 317 insertions(+), 6 deletions(-) create mode 100644 docs/source/Support/bskKnownIssuesSnippets/python-version-validation.rst create mode 100644 docs/source/Support/bskReleaseNotesSnippets/python-version-validation.rst create mode 100644 src/buildSupport/checkPythonVersion.py create mode 100644 src/cmake/bskCheckPythonVersion.cmake create mode 100644 src/tests/test_python_version.py diff --git a/.github/workflows/requirements_dev.txt b/.github/workflows/requirements_dev.txt index ed7adc2edec..2ded5d0c8b9 100644 --- a/.github/workflows/requirements_dev.txt +++ b/.github/workflows/requirements_dev.txt @@ -2,6 +2,7 @@ wheel cmake conan packaging +tomli; python_version < "3.11" setuptools setuptools-scm swig diff --git a/conanfile.py b/conanfile.py index 8779f975bad..3b27fb86c56 100644 --- a/conanfile.py +++ b/conanfile.py @@ -1,6 +1,7 @@ import argparse import os import json +import runpy import shutil import shlex import stat @@ -13,12 +14,20 @@ import importlib.metadata +REPOSITORY_ROOT = Path(__file__).resolve().parent +if __name__ == "__main__": + # Reject unsupported interpreters before loading build tools or changing files. + version_check = runpy.run_path( + str(REPOSITORY_ROOT / "src" / "buildSupport" / "checkPythonVersion.py") + ) + if version_check["main"]([str(REPOSITORY_ROOT / "pyproject.toml")]): + sys.exit(1) + from conan import ConanFile from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps from conan.tools.microsoft import is_msvc from conan.tools.files import copy -REPOSITORY_ROOT = Path(__file__).resolve().parent sys.path.insert(1, str(REPOSITORY_ROOT / "src" / "utilities")) import makeDraftModule @@ -574,7 +583,7 @@ class BasiliskConan(ConanFile): } # Sources are located in the same place as this recipe, copy them to the recipe - exports_sources = "setup.py", "CMakeLists.txt", "src/*", "include/*" + exports_sources = "setup.py", "pyproject.toml", "CMakeLists.txt", "src/*", "include/*" for opt, value in bskModuleOptionsBool.items(): options.update({opt: value[0]}) diff --git a/docs/source/Build/installBuild.rst b/docs/source/Build/installBuild.rst index 03dafed962f..506785141d0 100644 --- a/docs/source/Build/installBuild.rst +++ b/docs/source/Build/installBuild.rst @@ -47,6 +47,12 @@ shown in the platform-specific installation instructions. The editable installat virtual environment; subsequent incremental or clean native rebuilds update the same ``dist3`` package in place and do not require another editable installation. +Before starting build operations, ``conanfile.py`` checks the running Python version against +``project.requires-python`` in ``pyproject.toml``. CMake checks its selected Python interpreter against +the same range during configuration, including when invoked directly. An unsupported version stops the +build with an error identifying the interpreter and supported range. Create or activate a virtual +environment with a supported Python version before retrying. + By default the build is for Python3 with the support for :ref:`vizInterface` included to enable recording data for or live-streaming to :ref:`Vizard `. diff --git a/docs/source/Support/bskKnownIssuesSnippets/python-version-validation.rst b/docs/source/Support/bskKnownIssuesSnippets/python-version-validation.rst new file mode 100644 index 00000000000..5fbbbea8a0f --- /dev/null +++ b/docs/source/Support/bskKnownIssuesSnippets/python-version-validation.rst @@ -0,0 +1 @@ +- In Basilisk 2.12.1, direct ``conanfile.py`` and CMake builds could accept Python versions outside the range declared in package metadata, even though package installation rejected them. Both build paths now enforce ``project.requires-python`` from ``pyproject.toml``. For affected releases, create a virtual environment with a supported Python version before building and installing Basilisk. diff --git a/docs/source/Support/bskReleaseNotesSnippets/python-version-validation.rst b/docs/source/Support/bskReleaseNotesSnippets/python-version-validation.rst new file mode 100644 index 00000000000..8b2cb2973d5 --- /dev/null +++ b/docs/source/Support/bskReleaseNotesSnippets/python-version-validation.rst @@ -0,0 +1 @@ +- The build script now rejects unsupported Python versions before starting build operations, and CMake validates its selected interpreter against the same ``project.requires-python`` range in ``pyproject.toml``. diff --git a/pyproject.toml b/pyproject.toml index ab2610e4a2a..650076dce3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ requires = [ "setuptools>=77.0.3,<=84.0.0", # Required for "bdist_wheel" to work correctly and for PEP 639 license metadata. "setuptools-scm>=8.0,<=10.3.4", # Automatically include all Git-controlled files in sdist "packaging>=24.0,<=26.3", + "tomli>=2.0.1,<3; python_version < '3.11'", # needed to find the numpy include headers, # pin to lowest runtime allowed version to take advantage of backwards compatibility diff --git a/requirements_dev.txt b/requirements_dev.txt index 2a4e2805f16..8481328a1a1 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -2,6 +2,7 @@ wheel>=0.45.1,<=0.48.0 cmake>=3.26,<=4.4.4 conan>=2.4.0,<=2.33.0 packaging>=24,<=26.3 +tomli>=2.0.1,<3; python_version < "3.11" setuptools>=70.1.0,<=84.0.0 setuptools-scm>=8.0,<=10.3.4 swig>=4.4.1,<5 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3bf460e932f..333f8ba4f58 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -70,10 +70,10 @@ list(FILTER BSK_CUSTOM_CMAKE_FILES INCLUDE REGEX "(^|/)Custom\\.cmake$") # Find necessary packages find_package(Python3 ${PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development.Module OPTIONAL_COMPONENTS Development.SABIModule) -# enforce minimum supported Python version and warn on deprecated versions -if(Python3_VERSION VERSION_LESS "3.9") - message(FATAL_ERROR "Basilisk requires Python 3.9 or newer, but found Python ${Python3_VERSION}.") -elseif(Python3_VERSION VERSION_LESS "3.10") +# Enforce the package's supported Python range and warn on deprecated versions. +include(bskCheckPythonVersion) +bsk_check_python_version("${CMAKE_CURRENT_SOURCE_DIR}/../pyproject.toml") +if(Python3_VERSION VERSION_LESS "3.10") message(WARNING "You are using Python ${Python3_VERSION}, but support for Python 3.9 is deprecated and will be removed in March 2027.") endif() diff --git a/src/buildSupport/checkPythonVersion.py b/src/buildSupport/checkPythonVersion.py new file mode 100644 index 00000000000..1a0494a7aa3 --- /dev/null +++ b/src/buildSupport/checkPythonVersion.py @@ -0,0 +1,80 @@ +# ISC License +# +# Copyright (c) 2026, Autonomous Vehicle Systems Lab, University of Colorado at Boulder +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Check build interpreters against Basilisk's package metadata.""" + +import argparse +from pathlib import Path +import sys + + +def check_python_version(pyproject_path, version=None): + """Reject Python versions outside ``project.requires-python``. + + :param pyproject_path: Path to Basilisk's ``pyproject.toml``. + :param version: Version string to check; defaults to the running interpreter. + :raises ValueError: If the metadata is invalid or the version is unsupported. + """ + try: + import tomllib + except ModuleNotFoundError: + import tomli as tomllib + from packaging.specifiers import SpecifierSet + + with Path(pyproject_path).open("rb") as metadata_file: + metadata = tomllib.load(metadata_file) + project = metadata.get("project", {}) + requires_python = project.get("requires-python") if isinstance(project, dict) else None + if not isinstance(requires_python, str) or not requires_python.strip(): + raise ValueError("pyproject.toml must define project.requires-python.") + supported_versions = SpecifierSet(requires_python) + if version is None: + # Match the release tuple used by installers for Requires-Python checks. + version = ".".join(str(component) for component in sys.version_info[:3]) + if not supported_versions.contains(version, prereleases=True): + raise ValueError( + f"Basilisk requires Python {requires_python}, but found Python {version} " + f"({sys.executable}). Create or activate a virtual environment with a " + "supported Python version and rerun the build." + ) + + +def main(argv=None): + """Return a command-line status after checking the selected metadata file. + + :param argv: Command-line arguments; defaults to ``sys.argv[1:]``. + :return: Zero on success, or one if validation fails. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("pyproject_path", type=Path) + args = parser.parse_args(argv) + try: + check_python_version(args.pyproject_path) + except ImportError as error: + print( + f"Python version check failed: {error}. Install the build requirements " + "from requirements_dev.txt.", + file=sys.stderr, + ) + return 1 + except (OSError, ValueError) as error: + print(f"Python version check failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/cmake/bskCheckPythonVersion.cmake b/src/cmake/bskCheckPythonVersion.cmake new file mode 100644 index 00000000000..5cbcb751de7 --- /dev/null +++ b/src/cmake/bskCheckPythonVersion.cmake @@ -0,0 +1,36 @@ +# ISC License +# +# Copyright (c) 2026, Autonomous Vehicle Systems Lab, University of Colorado at Boulder +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +include_guard(GLOBAL) + +function(bsk_check_python_version PYPROJECT_PATH) + set(VERSION_CHECK_SCRIPT + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../buildSupport/checkPythonVersion.py") + execute_process( + COMMAND "${Python3_EXECUTABLE}" "${VERSION_CHECK_SCRIPT}" "${PYPROJECT_PATH}" + RESULT_VARIABLE VERSION_CHECK_RESULT + OUTPUT_VARIABLE VERSION_CHECK_OUTPUT + ERROR_VARIABLE VERSION_CHECK_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE) + if(NOT "${VERSION_CHECK_RESULT}" STREQUAL "0") + message(FATAL_ERROR + "Python version validation failed (${VERSION_CHECK_RESULT}).\n" + "${VERSION_CHECK_OUTPUT}${VERSION_CHECK_ERROR}") + endif() + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + "${PYPROJECT_PATH}" "${VERSION_CHECK_SCRIPT}") +endfunction() diff --git a/src/tests/test_conan_command.py b/src/tests/test_conan_command.py index ee0700a5a2d..9cc1253d15f 100644 --- a/src/tests/test_conan_command.py +++ b/src/tests/test_conan_command.py @@ -113,7 +113,9 @@ def test_script_builds_repository_from_another_directory( repo_root = tmp_path / "repository with spaces" for resource in ( "conanfile.py", + "pyproject.toml", "docs/source/bskVersion.txt", + "src/buildSupport/checkPythonVersion.py", "src/utilities/makeDraftModule.py", ): destination = repo_root / resource diff --git a/src/tests/test_python_version.py b/src/tests/test_python_version.py new file mode 100644 index 00000000000..4fcec19516f --- /dev/null +++ b/src/tests/test_python_version.py @@ -0,0 +1,173 @@ +# ISC License +# +# Copyright (c) 2026, Autonomous Vehicle Systems Lab, University of Colorado at Boulder +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Verify Python compatibility checks without building native modules.""" + +import os +from pathlib import Path +import runpy +import shutil +import subprocess +import sys + +import pytest + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +VERSION_CHECK_SCRIPT = Path("src/buildSupport/checkPythonVersion.py") +CMAKE_NAME = "cmake.exe" if os.name == "nt" else "cmake" +ENVIRONMENT_CMAKE = Path(sys.executable).with_name(CMAKE_NAME) +CMAKE = shutil.which(CMAKE_NAME) or ( + str(ENVIRONMENT_CMAKE) if ENVIRONMENT_CMAKE.is_file() else None +) + + +@pytest.fixture(scope="module") +def version_checker(): + """Load the shared validator without importing build tools.""" + return runpy.run_path(str(REPOSITORY_ROOT / VERSION_CHECK_SCRIPT)) + + +@pytest.mark.parametrize("version, supported", [ + ("3.8.20", False), + ("3.9.0", True), + ("3.10.0", True), + ("3.11.0", True), + ("3.12.0", True), + ("3.13.0", True), + ("3.14.0", True), + ("3.14.99", True), + ("3.15.0", False), + ("3.15.2", False), + ("4.0.0", False), +]) +def test_python_version_boundaries(tmp_path, version_checker, version, supported): + """Enforce both endpoints while preserving every supported minor version.""" + metadata = tmp_path / "pyproject.toml" + metadata.write_text('[project]\nrequires-python = ">=3.9, <3.15"\n', encoding="utf-8") + check_version = version_checker["check_python_version"] + if supported: + check_version(metadata, version) + else: + with pytest.raises(ValueError, match=r"Basilisk requires Python >=3.9, <3.15") as error: + check_version(metadata, version) + assert f"found Python {version}" in str(error.value) + + +def test_updated_metadata_controls_supported_versions(tmp_path, version_checker): + """Honor changed bounds and exclusions without changing the validator.""" + metadata = tmp_path / "pyproject.toml" + metadata.write_text( + '[project]\nrequires-python = ">=3.10, <3.16, !=3.12.*"\n', encoding="utf-8", + ) + check_version = version_checker["check_python_version"] + check_version(metadata, "3.15.2") + for version in ("3.9.20", "3.12.0", "3.16.0"): + with pytest.raises(ValueError, match="Basilisk requires Python"): + check_version(metadata, version) + + +@pytest.mark.parametrize("content", [ + "[project]\n", + 'project = "invalid"\n', + '[project]\nrequires-python = ""\n', + '[project]\nrequires-python = 315\n', + '[project]\nrequires-python = "invalid"\n', + "[project\n", +]) +def test_invalid_metadata_stops_validation(tmp_path, version_checker, capsys, content): + """Fail with a diagnostic instead of silently accepting incomplete metadata.""" + metadata = tmp_path / "pyproject.toml" + metadata.write_text(content, encoding="utf-8") + assert version_checker["main"]([str(metadata)]) == 1 + assert "Python version check failed:" in capsys.readouterr().err + + +def test_missing_metadata_stops_validation(tmp_path, version_checker, capsys): + """Report a missing metadata file as a failed check.""" + assert version_checker["main"]([str(tmp_path / "missing.toml")]) == 1 + assert "missing.toml" in capsys.readouterr().err + + +def test_conan_script_rejects_before_imports_or_clean(tmp_path): + """Reject an unsupported interpreter before importing Conan or touching outputs.""" + repository = tmp_path / "repository with spaces" + for resource in (Path("conanfile.py"), VERSION_CHECK_SCRIPT): + destination = repository / resource + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(REPOSITORY_ROOT / resource, destination) + (repository / "pyproject.toml").write_text( + '[project]\nrequires-python = ">=99"\n', encoding="utf-8", + ) + (repository / "conan.py").write_text( + 'raise RuntimeError("Conan must not be imported")\n', encoding="utf-8", + ) + existing_output = repository / "dist3" / "keep.txt" + existing_output.parent.mkdir() + existing_output.write_text("existing build", encoding="utf-8") + # The invoking directory's metadata must not override the repository's range. + (tmp_path / "pyproject.toml").write_text( + '[project]\nrequires-python = ">=0"\n', encoding="utf-8", + ) + + result = subprocess.run( + [sys.executable, str(repository / "conanfile.py"), "--clean"], + cwd=tmp_path, capture_output=True, text=True, check=False, timeout=30, # [s] + ) + + assert result.returncode == 1 + assert "Basilisk requires Python >=99" in result.stderr + assert sys.executable in result.stderr + assert "Conan must not be imported" not in result.stderr + assert "Traceback" not in result.stderr + assert existing_output.read_text(encoding="utf-8") == "existing build" + + +@pytest.mark.skipif(CMAKE is None, reason="CMake is required") +def test_cmake_checks_selected_interpreter_and_rechecks_changed_metadata(tmp_path): + """Validate the actual executable and rerun the check when metadata changes.""" + source = tmp_path / "source with spaces" + source.mkdir() + metadata = source / "pyproject.toml" + metadata.write_text('[project]\nrequires-python = ">=0"\n', encoding="utf-8") + module = (REPOSITORY_ROOT / "src/cmake/bskCheckPythonVersion.cmake").as_posix() + (source / "CMakeLists.txt").write_text( + f'''cmake_minimum_required(VERSION 3.26) +project(PythonVersionCheck NONE) +include("{module}") +set(Python3_EXECUTABLE "{Path(sys.executable).as_posix()}") +# Deliberately inconsistent: validation must inspect the executable itself. +set(Python3_VERSION "99.0.0") +bsk_check_python_version("${{CMAKE_CURRENT_SOURCE_DIR}}/pyproject.toml") +''', encoding="utf-8", + ) + build = tmp_path / "build" + configure = subprocess.run( + [CMAKE, "-S", str(source), "-B", str(build)], + capture_output=True, text=True, check=False, timeout=30, # [s] + ) + assert configure.returncode == 0, configure.stdout + configure.stderr + + metadata.write_text('[project]\nrequires-python = ">=99"\n', encoding="utf-8") + rebuild = subprocess.run( + [CMAKE, "--build", str(build)], + capture_output=True, text=True, check=False, timeout=30, # [s] + ) + assert rebuild.returncode != 0 + output = " ".join((rebuild.stdout + rebuild.stderr).split()) + assert "Basilisk requires Python >=99" in output + actual_version = ".".join(str(component) for component in sys.version_info[:3]) + assert f"found Python {actual_version}" in output From 921d98cd6fed50cbba9be16f53e550e74c3694db Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Thu, 29 Oct 2026 19:27:07 -0600 Subject: [PATCH 03/13] [#1663] Add Python 3.15 support to packaging and dependencies Allow Python 3.9 through 3.15 and select compatible NumPy and Numba requirements for Python 3.15 while retaining the existing ranges for older interpreters. Update version-boundary and dependency regression tests. --- pyproject.toml | 12 +++-- requirements.txt | 3 +- requirements_dev.txt | 4 +- src/tests/test_python_version.py | 81 +++++++++++++++++++++++++++++--- 4 files changed, 88 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 650076dce3a..bb46ec06ddc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,9 @@ requires = [ # needed to find the numpy include headers, # pin to lowest runtime allowed version to take advantage of backwards compatibility "numpy>=1.24.4,<2.4.0; python_version < '3.13'", - "numpy>=2.0,<2.4.0; python_version >= '3.13'", + "numpy>=2.0,<2.4.0; python_version >= '3.13' and python_version < '3.15'", + # NumPy 2.5.2 introduced wheels for standard CPython 3.15. + "numpy>=2.5.2,<=2.5.4; python_version >= '3.15'", # Requirements for building Basilisk through conanfile "conan>=2.4.0,<=2.33.0", @@ -21,7 +23,7 @@ requires = [ [project] name = 'bsk' dynamic = ["version", "dependencies"] -requires-python = ">=3.9, <3.15" +requires-python = ">=3.9, <3.16" readme = "README.md" license = "ISC" @@ -36,6 +38,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Scientific/Engineering", ] @@ -72,8 +75,9 @@ examples = [ 'scipy==1.13.1; python_version < "3.10"', 'scipy==1.15.3; python_version >= "3.10" and python_version < "3.11"', 'scipy>=1.17.1,<=1.18.1; python_version >= "3.11"', - # Match development requirements so NumPy 2.5 upgrades also update Numba. - 'numba>=0.67.0,<=0.68.0; python_version >= "3.12"', + # Match development requirements for NumPy 2.5 and Python 3.15 compatibility. + 'numba>=0.68.0,<=0.68.0; python_version >= "3.15"', + 'numba>=0.67.0,<=0.68.0; python_version >= "3.12" and python_version < "3.15"', 'numba<=0.68.0; python_version < "3.12"', ] opnav = [ diff --git a/requirements.txt b/requirements.txt index 97f5a483720..ca18240ecbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ pandas>=2.0.3,<=3.0.6 matplotlib>=3.7.5,<=3.11.2 numpy>=1.24.4,<=2.5.4; python_version < "3.13" -numpy>=2.0,<=2.5.4; python_version >= "3.13" +numpy>=2.0,<=2.5.4; python_version >= "3.13" and python_version < "3.15" +numpy>=2.5.2,<=2.5.4; python_version >= "3.15" colorama==0.4.6 tqdm>=4.67.3,<=4.70.1 pillow>=10.4.0,<=13.0.0 diff --git a/requirements_dev.txt b/requirements_dev.txt index 8481328a1a1..6c29a055157 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -9,7 +9,9 @@ swig>=4.4.1,<5 libclang>=15.0.6.1,<=18.1.1 # NumPy 2.5 is available on Python 3.12+ and requires Numba 0.67 or newer. -numba>=0.67.0,<=0.68.0; python_version >= "3.12" +# Python 3.15 requires Numba 0.68; older Python versions retain their choices. +numba>=0.68.0,<=0.68.0; python_version >= "3.15" +numba>=0.67.0,<=0.68.0; python_version >= "3.12" and python_version < "3.15" numba<=0.68.0; python_version < "3.12" pytest>=8.3.5,<=9.1.1 diff --git a/src/tests/test_python_version.py b/src/tests/test_python_version.py index 4fcec19516f..fc5fd4cdab7 100644 --- a/src/tests/test_python_version.py +++ b/src/tests/test_python_version.py @@ -24,6 +24,13 @@ import sys import pytest +from packaging.markers import default_environment +from packaging.requirements import Requirement + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib REPOSITORY_ROOT = Path(__file__).resolve().parents[2] @@ -50,23 +57,85 @@ def version_checker(): ("3.13.0", True), ("3.14.0", True), ("3.14.99", True), - ("3.15.0", False), - ("3.15.2", False), + ("3.15.0", True), + ("3.15.2", True), + ("3.15.99", True), + ("3.16.0", False), ("4.0.0", False), ]) -def test_python_version_boundaries(tmp_path, version_checker, version, supported): +def test_python_version_boundaries(version_checker, version, supported): """Enforce both endpoints while preserving every supported minor version.""" - metadata = tmp_path / "pyproject.toml" - metadata.write_text('[project]\nrequires-python = ">=3.9, <3.15"\n', encoding="utf-8") + metadata = REPOSITORY_ROOT / "pyproject.toml" check_version = version_checker["check_python_version"] if supported: check_version(metadata, version) else: - with pytest.raises(ValueError, match=r"Basilisk requires Python >=3.9, <3.15") as error: + with pytest.raises(ValueError, match="Basilisk requires Python") as error: check_version(metadata, version) assert f"found Python {version}" in str(error.value) +@pytest.fixture(scope="module") +def dependency_groups(): + """Read the build, runtime, development, and example requirements.""" + with (REPOSITORY_ROOT / "pyproject.toml").open("rb") as metadata_file: + metadata = tomllib.load(metadata_file) + return { + "build": metadata["build-system"]["requires"], + "runtime": (REPOSITORY_ROOT / "requirements.txt").read_text(encoding="utf-8").splitlines(), + "development": (REPOSITORY_ROOT / "requirements_dev.txt").read_text(encoding="utf-8").splitlines(), + "examples": metadata["project"]["optional-dependencies"]["examples"], + } + + +def active_requirement(requirements, name, python_version): + """Select the unique requirement for a dependency on a target interpreter.""" + environment = default_environment() + environment.update(python_version=python_version, python_full_version=f"{python_version}.0") + active = [] + for line in requirements: + if not line.strip() or line.lstrip().startswith("#"): + continue + requirement = Requirement(line) + if requirement.name == name and ( + requirement.marker is None or requirement.marker.evaluate(environment) + ): + active.append(requirement) + assert len(active) == 1, (name, python_version, active) + return active[0] + + +@pytest.mark.parametrize("python_version, numpy_version, numba_version", [ + ("3.9", "2.0.2", "0.60.0"), + ("3.10", "2.2.6", "0.68.0"), + ("3.11", "2.3.5", "0.68.0"), + ("3.12", "2.3.5", "0.67.0"), + ("3.13", "2.3.5", "0.67.0"), + ("3.14", "2.3.5", "0.67.0"), + ("3.15", "2.5.2", "0.68.0"), +]) +def test_dependency_ranges_preserve_supported_interpreters( + dependency_groups, python_version, numpy_version, numba_version, +): + """Keep compatible dependency choices available across Python 3.9 through 3.15.""" + for group in ("build", "runtime"): + requirement = active_requirement(dependency_groups[group], "numpy", python_version) + assert requirement.specifier.contains(numpy_version) + for group in ("development", "examples"): + requirement = active_requirement(dependency_groups[group], "numba", python_version) + assert requirement.specifier.contains(numba_version) + + +def test_python315_excludes_incompatible_dependencies(dependency_groups): + """Prevent older NumPy and Numba releases from being selected on Python 3.15.""" + for group in ("build", "runtime"): + requirement = active_requirement(dependency_groups[group], "numpy", "3.15") + assert not requirement.specifier.contains("2.5.1") + for group in ("development", "examples"): + requirement = active_requirement(dependency_groups[group], "numba", "3.15") + assert not requirement.specifier.contains("0.67.0") + + def test_updated_metadata_controls_supported_versions(tmp_path, version_checker): """Honor changed bounds and exclusions without changing the validator.""" metadata = tmp_path / "pyproject.toml" From 7bf200e6e80bad4c9553d86658904475bf3c4d77 Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Thu, 29 Oct 2026 19:27:29 -0600 Subject: [PATCH 04/13] [#1663] Update existing CI and wheel builds for Python 3.15 Upgrade cibuildwheel, add the cp315 selector, and move existing maximum-version builds and wheel installation checks to Python 3.15. Preserve the Python 3.9 build, older wheel selectors, and the existing CI test structure. --- .github/actions/build-wheel/action.yml | 4 ++-- .github/workflows/canary.yml | 2 +- .github/workflows/merge.yml | 2 +- .github/workflows/nightly-wheels.yml | 2 +- .github/workflows/publish-wheels.yml | 4 ++-- .github/workflows/pull-request.yml | 6 +++--- .github/workflows/test-optional-wheels.yml | 4 ++-- pyproject.toml | 4 +++- 8 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/actions/build-wheel/action.yml b/.github/actions/build-wheel/action.yml index a94fc3f288b..ead4b153b97 100644 --- a/.github/actions/build-wheel/action.yml +++ b/.github/actions/build-wheel/action.yml @@ -134,7 +134,7 @@ runs: - name: Build wheel if: ${{ inputs.cibw-build == '' }} - uses: pypa/cibuildwheel@v3.4.1 + uses: pypa/cibuildwheel@v4.3.5 with: output-dir: ${{ inputs.output-dir }} env: @@ -143,7 +143,7 @@ runs: - name: Build selected wheel if: ${{ inputs.cibw-build != '' }} - uses: pypa/cibuildwheel@v3.4.1 + uses: pypa/cibuildwheel@v4.3.5 with: output-dir: ${{ inputs.output-dir }} env: diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 8beb5b424bb..0e48b003efa 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -17,7 +17,7 @@ jobs: timeout-minutes: 75 strategy: matrix: - python-version: ["3.14"] + python-version: ["3.15"] steps: - name: Checkout code uses: actions/checkout@v5 diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 0700872af17..42e3bef4a78 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -37,7 +37,7 @@ jobs: - name: Build Basilisk uses: ./.github/actions/build with: - python-version: 3.14 + python-version: "3.15" conan-args: ${{ matrix.conan-args }} save-conan-cache: true diff --git a/.github/workflows/nightly-wheels.yml b/.github/workflows/nightly-wheels.yml index 8f2ad4f3aca..64bd5c62103 100644 --- a/.github/workflows/nightly-wheels.yml +++ b/.github/workflows/nightly-wheels.yml @@ -96,7 +96,7 @@ jobs: - uses: actions/setup-python@v6 with: - python-version: 3.14 + python-version: "3.15" - name: Download core wheel input uses: actions/download-artifact@v6 diff --git a/.github/workflows/publish-wheels.yml b/.github/workflows/publish-wheels.yml index 638687b1a56..ea690ff49af 100644 --- a/.github/workflows/publish-wheels.yml +++ b/.github/workflows/publish-wheels.yml @@ -128,7 +128,7 @@ jobs: - uses: actions/setup-python@v6 with: - python-version: 3.14 + python-version: "3.15" - name: Download core wheel input uses: actions/download-artifact@v6 @@ -197,7 +197,7 @@ jobs: - uses: actions/setup-python@v6 with: - python-version: 3.14 + python-version: "3.15" - name: Download sdist uses: actions/download-artifact@v6 diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 8d0519bcbfd..82c18ea36fd 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -62,7 +62,7 @@ jobs: # Windows, and macOS. - uses: ./.github/actions/build with: - python-version: 3.14 + python-version: "3.15" conan-args: --opNav True --mujoco True --rustModules True save-conan-cache: true - name: Verify Corrosion configuration @@ -125,7 +125,7 @@ jobs: - uses: ./.github/actions/data-cache - uses: ./.github/actions/build with: - python-version: 3.14 + python-version: "3.15" conan-args: --opNav True --mujoco True --rustModules True save-conan-cache: true - name: Verify Corrosion configuration @@ -186,7 +186,7 @@ jobs: ] include: # An extra plain Basilisk build without optional opNav, MuJoCo, or visualization support - - python: "3.14" + - python: "3.15" python_label: "MAX Python" pytest_flags: -n auto -m "not ciSkip" -rs --timeout=300 --timeout-method=thread --durations=25 -v conan_args: --opNav False --mujoco False --vizInterface False diff --git a/.github/workflows/test-optional-wheels.yml b/.github/workflows/test-optional-wheels.yml index c6afaa6ab28..e4ed09059a9 100644 --- a/.github/workflows/test-optional-wheels.yml +++ b/.github/workflows/test-optional-wheels.yml @@ -65,7 +65,7 @@ jobs: rust-toolchain: ${{ needs.resolve-rust-toolchain.outputs.version }} output-dir: ${{ matrix.output-dir }} conan-args: ${{ matrix.conan-args }} - cibw-build: cp314-* + cibw-build: cp315-* save-conan-cache: ${{ matrix.save-conan-cache }} - name: Upload wheel input @@ -110,7 +110,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: "3.14" + python-version: "3.15" - name: Download core wheel input uses: actions/download-artifact@v6 diff --git a/pyproject.toml b/pyproject.toml index bb46ec06ddc..280307f3d37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,9 +88,11 @@ all = [ ] [tool.cibuildwheel] -build = ["cp39-*", "cp310-*", "cp311-*", "cp312-*", "cp313-*", "cp314-*"] +build = ["cp39-*", "cp310-*", "cp311-*", "cp312-*", "cp313-*", "cp314-*", "cp315-*"] skip = ["*-win32", "*-musllinux_*"] build-verbosity = 1 +# Preserve the existing wheel checks; cibuildwheel 4 enables an ABI audit by default. +audit-command = "" before-build = "python .github/scripts/configure_conan_retries.py && pip install \"libclang>=15.0.6.1,<=18.1.1\" ninja" test-extras = ["test"] test-command = "bskLargeData && pytest -v {project}/src/tests -m \"not ciSkip and not scenarioTest\"" From 95613447ac174e023afa2f504c39fa719cba909f Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Thu, 29 Oct 2026 19:27:41 -0600 Subject: [PATCH 05/13] [#1663] Document Python 3.15 support Update the platform installation guides to list standard CPython 3.9 through 3.15 and add a release note covering dependency compatibility and existing CI version updates. --- docs/source/Build/installOnLinux.rst | 2 +- docs/source/Build/installOnMacOS.rst | 2 +- docs/source/Build/installOnWindows.rst | 2 +- .../Support/bskReleaseNotesSnippets/python-315-support.rst | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 docs/source/Support/bskReleaseNotesSnippets/python-315-support.rst diff --git a/docs/source/Build/installOnLinux.rst b/docs/source/Build/installOnLinux.rst index bf5a56aa25f..eae0d719c91 100644 --- a/docs/source/Build/installOnLinux.rst +++ b/docs/source/Build/installOnLinux.rst @@ -12,7 +12,7 @@ Software setup In order to run Basilisk, the following software will be necessary. This document outline how to install this support software. -- `Python `__ 3.9 to 3.14. Version 3.9 is deprecated and will be removed March 2027. +- Standard `CPython `__ 3.9 to 3.15. Version 3.9 is deprecated and will be removed March 2027. - `GCC `__ - (Optional) Get the `GitKraken `__ application to be able to pull and manage a copy of Basilisk diff --git a/docs/source/Build/installOnMacOS.rst b/docs/source/Build/installOnMacOS.rst index b76823792e7..51fa8412ff2 100644 --- a/docs/source/Build/installOnMacOS.rst +++ b/docs/source/Build/installOnMacOS.rst @@ -8,7 +8,7 @@ Setup On macOS ============== These instruction outline how to install Basilisk (BSK) on a clean version of macOS. -Basilisk requires the use of Python 3.9 to 3.14. Version 3.9 is deprecated and will be removed March 2027. +Basilisk supports standard CPython 3.9 to 3.15. Version 3.9 is deprecated and will be removed March 2027. The following python package dependencies are automatically checked and installed in the steps below. diff --git a/docs/source/Build/installOnWindows.rst b/docs/source/Build/installOnWindows.rst index fbc3bde1f7c..41f88a9e412 100644 --- a/docs/source/Build/installOnWindows.rst +++ b/docs/source/Build/installOnWindows.rst @@ -13,7 +13,7 @@ Software setup In order to run Basilisk, the following software will be necessary: -- `Python `__ 3.9 to 3.14. +- Standard `CPython `__ 3.9 to 3.15. Version 3.9 is deprecated and will be removed March 2027. - `pip `__ - Default compiler is Visual Studios 17 2022 diff --git a/docs/source/Support/bskReleaseNotesSnippets/python-315-support.rst b/docs/source/Support/bskReleaseNotesSnippets/python-315-support.rst new file mode 100644 index 00000000000..108ca0407e3 --- /dev/null +++ b/docs/source/Support/bskReleaseNotesSnippets/python-315-support.rst @@ -0,0 +1 @@ +- Added standard CPython 3.15 support while retaining Python 3.9 through 3.14. Python 3.15 uses NumPy 2.5.2 or newer and Numba 0.68 for examples and development; older interpreters retain their dependency ranges. Updated source-build CI and wheel tooling, and moved the existing wheel installation checks to Python 3.15. From 71a8966bb58328c965a1d6235344a708fead8450 Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Thu, 29 Oct 2026 20:21:30 -0600 Subject: [PATCH 06/13] [#1663] Defer support-data tag checks until a download is needed --- .../Learn/bskPrinciples/bskPrinciples-6a.rst | 7 ++ .../defer-support-data-tag-check.rst | 6 + .../defer-support-data-tag-check.rst | 1 + src/tests/test_dataFetcher.py | 119 ++++++++++++++++++ src/utilities/supportDataTools/dataFetcher.py | 19 ++- 5 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 docs/source/Support/bskKnownIssuesSnippets/defer-support-data-tag-check.rst create mode 100644 docs/source/Support/bskReleaseNotesSnippets/defer-support-data-tag-check.rst diff --git a/docs/source/Learn/bskPrinciples/bskPrinciples-6a.rst b/docs/source/Learn/bskPrinciples/bskPrinciples-6a.rst index 6adc507fd9b..b51dfc8bd4e 100644 --- a/docs/source/Learn/bskPrinciples/bskPrinciples-6a.rst +++ b/docs/source/Learn/bskPrinciples/bskPrinciples-6a.rst @@ -12,6 +12,13 @@ The data importing is handled with :ref:`dataFetcher`. This process pulls the d file from the local ``basilisk/supportData`` folder if it exists (repo was cloned), or from the ``pooch`` cache if it was downloaded by ``pooch`` data management package. +Importing the data fetching tools and using local or valid cached files do not +contact GitHub. When a file must be downloaded from Basilisk's GitHub repository, +the downloader checks whether the installed release tag exists and falls back to +``develop`` if it is unavailable. The tag result is cached for subsequent +downloads in the same Python process. Files with explicit external URLs, such +as NASA ephemeris files, do not need this tag check. + At the top of your simulation script, import the data fetching tools using:: from Basilisk.utilities.supportDataTools.dataFetcher import get_path, DataFile diff --git a/docs/source/Support/bskKnownIssuesSnippets/defer-support-data-tag-check.rst b/docs/source/Support/bskKnownIssuesSnippets/defer-support-data-tag-check.rst new file mode 100644 index 00000000000..3eb0b600d32 --- /dev/null +++ b/docs/source/Support/bskKnownIssuesSnippets/defer-support-data-tag-check.rst @@ -0,0 +1,6 @@ +- In Basilisk 2.12.1, importing the support-data loader checks GitHub for a release + tag even when local or cached files are available, adding network-dependent + startup delays. The lookup is now deferred until a repository-hosted file + needs downloading. Local files, valid cached files, and explicit external + download URLs avoid the lookup; release-tag selection and the fallback to + ``develop`` are preserved when a download is required. diff --git a/docs/source/Support/bskReleaseNotesSnippets/defer-support-data-tag-check.rst b/docs/source/Support/bskReleaseNotesSnippets/defer-support-data-tag-check.rst new file mode 100644 index 00000000000..d5ee1e2a804 --- /dev/null +++ b/docs/source/Support/bskReleaseNotesSnippets/defer-support-data-tag-check.rst @@ -0,0 +1 @@ +- Deferred the support-data GitHub release-tag lookup until a repository-hosted file needs downloading, eliminating this network request during imports, local and valid cached file access, and external data downloads. diff --git a/src/tests/test_dataFetcher.py b/src/tests/test_dataFetcher.py index 9d7851e000d..a43b1a69a3c 100644 --- a/src/tests/test_dataFetcher.py +++ b/src/tests/test_dataFetcher.py @@ -20,7 +20,10 @@ Unit tests for Basilisk supportData fetching (Pooch + local fallback). """ +import hashlib +import importlib.util from pathlib import Path +from unittest.mock import Mock import pytest import requests @@ -59,6 +62,122 @@ def fake_fetch(monkeypatch, tmp_path): return dummy +@pytest.fixture +def isolated_fetcher(monkeypatch, tmp_path): + """Import fresh fetcher state with network requests forbidden.""" + def reject_request(*args, **kwargs): + pytest.fail("Unexpected network request while accessing support data") + + monkeypatch.setattr(requests.sessions.Session, "request", reject_request) + monkeypatch.setenv("BSK_SUPPORT_DATA_CACHE", str(tmp_path / "cache")) + spec = importlib.util.spec_from_file_location( + "isolated_data_fetcher", dataFetcher.__file__ + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def recorded_downloads(isolated_fetcher, monkeypatch): + """Use real Pooch cache handling with an in-memory HTTP response.""" + content = b"test support data\n" + urls = [] + + def download(url, output_file, pooch_obj, check_only=False): + urls.append(url) + Path(output_file).write_bytes(content) + + downloader_type = isolated_fetcher.RetryingHTTPDownloader + monkeypatch.setattr( + isolated_fetcher, + "RetryingHTTPDownloader", + lambda **kwargs: downloader_type( + downloader_factory=lambda **options: download, **kwargs + ), + ) + monkeypatch.setattr(isolated_fetcher, "LOCAL_SUPPORT", None) + return content, urls + + +def test_import_does_not_access_network(isolated_fetcher): + """Importing the fetcher must initialize its registry without HTTP requests.""" + assert isolated_fetcher.POOCH.registry + + +def test_local_support_file_does_not_access_network(isolated_fetcher, monkeypatch, tmp_path): + """A checkout's support file must be usable without a GitHub tag lookup.""" + file_enum = isolated_fetcher.DataFile.MagneticFieldData.WMM + rel = isolated_fetcher.relpath(file_enum) + local_root = tmp_path / "supportData" + local_file = tmp_path / rel + local_file.parent.mkdir(parents=True) + local_file.write_text("local data") + monkeypatch.setattr(isolated_fetcher, "LOCAL_SUPPORT", local_root) + + assert isolated_fetcher.get_path(file_enum) == local_file + + +def test_cached_support_file_does_not_access_network(isolated_fetcher, monkeypatch): + """A valid Pooch cache entry must be returned without HTTP requests.""" + file_enum = isolated_fetcher.DataFile.MagneticFieldData.WMM + rel = isolated_fetcher.relpath(file_enum) + cached_file = isolated_fetcher.POOCH.abspath / rel + cached_file.parent.mkdir(parents=True) + content = b"cached data" + cached_file.write_bytes(content) + monkeypatch.setattr(isolated_fetcher, "LOCAL_SUPPORT", None) + monkeypatch.setitem(isolated_fetcher.POOCH.registry, rel, hashlib.sha256(content).hexdigest()) + + assert isolated_fetcher.get_path(file_enum) == cached_file + + +@pytest.mark.parametrize("cache_state", ["missing", "stale"]) +@pytest.mark.parametrize("tag_status", [200, 404, "timeout"]) +def test_remote_fetch_resolves_tag_once( + isolated_fetcher, recorded_downloads, monkeypatch, cache_state, tag_status +): + """Resolve release or develop URLs on download and reuse the tag result.""" + content, downloaded_urls = recorded_downloads + file_enum = isolated_fetcher.DataFile.MagneticFieldData.WMM + rel = isolated_fetcher.relpath(file_enum) + another_rel = "supportData/another.dat" + content_hash = hashlib.sha256(content).hexdigest() + monkeypatch.setitem(isolated_fetcher.POOCH.registry, rel, content_hash) + monkeypatch.setitem(isolated_fetcher.POOCH.registry, another_rel, content_hash) + cached_file = isolated_fetcher.POOCH.abspath / rel + if cache_state == "stale": + cached_file.parent.mkdir(parents=True) + cached_file.write_text("outdated data") + + if tag_status == "timeout": + head = Mock(side_effect=requests.exceptions.Timeout("tag lookup timed out")) + else: + response = requests.Response() + response.status_code = tag_status + head = Mock(return_value=response) + monkeypatch.setattr(requests, "head", head) + + assert isolated_fetcher.get_path(file_enum).read_bytes() == content + assert isolated_fetcher.get_path(file_enum) == cached_file + assert Path(isolated_fetcher.fetch_support_data(another_rel)).read_bytes() == content + + version = isolated_fetcher.DATA_VERSION if tag_status == 200 else "develop" + base_url = f"https://raw.githubusercontent.com/AVSLab/basilisk/{version}/" + assert downloaded_urls == [base_url + rel, base_url + another_rel] + assert head.call_count == 1 + + +def test_external_download_does_not_check_github(isolated_fetcher, recorded_downloads): + """Explicit external data URLs must not require Basilisk's release tag.""" + content, downloaded_urls = recorded_downloads + file_enum = isolated_fetcher.DataFile.EphemerisData.de430 + rel = isolated_fetcher.relpath(file_enum) + + assert isolated_fetcher.get_path(file_enum).read_bytes() == content + assert downloaded_urls == [isolated_fetcher.EXTERNAL_KERNEL_URLS[rel]] + + def test_all_enum_entries_are_in_registry(): """Every DataFile enum value must correspond to a key in REGISTRY.""" missing = [] diff --git a/src/utilities/supportDataTools/dataFetcher.py b/src/utilities/supportDataTools/dataFetcher.py index 37fdb2c66d5..4e8f5e90a8f 100644 --- a/src/utilities/supportDataTools/dataFetcher.py +++ b/src/utilities/supportDataTools/dataFetcher.py @@ -124,14 +124,12 @@ def find_local_support_data() -> Optional[Path]: return None -# Compute the base GitHub URL once at import time. LOCAL_SUPPORT = find_local_support_data() -# For remote fetches (wheel installs), check if the version tag exists. Fall -# back to develop if not +# Start with the versioned URL without accessing GitHub. The downloader resolves +# the develop fallback only after Pooch determines that a download is needed. _version_tag_url = f"https://github.com/AVSLab/basilisk/releases/tag/{DATA_VERSION}" -_remote_version = DATA_VERSION if tag_exists(_version_tag_url) else "develop" -BASE_URL = f"https://raw.githubusercontent.com/AVSLab/basilisk/{_remote_version}/" +BASE_URL = f"https://raw.githubusercontent.com/AVSLab/basilisk/{DATA_VERSION}/" POOCH = pooch.create( path=_POOCH_PATH, @@ -141,6 +139,16 @@ def find_local_support_data() -> Optional[Path]: ) +def _resolve_github_url(url: str) -> str: + """Resolve the release-tag fallback only for Basilisk GitHub downloads.""" + if url.startswith(BASE_URL) and not tag_exists(_version_tag_url): + return ( + "https://raw.githubusercontent.com/AVSLab/basilisk/develop/" + + url[len(BASE_URL):] + ) + return url + + def _is_retryable_error(error: Exception) -> bool: """Return ``True`` if a support data download error should be retried.""" if isinstance(error, requests.exceptions.HTTPError): @@ -178,6 +186,7 @@ def __init__( self.backup_urls_for = backup_urls_for def __call__(self, url, output_file, pooch_obj, check_only=False): + url = _resolve_github_url(url) last_error = None downloader = self.downloader_factory( progressbar=self.progressbar, From 12fbdb84c67ce2efde728fae7b656f9e63eb26e0 Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Thu, 29 Oct 2026 21:01:33 -0600 Subject: [PATCH 07/13] [#1663] Defer SimulationBaseClass Matplotlib imports on Python 3.15 Use __lazy_modules__ to defer plotting dependencies on Python 3.15 while retaining eager imports on Python 3.9-3.14. Add one compact subprocess test for import behavior and first-use execution plotting, plus a release note. Validation: the subprocess test passed on each Python version from 3.9 through 3.15. The existing 87 SimulationBaseClass tests previously passed on the same versions. Median SimulationBaseClass import time fell from 266 ms to 37 ms across 12 paired fresh-process trials on Python 3.15.2. --- .../lazy-simulation-matplotlib.rst | 1 + src/utilities/SimulationBaseClass.py | 4 ++ .../tests/test_simulationBaseClass_imports.py | 65 +++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 docs/source/Support/bskReleaseNotesSnippets/lazy-simulation-matplotlib.rst create mode 100644 src/utilities/tests/test_simulationBaseClass_imports.py diff --git a/docs/source/Support/bskReleaseNotesSnippets/lazy-simulation-matplotlib.rst b/docs/source/Support/bskReleaseNotesSnippets/lazy-simulation-matplotlib.rst new file mode 100644 index 00000000000..03b6a871166 --- /dev/null +++ b/docs/source/Support/bskReleaseNotesSnippets/lazy-simulation-matplotlib.rst @@ -0,0 +1 @@ +- On Python 3.15, ``SimulationBaseClass`` defers its Matplotlib imports until plotting functionality is used, reducing startup work for simulations that do not draw diagrams. Python 3.9–3.14 retain eager imports and the same plotting interfaces. diff --git a/src/utilities/SimulationBaseClass.py b/src/utilities/SimulationBaseClass.py index 174140c974e..362a979fca7 100755 --- a/src/utilities/SimulationBaseClass.py +++ b/src/utilities/SimulationBaseClass.py @@ -15,6 +15,10 @@ # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# Python 3.15 defers these imports until a diagram is drawn. Older Python +# versions keep the ordinary eager imports below. +__lazy_modules__ = ("matplotlib.pyplot", "matplotlib.patches") + # Import some architectural stuff that we will probably always use import html import math diff --git a/src/utilities/tests/test_simulationBaseClass_imports.py b/src/utilities/tests/test_simulationBaseClass_imports.py new file mode 100644 index 00000000000..7c37408aa18 --- /dev/null +++ b/src/utilities/tests/test_simulationBaseClass_imports.py @@ -0,0 +1,65 @@ +# ISC License +# +# Copyright (c) 2026, Autonomous Vehicle Systems Lab, University of Colorado at Boulder +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Check Matplotlib import behavior and first plotting use in a fresh process.""" + +import os +from pathlib import Path +import subprocess +import sys +import textwrap + +import Basilisk +from Basilisk.utilities import SimulationBaseClass + + +def test_matplotlib_imports_and_first_diagram(): + """Defer imports on 3.15 while preserving first-use plotting on all versions.""" + script = textwrap.dedent(""" + import sys + + sys.path.insert(0, sys.argv[1]) + from Basilisk import utilities + utilities.__path__.insert(0, sys.argv[2]) + from Basilisk.utilities import SimulationBaseClass + + eager = sys.version_info < (3, 15) + assert ("matplotlib.pyplot" in sys.modules) == eager + assert ("matplotlib.patches" in sys.modules) == eager + + sim = SimulationBaseClass.SimBaseClass() + process = sim.CreateNewProcess("testProcess") + time_step = 1_000_000_000 # [ns] + process.addTask(sim.CreateNewTask("testTask", time_step)) + figure = sim.ShowExecutionFigure(show_plots=False) + figure.canvas.draw() + assert "matplotlib.pyplot" in sys.modules + assert figure.axes + """) + result = subprocess.run( + [ + sys.executable, + "-c", + script, + str(Path(Basilisk.__path__[0]).resolve().parent), + str(Path(SimulationBaseClass.__file__).resolve().parent), + ], + env=dict(os.environ, MPLBACKEND="Agg", PYTHON_LAZY_IMPORTS="normal"), + capture_output=True, + text=True, + timeout=60, # [s] + ) + assert result.returncode == 0, result.stdout + result.stderr From 99c3575dd4e2d479bab207df5cd89c7dfa6aa439 Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Thu, 29 Oct 2026 22:24:53 -0600 Subject: [PATCH 08/13] [#1663] Remove redundant Monte Carlo retention delay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the one-second pause after queuing retained data, relying on existing writer shutdown synchronization. Preserve worker-pool behavior. Add regression coverage for delayed writers and document the change. Monte Carlo benchmarks show 28–62% shorter batch times with identical retained results. --- .../1663-monte-carlo-retention-delay.rst | 1 + src/utilities/MonteCarlo/Controller.py | 3 +- .../MonteCarlo/_UnitTests/test_controller.py | 72 +++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-retention-delay.rst diff --git a/docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-retention-delay.rst b/docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-retention-delay.rst new file mode 100644 index 00000000000..915dae288f5 --- /dev/null +++ b/docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-retention-delay.rst @@ -0,0 +1 @@ +- Removed the fixed one-second delay after each Monte Carlo run queues retained data. The controller still waits for the data writer to finish before returning; worker-pool behavior is unchanged. diff --git a/src/utilities/MonteCarlo/Controller.py b/src/utilities/MonteCarlo/Controller.py index c6dd3e068e5..519eeba0374 100644 --- a/src/utilities/MonteCarlo/Controller.py +++ b/src/utilities/MonteCarlo/Controller.py @@ -876,8 +876,9 @@ def __call__(cls, params): print("Retaining data for run in", retentionFile) retainedData = RetentionPolicy.getDataForRetention(simInstance, simParams.retentionPolicies) + # Manager.Queue.put() completes before returning; the controller + # joins the writer before shutting down the queue manager. dataOutQueue.put((retainedData, simParams.index, None)) - time.sleep(1) with gzip.open(retentionFile, "w") as archive: retainedData["index"] = simParams.index # add run index diff --git a/src/utilities/MonteCarlo/_UnitTests/test_controller.py b/src/utilities/MonteCarlo/_UnitTests/test_controller.py index d60b945fe98..8bb508f9010 100644 --- a/src/utilities/MonteCarlo/_UnitTests/test_controller.py +++ b/src/utilities/MonteCarlo/_UnitTests/test_controller.py @@ -17,13 +17,18 @@ # import inspect +import multiprocessing as mp +import signal import numpy as np +import pandas as pd import pytest from Basilisk.simulation import motorVoltageInterface, spacecraft from Basilisk.utilities import simHelpers +from Basilisk.utilities.MonteCarlo import Controller as controller_module from Basilisk.utilities.MonteCarlo.Controller import SimulationExecutor, SimulationParameters +from Basilisk.utilities.MonteCarlo.DataWriter import DataWriter from Basilisk.utilities.MonteCarlo.Dispersions import ( NormalVectorAngleDispersion, NormalVectorDispersion, @@ -76,6 +81,73 @@ def get_DynModel(self): return self.dynModel +class DeferredDataWriter(DataWriter): + """Hold retained data in the queue until the controller starts shutdown.""" + + def __init__(self, queue): + super().__init__(queue) + self.release = mp.Event() + + def run(self): + """Drain and consolidate the backlog after all producers finish.""" + timeout = 30.0 # [s] + assert self.release.wait(timeout), "Controller did not release the writer" + super().run() + + +def execute_retention_simulation(sim): + times = np.arange(32) # [ns] + sim.retained = np.column_stack((times, np.random.random((32, 3)))) + + +def retain_simulation_data(sim): + return {"samples": sim.retained} + + +@pytest.mark.parametrize("workers", [1, 2, 4]) +def test_controller_drains_retained_data_before_return(tmp_path, monkeypatch, workers): + """Verify a delayed writer saves every run before the controller returns.""" + controller = controller_module.Controller() + controller.setSimulationFunction(DummySimulation) + controller.setExecutionFunction(execute_retention_simulation) + controller.setExecutionCount(8) + controller.setThreadCount(workers) + controller.setArchiveDir(str(tmp_path / "archive")) + controller.setVerbose(False) + controller.setShowProgressBar(False) + policy = controller_module.RetentionPolicy() + policy.addRetentionFunction(retain_simulation_data) + controller.addRetentionPolicy(policy) + shutdown = controller_module.Controller._shutdown_data_writer + + def release_and_shutdown(self): + writer = self.dataWriter + writer.release.set() + shutdown(self) + assert writer.exitcode == 0 + + monkeypatch.setattr(controller_module, "DataWriter", DeferredDataWriter) + monkeypatch.setattr( + controller_module.Controller, "_shutdown_data_writer", release_and_shutdown + ) + interrupt_handler = signal.getsignal(signal.SIGINT) + try: + assert controller.executeSimulations() == [] + finally: + signal.signal(signal.SIGINT, interrupt_handler) + + combined = pd.read_pickle(tmp_path / "archive" / "samples.data") + expected_columns = [(run, component) for run in range(8) for component in range(3)] + assert list(combined.columns) == expected_columns + np.testing.assert_array_equal(combined.index, np.arange(32)) + for run in range(8): + expected = np.random.RandomState(run * 10).random_sample((32, 3)) + retained = controller.getRetainedData(run) + assert retained["index"] == run + np.testing.assert_array_equal(retained["custom"]["samples"][:, 1:], expected) + np.testing.assert_array_equal(combined[run].to_numpy(), expected) + + @pytest.mark.parametrize( "dispersion_class", [SingleVariableDispersion, VectorVariableDispersion], From 02f0d4ae275405dc8111e7ed2067b8ec4c3a4903 Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Fri, 30 Oct 2026 09:53:30 -0600 Subject: [PATCH 09/13] [#1663] Reuse Monte Carlo workers with bounded retention - Reuse one process pool across batches with configurable worker recycling. - Bound the retention queue and raise RuntimeError on writer failures. - Clean up workers, writers, and managers after failures or interruption. - Preserve failed-index reporting for task preparation and submission errors. - Support NumPy case indices in initial-condition reruns. - Add regression tests and document the resource controls. Validated: all 68 Monte Carlo tests pass on Python 3.9, 3.14, and 3.15. --- .../1663-monte-carlo-writer-failure.rst | 1 + .../1663-monte-carlo-pool-reuse.rst | 1 + .../1663-monte-carlo-retention-delay.rst | 2 +- src/utilities/MonteCarlo/Controller.py | 432 +++++++++--------- src/utilities/MonteCarlo/README.md | 29 ++ .../MonteCarlo/_UnitTests/test_controller.py | 268 ++++++++++- 6 files changed, 521 insertions(+), 212 deletions(-) create mode 100644 docs/source/Support/bskKnownIssuesSnippets/1663-monte-carlo-writer-failure.rst create mode 100644 docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-pool-reuse.rst diff --git a/docs/source/Support/bskKnownIssuesSnippets/1663-monte-carlo-writer-failure.rst b/docs/source/Support/bskKnownIssuesSnippets/1663-monte-carlo-writer-failure.rst new file mode 100644 index 00000000000..35d446487c7 --- /dev/null +++ b/docs/source/Support/bskKnownIssuesSnippets/1663-monte-carlo-writer-failure.rst @@ -0,0 +1 @@ +- In Basilisk 2.12.1, a failed Monte Carlo data writer can leave incomplete combined dataframes without reporting an error to the caller; initial-condition runs can also wait indefinitely for queued data. The controller now detects writer exit, raises ``RuntimeError``, and cleans up its workers and queue manager. Treat partial archives from failed or interrupted campaigns as incomplete and rerun after resolving the writer error. diff --git a/docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-pool-reuse.rst b/docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-pool-reuse.rst new file mode 100644 index 00000000000..df61720b0e6 --- /dev/null +++ b/docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-pool-reuse.rst @@ -0,0 +1 @@ +- Monte Carlo campaigns now reuse their process pool, recycling each worker after eight simulations by default. Configure worker lifetime with ``setMaxTasksPerChild()`` and bounded retention-queue capacity with ``setRetentionQueueSize()``. Writer failures now raise ``RuntimeError`` and release campaign resources, including during initial-condition reruns. diff --git a/docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-retention-delay.rst b/docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-retention-delay.rst index 915dae288f5..75396ff9274 100644 --- a/docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-retention-delay.rst +++ b/docs/source/Support/bskReleaseNotesSnippets/1663-monte-carlo-retention-delay.rst @@ -1 +1 @@ -- Removed the fixed one-second delay after each Monte Carlo run queues retained data. The controller still waits for the data writer to finish before returning; worker-pool behavior is unchanged. +- Removed the fixed one-second delay after each Monte Carlo run queues retained data. The controller still waits for the data writer to finish before returning. diff --git a/src/utilities/MonteCarlo/Controller.py b/src/utilities/MonteCarlo/Controller.py index 519eeba0374..a98fabf3005 100644 --- a/src/utilities/MonteCarlo/Controller.py +++ b/src/utilities/MonteCarlo/Controller.py @@ -27,7 +27,6 @@ import os import random import shutil -import sys import traceback import warnings import ast @@ -38,7 +37,9 @@ import gzip import json import signal -import time +from numbers import Integral +from operator import index +from queue import Full import numpy as np import multiprocessing as mp import pickle as pickle @@ -47,6 +48,40 @@ from Basilisk.utilities.simulationProgessBar import SimulationProgressBar +_WRITER_POLL_INTERVAL = 0.1 # [s] + + +def _check_writer(writer): + """Raise if a writer exits before receiving its end marker.""" + if writer is not None and writer.exitcode is not None: + raise RuntimeError( + f"Monte Carlo data writer exited unexpectedly (exit code {writer.exitcode})" + ) + + +def _initialize_worker(): + """Let the controller handle keyboard interrupts for pool workers.""" + signal.signal(signal.SIGINT, signal.SIG_IGN) + + +class _CheckedRetentionQueue: + """Check writer health while the calling process waits for queue space.""" + + def __init__(self, queue, writer): + self.queue = queue + self.writer = writer + + def put(self, item): + """Enqueue an item or raise if the writer has exited.""" + while True: + _check_writer(self.writer) + try: + self.queue.put(item, timeout=_WRITER_POLL_INTERVAL) + return + except Full: + continue + + class Controller: """ The MonteCarloController class is used to run a monte carlo simulation. @@ -61,6 +96,8 @@ def __init__(self): self.archiveDir = None self.varCast = None self.numProcess = mp.cpu_count() + self.maxTasksPerChild = 8 + self.retentionQueueSize = None self.multiProcManager = None self.dataOutQueue = None self.dataWriter = None @@ -77,26 +114,137 @@ def __init__(self): icfilename="" ) - def _shutdown_data_writer(self): - if self.dataWriter is not None: - try: - if self.dataOutQueue is not None and getattr(self.dataWriter, "pid", None) is not None: - self.dataOutQueue.put((None, None, True)) - except Exception: - pass + def _start_data_writer(self, logDir, workers): + """Start a writer with bounded retention storage when requested.""" + if logDir is None: + return + self.multiProcManager = mp.Manager() + capacity = getattr(self, "retentionQueueSize", None) or 2 * workers + self.dataOutQueue = self.multiProcManager.Queue(maxsize=capacity) + self.dataWriter = DataWriter(self.dataOutQueue) + self.dataWriter.daemon = False + self.dataWriter.setLogDir(logDir) + self.dataWriter.setVarCast(self.varCast) + self.dataWriter.start() + + def _shutdown_data_writer(self, abort=False): + """Finish retained data, or stop immediately when aborting a campaign. + + :param bool abort: Stop the writer without consolidating partial data. + :raises RuntimeError: The writer exits before completing its output. + """ + writer = self.dataWriter + try: + if writer is not None and writer.pid is not None and not abort: + queue = _CheckedRetentionQueue(self.dataOutQueue, writer) + queue.put((None, None, True)) + writer.join() + if writer.exitcode != 0: + raise RuntimeError( + f"Monte Carlo data writer failed (exit code {writer.exitcode})" + ) + finally: try: - if getattr(self.dataWriter, "pid", None) is not None: - self.dataWriter.join() - except Exception: - pass - if self.multiProcManager is not None: + if writer is not None and writer.pid is not None: + if writer.is_alive(): + writer.terminate() + writer.join() + finally: + try: + if self.multiProcManager is not None: + self.multiProcManager.shutdown() + finally: + self.dataWriter = None + self.dataOutQueue = None + self.multiProcManager = None + + def _execute_cases(self, caseList, generator, logDir): + """Execute bounded batches in one pool and supervise retained output.""" + if len(caseList) == 0: + return [] + workers = min(self.numProcess, len(caseList)) + if workers < 1: + raise ValueError("Monte Carlo worker count must be positive") + failed = [] + finished = 0 + pool = None + progressBar = SimulationProgressBar(len(caseList), self.simParams.showProgressBar) + try: + self._start_data_writer(logDir, workers) + executor = SimulationExecutor() + if self.numProcess > 1: + pool = mp.Pool( + workers, + initializer=_initialize_worker, + maxtasksperchild=getattr(self, "maxTasksPerChild", 8), + ) + queue = self.dataOutQueue + if pool is None and queue is not None: + queue = _CheckedRetentionQueue(queue, self.dataWriter) + for start in range(0, len(caseList), workers): + _check_writer(self.dataWriter) + # Normalize NumPy indices without materializing the entire campaign. + cases = [index(case) for case in caseList[start:start + workers]] + try: + batch = [(sim, queue) for sim in generator(cases)] + except Exception as error: + _check_writer(self.dataWriter) + print("Error preparing Monte Carlo simulations:", error) + traceback.print_exc() + # Nothing in this batch has been submitted yet. + failed.extend(cases) + finished += len(cases) + progressBar.update(finished) + continue + if pool is None: + results = map(executor, batch) + else: + results = pool.imap_unordered(executor, batch, chunksize=1) + pending = [sim.index for sim, _ in batch] + remaining = len(batch) + while remaining: + _check_writer(self.dataWriter) + try: + if pool is None: + result = next(results) + else: + result = results.next(_WRITER_POLL_INTERVAL) + except mp.TimeoutError: + continue + except Exception as error: + if pool is None: + raise + # A submission error consumes one result without its + # run index. Collect the missing indices after draining + # the rest of this batch, including successful runs. + print("Error receiving Monte Carlo worker result:", error) + traceback.print_exc() + else: + pending.remove(result[1]) + if not result[0]: + failed.append(result[1]) + _check_writer(self.dataWriter) + remaining -= 1 + finished += 1 + progressBar.update(finished) + failed.extend(pending) + if pool is not None: + pool.close() + pool.join() + pool = None + self._shutdown_data_writer() + progressBar.markComplete() + except BaseException: try: - self.multiProcManager.shutdown() - except Exception: - pass - self.dataWriter = None - self.dataOutQueue = None - self.multiProcManager = None + if pool is not None: + pool.terminate() + pool.join() + finally: + self._shutdown_data_writer(abort=True) + raise + finally: + progressBar.close() + return sorted(failed) def setShowProgressBar(self, value): """ @@ -212,6 +360,39 @@ def setThreadCount(self, threads): """ self.numProcess = threads + def setMaxTasksPerChild(self, count): + """Set how many simulations each pool worker runs before replacement. + + The default is eight simulations. One restores a fresh process for + each simulation; ``None`` keeps workers for the entire campaign. + This setting does not affect the one-worker path, which runs in the + calling process. + + :param int count: Positive simulation count, or ``None`` for no limit. + :raises ValueError: The count is neither a positive integer nor ``None``. + """ + if count is not None and ( + isinstance(count, bool) or not isinstance(count, Integral) or count < 1 + ): + raise ValueError("Worker task limit must be a positive integer or None") + self.maxTasksPerChild = None if count is None else int(count) + + def setRetentionQueueSize(self, count): + """Set the maximum number of retained results awaiting the writer. + + ``None`` selects twice the active worker count. The limit counts + simulation results, not bytes; large retained arrays still require + corresponding memory. Producers wait when the queue is full. + + :param int count: Positive queue capacity, or ``None`` for automatic sizing. + :raises ValueError: The capacity is neither a positive integer nor ``None``. + """ + if count is not None and ( + isinstance(count, bool) or not isinstance(count, Integral) or count < 1 + ): + raise ValueError("Retention queue size must be a positive integer or None") + self.retentionQueueSize = None if count is None else int(count) + def setVerbose(self, verbose): """ Use verbose output for this MonteCarlo run @@ -380,18 +561,14 @@ def reRunCases(self, caseList): def runInitialConditions(self, caseList): """ - Run initial conditions given in a file + Execute saved initial conditions with the configured worker lifecycle. - Args: - caseList: int[] - The list of runs to repeat, a list of numbers. - Returns: - failures: int[] - The list of failed runs. + :param caseList: Sequence of integer indices of the saved simulations to execute. + :return: Sorted indices of failed simulations. + :rtype: list[int] + :raises RuntimeError: The data writer exits before finishing the archive. + :raises ValueError: The output archive would overwrite the input archive. """ - # the list of failures - failed = [] - assert self.icDirectory != "", "No initial condition directory was given" assert self.ICrunFlag is not False, "IC run flag was not set" @@ -410,91 +587,17 @@ def runInitialConditions(self, caseList): except Exception as e: print("Unknown exception while trying to pickle monte-carlo-controller... \ncontinuing...\n\n", e) - # Create Queue, but don't ever start it. - self.multiProcManager = mp.Manager() - self.dataOutQueue = self.multiProcManager.Queue() - self.dataWriter = DataWriter(self.dataOutQueue) - self.dataWriter.daemon = False - - # If archiving the rerun data -- make sure not to delete the original data! + # Preserve the input archive when saving consolidated rerun data. if self.archiveDir is not None: - if self.archiveDir != self.icDirectory: - if os.path.exists(self.archiveDir): - shutil.rmtree(self.archiveDir) - os.mkdir(self.archiveDir) - self.dataWriter.setLogDir(self.archiveDir) - self.dataWriter.start() - else: - print("ERROR: The archive directory is set as the icDirectory. Proceeding would have overwriten all data " \ - "within: " + self.archiveDir + " with the select rerun cases! Exiting.\n") - sys.exit("Change the archive directory to a new location when rerunning cases.") - else: - print("No archive data specified; no data will be logged to dataframes") - - jobsFinished = 0 # keep track of what simulations have finished - - # The simulation executor is responsible for executing simulation given a simulation's parameters - # It is called within worker threads with each worker's simulation parameters - simulationExecutor = SimulationExecutor() - # - progressBar = SimulationProgressBar(len(caseList), self.simParams.showProgressBar) - if self.numProcess == 1: # don't make child thread - if self.simParams.verbose: - print("Executing sequentially...") - i = 0 - for i in range(len(caseList)): - simGenerator = self.generateICSims(caseList[i:i+1]) - for sim in simGenerator: - try: - simulationExecutor([sim, self.dataOutQueue]) - except: - failed.append(i) - i += 1 - progressBar.update(i) - else: - numSims = len(caseList) - if self.numProcess > numSims: - print("Fewer MCs spawned than processes assigned (%d < %d). Changing processes count to %d." % (numSims, self.numProcess, numSims)) - self.numProcess = numSims - for i in range(numSims//self.numProcess): - # If number of sims doesn't factor evenly into the number of processes: - if numSims % self.numProcess != 0 and i == len(list(range(numSims//self.numProcess)))-1: - offset = numSims % self.numProcess - else: - offset = 0 - - simGenerator = self.generateICSims(caseList[self.numProcess*i:self.numProcess*(i+1)+offset]) - pool = mp.Pool(self.numProcess) - try: - # yields results *as* the workers finish jobs - for result in pool.imap_unordered(simulationExecutor, [(x, self.dataOutQueue) for x in simGenerator]): - if result[0] is not True: # workers return True on success - failed.append(result[1]) # add failed jobs to the list of failures - print("Job", result[1], "failed...") - - jobsFinished += 1 - progressBar.update(jobsFinished) - pool.close() - except KeyboardInterrupt as e: - print("Ctrl-C was hit, closing pool") - # failed.extend(range(jobsFinished, numSims)) # fail all potentially running jobs... - pool.terminate() - raise e - except Exception as e: - print("Unknown exception while running simulations:", e) - # failed.extend(range(jobsFinished, numSims)) # fail all potentially running jobs... - traceback.print_exc() - pool.terminate() - finally: - pool.join() + if self.archiveDir == self.icDirectory: + raise ValueError( + "The archive directory must differ from the initial-condition directory" + ) + if os.path.exists(self.archiveDir): + shutil.rmtree(self.archiveDir) + os.mkdir(self.archiveDir) - progressBar.markComplete() - progressBar.close() - # If the data was archiving, close the queue. - if self.archiveDir is not None and self.archiveDir != self.icDirectory: - while not self.dataOutQueue.empty(): - time.sleep(1) - self._shutdown_data_writer() + failed = self._execute_cases(caseList, self.generateICSims, self.archiveDir) # if there are failures if len(failed) > 0: @@ -594,10 +697,11 @@ def executeCallbacks(self, rng=None, retentionPolicies=[]): def executeSimulations(self): """ - Execute simulations in parallel + Execute simulations with a reusable worker pool and bounded retention. - :return: failed: int[] - A list of the indices of all failed simulation runs. + :return: Sorted indices of failed simulations. + :rtype: list[int] + :raises RuntimeError: The data writer exits before finishing the archive. """ if self.simParams.verbose: @@ -615,94 +719,9 @@ def executeSimulations(self): except Exception as e: print("Unknown exception while trying to pickle monte-carlo-controller... \ncontinuing...\n\n", e) - self.multiProcManager = mp.Manager() - self.dataOutQueue = self.multiProcManager.Queue() - self.dataWriter = DataWriter(self.dataOutQueue) - self.dataWriter.daemon = False - - numSims = self.executionCount - - # start data writer process - self.dataWriter.setLogDir(self.archiveDir) - self.dataWriter.setVarCast(self.varCast) - self.dataWriter.start() - - # Avoid building a full list of all simulations to run in memory, - # instead only generating simulations right before they are needed by a waiting worker - # This is accomplished using a generator and pool.imap, -- simulations are only built - # when they are about to be passed to a worker, avoiding memory overhead of first building simulations - # There is a system-dependent chunking behavior, sometimes 10-20 are generated at a time. - # simGenerator = self.generateSims(range(numSims)) - failed = [] # keep track of the indices of failed simulations - jobsFinished = 0 # keep track of what simulations have finished - - # The simulation executor is responsible for executing simulation given a simulation's parameters - # It is called within worker threads with each worker's simulation parameters - simulationExecutor = SimulationExecutor() - - progressBar = SimulationProgressBar(numSims, self.simParams.showProgressBar) - - # The outermost for-loop for both the serial and multiprocessed sim generator is not necessary. It - # is a temporary fix to a memory leak which is assumed to be a result of the simGenerator not collecting - # garbage properly. # TODO: Find a more permenant solution to the leak. - - try: - if self.numProcess == 1: # don't make child thread - if self.simParams.verbose: - print("Executing sequentially...") - i = 0 - for i in range(numSims): - simGenerator = self.generateSims(list(range(i,i+1))) - for sim in simGenerator: - try: - run_ok = simulationExecutor([sim, self.dataOutQueue])[0] - except: - failed.append(i) - else: - if not run_ok: - failed.append(i) - i += 1 - progressBar.update(i) - else: - if self.numProcess > numSims: - print("Fewer MCs spawned than processes assigned (%d < %d). Changing processes count to %d." % (numSims, self.numProcess, numSims)) - self.numProcess = numSims - for i in range(numSims//self.numProcess): - # If number of sims doesn't factor evenly into the number of processes: - if numSims % self.numProcess != 0 and i == len(list(range(numSims//self.numProcess)))-1: - offset = numSims % self.numProcess - else: - offset = 0 - simGenerator = self.generateSims(list(range(self.numProcess*i, self.numProcess*(i+1)+offset))) - pool = mp.Pool(self.numProcess) - try: - # yields results *as* the workers finish jobs - for result in pool.imap_unordered(simulationExecutor, [(x, self.dataOutQueue) for x in simGenerator]): - if result[0] is not True: # workers return True on success - failed.append(result[1]) # add failed jobs to the list of failures - print("Job", result[1], "failed...") - - jobsFinished += 1 - progressBar.update(jobsFinished) - pool.close() - except KeyboardInterrupt as e: - print("Ctrl-C was hit, closing pool") - failed.extend(list(range(jobsFinished, numSims))) # fail all potentially running jobs... - pool.terminate() - raise e - except Exception as e: - print("Unknown exception while running simulations:", e) - failed.extend(list(range(jobsFinished, numSims))) # fail all potentially running jobs... - traceback.print_exc() - pool.terminate() - finally: - # Wait until all data is logged from the spawned runs before proceeding with the next set. - pool.join() - - progressBar.markComplete() - finally: - progressBar.close() - self._shutdown_data_writer() + failed = self._execute_cases( + range(self.executionCount), self.generateSims, self.archiveDir + ) # if there are failures if len(failed) > 0: @@ -784,8 +803,6 @@ def __call__(cls, params): dataOutQueue = params[1] try: - signal.signal(signal.SIGINT, signal.SIG_IGN) # On ctrl-c ignore the signal... let the parent deal with it. - # must make new random seed on each new thread. np.random.seed(simParams.index * 10) random.seed(simParams.index * 10) @@ -878,7 +895,8 @@ def __call__(cls, params): retainedData = RetentionPolicy.getDataForRetention(simInstance, simParams.retentionPolicies) # Manager.Queue.put() completes before returning; the controller # joins the writer before shutting down the queue manager. - dataOutQueue.put((retainedData, simParams.index, None)) + if dataOutQueue is not None: + dataOutQueue.put((retainedData, simParams.index, None)) with gzip.open(retentionFile, "w") as archive: retainedData["index"] = simParams.index # add run index diff --git a/src/utilities/MonteCarlo/README.md b/src/utilities/MonteCarlo/README.md index a4013639c73..95173b75851 100644 --- a/src/utilities/MonteCarlo/README.md +++ b/src/utilities/MonteCarlo/README.md @@ -164,9 +164,38 @@ monteCarlo.setExecutionCount(100) monteCarlo.setThreadCount(4) monteCarlo.setVerbose(False) +# Optional resource limits (these values are the defaults for four workers). +monteCarlo.setMaxTasksPerChild(8) +monteCarlo.setRetentionQueueSize(8) + failures = monteCarlo.executeSimulations() ``` +Parallel campaigns reuse one process pool across batches. By default, each worker +is replaced after eight simulations. `setMaxTasksPerChild(1)` gives each simulation +a fresh worker; `setMaxTasksPerChild(None)` allows unlimited reuse. Application +globals and native caches can persist between simulations in a reused worker. +The one-worker path still executes in the calling process, where this recycling +setting does not apply. + +The retention queue defaults to twice the active worker count. Set a positive +capacity with `setRetentionQueueSize(count)`, or restore automatic sizing with +`setRetentionQueueSize(None)`. The capacity counts retained simulation results, +not bytes. Producers wait when the queue is full, limiting queued data without +discarding results. Individual results and the writer's final dataframe +consolidation still require memory proportional to their size. + +These settings also apply to `runInitialConditions()`. Initial-condition runs +without a separate output archive save individual run data without creating a +dataframe writer or retention queue. + +Successful execution waits for the writer to finish all output. If the writer +exits unexpectedly or fails during consolidation, the controller raises +`RuntimeError` and cleans up its workers and queue manager. A keyboard interrupt +also stops those processes. After an aborted campaign, individual run files may +remain, but combined dataframes may be incomplete and must not be treated as a +successful archive. + `executeSimulations()` returns the indices of failed runs. Once execution is complete, the controller and retained data can be reloaded in the same script or a later analysis script: diff --git a/src/utilities/MonteCarlo/_UnitTests/test_controller.py b/src/utilities/MonteCarlo/_UnitTests/test_controller.py index 8bb508f9010..119dc80d57b 100644 --- a/src/utilities/MonteCarlo/_UnitTests/test_controller.py +++ b/src/utilities/MonteCarlo/_UnitTests/test_controller.py @@ -17,6 +17,11 @@ # import inspect +import json +import os +import _thread +from collections import Counter +import threading import multiprocessing as mp import signal @@ -111,6 +116,7 @@ def test_controller_drains_retained_data_before_return(tmp_path, monkeypatch, wo controller.setSimulationFunction(DummySimulation) controller.setExecutionFunction(execute_retention_simulation) controller.setExecutionCount(8) + controller.setRetentionQueueSize(8) controller.setThreadCount(workers) controller.setArchiveDir(str(tmp_path / "archive")) controller.setVerbose(False) @@ -120,11 +126,13 @@ def test_controller_drains_retained_data_before_return(tmp_path, monkeypatch, wo controller.addRetentionPolicy(policy) shutdown = controller_module.Controller._shutdown_data_writer - def release_and_shutdown(self): + def release_and_shutdown(self, abort=False): writer = self.dataWriter - writer.release.set() - shutdown(self) - assert writer.exitcode == 0 + if writer is not None: + writer.release.set() + shutdown(self, abort=abort) + if writer is not None and not abort: + assert writer.exitcode == 0 monkeypatch.setattr(controller_module, "DataWriter", DeferredDataWriter) monkeypatch.setattr( @@ -148,6 +156,258 @@ def release_and_shutdown(self): np.testing.assert_array_equal(combined[run].to_numpy(), expected) +class ExitingDataWriter(DataWriter): + """Exit abruptly after receiving the first retained result.""" + + def run(self): + """Simulate a writer process dying while producers are active.""" + self._queue.get() + os._exit(7) + + +class FailedConsolidationWriter(DataWriter): + """Fail after accepting all retained results and the end marker.""" + + def run(self): + """Simulate an output error during final consolidation.""" + while not self._queue.get()[2]: + pass + raise OSError("Simulated consolidation failure") + + +def retain_worker_pid(sim): + return {"worker_pid": np.array([[0, os.getpid()]])} + + +def fail_simulation(sim): + raise ValueError("Simulated run failure") + + +def make_retention_controller(tmp_path, workers, archive=True): + controller = controller_module.Controller() + controller.setSimulationFunction(DummySimulation) + controller.setExecutionFunction(execute_retention_simulation) + controller.setExecutionCount(9) + controller.setThreadCount(workers) + controller.setRetentionQueueSize(1) + controller.setVerbose(False) + controller.setShowProgressBar(False) + if archive: + controller.setArchiveDir(str(tmp_path / "archive")) + policy = controller_module.RetentionPolicy() + policy.addRetentionFunction(retain_simulation_data) + policy.addRetentionFunction(retain_worker_pid) + controller.addRetentionPolicy(policy) + return controller + + +def assert_controller_closed(controller, original_children): + assert controller.dataWriter is None + assert controller.dataOutQueue is None + assert controller.multiProcManager is None + assert {child.pid for child in mp.active_children()} == original_children + + +@pytest.mark.timeout(30) # [s] +def test_pool_reuse_recycling_and_repeated_execution(tmp_path): + """Reuse workers, bound their lifetimes, and finish a partial final batch.""" + controller = make_retention_controller(tmp_path, 2) + original_children = {child.pid for child in mp.active_children()} + for limit in (None, 2): + controller.setMaxTasksPerChild(limit) + assert controller.executeSimulations() == [] + assert_controller_closed(controller, original_children) + counts = Counter( + int(controller.getRetainedData(run)["custom"]["worker_pid"][0, 1]) + for run in range(9) + ) + if limit is None: + assert len(counts) <= 2 + assert max(counts.values()) > 1 + else: + assert max(counts.values()) <= limit + assert len(counts) >= 5 + combined = pd.read_pickle(tmp_path / "archive" / "samples.data") + for run in range(9): + expected = np.random.RandomState(run * 10).random_sample((32, 3)) + np.testing.assert_array_equal(combined[run].to_numpy(), expected) + + +@pytest.mark.timeout(30) # [s] +@pytest.mark.parametrize("initial_conditions", [False, True]) +@pytest.mark.parametrize("failure_stage", ["preparation", "submission"]) +def test_task_failures_report_indices_and_preserve_successes( + tmp_path, monkeypatch, initial_conditions, failure_stage +): + """Report preparation and submission errors while retaining later runs.""" + controller = make_retention_controller(tmp_path, 2) + controller.setExecutionCount(6) + original_children = {child.pid for child in mp.active_children()} + cases = list(range(6)) + archive = tmp_path / "archive" + failure_directory = archive + generator_name = "generateSims" + if initial_conditions: + cases = [2, 4, 6, 8, 10, 12] + failure_directory = tmp_path / "initial" + failure_directory.mkdir() + for run in cases: + (failure_directory / f"run{run}.json").write_text("{}") + controller.setICDir(str(failure_directory)) + controller.setICRunFlag(True) + generator_name = "generateICSims" + + rejected = [cases[0], cases[1], cases[3]] + failed = cases[:4] if failure_stage == "preparation" else rejected + generate = getattr(controller_module.Controller, generator_name) + + def generate_with_errors(self, case_list): + for sim in generate(self, case_list): + if sim.index in rejected: + if failure_stage == "preparation": + raise TypeError("Simulated task-preparation failure") + sim.configureFunction = lambda sim: None + yield sim + + monkeypatch.setattr( + controller_module.Controller, generator_name, generate_with_errors + ) + if initial_conditions: + result = controller.runInitialConditions(cases) + else: + result = controller.executeSimulations() + assert result == failed + assert json.loads((failure_directory / "failures.txt").read_text()) == failed + assert_controller_closed(controller, original_children) + combined = pd.read_pickle(archive / "samples.data") + successful = [run for run in cases if run not in failed] + assert list(combined.dropna(axis=1, how="all").columns) == [ + (run, component) for run in successful for component in range(3) + ] + for run in successful: + expected = np.random.RandomState(run * 10).random_sample((32, 3)) + np.testing.assert_array_equal(combined[run].to_numpy(), expected) + + +@pytest.mark.parametrize("method", ["setMaxTasksPerChild", "setRetentionQueueSize"]) +@pytest.mark.parametrize("invalid", [0, -1, True, 1.5, "8"]) +def test_worker_settings_reject_invalid_values(method, invalid): + """Reject settings that cannot provide a positive worker or queue limit.""" + with pytest.raises(ValueError, match="positive integer or None"): + getattr(controller_module.Controller(), method)(invalid) + + +@pytest.mark.timeout(30) # [s] +@pytest.mark.parametrize("workers", [1, 2]) +@pytest.mark.parametrize("writer", [ExitingDataWriter, FailedConsolidationWriter]) +def test_writer_failure_raises_and_cleans_up(tmp_path, monkeypatch, workers, writer): + """Report writer failure during production or shutdown without hanging.""" + controller = make_retention_controller(tmp_path, workers) + original_children = {child.pid for child in mp.active_children()} + monkeypatch.setattr(controller_module, "DataWriter", writer) + with pytest.raises(RuntimeError, match="Monte Carlo data writer"): + controller.executeSimulations() + assert_controller_closed(controller, original_children) + + +@pytest.mark.timeout(30) # [s] +@pytest.mark.parametrize("workers", [1, 2]) +def test_interrupt_with_full_retention_queue_cleans_up(tmp_path, monkeypatch, workers): + """Cancel blocked producers and the deferred writer on a keyboard interrupt.""" + controller = make_retention_controller(tmp_path, workers) + original_children = {child.pid for child in mp.active_children()} + monkeypatch.setattr(controller_module, "DataWriter", DeferredDataWriter) + stopped = threading.Event() + + def interrupt_when_full(): + interval = 0.01 # [s] + while not stopped.wait(interval): + queue = controller.dataOutQueue + if queue is not None and queue.full(): + _thread.interrupt_main() + return + + watcher = threading.Thread(target=interrupt_when_full) + watcher.start() + try: + with pytest.raises(KeyboardInterrupt): + controller.executeSimulations() + finally: + stopped.set() + watcher.join() + assert_controller_closed(controller, original_children) + + +@pytest.mark.timeout(30) # [s] +@pytest.mark.parametrize("workers", [1, 2]) +@pytest.mark.parametrize("archive", [False, True]) +def test_initial_conditions_with_and_without_writer(tmp_path, workers, archive): + """Run saved cases with a bounded writer or only their individual archives.""" + controller = make_retention_controller(tmp_path, workers, archive=archive) + original_children = {child.pid for child in mp.active_children()} + initial = tmp_path / "initial" + initial.mkdir() + cases = [1, 2, 3, 4, 5] + for run in cases: + (initial / f"run{run}.json").write_text("{}") + controller.setICDir(str(initial)) + controller.setICRunFlag(True) + controller.setMaxTasksPerChild(2) + assert controller.runInitialConditions(cases) == [] + assert_controller_closed(controller, original_children) + for run in cases: + expected = np.random.RandomState(run * 10).random_sample((32, 3)) + retained = controller.getRetainedData(run) + np.testing.assert_array_equal(retained["custom"]["samples"][:, 1:], expected) + if archive: + combined = pd.read_pickle(tmp_path / "archive" / "samples.data") + np.testing.assert_array_equal(combined[run].to_numpy(), expected) + controller.setExecutionFunction(fail_simulation) + assert controller.runInitialConditions([2, 5]) == [2, 5] + assert json.loads((initial / "failures.txt").read_text()) == [2, 5] + assert_controller_closed(controller, original_children) + + +@pytest.mark.timeout(30) # [s] +@pytest.mark.parametrize("cases", [[], [0], [0, 2]]) +def test_initial_conditions_accept_numpy_indices(tmp_path, cases): + """Execute NumPy case arrays, including case zero and an empty selection.""" + controller = make_retention_controller(tmp_path, 2) + original_children = {child.pid for child in mp.active_children()} + initial = tmp_path / "initial" + initial.mkdir() + for run in cases: + (initial / f"run{run}.json").write_text("{}") + controller.setICDir(str(initial)) + controller.setICRunFlag(True) + + assert controller.runInitialConditions(np.array(cases, dtype=np.int64)) == [] + assert_controller_closed(controller, original_children) + for run in cases: + expected = np.random.RandomState(run * 10).random_sample((32, 3)) + retained = controller.getRetainedData(run) + np.testing.assert_array_equal(retained["custom"]["samples"][:, 1:], expected) + + +def test_empty_campaign_and_legacy_controller_settings(tmp_path): + """Handle empty campaigns and controller archives predating the new settings.""" + controller = make_retention_controller(tmp_path, 2) + controller.setExecutionCount(0) + original_children = {child.pid for child in mp.active_children()} + assert controller.executeSimulations() == [] + assert_controller_closed(controller, original_children) + del controller.maxTasksPerChild + del controller.retentionQueueSize + controller.setExecutionCount(3) + assert controller.executeSimulations() == [] + loaded = controller_module.Controller.load(str(tmp_path / "archive")) + loaded.setExecutionCount(1) + assert loaded.executeSimulations() == [] + worker_pid = loaded.getRetainedData(0)["custom"]["worker_pid"][0, 1] + assert worker_pid != os.getpid() + assert_controller_closed(loaded, original_children) + + @pytest.mark.parametrize( "dispersion_class", [SingleVariableDispersion, VectorVariableDispersion], From a585c7ad3b8e50749680ff6241ed2c16d4093c61 Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Fri, 30 Oct 2026 10:51:52 -0600 Subject: [PATCH 10/13] [#1663] TMP: debug Windows CI failure --- .github/workflows/pull-request.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 82c18ea36fd..766cb2902a1 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -134,12 +134,23 @@ jobs: grep -F "BUILD_RUST_MODULES:BOOL=ON" dist3/CMakeCache.txt python -c "from Basilisk import getBuildInfo; assert getBuildInfo()['diagnostics']['tools']['corrosion']" - name: Pytest + id: windows-pytest shell: pwsh working-directory: src run: | pip install pytest-error-for-skips pytest-timeout pytest -n auto -m "not ciSkip" -rs --error-for-skips --timeout=300 --timeout-method=thread --durations=25 -v if(($LastExitCode -ne 0) -and ($LastExitCode -ne 5)) {exit 1} + - name: Diagnose Monte Carlo interruption failure + if: ${{ failure() && steps.windows-pytest.outcome == 'failure' }} + shell: pwsh + working-directory: src + timeout-minutes: 2 # [min] + run: | + # Run outside xdist with live output so a terminated worker cannot lose the stacks. + # Dump all thread stacks after 10 s and retain the test's 30 s timeout. + python -u -X faulthandler -m pytest -n 0 -s -vv -o faulthandler_timeout=10 --timeout=30 --timeout-method=thread "utilities/MonteCarlo/_UnitTests/test_controller.py::test_interrupt_with_full_retention_queue_cleans_up[1]" + if ($LastExitCode -ne 0) { exit $LastExitCode } - name: Build integration tests shell: pwsh working-directory: src From f8b6018907c6aca9db7a121040ed3e9d10d3ee21 Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Fri, 30 Oct 2026 16:05:39 -0600 Subject: [PATCH 11/13] [#1663] Make Monte Carlo interruption test deterministic --- .../MonteCarlo/_UnitTests/test_controller.py | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/src/utilities/MonteCarlo/_UnitTests/test_controller.py b/src/utilities/MonteCarlo/_UnitTests/test_controller.py index 119dc80d57b..60996e7e76a 100644 --- a/src/utilities/MonteCarlo/_UnitTests/test_controller.py +++ b/src/utilities/MonteCarlo/_UnitTests/test_controller.py @@ -19,9 +19,7 @@ import inspect import json import os -import _thread from collections import Counter -import threading import multiprocessing as mp import signal @@ -313,28 +311,22 @@ def test_writer_failure_raises_and_cleans_up(tmp_path, monkeypatch, workers, wri @pytest.mark.timeout(30) # [s] @pytest.mark.parametrize("workers", [1, 2]) def test_interrupt_with_full_retention_queue_cleans_up(tmp_path, monkeypatch, workers): - """Cancel blocked producers and the deferred writer on a keyboard interrupt.""" + """Verify interrupt cleanup with a full retention queue and paused writer.""" controller = make_retention_controller(tmp_path, workers) original_children = {child.pid for child in mp.active_children()} monkeypatch.setattr(controller_module, "DataWriter", DeferredDataWriter) - stopped = threading.Event() - - def interrupt_when_full(): - interval = 0.01 # [s] - while not stopped.wait(interval): - queue = controller.dataOutQueue - if queue is not None and queue.full(): - _thread.interrupt_main() - return - - watcher = threading.Thread(target=interrupt_when_full) - watcher.start() - try: - with pytest.raises(KeyboardInterrupt): - controller.executeSimulations() - finally: - stopped.set() - watcher.join() + check_writer = controller_module._check_writer + + def interrupt_when_full(writer): + check_writer(writer) + if controller.dataOutQueue.full(): + # Raise in the parent after IPC returns: Windows pipe reads can + # suppress an asynchronous KeyboardInterrupt when a read completes. + raise KeyboardInterrupt("Simulated interrupt with a full retention queue") + + monkeypatch.setattr(controller_module, "_check_writer", interrupt_when_full) + with pytest.raises(KeyboardInterrupt, match="Simulated interrupt with a full retention queue"): + controller.executeSimulations() assert_controller_closed(controller, original_children) From 5092951edd825eaccf19f5f98040eb9edf67c05f Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Fri, 30 Oct 2026 20:15:17 -0600 Subject: [PATCH 12/13] [#1663] Remove temporary Windows CI diagnostics --- .github/workflows/pull-request.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 766cb2902a1..82c18ea36fd 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -134,23 +134,12 @@ jobs: grep -F "BUILD_RUST_MODULES:BOOL=ON" dist3/CMakeCache.txt python -c "from Basilisk import getBuildInfo; assert getBuildInfo()['diagnostics']['tools']['corrosion']" - name: Pytest - id: windows-pytest shell: pwsh working-directory: src run: | pip install pytest-error-for-skips pytest-timeout pytest -n auto -m "not ciSkip" -rs --error-for-skips --timeout=300 --timeout-method=thread --durations=25 -v if(($LastExitCode -ne 0) -and ($LastExitCode -ne 5)) {exit 1} - - name: Diagnose Monte Carlo interruption failure - if: ${{ failure() && steps.windows-pytest.outcome == 'failure' }} - shell: pwsh - working-directory: src - timeout-minutes: 2 # [min] - run: | - # Run outside xdist with live output so a terminated worker cannot lose the stacks. - # Dump all thread stacks after 10 s and retain the test's 30 s timeout. - python -u -X faulthandler -m pytest -n 0 -s -vv -o faulthandler_timeout=10 --timeout=30 --timeout-method=thread "utilities/MonteCarlo/_UnitTests/test_controller.py::test_interrupt_with_full_retention_queue_cleans_up[1]" - if ($LastExitCode -ne 0) { exit $LastExitCode } - name: Build integration tests shell: pwsh working-directory: src From ea279a5fa17e25205aa0e4d0ee50eb97fe53ee16 Mon Sep 17 00:00:00 2001 From: Hanspeter Schaub Date: Sat, 31 Oct 2026 08:57:41 -0600 Subject: [PATCH 13/13] [#1663] Preserve NumPy 1.24 compatibility in Python 3.15-built modules Explicitly target the NumPy 1.24 C API in message bindings so newer build headers do not raise the minimum runtime requirement. Add a focused compile regression test and update the release note. Verify Python 3.15-built modules with Python 3.9 and NumPy 1.24.4: 46 compatibility tests passed; all 402 messaging tests passed on 3.15. --- .../python-315-support.rst | 2 +- .../_UnitTest/test_numpy_api_compatibility.py | 54 +++++++++++++++++++ src/architecture/messaging/newMessaging.ih | 3 ++ 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 src/architecture/messaging/_UnitTest/test_numpy_api_compatibility.py diff --git a/docs/source/Support/bskReleaseNotesSnippets/python-315-support.rst b/docs/source/Support/bskReleaseNotesSnippets/python-315-support.rst index 108ca0407e3..fd99bc10497 100644 --- a/docs/source/Support/bskReleaseNotesSnippets/python-315-support.rst +++ b/docs/source/Support/bskReleaseNotesSnippets/python-315-support.rst @@ -1 +1 @@ -- Added standard CPython 3.15 support while retaining Python 3.9 through 3.14. Python 3.15 uses NumPy 2.5.2 or newer and Numba 0.68 for examples and development; older interpreters retain their dependency ranges. Updated source-build CI and wheel tooling, and moved the existing wheel installation checks to Python 3.15. +- Added standard CPython 3.15 support while retaining Python 3.9 through 3.14. Python 3.15 uses NumPy 2.5.2 or newer and Numba 0.68 for examples and development; older interpreters retain their dependency ranges. Native message bindings explicitly target the NumPy 1.24 C API so Python 3.15-built wheels remain compatible with NumPy 1.24.4 on older interpreters. Updated source-build CI and wheel tooling, and moved the existing wheel installation checks to Python 3.15. diff --git a/src/architecture/messaging/_UnitTest/test_numpy_api_compatibility.py b/src/architecture/messaging/_UnitTest/test_numpy_api_compatibility.py new file mode 100644 index 00000000000..5aa2e3ab5b5 --- /dev/null +++ b/src/architecture/messaging/_UnitTest/test_numpy_api_compatibility.py @@ -0,0 +1,54 @@ +# ISC License +# +# Copyright (c) 2026, Autonomous Vehicle Systems Lab, University of Colorado at Boulder +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Check the NumPy C API required by the shared message bindings.""" + +from pathlib import Path +import re +import sysconfig + +import numpy as np +from setuptools._distutils.ccompiler import new_compiler +from setuptools._distutils.sysconfig import customize_compiler + + +def test_message_bindings_support_numpy124(tmp_path): + """Compile the real NumPy include block against the installed build headers. + + Newer headers must not silently raise the runtime requirement above NumPy + 1.24, even when the runtime executing this test is newer. + """ + template = (Path(__file__).parents[1] / "newMessaging.ih").read_text(encoding="utf-8") + numpy_blocks = [ + block for block in re.findall(r"%\{(.*?)%\}", template, flags=re.DOTALL) + if "#include " in block + ] + assert len(numpy_blocks) == 1 + source = tmp_path / "numpy_api_probe.cpp" + source.write_text( + numpy_blocks[0] + + '\nstatic_assert(NPY_FEATURE_VERSION <= NPY_1_24_API_VERSION, ' + '"Message bindings must support the NumPy 1.24 runtime");\n', + encoding="utf-8", + ) + compiler = new_compiler() + customize_compiler(compiler) + compiler.compile( + [str(source)], + output_dir=str(tmp_path), + include_dirs=[sysconfig.get_path("include"), np.get_include()], + macros=[("Py_LIMITED_API", "0x03090000")], + ) diff --git a/src/architecture/messaging/newMessaging.ih b/src/architecture/messaging/newMessaging.ih index a4dbb1966f0..cc81c9e97ea 100644 --- a/src/architecture/messaging/newMessaging.ih +++ b/src/architecture/messaging/newMessaging.ih @@ -28,6 +28,9 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. %} %{ +// Keep abi3 wheels compatible with the oldest supported NumPy runtime, +// including when the build uses newer headers with a higher default API target. +#define NPY_TARGET_VERSION NPY_1_24_API_VERSION #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include %}