From 1265cce1f0c4731df53140cbaed118171cf4c40b Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 16 Oct 2026 14:15:59 +0200 Subject: [PATCH 1/4] doc: link to the ewoks execute reference within the docs Co-Authored-By: Claude Opus 5.5 --- doc/explanations/task_input_priority.rst | 2 +- doc/reference/cli.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/explanations/task_input_priority.rst b/doc/explanations/task_input_priority.rst index f003e44..d08274e 100644 --- a/doc/explanations/task_input_priority.rst +++ b/doc/explanations/task_input_priority.rst @@ -7,7 +7,7 @@ Details A :term:`node ` in a :term:`workflow` can get inputs from three different sources: 1. Via the ``data_mapping`` :term:`link ` attribute of an incoming :term:`link ` (see `Link attributes `_) -2. Via the ``parameters`` CLI argument (or ``inputs`` for Python) when executing/submitting the :term:`workflow` (see `ewoks execute reference `_) +2. Via the ``parameters`` CLI argument (or ``inputs`` for Python) when executing/submitting the :term:`workflow` (see :ref:`ewoks execute reference `) 3. Via the ``default_inputs`` :term:`node ` attribute of the :term:`node ` itself (see `Node attributes `_) If the same input is specified by these different sources, :term:`Ewoks` applies the following priorities: diff --git a/doc/reference/cli.rst b/doc/reference/cli.rst index 4add2cd..a68fe21 100644 --- a/doc/reference/cli.rst +++ b/doc/reference/cli.rst @@ -63,6 +63,8 @@ ewoks convert **ewoks convert** can also be used to store ``inputs`` inside the destination :term:`workflow`. +.. _cli_execute: + ewoks execute ------------- From f23766173616a27ccf19b84f75761620762d00e2 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 16 Oct 2026 14:15:59 +0200 Subject: [PATCH 2/4] ewoks convert: record the ewoks version and the engine in the workflow requirements Co-Authored-By: Claude Opus 5.5 --- doc/howtoguides/requirements.rst | 2 ++ doc/tutorials/install.rst | 4 ++++ src/ewoks/_engines.py | 14 ++++++++++++++ src/ewoks/_requirements/__init__.py | 7 ++++++- src/ewoks/_requirements/utils/base_manager.py | 11 ++++++++++- src/ewoks/_requirements/utils/metadata/models.py | 16 ++++++++++++++++ src/ewoks/bindings.py | 4 ++++ src/ewoks/tests/requirements/test_managers.py | 2 ++ src/ewoks/tests/test_convert_cli.py | 12 ++++++++++++ 9 files changed, 70 insertions(+), 2 deletions(-) diff --git a/doc/howtoguides/requirements.rst b/doc/howtoguides/requirements.rst index 74867f6..e2caac2 100644 --- a/doc/howtoguides/requirements.rst +++ b/doc/howtoguides/requirements.rst @@ -13,6 +13,8 @@ What is stored ``ewoks convert`` and ``ewoks execute -o convert_destination=...`` store +* ``ewoks``: the version of :term:`ewoks` that generated the requirements and the engine + that saved the :term:`workflow`. * ``python`` and ``system``: the python interpreter and the operating system. * ``distributions``: every installed python package with its version and, when it was not installed from the python package index, the git commit or the archive it came from. Any diff --git a/doc/tutorials/install.rst b/doc/tutorials/install.rst index 96dd6f7..b1455ad 100644 --- a/doc/tutorials/install.rst +++ b/doc/tutorials/install.rst @@ -40,6 +40,7 @@ Store the requirements .. code-block:: json { + "ewoks": {"version": "7.0.0", "engine": {"name": "ewokscore", "version": "5.1.0"}}, "python": {"version": "3.12.11", "implementation": "CPython", "...": "..."}, "system": {"system": "Linux", "machine": "x86_64", "...": "..."}, "distributions": [ @@ -53,6 +54,9 @@ Store the requirements } } +* ``ewoks`` is the version of :term:`ewoks` that generated the requirements and the engine that + saved the :term:`workflow`: ``ewokscore`` for a ``.json`` or ``.yaml`` file and + ``ewoksorange`` for an ``.ows`` file. * ``distributions`` are the installed python packages. Any :term:`package manager` can recreate the environment from this list. * ``manager`` is the :term:`package manager` that generated the requirements together with diff --git a/src/ewoks/_engines.py b/src/ewoks/_engines.py index 83e9c33..951ca71 100644 --- a/src/ewoks/_engines.py +++ b/src/ewoks/_engines.py @@ -1,3 +1,4 @@ +import importlib.metadata import logging from functools import lru_cache from typing import Any @@ -86,6 +87,19 @@ def get_serialization_engine( return engine, core_representation +def get_engine_distribution(engine: WorkflowEngine) -> Optional[Tuple[str, str]]: + """Name and version of the distribution that provides the engine.""" + module = type(engine).__module__ + for distribution in importlib.metadata.distributions(): + for entry_point in distribution.entry_points: + if ( + entry_point.group == "ewoks.engines" + and entry_point.value.split(":")[0] == module + ): + return distribution.metadata["Name"], distribution.version + return None + + _EngineClassLoaderGenerator = Generator[ Tuple[str, Callable[[], Type[WorkflowEngine]]], None, None ] diff --git a/src/ewoks/_requirements/__init__.py b/src/ewoks/_requirements/__init__.py index b58ff9c..475e288 100644 --- a/src/ewoks/_requirements/__init__.py +++ b/src/ewoks/_requirements/__init__.py @@ -21,6 +21,7 @@ from .utils.detect import get_manager from .utils.environment import Environment from .utils.metadata import last_resort +from .utils.metadata.models import EngineInfo logger = logging.getLogger(__file__) @@ -49,12 +50,16 @@ def add_requirements( graph: TaskGraph, manager_name: Optional[str] = None, manager_command: Tuple[str, ...] = tuple(), + engine: Optional[Tuple[str, str]] = None, ) -> str: """Add requirements to a workflow definition in-place. Returns the name of the - package manager that generated them. + package manager that generated them. `engine` is the name and version of the + distribution of the engine that saves the workflow. """ manager = get_manager(manager_name=manager_name, manager_command=manager_command) requirements = manager.gather_requirements() + if requirements.ewoks and engine: + requirements.ewoks.engine = EngineInfo(name=engine[0], version=engine[1]) graph.graph.graph["requirements"] = requirements.model_dump() return manager.NAME diff --git a/src/ewoks/_requirements/utils/base_manager.py b/src/ewoks/_requirements/utils/base_manager.py index 2ae0ef6..b87575f 100644 --- a/src/ewoks/_requirements/utils/base_manager.py +++ b/src/ewoks/_requirements/utils/base_manager.py @@ -1,3 +1,4 @@ +import importlib.metadata import logging from abc import abstractmethod from pathlib import Path @@ -45,6 +46,13 @@ class BaseManagerInfo(models.BaseModel): class BaseRequirements(models.BaseModel): + ewoks: Optional[models.EwoksInfo] = Field( + default=None, + description=( + "Ewoks that generated the requirements. Not set for requirements that " + "ewoks did not generate." + ), + ) system: models.SystemInfo = Field( description="Operating system on which the requirements were generated." ) @@ -170,7 +178,8 @@ def gather_requirements(self) -> BaseRequirements: ) manager = dict(name=self.NAME, version=manager_version, files=files) - return self.REQUIREMENTS_MODEL(manager=manager, **metadata) + ewoks = dict(version=importlib.metadata.version("ewoks")) + return self.REQUIREMENTS_MODEL(ewoks=ewoks, manager=manager, **metadata) def environment(self, location: Union[str, Path]) -> Environment: """Environment the manager creates at this location.""" diff --git a/src/ewoks/_requirements/utils/metadata/models.py b/src/ewoks/_requirements/utils/metadata/models.py index 72bfd66..8c333fe 100644 --- a/src/ewoks/_requirements/utils/metadata/models.py +++ b/src/ewoks/_requirements/utils/metadata/models.py @@ -5,6 +5,22 @@ from pydantic import Field +class EngineInfo(BaseModel): + name: str = Field(description="Distribution name.", examples=["ewoksorange"]) + version: str = Field(description="Distribution version.", examples=["2.2.0"]) + + +class EwoksInfo(BaseModel): + version: str = Field(description="Ewoks version.", examples=["7.0.0"]) + engine: Optional[EngineInfo] = Field( + default=None, + description=( + "Distribution of the engine that saved the workflow, which is needed " + "to load it." + ), + ) + + class SystemInfo(BaseModel): system: str = Field(description="Operating system name.", examples=["Linux"]) release: str = Field( diff --git a/src/ewoks/bindings.py b/src/ewoks/bindings.py index 91d9f2f..ecfa770 100644 --- a/src/ewoks/bindings.py +++ b/src/ewoks/bindings.py @@ -222,10 +222,14 @@ def convert_graph( elif isinstance(package_manager_command, str): package_manager_command = _split_command(package_manager_command) + engine, _ = _engines.get_serialization_engine( + destination, representation=save_options.get("representation") + ) manager_name = _requirements.add_requirements( graph, manager_name=package_manager_name, manager_command=package_manager_command, + engine=_engines.get_engine_distribution(engine), ) logger.info("Requirements generated with the %r package manager", manager_name) else: diff --git a/src/ewoks/tests/requirements/test_managers.py b/src/ewoks/tests/requirements/test_managers.py index 402935c..07d55b5 100644 --- a/src/ewoks/tests/requirements/test_managers.py +++ b/src/ewoks/tests/requirements/test_managers.py @@ -1,5 +1,6 @@ """Tests that apply to all package managers.""" +import importlib.metadata from typing import Iterator import pytest @@ -83,6 +84,7 @@ def test_gather_requirements(manager, manager_case, monkeypatch): requirements = manager.gather_requirements() assert isinstance(requirements, manager_case.MANAGER_CLS.REQUIREMENTS_MODEL) + assert requirements.ewoks.version == importlib.metadata.version("ewoks") assert requirements.manager.name == manager_case.NAME assert requirements.manager.version assert requirements.python.version == PYTHON_VERSION diff --git a/src/ewoks/tests/test_convert_cli.py b/src/ewoks/tests/test_convert_cli.py index dc814d1..911e331 100644 --- a/src/ewoks/tests/test_convert_cli.py +++ b/src/ewoks/tests/test_convert_cli.py @@ -1,3 +1,5 @@ +import importlib.metadata +import json import os import sys @@ -34,6 +36,10 @@ def test_convert_to_json(graph_name, tmpdir): graph = load_graph(destination) assert_in_graph_requirements(graph, "ewokscore") + assert graph.graph.graph["requirements"]["ewoks"]["engine"] == { + "name": "ewokscore", + "version": importlib.metadata.version("ewokscore"), + } def test_convert_with_all_inputs(tmpdir): @@ -133,6 +139,12 @@ def test_convert_to_ows(graph_name, tmpdir): tree = ElementTree.parse(destination) root = tree.getroot() + graph_attrs = json.loads(root.find("./ewoks_graph_attrs").text) + assert graph_attrs["requirements"]["ewoks"]["engine"] == { + "name": "ewoksorange", + "version": importlib.metadata.version("ewoksorange"), + } + for node in root.findall("./nodes/node"): try: assert issubclass(import_qualname(node.get("qualified_name")), OWBaseWidget) From 2dbce90feda0673fda4679dd9d7b19fa7826a778 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 16 Oct 2026 14:15:59 +0200 Subject: [PATCH 3/4] ewoks install: print execute and remove commands that work from any terminal Co-Authored-By: Claude Opus 5.5 --- doc/tutorials/install.rst | 8 +-- src/ewoks/__main__.py | 65 +++++++++++++++---- .../tests/requirements/test_install_cli.py | 14 ++++ 3 files changed, 70 insertions(+), 17 deletions(-) diff --git a/doc/tutorials/install.rst b/doc/tutorials/install.rst index b1455ad..d05b5b1 100644 --- a/doc/tutorials/install.rst +++ b/doc/tutorials/install.rst @@ -67,14 +67,14 @@ Create the environment ``ewoks install`` creates a python environment for the :term:`workflow`. It prints the python interpreter of the environment it created, the command to execute the :term:`workflow` in it -and the command to remove it again +and the command to remove it again. These commands work from any terminal and directory .. code-block:: text Installed requirements for demo.json - Python : ewoks_envs/demo/bin/python - Execute: ewoks execute --env ewoks_envs/demo demo.json - Remove : rm -rf ewoks_envs/demo + Python : /home/user/ewoks_envs/demo/bin/python + Execute: /home/user/ewoks_envs/demo/bin/python -m ewoks execute /home/user/demo.json + Remove : rm -rf /home/user/ewoks_envs/demo Without ``--yes`` you are asked to confirm after the packages have been listed. The walk-throughs use ``--env-root`` to create the environment in the working directory. Without diff --git a/src/ewoks/__main__.py b/src/ewoks/__main__.py index 1383a56..b136400 100644 --- a/src/ewoks/__main__.py +++ b/src/ewoks/__main__.py @@ -1,3 +1,6 @@ +import os +import re +import shlex import subprocess import sys import traceback @@ -207,27 +210,63 @@ def command_install( print(f"Install aborted for {workflow}") else: print(f"Installed requirements for {workflow}") + print(f" Python : {environment.python}") + if environment.distribution_version("ewoks"): + execute = _shell_command( + [ + str(environment.python), + "-m", + "ewoks", + "execute", + _absolute_workflow(workflow), + ] + ) + print(f" Execute: {execute}") + else: + print( + " Execute: the environment has no ewoks to execute the " + "workflow with (use '--with-ewoks')" + ) if not cli_args.in_place: - location = environment.location - print(f" Python : {environment.python}") - if environment.distribution_version("ewoks"): - print(f" Execute: ewoks execute --env {location} {workflow}") - else: - print( - " Execute: the environment has no ewoks to execute the " - "workflow with (use '--with-ewoks')" - ) - print(f" Remove : {_remove_command(location)}") + print(f" Remove : {_remove_command(environment.location)}") if shell: return 0 return None def _remove_command(location: Path) -> str: - """Shell command that removes a python environment.""" + """Shell command that removes a python environment: PowerShell on Windows.""" + if sys.platform == "win32": + return f"Remove-Item -Recurse -Force {_powershell_quote(str(location))}" + return f"rm -rf {shlex.quote(str(location))}" + + +def _shell_command(arguments: List[str]) -> str: + """Command line for the shell of the platform: PowerShell on Windows.""" if sys.platform == "win32": - return f'rmdir /s /q "{location}"' - return f"rm -rf {location}" + command = " ".join(_powershell_quote(argument) for argument in arguments) + if command.startswith("'"): + # A quoted string is not a command in PowerShell + return f"& {command}" + return command + return shlex.join(arguments) + + +_POWERSHELL_SAFE = re.compile(r"[\w./\\:=+-]+") + + +def _powershell_quote(argument: str) -> str: + """Argument quoted for PowerShell when needed.""" + if _POWERSHELL_SAFE.fullmatch(argument): + return argument + return "'" + argument.replace("'", "''") + "'" + + +def _absolute_workflow(workflow: str) -> str: + """Workflow that does not depend on the working directory.""" + if os.path.isfile(workflow): + return os.path.abspath(workflow) + return workflow def command_in_environment( diff --git a/src/ewoks/tests/requirements/test_install_cli.py b/src/ewoks/tests/requirements/test_install_cli.py index a0b4342..3305b46 100644 --- a/src/ewoks/tests/requirements/test_install_cli.py +++ b/src/ewoks/tests/requirements/test_install_cli.py @@ -12,6 +12,8 @@ import pytest +from ...__main__ import _remove_command +from ...__main__ import _shell_command from ..._requirements import environment_name from ..._requirements.utils.environment import Environment from ...bindings import load_graph @@ -27,6 +29,18 @@ environment created for them.""" +def test_printed_commands_with_spaces(tmp_path): + """The printed commands work in the shell of the platform.""" + location = tmp_path / "ewoks envs" + python = str(location / "python") + if sys.platform == "win32": + assert _shell_command([python, "-m", "ewoks"]) == f"& '{python}' -m ewoks" + assert _remove_command(location) == f"Remove-Item -Recurse -Force '{location}'" + else: + assert _shell_command([python, "-m", "ewoks"]) == f"'{python}' -m ewoks" + assert _remove_command(location) == f"rm -rf '{location}'" + + def test_install_with_files(fast_manager_case, env_root): files = fast_manager_case.native_files(DISTRIBUTIONS, PYTHON_VERSION) if not files: From def17235d7f44eb2ad43c79c0b4ac20e70d7ab97 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 16 Oct 2026 14:15:59 +0200 Subject: [PATCH 4/4] Add the ewoks-install scripts that install a workflow environment without ewoks Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test.yml | 19 + CHANGELOG.md | 1 + doc/conf.py | 49 ++ doc/howtoguides/requirements.rst | 3 +- doc/tutorials/install.rst | 104 +++ doc/tutorials/install/conda.rst | 49 ++ doc/tutorials/install/pip_venv.rst | 49 ++ doc/tutorials/install/pixi.rst | 49 ++ doc/tutorials/install/poetry.rst | 49 ++ doc/tutorials/install/uv.rst | 49 ++ pyproject.toml | 1 + src/ewoks/__main__.py | 64 +- src/ewoks/_bootstrap/ewoks-install.ps1 | 716 ++++++++++++++++++ src/ewoks/_bootstrap/ewoks-install.sh | 577 ++++++++++++++ src/ewoks/tests/conftest.py | 26 + src/ewoks/tests/requirements/conftest.py | 3 + .../tests/requirements/test_bootstrap.py | 422 +++++++++++ .../tests/requirements/test_install_cli.py | 28 + 18 files changed, 2235 insertions(+), 23 deletions(-) create mode 100644 src/ewoks/_bootstrap/ewoks-install.ps1 create mode 100644 src/ewoks/_bootstrap/ewoks-install.sh create mode 100644 src/ewoks/tests/requirements/test_bootstrap.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 798e431..e42f7b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,6 +79,25 @@ jobs: enable-coverage: "false" extra-pytest-warnings: "" + # Run the tests that install package managers, on runners without them + tests-package-manager-install: + needs: build + uses: ewoks-kit/.github/.github/workflows/python-tests.yml@main + with: + os: ${{ matrix.os }} + python-version: "3.12" + pytest-args: >- + -v -ra -W error -m package_manager_install + -W ignore::DeprecationWarning:jupyter_client.session + -W ignore::DeprecationWarning:orangecanvas.utils.localization + -W ignore::DeprecationWarning:orangecanvas.utils.localization.si + -W ignore::DeprecationWarning:matplotlib._fontconfig_pattern + -W ignore::DeprecationWarning:matplotlib._mathtext + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + # Run linter / checks checks: uses: ewoks-kit/.github/.github/workflows/python-check.yml@main diff --git a/CHANGELOG.md b/CHANGELOG.md index d98152d..f01ee7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - add `--in-place`, `--python-version` and `--with-ewoks` arguments. - `ewoks convert`: add `--package-manager-name` and `--package-manager-command` arguments. - `ewoks execute`: add `--env` argument. +- Add `ewoks-install.sh` and `ewoks-install.ps1` scripts to install the environment of a workflow without ewoks. - Add `ewoks lint` CLI command to check if a workflow is conform to the Ewoks specification. ### Changed diff --git a/doc/conf.py b/doc/conf.py index fcef75b..3e47c28 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -3,6 +3,13 @@ import importlib.metadata +import os +from typing import Optional + +from ewokscore.tests.examples.graphs import get_graph +from sphinx.application import Sphinx + +from ewoks.bindings import convert_graph release = importlib.metadata.version("ewoks") @@ -76,3 +83,45 @@ "footer_start": ["copyright"], "footer_end": ["footer_end"], } + + +# The ewoks-install scripts +html_extra_path = ["../src/ewoks/_bootstrap"] + +# Root of the documentation version that is built: on Read the Docs, for example +# .../en/stable, or on GitLab Pages +_DOCS_URL = ( + os.environ.get("READTHEDOCS_CANONICAL_URL") + or os.environ.get("CI_PAGES_URL") + or "https://ewoks.readthedocs.io/en/latest" +).rstrip("/") + + +def _substitute_docs_url(app, docname, source): + # Substitutions are not supported in code blocks + source[0] = source[0].replace("|docs_url|", _DOCS_URL) + + +def _save_demo_workflow(app: Sphinx, exception: Optional[Exception]) -> None: + """Save the demo workflow next to the ewoks-install scripts, with only ewoks and + its engine as requirements instead of all the packages of the documentation + build.""" + if exception is not None or app.builder.format != "html": + return + graph, _ = get_graph("demo") + workflow = convert_graph(graph, None) + requirements = workflow["graph"]["requirements"] + names = ["ewoks", requirements["ewoks"]["engine"]["name"]] + requirements["distributions"] = [ + {"name": name, "version": importlib.metadata.version(name)} for name in names + ] + requirements["manager"]["files"] = {} + filename = os.path.join(app.outdir, "demo.json") + convert_graph( + workflow, filename, save_options={"indent": 2}, save_requirements=False + ) + + +def setup(app): + app.connect("source-read", _substitute_docs_url) + app.connect("build-finished", _save_demo_workflow) diff --git a/doc/howtoguides/requirements.rst b/doc/howtoguides/requirements.rst index e2caac2..58c7c7f 100644 --- a/doc/howtoguides/requirements.rst +++ b/doc/howtoguides/requirements.rst @@ -14,7 +14,8 @@ What is stored ``ewoks convert`` and ``ewoks execute -o convert_destination=...`` store * ``ewoks``: the version of :term:`ewoks` that generated the requirements and the engine - that saved the :term:`workflow`. + that saved the :term:`workflow`. The + :ref:`ewoks-install script ` installs both to run ``ewoks install``. * ``python`` and ``system``: the python interpreter and the operating system. * ``distributions``: every installed python package with its version and, when it was not installed from the python package index, the git commit or the archive it came from. Any diff --git a/doc/tutorials/install.rst b/doc/tutorials/install.rst index d05b5b1..73c0263 100644 --- a/doc/tutorials/install.rst +++ b/doc/tutorials/install.rst @@ -80,6 +80,110 @@ Without ``--yes`` you are asked to confirm after the packages have been listed. walk-throughs use ``--env-root`` to create the environment in the working directory. Without it the environment is created where the :term:`package manager` creates named environments. +.. _install_bootstrap: + +Create the environment without ewoks +++++++++++++++++++++++++++++++++++++ + +``ewoks install`` needs a python environment with :term:`ewoks`. The ``ewoks-install`` script +only needs a :term:`package manager`: it installs the version of :term:`ewoks` that generated +the requirements, and the engine that saved the :term:`workflow`, in a bootstrap environment +and runs ``ewoks install`` from there. All +arguments are passed to ``ewoks install``, except for the options of the script itself + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + +Without a :term:`workflow` at hand, try it with the demo :term:`workflow` of this +documentation, which only requires :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + irm |docs_url|/demo.json -OutFile demo.json + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + +Alternatively, download the script first, for example to read it before running it + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf -o ewoks-install.sh |docs_url|/ewoks-install.sh + sh ewoks-install.sh demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf -o ewoks-install.sh |docs_url|/ewoks-install.sh + sh ewoks-install.sh demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + irm |docs_url|/ewoks-install.ps1 -OutFile ewoks-install.ps1 + powershell -ExecutionPolicy Bypass -File ewoks-install.ps1 demo.json + +* ``--package-manager-name`` and ``--package-manager-command`` select the + :term:`package manager` that creates the bootstrap environment. Without them it is the first + one that is available of uv, pixi, conda, poetry and pip-venv. +* ``--print-command`` prints the ``ewoks install`` command instead of running it. +* ``--bootstrap-requirement`` installs a package in the bootstrap environment instead of the + version of :term:`ewoks` and the engine of the requirements, for example + ``--bootstrap-requirement "ewoks>=7"``. Repeat it to install an engine as well, for example + ``--bootstrap-requirement ewoks --bootstrap-requirement ewoksorange``. The latest version of + :term:`ewoks` is installed when the requirements do not provide one. +* ``--bootstrap-dir`` is the directory of the bootstrap environments. +* ``--install-package-manager`` installs the :term:`package manager` of + ``--package-manager-name`` in the bootstrap directory when it is not installed. Conda + cannot be installed in a directory with spaces. + +The script keeps the bootstrap environment in ``~/.ewoks/bootstrap`` and uses it again when +it installs another :term:`workflow` whose requirements were generated by the same version of +:term:`ewoks`. The bootstrap environment only runs ``ewoks install``: +it is not the environment of the :term:`workflow`. Before the environment of the +:term:`workflow`, the script prints the python interpreter of the bootstrap environment, the +command to run ``ewoks install`` again and the command to remove the bootstrap environment. When the script fails to create the bootstrap +environment, it prints how to install without it. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/conda.rst b/doc/tutorials/install/conda.rst index ee665bf..2325f3c 100644 --- a/doc/tutorials/install/conda.rst +++ b/doc/tutorials/install/conda.rst @@ -89,6 +89,29 @@ packages of the python package index. Re-producer side ---------------- +Without the producer side, download the demo :term:`workflow` of this documentation, which +only requires :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + irm |docs_url|/demo.json -OutFile demo.json + Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` .. tabs:: @@ -124,6 +147,32 @@ The environment of the :term:`workflow` is a conda environment in ``ewoks_envs/d ``--env-root`` it is created in the first environment directory of conda, where ``conda activate demo`` finds it. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + +It selects the :term:`package manager` itself and prints the command to execute the +:term:`workflow` in the environment it creates. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/pip_venv.rst b/doc/tutorials/install/pip_venv.rst index 2ee3857..63a4b72 100644 --- a/doc/tutorials/install/pip_venv.rst +++ b/doc/tutorials/install/pip_venv.rst @@ -50,6 +50,29 @@ Install any package that provides :term:`tasks ` instead of Re-producer side ---------------- +Without the producer side, download the demo :term:`workflow` of this documentation, which +only requires :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + irm |docs_url|/demo.json -OutFile demo.json + Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` .. tabs:: @@ -83,6 +106,32 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a virtual environment in ``ewoks_envs/demo``. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + +It selects the :term:`package manager` itself and prints the command to execute the +:term:`workflow` in the environment it creates. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/pixi.rst b/doc/tutorials/install/pixi.rst index 3a420dd..6b21ba2 100644 --- a/doc/tutorials/install/pixi.rst +++ b/doc/tutorials/install/pixi.rst @@ -85,6 +85,29 @@ workflow paths are relative to it and not to the workspace. Re-producer side ---------------- +Without the producer side, download the demo :term:`workflow` of this documentation, which +only requires :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + irm |docs_url|/demo.json -OutFile demo.json + Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` .. tabs:: @@ -119,6 +142,32 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a pixi workspace in ``ewoks_envs/demo`` with the environment in its ``.pixi/envs/default`` directory. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + +It selects the :term:`package manager` itself and prints the command to execute the +:term:`workflow` in the environment it creates. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/poetry.rst b/doc/tutorials/install/poetry.rst index f4360bd..fe5028a 100644 --- a/doc/tutorials/install/poetry.rst +++ b/doc/tutorials/install/poetry.rst @@ -75,6 +75,29 @@ relative to the project directory, so the workflow paths are absolute. Re-producer side ---------------- +Without the producer side, download the demo :term:`workflow` of this documentation, which +only requires :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + irm |docs_url|/demo.json -OutFile demo.json + Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` .. tabs:: @@ -106,6 +129,32 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a poetry project in ``ewoks_envs/demo`` with the virtual environment in its ``.venv`` directory. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + +It selects the :term:`package manager` itself and prints the command to execute the +:term:`workflow` in the environment it creates. + Execute the workflow -------------------- diff --git a/doc/tutorials/install/uv.rst b/doc/tutorials/install/uv.rst index 11db413..08c5599 100644 --- a/doc/tutorials/install/uv.rst +++ b/doc/tutorials/install/uv.rst @@ -79,6 +79,29 @@ Install any package that provides :term:`tasks ` instead of Re-producer side ---------------- +Without the producer side, download the demo :term:`workflow` of this documentation, which +only requires :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSfO |docs_url|/demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + irm |docs_url|/demo.json -OutFile demo.json + Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` .. tabs:: @@ -113,6 +136,32 @@ Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` The environment of the :term:`workflow` is a uv project in ``ewoks_envs/demo`` with the virtual environment in its ``.venv`` directory. +Alternatively, the :ref:`ewoks-install script ` recreates the +environment without an environment with :term:`ewoks` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + curl -LsSf |docs_url|/ewoks-install.sh | sh -s -- demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + & ([scriptblock]::Create((irm |docs_url|/ewoks-install.ps1))) demo.json + +It selects the :term:`package manager` itself and prints the command to execute the +:term:`workflow` in the environment it creates. + Execute the workflow -------------------- diff --git a/pyproject.toml b/pyproject.toml index cebf15e..6b60671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ ewoks = "ewoks.__main__:main" [tool.setuptools.package-data] "ewoks.tests.notebooks" = ["*.ipynb"] +"ewoks._bootstrap" = ["*.sh", "*.ps1"] [tool.ruff.lint] select = [ diff --git a/src/ewoks/__main__.py b/src/ewoks/__main__.py index b136400..be0dfa0 100644 --- a/src/ewoks/__main__.py +++ b/src/ewoks/__main__.py @@ -185,10 +185,16 @@ def command_convert( return None +# Environment variable with text that is printed before the summary, like the +# ewoks-install scripts do for their bootstrap environment +_SUMMARY_HEADER_VARIABLE = "EWOKS_INSTALL_SUMMARY_HEADER" + + def command_install( cli_args: Namespace, shell: bool = False ) -> Optional[Literal[0, 1]]: cli_install_utils.parse_install_arguments(cli_args, shell=shell) + header = os.environ.get(_SUMMARY_HEADER_VARIABLE) for workflow, source in zip(cli_args.workflows, cli_args.graphs): try: environment = install_graph( @@ -205,35 +211,49 @@ def command_install( ) except (CalledProcessError, RuntimeError, ValueError): traceback.print_exc() - print(f"Install failed for {workflow}") + summary = [f"Install failed for {workflow}"] except AbortException: - print(f"Install aborted for {workflow}") + summary = [f"Install aborted for {workflow}"] else: - print(f"Installed requirements for {workflow}") - print(f" Python : {environment.python}") - if environment.distribution_version("ewoks"): - execute = _shell_command( - [ - str(environment.python), - "-m", - "ewoks", - "execute", - _absolute_workflow(workflow), - ] - ) - print(f" Execute: {execute}") - else: - print( - " Execute: the environment has no ewoks to execute the " - "workflow with (use '--with-ewoks')" - ) - if not cli_args.in_place: - print(f" Remove : {_remove_command(environment.location)}") + summary = _install_summary(workflow, environment, cli_args.in_place) + if header: + print(header) + header = None + print("\n".join(summary)) if shell: return 0 return None +def _install_summary( + workflow: str, environment: Environment, in_place: bool +) -> List[str]: + """Lines that describe the python environment in which a workflow is installed.""" + summary = [ + f"Installed requirements for {workflow}", + f" Python : {environment.python}", + ] + if environment.distribution_version("ewoks"): + execute = _shell_command( + [ + str(environment.python), + "-m", + "ewoks", + "execute", + _absolute_workflow(workflow), + ] + ) + summary.append(f" Execute: {execute}") + else: + summary.append( + " Execute: the environment has no ewoks to execute the workflow with " + "(use '--with-ewoks')" + ) + if not in_place: + summary.append(f" Remove : {_remove_command(environment.location)}") + return summary + + def _remove_command(location: Path) -> str: """Shell command that removes a python environment: PowerShell on Windows.""" if sys.platform == "win32": diff --git a/src/ewoks/_bootstrap/ewoks-install.ps1 b/src/ewoks/_bootstrap/ewoks-install.ps1 new file mode 100644 index 0000000..4539f56 --- /dev/null +++ b/src/ewoks/_bootstrap/ewoks-install.ps1 @@ -0,0 +1,716 @@ +# Install the python environment of an ewoks workflow with nothing but a package +# manager. For example +# +# & ([scriptblock]::Create((irm https://ewoks.readthedocs.io/en/stable/ewoks-install.ps1))) demo.json --package-manager-name uv +# +# or download the script first +# +# irm https://ewoks.readthedocs.io/en/stable/ewoks-install.ps1 -OutFile ewoks-install.ps1 +# powershell -ExecutionPolicy Bypass -File ewoks-install.ps1 demo.json --package-manager-name uv +# +# Steps (see `Invoke-EwoksInstall` at the end), like ewoks-install.sh: +# +# 1. Split the arguments in options of this script and arguments of `ewoks install`. +# 2. Select the package manager. +# 3. Read the ewoks, engine and python versions from the workflow requirements. +# 4. Create a bootstrap environment with that ewoks and engine, unless it exists. +# 5. Run `ewoks install` in the bootstrap environment. +# +# The bootstrap environments are cached in ~/.ewoks/bootstrap// +# where the name is, for example, `ewoks=7.0.0@ewokscore=5.1.0@python=3.12`. +# +# Errors are thrown instead of calling `exit`, which would close the PowerShell +# session that runs the script block. Native commands run with the "Continue" +# error action: Windows PowerShell 5.1 turns their stderr into errors. + +$ErrorActionPreference = "Stop" + +$usage = @' +Usage: ewoks-install.ps1 [OPTIONS] WORKFLOW [ARGS ...] + +Install the python environment of an ewoks workflow with nothing but a package +manager. The ewoks version that generated the workflow requirements, and the +engine that saved the workflow, are installed in a bootstrap environment, which +then runs + + ewoks install WORKFLOW ARGS + +See 'ewoks install --help' for ARGS. The package manager that creates the +bootstrap environment is selected with '--package-manager-name' and +'--package-manager-command', which are passed to 'ewoks install' as well. +Without them it is the first available of uv, pixi, conda, poetry and pip-venv. +A package manager that is not installed can be installed in the bootstrap +directory with '--install-package-manager'. + +Options: + --print-command Print the 'ewoks install' command instead of + running it. + --bootstrap-requirement REQ + Install REQ in the bootstrap environment instead of + the ewoks version and engine of the workflow, for + example 'ewoks>=7' or a wheel file. Repeat it to + install an engine as well, for example + '--bootstrap-requirement ewoks + --bootstrap-requirement ewoksorange'. + --bootstrap-dir DIR Directory of the bootstrap environments. + Default: ~/.ewoks/bootstrap + --install-package-manager + Install the package manager of + '--package-manager-name' when it is not installed. + -h, --help Show this help. +'@ + +$onWindows = ($PSVersionTable.PSEdition -eq "Desktop") -or $IsWindows + +# ============================================================================= +# Helpers +# ============================================================================= + +function Write-Info([string]$message) { + [Console]::Error.WriteLine("ewoks-install: $message") +} + +# Argument quoted for PowerShell when needed +function Format-Argument([string]$argument) { + if ($argument -match '^[A-Za-z0-9_./=:+\\-]+$') { + return $argument + } + return "'" + $argument.Replace("'", "''") + "'" +} + +# Arguments quoted for PowerShell and joined with spaces +function Format-Arguments([string[]]$arguments) { + return @($arguments | ForEach-Object { Format-Argument $_ }) -join " " +} + +function Test-Command([string]$name) { + return [bool](Get-Command $name -CommandType Application -ErrorAction SilentlyContinue) +} + +# Run a native command with its output on stderr instead of in the pipeline +function Invoke-Native([string[]]$command) { + $ErrorActionPreference = "Continue" + $arguments = @() + if ($command.Count -gt 1) { + $arguments = $command[1..($command.Count - 1)] + } + & $command[0] @arguments 2>&1 | ForEach-Object { [Console]::Error.WriteLine($_) } + if ($LASTEXITCODE -ne 0) { + throw "command failed: $($command -join ' ')" + } +} + +# Run a native command with additional environment variables +function Invoke-NativeWithEnv([hashtable]$variables, [string[]]$command) { + $previous = @{} + foreach ($name in $variables.Keys) { + $previous[$name] = [Environment]::GetEnvironmentVariable($name) + [Environment]::SetEnvironmentVariable($name, $variables[$name]) + } + try { + Invoke-Native $command + } finally { + foreach ($name in $variables.Keys) { + [Environment]::SetEnvironmentVariable($name, $previous[$name]) + } + } +} + +# Output of a native command +function Get-NativeOutput([string[]]$command) { + $ErrorActionPreference = "Continue" + $arguments = @() + if ($command.Count -gt 1) { + $arguments = $command[1..($command.Count - 1)] + } + $output = & $command[0] @arguments + if ($LASTEXITCODE -ne 0) { + throw "command failed: $($command -join ' ')" + } + return "$output".Trim() +} + +# Python interpreter with this major.minor version, or the default one +function Find-Python([string]$majorMinor) { + $ErrorActionPreference = "Continue" + if ($onWindows) { + # The python of the PATH, like the one of an active environment, when it has this version + if ($majorMinor -and (Test-Command "python")) { + try { + $pathMajorMinor = Get-PythonMajorMinor @("python") + } catch { + $pathMajorMinor = "" + } + if ($pathMajorMinor -eq $majorMinor) { + return @("python") + } + } + if (Test-Command "py") { + if ($majorMinor) { + & py "-$majorMinor" -c "pass" 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { + return @("py", "-$majorMinor") + } + Write-Info "warning: python $majorMinor of the workflow is not found: using py -3" + } + return @("py", "-3") + } + } elseif ($majorMinor -and (Test-Command "python$majorMinor")) { + return @("python$majorMinor") + } elseif (Test-Command "python3") { + if ($majorMinor) { Write-Info "warning: python$majorMinor of the workflow is not found: using python3" } + return @("python3") + } + if (Test-Command "python") { + if ($majorMinor) { Write-Info "warning: python $majorMinor of the workflow is not found: using python" } + return @("python") + } + throw "no python interpreter found (see --package-manager-name uv --install-package-manager)" +} + +# major.minor version of a python interpreter +function Get-PythonMajorMinor([string[]]$python) { + return Get-NativeOutput ($python + @("-c", 'import sys; print(''%d.%d'' % sys.version_info[:2])')) +} + +# Version of an installed distribution, or nothing when it is not installed +function Get-DistributionVersion([string]$python, [string]$name) { + $ErrorActionPreference = "Continue" + try { + return "$(& $python -c 'import importlib.metadata as m, sys; print(m.version(sys.argv[1]))' $name 2>$null)".Trim() + } catch { + return "" + } +} + +# ============================================================================= +# Workflow requirements +# ============================================================================= + +# Value in the requirements of a workflow file: ewoks.version, ewoks.engine.name, +# ewoks.engine.version or python.version. The requirements are JSON, which can be +# embedded in another format like the XML of .ows files, or YAML. +function Get-RequirementValue([string]$content, [string]$key) { + $value = Get-JsonRequirementValue $content $key + if (-not $value) { + $value = Get-YamlRequirementValue $content $key + } + return $value +} + +# Value in JSON without a JSON parser. For example `ewoks.version` in +# +# "ewoks": {"engine": {"name": "ewokscore", "version": "5.1.0"}, "version": "7.0.0"} +function Get-JsonRequirementValue([string]$content, [string]$key) { + $q = "[""']" + $value = "$q\s*:\s*$q([^""']*)$q" + # The members of an object, which can contain objects themselves + $members = "[^{}]*(?:\{[^{}]*\}[^{}]*)*" + switch ($key) { + "ewoks.version" { $pattern = "${q}ewoks$q\s*:\s*\{$members${q}version$value" } + "ewoks.engine.name" { $pattern = "${q}ewoks$q\s*:\s*\{[^{}]*${q}engine$q\s*:\s*\{[^{}]*${q}name$value" } + "ewoks.engine.version" { $pattern = "${q}ewoks$q\s*:\s*\{[^{}]*${q}engine$q\s*:\s*\{[^{}]*${q}version$value" } + "python.version" { $pattern = "${q}python$q\s*:\s*\{[^{}]*${q}version$value" } + } + # With the quotes that XML escapes + $found = [regex]::Matches($content.Replace(""", '"'), $pattern) + if ($found.Count -eq 0) { + return "" + } + return $found[$found.Count - 1].Groups[1].Value +} + +# Value in YAML without a YAML parser: the last `key: value` line whose key path +# ends with requirements. +function Get-YamlRequirementValue([string]$content, [string]$key) { + $wanted = "requirements.$key" + $indents = New-Object System.Collections.Generic.List[int] + $keys = New-Object System.Collections.Generic.List[string] + $found = "" + foreach ($line in ($content -split "\r?\n")) { + if ($line -match '^\s*(#|$)') { + continue + } + # The dash of a list item belongs to the indentation of its first key + $match = [regex]::Match($line, '^([ -]*)([A-Za-z0-9_]+):\s*(.*)$') + if (-not $match.Success) { + continue + } + # Key path of this line, from the keys of the lines with less indentation + $indent = $match.Groups[1].Length + while ($indents.Count -gt 0 -and $indents[$indents.Count - 1] -ge $indent) { + $indents.RemoveAt($indents.Count - 1) + $keys.RemoveAt($keys.Count - 1) + } + $indents.Add($indent) + $keys.Add($match.Groups[2].Value) + $path = $keys -join "." + $value = $match.Groups[3].Value + if ($value -and (($path -eq $wanted) -or $path.EndsWith(".$wanted"))) { + $found = $value -replace "^[""']|[""']$", "" + } + } + return $found +} + +# ============================================================================= +# Package managers +# ============================================================================= + +# Arguments of a '--package-manager-command' argument, split like `ewoks install` does +function Split-Command([string]$command) { + if ($onWindows) { + # Spaces escaped with a backslash are part of an argument + return @([regex]::Split($command, '(?=7") + $tag = "ewoks=latest" + } + if ($engineVersion) { + $requirements += "$engine==$engineVersion" + $tag = "$tag@$engine=$engineVersion" + } elseif ($engine) { + $requirements += $engine + $tag = "$tag@$engine" + } + $state.Version = $version + $state.Engine = $engine + $state.EngineVersion = $engineVersion + $state.Requirements = $requirements + $state.Tag = $tag +} + +# Install the package manager in the bootstrap directory when it is not installed. +# `ewoks install` needs it as well. +function Initialize-Manager([hashtable]$state) { + if (-not $state.Command -or $state.ManagerCommand -or (Test-Command $state.Command[0])) { + return + } + $manager = $state.Manager + $managerDir = Join-Path (Join-Path $state.Root "package-managers") $manager + $executable = Get-ManagerExecutable $manager $managerDir + if (-not (Test-Path -LiteralPath $executable -PathType Leaf)) { + if (-not $state.InstallPackageManager) { + throw "$manager is not installed (see --install-package-manager)" + } + Install-Manager $manager $managerDir + if (-not (Test-Path -LiteralPath $executable -PathType Leaf)) { + throw "cannot install $manager in $managerDir" + } + } + $state.Command = @($executable) + $state.Forwarded += @("--package-manager-command", (Format-Command $executable)) +} + +# Set `Python` and `Location`: the interpreter and directory of the bootstrap +# environment, which is created when it does not exist +function New-BootstrapEnvironment([hashtable]$state) { + $manager = $state.Manager + $command = $state.Command + + # pip-venv and poetry cannot provide a python version: the interpreter decides + $pythonVersion = $state.PythonVersion + $basePython = @() + if ($manager -eq "pip-venv") { + if (-not $command) { + $command = @(Find-Python $pythonVersion) + $state.Command = $command + } + $pythonVersion = Get-PythonMajorMinor $command + } elseif ($manager -eq "poetry") { + $basePython = @(Find-Python $pythonVersion) + $pythonVersion = Get-PythonMajorMinor $basePython + } + + $name = $state.Tag + if ($pythonVersion) { + $name = "$name@python=$pythonVersion" + } + $location = Join-Path (Join-Path $state.Root $manager) $name + if ($onWindows) { + $venvPython = Join-Path "Scripts" "python.exe" + $prefixPython = "python.exe" + } else { + $venvPython = Join-Path "bin" "python" + $prefixPython = $venvPython + } + switch ($manager) { + "pixi" { $python = Join-Path $location (Join-Path ".pixi/envs/default" $prefixPython) } + "conda" { $python = Join-Path $location $prefixPython } + "poetry" { $python = Join-Path $location (Join-Path ".venv" $venvPython) } + default { $python = Join-Path $location $venvPython } + } + $state.Location = $location + $state.Python = $python + if (Test-Path -LiteralPath $python -PathType Leaf) { + return + } + + Write-Info "create the bootstrap environment $location" + # A previous attempt failed + if (Test-Path -LiteralPath $location) { + Remove-Item -LiteralPath $location -Recurse -Force + } + New-Item -ItemType Directory -Path $location -Force | Out-Null + $pythonSpec = "python" + if ($pythonVersion) { + $pythonSpec = "python=$pythonVersion" + } + switch ($manager) { + "uv" { + $create = $command + @("venv", "--quiet") + if ($pythonVersion) { $create += @("--python", $pythonVersion) } + Invoke-Native ($create + @($location)) + } + "pixi" { + Invoke-Native ($command + @("init", $location)) + Invoke-Native ($command + @("add", "--manifest-path", (Join-Path $location "pixi.toml"), $pythonSpec, "pip")) + } + "conda" { + Invoke-Native ($command + @("create", "--yes", "--quiet", "--prefix", $location, $pythonSpec, "pip")) + } + "poetry" { + $pyproject = "[tool.poetry]`npackage-mode = false`n`n[tool.poetry.dependencies]`npython = `"*`"`n" + [System.IO.File]::WriteAllText((Join-Path $location "pyproject.toml"), $pyproject) + $interpreter = Get-NativeOutput ($basePython + @("-c", "import sys; print(sys.executable)")) + # Poetry uses an active virtual or conda environment instead of the one of the project. + # Without its own python it runs the `python` command, which can be python 2. + $variables = @{ + VIRTUAL_ENV = $null + CONDA_PREFIX = $null + POETRY_VIRTUALENVS_IN_PROJECT = "true" + POETRY_VIRTUALENVS_USE_POETRY_PYTHON = "true" + } + Invoke-NativeWithEnv $variables ($command + @("-C", $location, "env", "use", $interpreter)) + } + "pip-venv" { + Invoke-Native ($command + @("-m", "venv", $location)) + } + } +} + +# Install the requirements in the bootstrap environment unless it has their pinned versions +function Install-BootstrapRequirements([hashtable]$state) { + $python = $state.Python + if ($state.Version -and ((Get-DistributionVersion $python "ewoks") -eq $state.Version)) { + if (-not $state.Engine) { + return + } + $installed = Get-DistributionVersion $python $state.Engine + if ($installed -and (-not $state.EngineVersion -or $installed -eq $state.EngineVersion)) { + return + } + } + + $requirements = $state.Requirements + Write-Info "install $(Format-Arguments $requirements) in $($state.Location)" + try { + if ($state.Manager -eq "uv") { + Invoke-Native ($state.Command + @("pip", "install", "--quiet", "--upgrade", "--python", $python) + $requirements) + } else { + Invoke-Native (@($python, "-m", "pip", "install", "--quiet", "--disable-pip-version-check", "--upgrade") + $requirements) + } + } catch { + throw "cannot install $(Format-Arguments $requirements) (see --bootstrap-requirement)" + } +} + +# Run `ewoks install` in the bootstrap environment, or print the command +function Invoke-EwoksInstallCommand([hashtable]$state) { + $install = @("-m", "ewoks", "install") + $state.Forwarded + $command = "& " + (Format-Arguments (@($state.Python) + $install)) + if ($state.PrintCommand) { + Write-Output $command + return + } + # Printed by `ewoks install` before its summary + $header = @( + "Bootstrap environment" + " Python : $($state.Python)" + " Install: $command" + " Remove : Remove-Item -Recurse -Force $(Format-Argument $state.Location)" + ) -join "`n" + $previous = $env:EWOKS_INSTALL_SUMMARY_HEADER + $env:EWOKS_INSTALL_SUMMARY_HEADER = $header + try { + $ErrorActionPreference = "Continue" + & $state.Python @install + } finally { + # Not left in the PowerShell session that runs the script block + $env:EWOKS_INSTALL_SUMMARY_HEADER = $previous + } +} + +# Explain how to install without this script +function Write-ManualRoute([hashtable]$state) { + $python = "python" + if ($state.PythonVersion) { $python = "python $($state.PythonVersion)" } + Write-Info "the bootstrap failed. To install without this script:" + Write-Info " 1. create a $python environment and run in it: pip install $(Format-Arguments $state.Requirements)" + Write-Info " 2. run in that environment: python -m ewoks install $(Format-Arguments $state.Forwarded)" +} + +function Invoke-EwoksInstall([object[]]$arguments) { + $state = Read-Arguments $arguments + if ($state.Help) { + Write-Output $usage + return + } + if (-not $state.Workflow) { + Write-Info $usage + throw "provide the file of the workflow (no existing file in the arguments)" + } + Select-Manager $state + Read-WorkflowRequirements $state + + try { + Initialize-Manager $state + New-BootstrapEnvironment $state + Install-BootstrapRequirements $state + } catch { + # An interruption is not caught + Write-ManualRoute $state + throw + } + Invoke-EwoksInstallCommand $state +} + +Invoke-EwoksInstall $args diff --git a/src/ewoks/_bootstrap/ewoks-install.sh b/src/ewoks/_bootstrap/ewoks-install.sh new file mode 100644 index 0000000..371f84c --- /dev/null +++ b/src/ewoks/_bootstrap/ewoks-install.sh @@ -0,0 +1,577 @@ +#!/bin/sh +# Install the python environment of an ewoks workflow with nothing but a package +# manager. For example +# +# curl -LsSf https://ewoks.readthedocs.io/en/stable/ewoks-install.sh | sh -s -- demo.json --package-manager-name uv +# +# or download the script first +# +# curl -LsSf -o ewoks-install.sh https://ewoks.readthedocs.io/en/stable/ewoks-install.sh +# sh ewoks-install.sh demo.json --package-manager-name uv +# +# Steps (see `main` at the end): +# +# 1. Split the arguments in options of this script and arguments of `ewoks install`. +# 2. Select the package manager. +# 3. Read the ewoks, engine and python versions from the workflow requirements. +# 4. Create a bootstrap environment with that ewoks and engine, unless it exists. +# 5. Run `ewoks install` in the bootstrap environment. +# +# The bootstrap environments are cached in ~/.ewoks/bootstrap// +# where the name is, for example, `ewoks=7.0.0@ewokscore=5.1.0@python=3.12`. +set -eu + +usage() { + cat <<'EOF' +Usage: ewoks-install.sh [OPTIONS] WORKFLOW [ARGS ...] + +Install the python environment of an ewoks workflow with nothing but a package +manager. The ewoks version that generated the workflow requirements, and the +engine that saved the workflow, are installed in a bootstrap environment, which +then runs + + ewoks install WORKFLOW ARGS + +See 'ewoks install --help' for ARGS. The package manager that creates the +bootstrap environment is selected with '--package-manager-name' and +'--package-manager-command', which are passed to 'ewoks install' as well. +Without them it is the first available of uv, pixi, conda, poetry and pip-venv. +A package manager that is not installed can be installed in the bootstrap +directory with '--install-package-manager'. + +Options: + --print-command Print the 'ewoks install' command instead of + running it. + --bootstrap-requirement REQ + Install REQ in the bootstrap environment instead of + the ewoks version and engine of the workflow, for + example 'ewoks>=7' or a wheel file. Repeat it to + install an engine as well, for example + '--bootstrap-requirement ewoks + --bootstrap-requirement ewoksorange'. + --bootstrap-dir DIR Directory of the bootstrap environments. + Default: ~/.ewoks/bootstrap + --install-package-manager + Install the package manager of + '--package-manager-name' when it is not installed. + -h, --help Show this help. +EOF +} + +# ============================================================================= +# Helpers +# ============================================================================= + +say() { + printf 'ewoks-install: %s\n' "$*" >&2 +} + +die() { + say "error: $*" + exit 1 +} + +# Argument quoted for the shell when needed +quote() { + case $1 in + '' | *[!A-Za-z0-9_./=:@%+,-]*) + printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" + ;; + *) + printf '%s' "$1" + ;; + esac +} + +# Arguments quoted for the shell and joined with spaces +quote_all() { + for arg in "$@"; do + printf '%s ' "$(quote "$arg")" + done | sed 's/ $//' +} + +# Absolute path of a file +absolute_path() { + case $1 in + /*) printf '%s' "$1" ;; + *) printf '%s/%s' "$(pwd)" "$1" ;; + esac +} + +# Python interpreter with this major.minor version, or the default one +find_python() { + if [ -n "$1" ] && command -v "python$1" >/dev/null 2>&1; then + printf 'python%s' "$1" + elif command -v python3 >/dev/null 2>&1; then + [ -z "$1" ] || say "warning: python$1 of the workflow is not found: using python3" + printf 'python3' + else + die "no python interpreter found (see --package-manager-name uv --install-package-manager)" + fi +} + +# major.minor version of a python interpreter +python_major_minor() { + "$@" -c 'import sys; print("%d.%d" % sys.version_info[:2])' +} + +# Version of an installed distribution, or nothing when it is not installed +distribution_version() { + "$1" -c 'import importlib.metadata as m, sys; print(m.version(sys.argv[1]))' "$2" 2>/dev/null || true +} + +# Download a URL to a file +download() { + if command -v curl >/dev/null 2>&1; then + curl -LsSf -o "$2" "$1" || die "cannot download $1" + elif command -v wget >/dev/null 2>&1; then + wget -q -O "$2" "$1" || die "cannot download $1" + else + die "curl or wget is needed to download $1" + fi +} + +# ============================================================================= +# Workflow requirements +# ============================================================================= + +# Value in the requirements of a workflow file: ewoks.version, ewoks.engine.name, +# ewoks.engine.version or python.version. The requirements are JSON, which can be +# embedded in another format like the XML of .ows files, or YAML. +requirement_value() { + value=$(json_requirement_value "$1" "$2") + [ -n "$value" ] || value=$(yaml_requirement_value "$1" "$2") + printf '%s' "$value" +} + +# Value in JSON without a JSON parser. For example `ewoks.version` in +# +# "ewoks": {"engine": {"name": "ewokscore", "version": "5.1.0"}, "version": "7.0.0"} +json_requirement_value() { + q="[\"']" + s='[[:space:]]*' + value="${q}$s:$s${q}\([^\"']*\)${q}" + # The members of an object, which can contain objects themselves + members='[^{}]*\({[^{}]*}[^{}]*\)*' + case $2 in + ewoks.version) + pattern="${q}ewoks${q}$s:$s{$members${q}version$value" group=2 + ;; + ewoks.engine.name) + pattern="${q}ewoks${q}$s:$s{[^{}]*${q}engine${q}$s:$s{[^{}]*${q}name$value" group=1 + ;; + ewoks.engine.version) + pattern="${q}ewoks${q}$s:$s{[^{}]*${q}engine${q}$s:$s{[^{}]*${q}version$value" group=1 + ;; + python.version) + pattern="${q}python${q}$s:$s{[^{}]*${q}version$value" group=1 + ;; + esac + # On one line, with the quotes that XML escapes + tr -d '\n\r' <"$1" | sed 's/"/"/g' | sed -n "s/.*$pattern.*/\\$group/p" +} + +# Value in YAML without a YAML parser: the last `key: value` line whose key path +# ends with requirements. +yaml_requirement_value() { + awk -v wanted="requirements.$2" -v sq="'" ' + { + sub(/\r$/, "") + if ($0 ~ /^[ \t]*(#|$)/) next + # The dash of a list item belongs to the indentation of its first key + match($0, /^[ -]*/) + indent = RLENGTH + rest = substr($0, indent + 1) + if (rest !~ /^[A-Za-z0-9_]+:/) next + key = rest + sub(/:.*/, "", key) + value = rest + sub(/^[^:]*:[ \t]*/, "", value) + # Key path of this line, from the keys of the lines with less indentation + while (depth > 0 && indents[depth] >= indent) depth-- + depth++ + indents[depth] = indent + keys[depth] = key + path = keys[1] + for (i = 2; i <= depth; i++) path = path "." keys[i] + n = length(path) - length(wanted) + if (value == "" || n < 0 || substr(path, n + 1) != wanted) next + if (n > 0 && substr(path, n, 1) != ".") next + gsub("^[\"" sq "]|[\"" sq "]$", "", value) + found = value + } + END { printf "%s", found } + ' "$1" +} + +# ============================================================================= +# Package managers +# ============================================================================= + +# Run the package manager command `cmd`, which is quoted for the shell like the +# '--package-manager-command' argument of `ewoks install` +run_manager() { + eval "set -- $cmd \"\$@\"" + "$@" +} + +# Executable of a package manager installed in a directory by this script +manager_executable() { + case $1 in + uv) printf '%s/bin/uv' "$2" ;; + pixi) printf '%s/bin/pixi' "$2" ;; + poetry) printf '%s/bin/poetry' "$2" ;; + conda) printf '%s/bin/conda' "$2" ;; + esac +} + +# Install a package manager in a directory with its official installer +install_manager() { + if [ "$1" = conda ]; then + case $2 in + *" "*) die "Miniforge cannot be installed in a directory with spaces: $2 (see --bootstrap-dir)" ;; + esac + fi + say "install $1 in $2" + rm -rf "$2" + mkdir -p "$(dirname "$2")" + # Removed by `on_exit` when the installation fails. + # The Miniforge installer requires the '.sh' suffix. + installer_dir=$(mktemp -d) + installer=$installer_dir/installer.sh + case $1 in + uv) + download https://astral.sh/uv/install.sh "$installer" + UV_INSTALL_DIR=$2/bin UV_NO_MODIFY_PATH=1 sh "$installer" >&2 || + die "the uv installer failed" + ;; + pixi) + download https://pixi.sh/install.sh "$installer" + PIXI_HOME=$2 PIXI_NO_PATH_UPDATE=1 sh "$installer" >&2 || + die "the pixi installer failed" + ;; + poetry) + download https://install.python-poetry.org "$installer" + POETRY_HOME=$2 "$(find_python "")" "$installer" >&2 || + die "the poetry installer failed" + ;; + conda) + system=$(uname -s) + [ "$system" != Darwin ] || system=MacOSX + download "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$system-$(uname -m).sh" "$installer" + bash "$installer" -b -p "$2" >&2 || + die "the Miniforge installer failed" + ;; + esac + rm -rf "$installer_dir" + installer_dir="" +} + +# ============================================================================= +# Steps of `main`, which share their variables +# ============================================================================= + +# Set the options of this script and `forwarded`: the other arguments, which are +# for `ewoks install`, quoted for the shell +parse_arguments() { + print_command=0 + install_package_manager=0 + custom_requirements="" # Quoted for the shell + root=$HOME/.ewoks/bootstrap + manager="" + manager_command="" + workflow="" + forwarded="" + while [ $# -gt 0 ]; do + arg=$1 + shift + case $arg in + -h | --help) + usage + exit 0 + ;; + --print-command) + print_command=1 + ;; + --install-package-manager) + install_package_manager=1 + ;; + --bootstrap-requirement) + [ $# -gt 0 ] || die "$arg requires a value" + custom_requirements="$custom_requirements $(quote "$1")" + shift + ;; + --bootstrap-requirement=*) + custom_requirements="$custom_requirements $(quote "${arg#*=}")" + ;; + --bootstrap-dir) + [ $# -gt 0 ] || die "$arg requires a value" + root=$1 + shift + ;; + --bootstrap-dir=*) + root=${arg#*=} + ;; + --package-manager-name | --package-manager-command) + [ $# -gt 0 ] || die "$arg requires a value" + if [ "$arg" = --package-manager-name ]; then + manager=$1 + else + manager_command=$1 + fi + forwarded="$forwarded $arg $(quote "$1")" + shift + ;; + --package-manager-name=*) + manager=${arg#*=} + forwarded="$forwarded $(quote "$arg")" + ;; + --package-manager-command=*) + manager_command=${arg#*=} + forwarded="$forwarded $(quote "$arg")" + ;; + *) + # The workflow is the first argument that is an existing file. + # Its absolute path makes the printed commands work from any directory. + if [ -z "$workflow" ] && [ -f "$arg" ]; then + arg=$(absolute_path "$arg") + workflow=$arg + fi + forwarded="$forwarded $(quote "$arg")" + ;; + esac + done + custom_requirements=${custom_requirements# } + forwarded=${forwarded# } + + if [ -z "$workflow" ]; then + usage >&2 + die "provide the file of the workflow (no existing file in the arguments)" + fi +} + +# Set `manager` and `cmd`: the command of the package manager, quoted for the shell. +# `cmd` is empty for pip-venv without '--package-manager-command'. +select_manager() { + manager=$(printf '%s' "$manager" | tr '[:upper:]' '[:lower:]') + if [ "$install_package_manager" = 1 ] && [ -z "$manager" ]; then + die "--install-package-manager requires --package-manager-name" + fi + if [ -z "$manager" ]; then + [ -z "$manager_command" ] || die "--package-manager-command requires --package-manager-name" + for name in uv pixi conda poetry; do + if command -v "$name" >/dev/null 2>&1; then + manager=$name + break + fi + done + manager=${manager:-pip-venv} + fi + + case $manager in + uv | pixi | conda | poetry) cmd=${manager_command:-$manager} ;; + pip-venv) cmd=$manager_command ;; + *) die "package manager '$manager' is not supported" ;; + esac +} + +# Set `python_version` (major.minor), `requirements` of the bootstrap environment +# (quoted for the shell), `tag` (the name of the bootstrap environment without the +# python version) and, to check whether an existing bootstrap environment is up to +# date, `version`, `engine` and `engine_version` +read_workflow_requirements() { + python_version=$(requirement_value "$workflow" python.version) + python_version=$(printf '%s' "$python_version" | sed -n 's/^\([0-9]*\.[0-9]*\).*/\1/p') + + # The `@` and `=` separators of the tag cannot be part of a distribution name or version + if [ -n "$custom_requirements" ]; then + version="" + engine="" + requirements=$custom_requirements + tag=custom=$(printf '%s' "$requirements" | cksum | cut -d ' ' -f 1) + return + fi + + version=$(requirement_value "$workflow" ewoks.version) + engine=$(requirement_value "$workflow" ewoks.engine.name) + engine_version=$(requirement_value "$workflow" ewoks.engine.version) + [ -n "$engine_version" ] || engine="" + if [ -z "$engine" ]; then + case $workflow in + *.ows) + # Workflows saved before ewoks recorded the engine + say "the workflow does not provide its engine: using the latest ewoksorange" + engine=ewoksorange + ;; + esac + fi + + if [ -n "$version" ]; then + requirements=$(quote "ewoks==$version") + tag=ewoks=$version + else + say "the workflow does not provide the ewoks version: using the latest one" + requirements=$(quote "ewoks>=7") + tag=ewoks=latest + fi + if [ -n "$engine" ]; then + requirements="$requirements $(quote "$engine${engine_version:+==$engine_version}")" + tag=$tag@$engine${engine_version:+=$engine_version} + fi +} + +# Install the package manager in the bootstrap directory when it is not installed. +# `ewoks install` needs it as well. +ensure_manager() { + if [ -z "$cmd" ] || [ -n "$manager_command" ] || command -v "$cmd" >/dev/null 2>&1; then + return + fi + manager_dir=$root/package-managers/$manager + executable=$(manager_executable "$manager" "$manager_dir") + if [ ! -x "$executable" ]; then + [ "$install_package_manager" = 1 ] || + die "$manager is not installed (see --install-package-manager)" + install_manager "$manager" "$manager_dir" + [ -x "$executable" ] || die "cannot install $manager in $manager_dir" + fi + cmd=$(quote "$executable") + forwarded="$forwarded --package-manager-command $(quote "$cmd")" +} + +# Set `python`: the interpreter of the bootstrap environment, which is created when +# it does not exist +create_bootstrap_environment() { + # pip-venv and poetry cannot provide a python version: the interpreter decides + env_python_version=$python_version + case $manager in + pip-venv) + [ -n "$cmd" ] || cmd=$(find_python "$python_version") + env_python_version=$(python_major_minor run_manager) + ;; + poetry) + base_python=$(find_python "$python_version") + env_python_version=$(python_major_minor "$base_python") + ;; + esac + + location=$root/$manager/$tag${env_python_version:+@python=$env_python_version} + case $manager in + pixi) python=$location/.pixi/envs/default/bin/python ;; + poetry) python=$location/.venv/bin/python ;; + *) python=$location/bin/python ;; + esac + if [ -x "$python" ]; then + return + fi + + say "create the bootstrap environment $location" + # A previous attempt failed + rm -rf "$location" + mkdir -p "$location" + case $manager in + uv) + run_manager venv --quiet ${env_python_version:+--python "$env_python_version"} "$location" >&2 + ;; + pixi) + run_manager init "$location" >&2 + run_manager add --manifest-path "$location/pixi.toml" "python${env_python_version:+=$env_python_version}" pip >&2 + ;; + conda) + run_manager create --yes --quiet --prefix "$location" "python${env_python_version:+=$env_python_version}" pip >&2 + ;; + poetry) + printf '[tool.poetry]\npackage-mode = false\n\n[tool.poetry.dependencies]\npython = "*"\n' >"$location/pyproject.toml" + interpreter=$("$base_python" -c 'import sys; print(sys.executable)') + # Poetry uses an active virtual or conda environment instead of the one of the project. + # Without its own python it runs the `python` command, which can be python 2. + ( + unset VIRTUAL_ENV CONDA_PREFIX + export POETRY_VIRTUALENVS_IN_PROJECT=true POETRY_VIRTUALENVS_USE_POETRY_PYTHON=true + run_manager -C "$location" env use "$interpreter" >&2 + ) + ;; + pip-venv) + run_manager -m venv "$location" >&2 + ;; + esac +} + +# Install the requirements in the bootstrap environment unless it has their pinned versions +install_bootstrap_requirements() { + if [ -n "$version" ] && [ "$(distribution_version "$python" ewoks)" = "$version" ]; then + if [ -z "$engine" ]; then + return + fi + installed=$(distribution_version "$python" "$engine") + if [ -n "$installed" ] && [ "${engine_version:-$installed}" = "$installed" ]; then + return + fi + fi + + say "install $requirements in $location" + if [ "$manager" = uv ]; then + eval "run_manager pip install --quiet --upgrade --python \"\$python\" $requirements" >&2 || + die "cannot install $requirements (see --bootstrap-requirement)" + else + eval "\"\$python\" -m pip install --quiet --disable-pip-version-check --upgrade $requirements" >&2 || + die "cannot install $requirements (see --bootstrap-requirement)" + fi +} + +# Run `ewoks install` in the bootstrap environment, or print the command +run_ewoks_install() { + eval "set -- \"\$python\" -m ewoks install $forwarded" + if [ "$print_command" = 1 ]; then + quote_all "$@" + printf '\n' + return + fi + + # Printed by `ewoks install` before its summary + EWOKS_INSTALL_SUMMARY_HEADER=$( + printf 'Bootstrap environment\n' + printf ' Python : %s\n' "$python" + printf ' Install: %s\n' "$(quote_all "$@")" + printf ' Remove : rm -rf %s' "$(quote "$location")" + ) + export EWOKS_INSTALL_SUMMARY_HEADER + if [ ! -t 0 ] && (exec /dev/null; then + # Confirmation prompts do not read stdin, which is this script with `curl | sh` + exec "$@" None: + config.addinivalue_line( + "markers", + "package_manager_install: installs package managers from the internet, " + "only runs when selected with '-m package_manager_install'", + ) + + +_PACKAGE_MANAGER_INSTALL = "package_manager_install" + + +def pytest_collection_modifyitems( + config: pytest.Config, items: List[pytest.Item] +) -> None: + """Skip the tests that install package managers unless they are selected.""" + if _PACKAGE_MANAGER_INSTALL in config.getoption("markexpr", ""): + return + skip = pytest.mark.skip( + reason=f"installs package managers: select with '-m {_PACKAGE_MANAGER_INSTALL}'" + ) + for item in items: + if _PACKAGE_MANAGER_INSTALL in item.keywords: + item.add_marker(skip) + + @pytest.fixture def mock_icat_client(monkeypatch): mock_defaults = MagicMock() diff --git a/src/ewoks/tests/requirements/conftest.py b/src/ewoks/tests/requirements/conftest.py index cb50a1a..92c3a5d 100644 --- a/src/ewoks/tests/requirements/conftest.py +++ b/src/ewoks/tests/requirements/conftest.py @@ -52,6 +52,9 @@ def _home_variables(home: Path) -> Dict[str, Path]: # Directories in which package managers create named environments "CONDA_ENVS_DIRS": home / "conda" / "envs", "POETRY_VIRTUALENVS_PATH": home / "poetry" / "virtualenvs", + # Pixi cannot find its cache directory without a user profile, like on CI + # runners that run as a service + "PIXI_CACHE_DIR": home / "pixi" / "cache", # Channels are no longer provided by the home directory of the user "CONDARC": condarc, } diff --git a/src/ewoks/tests/requirements/test_bootstrap.py b/src/ewoks/tests/requirements/test_bootstrap.py new file mode 100644 index 0000000..f7103b6 --- /dev/null +++ b/src/ewoks/tests/requirements/test_bootstrap.py @@ -0,0 +1,422 @@ +"""Tests of the scripts that run `ewoks install` with nothing but a package manager. + +The workflows are saved by `ewoks convert`. A fake bootstrap environment is +prepared in advance so the scripts do not install anything: its python interpreter +runs the ewoks under test, which is not on the python package index. Except for +the tests marked `package_manager_install`, which install package managers and +ewoks from the internet. +""" + +import os +import shlex +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Callable +from typing import Dict +from typing import List +from typing import Optional + +import pytest + +from ...bindings import convert_graph +from ...bindings import load_graph +from .utils import PYTHON_MINOR_VERSION + +_SCRIPTS = Path(__file__).parents[2] / "_bootstrap" +_INSTALLABLE_MANAGERS = ["uv", "pixi", "poetry", "conda"] + + +@pytest.fixture(params=["sh", "pwsh", "powershell"]) +def script(request) -> List[str]: + """Command that runs a bootstrap script. `powershell` is Windows PowerShell 5.1.""" + if request.param == "sh": + if sys.platform == "win32": + pytest.skip("sh is not available on Windows") + return ["sh", str(_SCRIPTS / "ewoks-install.sh")] + if request.param == "powershell" and sys.platform != "win32": + pytest.skip("Windows PowerShell is only available on Windows") + powershell = shutil.which(request.param) + if powershell is None: + pytest.skip(f"{request.param} is not installed") + return [ + powershell, + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(_SCRIPTS / "ewoks-install.ps1"), + ] + + +@pytest.fixture +def no_managers_env(tmp_path) -> Dict[str, str]: + """Environment variables with a PATH that only provides python and the + system tools, which can include package managers (see `_skip_if_installed`).""" + if sys.platform == "win32": + system_root = os.environ.get("SystemRoot", r"C:\Windows") + directories = [ + os.path.dirname(sys.executable), + os.path.join(system_root, "System32"), + system_root, + os.path.join(system_root, "System32", "WindowsPowerShell", "v1.0"), + ] + else: + bin_dir = tmp_path / "python-bin" + bin_dir.mkdir() + for name in ("python3", f"python{PYTHON_MINOR_VERSION}"): + _python_wrapper(bin_dir / name) + directories = [str(bin_dir), "/usr/bin", "/bin", "/usr/sbin", "/sbin"] + path = os.pathsep.join(directories) + env = { + name: value + for name, value in os.environ.items() + if name not in ("VIRTUAL_ENV", "CONDA_PREFIX") + } + env["PATH"] = path + return env + + +@pytest.fixture(scope="module") +def saved_workflow(tmp_path_factory) -> Callable[[str], Path]: + """Workflow saved by `ewoks convert` in a format: json, yaml or ows. Every + format is saved once because generating the requirements takes seconds.""" + directory = tmp_path_factory.mktemp("workflows") + workflows: Dict[str, Path] = {} + + def get(workflow_format: str) -> Path: + if workflow_format not in workflows: + workflows[workflow_format] = _save_workflow(directory, workflow_format) + return workflows[workflow_format] + + return get + + +@pytest.fixture +def workflow(saved_workflow) -> Path: + """Workflow saved by `ewoks convert` in JSON.""" + return saved_workflow("json") + + +@pytest.fixture +def bootstrap_dir(tmp_path, workflow) -> Path: + """Directory of the bootstrap environments with a fake pip-venv environment + for the workflow.""" + root = tmp_path / "boot strap" + _fake_bootstrap_environment(root, workflow) + return root + + +def test_print_command(script, workflow, bootstrap_dir): + """The bootstrap environment of the ewoks version of the workflow runs + `ewoks install` with the arguments that are not for the script.""" + arguments = [str(workflow), *_pip_venv_arguments(), "--yes"] + + result = _run(script, ["--print-command", *arguments], bootstrap_dir) + + assert result.returncode == 0, result.stderr + command = result.stdout.strip() + python = _bootstrap_python(bootstrap_dir, workflow) + assert str(python) in command + assert command.endswith(f"-m ewoks install {' '.join(arguments)}") + + +def test_run_ewoks_install(script, workflow, bootstrap_dir, tmp_path): + """`ewoks install` runs in the bootstrap environment and prints how to run it + again before its summary.""" + # A directory that is not a python environment, so nothing is installed + env_root = tmp_path / "envs" + (env_root / "bootstrap").mkdir(parents=True) + arguments = [ + str(workflow), + *_pip_venv_arguments(), + "--yes", + "--clean", + "--env-root", + str(env_root), + ] + + result = _run(script, arguments, bootstrap_dir) + + assert "is not a python environment" in result.stderr + python = _bootstrap_python(bootstrap_dir, workflow) + header, summary = result.stdout.split("Install failed for") + assert "Bootstrap environment" in header + assert str(python) in header + assert f"-m ewoks install {workflow}" in header.replace("'", "") + + +def test_manager_command_with_spaces(script, workflow, bootstrap_dir, tmp_path): + """The package manager command is quoted like the one of `ewoks install`.""" + if script[0] != "sh": + pytest.skip("PowerShell splits the command like `ewoks install` on Windows") + python = tmp_path / "python bin" / "python" + python.parent.mkdir() + _python_wrapper(python) + arguments = [ + str(workflow), + "--package-manager-name", + "pip-venv", + "--package-manager-command", + shlex.quote(str(python)), + ] + + result = _run(script, ["--print-command", *arguments], bootstrap_dir) + + assert result.returncode == 0, result.stderr + assert "@python=" in result.stdout + + +def test_installed_manager_with_spaces(workflow, no_managers_env, tmp_path): + """The package manager installed by the script is passed to `ewoks install` + quoted for the shell.""" + _skip_if_installed("uv", no_managers_env) + if sys.platform == "win32": + pytest.skip("sh is not available on Windows") + bootstrap_dir = tmp_path / "boot strap" + uv = bootstrap_dir / "package-managers" / "uv" / "bin" / "uv" + uv.parent.mkdir(parents=True) + uv.write_text("#!/bin/sh\nexit 1\n") + uv.chmod(0o755) + _fake_bootstrap_environment(bootstrap_dir, workflow, manager="uv") + + result = _run( + ["sh", str(_SCRIPTS / "ewoks-install.sh")], + ["--print-command", str(workflow), "--package-manager-name", "uv"], + bootstrap_dir, + env=no_managers_env, + ) + + assert result.returncode == 0, result.stderr + command = shlex.split(result.stdout) + manager_command = command[command.index("--package-manager-command") + 1] + assert shlex.split(manager_command) == [str(uv)] + + +def test_relative_workflow(script, workflow, bootstrap_dir): + """`ewoks install` gets the absolute path of the workflow.""" + result = _run( + script, + ["--print-command", workflow.name, *_pip_venv_arguments()], + bootstrap_dir, + cwd=workflow.parent, + ) + + assert result.returncode == 0, result.stderr + assert f"-m ewoks install {workflow}" in result.stdout.replace("'", "") + + +def test_no_workflow(script, tmp_path): + result = _run(script, ["--test", "demo"], tmp_path / "bootstrap") + + assert result.returncode != 0 + assert "provide the file of the workflow" in result.stderr + + +def test_unsupported_manager(script, workflow, tmp_path): + + result = _run( + script, + [str(workflow), "--package-manager-name", "unknown"], + tmp_path / "bootstrap", + ) + + assert result.returncode != 0 + assert "package manager 'unknown' is not supported" in result.stderr + + +def test_install_manager_requires_name(script, workflow, tmp_path): + + result = _run( + script, + [str(workflow), "--install-package-manager"], + tmp_path / "bootstrap", + ) + + assert result.returncode != 0 + assert "--install-package-manager requires --package-manager-name" in ( + result.stderr + ) + + +def test_manager_not_installed(script, workflow, no_managers_env, tmp_path): + _skip_if_installed("pixi", no_managers_env) + + result = _run( + script, + [str(workflow), "--package-manager-name", "pixi"], + tmp_path / "bootstrap", + env=no_managers_env, + ) + + assert result.returncode != 0 + assert "pixi is not installed (see --install-package-manager)" in result.stderr + + +def test_install_conda_in_directory_with_spaces( + script, workflow, no_managers_env, tmp_path +): + _skip_if_installed("conda", no_managers_env) + result = _run( + script, + [ + "--install-package-manager", + str(workflow), + "--package-manager-name", + "conda", + ], + tmp_path / "boot strap", + env=no_managers_env, + ) + + assert result.returncode != 0 + assert "Miniforge cannot be installed in a directory with spaces" in result.stderr + assert "install conda in" not in result.stderr + + +@pytest.mark.package_manager_install +@pytest.mark.parametrize("manager", _INSTALLABLE_MANAGERS) +def test_install_manager(script, workflow, no_managers_env, tmp_path, manager): + """The package manager is installed in the bootstrap directory, used again + by the next run and passed to `ewoks install`.""" + _skip_if_installed(manager, no_managers_env) + if manager == "conda": + # Miniforge cannot be installed in a directory with spaces + bootstrap_dir = tmp_path / "bootstrap" + else: + # A directory with a space, like the home directory of some Windows users + bootstrap_dir = tmp_path / "boot strap" + arguments = [ + "--print-command", + "--bootstrap-requirement", + "ewoks", + str(workflow), + "--package-manager-name", + manager, + "--yes", + ] + + result = _run( + script, + ["--install-package-manager", *arguments], + bootstrap_dir, + env=no_managers_env, + timeout=1800, + ) + + assert result.returncode == 0, result.stderr + assert f"install {manager} in" in result.stderr + manager_dir = bootstrap_dir / "package-managers" / manager + command = result.stdout.strip() + if sys.platform == "win32": + # `ewoks install` splits the command on spaces that are not escaped + assert str(manager_dir).replace(" ", "\\ ") in command + else: + assert str(manager_dir) in command + + result = _run(script, arguments, bootstrap_dir, env=no_managers_env) + + assert result.returncode == 0, result.stderr + assert f"install {manager} in" not in result.stderr + assert result.stdout.strip() == command + + +@pytest.mark.parametrize("workflow_format", ["json", "yaml", "ows"]) +def test_requirements_of_any_format(script, saved_workflow, tmp_path, workflow_format): + """The bootstrap environment of the ewoks version and the engine of the + workflow is used, whatever the format of the workflow.""" + workflow = saved_workflow(workflow_format) + bootstrap_dir = tmp_path / "bootstrap" + python = _fake_bootstrap_environment(bootstrap_dir, workflow) + + result = _run( + script, + ["--print-command", str(workflow), *_pip_venv_arguments()], + bootstrap_dir, + ) + + assert result.returncode == 0, result.stderr + assert "install ewoks" not in result.stderr + assert str(python) in result.stdout + + +def _skip_if_installed(manager: str, env: Dict[str, str]) -> None: + """Skip a test that needs a package manager to be missing from the PATH.""" + if shutil.which(manager, path=env["PATH"]): + pytest.skip(f"{manager} is installed in a system directory") + + +def _fake_bootstrap_environment( + root: Path, workflow: Path, manager: str = "pip-venv" +) -> Path: + """Bootstrap environment of a workflow, as the scripts cache it, whose python + interpreter is the one of the tests. The scripts find the ewoks under test + installed in it, so they do not install anything. Returns the python + interpreter.""" + if sys.platform == "win32": + pytest.skip("the fake bootstrap environment is a POSIX shell script") + python = _bootstrap_python(root, workflow, manager) + python.parent.mkdir(parents=True) + _python_wrapper(python) + return python + + +def _bootstrap_python(root: Path, workflow: Path, manager: str = "pip-venv") -> Path: + """Python interpreter of the bootstrap environment of a workflow.""" + ewoks = load_graph(str(workflow)).graph.graph["requirements"]["ewoks"] + engine = ewoks["engine"] + name = ( + f"ewoks={ewoks['version']}@{engine['name']}={engine['version']}" + f"@python={PYTHON_MINOR_VERSION}" + ) + return root / manager / name / "bin" / "python" + + +def _python_wrapper(path: Path) -> None: + """POSIX shell script that runs the python interpreter of the tests.""" + with open(path, "w", encoding="utf-8") as fh: + fh.write(f'#!/bin/sh\nexec "{sys.executable}" "$@"\n') + path.chmod(0o755) + + +def _save_workflow(directory: Path, workflow_format: str) -> Path: + """Workflow saved by `ewoks convert` with its requirements.""" + if workflow_format == "ows": + pytest.importorskip("ewoksorange") + filename = directory / f"workflow.{workflow_format}" + convert_graph( + {"graph": {"id": "bootstrap"}, "nodes": [], "links": []}, + str(filename), + package_manager_name="pip-venv", + package_manager_command=(sys.executable,), + ) + return filename + + +def _pip_venv_arguments() -> List[str]: + return [ + "--package-manager-name", + "pip-venv", + "--package-manager-command", + sys.executable, + ] + + +def _run( + script: List[str], + arguments: List[str], + bootstrap_dir: Path, + env: Optional[Dict[str, str]] = None, + timeout: float = 300, + cwd: Optional[Path] = None, +) -> "subprocess.CompletedProcess[str]": + return subprocess.run( # noqa: S603 - Script under test + [*script, "--bootstrap-dir", str(bootstrap_dir), *arguments], + capture_output=True, + text=True, + env=env, + cwd=cwd, + stdin=subprocess.DEVNULL, + timeout=timeout, + ) diff --git a/src/ewoks/tests/requirements/test_install_cli.py b/src/ewoks/tests/requirements/test_install_cli.py index 3305b46..3082fd0 100644 --- a/src/ewoks/tests/requirements/test_install_cli.py +++ b/src/ewoks/tests/requirements/test_install_cli.py @@ -1,6 +1,7 @@ """Tests of `ewoks install`, which does not depend on the package manager.""" import json +import os import shutil import subprocess import sys @@ -41,6 +42,33 @@ def test_printed_commands_with_spaces(tmp_path): assert _remove_command(location) == f"rm -rf '{location}'" +def test_summary_header(tmp_path): + """The text of the environment variable is printed before the summary.""" + graph = _graph(manager_requirements("pip-venv", python_version=PYTHON_VERSION)) + # A directory that is not a python environment, so nothing is installed + (tmp_path / _GRAPH_ID).mkdir() + argv = [ + sys.executable, + "-m", + "ewoks", + "install", + json.dumps(graph), + "--yes", + "--clean", + "--env-root", + str(tmp_path), + "--package-manager-name", + "pip-venv", + ] + env = {**os.environ, "EWOKS_INSTALL_SUMMARY_HEADER": "Header\n line"} + + result = subprocess.run( # noqa: S603 - Trusted test command + argv, capture_output=True, text=True, env=env + ) + + assert result.stdout.startswith("Header\n line\nInstall failed for") + + def test_install_with_files(fast_manager_case, env_root): files = fast_manager_case.native_files(DISTRIBUTIONS, PYTHON_VERSION) if not files: