diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..6324d40 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/ansible/roles/astral/defaults/main.yml b/ansible/roles/astral/defaults/main.yml new file mode 100644 index 0000000..aa4af2d --- /dev/null +++ b/ansible/roles/astral/defaults/main.yml @@ -0,0 +1,8 @@ +--- +# The Python uv uses by default, pinned as each user's global uv pin. uv +# installs it only where the system python3 is older. Follows the HyperI floor. +# +# The role rewrites that global pin on every converge, so a user's own +# `uv python pin --global` does not survive. Set this instead, in +# local-config/vars.yml, to keep a different version. +astral_python_version: "3.14" diff --git a/ansible/roles/astral/tasks/main.yml b/ansible/roles/astral/tasks/main.yml index 80460a7..4dbc6dd 100644 --- a/ansible/roles/astral/tasks/main.yml +++ b/ansible/roles/astral/tasks/main.yml @@ -24,7 +24,7 @@ - name: Load macOS variables ansible.builtin.include_vars: macos.yml when: ansible_facts['distribution'] == 'MacOSX' - tags: ['astral', 'uv', 'ruff', 'ty'] + tags: ['astral', 'uv', 'ruff', 'ty', 'python'] # ============================================================================ # uv - must be first: ruff/ty on Ubuntu install THROUGH uv @@ -150,6 +150,74 @@ when: ansible_facts['distribution'] == 'Ubuntu' tags: ['astral', 'ruff', 'ty'] +# ============================================================================ +# Python - uv's default interpreter, without touching the system python3 +# ============================================================================ +# Where the system python3 is already new enough (Fedora 43+, Ubuntu 26.04) uv +# uses it, and Fedora's packaged uv prefers it regardless. Elsewhere (Ubuntu +# 24.04, macOS) uv installs its own build, adding only a versioned python3.14 to +# ~/.local/bin. No `--default`: it also puts python3 there, ahead of the system +# python3 that distro tools and Ansible run on. + +# /usr/bin/python3 rather than ansible_facts['python']: the interpreter Ansible +# runs with is not necessarily the system one. +- name: Read the system python3 version + ansible.builtin.command: + argv: [/usr/bin/python3, -c, "import sys; print('%d.%d' % sys.version_info[:2])"] + register: astral_system_python + changed_when: false + failed_when: false + check_mode: false + tags: ['astral', 'uv', 'python'] + +# A download that cannot complete (offline, an upstream outage) is a warning, +# not a failed base converge. +- name: Set uv's default Python + become: "{{ ansible_facts['distribution'] != 'MacOSX' }}" + become_user: "{{ actual_user if ansible_facts['distribution'] != 'MacOSX' else omit }}" + environment: "{{ astral_uv_env }}" + tags: ['astral', 'uv', 'python'] + block: + - name: Install the default Python for uv + ansible.builtin.command: + argv: [uv, python, install, "{{ astral_python_version }}"] + register: astral_python_install + changed_when: "'already installed' not in (astral_python_install.stderr | default(''))" + when: >- + astral_system_python.rc != 0 + or not (astral_system_python.stdout | trim is version(astral_python_version, '>=')) + + # Read first because `uv python pin` reports "Pinned" whether or not it + # changed anything. With no global pin it exits 2 and stdout is empty. + - name: Read uv's global Python pin + ansible.builtin.command: + argv: [uv, python, pin, --global] + register: astral_python_pin + changed_when: false + failed_when: false + check_mode: false + + # Global, so it applies wherever no project names its own version, and it + # replaces any global pin the user set: astral_python_version is the override. + # Skipped when the install fails, since a pin to a missing Python breaks uv. + - name: Pin uv's global default Python + ansible.builtin.command: + argv: [uv, python, pin, --global, "{{ astral_python_version }}"] + when: astral_python_pin.stdout | trim != astral_python_version + changed_when: true + + rescue: + - name: Record that uv's default Python was not set + # noqa: var-naming[no-role-prefix] -- deploy_warnings is the shared + # accumulator reported by playbooks/main.yml post_tasks. + ansible.builtin.set_fact: + deploy_warnings: >- + {{ (deploy_warnings | default([]) + + ['uv python ' ~ astral_python_version ~ ': ' + ~ ((ansible_failed_result.stderr | default('', true) + or ansible_failed_result.msg | default('failed')) | trim | truncate(120))]) + | unique }} + - name: Display the Astral suite info ansible.builtin.debug: msg: | diff --git a/ansible/roles/astral/vars/main.yml b/ansible/roles/astral/vars/main.yml new file mode 100644 index 0000000..e990341 --- /dev/null +++ b/ansible/roles/astral/vars/main.yml @@ -0,0 +1,7 @@ +--- +# Where uv is on PATH for the target user: brew's prefix on macOS, the +# user-owned ~/.local/bin on Ubuntu, and /usr/bin (already on PATH) on Fedora. +astral_uv_env: + PATH: >- + {{ homebrew_env.PATH if ansible_facts['distribution'] == 'MacOSX' + else user_home ~ '/.local/bin:' ~ ansible_facts['env'].PATH }} diff --git a/ansible/roles/developer-rust/files/hyperi-rust-cache-prune b/ansible/roles/developer-rust/files/hyperi-rust-cache-prune index 2b2f016..852e873 100644 --- a/ansible/roles/developer-rust/files/hyperi-rust-cache-prune +++ b/ansible/roles/developer-rust/files/hyperi-rust-cache-prune @@ -90,8 +90,16 @@ hunting. It does not touch the cargo registry or the sccache/ccache stores either. Those have their own eviction and are reported on only. + +Python +------ +Stdlib only, and runs on the oldest python3 a supported OS ships: 3.9 on macOS, +where launchd resolves `python3` to the Command Line Tools' /usr/bin/python3. """ +# Keeps `int | str` annotations unevaluated, which 3.9 cannot evaluate. +from __future__ import annotations + import argparse import fcntl import os @@ -360,12 +368,57 @@ def cargo_config() -> dict: config = Path(os.environ.get("CARGO_HOME") or Path.home() / ".cargo") / "config.toml" if not config.is_file(): return {} + try: + text = config.read_text(encoding="utf-8") + except OSError: + return {} try: import tomllib + except ImportError: + return {"build": build_table_without_tomllib(text)} + try: + return tomllib.loads(text) + except ValueError: + return {} - return tomllib.loads(config.read_text(encoding="utf-8")) - except (OSError, ValueError): + +_BUILD_HEADER = re.compile(r"^\s*\[\s*build\s*\]\s*(?:#.*)?$") +_STRING_KEY = re.compile(r"""^\s*([A-Za-z0-9_-]+)\s*=\s*(?:"([^"\\]*)"|'([^']*)')\s*(?:#.*)?$""") + + +def build_table_without_tomllib(text: str) -> dict[str, str]: + """The plain string keys of cargo's `[build]` table, for a Python with no tomllib. + + tomllib arrived in 3.11 and macOS ships 3.9. This reads only what the tool + needs, `build-dir` and `rustc-wrapper`, in the `key = "value"` form + hyperi-rust-setup writes. A value it cannot read is left out, which reads as + unconfigured rather than as a wrong path to delete under. + + A file this line reader could misread returns nothing at all: a multi-line + string can carry a `[build]` line that is not a header, and a repeated key + or table is invalid TOML that cargo itself rejects. + """ + if "'''" in text or '"""' in text: return {} + table: dict[str, str] = {} + in_build = False + seen_build = False + for line in text.splitlines(): + if line.lstrip().startswith("["): + in_build = _BUILD_HEADER.match(line) is not None + if in_build and seen_build: + return {} + seen_build = seen_build or in_build + continue + if not in_build: + continue + pair = _STRING_KEY.match(line) + if pair: + if pair.group(1) in table: + return {} + value = pair.group(2) if pair.group(2) is not None else pair.group(3) + table[pair.group(1)] = value + return table def configured_wrapper() -> Path | None: diff --git a/ansible/roles/developer-rust/files/hyperi-rust-setup b/ansible/roles/developer-rust/files/hyperi-rust-setup index a7c2124..6d0de74 100644 --- a/ansible/roles/developer-rust/files/hyperi-rust-setup +++ b/ansible/roles/developer-rust/files/hyperi-rust-setup @@ -64,12 +64,27 @@ import shutil import subprocess import sys import tempfile -import tomllib import urllib.request from datetime import datetime, timezone from pathlib import Path from typing import NamedTuple +# tomllib is 3.11+, and `python3` on macOS is the Command Line Tools' 3.9. Hand +# over to the astral role's uv-managed Python when there is one. +try: + import tomllib +except ModuleNotFoundError: + _UV_PYTHON = os.path.expanduser("~/.local/bin/python3.14") + if os.access(_UV_PYTHON, os.X_OK): + os.execv(_UV_PYTHON, [_UV_PYTHON, *sys.argv]) + print( + f"hyperi-rust-setup: needs Python 3.11 or later, and this is " + f"{platform.python_version()}. Run it with a newer python3, or install " + "one with `uv python install 3.14`.", + file=sys.stderr, + ) + sys.exit(2) + MANAGED_HEADER = "# MANAGED BY hyperi-rust-setup" MANAGED_ENV_BEGIN = "# BEGIN hyperi-rust-setup cache caps" MANAGED_ENV_END = "# END hyperi-rust-setup cache caps" diff --git a/ansible/roles/developer-rust/meta/main.yml b/ansible/roles/developer-rust/meta/main.yml new file mode 100644 index 0000000..7a0173b --- /dev/null +++ b/ansible/roles/developer-rust/meta/main.yml @@ -0,0 +1,9 @@ +--- +# Developer-Rust -- the Rust tier. +# +# Depends on astral so its uv-managed Python 3.14 exists before hyperi-rust-setup +# runs, including on `--tags developer-rust` alone and in personas that list +# this role ahead of the base. + +dependencies: + - role: astral diff --git a/ansible/roles/developer-rust/tasks/rust.yml b/ansible/roles/developer-rust/tasks/rust.yml index 6eaebcb..e1a9828 100644 --- a/ansible/roles/developer-rust/tasks/rust.yml +++ b/ansible/roles/developer-rust/tasks/rust.yml @@ -776,11 +776,24 @@ when: ansible_facts['distribution'] == 'MacOSX' tags: ['rust-cache'] +# The script needs Python 3.11+ for tomllib, and macOS's /usr/bin/python3 is +# 3.9. The astral role's uv-managed python3.14 is used wherever it exists, on +# every OS, and python3 otherwise. +- name: Look for the uv-managed Python + ansible.builtin.stat: + path: "{{ user_home }}/.local/bin/python3.14" + follow: true + register: developer_rust_uv_python + tags: ['rust-cache'] + # --yes because Ansible is the unattended caller; a human running it by hand # gets the confirmation prompt instead. - name: Configure the Rust build environment ansible.builtin.command: cmd: >- + {{ developer_rust_uv_python.stat.path + if developer_rust_uv_python.stat.exists and developer_rust_uv_python.stat.executable + else 'python3' }} /usr/local/bin/hyperi-rust-setup --yes --sccache-size {{ rust_cache_sccache_max }} --ccache-size {{ rust_cache_ccache_max }} @@ -798,9 +811,27 @@ failed_when: false tags: ['rust-cache'] +# stderr as well: a missing cargo and a Python traceback both land there. - name: Report Rust build environment setup ansible.builtin.debug: - msg: "{{ developer_rust_setup.stdout_lines | default(['(no output)']) }}" + msg: >- + {{ (developer_rust_setup.stdout_lines | default([])) + + (developer_rust_setup.stderr_lines | default([])) or ['(no output)'] }} + when: developer_rust_setup.rc | default(0) != 0 + tags: ['rust-cache'] + +# The script exits 0 on warnings, so non-zero is a real failure. It goes in the +# end-of-run summary rather than aborting, for the reason above. +- name: Warn if hyperi-rust-setup failed + # noqa: var-naming[no-role-prefix] -- deploy_warnings is the play-wide list + # the playbook's post_tasks report from. + ansible.builtin.set_fact: + deploy_warnings: >- + {{ (deploy_warnings | default([]) + + ['hyperi-rust-setup: exited ' ~ developer_rust_setup.rc | string ~ ' -- ' + ~ ((developer_rust_setup.stderr_lines | default([], true) | last | default('', true) + or developer_rust_setup.msg | default('no output')) | trim | truncate(120))]) + | unique }} when: developer_rust_setup.rc | default(0) != 0 tags: ['rust-cache'] diff --git a/ansible/roles/developer/tasks/utilities.yml b/ansible/roles/developer/tasks/utilities.yml index 5b8107c..77d4b9e 100644 --- a/ansible/roles/developer/tasks/utilities.yml +++ b/ansible/roles/developer/tasks/utilities.yml @@ -77,12 +77,10 @@ when: ansible_facts['distribution'] == 'Ubuntu' # macOS utility list intentionally omits `ansible` and `python@3`: -# - ansible: installed via uv as a tool (`uv tool install ansible-core -# --with ansible`) so it lives in its own isolated env and tracks uv's -# Python. A brew ansible would shadow it and create two ansibles on PATH. -# - python@3: brew's keg-managed Python is unnecessary — uv-managed Python -# versions (via `uv python install`) are the corporate standard and -# self-contained per-tool. +# - ansible: install.sh runs the playbook from a throwaway venv, so nothing +# here installs a persistent one. +# - python@3: the astral role installs a uv-managed Python and pins it as uv's +# global default. The CLT /usr/bin/python3 stays as the system interpreter. - name: Install CLI utilities (macOS) community.general.homebrew: name: diff --git a/docs/install-matrix.md b/docs/install-matrix.md index c7e77aa..8bfebd9 100644 --- a/docs/install-matrix.md +++ b/docs/install-matrix.md @@ -168,6 +168,7 @@ reach for instead, are not in that manifest and have no published digest at all. | Tool(s) | Platforms | Method | |---|---|---| | astral suite: uv, ruff, ty (uv bundles `uv audit` + `uv check`) | all | Fedora dnf / macOS brew; Ubuntu has no apt package (see Auto-update) | +| Python 3.14 for uv (`astral_python_version`), pinned as uv's global default; system `python3` untouched | all | system python3 where it is 3.14+ (Fedora, Ubuntu 26.04), else `uv python install`; `uv python pin --global` per user | | CLI utils (jq, gron, bat, fzf, ripgrep, fd, git-delta, moreutils, miller, rsync, tmux, htop, wget, shellcheck, age, parallel, ...) | all | distro repo / brew | | sd | all | distro (apt/dnf) / brew | | yq (mikefarah; apt `yq` is kislyuk/yq, a different tool) | all | Fedora dnf / Ubuntu re-fetch (Tier 3) / brew | diff --git a/tools/check_release_matrix.py b/tools/check_release_matrix.py old mode 100644 new mode 100755 index 5ca0de2..329d775 --- a/tools/check_release_matrix.py +++ b/tools/check_release_matrix.py @@ -22,8 +22,6 @@ checked. """ -from __future__ import annotations - import argparse import re import sys diff --git a/tools/check_role_file_refs.py b/tools/check_role_file_refs.py index ba4dc2e..f61b21b 100644 --- a/tools/check_role_file_refs.py +++ b/tools/check_role_file_refs.py @@ -13,8 +13,6 @@ Run: python3 tools/check_role_file_refs.py """ -from __future__ import annotations - import pathlib import re import sys diff --git a/tools/hyperi-doctor b/tools/hyperi-doctor index c417a68..bf3d52d 100755 --- a/tools/hyperi-doctor +++ b/tools/hyperi-doctor @@ -9,5 +9,23 @@ set -euo pipefail -command -v python3 >/dev/null 2>&1 || { echo "hyperi-doctor: python3 is required and was not found on PATH" >&2; exit 1; } -exec python3 "$(dirname "$0")/hyperi_doctor.py" "$@" +script="$(dirname "$0")/hyperi_doctor.py" + +# The same rule on every OS: the first interpreter that is 3.11+ (datetime.UTC) +# and has PyYAML. The astral role's uv-managed python3.14 comes first, but it +# carries no PyYAML, so the distro python3 is what passes on Linux. +for py in "$HOME/.local/bin/python3.14" python3; do + if command -v "$py" >/dev/null 2>&1 \ + && "$py" -c 'import sys, yaml; sys.exit(sys.version_info < (3, 11))' >/dev/null 2>&1; then + exec "$py" "$script" "$@" + fi +done + +# macOS has no interpreter that passes: the CLT python3 is 3.9 and nothing +# installs PyYAML. uv supplies both, fetching PyYAML once into its cache. +if command -v uv >/dev/null 2>&1; then + exec uv run --quiet --no-project --python 3.14 --with pyyaml "$script" "$@" +fi + +echo "hyperi-doctor: needs Python 3.11+ with PyYAML, or uv, and found neither" >&2 +exit 1 diff --git a/tools/hyperi_doctor.py b/tools/hyperi_doctor.py old mode 100644 new mode 100755 index c4b1ded..647282e --- a/tools/hyperi_doctor.py +++ b/tools/hyperi_doctor.py @@ -13,8 +13,6 @@ Licensed under the Apache License, Version 2.0 """ -from __future__ import annotations - import argparse import json import os diff --git a/tools/tests/test_check_role_file_refs.py b/tools/tests/test_check_role_file_refs.py index 9ffbf1b..2584750 100644 --- a/tools/tests/test_check_role_file_refs.py +++ b/tools/tests/test_check_role_file_refs.py @@ -4,8 +4,6 @@ tree that is wrong in one specific way and asserts the checker says so. """ -from __future__ import annotations - import importlib.util import pathlib import sys diff --git a/tools/tests/test_hyperi_doctor.py b/tools/tests/test_hyperi_doctor.py index 7bba422..fb6676a 100644 --- a/tools/tests/test_hyperi_doctor.py +++ b/tools/tests/test_hyperi_doctor.py @@ -6,8 +6,6 @@ dpkg database. """ -from __future__ import annotations - import sys from pathlib import Path diff --git a/tools/tests/test_rust_cache_prune.py b/tools/tests/test_rust_cache_prune.py index 6e3bcbc..4de2c8c 100644 --- a/tools/tests/test_rust_cache_prune.py +++ b/tools/tests/test_rust_cache_prune.py @@ -89,6 +89,55 @@ def test_malformed_config_does_not_raise(prune, tmp_path, monkeypatch): assert prune.configured_pool() is None +SETUP_SHAPED_CONFIG = """\ +# Managed by hyperi-rust-setup. +[build] +rustc-wrapper = "/usr/local/bin/sccache" +# Intermediate artefacts only; final binaries stay in the project's target/. +build-dir = '/home/someone/.cache/pool/{workspace-path-hash}' # trailing comment + +[target.x86_64-unknown-linux-gnu] +linker = "/usr/bin/clang" +rustflags = [ + "-C", "link-arg=-fuse-ld=/usr/bin/mold", +] + +[env] +build-dir = "/not/the/build/table" +""" + + +def test_the_fallback_reads_the_build_table_as_tomllib_does(prune): + """macOS ships Python 3.9, which has no tomllib, and launchd runs the tool there.""" + import tomllib + + expected = tomllib.loads(SETUP_SHAPED_CONFIG)["build"] + assert prune.build_table_without_tomllib(SETUP_SHAPED_CONFIG) == expected + + +@pytest.mark.parametrize( + "text", + [ + # A multi-line string whose content looks like a [build] table. + "[env]\nNOTE = '''\n[build]\nbuild-dir = \"/home/u/.cache/victim\"\n'''\n", + '[build]\nbuild-dir = """/home/u/.cache/x"""\n', + # Invalid TOML that a line reader would otherwise resolve to one value. + '[build]\nbuild-dir = "/home/u/.cache/a"\nbuild-dir = "/home/u/.cache/b"\n', + '[build]\nbuild-dir = "/home/u/.cache/a"\n[env]\nX = "1"\n[build]\njobs = "2"\n', + ], +) +def test_the_fallback_reads_nothing_from_a_file_it_could_misread(prune, text): + assert prune.build_table_without_tomllib(text) == {} + + +def test_a_python_without_tomllib_still_finds_the_pool(prune, tmp_path, monkeypatch): + cargo_home = write_cargo_config(tmp_path, SETUP_SHAPED_CONFIG) + monkeypatch.setenv("CARGO_HOME", str(cargo_home)) + monkeypatch.setitem(sys.modules, "tomllib", None) + assert prune.configured_pool() == Path("/home/someone/.cache/pool") + assert prune.configured_wrapper() == Path("/usr/local/bin/sccache") + + def test_auto_size_is_a_share_of_the_disk_with_a_floor(prune, tmp_path, monkeypatch): """`auto` derives from the filesystem total, and never drops below the floor.""" monkeypatch.setenv("HOME", str(tmp_path)) diff --git a/tools/tests/test_rust_setup_interpreter.py b/tools/tests/test_rust_setup_interpreter.py new file mode 100644 index 0000000..e83fb33 --- /dev/null +++ b/tools/tests/test_rust_setup_interpreter.py @@ -0,0 +1,65 @@ +"""Tests for hyperi-rust-setup's hand-over on a Python without tomllib. + +macOS's python3 is 3.9, which has no tomllib. The script is run here with +tomllib hidden, which is the same import failure. +""" + +import os +import subprocess +import sys +from pathlib import Path + +SCRIPT = ( + Path(__file__).resolve().parents[2] + / "ansible/roles/developer-rust/files/hyperi-rust-setup" +) + +RUN_WITHOUT_TOMLLIB = ( + "import runpy, sys; sys.modules['tomllib'] = None; " + "sys.argv = sys.argv[1:]; runpy.run_path(sys.argv[0], run_name='__main__')" +) + + +def run_without_tomllib(home: Path) -> subprocess.CompletedProcess: + env = {**os.environ, "HOME": str(home)} + return subprocess.run( + [sys.executable, "-c", RUN_WITHOUT_TOMLLIB, str(SCRIPT), "--check"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + env=env, + check=False, + ) + + +def test_it_hands_over_to_the_uv_python_with_the_same_arguments(tmp_path): + bin_dir = tmp_path / ".local" / "bin" + bin_dir.mkdir(parents=True) + fake = bin_dir / "python3.14" + fake.write_text('#!/bin/sh\necho "handed over: $*"\n', encoding="utf-8") + fake.chmod(0o755) + + result = run_without_tomllib(tmp_path) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == f"handed over: {SCRIPT} --check" + + +def test_it_names_the_python_it_needs_when_there_is_none(tmp_path): + result = run_without_tomllib(tmp_path) + + assert result.returncode == 2 + assert "needs Python 3.11 or later" in result.stderr + assert "Traceback" not in result.stderr + + +def test_a_dangling_uv_python_is_not_handed_over_to(tmp_path): + bin_dir = tmp_path / ".local" / "bin" + bin_dir.mkdir(parents=True) + (bin_dir / "python3.14").symlink_to(tmp_path / "gone" / "python3.14") + + result = run_without_tomllib(tmp_path) + + assert result.returncode == 2 + assert "needs Python 3.11 or later" in result.stderr