diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 36d7e36b..1acc1c5a 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -42,7 +42,7 @@ Sister modules in the ecosystem: ARAGOG (interior thermal evolution, entropy for 1. A C compiler (gcc or clang) and GNU make. 2. PETSc with SUNDIALS2 (`--download-sundials2`). Two supported paths: - - Installer: `./tools/get_petsc.sh` (downloads the PETSc 3.19.0 source archive from OSF and builds it; sets `arch-linux-c-opt` / `arch-darwin-c-opt`). + - Installer: `./tools/get_petsc.sh` (downloads the PETSc 3.19.0 source archive from Zenodo, or its DataverseNL mirror, checks its SHA-256 and builds it; sets `arch-linux-c-opt` / `arch-darwin-c-opt`). - From source: clone `https://gitlab.com/petsc/petsc.git`, configure with `--with-fc=0 --with-cxx=0 --download-sundials2 --download-mpich --download-f2cblaslapack`. 3. Python 3.12+ with `pip install -r py/requirements.txt` plus `pytest pytest-timeout` for the test suite. diff --git a/tests/test_get_petsc.py b/tests/test_get_petsc.py new file mode 100644 index 00000000..1ca350bc --- /dev/null +++ b/tests/test_get_petsc.py @@ -0,0 +1,98 @@ +"""Tests for the verified PETSc download in ``tools/get_petsc.sh``. + +The function and the pins are read from the shipped script, so a change to the +script re-runs these cases against its new text. Sources are ``file://`` URLs: +no network and no build. +""" + +from __future__ import annotations + +import hashlib +import re +import subprocess +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] + +SCRIPT = Path(__file__).resolve().parents[1] / 'tools' / 'get_petsc.sh' + + +def _function() -> str: + """Return the shipped fetch_verified definition with the console helpers it calls.""" + text = SCRIPT.read_text() + helpers = re.search(r'^console\(\) \{.*?^\}\n\nannounce\(\) \{.*?^\}\n', text, re.S | re.M) + body = re.search(r'^fetch_verified\(\) \{.*?^\}\n', text, re.S | re.M) + return 'set -euo pipefail\nexec 3>&1\n' + helpers.group(0) + body.group(0) + + +def _run(body: str) -> subprocess.CompletedProcess: + """Run ``body`` after the shipped function definitions.""" + return subprocess.run(['bash', '-c', _function() + body], capture_output=True, text=True) + + +def _sha256(data: bytes) -> str: + """Return the SHA-256 hex digest of ``data``.""" + return hashlib.sha256(data).hexdigest() + + +def test_the_first_source_with_the_pinned_hash_wins(tmp_path): + """A failed download, an empty mirror and a wrong file each give way to the next source, + so only an archive with the pinned SHA-256 reaches unzip and the PETSc build.""" + good, bad = tmp_path / 'good.zip', tmp_path / 'bad.zip' + good.write_bytes(b'petsc archive') + bad.write_bytes(b'an error page') + dest = tmp_path / 'petsc.zip' + result = _run( + f'fetch_verified {_sha256(b"petsc archive")} "{dest}" ' + f'"file://{tmp_path}/missing.zip" "" "file://{bad}" "file://{good}"\necho REACHED_UNZIP\n' + ) + assert result.returncode == 0, result.stderr + assert dest.read_bytes() == b'petsc archive' + assert 'REACHED_UNZIP' in result.stdout + assert f'download from file://{tmp_path}/missing.zip failed' in result.stdout + assert f'served a file with SHA-256 {_sha256(b"an error page")}' in result.stdout + + +def test_no_matching_source_stops_before_unzip(tmp_path): + """With no source serving the pinned file the script stops and leaves no archive.""" + bad = tmp_path / 'bad.zip' + bad.write_bytes(b'an error page') + dest = tmp_path / 'petsc.zip' + result = _run(f'fetch_verified {"0" * 64} "{dest}" "file://{bad}" ""\necho REACHED_UNZIP\n') + assert result.returncode == 1 + assert 'REACHED_UNZIP' not in result.stdout + assert not dest.exists() + assert f'no source served petsc.zip with SHA-256 {"0" * 64}' in result.stdout + + +def test_an_empty_hash_or_no_hash_tool_stops_before_any_download(tmp_path): + """A blank pin, or no shasum and no sha256sum, stops at once instead of accepting whatever a + source serves; the shipped script pins the Zenodo archive, its mirror and a 64-digit hash.""" + good = tmp_path / 'good.zip' + good.write_bytes(b'petsc archive') + call = f'fetch_verified "{{}}" "{tmp_path}/p.zip" "file://{good}"\necho REACHED_UNZIP\n' + blank = _run(call.format('')) + no_tool = subprocess.run( + ['/bin/bash', '-c', _function() + call.format(_sha256(b'petsc archive'))], + capture_output=True, + text=True, + env={'PATH': str(tmp_path / 'empty')}, + ) + assert blank.returncode == 1 and 'no SHA-256 pin for p.zip' in blank.stdout + assert no_tool.returncode == 1 + assert 'neither shasum nor sha256sum is installed' in no_tool.stdout + for result in (blank, no_tool): + assert 'Downloading' not in result.stdout and 'REACHED_UNZIP' not in result.stdout + text = SCRIPT.read_text() + assert ( + 'url="${PETSC_URL:-https://zenodo.org/records/15805756/files/petsc.zip?download=1}"' + in text + ) + assert ( + 'mirror_url="${PETSC_MIRROR_URL:-https://dataverse.nl/api/access/datafile/683669}"' + in text + ) + assert re.search(r'^petsc_sha256="[0-9a-f]{64}"$', text, re.M) + assert 'fetch_verified "$petsc_sha256" "$zipfile" "$url" "$mirror_url"' in text diff --git a/tools/get_petsc.sh b/tools/get_petsc.sh index 512a714a..57063809 100755 --- a/tools/get_petsc.sh +++ b/tools/get_petsc.sh @@ -3,7 +3,8 @@ # get_petsc.sh — Download, configure, and compile PETSc for SPIDER # ============================================================================= # -# Downloads PETSc 3.19.0 from OSF and builds it with sundials2 support. +# Downloads the PETSc 3.19.0 source archive from Zenodo, or its DataverseNL +# mirror, checks its SHA-256, and builds it with sundials2 support. # SPIDER is a pure C code, so C++ and Fortran compilers are disabled. # # This script is intended to live inside the SPIDER repository: @@ -118,11 +119,46 @@ safe_to_remove_dir() { return 0 } +# ----------------------------------------------------------------------------- +# Verified download: the first source that serves the pinned SHA-256 wins +# ----------------------------------------------------------------------------- +fetch_verified() { + local sha256="$1" dest="$2" url got + shift 2 + if [[ -z "$sha256" ]]; then + console "ERROR: no SHA-256 pin for $(basename "$dest")" + return 1 + fi + if ! command -v shasum >/dev/null 2>&1 && ! command -v sha256sum >/dev/null 2>&1; then + console "ERROR: neither shasum nor sha256sum is installed; install one of them" + return 1 + fi + for url in "$@"; do + [[ -n "$url" ]] || continue + announce "Downloading $url" + if curl -fLsS --retry 3 --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + "$url" -o "$dest"; then + got=$( (shasum -a 256 "$dest" 2>/dev/null || sha256sum "$dest") | awk '{print $1}') + [[ "$got" == "$sha256" ]] && return 0 + console "WARNING: $url served a file with SHA-256 $got, not $sha256" + else + console "WARNING: download from $url failed" + fi + done + rm -f "$dest" + console "ERROR: no source served $(basename "$dest") with SHA-256 $sha256" + return 1 +} + # ----------------------------------------------------------------------------- # Error handling: report which step failed on any non-zero exit # ----------------------------------------------------------------------------- current_step="initialising" -url="https://osf.io/download/p5vxq/" +# PETSc 3.19.0 source archive on Zenodo (15805756) and its DataverseNL mirror (10.34894/BZDJPU); +# PETSC_URL and PETSC_MIRROR_URL override them. +url="${PETSC_URL:-https://zenodo.org/records/15805756/files/petsc.zip?download=1}" +mirror_url="${PETSC_MIRROR_URL:-https://dataverse.nl/api/access/datafile/683669}" +petsc_sha256="c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d" logfile="" on_error() { @@ -153,8 +189,8 @@ on_error() { ;; *"Download"*) console " - Check your internet connection" - console " - Verify the OSF URL is accessible: $url" - console " - Try downloading manually: curl -fLsS \"$url\" -o petsc.zip" + console " - Sources tried: $url ${mirror_url:-(no mirror)}" + console " - PETSC_URL and PETSC_MIRROR_URL override them; the SHA-256 must match" ;; *"Decompress"*) console " - The downloaded archive may be corrupted" @@ -267,15 +303,13 @@ fi mkdir -p "$workpath" # ----------------------------------------------------------------------------- -# 6. Download PETSc 3.19.0 from OSF +# 6. Download the PETSc 3.19.0 archive: Zenodo first, then the DataverseNL mirror # ----------------------------------------------------------------------------- -current_step="Downloading PETSc archive from OSF" +current_step="Downloading PETSc archive" zipfile="$workpath/petsc.zip" announce "" -announce "Downloading PETSc archive from OSF..." -announce " $url -> $zipfile" -curl -fLsS "$url" -o "$zipfile" +fetch_verified "$petsc_sha256" "$zipfile" "$url" "$mirror_url" current_step="Decompressing PETSc archive" announce "Decompressing..."