From ce1f759cb910edd700cbf602d5b01dd245bfefe8 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Tue, 6 Oct 2026 21:06:00 +0200 Subject: [PATCH 1/4] Download PETSc from Zenodo, with the DataverseNL mirror as the fallback get_petsc.sh fetches the PETSc 3.19.0 source archive from Zenodo 15805756 instead of OSF, falls back to the DataverseNL mirror when Zenodo fails, and accepts only a file with the pinned SHA-256. The mirror URL stays empty until the mirror is published; PETSC_URL and PETSC_MIRROR_URL override the sources. --- .github/copilot-instructions.md | 2 +- tests/test_get_petsc.py | 81 +++++++++++++++++++++++++++++++++ tools/get_petsc.sh | 43 +++++++++++++---- 3 files changed, 116 insertions(+), 10 deletions(-) create mode 100644 tests/test_get_petsc.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 36d7e36b..1acc1c5a 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -42,7 +42,7 @@ Sister modules in the ecosystem: ARAGOG (interior thermal evolution, entropy for 1. A C compiler (gcc or clang) and GNU make. 2. PETSc with SUNDIALS2 (`--download-sundials2`). Two supported paths: - - Installer: `./tools/get_petsc.sh` (downloads the PETSc 3.19.0 source archive from OSF and builds it; sets `arch-linux-c-opt` / `arch-darwin-c-opt`). + - Installer: `./tools/get_petsc.sh` (downloads the PETSc 3.19.0 source archive from Zenodo, or its DataverseNL mirror, checks its SHA-256 and builds it; sets `arch-linux-c-opt` / `arch-darwin-c-opt`). - From source: clone `https://gitlab.com/petsc/petsc.git`, configure with `--with-fc=0 --with-cxx=0 --download-sundials2 --download-mpich --download-f2cblaslapack`. 3. Python 3.12+ with `pip install -r py/requirements.txt` plus `pytest pytest-timeout` for the test suite. diff --git a/tests/test_get_petsc.py b/tests/test_get_petsc.py new file mode 100644 index 00000000..a3e532f0 --- /dev/null +++ b/tests/test_get_petsc.py @@ -0,0 +1,81 @@ +"""Tests for the verified PETSc download in ``tools/get_petsc.sh``. + +The function and the pins are read from the shipped script, so a change to the +script re-runs these cases against its new text. Sources are ``file://`` URLs: +no network and no build. +""" + +from __future__ import annotations + +import hashlib +import re +import subprocess +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] + +SCRIPT = Path(__file__).resolve().parents[1] / 'tools' / 'get_petsc.sh' + + +def _function() -> str: + """Return the shipped fetch_verified definition with the console helpers it calls.""" + text = SCRIPT.read_text() + helpers = re.search(r'^console\(\) \{.*?^\}\n\nannounce\(\) \{.*?^\}\n', text, re.S | re.M) + body = re.search(r'^fetch_verified\(\) \{.*?^\}\n', text, re.S | re.M) + return 'set -euo pipefail\nexec 3>&1\n' + helpers.group(0) + body.group(0) + + +def _run(body: str) -> subprocess.CompletedProcess: + """Run ``body`` after the shipped function definitions.""" + return subprocess.run(['bash', '-c', _function() + body], capture_output=True, text=True) + + +def _sha256(data: bytes) -> str: + """Return the SHA-256 hex digest of ``data``.""" + return hashlib.sha256(data).hexdigest() + + +def test_the_first_source_with_the_pinned_hash_wins(tmp_path): + """A failed download, an empty mirror and a wrong file each give way to the next source, + so only an archive with the pinned SHA-256 reaches unzip and the PETSc build.""" + good, bad = tmp_path / 'good.zip', tmp_path / 'bad.zip' + good.write_bytes(b'petsc archive') + bad.write_bytes(b'an error page') + dest = tmp_path / 'petsc.zip' + result = _run( + f'fetch_verified {_sha256(b"petsc archive")} "{dest}" ' + f'"file://{tmp_path}/missing.zip" "" "file://{bad}" "file://{good}"\necho REACHED_UNZIP\n' + ) + assert result.returncode == 0, result.stderr + assert dest.read_bytes() == b'petsc archive' + assert 'REACHED_UNZIP' in result.stdout + assert f'download from file://{tmp_path}/missing.zip failed' in result.stdout + assert f'served a file with SHA-256 {_sha256(b"an error page")}' in result.stdout + + +def test_no_matching_source_stops_before_unzip(tmp_path): + """With no source serving the pinned file the script stops and leaves no archive.""" + bad = tmp_path / 'bad.zip' + bad.write_bytes(b'an error page') + dest = tmp_path / 'petsc.zip' + result = _run(f'fetch_verified {"0" * 64} "{dest}" "file://{bad}" ""\necho REACHED_UNZIP\n') + assert result.returncode == 1 + assert 'REACHED_UNZIP' not in result.stdout + assert not dest.exists() + assert f'no source served petsc.zip with SHA-256 {"0" * 64}' in result.stdout + + +def test_the_script_pins_the_zenodo_archive_and_its_hash(): + """The default source is the Zenodo archive, with its SHA-256 pinned beside it and both + sources overridable for a mirror or a test.""" + text = SCRIPT.read_text() + assert ( + 'url="${PETSC_URL:-https://zenodo.org/records/15805756/files/petsc.zip?download=1}"' + in text + ) + assert 'mirror_url="${PETSC_MIRROR_URL:-}"' in text + (sha,) = re.findall(r'^petsc_sha256="([0-9a-f]{64})"$', text, re.M) + assert sha == 'c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d' + assert 'fetch_verified "$petsc_sha256" "$zipfile" "$url" "$mirror_url"' in text diff --git a/tools/get_petsc.sh b/tools/get_petsc.sh index fd9adffd..c20f5eb5 100755 --- a/tools/get_petsc.sh +++ b/tools/get_petsc.sh @@ -3,7 +3,8 @@ # get_petsc.sh — Download, configure, and compile PETSc for SPIDER # ============================================================================= # -# Downloads PETSc 3.19.0 from OSF and builds it with sundials2 support. +# Downloads the PETSc 3.19.0 source archive from Zenodo, or its DataverseNL +# mirror, checks its SHA-256, and builds it with sundials2 support. # SPIDER is a pure C code, so C++ and Fortran compilers are disabled. # # This script is intended to live inside the SPIDER repository: @@ -118,11 +119,37 @@ safe_to_remove_dir() { return 0 } +# ----------------------------------------------------------------------------- +# Verified download: the first source that serves the pinned SHA-256 wins +# ----------------------------------------------------------------------------- +fetch_verified() { + local sha256="$1" dest="$2" url got + shift 2 + for url in "$@"; do + [[ -n "$url" ]] || continue + announce "Downloading $url" + if curl -fLsS --retry 3 "$url" -o "$dest"; then + got=$( (shasum -a 256 "$dest" 2>/dev/null || sha256sum "$dest") | awk '{print $1}') + [[ "$got" == "$sha256" ]] && return 0 + console "WARNING: $url served a file with SHA-256 $got, not $sha256" + else + console "WARNING: download from $url failed" + fi + done + rm -f "$dest" + console "ERROR: no source served $(basename "$dest") with SHA-256 $sha256" + return 1 +} + # ----------------------------------------------------------------------------- # Error handling: report which step failed on any non-zero exit # ----------------------------------------------------------------------------- current_step="initialising" -url="https://osf.io/download/p5vxq/" +# PETSc 3.19.0 source archive on Zenodo (15805756) and its DataverseNL mirror (empty: none); +# PETSC_URL and PETSC_MIRROR_URL override them. +url="${PETSC_URL:-https://zenodo.org/records/15805756/files/petsc.zip?download=1}" +mirror_url="${PETSC_MIRROR_URL:-}" +petsc_sha256="c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d" logfile="" on_error() { @@ -153,8 +180,8 @@ on_error() { ;; *"Download"*) console " - Check your internet connection" - console " - Verify the OSF URL is accessible: $url" - console " - Try downloading manually: curl -fLsS \"$url\" -o petsc.zip" + console " - Sources tried: $url ${mirror_url:-(no mirror)}" + console " - PETSC_URL and PETSC_MIRROR_URL override them; the SHA-256 must match" ;; *"Decompress"*) console " - The downloaded archive may be corrupted" @@ -267,15 +294,13 @@ fi mkdir -p "$workpath" # ----------------------------------------------------------------------------- -# 6. Download PETSc 3.19.0 from OSF +# 6. Download the PETSc 3.19.0 archive: Zenodo first, then the DataverseNL mirror # ----------------------------------------------------------------------------- -current_step="Downloading PETSc archive from OSF" +current_step="Downloading PETSc archive" zipfile="$workpath/petsc.zip" announce "" -announce "Downloading PETSc archive from OSF..." -announce " $url -> $zipfile" -curl -fLsS "$url" -o "$zipfile" +fetch_verified "$petsc_sha256" "$zipfile" "$url" "$mirror_url" current_step="Decompressing PETSc archive" announce "Decompressing..." From a1cfe3635728715f4f53eaf0dcb12bb26eb16328 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Wed, 7 Oct 2026 06:23:09 +0200 Subject: [PATCH 2/4] Point the PETSc download at its DataverseNL mirror The PETSc archive is mirrored as DataverseNL 10.34894/BZDJPU (one file, petsc.zip, same bytes as Zenodo 15805756), so get_petsc.sh falls back to that file when the Zenodo download fails or its SHA-256 does not match. --- tests/test_get_petsc.py | 2 +- tools/get_petsc.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_get_petsc.py b/tests/test_get_petsc.py index a3e532f0..b5c83a81 100644 --- a/tests/test_get_petsc.py +++ b/tests/test_get_petsc.py @@ -75,7 +75,7 @@ def test_the_script_pins_the_zenodo_archive_and_its_hash(): 'url="${PETSC_URL:-https://zenodo.org/records/15805756/files/petsc.zip?download=1}"' in text ) - assert 'mirror_url="${PETSC_MIRROR_URL:-}"' in text + assert 'mirror_url="${PETSC_MIRROR_URL:-https://dataverse.nl/api/access/datafile/683669}"' in text (sha,) = re.findall(r'^petsc_sha256="([0-9a-f]{64})"$', text, re.M) assert sha == 'c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d' assert 'fetch_verified "$petsc_sha256" "$zipfile" "$url" "$mirror_url"' in text diff --git a/tools/get_petsc.sh b/tools/get_petsc.sh index e0b8bb18..62ca71b4 100755 --- a/tools/get_petsc.sh +++ b/tools/get_petsc.sh @@ -145,10 +145,10 @@ fetch_verified() { # Error handling: report which step failed on any non-zero exit # ----------------------------------------------------------------------------- current_step="initialising" -# PETSc 3.19.0 source archive on Zenodo (15805756) and its DataverseNL mirror (empty: none); +# PETSc 3.19.0 source archive on Zenodo (15805756) and its DataverseNL mirror (10.34894/BZDJPU); # PETSC_URL and PETSC_MIRROR_URL override them. url="${PETSC_URL:-https://zenodo.org/records/15805756/files/petsc.zip?download=1}" -mirror_url="${PETSC_MIRROR_URL:-}" +mirror_url="${PETSC_MIRROR_URL:-https://dataverse.nl/api/access/datafile/683669}" petsc_sha256="c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d" logfile="" From fb246ac65337f4fe821af9bce3613db4f2d7f738 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Wed, 7 Oct 2026 08:16:20 +0200 Subject: [PATCH 3/4] Stop on an empty PETSc hash and time out stalled downloads fetch_verified stops before any download when the SHA-256 pin is empty, and curl gives up on a connection or a stalled transfer so the next source is tried. --- tests/test_get_petsc.py | 15 ++++++++++++++- tools/get_petsc.sh | 7 ++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/test_get_petsc.py b/tests/test_get_petsc.py index b5c83a81..a3409c98 100644 --- a/tests/test_get_petsc.py +++ b/tests/test_get_petsc.py @@ -67,6 +67,16 @@ def test_no_matching_source_stops_before_unzip(tmp_path): assert f'no source served petsc.zip with SHA-256 {"0" * 64}' in result.stdout +def test_an_empty_hash_stops_before_any_download(tmp_path): + """A blank pin stops at once instead of accepting whatever a source serves.""" + good = tmp_path / 'good.zip' + good.write_bytes(b'petsc archive') + result = _run(f'fetch_verified "" "{tmp_path}/p.zip" "file://{good}"\necho REACHED_UNZIP\n') + assert result.returncode == 1 + assert 'no SHA-256 pin for p.zip' in result.stdout + assert 'Downloading' not in result.stdout and 'REACHED_UNZIP' not in result.stdout + + def test_the_script_pins_the_zenodo_archive_and_its_hash(): """The default source is the Zenodo archive, with its SHA-256 pinned beside it and both sources overridable for a mirror or a test.""" @@ -75,7 +85,10 @@ def test_the_script_pins_the_zenodo_archive_and_its_hash(): 'url="${PETSC_URL:-https://zenodo.org/records/15805756/files/petsc.zip?download=1}"' in text ) - assert 'mirror_url="${PETSC_MIRROR_URL:-https://dataverse.nl/api/access/datafile/683669}"' in text + assert ( + 'mirror_url="${PETSC_MIRROR_URL:-https://dataverse.nl/api/access/datafile/683669}"' + in text + ) (sha,) = re.findall(r'^petsc_sha256="([0-9a-f]{64})"$', text, re.M) assert sha == 'c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d' assert 'fetch_verified "$petsc_sha256" "$zipfile" "$url" "$mirror_url"' in text diff --git a/tools/get_petsc.sh b/tools/get_petsc.sh index 62ca71b4..caf4da07 100755 --- a/tools/get_petsc.sh +++ b/tools/get_petsc.sh @@ -125,10 +125,15 @@ safe_to_remove_dir() { fetch_verified() { local sha256="$1" dest="$2" url got shift 2 + if [[ -z "$sha256" ]]; then + console "ERROR: no SHA-256 pin for $(basename "$dest")" + return 1 + fi for url in "$@"; do [[ -n "$url" ]] || continue announce "Downloading $url" - if curl -fLsS --retry 3 "$url" -o "$dest"; then + if curl -fLsS --retry 3 --connect-timeout 30 --speed-limit 1024 --speed-time 60 \ + "$url" -o "$dest"; then got=$( (shasum -a 256 "$dest" 2>/dev/null || sha256sum "$dest") | awk '{print $1}') [[ "$got" == "$sha256" ]] && return 0 console "WARNING: $url served a file with SHA-256 $got, not $sha256" From a706b9d1e89f04f6070305d37cb99f05ee9a2be5 Mon Sep 17 00:00:00 2001 From: timlichtenberg Date: Wed, 7 Oct 2026 09:56:28 +0200 Subject: [PATCH 4/4] Stop the PETSc download when no hash tool is installed fetch_verified stops before any download, naming shasum and sha256sum, when neither is installed, instead of failing inside the hash step. The pin assertions are part of the test that stops on a blank pin or a missing hash tool. --- tests/test_get_petsc.py | 30 +++++++++++++++++------------- tools/get_petsc.sh | 4 ++++ 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/tests/test_get_petsc.py b/tests/test_get_petsc.py index a3409c98..1ca350bc 100644 --- a/tests/test_get_petsc.py +++ b/tests/test_get_petsc.py @@ -67,19 +67,24 @@ def test_no_matching_source_stops_before_unzip(tmp_path): assert f'no source served petsc.zip with SHA-256 {"0" * 64}' in result.stdout -def test_an_empty_hash_stops_before_any_download(tmp_path): - """A blank pin stops at once instead of accepting whatever a source serves.""" +def test_an_empty_hash_or_no_hash_tool_stops_before_any_download(tmp_path): + """A blank pin, or no shasum and no sha256sum, stops at once instead of accepting whatever a + source serves; the shipped script pins the Zenodo archive, its mirror and a 64-digit hash.""" good = tmp_path / 'good.zip' good.write_bytes(b'petsc archive') - result = _run(f'fetch_verified "" "{tmp_path}/p.zip" "file://{good}"\necho REACHED_UNZIP\n') - assert result.returncode == 1 - assert 'no SHA-256 pin for p.zip' in result.stdout - assert 'Downloading' not in result.stdout and 'REACHED_UNZIP' not in result.stdout - - -def test_the_script_pins_the_zenodo_archive_and_its_hash(): - """The default source is the Zenodo archive, with its SHA-256 pinned beside it and both - sources overridable for a mirror or a test.""" + call = f'fetch_verified "{{}}" "{tmp_path}/p.zip" "file://{good}"\necho REACHED_UNZIP\n' + blank = _run(call.format('')) + no_tool = subprocess.run( + ['/bin/bash', '-c', _function() + call.format(_sha256(b'petsc archive'))], + capture_output=True, + text=True, + env={'PATH': str(tmp_path / 'empty')}, + ) + assert blank.returncode == 1 and 'no SHA-256 pin for p.zip' in blank.stdout + assert no_tool.returncode == 1 + assert 'neither shasum nor sha256sum is installed' in no_tool.stdout + for result in (blank, no_tool): + assert 'Downloading' not in result.stdout and 'REACHED_UNZIP' not in result.stdout text = SCRIPT.read_text() assert ( 'url="${PETSC_URL:-https://zenodo.org/records/15805756/files/petsc.zip?download=1}"' @@ -89,6 +94,5 @@ def test_the_script_pins_the_zenodo_archive_and_its_hash(): 'mirror_url="${PETSC_MIRROR_URL:-https://dataverse.nl/api/access/datafile/683669}"' in text ) - (sha,) = re.findall(r'^petsc_sha256="([0-9a-f]{64})"$', text, re.M) - assert sha == 'c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d' + assert re.search(r'^petsc_sha256="[0-9a-f]{64}"$', text, re.M) assert 'fetch_verified "$petsc_sha256" "$zipfile" "$url" "$mirror_url"' in text diff --git a/tools/get_petsc.sh b/tools/get_petsc.sh index caf4da07..57063809 100755 --- a/tools/get_petsc.sh +++ b/tools/get_petsc.sh @@ -129,6 +129,10 @@ fetch_verified() { console "ERROR: no SHA-256 pin for $(basename "$dest")" return 1 fi + if ! command -v shasum >/dev/null 2>&1 && ! command -v sha256sum >/dev/null 2>&1; then + console "ERROR: neither shasum nor sha256sum is installed; install one of them" + return 1 + fi for url in "$@"; do [[ -n "$url" ]] || continue announce "Downloading $url"