Download PETSc from Zenodo, with the DataverseNL mirror as the fallback

.github/copilot-instructions.md

@@ -42,7 +42,7 @@ Sister modules in the ecosystem: ARAGOG (interior thermal evolution, entropy for
 
 1. A C compiler (gcc or clang) and GNU make.
 2. PETSc with SUNDIALS2 (`--download-sundials2`). Two supported paths:
-   - Installer: `./tools/get_petsc.sh` (downloads the PETSc 3.19.0 source archive from OSF and builds it; sets `arch-linux-c-opt` / `arch-darwin-c-opt`).
+   - Installer: `./tools/get_petsc.sh` (downloads the PETSc 3.19.0 source archive from Zenodo, or its DataverseNL mirror, checks its SHA-256 and builds it; sets `arch-linux-c-opt` / `arch-darwin-c-opt`).
    - From source: clone `https://gitlab.com/petsc/petsc.git`, configure with `--with-fc=0 --with-cxx=0 --download-sundials2 --download-mpich --download-f2cblaslapack`.
 3. Python 3.12+ with `pip install -r py/requirements.txt` plus `pytest pytest-timeout` for the test suite.
 

tests/test_get_petsc.py

@@ -0,0 +1,98 @@
+"""Tests for the verified PETSc download in ``tools/get_petsc.sh``.
+
+The function and the pins are read from the shipped script, so a change to the
+script re-runs these cases against its new text. Sources are ``file://`` URLs:
+no network and no build.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+import subprocess
+from pathlib import Path
+
+import pytest
+
+pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)]
+
+SCRIPT = Path(__file__).resolve().parents[1] / 'tools' / 'get_petsc.sh'
+
+
+def _function() -> str:
+    """Return the shipped fetch_verified definition with the console helpers it calls."""
+    text = SCRIPT.read_text()
+    helpers = re.search(r'^console\(\) \{.*?^\}\n\nannounce\(\) \{.*?^\}\n', text, re.S | re.M)
+    body = re.search(r'^fetch_verified\(\) \{.*?^\}\n', text, re.S | re.M)
+    return 'set -euo pipefail\nexec 3>&1\n' + helpers.group(0) + body.group(0)
+
+
+def _run(body: str) -> subprocess.CompletedProcess:
+    """Run ``body`` after the shipped function definitions."""
+    return subprocess.run(['bash', '-c', _function() + body], capture_output=True, text=True)
+
+
+def _sha256(data: bytes) -> str:
+    """Return the SHA-256 hex digest of ``data``."""
+    return hashlib.sha256(data).hexdigest()
+
+
+def test_the_first_source_with_the_pinned_hash_wins(tmp_path):
+    """A failed download, an empty mirror and a wrong file each give way to the next source,
+    so only an archive with the pinned SHA-256 reaches unzip and the PETSc build."""
+    good, bad = tmp_path / 'good.zip', tmp_path / 'bad.zip'
+    good.write_bytes(b'petsc archive')
+    bad.write_bytes(b'an error page')
+    dest = tmp_path / 'petsc.zip'
+    result = _run(
+        f'fetch_verified {_sha256(b"petsc archive")} "{dest}" '
+        f'"file://{tmp_path}/missing.zip" "" "file://{bad}" "file://{good}"\necho REACHED_UNZIP\n'
+    )
+    assert result.returncode == 0, result.stderr
+    assert dest.read_bytes() == b'petsc archive'
+    assert 'REACHED_UNZIP' in result.stdout
+    assert f'download from file://{tmp_path}/missing.zip failed' in result.stdout
+    assert f'served a file with SHA-256 {_sha256(b"an error page")}' in result.stdout
+
+
+def test_no_matching_source_stops_before_unzip(tmp_path):
+    """With no source serving the pinned file the script stops and leaves no archive."""
+    bad = tmp_path / 'bad.zip'
+    bad.write_bytes(b'an error page')
+    dest = tmp_path / 'petsc.zip'
+    result = _run(f'fetch_verified {"0" * 64} "{dest}" "file://{bad}" ""\necho REACHED_UNZIP\n')
+    assert result.returncode == 1
+    assert 'REACHED_UNZIP' not in result.stdout
+    assert not dest.exists()
+    assert f'no source served petsc.zip with SHA-256 {"0" * 64}' in result.stdout
+
+
+def test_an_empty_hash_or_no_hash_tool_stops_before_any_download(tmp_path):
+    """A blank pin, or no shasum and no sha256sum, stops at once instead of accepting whatever a
+    source serves; the shipped script pins the Zenodo archive, its mirror and a 64-digit hash."""
+    good = tmp_path / 'good.zip'
+    good.write_bytes(b'petsc archive')
+    call = f'fetch_verified "{{}}" "{tmp_path}/p.zip" "file://{good}"\necho REACHED_UNZIP\n'
+    blank = _run(call.format(''))
+    no_tool = subprocess.run(
+        ['/bin/bash', '-c', _function() + call.format(_sha256(b'petsc archive'))],
+        capture_output=True,
+        text=True,
+        env={'PATH': str(tmp_path / 'empty')},
+    )
+    assert blank.returncode == 1 and 'no SHA-256 pin for p.zip' in blank.stdout
+    assert no_tool.returncode == 1
+    assert 'neither shasum nor sha256sum is installed' in no_tool.stdout
+    for result in (blank, no_tool):
+        assert 'Downloading' not in result.stdout and 'REACHED_UNZIP' not in result.stdout
+    text = SCRIPT.read_text()
+    assert (
+        'url="${PETSC_URL:-https://zenodo.org/records/15805756/files/petsc.zip?download=1}"'
+        in text
+    )
+    assert (
+        'mirror_url="${PETSC_MIRROR_URL:-https://dataverse.nl/api/access/datafile/683669}"'
+        in text
+    )
+    assert re.search(r'^petsc_sha256="[0-9a-f]{64}"$', text, re.M)
+    assert 'fetch_verified "$petsc_sha256" "$zipfile" "$url" "$mirror_url"' in text

tools/get_petsc.sh

@@ -3,7 +3,8 @@
 # get_petsc.sh — Download, configure, and compile PETSc for SPIDER
 # =============================================================================
 #
-# Downloads PETSc 3.19.0 from OSF and builds it with sundials2 support.
+# Downloads the PETSc 3.19.0 source archive from Zenodo, or its DataverseNL
+# mirror, checks its SHA-256, and builds it with sundials2 support.
 # SPIDER is a pure C code, so C++ and Fortran compilers are disabled.
 #
 # This script is intended to live inside the SPIDER repository:
@@ -118,11 +119,46 @@ safe_to_remove_dir() {
     return 0
 }
 
+# -----------------------------------------------------------------------------
+# Verified download: the first source that serves the pinned SHA-256 wins
+# -----------------------------------------------------------------------------
+fetch_verified() {
+    local sha256="$1" dest="$2" url got
+    shift 2
+    if [[ -z "$sha256" ]]; then
+        console "ERROR: no SHA-256 pin for $(basename "$dest")"
+        return 1
+    fi
+    if ! command -v shasum >/dev/null 2>&1 && ! command -v sha256sum >/dev/null 2>&1; then
+        console "ERROR: neither shasum nor sha256sum is installed; install one of them"
+        return 1
+    fi
+    for url in "$@"; do
+        [[ -n "$url" ]] || continue
+        announce "Downloading $url"
+        if curl -fLsS --retry 3 --connect-timeout 30 --speed-limit 1024 --speed-time 60 \
+            "$url" -o "$dest"; then
+            got=$( (shasum -a 256 "$dest" 2>/dev/null || sha256sum "$dest") | awk '{print $1}')
+            [[ "$got" == "$sha256" ]] && return 0
+            console "WARNING: $url served a file with SHA-256 $got, not $sha256"
+        else
+            console "WARNING: download from $url failed"
+        fi
+    done
+    rm -f "$dest"
+    console "ERROR: no source served $(basename "$dest") with SHA-256 $sha256"
+    return 1
+}
+
 # -----------------------------------------------------------------------------
 # Error handling: report which step failed on any non-zero exit
 # -----------------------------------------------------------------------------
 current_step="initialising"
-url="https://osf.io/download/p5vxq/"
+# PETSc 3.19.0 source archive on Zenodo (15805756) and its DataverseNL mirror (10.34894/BZDJPU);
+# PETSC_URL and PETSC_MIRROR_URL override them.
+url="${PETSC_URL:-https://zenodo.org/records/15805756/files/petsc.zip?download=1}"
+mirror_url="${PETSC_MIRROR_URL:-https://dataverse.nl/api/access/datafile/683669}"
+petsc_sha256="c5bdb75048b609627bac7fdc83042078a629f5de0c6508b50166a351d2aa045d"
 logfile=""
 
 on_error() {
@@ -153,8 +189,8 @@ on_error() {
             ;;
         *"Download"*)
             console "   - Check your internet connection"
-            console "   - Verify the OSF URL is accessible: $url"
-            console "   - Try downloading manually: curl -fLsS \"$url\" -o petsc.zip"
+            console "   - Sources tried: $url ${mirror_url:-(no mirror)}"
+            console "   - PETSC_URL and PETSC_MIRROR_URL override them; the SHA-256 must match"
             ;;
         *"Decompress"*)
             console "   - The downloaded archive may be corrupted"
@@ -267,15 +303,13 @@ fi
 mkdir -p "$workpath"
 
 # -----------------------------------------------------------------------------
-# 6. Download PETSc 3.19.0 from OSF
+# 6. Download the PETSc 3.19.0 archive: Zenodo first, then the DataverseNL mirror
 # -----------------------------------------------------------------------------
-current_step="Downloading PETSc archive from OSF"
+current_step="Downloading PETSc archive"
 
 zipfile="$workpath/petsc.zip"
 announce ""
-announce "Downloading PETSc archive from OSF..."
-announce "    $url -> $zipfile"
-curl -fLsS "$url" -o "$zipfile"
+fetch_verified "$petsc_sha256" "$zipfile" "$url" "$mirror_url"
 
 current_step="Decompressing PETSc archive"
 announce "Decompressing..."