chore: fall back to the toolchain mirror for public downloads

docs/build/building.md

@@ -30,6 +30,9 @@ Other options:
 
 - Downloads the public toolchain into `tools/` and checks every file against a
   pinned SHA-256. The layout is in [tools/README.md](../../tools/README.md).
+  If an upstream link is gone, it takes the same file from
+  [toolchain-mirror](https://github.com/lombyte-project/toolchain-mirror),
+  which also explains why the toolchain looks the way it does.
 - Builds the game compiler from its GPL source and our
   [patches](../../patches/sce-991111b/README.md).
 - Creates `.venv` with the Python packages from `requirements.txt`.

scripts/build-game-compiler.py

@@ -36,9 +36,17 @@
 PATCH_DIR = ROOT / "patches" / "sce-991111b"
 
 ARCHIVE_NAME = "gnu-ee-binutils-gcc-1.1.tar.gz"
+# Our copy of the open-source downloads (github.com/lombyte-project/toolchain-mirror),
+# pinned to a commit; tried after the upstream URLs.
+MIRROR = (
+    "https://raw.githubusercontent.com/lombyte-project/toolchain-mirror/"
+    "915a96d32d1b23d58558b82e4b2ba7a4f2820f51"
+)
+
 ARCHIVE_URLS = (
     "https://web.archive.org/web/20060518234647id_/"
     "http://ps2dev.sourceforge.net:80/downloads/ee/gnu-ee-binutils-gcc-1.1.tar.gz",
+    f"{MIRROR}/gnu-ee-binutils-gcc-1.1.tar.gz",
 )
 ARCHIVE_SHA256 = "1f518043e252d6eda726386971d52eda26541ab936ea73a9783d73712b595f92"
 ARCHIVE_TOP = "gnu-ee-binutils-gcc"
@@ -59,6 +67,7 @@
     "https://ftp.gnu.org/gnu/bison/bison-1.28.tar.gz",
     "https://mirrors.kernel.org/gnu/bison/bison-1.28.tar.gz",
     "https://ftpmirror.gnu.org/gnu/bison/bison-1.28.tar.gz",
+    f"{MIRROR}/bison-1.28.tar.gz",
 )
 BISON_SHA256 = "c5d3e4858e17cb440cee9de7837f07277bcfb03507e9d2f0c506cab5efe36c3a"
 

scripts/build-sn-compiler.py

@@ -37,6 +37,7 @@
     "https://ftp.gnu.org/gnu/gcc/gcc-2.95.2.tar.gz",
     "https://mirrors.kernel.org/gnu/gcc/gcc-2.95.2.tar.gz",
     "https://ftpmirror.gnu.org/gnu/gcc/gcc-2.95.2.tar.gz",
+    f"{game.MIRROR}/gcc-2.95.2.tar.gz",
 )
 GCC_SHA256 = "064e1cb06ea5d2f4a07ec46c1c64d771f74d04f404b6a6766bca2477f7d72482"
 GCC_TOP = "gcc-2.95.2"

setup.sh

@@ -28,6 +28,9 @@ ELF_TARGET="$ROOT/config/us/SCUS_971.99"
 ELF_SHA=e050581032e4bb3f20341307da5b69b76f1574910519155380ea771e55c3c0c9
 
 # Pinned public downloads (SHA-256 checked before anything is installed).
+# A dead upstream link falls back to our copy of the open-source files
+# (github.com/lombyte-project/toolchain-mirror), pinned to a commit.
+MIRROR_URL=https://raw.githubusercontent.com/lombyte-project/toolchain-mirror/915a96d32d1b23d58558b82e4b2ba7a4f2820f51
 BINUTILS_URL=https://github.com/decompals/binutils-mips-ps2-decompals/releases/download/v0.10/binutils-mips-ps2-decompals-linux-x86-64.tar.gz
 BINUTILS_SHA=9fe31ea3ee1a37536f9f0e2e12c668e7c3cb99e59f5154bd2f1fc4762473094c
 OBJDIFF_URL=https://github.com/encounter/objdiff/releases/download/v3.8.0/objdiff-cli-linux-x86_64
@@ -51,6 +54,8 @@ SDK_MIRROR_COMMIT=5c8bdf31f6bdd82ba4456413594198b5bd469f43
 SDK_MIRROR_INCLUDE=local/sce/ee/gcc/lib/gcc-lib/ee/2.95.2/include
 SDK_MIRROR_STDARG_SHA=84945e1a49c2c9aeee025705c53d94f1b91999bd30e2cafa639d800cad852966
 SDK_MIRROR_VA_MIPS_SHA=af46841480f84b4620ca62c3f955cce20aac166928db5cdf17109ed71de41ce6
+SDK_HEADERS_FILE=gcc-2.95.2-ee-include.tar.xz
+SDK_HEADERS_SHA=237d1e6ba8bf9c40272f235ad062cba5ebe3d9677895b64efbdc6942a72860b6
 
 DOCKER_IMAGE=lombyte-dev
 DOCKER_BASE=ubuntu:24.04
@@ -119,16 +124,25 @@ replace_dir() { # target staged
     mv -- "$staged" "$target"
 }
 
-download() { # url sha path
+fetch() { # url path
+    curl --fail --location --retry 3 --retry-delay 2 --show-error --progress-bar "$1" --output "$2"
+}
+
+# The optional mirror-file is the file's name in the toolchain mirror, tried
+# when the url fails.
+download() { # url sha path [mirror-file]
     local url=$1 sha=$2 path=$3
     mkdir -p "$(dirname -- "$path")"
     if [[ -f "$path" ]] && sha_ok "$path" "$sha"; then
         return
     fi
     say "downloading $(basename -- "$path")"
     rm -f -- "$path"
-    curl --fail --location --retry 3 --retry-delay 2 --show-error --progress-bar \
-        "$url" --output "$path.part"
+    if ! fetch "$url" "$path.part"; then
+        [[ -n "${4:-}" ]] || die "could not download $url"
+        warn "$url failed; trying the toolchain mirror"
+        fetch "$MIRROR_URL/$4" "$path.part" || die "could not download $(basename -- "$path") from the toolchain mirror either"
+    fi
     if ! sha_ok "$path.part" "$sha"; then
         rm -f -- "$path.part"
         die "$(basename -- "$path") failed its SHA-256 check; the download is corrupt or the mirror changed"
@@ -291,7 +305,7 @@ install_binutils() {
     local target="$TOOLS/binutils-mips-ps2-decompals"
     [[ -x "$target/mips-ps2-decompals-ld" ]] && return
     local archive="$DOWNLOADS/binutils-mips-ps2-decompals-v0.10.tar.gz"
-    download "$BINUTILS_URL" "$BINUTILS_SHA" "$archive"
+    download "$BINUTILS_URL" "$BINUTILS_SHA" "$archive" binutils-mips-ps2-decompals-v0.10.tar.gz
     rm -rf -- "$STAGE/binutils"; mkdir -p "$STAGE/binutils"
     tar -xzf "$archive" -C "$STAGE/binutils"
     chmod 0755 "$STAGE/binutils"/mips-ps2-decompals-*
@@ -303,7 +317,7 @@ install_objdiff() {
     local target="$TOOLS/objdiff/objdiff-cli"
     [[ -x "$target" ]] && sha_ok "$target" "$OBJDIFF_SHA" && return
     local file="$DOWNLOADS/objdiff-cli-v3.8.0-linux-x86_64"
-    download "$OBJDIFF_URL" "$OBJDIFF_SHA" "$file"
+    download "$OBJDIFF_URL" "$OBJDIFF_SHA" "$file" objdiff-cli-v3.8.0-linux-x86_64
     mkdir -p "$(dirname -- "$target")"
     install -m 0755 "$file" "$target"
 }
@@ -320,6 +334,19 @@ install_sdk_compiler() {
     replace_dir "$target" "$STAGE/sdk"
 }
 
+# Sparse checkout of the pinned header folder; fails on any network error
+# (the checkout fetches blobs lazily, so it is a network step too).
+sdk_headers_from_git() { # dir
+    local mirror=$1
+    if [[ ! -d "$mirror/.git" ]]; then
+        say "fetching GCC 2.95.2 headers from the SDK mirror"
+        rm -rf -- "$mirror"
+        git clone --quiet --filter=blob:none --no-checkout "$SDK_MIRROR_URL" "$mirror" || return 1
+    fi
+    git -C "$mirror" sparse-checkout set --no-cone "$SDK_MIRROR_INCLUDE/*" &&
+        git -C "$mirror" checkout --quiet --detach "$SDK_MIRROR_COMMIT"
+}
+
 install_sn_compiler() {
     local target="$COMPILERS/sn-compiler"
     local include="$target/lib/gcc-lib/ee/2.95.2/include"
@@ -339,13 +366,13 @@ install_sn_compiler() {
     python3 "$ROOT/scripts/patch-ps2eeas.py" "$assembler" "$STAGE/sn/ee/bin/Ps2EeAs.exe"
     # GCC 2.95.2 headers (stdarg.h, ...) from the Sony SDK mirror, pinned.
     local mirror="$DOWNLOADS/sce_ps2_sdk_24"
-    if [[ ! -d "$mirror/.git" ]]; then
-        say "fetching GCC 2.95.2 headers from the SDK mirror"
-        rm -rf -- "$mirror"
-        git clone --quiet --filter=blob:none --no-checkout "$SDK_MIRROR_URL" "$mirror"
+    if ! sdk_headers_from_git "$mirror"; then
+        # Same folder at the same commit, from the toolchain mirror.
+        warn "$SDK_MIRROR_URL failed; trying the toolchain mirror"
+        rm -rf -- "$mirror"; mkdir -p "$mirror"
+        download "$MIRROR_URL/$SDK_HEADERS_FILE" "$SDK_HEADERS_SHA" "$DOWNLOADS/$SDK_HEADERS_FILE"
+        tar -xJf "$DOWNLOADS/$SDK_HEADERS_FILE" -C "$mirror"
     fi
-    git -C "$mirror" sparse-checkout set --no-cone "$SDK_MIRROR_INCLUDE/*"
-    git -C "$mirror" checkout --quiet --detach "$SDK_MIRROR_COMMIT"
     assert_sha "$mirror/$SDK_MIRROR_INCLUDE/stdarg.h" "$SDK_MIRROR_STDARG_SHA"
     assert_sha "$mirror/$SDK_MIRROR_INCLUDE/va-mips.h" "$SDK_MIRROR_VA_MIPS_SHA"
     mkdir -p "$(dirname -- "$STAGE/sn/lib/gcc-lib/ee/2.95.2/include")"