From 050d9d9fbf77fe1030936372b9e7d4c673e033ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20K=C5=82ysz?= Date: Mon, 2 Nov 2026 13:14:22 +0000 Subject: [PATCH 1/3] chore: fall back to the toolchain mirror for public downloads setup.sh and the compiler build scripts try lombyte-project/toolchain-mirror (pinned to a commit) when an upstream download fails: binutils, objdiff, the EE GCC source, gcc-2.95.2, bison 1.28 and the GCC 2.95.2 headers. Same SHA-256 pins as before. --- docs/build/building.md | 3 +++ scripts/build-game-compiler.py | 9 +++++++++ scripts/build-sn-compiler.py | 1 + setup.sh | 33 +++++++++++++++++++++++++-------- 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/docs/build/building.md b/docs/build/building.md index bab55b7f8c..5935d76a76 100644 --- a/docs/build/building.md +++ b/docs/build/building.md @@ -30,6 +30,9 @@ Other options: - Downloads the public toolchain into `tools/` and checks every file against a pinned SHA-256. The layout is in [tools/README.md](../../tools/README.md). + If an upstream link is gone, it takes the same file from + [toolchain-mirror](https://github.com/lombyte-project/toolchain-mirror), + which also explains why the toolchain looks the way it does. - Builds the game compiler from its GPL source and our [patches](../../patches/sce-991111b/README.md). - Creates `.venv` with the Python packages from `requirements.txt`. diff --git a/scripts/build-game-compiler.py b/scripts/build-game-compiler.py index 1539dd8b1c..b5a53d8d11 100755 --- a/scripts/build-game-compiler.py +++ b/scripts/build-game-compiler.py @@ -36,9 +36,17 @@ PATCH_DIR = ROOT / "patches" / "sce-991111b" ARCHIVE_NAME = "gnu-ee-binutils-gcc-1.1.tar.gz" +# Our copy of every download (github.com/lombyte-project/toolchain-mirror), +# pinned to a commit; tried after the upstream URLs. +MIRROR = ( + "https://raw.githubusercontent.com/lombyte-project/toolchain-mirror/" + "51d9a4db241f3f7fe5e8ae137561245964f333c1" +) + ARCHIVE_URLS = ( "https://web.archive.org/web/20060518234647id_/" "http://ps2dev.sourceforge.net:80/downloads/ee/gnu-ee-binutils-gcc-1.1.tar.gz", + f"{MIRROR}/gnu-ee-binutils-gcc-1.1.tar.gz", ) ARCHIVE_SHA256 = "1f518043e252d6eda726386971d52eda26541ab936ea73a9783d73712b595f92" ARCHIVE_TOP = "gnu-ee-binutils-gcc" @@ -59,6 +67,7 @@ "https://ftp.gnu.org/gnu/bison/bison-1.28.tar.gz", "https://mirrors.kernel.org/gnu/bison/bison-1.28.tar.gz", "https://ftpmirror.gnu.org/gnu/bison/bison-1.28.tar.gz", + f"{MIRROR}/bison-1.28.tar.gz", ) BISON_SHA256 = "c5d3e4858e17cb440cee9de7837f07277bcfb03507e9d2f0c506cab5efe36c3a" diff --git a/scripts/build-sn-compiler.py b/scripts/build-sn-compiler.py index a03629c058..faccc91aa7 100644 --- a/scripts/build-sn-compiler.py +++ b/scripts/build-sn-compiler.py @@ -37,6 +37,7 @@ "https://ftp.gnu.org/gnu/gcc/gcc-2.95.2.tar.gz", "https://mirrors.kernel.org/gnu/gcc/gcc-2.95.2.tar.gz", "https://ftpmirror.gnu.org/gnu/gcc/gcc-2.95.2.tar.gz", + f"{game.MIRROR}/gcc-2.95.2.tar.gz", ) GCC_SHA256 = "064e1cb06ea5d2f4a07ec46c1c64d771f74d04f404b6a6766bca2477f7d72482" GCC_TOP = "gcc-2.95.2" diff --git a/setup.sh b/setup.sh index 801decdb34..269fe9f79f 100755 --- a/setup.sh +++ b/setup.sh @@ -28,6 +28,9 @@ ELF_TARGET="$ROOT/config/us/SCUS_971.99" ELF_SHA=e050581032e4bb3f20341307da5b69b76f1574910519155380ea771e55c3c0c9 # Pinned public downloads (SHA-256 checked before anything is installed). +# A dead upstream link falls back to our copy of the open-source files +# (github.com/lombyte-project/toolchain-mirror), pinned to a commit. +MIRROR_URL=https://raw.githubusercontent.com/lombyte-project/toolchain-mirror/51d9a4db241f3f7fe5e8ae137561245964f333c1 BINUTILS_URL=https://github.com/decompals/binutils-mips-ps2-decompals/releases/download/v0.10/binutils-mips-ps2-decompals-linux-x86-64.tar.gz BINUTILS_SHA=9fe31ea3ee1a37536f9f0e2e12c668e7c3cb99e59f5154bd2f1fc4762473094c OBJDIFF_URL=https://github.com/encounter/objdiff/releases/download/v3.8.0/objdiff-cli-linux-x86_64 @@ -51,6 +54,8 @@ SDK_MIRROR_COMMIT=5c8bdf31f6bdd82ba4456413594198b5bd469f43 SDK_MIRROR_INCLUDE=local/sce/ee/gcc/lib/gcc-lib/ee/2.95.2/include SDK_MIRROR_STDARG_SHA=84945e1a49c2c9aeee025705c53d94f1b91999bd30e2cafa639d800cad852966 SDK_MIRROR_VA_MIPS_SHA=af46841480f84b4620ca62c3f955cce20aac166928db5cdf17109ed71de41ce6 +SDK_HEADERS_FILE=gcc-2.95.2-ee-include.tar.xz +SDK_HEADERS_SHA=237d1e6ba8bf9c40272f235ad062cba5ebe3d9677895b64efbdc6942a72860b6 DOCKER_IMAGE=lombyte-dev DOCKER_BASE=ubuntu:24.04 @@ -119,7 +124,7 @@ replace_dir() { # target staged mv -- "$staged" "$target" } -download() { # url sha path +download() { # url sha path [mirror-file] local url=$1 sha=$2 path=$3 mkdir -p "$(dirname -- "$path")" if [[ -f "$path" ]] && sha_ok "$path" "$sha"; then @@ -127,8 +132,12 @@ download() { # url sha path fi say "downloading $(basename -- "$path")" rm -f -- "$path" - curl --fail --location --retry 3 --retry-delay 2 --show-error --progress-bar \ - "$url" --output "$path.part" + if ! curl --fail --location --retry 3 --retry-delay 2 --show-error --progress-bar \ + "$url" --output "$path.part" && [[ -n "${4:-}" ]]; then + warn "$url failed; trying the toolchain mirror" + curl --fail --location --retry 3 --retry-delay 2 --show-error --progress-bar \ + "$MIRROR_URL/$4" --output "$path.part" + fi if ! sha_ok "$path.part" "$sha"; then rm -f -- "$path.part" die "$(basename -- "$path") failed its SHA-256 check; the download is corrupt or the mirror changed" @@ -291,7 +300,7 @@ install_binutils() { local target="$TOOLS/binutils-mips-ps2-decompals" [[ -x "$target/mips-ps2-decompals-ld" ]] && return local archive="$DOWNLOADS/binutils-mips-ps2-decompals-v0.10.tar.gz" - download "$BINUTILS_URL" "$BINUTILS_SHA" "$archive" + download "$BINUTILS_URL" "$BINUTILS_SHA" "$archive" binutils-mips-ps2-decompals-v0.10.tar.gz rm -rf -- "$STAGE/binutils"; mkdir -p "$STAGE/binutils" tar -xzf "$archive" -C "$STAGE/binutils" chmod 0755 "$STAGE/binutils"/mips-ps2-decompals-* @@ -303,7 +312,7 @@ install_objdiff() { local target="$TOOLS/objdiff/objdiff-cli" [[ -x "$target" ]] && sha_ok "$target" "$OBJDIFF_SHA" && return local file="$DOWNLOADS/objdiff-cli-v3.8.0-linux-x86_64" - download "$OBJDIFF_URL" "$OBJDIFF_SHA" "$file" + download "$OBJDIFF_URL" "$OBJDIFF_SHA" "$file" objdiff-cli-v3.8.0-linux-x86_64 mkdir -p "$(dirname -- "$target")" install -m 0755 "$file" "$target" } @@ -342,10 +351,18 @@ install_sn_compiler() { if [[ ! -d "$mirror/.git" ]]; then say "fetching GCC 2.95.2 headers from the SDK mirror" rm -rf -- "$mirror" - git clone --quiet --filter=blob:none --no-checkout "$SDK_MIRROR_URL" "$mirror" + if ! git clone --quiet --filter=blob:none --no-checkout "$SDK_MIRROR_URL" "$mirror"; then + # Same folder at the same commit, from the toolchain mirror. + warn "$SDK_MIRROR_URL failed; trying the toolchain mirror" + rm -rf -- "$mirror"; mkdir -p "$mirror" + download "$MIRROR_URL/$SDK_HEADERS_FILE" "$SDK_HEADERS_SHA" "$DOWNLOADS/$SDK_HEADERS_FILE" + tar -xJf "$DOWNLOADS/$SDK_HEADERS_FILE" -C "$mirror" + fi + fi + if [[ -d "$mirror/.git" ]]; then + git -C "$mirror" sparse-checkout set --no-cone "$SDK_MIRROR_INCLUDE/*" + git -C "$mirror" checkout --quiet --detach "$SDK_MIRROR_COMMIT" fi - git -C "$mirror" sparse-checkout set --no-cone "$SDK_MIRROR_INCLUDE/*" - git -C "$mirror" checkout --quiet --detach "$SDK_MIRROR_COMMIT" assert_sha "$mirror/$SDK_MIRROR_INCLUDE/stdarg.h" "$SDK_MIRROR_STDARG_SHA" assert_sha "$mirror/$SDK_MIRROR_INCLUDE/va-mips.h" "$SDK_MIRROR_VA_MIPS_SHA" mkdir -p "$(dirname -- "$STAGE/sn/lib/gcc-lib/ee/2.95.2/include")" From b074b133e966bad649dac66c11c57cca9f88111d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20K=C5=82ysz?= Date: Mon, 2 Nov 2026 13:17:59 +0000 Subject: [PATCH 2/3] chore: pin the toolchain mirror to its rewritten history --- scripts/build-game-compiler.py | 2 +- setup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build-game-compiler.py b/scripts/build-game-compiler.py index b5a53d8d11..b0a55ccd07 100755 --- a/scripts/build-game-compiler.py +++ b/scripts/build-game-compiler.py @@ -40,7 +40,7 @@ # pinned to a commit; tried after the upstream URLs. MIRROR = ( "https://raw.githubusercontent.com/lombyte-project/toolchain-mirror/" - "51d9a4db241f3f7fe5e8ae137561245964f333c1" + "915a96d32d1b23d58558b82e4b2ba7a4f2820f51" ) ARCHIVE_URLS = ( diff --git a/setup.sh b/setup.sh index 269fe9f79f..b1f17a04aa 100755 --- a/setup.sh +++ b/setup.sh @@ -30,7 +30,7 @@ ELF_SHA=e050581032e4bb3f20341307da5b69b76f1574910519155380ea771e55c3c0c9 # Pinned public downloads (SHA-256 checked before anything is installed). # A dead upstream link falls back to our copy of the open-source files # (github.com/lombyte-project/toolchain-mirror), pinned to a commit. -MIRROR_URL=https://raw.githubusercontent.com/lombyte-project/toolchain-mirror/51d9a4db241f3f7fe5e8ae137561245964f333c1 +MIRROR_URL=https://raw.githubusercontent.com/lombyte-project/toolchain-mirror/915a96d32d1b23d58558b82e4b2ba7a4f2820f51 BINUTILS_URL=https://github.com/decompals/binutils-mips-ps2-decompals/releases/download/v0.10/binutils-mips-ps2-decompals-linux-x86-64.tar.gz BINUTILS_SHA=9fe31ea3ee1a37536f9f0e2e12c668e7c3cb99e59f5154bd2f1fc4762473094c OBJDIFF_URL=https://github.com/encounter/objdiff/releases/download/v3.8.0/objdiff-cli-linux-x86_64 From f2b1d750e27aa8d5218eb3cf1f51ac903ab47fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20K=C5=82ysz?= Date: Mon, 2 Nov 2026 13:24:33 +0000 Subject: [PATCH 3/3] fix: clear errors when a download and its mirror both fail setup.sh stops with 'could not download' instead of a misleading hash error, and a failed sparse checkout of the SDK headers also falls back to the toolchain mirror. --- scripts/build-game-compiler.py | 2 +- setup.sh | 46 +++++++++++++++++++++------------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/scripts/build-game-compiler.py b/scripts/build-game-compiler.py index b0a55ccd07..e79540f981 100755 --- a/scripts/build-game-compiler.py +++ b/scripts/build-game-compiler.py @@ -36,7 +36,7 @@ PATCH_DIR = ROOT / "patches" / "sce-991111b" ARCHIVE_NAME = "gnu-ee-binutils-gcc-1.1.tar.gz" -# Our copy of every download (github.com/lombyte-project/toolchain-mirror), +# Our copy of the open-source downloads (github.com/lombyte-project/toolchain-mirror), # pinned to a commit; tried after the upstream URLs. MIRROR = ( "https://raw.githubusercontent.com/lombyte-project/toolchain-mirror/" diff --git a/setup.sh b/setup.sh index b1f17a04aa..448e217cef 100755 --- a/setup.sh +++ b/setup.sh @@ -124,6 +124,12 @@ replace_dir() { # target staged mv -- "$staged" "$target" } +fetch() { # url path + curl --fail --location --retry 3 --retry-delay 2 --show-error --progress-bar "$1" --output "$2" +} + +# The optional mirror-file is the file's name in the toolchain mirror, tried +# when the url fails. download() { # url sha path [mirror-file] local url=$1 sha=$2 path=$3 mkdir -p "$(dirname -- "$path")" @@ -132,11 +138,10 @@ download() { # url sha path [mirror-file] fi say "downloading $(basename -- "$path")" rm -f -- "$path" - if ! curl --fail --location --retry 3 --retry-delay 2 --show-error --progress-bar \ - "$url" --output "$path.part" && [[ -n "${4:-}" ]]; then + if ! fetch "$url" "$path.part"; then + [[ -n "${4:-}" ]] || die "could not download $url" warn "$url failed; trying the toolchain mirror" - curl --fail --location --retry 3 --retry-delay 2 --show-error --progress-bar \ - "$MIRROR_URL/$4" --output "$path.part" + fetch "$MIRROR_URL/$4" "$path.part" || die "could not download $(basename -- "$path") from the toolchain mirror either" fi if ! sha_ok "$path.part" "$sha"; then rm -f -- "$path.part" @@ -329,6 +334,19 @@ install_sdk_compiler() { replace_dir "$target" "$STAGE/sdk" } +# Sparse checkout of the pinned header folder; fails on any network error +# (the checkout fetches blobs lazily, so it is a network step too). +sdk_headers_from_git() { # dir + local mirror=$1 + if [[ ! -d "$mirror/.git" ]]; then + say "fetching GCC 2.95.2 headers from the SDK mirror" + rm -rf -- "$mirror" + git clone --quiet --filter=blob:none --no-checkout "$SDK_MIRROR_URL" "$mirror" || return 1 + fi + git -C "$mirror" sparse-checkout set --no-cone "$SDK_MIRROR_INCLUDE/*" && + git -C "$mirror" checkout --quiet --detach "$SDK_MIRROR_COMMIT" +} + install_sn_compiler() { local target="$COMPILERS/sn-compiler" local include="$target/lib/gcc-lib/ee/2.95.2/include" @@ -348,20 +366,12 @@ install_sn_compiler() { python3 "$ROOT/scripts/patch-ps2eeas.py" "$assembler" "$STAGE/sn/ee/bin/Ps2EeAs.exe" # GCC 2.95.2 headers (stdarg.h, ...) from the Sony SDK mirror, pinned. local mirror="$DOWNLOADS/sce_ps2_sdk_24" - if [[ ! -d "$mirror/.git" ]]; then - say "fetching GCC 2.95.2 headers from the SDK mirror" - rm -rf -- "$mirror" - if ! git clone --quiet --filter=blob:none --no-checkout "$SDK_MIRROR_URL" "$mirror"; then - # Same folder at the same commit, from the toolchain mirror. - warn "$SDK_MIRROR_URL failed; trying the toolchain mirror" - rm -rf -- "$mirror"; mkdir -p "$mirror" - download "$MIRROR_URL/$SDK_HEADERS_FILE" "$SDK_HEADERS_SHA" "$DOWNLOADS/$SDK_HEADERS_FILE" - tar -xJf "$DOWNLOADS/$SDK_HEADERS_FILE" -C "$mirror" - fi - fi - if [[ -d "$mirror/.git" ]]; then - git -C "$mirror" sparse-checkout set --no-cone "$SDK_MIRROR_INCLUDE/*" - git -C "$mirror" checkout --quiet --detach "$SDK_MIRROR_COMMIT" + if ! sdk_headers_from_git "$mirror"; then + # Same folder at the same commit, from the toolchain mirror. + warn "$SDK_MIRROR_URL failed; trying the toolchain mirror" + rm -rf -- "$mirror"; mkdir -p "$mirror" + download "$MIRROR_URL/$SDK_HEADERS_FILE" "$SDK_HEADERS_SHA" "$DOWNLOADS/$SDK_HEADERS_FILE" + tar -xJf "$DOWNLOADS/$SDK_HEADERS_FILE" -C "$mirror" fi assert_sha "$mirror/$SDK_MIRROR_INCLUDE/stdarg.h" "$SDK_MIRROR_STDARG_SHA" assert_sha "$mirror/$SDK_MIRROR_INCLUDE/va-mips.h" "$SDK_MIRROR_VA_MIPS_SHA"