diff --git a/.github/workflows/mirror-gisinternals-sdk.yml b/.github/workflows/mirror-gisinternals-sdk.yml new file mode 100644 index 000000000..83ea32a20 --- /dev/null +++ b/.github/workflows/mirror-gisinternals-sdk.yml @@ -0,0 +1,86 @@ +name: Mirror GisInternals SDK + +# Manual, on-demand job. download.gisinternals.com is a single-maintainer +# host that goes down often, which breaks the Windows build whenever its +# cache misses (see win/partials.psm1 Get-GdalSdkIsAvailable). This workflow +# downloads the GDAL SDK zip once and uploads it as an asset on a dedicated, +# non-production release tag so win/partials.psm1 can mirror it from this +# repo's GitHub releases first. It never touches the project's real +# releases. + +on: + workflow_dispatch: + inputs: + sdk-url: + description: "URL to download the SDK zip from" + type: string + required: false + default: "http://download.gisinternals.com/sdk/downloads/release-1930-x64-dev.zip" + sdk-zip-name: + description: "Asset file name to upload" + type: string + required: false + default: "release-1930-x64-dev.zip" + +permissions: + contents: write + +env: + MIRROR_TAG: deps-gisinternals-sdk + +jobs: + mirror: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5.0.1 + + - name: Download SDK zip + run: | + curl --location --fail --show-error \ + --connect-timeout 30 --retry 5 --retry-delay 10 --retry-all-errors \ + --output "${{ inputs.sdk-zip-name }}" \ + "${{ inputs.sdk-url }}" + + - name: Print SHA256 + id: checksum + run: | + sha256sum "${{ inputs.sdk-zip-name }}" + echo "sha256=$(sha256sum "${{ inputs.sdk-zip-name }}" | cut -d ' ' -f1)" >> "$GITHUB_OUTPUT" + + - name: Create or update mirror release + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + TAG="$MIRROR_TAG" + NOTES="Build dependency mirror. Not a project release. + + This release hosts a pinned copy of the GisInternals SDK zip used by + win/partials.psm1 so Windows CI does not depend on + download.gisinternals.com being reachable. See win/README.md for + details on refreshing this mirror. + + SHA256 of ${{ inputs.sdk-zip-name }}: ${{ steps.checksum.outputs.sha256 }}" + + if gh release view "$TAG" >/dev/null 2>&1; then + gh release upload "$TAG" "${{ inputs.sdk-zip-name }}" --clobber + gh release edit "$TAG" --notes "$NOTES" + else + gh release create "$TAG" "${{ inputs.sdk-zip-name }}" \ + --title "GisInternals SDK mirror" \ + --notes "$NOTES" \ + --prerelease \ + --latest=false + fi + + - name: Summary + run: | + { + echo "### GisInternals SDK mirror updated" + echo "" + echo "- Tag: \`$MIRROR_TAG\`" + echo "- Asset: \`${{ inputs.sdk-zip-name }}\`" + echo "- SHA256: \`${{ steps.checksum.outputs.sha256 }}\`" + echo "" + echo "Update \$script:SDK_ZIP_SHA256 in win/partials.psm1 with the SHA256 above to enable checksum verification." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index ba6e12ec8..a897cbe94 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -63,10 +63,28 @@ jobs: restore-keys: | vcpkg-${{ runner.os }}- - # Layer 2: Build outputs (PROJ, GDAL, VCPKG installed, SDK) + # Layer 2: GisInternals SDK zip. Cached on its own, keyed only on the + # SDK name, so it survives build-script changes that would otherwise + # bust the builds cache below and force a re-download from an + # unreliable host (download.gisinternals.com). We cache the pristine + # downloaded zip rather than the extracted build-win/sdk/release-1930-x64/ + # directory: install.ps1 rewrites absolute paths inside the extracted + # SDK, so a cached extraction would be tied to this runner's workspace + # layout. Re-extracting the zip on a cache hit is cheap. + - name: Cache GisInternals SDK zip + id: cache-sdk + uses: actions/cache@v4 + with: + path: | + build-win/downloads/release-1930-x64-dev.zip + key: gis-sdk-${{ runner.os }}-release-1930-x64-dev + + # Layer 3: Build outputs (PROJ, GDAL, VCPKG installed) # Keep this cache narrow enough that the separate VCPKG archive cache # is less likely to be evicted. Source clones and downloads are rebuilt # on demand, but the installed outputs/state stamps stay reusable. + # The SDK zip has its own cache above and is intentionally not listed + # here; the extracted SDK dir is rebuilt from it on every run. - name: Cache build outputs id: cache-builds uses: actions/cache@v4 @@ -76,7 +94,6 @@ jobs: build-win/gdal-build/ build-win/proj-build/ build-win/gdal-cmake-temp/swig/csharp/ - build-win/sdk/release-1930-x64/ build-win/vcpkg/ !build-win/vcpkg/buildtrees/ !build-win/vcpkg/downloads/ @@ -85,7 +102,7 @@ jobs: # by prefix mixes them with a different vcpkg baseline. key: builds-${{ runner.os }}-${{ hashFiles('shared/swig.lock', 'shared/GdalCore.opt', 'win/install.ps1', 'win/functions.psm1', 'win/partials.psm1', 'win/vcpkg-makefile.vc', 'shared/vcpkg.json', 'shared/vcpkg-configuration.json', 'shared/vcpkg-lock.json') }} - # Layer 3: .NET SDK packages + # Layer 4: .NET SDK packages - name: Cache .NET packages uses: actions/cache@v4 with: diff --git a/win/README.md b/win/README.md index fc384e085..96c0c2920 100644 --- a/win/README.md +++ b/win/README.md @@ -15,6 +15,8 @@ In this folder contains Powershell and NMake scripts for building Windows runtim 1. [Visual Studio Build Tools (with ATL)](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools&rel=16). Recommended versions - vs16.0(2019) and vs17.0(2022) or greater (**nmake** and to retarget **libpng** to v143 toolset) + The scripts locate it with `vswhere.exe`, which the Visual Studio Installer places in `%ProgramFiles(x86)%\Microsoft Visual Studio\Installer`. No PowerShell Gallery modules are needed. + 2. [.NET Core SDK](https://dotnet.microsoft.com/en-us/download/dotnet/7.0) and [Nuget.exe](https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools) - for building and publishing packages respectively. The Windows build uses the same shared VCPKG manifest authority as Unix and macOS: @@ -44,6 +46,18 @@ This will install all required VCPKG packages and tools, build GDAL and PROJ, an On CI, cache-warm runs restore the VCPKG archive cache separately from the Windows build-output cache. When the manifest/build inputs match, the build-state stamps allow the workflow to reuse cached VCPKG, PROJ, and GDAL outputs instead of recompiling them from scratch. +#### GisInternals SDK mirror + +`download.gisinternals.com` is a single-maintainer host that is frequently unreachable, which breaks Windows CI whenever the SDK cache misses. `win/partials.psm1` first tries to download `release-1930-x64-dev.zip` from a mirror release on this repo (`deps-gisinternals-sdk`) and only falls back to GisInternals if that fails. Both URLs are configurable via the `SDK_MIRROR_URL` and `SDK_URL` environment variables. Windows CI also caches the pristine downloaded zip under its own cache key (`gis-sdk--release-1930-x64-dev`) so build-script changes do not force a re-download. + +To refresh the mirror, either run the `Mirror GisInternals SDK` workflow (`.github/workflows/mirror-gisinternals-sdk.yml`, manual `workflow_dispatch`) or, from a machine that already has the zip: + +``` +gh release upload deps-gisinternals-sdk release-1930-x64-dev.zip --clobber +``` + +After uploading, set `$script:SDK_ZIP_SHA256` in `win/partials.psm1` to the zip's SHA256 so the build can verify it. Until that value is set away from its placeholder, checksum verification is skipped with a warning. + 2. Call `./test.ps1` to test runtime and core packages.
If everything runs smoothly, you can use a local nuget feed to include packages in your project. diff --git a/win/functions.psm1 b/win/functions.psm1 index 10f57119d..f1f40fb94 100644 --- a/win/functions.psm1 +++ b/win/functions.psm1 @@ -118,7 +118,32 @@ function Set-ReplaceContentInFilesByRegex { function Get-VisualStudioVars { Write-BuildStep "Setting Visual Studio Environment" - Import-VisualStudioVars -Architecture $env:CMAKE_ARCHITECTURE + # vswhere.exe ships with the Visual Studio Installer (2017 and later) at a fixed path. + $vswhere = Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe" + if (!(Test-Path -Path $vswhere -PathType Leaf)) { + throw "vswhere.exe was not found at '$vswhere'. Install Visual Studio or Build Tools with the C++ workload." + } + $vsPath = & $vswhere -latest -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath | + Select-Object -First 1 + if (!$vsPath) { + throw "No Visual Studio installation with the C++ x86/x64 build tools was found." + } + $vcvarsall = Join-Path $vsPath "VC\Auxiliary\Build\vcvarsall.bat" + + # Run vcvarsall in cmd and import the environment it leaves behind into this session. + $marker = "__GDAL_NETCORE_VCVARS_ENV__" + [string[]] $output = @(& cmd.exe /c " `"$vcvarsall`" $env:CMAKE_ARCHITECTURE && echo $marker&& set ") + $exitCode = $LASTEXITCODE + $markerIndex = [Array]::IndexOf($output, $marker) + if ($exitCode -ne 0 -or $markerIndex -lt 0) { + $output | Write-Host + throw "vcvarsall.bat $env:CMAKE_ARCHITECTURE failed (exit code $exitCode)." + } + foreach ($line in ($output | Select-Object -Skip ($markerIndex + 1))) { + if ($line -match '^([^=]+)=(.*)$') { + Set-Item -Path "env:$($Matches[1])" -Value $Matches[2] + } + } Write-BuildStep "Visual Studio Environment was initialized" } @@ -156,30 +181,10 @@ function Install-PwshModuleRequirements { Write-Information "CMake was installed" } - if (!(Get-PackageProvider -Name "NuGet" -Force -ErrorAction SilentlyContinue)) { - Import-PackageProvider NuGet -Scope CurrentUser - } - - if (!(Get-PSRepository -Name "PSGallery" -ErrorAction SilentlyContinue)) { - Set-PSRepository -Name PSGallery -InstallationPolicy Trusted - } + # No PowerShell Gallery modules are required: Visual Studio is located with + # vswhere.exe (Get-VisualStudioVars) and 7-Zip is unpacked with 7zr.exe + # (Get-7ZipInstallation), so the build does not depend on PSGallery uptime. - if (!(Get-Module -Name "VSSetup" -ErrorAction SilentlyContinue)) { - Install-Module -Name VSSetup -RequiredVersion 2.2.5 -Scope CurrentUser -Force - } - - if (!(Get-Module -Name "Pscx" -ErrorAction SilentlyContinue)) { - Install-Module -Name Pscx -AllowClobber -AllowPrerelease -Scope CurrentUser -Force - } - - if (!(Get-Command "Invoke-WebRequest" -ErrorAction SilentlyContinue)) { - Install-Module -Name WebKitDev -RequiredVersion 0.4.0 -Force -Scope CurrentUser - } - - if (!(Get-Command "choco" -ErrorAction SilentlyContinue)) { - Install-Module -Name Choco -Scope CurrentUser -Force - } - $swigLockPath = Join-Path $PSScriptRoot "../shared/swig.lock" $swigLock = (Get-Content -Raw $swigLockPath).Trim() -split '\s+' if ($swigLock.Count -ne 2 -or $swigLock[0] -notmatch '^\d+\.\d+\.\d+$' -or $swigLock[1] -notmatch '^[0-9a-f]{64}$') { diff --git a/win/partials.psm1 b/win/partials.psm1 index cd87902df..587d60b74 100644 --- a/win/partials.psm1 +++ b/win/partials.psm1 @@ -13,7 +13,16 @@ function Set-GdalVariables { $env:VS_VERSION = "Visual Studio 17 2022" $env:SDK = "release-1930-x64" #2022 x64 $env:SDK_ZIP = "$env:SDK" + "-dev.zip" - $env:SDK_URL = "http://download.gisinternals.com/sdk/downloads/$env:SDK_ZIP" + # GisInternals is an unreliable, single-maintainer host that is frequently + # unreachable. We mirror the exact zip on this repo's GitHub releases and + # try that first; GisInternals stays as a fallback. Both are configurable + # via env vars so CI or a local build can override them. + if (-Not $env:SDK_MIRROR_URL) { + $env:SDK_MIRROR_URL = "https://github.com/MaxRev-Dev/gdal.netcore/releases/download/deps-gisinternals-sdk/$env:SDK_ZIP" + } + if (-Not $env:SDK_URL) { + $env:SDK_URL = "http://download.gisinternals.com/sdk/downloads/$env:SDK_ZIP" + } $env:LIBWEBP_URL = "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.3.2-windows-x64.zip" $env:LIBZSTD_URL = "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-v1.5.5-win64.zip" $env:LIBDEFLATE_URL = "https://github.com/ebiggers/libdeflate/releases/download/v1.19/libdeflate-1.19-windows-x86_64-bin.zip" @@ -134,8 +143,10 @@ function Get-7ZipInstallation { New-FolderIfNotExists $env:7Z_ROOT $env:7Z_URL = "https://www.7-zip.org/a/7z2501-extra.7z" if (-Not (Test-Path -Path "$env:7Z_ROOT\7za.exe" -PathType Leaf)) { + # 7zr.exe is the standalone 7z-only extractor; it unpacks the 7za console build. + Invoke-WebRequest "https://www.7-zip.org/a/7zr.exe" -OutFile "$env:7Z_ROOT\7zr.exe" Invoke-WebRequest "$env:7Z_URL" -OutFile "$env:7Z_ROOT\7z.7z" - Expand-PscxArchive -Path "$env:7Z_ROOT\7z.7z" -OutputPath "$env:7Z_ROOT\" + exec { & "$env:7Z_ROOT\7zr.exe" x "$env:7Z_ROOT\7z.7z" "-o$env:7Z_ROOT" -y } Write-BuildStep "Installed 7z into $env:7Z_ROOT" } else { @@ -144,15 +155,78 @@ function Get-7ZipInstallation { $env:PATH += ";$env:7Z_ROOT" } +# Pinned SHA256 of $env:SDK_ZIP (release-1930-x64-dev.zip). Set this once the +# mirror release has the real asset uploaded (see win/README.md). While this +# stays the placeholder below, verification is skipped with a warning instead +# of failing the build. +$script:SDK_ZIP_SHA256 = "TODO-PIN-SHA256-AFTER-FIRST-MIRROR-UPLOAD" + +function Get-SdkZipFrom { + param( + [Parameter(Mandatory)] [string] $Url, + [Parameter(Mandatory)] [string] $OutFile, + [string] $Label = $Url + ) + $maxAttempts = 3 + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { + try { + Invoke-WebRequest $Url -OutFile $OutFile -TimeoutSec 60 -MaximumRetryCount 2 -RetryIntervalSec 5 + return $true + } + catch { + Write-BuildWarn "Attempt $attempt/$maxAttempts to download SDK from $Label failed: $($_.Exception.Message)" + if (Test-Path -Path $OutFile -PathType Leaf) { + Remove-Item -Path $OutFile -Force -ErrorAction SilentlyContinue + } + if ($attempt -lt $maxAttempts) { + Start-Sleep -Seconds 5 + } + } + } + return $false +} + +function Test-SdkZipChecksum { + param([Parameter(Mandatory)] [string] $ZipPath) + + if ($script:SDK_ZIP_SHA256 -eq "TODO-PIN-SHA256-AFTER-FIRST-MIRROR-UPLOAD") { + Write-BuildWarn "SDK_ZIP_SHA256 is still the placeholder; skipping checksum verification" + return + } + + $actual = (Get-FileHash -Path $ZipPath -Algorithm SHA256).Hash + if ($actual -ne $script:SDK_ZIP_SHA256) { + Write-BuildError "SDK zip checksum mismatch: expected $($script:SDK_ZIP_SHA256), got $actual" + Remove-Item -Path $ZipPath -Force -ErrorAction SilentlyContinue + throw "GDAL SDK checksum verification failed for $ZipPath" + } + Write-BuildStep "GDAL SDK checksum verified" +} + function Get-GdalSdkIsAvailable { New-FolderIfNotExistsAndSetCurrentLocation $env:DOWNLOADS_DIR - Write-BuildStep "Checking for GDAL SDK from GisInternals" - if (-Not (Test-Path -Path $env:SDK_ZIP -PathType Leaf)) { - Write-BuildInfo "Downloading GDAL SDK from GisInternals" - Invoke-WebRequest "$env:SDK_URL" -OutFile "$env:SDK_ZIP" + Write-BuildStep "Checking for GDAL SDK" + if (-Not (Test-Path -Path $env:SDK_ZIP -PathType Leaf)) { + Write-BuildInfo "Downloading GDAL SDK from mirror: $env:SDK_MIRROR_URL" + $downloaded = Get-SdkZipFrom -Url $env:SDK_MIRROR_URL -OutFile $env:SDK_ZIP -Label "mirror" + if (-Not $downloaded) { + Write-BuildWarn "Mirror download failed, falling back to GisInternals: $env:SDK_URL" + $downloaded = Get-SdkZipFrom -Url $env:SDK_URL -OutFile $env:SDK_ZIP -Label "GisInternals" + } + if (-Not $downloaded) { + Write-BuildError "Failed to download GDAL SDK from both mirror and GisInternals" + throw "Unable to download GDAL SDK ($env:SDK_ZIP)" + } Write-BuildStep "GDAL SDK was downloaded" } - New-FolderIfNotExistsAndSetCurrentLocation "$env:BUILD_ROOT\sdk" + + # The zip may have just been downloaded, restored from cache, or left behind + # by a prior run. Verify it every time it is about to be used, and remove it + # on mismatch so the next run re-downloads instead of extracting a corrupt + # or tampered archive. + Test-SdkZipChecksum -ZipPath "$env:DOWNLOADS_DIR\$env:SDK_ZIP" + + New-FolderIfNotExistsAndSetCurrentLocation "$env:BUILD_ROOT\sdk" if (-Not (Test-Path -Path "$env:BUILD_ROOT\sdk\release-*" -PathType Container)) { exec { 7za x "$env:DOWNLOADS_DIR\$env:SDK_ZIP" -aoa }