diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70165d9..58ef96c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python-version: ["3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..34ddd4b --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,129 @@ +name: Wheels + +on: + workflow_dispatch: + push: + # branches: ['**'] looks redundant with no branches actually excluded, + # but it's load-bearing: adding `tags:` below with no `branches:` key + # at all silently suppresses ALL branch-push triggering (confirmed by + # this exact push not triggering a run), not just narrows it -- this + # restores the "any branch, filtered by paths" behavior alongside it. + branches: + - '**' + paths: + - pyproject.toml + - meson.build + - 'tools/build_*.sh' + - 'tools/cibw_*.sh' + - .github/workflows/wheels.yml + tags: + # Release tags (see RELEASING.md's `git tag vX.Y.Z`) trigger the real + # PyPI publish job below. Safe to combine with `paths:` above even + # though a tag push touches none of those files: GitHub doesn't apply + # `paths` filtering to tag pushes at all (only to branch pushes), so + # this fires unconditionally on any v*-matching tag regardless. + - 'v*' + +concurrency: + group: wheels-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: build (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # macos-14 = Apple Silicon (arm64) -- spelled out explicitly rather + # than macos-latest, whose target architecture has changed before + # and would silently narrow this matrix if it did again. + # + # No Intel macOS (macos-13): dropped intentionally, not an + # oversight -- GitHub's Intel-macOS runner queue is badly backlogged + # (one build sat queued for 3+ hours here), consistent with GitHub + # winding that pool down as Apple's own Intel Mac lineup ages out. + # Revisit if that changes. + os: [ubuntu-latest, macos-14] + steps: + - uses: actions/checkout@v4 + + - uses: pypa/cibuildwheel@v4.2.0 + + - name: Verify a wheel is fully self-contained + shell: bash + run: | + set -euo pipefail + wheel=$(ls wheelhouse/*.whl | head -1) + workdir=$(mktemp -d) + unzip -q "$wheel" -d "$workdir" + so=$(find "$workdir" -name 'pyPamtraLib*.so' -o -name 'pyPamtraLib*.pyd' | head -1) + echo "Checking $so from $wheel" + if [[ "${{ matrix.os }}" == ubuntu-* ]]; then + echo "--- exported symbols (should be exactly one: PyInit_pyPamtraLib) ---" + nm -D "$so" | awk '$2=="T"' + echo "--- linked libs (should show no libopenblas/libfftw3) ---" + ldd "$so" | grep -iE "openblas|fftw3" && exit 1 || echo "OK: neither linked" + else + echo "--- exported symbols (should be exactly one: PyInit_pyPamtraLib) ---" + nm -gU "$so" 2>/dev/null + echo "--- linked libs (should show no libopenblas/libfftw3) ---" + otool -L "$so" | grep -iE "openblas|fftw3" && exit 1 || echo "OK: neither linked" + fi + + - uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: wheelhouse/*.whl + + publish-testpypi: + name: Publish to TestPyPI + needs: build + # Deliberately not on every packaging-file push (this workflow's other + # trigger): TestPyPI rejects re-uploading the same version, so that + # would just fail on the second push after the first successful + # publish. Manual dispatch only -- kept around as an on-demand dry-run + # tool independent of publish-pypi below (the real, tag-gated job). + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/p/pamtra + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: wheels-* + path: dist + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + packages-dir: dist + + publish-pypi: + name: Publish to PyPI + needs: build + # Only on an actual release tag (see RELEASING.md's `git tag vX.Y.Z`) -- + # never on the workflow's other triggers (workflow_dispatch, packaging- + # file pushes), so an ordinary commit or manual dry run can never + # accidentally publish a real release. + if: startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/pamtra + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: wheels-* + path: dist + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist diff --git a/.gitignore b/.gitignore index a2fa890..13363de 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ mm_notes.txt scripts/ tools/lapack-3.5.0 build +build-deps/ lib doc/build *.pyf diff --git a/AI.md b/AI.md index 5c5d24f..8343505 100644 --- a/AI.md +++ b/AI.md @@ -93,6 +93,39 @@ binary has no such auto-fetch -- it always needs `PAMTRA_DATADIR` set manually. External library dependencies for the Fortran build: LAPACK/BLAS (or OpenBLAS), FFTW3, NetCDF (Fortran bindings), and a Fortran 90 compiler (gfortran assumed by both build systems). +`tools/build_openblas_static.sh` builds a private, static, single-threaded OpenBLAS with hidden +symbol visibility and installs its pkg-config file; point `meson.build`'s pkg-config-based +`dependency('openblas', ...)` lookup at it with `PKG_CONFIG_PATH=/lib/pkgconfig pip install +.` (no meson.build changes needed for discovery). This exists for eventual PyPI wheel builds: a +normal dynamic OpenBLAS bundled into a wheel would collide at runtime with numpy/scipy's own +bundled copy in the same process. See [RELEASING.md](RELEASING.md) for the "why not PyPI" context +this is a building block for. + +`tools/build_fftw_static.sh` and `tools/build_netcdf_stack.sh` are the same idea for the other two +dependencies -- static for FFTW (no collision risk like OpenBLAS, just kept static to give +auditwheel/delocate one less shared object to chase), dynamic for HDF5/netCDF-C/netCDF-Fortran +(large, complex chain where the standard auditwheel/delocate bundling path is lower-risk than +statically linking it ourselves; also what netCDF4's and h5py's own PyPI wheels do). Only the +standalone `pamtra` CLI executable needs netCDF at all -- `pyPamtraLib` (what `import pyPamtra` +loads) needs no direct netCDF link, since its NetCDF I/O goes through the pure-Python `netCDF4` +package instead. **Local testing gotcha**: unlike the static builds, these are ordinary dynamic +libraries installed to a non-standard prefix, so running anything against them locally (not +through a repaired wheel) needs `LD_LIBRARY_PATH`/`DYLD_LIBRARY_PATH` set to `/lib` -- +without it, the loader can silently resolve to a same-SONAME system copy instead (e.g. Debian's +own `libnetcdff.so.7` package) and produce corrupted output rather than an error. The eventual +wheel doesn't have this problem: `auditwheel`/`delocate` rewrite the built library to load its own +bundled copy via a relative rpath. + +**PyPI wheels drop the standalone `pamtra` CLI executable** (`meson_options.txt`'s `build_cli` +option, off via `-Dbuild_cli=false` in `[tool.cibuildwheel]`'s `config-settings`) -- it kept +segfaulting specifically inside cibuildwheel's manylinux container in a way that didn't reproduce +in any manually-built-and-`auditwheel`-repaired wheel tested outside that container, and the CLI +isn't the point of a PyPI wheel; `pyPamtraLib` (`import pyPamtra`) needs no netCDF at all (see +above), so this also means wheel builds skip `tools/build_netcdf_stack.sh` entirely -- +`tools/cibw_before_all.sh` only calls the OpenBLAS/FFTW scripts. `pip install .`/the conda-forge +recipe are unaffected (`build_cli` defaults to `true`), so the CLI is still available everywhere +except the PyPI wheel. + ## Architecture ### Fortran core (`src/`) diff --git a/RELEASING.md b/RELEASING.md index 0d090f2..c6946cd 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,9 +1,8 @@ # Making a PAMTRA release -This describes how to cut a new PAMTRA version and get it published on -conda-forge. It's maintainer-facing (parallel to [AI.md](AI.md), which covers -day-to-day build/test); PyPI publishing is intentionally out of scope (see -"Why not PyPI" below). +This describes how to cut a new PAMTRA version and get it published on both +conda-forge and PyPI. It's maintainer-facing (parallel to [AI.md](AI.md), +which covers day-to-day build/test). ## 1. Bump the version number @@ -23,6 +22,14 @@ git tag vX.Y.Z git push origin vX.Y.Z ``` +Pushing the tag alone (before creating any GitHub Release) already triggers +[wheels.yml](.github/workflows/wheels.yml)'s `publish-pypi` job: it builds +wheels for every supported platform/Python version and publishes them to +PyPI via trusted publishing (OIDC, no stored token) — no separate manual +step. See "PyPI" below for the one-time setup this depends on, and check +[the Actions tab](https://github.com/igmk/pamtra/actions/workflows/wheels.yml) +to confirm it went green before moving on. + Then create a GitHub Release from that tag (`gh release create vX.Y.Z` or via the GitHub UI) — GitHub's auto-generated source tarball for the tag is what the conda-forge recipe points at, so the release doesn't need any attached @@ -96,16 +103,41 @@ release needs a manual PR against `conda-forge/pamtra-feedstock`: before merging - [ ] After merge, `conda install -c conda-forge pamtra` works in a clean environment - -## Why not PyPI - -PAMTRA links against netCDF (C + Fortran bindings, which pull in HDF5/zlib/ -curl), FFTW, and OpenBLAS. A PyPI wheel can't depend on system packages the -way a conda package can — those libraries would have to be bundled *inside* -each wheel (via `cibuildwheel` + `auditwheel`/`delocate`/`delvewheel`), which -is a second, non-trivial CI pipeline. conda-forge gets this for free because -conda already manages those dependencies as packages. If PyPI becomes -worthwhile later, an sdist-only release (no compiled wheel, `pip install` -compiles from source using the user's local toolchain — same as `pip -install .` today) would be the low-effort first step, and would also let -conda-forge's auto-tick bot pick up new versions automatically. +- [ ] `wheels.yml`'s `publish-pypi` job (triggered by the tag push in step 2) + is green, and `pip install pamtra` works in a clean environment/venv + with no system libraries preinstalled + +## PyPI + +`pip install pamtra` works via [wheels.yml](.github/workflows/wheels.yml)'s +`build` + `publish-pypi` jobs (`cibuildwheel`, triggered on `vX.Y.Z` tags). +No conda/system libraries needed at install time — PAMTRA's own C/Fortran +dependencies are bundled into the wheel itself: + +- **OpenBLAS**: `tools/build_openblas_static.sh` builds it single-threaded + (`USE_THREAD=0` — PAMTRA only uses it for small per-particle T-matrix + solves, not large GEMMs, so this costs nothing in practice) and statically, + and `meson.build`'s `pyPamtraLib` target link-time-restricts its exported + symbol table to just `PyInit_pyPamtraLib`. Both matter because numpy/scipy + wheels already bundle their own dynamically-linked OpenBLAS, and two + copies loaded into the same process can collide (duplicate global symbols, + shared thread-pool state) regardless of whether the versions match — + matching versions alone doesn't rename or namespace anything. +- **FFTW**: `tools/build_fftw_static.sh`, static for the same + one-less-shared-object reason as OpenBLAS, though it has no collision risk + of its own (nothing else commonly bundles it). +- **netCDF-C/-Fortran/HDF5**: only needed by the standalone `pamtra` CLI + executable, not by `pyPamtraLib` (`import pyPamtra` gets its NetCDF I/O + through the pure-Python `netCDF4` package instead) — so wheel builds pass + `-Dbuild_cli=false` (`meson_options.txt`) and skip this dependency chain + entirely rather than bundle it. `tools/build_netcdf_stack.sh` still exists + for `pip install .`/conda builds, which build the CLI by default. + +**One-time setup this depends on**: a +[trusted publisher](https://pypi.org/manage/account/publishing/) registered +on pypi.org for project `pamtra`, GitHub repo `igmk/pamtra`, workflow +`wheels.yml`, environment `pypi` — no API token stored anywhere. Without +this, `publish-pypi` fails at the trusted-publishing handshake even though +the build itself succeeds. A `testpypi`-environment publisher (same setup, +on test.pypi.org) backs the separate `publish-testpypi` job, a manual +(`workflow_dispatch`-only) dry-run path independent of tag pushes. diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 97faf3e..a354cc8 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -29,6 +29,24 @@ binary (:ref:`pamtra`) in one step via `meson-python version of GNU Fortran``. +pip install (prebuilt wheels, quickest) +***************************************** + +For **Linux (x86_64)** and **macOS (Apple Silicon / arm64) only**:: + + pip install pamtra + +This installs a self-contained wheel with FFTW and OpenBLAS already bundled +in -- no system libraries, compiler, or conda/pixi environment needed. It +does **not** include the standalone ``pamtra`` CLI binary (:ref:`pamtra`), +which needs netCDF-Fortran (not bundled into the wheel); use one of the +from-source installs below if you need it. + +Not available for **Windows** or **Intel macOS** (``osx-64``) -- no wheels +are built for either platform. Use conda-forge/pixi below (covers Intel +macOS) or WSL2 below (Windows) instead. + + Get the code ************* diff --git a/meson.build b/meson.build index 30376b7..2c7e5d2 100644 --- a/meson.build +++ b/meson.build @@ -187,6 +187,19 @@ if not openblasdep.found() 'on other platforms, make sure its pkg-config file is on PKG_CONFIG_PATH.') endif +# Both netCDF-C and netCDF-Fortran below are needed only by the +# standalone `pamtra` executable further down (write_nc_results.f90) -- +# pyPamtraLib itself never calls netCDF at all, getting its NetCDF I/O +# purely through the Python netCDF4 package instead. Skip resolving either +# (and the CLI executable target itself, via disabler() propagation) when +# build_cli is off, so a wheel build -- which doesn't want the CLI's +# netCDF-C/netCDF-Fortran/HDF5 dependency chain bundled at all -- doesn't +# need any of it present or even attempt to find it. +build_cli = get_option('build_cli') +netcdfdep = disabler() +netcdffdep = disabler() + +if build_cli netcdfdep = dependency('netcdf') # Best-effort early warning for the same class of problem as the @@ -221,9 +234,16 @@ if netcdffdep.found() # source of truth for where the .mod files actually live. netcdff_includedir = netcdffdep.get_variable(pkgconfig: 'includedir', default_value: '') if netcdff_includedir != '' + # A raw '-I' compile_args string, not include_directories(): the latter + # rejects absolute paths that resolve inside the source tree (meson + # wants those expressed as relative paths instead), which a + # self-built netcdf-fortran can easily hit if its prefix happens to + # live under the project checkout (e.g. tools/build_netcdf_stack.sh's + # default of /build-deps/netcdf). compile_args has no such + # restriction -- it's passed to the compiler verbatim. netcdffdep = declare_dependency( dependencies: netcdffdep, - include_directories: include_directories(netcdff_includedir), + compile_args: ['-I' + netcdff_includedir], ) endif endif @@ -268,6 +288,7 @@ if not netcdffdep.found() 'or `brew install netcdf-fortran`), and either make sure pkg-config itself is installed ' + 'in the active environment, or export PKG_CONFIG_PATH to include its lib/pkgconfig dir.') endif +endif # build_cli # List sources csources = ['src/scatdb.c'] @@ -418,6 +439,32 @@ foreach ff : extra_fflags_list endif endforeach +# Restrict pyPamtraLib's exported dynamic symbol table to just the one +# symbol CPython's import machinery actually needs (dlopen + dlsym for +# PyInit_pyPamtraLib). Nothing else in the process ever looks up symbols in +# this .so by name, so this is safe -- and it matters for OpenBLAS: if +# another package in the same process (e.g. numpy/scipy) bundles its own +# copy of OpenBLAS, two same-named global symbols (dgemm_, thread-pool +# state, ...) loaded into one process can collide. `-fvisibility=hidden` +# passed to OpenBLAS's own build (see tools/build_openblas_static.sh) +# handles most of it, but not all: several of its ARM64 kernels are +# hand-written assembly with their own `.globl` directives, which ignore +# C-compiler visibility flags entirely and still end up global in +# libopenblas.a. A link-time whitelist here closes that gap regardless of +# where a leak comes from. +pyPamtraLib_link_args = [] +pyPamtraLib_link_depends = [] +if is_mac + pyPamtraLib_link_args = ['-Wl,-exported_symbol,_PyInit_pyPamtraLib'] +elif not is_windows + pyPamtraLib_map = meson.current_source_dir() / 'src/pyPamtraLib.map' + pyPamtraLib_link_args = ['-Wl,--version-script=' + pyPamtraLib_map] + pyPamtraLib_link_depends = [pyPamtraLib_map] +endif +# Windows is intentionally left alone: PAMTRA has no Windows build to test +# this against (see RELEASING.md), and MSVC/MinGW handle symbol export via +# .def files / __declspec, not GNU-ld version scripts or ld64 flags. + # Declare the fortran extension module py3.extension_module('pyPamtraLib', # extension module should have the same name ftmatrix as the target of f2py to be linked... at least when build with meson [fsources, csources, fortran_pamtra_source, fortranobject_c, versionNumberAuto], @@ -425,7 +472,9 @@ py3.extension_module('pyPamtraLib', # extension module should have the same name fortran_args: extra_fflags, include_directories: inc_dirs, link_with: fortranobject_lib, - dependencies : [py3_dep, fortranobject_dep, fftw3dep, openblasdep, netcdfdep], + link_args: pyPamtraLib_link_args, + link_depends: pyPamtraLib_link_depends, + dependencies : [py3_dep, fortranobject_dep, fftw3dep, openblasdep], subdir: 'pyPamtra/', install : true) diff --git a/meson_options.txt b/meson_options.txt new file mode 100644 index 0000000..e272ee7 --- /dev/null +++ b/meson_options.txt @@ -0,0 +1,2 @@ +option('build_cli', type: 'boolean', value: true, + description: 'Build and install the standalone pamtra CLI executable (needs netCDF-Fortran). Off for wheel builds -- see pyproject.toml [tool.cibuildwheel].') diff --git a/pyproject.toml b/pyproject.toml index 746bf25..14b15de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ requires = [ ] [project] -name = "pyPamtra" +name = "pamtra" license = {file = "LICENSE"} version = "1.0.3" description = "Python module pyPamtra" @@ -53,7 +53,6 @@ classifiers = [ "Topic :: Software Development :: Libraries", "Topic :: Scientific/Engineering :: Atmospheric Science", "Topic :: Utilities", - "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: POSIX", "Operating System :: Unix", @@ -65,6 +64,46 @@ classifiers = [ test = ["pytest", "pytest-cov"] #doc = ["sphinx>=1.3", "sphinx-rtd-theme"] +[tool.cibuildwheel] +# Matches the pip-build matrix in .github/workflows/ci.yml. +build = "cp310-* cp311-* cp312-* cp313-* cp314-*" +skip = ["*-musllinux*", "*-win*", "pp*"] +build-verbosity = 1 +before-all = "bash {project}/tools/cibw_before_all.sh" +environment = { PAMTRA_DEPS_PREFIX = "/tmp/pamtra-deps", PKG_CONFIG_PATH = "/tmp/pamtra-deps/openblas/lib/pkgconfig:/tmp/pamtra-deps/fftw/lib/pkgconfig" } +# -Dbuild_cli=false: the standalone pamtra CLI executable (needs netCDF-C/ +# -Fortran/HDF5, unlike pyPamtraLib itself -- see meson.build/meson_options.txt) +# is dropped from wheel builds specifically. It kept segfaulting inside +# cibuildwheel's manylinux container in ways that didn't reproduce in any +# manually-built-and-repaired wheel tested outside that container, and the +# CLI isn't the point of a PyPI wheel -- `pip install .`/conda still build +# it by default (build_cli defaults to true there). Worth revisiting if the +# segfault's cause is ever pinned down, but not worth blocking on. +config-settings = { setup-args = "-Dbuild_cli=false" } +test-extras = ["test"] +# {project}/tests isn't part of the installed wheel (meson.build only +# installs python/pyPamtra and pamtra_data.py) -- {project} points back at +# the source checkout specifically so the test suite is still reachable. +# PAMTRA_DATADIR="" opts out of the data auto-fetch (see AI.md), same as +# ci.yml. test_cli_binary.py skips itself cleanly when no pamtra binary is +# on PATH (see its pamtra_binary fixture), so no separate exclusion is +# needed for build_cli=false here. +test-command = "PAMTRA_DATADIR=\"\" pytest {project}/tests -v" + +[tool.cibuildwheel.linux] +manylinux-x86_64-image = "manylinux_2_28" + +[tool.cibuildwheel.macos] +# GitHub-hosted macOS runners' Homebrew gcc (needed for gfortran) requires +# macOS 14.0 as its own minimum target -- delocate refuses to repair a +# wheel tagged for an OLDER macOS (11.0, cibuildwheel's arm64 default) +# than the libraries it actually bundles (libgfortran/libquadmath), with +# "Library dependencies do not satisfy target MacOS version 11.0". Fully +# redeclaring `environment` here rather than just adding this one key: it +# was simpler to verify than relying on cibuildwheel's inheritance rules +# for how per-platform tables merge with the top-level one. +environment = { PAMTRA_DEPS_PREFIX = "/tmp/pamtra-deps", PKG_CONFIG_PATH = "/tmp/pamtra-deps/openblas/lib/pkgconfig:/tmp/pamtra-deps/fftw/lib/pkgconfig", MACOSX_DEPLOYMENT_TARGET = "14.0" } + [tool.coverage.run] source = ["pyPamtra", "pamtra_data"] diff --git a/readme.md b/readme.md index 90abe3e..47dbab5 100644 --- a/readme.md +++ b/readme.md @@ -8,7 +8,17 @@ Python/Fortran 90 package to solve the passive and active microwave radiative transfer in a plan parallel horizontally homogeneous atmosphere with hydrometeors ## Manual and Installation -See https://pamtra.readthedocs.io/ for documentation, including installation instructions. + +For Linux (x86_64) and macOS (Apple Silicon / arm64): + +``` +pip install pamtra +``` + +Not available as a wheel for Windows or Intel macOS -- see +https://pamtra.readthedocs.io/en/latest/installation.html for those and +other install options (conda-forge/pixi, building from source, HPC +clusters). ## Mailing list diff --git a/src/pyPamtraLib.map b/src/pyPamtraLib.map new file mode 100644 index 0000000..acf666e --- /dev/null +++ b/src/pyPamtraLib.map @@ -0,0 +1,4 @@ +{ + global: PyInit_pyPamtraLib; + local: *; +}; diff --git a/tools/build_fftw_static.sh b/tools/build_fftw_static.sh new file mode 100755 index 0000000..a46ba65 --- /dev/null +++ b/tools/build_fftw_static.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Builds a private, static FFTW and installs it (with a .pc file) into +# $PREFIX. Point meson at it with: +# +# PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig:$PKG_CONFIG_PATH" pip install . +# +# meson.build's `fftw3dep = dependency('fftw3')` call already tries +# pkg-config first, so this needs no meson.build changes -- it just has to +# win that pkg-config lookup ahead of any system/conda/Homebrew fftw3. +# +# Why this exists: for an eventual PyPI wheel, PAMTRA's own dependencies +# need to be self-contained. Unlike OpenBLAS, FFTW has no known symbol- +# collision risk with other commonly-bundled packages, so it doesn't need +# OpenBLAS's static+hidden-visibility treatment for correctness -- it's +# built static here purely to keep the wheel's runtime dependency surface +# minimal and avoid giving auditwheel/delocate one more shared object to +# chase down and bundle. +set -euo pipefail + +VERSION="${FFTW_VERSION:-3.3.10}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PREFIX="${PREFIX:-"$ROOT_DIR/build-deps/fftw"}" +JOBS="${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)}" +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +echo "Building static FFTW $VERSION -> $PREFIX" + +curl -sL "https://www.fftw.org/fftw-${VERSION}.tar.gz" -o "$WORK_DIR/fftw.tar.gz" +tar -xzf "$WORK_DIR/fftw.tar.gz" -C "$WORK_DIR" + +cd "$WORK_DIR/fftw-${VERSION}" + +# --enable-static is FFTW's default, spelled out here for clarity. +# --disable-shared: skip building the .so/.dylib entirely, so -lfftw3 +# unambiguously resolves to the static .a (mirrors the dangling-symlink +# lesson from build_openblas_static.sh: without this, `make install` would +# still leave a shared-library placeholder around to trip up later). +# --with-pic: the static objects need to be relocatable to link into +# pyPamtraLib's own shared object. +# Fortran wrappers are intentionally left enabled (the default) -- +# src/convolution.f90 includes FFTW's fftw3.f header directly, and that +# header's installation isn't worth risking over the tiny build-time +# saving from --disable-fortran. +./configure --prefix="$PREFIX" \ + --enable-static \ + --disable-shared \ + --with-pic + +make -j"$JOBS" +make install + +echo "Done. Static lib + pkg-config file:" +find "$PREFIX" -name 'libfftw3*' -o -name 'fftw3.pc' diff --git a/tools/build_netcdf_stack.sh b/tools/build_netcdf_stack.sh new file mode 100755 index 0000000..3310ac2 --- /dev/null +++ b/tools/build_netcdf_stack.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Builds a private, dynamic HDF5 -> netCDF-C -> netCDF-Fortran stack into one +# shared $PREFIX. Point meson at it with: +# +# PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig:$PKG_CONFIG_PATH" pip install . +# +# meson.build's `dependency('netcdf')`/`dependency('netcdf-fortran')` calls +# already try pkg-config first, so this needs no meson.build changes. +# +# Unlike tools/build_openblas_static.sh and tools/build_fftw_static.sh, this +# stack is built as ordinary DYNAMIC libraries, not static. OpenBLAS needed +# static+hidden-visibility specifically to avoid colliding with another +# package's bundled copy in the same process (see that script's header); +# HDF5/netCDF have no such collision risk (no other commonly-bundled PyPI +# package embeds them), so the standard `auditwheel repair`/`delocate-wheel` +# path -- which detects a wheel's dynamic library dependencies and bundles +# them automatically -- is the right, lower-risk tool here. It's also what +# netCDF4's and h5py's own PyPI wheels already do. That repair step is a +# separate, later part of the cibuildwheel pipeline, not this script. +# +# zlib is intentionally NOT built here: HDF5 links against whatever zlib +# headers/lib configure finds on the build machine (manylinux images and +# macOS both ship one), and it's stable/ABI-compatible enough across +# versions that building our own adds little. auditwheel/delocate bundle +# (or, per manylinux policy, may skip as an allowed baseline lib) whichever +# zlib the build actually picked up. +# +# NOTE for local (non-wheel) testing after running this script: these are +# dynamic libraries installed to a non-standard prefix, so unlike the static +# OpenBLAS/FFTW builds, `import pyPamtra` needs that prefix on the loader +# search path at runtime: +# macOS: DYLD_LIBRARY_PATH="$PREFIX/lib" python3 -c "import pyPamtra" +# Linux: LD_LIBRARY_PATH="$PREFIX/lib" python3 -c "import pyPamtra" +# The eventual wheel doesn't need this -- auditwheel/delocate rewrite the +# built .so to load its own bundled copies via a relative rpath instead. +set -euo pipefail + +HDF5_VERSION="${HDF5_VERSION:-1.14.6}" +NETCDF_C_VERSION="${NETCDF_C_VERSION:-4.10.1}" +NETCDF_FORTRAN_VERSION="${NETCDF_FORTRAN_VERSION:-4.6.4}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PREFIX="${PREFIX:-"$ROOT_DIR/build-deps/netcdf"}" +JOBS="${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)}" +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +# Each stage's configure step compiles and runs small test programs linked +# against the previous stage's just-built shared library, before it's +# installed anywhere the OS would normally find it -- so both the loader +# path and the compiler/linker flags need to point at $PREFIX from the +# start, not just at the end. +export CPPFLAGS="-I$PREFIX/include" +export LDFLAGS="-L$PREFIX/lib" +export LD_LIBRARY_PATH="$PREFIX/lib:${LD_LIBRARY_PATH:-}" +export DYLD_LIBRARY_PATH="$PREFIX/lib:${DYLD_LIBRARY_PATH:-}" + +echo "=== Building HDF5 $HDF5_VERSION -> $PREFIX ===" +curl -sL "https://github.com/HDFGroup/hdf5/releases/download/hdf5_${HDF5_VERSION}/hdf5-${HDF5_VERSION}.tar.gz" \ + -o "$WORK_DIR/hdf5.tar.gz" +tar -xzf "$WORK_DIR/hdf5.tar.gz" -C "$WORK_DIR" +( + cd "$WORK_DIR/hdf5-${HDF5_VERSION}" + ./configure --prefix="$PREFIX" \ + --enable-shared \ + --disable-static \ + --enable-fortran=no \ + --enable-cxx=no \ + --enable-build-mode=production + make -j"$JOBS" + make install +) + +echo "=== Building netCDF-C $NETCDF_C_VERSION -> $PREFIX ===" +curl -sL "https://github.com/Unidata/netcdf-c/archive/refs/tags/v${NETCDF_C_VERSION}.tar.gz" \ + -o "$WORK_DIR/netcdf-c.tar.gz" +tar -xzf "$WORK_DIR/netcdf-c.tar.gz" -C "$WORK_DIR" +( + cd "$WORK_DIR/netcdf-c-${NETCDF_C_VERSION}" + # --disable-dap/--disable-byterange: PAMTRA never does remote/OPeNDAP + # netCDF access (confirmed via grep across src/ and python/pyPamtra/), + # so drop the libcurl dependency entirely rather than bundle it unused. + # --disable-nczarr: PAMTRA only reads/writes classic netCDF files. + ./configure --prefix="$PREFIX" \ + --enable-shared \ + --disable-static \ + --disable-dap \ + --disable-byterange \ + --disable-nczarr + make -j"$JOBS" + make install +) + +echo "=== Building netCDF-Fortran $NETCDF_FORTRAN_VERSION -> $PREFIX ===" +curl -sL "https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v${NETCDF_FORTRAN_VERSION}.tar.gz" \ + -o "$WORK_DIR/netcdf-fortran.tar.gz" +tar -xzf "$WORK_DIR/netcdf-fortran.tar.gz" -C "$WORK_DIR" +( + cd "$WORK_DIR/netcdf-fortran-${NETCDF_FORTRAN_VERSION}" + ./configure --prefix="$PREFIX" \ + --enable-shared \ + --disable-static + make -j"$JOBS" + make install +) + +echo "Done. Shared libs + pkg-config files:" +find "$PREFIX/lib" -maxdepth 1 -name '*hdf5*' -o -name '*netcdf*' +find "$PREFIX/lib/pkgconfig" -name '*.pc' diff --git a/tools/build_openblas_static.sh b/tools/build_openblas_static.sh new file mode 100755 index 0000000..8e088fc --- /dev/null +++ b/tools/build_openblas_static.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Builds a private, static, single-threaded OpenBLAS and installs it (with a +# .pc file) into $PREFIX. Point meson at it with: +# +# PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig:$PKG_CONFIG_PATH" pip install . +# +# meson.build's `dependency('openblas', required: false)` call already tries +# pkg-config first, so this needs no meson.build changes -- it just has to +# win that pkg-config lookup ahead of any system/conda/Homebrew openblas. +# +# Why this exists: a normal dynamic OpenBLAS, if bundled into a PyPI wheel, +# would collide at runtime with numpy/scipy's own bundled OpenBLAS copy in +# the same process (duplicate global symbols, shared thread-pool state). +# Building it USE_THREAD=0 (PAMTRA only calls LAPACK for small per-particle +# T-matrix solves, not large GEMMs, so single-threaded is plenty fast) and +# statically with hidden visibility means its symbols are resolved at link +# time into pyPamtraLib's .so but never added to its exported dynamic symbol +# table -- so nothing else in the process can ever see or collide with this +# copy, regardless of what numpy/scipy bundle. +set -euo pipefail + +VERSION="${OPENBLAS_VERSION:-0.3.34}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PREFIX="${PREFIX:-"$ROOT_DIR/build-deps/openblas"}" +JOBS="${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)}" +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +echo "Building static sequential OpenBLAS $VERSION -> $PREFIX" + +curl -sL "https://github.com/OpenMathLib/OpenBLAS/releases/download/v${VERSION}/OpenBLAS-${VERSION}.tar.gz" \ + -o "$WORK_DIR/OpenBLAS.tar.gz" +tar -xzf "$WORK_DIR/OpenBLAS.tar.gz" -C "$WORK_DIR" + +# CCOMMON_OPT/FCOMMON_OPT must be environment variables, not `make VAR=val` +# command-line args: Makefile.system appends its own required flags to them +# (e.g. -DMAX_PARALLEL_NUMBER=...) via `+=`, and GNU Make silently drops +# makefile `+=` appends to a variable that was set on the command line +# (only an `override` directive in the makefile could change it back, and +# Makefile.system doesn't have one) -- so a command-line CCOMMON_OPT locks +# out those required flags instead of just adding ours on top. +export CCOMMON_OPT="-fvisibility=hidden" +export FCOMMON_OPT="-fvisibility=hidden" + +# BUILD_BFLOAT16=0: bfloat16 kernels are on by default on arm64, PAMTRA never +# needs them (only double-precision BLAS/LAPACK), and they fail to build +# under this toolchain (BGEMM_P/BGEMM_Q undeclared in gemm.c). This one is +# passed on the command line deliberately, to override Makefile.system's +# own arm64 default of BUILD_BFLOAT16=1. +# +# `shared` target explicitly, not the default `all` (which is `all :: +# tests`) and not `libs` alone. OpenBLAS's own Makefile graph is +# `tests : shared`, `shared : libs netlib $(RELA)`: `libs` alone only +# builds the BLAS kernels -- LAPACK (netlib) is a separate prerequisite of +# `shared`, so a `libs`-only build silently produces an archive missing +# LAPACK routines entirely (discovered the hard way: pyPamtraLib linked +# fine without ever calling into LAPACK, but the standalone pamtra +# executable failed with "Undefined symbols ... _dgetri_"). `shared` +# itself is safe to use with NO_SHARED=1: the actual .so/.dylib assembly +# in its recipe is wrapped in `ifneq ($(NO_SHARED), 1)`, so that part is a +# no-op and only its prerequisites (libs + netlib, i.e. everything the +# static archive needs) actually build -- without ever reaching `tests`, +# whose self-test programs (sblat2, dblat2, ...) don't link in +# cibuildwheel's sandboxed macOS environment (worked in a plain +# interactive shell, so not something worth chasing further -- we don't +# need those programs regardless). +make -C "$WORK_DIR/OpenBLAS-${VERSION}" -j"$JOBS" shared \ + USE_THREAD=0 \ + USE_LOCKING=1 \ + NO_SHARED=1 \ + BUILD_BFLOAT16=0 + +# OpenBLAS's own build output says it outright: "any flags passed to make +# during build should also be passed to make install to circumvent any +# install errors." NO_SHARED=1 is the one that actually matters here -- +# without it, `make install` unconditionally tries to `install` a shared +# library that was never built and dies with a hard, non-ignorable error on +# Linux (`install: cannot stat 'libopenblas...so': No such file or +# directory`, make exits non-zero). macOS's install step hits the same +# missing-file case but happens to use `cp` there in a make recipe line +# that's allowed to fail, so this went unnoticed when this script was first +# written and only tested on macOS. +make -C "$WORK_DIR/OpenBLAS-${VERSION}" install PREFIX="$PREFIX" \ + USE_THREAD=0 \ + USE_LOCKING=1 \ + NO_SHARED=1 \ + BUILD_BFLOAT16=0 + +# `make install` unconditionally creates libopenblas.{dylib,so}(.0) symlinks +# for the shared library even though NO_SHARED=1 means one was never built, +# leaving them dangling. A linker resolving `-lopenblas` could hit one of +# these before falling back to the .a, so remove them to force static +# linking deterministically. +find "$PREFIX/lib" -maxdepth 1 \( -name '*.dylib' -o -name '*.so*' \) ! -exec test -e {} \; -delete + +echo "Done. Static lib + pkg-config file:" +find "$PREFIX" -name 'libopenblas*' -o -name 'openblas.pc' diff --git a/tools/cibw_before_all.sh b/tools/cibw_before_all.sh new file mode 100755 index 0000000..4cd6d1a --- /dev/null +++ b/tools/cibw_before_all.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# cibuildwheel `before-all` hook: builds the two bundled dependencies wheel +# builds actually need (static OpenBLAS, static FFTW) into a fixed location +# OUTSIDE the checked-out source tree. That matters: a prefix under the +# repo (the build scripts' own default) can trip meson.build's +# netcdf-fortran includedir handling on absolute paths that resolve inside +# the source tree -- see the meson.build commit that fixed this for local +# testing. Building outside the tree sidesteps the whole question rather +# than relying on that one fix being complete. +# +# tools/build_netcdf_stack.sh is NOT called here: wheel builds pass +# -Dbuild_cli=false (see [tool.cibuildwheel] in pyproject.toml), which +# drops the standalone pamtra CLI executable -- the only thing that needs +# netCDF-C/-Fortran/HDF5 at all (pyPamtraLib gets its NetCDF I/O purely +# through the Python netCDF4 package). That script still exists for local/ +# dev use building the full CLI-included package. +# +# $PAMTRA_DEPS_PREFIX (set here, read by [tool.cibuildwheel.environment]'s +# PKG_CONFIG_PATH in pyproject.toml) must stay in sync with that setting. +set -euo pipefail + +PAMTRA_DEPS_PREFIX="${PAMTRA_DEPS_PREFIX:-/tmp/pamtra-deps}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [ "$(uname)" = "Darwin" ]; then + # GitHub-hosted macOS runners ship gfortran already, but cibuildwheel's + # manylinux-equivalent (a fresh container) does not -- brew install is + # a harmless no-op if it's already present. + brew install gcc +else + # manylinux images are minimal containers with no Fortran compiler and + # no zlib headers (only the runtime .so) by default. + (yum install -y gcc-gfortran zlib-devel) || (dnf install -y gcc-gfortran zlib-devel) +fi + +PREFIX="$PAMTRA_DEPS_PREFIX/openblas" "$ROOT_DIR/tools/build_openblas_static.sh" +PREFIX="$PAMTRA_DEPS_PREFIX/fftw" "$ROOT_DIR/tools/build_fftw_static.sh"