Make pip install pamtra work: static OpenBLAS/FFTW, cibuildwheel pipeline

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ["3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/wheels.yml

@@ -0,0 +1,129 @@
+name: Wheels
+
+on:
+  workflow_dispatch:
+  push:
+    # branches: ['**'] looks redundant with no branches actually excluded,
+    # but it's load-bearing: adding `tags:` below with no `branches:` key
+    # at all silently suppresses ALL branch-push triggering (confirmed by
+    # this exact push not triggering a run), not just narrows it -- this
+    # restores the "any branch, filtered by paths" behavior alongside it.
+    branches:
+      - '**'
+    paths:
+      - pyproject.toml
+      - meson.build
+      - 'tools/build_*.sh'
+      - 'tools/cibw_*.sh'
+      - .github/workflows/wheels.yml
+    tags:
+      # Release tags (see RELEASING.md's `git tag vX.Y.Z`) trigger the real
+      # PyPI publish job below. Safe to combine with `paths:` above even
+      # though a tag push touches none of those files: GitHub doesn't apply
+      # `paths` filtering to tag pushes at all (only to branch pushes), so
+      # this fires unconditionally on any v*-matching tag regardless.
+      - 'v*'
+
+concurrency:
+  group: wheels-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: build (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # macos-14 = Apple Silicon (arm64) -- spelled out explicitly rather
+        # than macos-latest, whose target architecture has changed before
+        # and would silently narrow this matrix if it did again.
+        #
+        # No Intel macOS (macos-13): dropped intentionally, not an
+        # oversight -- GitHub's Intel-macOS runner queue is badly backlogged
+        # (one build sat queued for 3+ hours here), consistent with GitHub
+        # winding that pool down as Apple's own Intel Mac lineup ages out.
+        # Revisit if that changes.
+        os: [ubuntu-latest, macos-14]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: pypa/cibuildwheel@v4.2.0
+
+      - name: Verify a wheel is fully self-contained
+        shell: bash
+        run: |
+          set -euo pipefail
+          wheel=$(ls wheelhouse/*.whl | head -1)
+          workdir=$(mktemp -d)
+          unzip -q "$wheel" -d "$workdir"
+          so=$(find "$workdir" -name 'pyPamtraLib*.so' -o -name 'pyPamtraLib*.pyd' | head -1)
+          echo "Checking $so from $wheel"
+          if [[ "${{ matrix.os }}" == ubuntu-* ]]; then
+            echo "--- exported symbols (should be exactly one: PyInit_pyPamtraLib) ---"
+            nm -D "$so" | awk '$2=="T"'
+            echo "--- linked libs (should show no libopenblas/libfftw3) ---"
+            ldd "$so" | grep -iE "openblas|fftw3" && exit 1 || echo "OK: neither linked"
+          else
+            echo "--- exported symbols (should be exactly one: PyInit_pyPamtraLib) ---"
+            nm -gU "$so" 2>/dev/null
+            echo "--- linked libs (should show no libopenblas/libfftw3) ---"
+            otool -L "$so" | grep -iE "openblas|fftw3" && exit 1 || echo "OK: neither linked"
+          fi
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.os }}
+          path: wheelhouse/*.whl
+
+  publish-testpypi:
+    name: Publish to TestPyPI
+    needs: build
+    # Deliberately not on every packaging-file push (this workflow's other
+    # trigger): TestPyPI rejects re-uploading the same version, so that
+    # would just fail on the second push after the first successful
+    # publish. Manual dispatch only -- kept around as an on-demand dry-run
+    # tool independent of publish-pypi below (the real, tag-gated job).
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/pamtra
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          path: dist
+          merge-multiple: true
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          packages-dir: dist
+
+  publish-pypi:
+    name: Publish to PyPI
+    needs: build
+    # Only on an actual release tag (see RELEASING.md's `git tag vX.Y.Z`) --
+    # never on the workflow's other triggers (workflow_dispatch, packaging-
+    # file pushes), so an ordinary commit or manual dry run can never
+    # accidentally publish a real release.
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pamtra
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          path: dist
+          merge-multiple: true
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist

.gitignore

@@ -50,6 +50,7 @@ mm_notes.txt
 scripts/
 tools/lapack-3.5.0
 build
+build-deps/
 lib
 doc/build
 *.pyf

AI.md

@@ -93,6 +93,39 @@ binary has no such auto-fetch -- it always needs `PAMTRA_DATADIR` set manually.
 External library dependencies for the Fortran build: LAPACK/BLAS (or OpenBLAS), FFTW3, NetCDF
 (Fortran bindings), and a Fortran 90 compiler (gfortran assumed by both build systems).
 
+`tools/build_openblas_static.sh` builds a private, static, single-threaded OpenBLAS with hidden
+symbol visibility and installs its pkg-config file; point `meson.build`'s pkg-config-based
+`dependency('openblas', ...)` lookup at it with `PKG_CONFIG_PATH=<prefix>/lib/pkgconfig pip install
+.` (no meson.build changes needed for discovery). This exists for eventual PyPI wheel builds: a
+normal dynamic OpenBLAS bundled into a wheel would collide at runtime with numpy/scipy's own
+bundled copy in the same process. See [RELEASING.md](RELEASING.md) for the "why not PyPI" context
+this is a building block for.
+
+`tools/build_fftw_static.sh` and `tools/build_netcdf_stack.sh` are the same idea for the other two
+dependencies -- static for FFTW (no collision risk like OpenBLAS, just kept static to give
+auditwheel/delocate one less shared object to chase), dynamic for HDF5/netCDF-C/netCDF-Fortran
+(large, complex chain where the standard auditwheel/delocate bundling path is lower-risk than
+statically linking it ourselves; also what netCDF4's and h5py's own PyPI wheels do). Only the
+standalone `pamtra` CLI executable needs netCDF at all -- `pyPamtraLib` (what `import pyPamtra`
+loads) needs no direct netCDF link, since its NetCDF I/O goes through the pure-Python `netCDF4`
+package instead. **Local testing gotcha**: unlike the static builds, these are ordinary dynamic
+libraries installed to a non-standard prefix, so running anything against them locally (not
+through a repaired wheel) needs `LD_LIBRARY_PATH`/`DYLD_LIBRARY_PATH` set to `<prefix>/lib` --
+without it, the loader can silently resolve to a same-SONAME system copy instead (e.g. Debian's
+own `libnetcdff.so.7` package) and produce corrupted output rather than an error. The eventual
+wheel doesn't have this problem: `auditwheel`/`delocate` rewrite the built library to load its own
+bundled copy via a relative rpath.
+
+**PyPI wheels drop the standalone `pamtra` CLI executable** (`meson_options.txt`'s `build_cli`
+option, off via `-Dbuild_cli=false` in `[tool.cibuildwheel]`'s `config-settings`) -- it kept
+segfaulting specifically inside cibuildwheel's manylinux container in a way that didn't reproduce
+in any manually-built-and-`auditwheel`-repaired wheel tested outside that container, and the CLI
+isn't the point of a PyPI wheel; `pyPamtraLib` (`import pyPamtra`) needs no netCDF at all (see
+above), so this also means wheel builds skip `tools/build_netcdf_stack.sh` entirely --
+`tools/cibw_before_all.sh` only calls the OpenBLAS/FFTW scripts. `pip install .`/the conda-forge
+recipe are unaffected (`build_cli` defaults to `true`), so the CLI is still available everywhere
+except the PyPI wheel.
+
 ## Architecture
 
 ### Fortran core (`src/`)

RELEASING.md

@@ -1,9 +1,8 @@
 # Making a PAMTRA release
 
-This describes how to cut a new PAMTRA version and get it published on
-conda-forge. It's maintainer-facing (parallel to [AI.md](AI.md), which covers
-day-to-day build/test); PyPI publishing is intentionally out of scope (see
-"Why not PyPI" below).
+This describes how to cut a new PAMTRA version and get it published on both
+conda-forge and PyPI. It's maintainer-facing (parallel to [AI.md](AI.md),
+which covers day-to-day build/test).
 
 ## 1. Bump the version number
 
@@ -23,6 +22,14 @@ git tag vX.Y.Z
 git push origin vX.Y.Z
 ```
 
+Pushing the tag alone (before creating any GitHub Release) already triggers
+[wheels.yml](.github/workflows/wheels.yml)'s `publish-pypi` job: it builds
+wheels for every supported platform/Python version and publishes them to
+PyPI via trusted publishing (OIDC, no stored token) — no separate manual
+step. See "PyPI" below for the one-time setup this depends on, and check
+[the Actions tab](https://github.com/igmk/pamtra/actions/workflows/wheels.yml)
+to confirm it went green before moving on.
+
 Then create a GitHub Release from that tag (`gh release create vX.Y.Z` or via
 the GitHub UI) — GitHub's auto-generated source tarball for the tag is what
 the conda-forge recipe points at, so the release doesn't need any attached
@@ -96,16 +103,41 @@ release needs a manual PR against `conda-forge/pamtra-feedstock`:
       before merging
 - [ ] After merge, `conda install -c conda-forge pamtra` works in a clean
       environment
-
-## Why not PyPI
-
-PAMTRA links against netCDF (C + Fortran bindings, which pull in HDF5/zlib/
-curl), FFTW, and OpenBLAS. A PyPI wheel can't depend on system packages the
-way a conda package can — those libraries would have to be bundled *inside*
-each wheel (via `cibuildwheel` + `auditwheel`/`delocate`/`delvewheel`), which
-is a second, non-trivial CI pipeline. conda-forge gets this for free because
-conda already manages those dependencies as packages. If PyPI becomes
-worthwhile later, an sdist-only release (no compiled wheel, `pip install`
-compiles from source using the user's local toolchain — same as `pip
-install .` today) would be the low-effort first step, and would also let
-conda-forge's auto-tick bot pick up new versions automatically.
+- [ ] `wheels.yml`'s `publish-pypi` job (triggered by the tag push in step 2)
+      is green, and `pip install pamtra` works in a clean environment/venv
+      with no system libraries preinstalled
+
+## PyPI
+
+`pip install pamtra` works via [wheels.yml](.github/workflows/wheels.yml)'s
+`build` + `publish-pypi` jobs (`cibuildwheel`, triggered on `vX.Y.Z` tags).
+No conda/system libraries needed at install time — PAMTRA's own C/Fortran
+dependencies are bundled into the wheel itself:
+
+- **OpenBLAS**: `tools/build_openblas_static.sh` builds it single-threaded
+  (`USE_THREAD=0` — PAMTRA only uses it for small per-particle T-matrix
+  solves, not large GEMMs, so this costs nothing in practice) and statically,
+  and `meson.build`'s `pyPamtraLib` target link-time-restricts its exported
+  symbol table to just `PyInit_pyPamtraLib`. Both matter because numpy/scipy
+  wheels already bundle their own dynamically-linked OpenBLAS, and two
+  copies loaded into the same process can collide (duplicate global symbols,
+  shared thread-pool state) regardless of whether the versions match —
+  matching versions alone doesn't rename or namespace anything.
+- **FFTW**: `tools/build_fftw_static.sh`, static for the same
+  one-less-shared-object reason as OpenBLAS, though it has no collision risk
+  of its own (nothing else commonly bundles it).
+- **netCDF-C/-Fortran/HDF5**: only needed by the standalone `pamtra` CLI
+  executable, not by `pyPamtraLib` (`import pyPamtra` gets its NetCDF I/O
+  through the pure-Python `netCDF4` package instead) — so wheel builds pass
+  `-Dbuild_cli=false` (`meson_options.txt`) and skip this dependency chain
+  entirely rather than bundle it. `tools/build_netcdf_stack.sh` still exists
+  for `pip install .`/conda builds, which build the CLI by default.
+
+**One-time setup this depends on**: a
+[trusted publisher](https://pypi.org/manage/account/publishing/) registered
+on pypi.org for project `pamtra`, GitHub repo `igmk/pamtra`, workflow
+`wheels.yml`, environment `pypi` — no API token stored anywhere. Without
+this, `publish-pypi` fails at the trusted-publishing handshake even though
+the build itself succeeds. A `testpypi`-environment publisher (same setup,
+on test.pypi.org) backs the separate `publish-testpypi` job, a manual
+(`workflow_dispatch`-only) dry-run path independent of tag pushes.

doc/source/installation.rst

@@ -29,6 +29,24 @@ binary (:ref:`pamtra`) in one step via `meson-python
    version of GNU Fortran``.
 
 
+pip install (prebuilt wheels, quickest)
+*****************************************
+
+For **Linux (x86_64)** and **macOS (Apple Silicon / arm64) only**::
+
+    pip install pamtra
+
+This installs a self-contained wheel with FFTW and OpenBLAS already bundled
+in -- no system libraries, compiler, or conda/pixi environment needed. It
+does **not** include the standalone ``pamtra`` CLI binary (:ref:`pamtra`),
+which needs netCDF-Fortran (not bundled into the wheel); use one of the
+from-source installs below if you need it.
+
+Not available for **Windows** or **Intel macOS** (``osx-64``) -- no wheels
+are built for either platform. Use conda-forge/pixi below (covers Intel
+macOS) or WSL2 below (Windows) instead.
+
+
 Get the code
 *************
 

meson.build

@@ -187,6 +187,19 @@ if not openblasdep.found()
         'on other platforms, make sure its pkg-config file is on PKG_CONFIG_PATH.')
 endif
 
+# Both netCDF-C and netCDF-Fortran below are needed only by the
+# standalone `pamtra` executable further down (write_nc_results.f90) --
+# pyPamtraLib itself never calls netCDF at all, getting its NetCDF I/O
+# purely through the Python netCDF4 package instead. Skip resolving either
+# (and the CLI executable target itself, via disabler() propagation) when
+# build_cli is off, so a wheel build -- which doesn't want the CLI's
+# netCDF-C/netCDF-Fortran/HDF5 dependency chain bundled at all -- doesn't
+# need any of it present or even attempt to find it.
+build_cli = get_option('build_cli')
+netcdfdep = disabler()
+netcdffdep = disabler()
+
+if build_cli
 netcdfdep = dependency('netcdf')
 
 # Best-effort early warning for the same class of problem as the
@@ -221,9 +234,16 @@ if netcdffdep.found()
   # source of truth for where the .mod files actually live.
   netcdff_includedir = netcdffdep.get_variable(pkgconfig: 'includedir', default_value: '')
   if netcdff_includedir != ''
+    # A raw '-I' compile_args string, not include_directories(): the latter
+    # rejects absolute paths that resolve inside the source tree (meson
+    # wants those expressed as relative paths instead), which a
+    # self-built netcdf-fortran can easily hit if its prefix happens to
+    # live under the project checkout (e.g. tools/build_netcdf_stack.sh's
+    # default of <repo>/build-deps/netcdf). compile_args has no such
+    # restriction -- it's passed to the compiler verbatim.
     netcdffdep = declare_dependency(
       dependencies: netcdffdep,
-      include_directories: include_directories(netcdff_includedir),
+      compile_args: ['-I' + netcdff_includedir],
     )
   endif
 endif
@@ -268,6 +288,7 @@ if not netcdffdep.found()
         'or `brew install netcdf-fortran`), and either make sure pkg-config itself is installed ' +
         'in the active environment, or export PKG_CONFIG_PATH to include its lib/pkgconfig dir.')
 endif
+endif # build_cli
 
 # List sources
 csources = ['src/scatdb.c']
@@ -418,14 +439,42 @@ foreach ff : extra_fflags_list
   endif
 endforeach
 
+# Restrict pyPamtraLib's exported dynamic symbol table to just the one
+# symbol CPython's import machinery actually needs (dlopen + dlsym for
+# PyInit_pyPamtraLib). Nothing else in the process ever looks up symbols in
+# this .so by name, so this is safe -- and it matters for OpenBLAS: if
+# another package in the same process (e.g. numpy/scipy) bundles its own
+# copy of OpenBLAS, two same-named global symbols (dgemm_, thread-pool
+# state, ...) loaded into one process can collide. `-fvisibility=hidden`
+# passed to OpenBLAS's own build (see tools/build_openblas_static.sh)
+# handles most of it, but not all: several of its ARM64 kernels are
+# hand-written assembly with their own `.globl` directives, which ignore
+# C-compiler visibility flags entirely and still end up global in
+# libopenblas.a. A link-time whitelist here closes that gap regardless of
+# where a leak comes from.
+pyPamtraLib_link_args = []
+pyPamtraLib_link_depends = []
+if is_mac
+  pyPamtraLib_link_args = ['-Wl,-exported_symbol,_PyInit_pyPamtraLib']
+elif not is_windows
+  pyPamtraLib_map = meson.current_source_dir() / 'src/pyPamtraLib.map'
+  pyPamtraLib_link_args = ['-Wl,--version-script=' + pyPamtraLib_map]
+  pyPamtraLib_link_depends = [pyPamtraLib_map]
+endif
+# Windows is intentionally left alone: PAMTRA has no Windows build to test
+# this against (see RELEASING.md), and MSVC/MinGW handle symbol export via
+# .def files / __declspec, not GNU-ld version scripts or ld64 flags.
+
 # Declare the fortran extension module
 py3.extension_module('pyPamtraLib', # extension module should have the same name ftmatrix as the target of f2py to be linked... at least when build with meson
   [fsources, csources, fortran_pamtra_source, fortranobject_c, versionNumberAuto],
   c_args: c_flags,
   fortran_args: extra_fflags,
   include_directories: inc_dirs,
   link_with: fortranobject_lib,
-  dependencies : [py3_dep, fortranobject_dep, fftw3dep, openblasdep, netcdfdep],
+  link_args: pyPamtraLib_link_args,
+  link_depends: pyPamtraLib_link_depends,
+  dependencies : [py3_dep, fortranobject_dep, fftw3dep, openblasdep],
   subdir: 'pyPamtra/',
   install : true)
 

meson_options.txt

@@ -0,0 +1,2 @@
+option('build_cli', type: 'boolean', value: true,
+  description: 'Build and install the standalone pamtra CLI executable (needs netCDF-Fortran). Off for wheel builds -- see pyproject.toml [tool.cibuildwheel].')