diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efbf921ca..c440e4ce0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -204,7 +204,7 @@ jobs: path: wheelhouse key: >- native-wheel-v1-${{ matrix.id }}-${{ hashFiles('native/**', '!native/tests/**', - '!native/benchmarks/**', '!native/**/*.md') }} + '!native/**/*.md') }} - if: steps.cache.outputs.cache-hit != 'true' && runner.os == 'Linux' name: Install the native toolchain run: | @@ -563,7 +563,7 @@ jobs: - name: Install the native toolchain run: | sudo apt-get update - sudo apt-get install --yes clang-tidy cmake g++-14 libfftw3-dev ninja-build pkg-config + sudo apt-get install --yes cmake g++-14 libfftw3-dev ninja-build pkg-config - run: uv python install 3.14 - run: uv sync --locked --no-install-package solprop-native - run: uv run poe check-native @@ -608,7 +608,6 @@ jobs: cmake --preset cuda \ -DCMAKE_C_COMPILER=gcc-14 -DCMAKE_CXX_COMPILER=g++-14 -DCMAKE_CUDA_HOST_COMPILER=g++-14 \ -DCUDA_ROOT=/usr/local/cuda-12.8 \ - -DSOLPROP_NATIVE_BUILD_BENCHMARKS=OFF \ -DPython_EXECUTABLE="$GITHUB_WORKSPACE/.venv/bin/python" - name: Compile, tests included working-directory: native @@ -656,7 +655,6 @@ jobs: cmake --preset hip \ -DCMAKE_CXX_COMPILER="$ROCM_PATH/bin/hipcc" -DCMAKE_PREFIX_PATH="$ROCM_PATH" \ -DCMAKE_CXX_SCAN_FOR_MODULES=OFF -DKokkos_ENABLE_ROCTHRUST=OFF \ - -DSOLPROP_NATIVE_BUILD_BENCHMARKS=OFF \ -DPython_EXECUTABLE="$GITHUB_WORKSPACE/.venv/bin/python" - name: Compile, tests included working-directory: native diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 0b59c4fc6..b756e2129 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -5,7 +5,8 @@ name: NVIDIA CUDA validation # Manual workflow for a self-hosted Linux runner labelled nvidia-gpu, with an NVIDIA -# GPU, the CUDA Toolkit (nvcc, cuFFT, cuBLAS, Compute Sanitizer) and GCC 13 or newer. +# GPU, the CUDA Toolkit 12.8 or newer (nvcc, cuFFT, cuBLAS, Compute Sanitizer) and GCC 14 +# or newer (CMakeLists.txt refuses an older host compiler). # Set the repository variable SOLPROP_NATIVE_CUDA_ARCH to the runner's Kokkos # architecture (for example AMPERE80, ADA89 or HOPPER90); HOPPER90 is used when unset. # The repository is public: keep this workflow dispatch-only and never let the @@ -58,3 +59,5 @@ jobs: run: | compute-sanitizer --tool memcheck --error-exitcode 1 ../.venv/bin/pytest tests/test_numerics.py compute-sanitizer --tool racecheck --error-exitcode 1 ../.venv/bin/pytest tests/test_numerics.py + compute-sanitizer --tool initcheck --error-exitcode 1 ../.venv/bin/pytest tests/test_numerics.py + compute-sanitizer --tool synccheck --error-exitcode 1 ../.venv/bin/pytest tests/test_numerics.py diff --git a/.github/workflows/native-deep.yml b/.github/workflows/native-deep.yml new file mode 100644 index 000000000..5795b3a64 --- /dev/null +++ b/.github/workflows/native-deep.yml @@ -0,0 +1,204 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +name: Native deep checks + +# Weekly, by hand, and on a pull request that changes this file: the native checks too +# slow for every pull request. Each job reports (an artifact and the job summary) and +# gates nothing. +# +# - coverage: Clang source coverage of the native code over the GoogleTest suite and +# the native pytest suite (SOLPROP_NATIVE_ENABLE_COVERAGE). Its uncovered lines are +# where to look for dead code. +# - extension-asan: the extension itself built with ASan and UBSan and run under the +# native pytest suite, which exercises the nanobind bindings the gate's sanitizer +# stage (GoogleTest only) never reaches. +# - include-cleaner: clang-tidy's misc-include-cleaner over the sources, as a report of +# missing and unused includes. + +on: + schedule: + - cron: "17 3 * * 1" + workflow_dispatch: + # A pull request that changes this workflow runs it, so that a change is seen to work + # before the next weekly run (dispatch needs the workflow on the default branch). + pull_request: + paths: + - .github/workflows/native-deep.yml + +permissions: + contents: read + +concurrency: + group: native-deep-${{ github.ref }} + cancel-in-progress: true + +jobs: + coverage: + runs-on: ubuntu-24.04 + timeout-minutes: 240 + env: + CC: clang-18 + CXX: clang++-18 + OMP_PROC_BIND: "false" + SOLPROP_NATIVE_THREADS: "2" + UV_NO_SYNC: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: true + - name: Install Clang, its OpenMP runtime and FFTW + run: | + sudo apt-get update + sudo apt-get install --yes clang-18 llvm-18 libomp-18-dev cmake libfftw3-dev ninja-build pkg-config + - run: uv python install 3.14 + - run: uv sync --locked --no-install-package solprop-native + - name: Build the instrumented extension and native tests + # FFTW's pthreads library: Ubuntu's OpenMP FFTW links GNU OpenMP, and Clang's + # build runs LLVM's. No vendor BLAS (the reference profile). + # Report-only: a warning from this job's compiler, which no pull-request lane + # builds with, does not stop the report. + env: + CMAKE_ARGS: >- + -DSOLPROP_NATIVE_WARNINGS_AS_ERRORS=OFF + -DSOLPROP_NATIVE_ENABLE_COVERAGE=ON + -DSOLPROP_NATIVE_PROFILE=REFERENCE + -DSOLPROP_NATIVE_FFTW_THREADING=PTHREADS + -DBUILD_TESTING=ON + run: | + uv pip install --no-deps --no-build-isolation -e native \ + --config-settings build-dir=build/coverage + - name: GoogleTest suite + continue-on-error: true + env: + LLVM_PROFILE_FILE: ${{ github.workspace }}/profiles/gtest-%p-%m.profraw + run: native/build/coverage/solprop_native_tests + - name: Native pytest suite + continue-on-error: true + working-directory: native + env: + LLVM_PROFILE_FILE: ${{ github.workspace }}/profiles/pytest-%p-%m.profraw + run: uv run pytest -n 2 -p no:cacheprovider + - name: Report + if: always() + run: | + extension=$(uv run python -c "import solprop_native._core as m; print(m.__file__)") + llvm-profdata-18 merge -sparse profiles/*.profraw -o coverage.profdata + objects=("$extension" -object native/build/coverage/solprop_native_tests) + sources=(native/src native/include/solprop_native) + mkdir -p native-coverage + llvm-cov-18 report "${objects[@]}" -instr-profile coverage.profdata "${sources[@]}" \ + | tee native-coverage/report.txt + llvm-cov-18 show "${objects[@]}" -instr-profile coverage.profdata "${sources[@]}" \ + -format=html -show-line-counts-or-regions -output-dir native-coverage/html + { echo '## Native source coverage'; echo '```'; tail -n 40 native-coverage/report.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: native-coverage + path: native-coverage + retention-days: 30 + + extension-asan: + runs-on: ubuntu-24.04 + timeout-minutes: 240 + env: + CXX: g++-14 + OMP_PROC_BIND: "false" + SOLPROP_NATIVE_THREADS: "2" + UV_NO_SYNC: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: true + - name: Install the native toolchain + run: | + sudo apt-get update + sudo apt-get install --yes cmake g++-14 libfftw3-dev ninja-build pkg-config + - run: uv python install 3.14 + - run: uv sync --locked --no-install-package solprop-native + - name: Build the extension with ASan, UBSan and Kokkos bounds checks + env: + CMAKE_ARGS: >- + -DSOLPROP_NATIVE_WARNINGS_AS_ERRORS=OFF + -DSOLPROP_NATIVE_ENABLE_SANITIZERS=ON + -DSOLPROP_NATIVE_PROFILE=REFERENCE + -DKokkos_ENABLE_DEBUG_BOUNDS_CHECK=ON + run: | + uv pip install --no-deps --no-build-isolation -e native \ + --config-settings build-dir=build/asan \ + --config-settings cmake.build-type=RelWithDebInfo + - name: Native pytest suite under the sanitizers + working-directory: native + # Python is not instrumented, so the ASan runtime is preloaded, with libstdc++ + # beside it: ASan resolves __cxa_throw when it starts, and Python loads no C++ + # runtime of its own, so without it every C++ exception aborts. CPython's own + # allocations at exit are not leaks of this package. setarch -R: Ubuntu's + # high-entropy ASLR can map a library inside ASan's shadow range. Report-only: + # the summary records the outcome and the log is the artifact. + run: | + set -o pipefail + export LD_PRELOAD="$(g++-14 -print-file-name=libasan.so):$(g++-14 -print-file-name=libstdc++.so)" + export ASAN_OPTIONS=detect_leaks=0:abort_on_error=1 + export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 + status=0 + setarch "$(uname -m)" -R uv run pytest -n 2 -p no:cacheprovider 2>&1 | tee ../asan.log || status=$? + { echo '## Native pytest under ASan and UBSan'; echo "exit status ${status}"; echo '```' + grep -E 'ERROR: AddressSanitizer|runtime error|^FAILED|passed|failed' ../asan.log | tail -n 40 + echo '```'; } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: native-extension-asan + path: asan.log + retention-days: 30 + + include-cleaner: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + CXX: g++-14 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: true + - name: Install the native toolchain + run: | + sudo apt-get update + sudo apt-get install --yes cmake g++-14 libfftw3-dev ninja-build pkg-config + - run: uv python install 3.14 + - run: uv sync --locked --no-install-package solprop-native + - name: Configure the cpu-tidy preset (compile commands and generated headers) + working-directory: native + run: | + cmake --preset cpu-tidy -DCMAKE_CXX_COMPILER="$(command -v g++-14)" \ + -DPython_EXECUTABLE="$GITHUB_WORKSPACE/.venv/bin/python" + # SLEEF's header comes from its own build. + cmake --build --preset cpu-tidy --target solprop_native_sleef_build + - name: misc-include-cleaner + run: | + build=native/build/presets/cpu-tidy + # The pinned clang-tidy and the arguments the build gives it. + mapfile -t tidy < "$build/clang_tidy_command.txt" + git ls-files -z -- 'native/src/*.cpp' 'native/tests/native/*.cpp' | + xargs -0 -n 1 -P "$(nproc)" "${tidy[@]}" --quiet -p "$build" \ + --checks='-*,misc-include-cleaner' --warnings-as-errors='' \ + > include-cleaner.txt 2>&1 || true + findings=$(grep -c 'misc-include-cleaner' include-cleaner.txt || true) + echo "## misc-include-cleaner: ${findings} findings" >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: always() + with: + name: native-include-cleaner + path: include-cleaner.txt + retention-days: 30 diff --git a/native/benchmarks/size_cliffs.py b/benchmarks/size_cliffs.py similarity index 96% rename from native/benchmarks/size_cliffs.py rename to benchmarks/size_cliffs.py index 873f28661..1504f4088 100644 --- a/native/benchmarks/size_cliffs.py +++ b/benchmarks/size_cliffs.py @@ -17,7 +17,7 @@ cheapest, because fixed per-evaluation costs make small grids dearer per sample. ``--check`` exits non-zero on one. Run on a quiet machine, with the threads pinned:: - OMP_PLACES=cores OMP_PROC_BIND=close python native/benchmarks/size_cliffs.py --check + OMP_PLACES=cores OMP_PROC_BIND=close python -m benchmarks.size_cliffs --check """ from __future__ import annotations @@ -31,6 +31,10 @@ import warnings from typing import Any, cast +from solprop.driver import Simulation +from solprop.grid import FieldGrid +from solprop.stepper import RungeKutta + FACTORS = (1.0, 1.125, 1.25, 1.375, 1.5, 1.75, 2.0) LENGTH = 1.5 STEP = LENGTH / 4000 @@ -39,10 +43,6 @@ def simulation(factor: float, fft_size: str, backend: str, steps: int) -> Any: """Return the RDW case on one grid, with `steps` capped Runge-Kutta steps.""" - from solprop.driver import Simulation - from solprop.grid import FieldGrid - from solprop.stepper import RungeKutta - grid = FieldGrid.from_limits( LENGTH, 800e-9, (90e-9, 4e-6), factor * 400e-15, fft_size=fft_size ) diff --git a/native/benchmarks/statistics_cost.py b/benchmarks/statistics_cost.py similarity index 69% rename from native/benchmarks/statistics_cost.py rename to benchmarks/statistics_cost.py index 93f2d1487..a79affa5d 100644 --- a/native/benchmarks/statistics_cost.py +++ b/benchmarks/statistics_cost.py @@ -2,7 +2,18 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -"""Measure the periodic statistics channel inside propagation.""" +"""Measure the periodic statistics channel inside propagation. + +Each case runs three ways through the backend's session, interleaved round by round: +``plain`` (no periodic statistics), ``stashed`` (the electron density's plasma stash +alone) and ``watched`` (every periodic row). The JSON line reports what the channel and +its collection cost per step and per row. ``m2_reduced`` is the two-mode helium RDW +capillary and ``ctc_reduced`` the one-mode hydrogen comb-to-continuum capillary below, +both at rtol 1e-6 and atol 1e-10: the workloads of the native package's former +``statistics_cost`` harness, reproduced exactly:: + + python -m benchmarks.statistics_cost --case m2_reduced --repeats 5 +""" from __future__ import annotations @@ -13,7 +24,17 @@ import os import time import warnings -from typing import TypedDict, cast +from typing import TypedDict + +import numpy as np + +from solprop.backends.registry import select +from solprop.backends.session import Start +from solprop.backends.sessions import start_session +from solprop.driver import Simulation +from solprop.exceptions import MaterialRangeWarning +from solprop.stats import StatsSpec +from solprop.translate.run_request import RunRequest PERIODIC = ("energy", "peak_power", "fwhm_t", "electron_density") @@ -26,41 +47,43 @@ class Arm(TypedDict): rows: int -class CaseTable(TypedDict): - """Geometry and sampling values of the M2 benchmark cases (``m2_cases.py``).""" - - radius: float - length: float - pressure: float | tuple[float, float] - modes: int - lambda_min: float - lambda_max: float - trange: float - n_save: int - expected_steps: int - +M2_REDUCED = { + "radius": 125e-6, + "length": 1.0, + "gas": "He", + "pressure": (1.2, 0.0), + "modes": 2, + "wavelength": 800e-9, + "wavelength_limits": (100e-9, 1.5e-6), + "trange": 200e-15, + "fwhm": 7.5e-15, + "energy": 275e-6, + "n_save": 51, +} +"""The reduced resonant-dispersive-wave capillary: the ``rdw`` case's 7.5 fs, 275 uJ +pulse at 800 nm in helium, over 1 m at 1.2 bar falling to vacuum, two modes.""" -class FamilyTable(TypedDict): - """Pulse and gas values of the M2 benchmark cases (``m2_cases.py``).""" +CTC_REDUCED = { + "radius": 16e-6, + "length": 0.1, + "gas": "H2", + "pressure": 40.0, + "modes": 1, + "wavelength": 515e-9, + "wavelength_limits": (250e-9, 2e-6), + "trange": 2e-12, + "fwhm": 400e-15, + "energy": 10e-6, + "n_save": 21, +} +"""The reduced comb-to-continuum capillary: 400 fs, 10 uJ at 515 nm in 40 bar of +hydrogen, lossless, with Raman, over 0.1 m in one mode.""" - gas: str - wavelength: float - tau_fwhm: float - energy: float - loss: bool - -def _case_tables() -> tuple[CaseTable, FamilyTable]: - from m2_cases import CASES, FAMILIES - - return cast(CaseTable, CASES["m2_reduced"]), cast( - FamilyTable, FAMILIES["m2_reduced"] - ) - - -def _worker( +def _worker( # noqa: PLR0915 - one measurement: set up, run the arms, report backend_name: str, case_name: str, + *, target: float, repeats: int, modes: int | None, @@ -71,7 +94,7 @@ def _worker( fftw_threads = 1 native_threads = int(os.environ.get("SOLPROP_NATIVE_THREADS", "1")) if backend_name == "native": - import solprop_native + import solprop_native # noqa: PLC0415 # pyright: ignore[reportMissingImports] build = solprop_native.build_info() blas_backend = build.blas_backend @@ -84,36 +107,43 @@ def _worker( else: blas_threads = "disabled" - import numpy as np - from solprop.backends.registry import select - from solprop.backends.session import Start - from solprop.backends.sessions import start_session - from solprop.driver import Simulation - from solprop.exceptions import MaterialRangeWarning - from solprop.stats import StatsSpec - from solprop.translate.run_request import RunRequest - with warnings.catch_warnings(): warnings.simplefilter("ignore", MaterialRangeWarning) if case_name == "ctc_reduced": - from ctc import build_simulation - - simulation = build_simulation(case_name, backend_name) + case = CTC_REDUCED + simulation = Simulation.from_capillary( + case["radius"], + case["length"], + case["gas"], + case["pressure"], + wavelength=case["wavelength"], + wavelength_limits=case["wavelength_limits"], + trange=case["trange"], + fwhm=case["fwhm"], + energy=case["energy"], + modes=case["modes"], + loss=False, + raman=True, + n_save=case["n_save"], + rtol=1e-6, + atol=1e-10, + backend=backend_name, + ) else: - case, family = _case_tables() + case = M2_REDUCED simulation = Simulation.from_capillary( case["radius"], case["length"], - family["gas"], + case["gas"], case["pressure"], pressure_floor=0.0, - wavelength=family["wavelength"], - wavelength_limits=(case["lambda_min"], case["lambda_max"]), + wavelength=case["wavelength"], + wavelength_limits=case["wavelength_limits"], trange=case["trange"], - fwhm=family["tau_fwhm"], - energy=family["energy"], + fwhm=case["fwhm"], + energy=case["energy"], modes=case["modes"] if modes is None else modes, - loss=family["loss"], + loss=True, n_save=case["n_save"], rtol=1e-6, atol=1e-10, @@ -249,10 +279,10 @@ def main() -> None: _worker( arguments.backend, arguments.case, - arguments.target, - arguments.repeats, - arguments.modes, - arguments.period, + target=arguments.target, + repeats=arguments.repeats, + modes=arguments.modes, + period=arguments.period, ) diff --git a/benchmarks/worker.py b/benchmarks/worker.py index 4e6919362..0c5263cf9 100644 --- a/benchmarks/worker.py +++ b/benchmarks/worker.py @@ -71,7 +71,7 @@ def emit(kind: str, **values: object) -> None: print(canonical({"event": kind, **values}), flush=True) -HOST_SPACES = ("OpenMP", "Threads", "Serial") +HOST_SPACES = ("OpenMP", "Serial") """Kokkos execution spaces of a native CPU build; any other is an accelerator's.""" diff --git a/changelog.d/native-build-options.removed.md b/changelog.d/native-build-options.removed.md new file mode 100644 index 000000000..fed266081 --- /dev/null +++ b/changelog.d/native-build-options.removed.md @@ -0,0 +1 @@ +Building solprop-native from source: removed the `SOLPROP_NATIVE_GEMM_PROVIDER` option (every build uses the hybrid provider, `build_info().gemm_provider` is always `"HYBRID"`), the `cuda-custom`, `cuda-native` and `cuda-tpl` presets, `SOLPROP_NATIVE_BUILD_BENCHMARKS` and its GEMM microbenchmark, `SOLPROP_NATIVE_ENABLE_THREADS` (the Kokkos Threads host space; CPU builds run OpenMP) and IntelLLVM (`icpx`) support (Linux CPU builds take GCC 14 or newer, or Clang 14 or newer). A CPU build no longer needs a single-precision FFTW, and the wheels no longer carry one. diff --git a/changelog.d/native-tooling.changed.md b/changelog.d/native-tooling.changed.md new file mode 100644 index 000000000..7fee63b7c --- /dev/null +++ b/changelog.d/native-tooling.changed.md @@ -0,0 +1 @@ +The native C++ checks: every tracked C++ file is formatted, clang-tidy is pinned to LLVM 23.1 and enables whole check families, the sources compile with stronger warnings, the sanitizer build stops at the first report and checks Kokkos view bounds, and `uv run poe fmt-native` and `uv run poe tidy-native` format and tidy native files locally. diff --git a/docs/dev/backends.md b/docs/dev/backends.md index 05871125a..0d4d0a2a2 100644 --- a/docs/dev/backends.md +++ b/docs/dev/backends.md @@ -803,13 +803,13 @@ mask and the in-stage absorber preserve it, and the measured saving are `nsz`, approximate functions and reciprocals, and flush-to-zero forbidden). On `native`, as `native/AGENTS.md` and `native/CMakeLists.txt` set it: - **Contraction.** A CPU build compiles with `-ffp-contract=fast` under GCC and - `-ffp-contract=fast-honor-pragmas` under Clang and `icpx` (plain `fast` makes Clang + `-ffp-contract=fast-honor-pragmas` under Clang (plain `fast` makes Clang ignore the pragmas), so the compiler may fuse any multiply and add into one rounding. Its one hazard is exact cancellation (`x*y - y*x` need not be zero). `build_info().optimization_policy` records the flag. - **Strict blocks.** A protected function or lambda carries `SOLPROP_NATIVE_STRICT_FUNCTION` and starts its body with `SOLPROP_NATIVE_STRICT_BLOCK` - (`include/solprop_native/fp.hpp`): under Clang and `icpx` a `#pragma clang fp + (`include/solprop_native/fp.hpp`): under Clang a `#pragma clang fp contract(off)`, which survives inlining; under GCC, which ignores the pragma, `optimize("fp-contract=off")` on the function, so a protected helper is a call there. Both settings are lexical, and `Kokkos::complex`'s operators keep their header's diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index b43d5cde4..0c2a23454 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -508,7 +508,7 @@ against 2.40 s), where it was 1.4 times. (smooth-size-cliffs)= ### Smooth grid sizes -`native/benchmarks/size_cliffs.py` times the four-mode RDW capillary per source +`python -m benchmarks.size_cliffs` times the four-mode RDW capillary per source evaluation (short and long capped Runge-Kutta windows, their difference) on 2-3-5-smooth and power-of-two grids, and flags a size whose cost per sample exceeds its nearest power of two's by more than 25 %. Two size boundaries made cliffs: the stage threshold (8192 diff --git a/docs/dev/dependencies.md b/docs/dev/dependencies.md index 461700234..1ea7326b4 100644 --- a/docs/dev/dependencies.md +++ b/docs/dev/dependencies.md @@ -119,6 +119,13 @@ drawsvg (the structure figures of `docs/dev/figures/build.py` and the geometry f `docs/guide/figures/geometries.py` only, run by hand with `poe figures`; the SVGs it writes are committed, so the documentation builds without it). +The native gate's C++ tools are pinned exactly in the dev group, so that its checks do +not depend on a machine's LLVM: **clang-format 23.1.0** (`poe fmt-native` and the lint +stage) and **clang-tidy 23.1.4** (`poe tidy-native` and the tidy stage; its PyPI wheels +start at 23.1.3, the nearest to the formatter's release). clang-tidy is installed on +Linux and macOS only: the gate that runs it is bash, and it has no PyPI wheel for +Windows on Arm, where a sync would build LLVM from source. + ## Candidates (not approved; only evaluate for a selected use case) | package | for | status | diff --git a/docs/dev/tooling.md b/docs/dev/tooling.md index 2810941f6..ce87bf1d5 100644 --- a/docs/dev/tooling.md +++ b/docs/dev/tooling.md @@ -159,6 +159,37 @@ ruff (check with `--fix`, then format), numpydoc validation, codespell, `uv-lock `reuse lint-file` on the staged files. Check the `rev` pins against current releases when updating. +## Native C++ tooling + +The native backend's C++ (`native/`) has its own tools, judged against this project: a +double-precision Kokkos code with one process-wide runtime, no untrusted input, a +Python oracle for every result, and end-to-end runs as the measure of speed. + +| tool | status | where, or why not | +|---|---|---| +| clang-format (pinned 23.1.0) | adopted | every tracked C++ file, in `check-native`'s lint stage; `poe fmt-native` formats in place | +| clang-tidy (pinned 23.1.4), whole families | adopted | `bugprone`, `performance`, `modernize`, `misc`, `readability`, `clang-analyzer` and `cppcoreguidelines-special-member-functions`, each exclusion stated in `native/.clang-tidy`; warnings are errors; the tidy stage; `poe tidy-native` for changed files | +| compiler warnings | adopted | `-Wall -Wextra -Wpedantic -Wconversion -Wsign-conversion -Wshadow -Wundef -Wnon-virtual-dtor -Woverloaded-virtual`, with Clang's `-Wimplicit-fallthrough` and GCC's `-Wduplicated-cond -Wduplicated-branches -Wlogical-op`; errors in development and CI builds | +| ASan + UBSan with Kokkos bounds checks | adopted | the `cpu-sanitize` preset over the GoogleTest suite, UBSan stopping at its first report | +| compute-sanitizer (memcheck, racecheck, initcheck, synccheck) | adopted | `.github/workflows/cuda.yml`, on a device | +| Clang source coverage | scheduled | `.github/workflows/native-deep.yml`, weekly: GoogleTest and native pytest, a report whose zero-count lines are the dead-code candidates | +| ASan + UBSan on the extension | scheduled | `native-deep.yml`: the native pytest suite through the bindings | +| `misc-include-cleaner` | scheduled | `native-deep.yml`, a report; too noisy across Kokkos's umbrella headers to gate | +| perf, Hotspot, Instruments, Nsight Systems and Compute, rocprof, optimisation remarks, llvm-mca, Compiler Explorer | ad hoc | profiling and code-generation questions, by hand; the deciding number is `python -m benchmarks ab` | +| cppcheck | rejected | overlaps clang-tidy and misreads Kokkos lambdas | +| include-what-you-use (the binary) | rejected | `misc-include-cleaner` gives the same report from the pinned clang-tidy | +| CodeChecker, scan-build | rejected | front ends to the Clang static analyser, which `clang-analyzer-*` runs | +| GCC `-fanalyzer` | rejected | C-oriented and noisy on C++ templates; `clang-analyzer-*` covers it | +| CodeQL, SonarQube | rejected | security analysis of untrusted input, which this code has none of | +| ThreadSanitizer | rejected for the gate | needs an instrumented libomp, FFTW and BLAS; the thread-invariance probes cover the outcomes. An on-demand campaign only | +| MemorySanitizer, TypeSanitizer | rejected | MSan needs every library instrumented; TySan is experimental and strict aliasing is not a risk here | +| Valgrind | rejected | ASan covers it, much faster, and it does not run on Apple silicon | +| Clang thread-safety annotations | rejected | there is one mutex | +| Google Benchmark | rejected | microbenchmarks mislead here; the deciding number is the end-to-end `benchmarks ab` | +| Tracy, HPCToolkit | rejected | Kokkos Tools and the profilers above cover them | +| fuzzers (libFuzzer, RapidCheck) | rejected | the seam's inputs are generated by Hypothesis in the Python tests | +| MPFR | rejected | mpmath oracles in the Python tests give the high-precision references | + ## Sphinx `docs/conf.py`: myst-parser, numpydoc (without napoleon), autosummary (recursive, `generate = True`), diff --git a/docs/guide/performance/building-native.md b/docs/guide/performance/building-native.md index bf3444f02..88e778c77 100644 --- a/docs/guide/performance/building-native.md +++ b/docs/guide/performance/building-native.md @@ -26,11 +26,11 @@ the arithmetic is double precision throughout, and no build enables fast-math. | CMake | 3.23 or newer (with build isolation, scikit-build-core fetches one from PyPI if none is found) | | build tool | Ninja is recommended; scikit-build-core uses it when available | | C++20 compiler, macOS | Clang 14 or newer; Homebrew LLVM is recommended and selected by default | -| C++20 compiler, Linux CPU | GCC 14 or newer, IntelLLVM (`icpx`) 2022 or newer, or Clang 14 or newer / AMD AOCC | +| C++20 compiler, Linux CPU | GCC 14 or newer, or Clang 14 or newer / AMD AOCC | | C++20 compiler, CUDA | GCC 14 or newer as the host compiler, with `nvcc` from CUDA 12.8 or newer (qualified on CUDA 13) | | C++20 compiler, HIP | ROCm's Clang (`hipcc` or `amdclang++`) | | OpenMP (CPU builds) | the compiler's runtime; on macOS Homebrew's `libomp` | -| FFT (CPU builds) | FFTW 3 with its threaded libraries, in double and single precision (or MKL's FFTW interface) | +| FFT (CPU builds) | FFTW 3 in double precision with its threaded libraries (or MKL's FFTW interface); the build needs no single-precision FFTW | | BLAS (CPU builds) | optional but recommended: Accelerate (macOS), OpenBLAS, MKL or another LP64 BLAS | | GPU libraries | cuFFT and cuBLAS (CUDA), or rocFFT and rocBLAS (HIP) | | network | configuring fetches the pinned dependencies below from GitHub (see [offline builds](#offline-builds)) | @@ -49,8 +49,7 @@ them: The Python build requirements are scikit-build-core 0.11 and nanobind 2.9 or newer. CPU builds contract multiply-adds (`-ffp-contract=fast` with GCC, -`-ffp-contract=fast-honor-pragmas` with Clang and `icpx`, after `-fno-fast-math` and -`-fp-model=precise`) except in a few protected blocks, as +`-ffp-contract=fast-honor-pragmas` with Clang, after `-fno-fast-math`) except in a few protected blocks, as [SEP 25](../../dev/seps/archive/sep-0025-rounding-level-reproducibility.md) sets out; `build_info().compiler_flags` and `optimization_policy` record the setting. GPU builds follow their compilers' defaults throughout ([GPU builds](#gpu-builds)). @@ -82,7 +81,7 @@ export CMAKE_ARGS="-DCMAKE_OSX_SYSROOT=$SDKROOT" ``` For an Apple silicon performance build, the pinned dependency recipe enables FFTW -3.3.11's NEON codelets in **both precisions**. Compiling scalar code with `-mcpu` +3.3.11's NEON codelets. Compiling scalar code with `-mcpu` does not enable those codelets. The recipe also retains the Arm cycle counter needed by measured planning, the Apple M1 distribution baseline and the macOS 14 deployment target. Its FFTW threading libraries and the native extension share one source-built @@ -101,9 +100,8 @@ uv run --no-sync python -m solprop_native Check the reported `fft_library` and `openmp_library` before benchmarking. The [pinned upstream configuration](https://github.com/FFTW/fftw3/blob/fftw-3.3.11/configure.ac) -supports `--enable-neon` for double and single precision on AArch64. The installed -libraries identify their codelet family in `fftw_version` and `fftwf_version`, which -both end in `-neon` for this recipe. Keep these prefixes separate from a comparison's +supports `--enable-neon` on AArch64. The installed library identifies its codelet family +in `fftw_version`, which ends in `-neon` for this recipe. Keep these prefixes separate from a comparison's baseline libraries. See [OpenMP runtimes](#openmp-runtimes) below for the limits of mixing independently bundled wheels. @@ -124,7 +122,7 @@ cd solprop CXX=g++-14 uv pip install --python /path/to/env/bin/python . ./native ``` -`libfftw3-dev` provides the OpenMP and pthreads FFTW libraries in both precisions. For +`libfftw3-dev` provides the OpenMP and pthreads FFTW libraries. For MKL instead of OpenBLAS, source oneAPI first (`source /opt/intel/oneapi/setvars.sh`): a set `MKLROOT` makes the automatic search choose MKL. Other compilers and libraries are in [Linux compilers and libraries](#linux-compilers-and-libraries). @@ -226,7 +224,7 @@ hand through `CXXFLAGS` appear in `compiler_flags`. sources under `native/` are not the ones the extension was compiled from (it names `uv sync`; a wheel skips this); when the installed solprop lacks a driver member or bundle field the build relies on (it names them); when the build enables fast-math; when the execution-space record is inconsistent; when a GPU build sees no -device; when the GEMM provider is unknown or its BLAS is missing or unidentified; when +device; when the GEMM provider is unknown or its BLAS is unidentified; when a reference build carries a vendor BLAS or machine-specific code, or a distribution build carries native code or no BLAS; and when a GPU build's BLAS is not the device's. Then, in solprop: @@ -305,7 +303,7 @@ order. Keep each profile in its own build directory; switching the profile in on rebuilds everything. Configuring prints one summary line, for example ```text --- solprop-native build profile: local; BLAS Accelerate (automatic: Accelerate (macOS)); GEMM provider HYBRID; native architecture ON; CPU baseline none; IPO ON (Release builds) +-- solprop-native build profile: local; BLAS Accelerate (automatic: Accelerate (macOS)); native architecture ON; CPU baseline none; IPO ON (Release builds) ``` and `build_info()` reports the same choice in `build_profile`, `cpu_baseline`, @@ -356,7 +354,6 @@ The options are read by `native/CMakeLists.txt` and `native/cmake/HostLibraries. | option | default | values and effect | |---|---|---| | `SOLPROP_NATIVE_ENABLE_OPENMP` | `ON` | the OpenMP host execution space and threaded FFTW. `OFF` in GPU builds, whose host space is Serial | -| `SOLPROP_NATIVE_ENABLE_THREADS` | `OFF` | the Kokkos Threads (C++ threads) host space with FFTW's threads library, for a build that adds no OpenMP runtime (a fallback for Windows builds); requires `SOLPROP_NATIVE_ENABLE_OPENMP=OFF`. Kernels then run at the runtime's thread count, without per-operation sizing | | `SOLPROP_NATIVE_ENABLE_CUDA` | `OFF` | the CUDA execution space with cuFFT (Linux, GNU host compiler) | | `SOLPROP_NATIVE_ENABLE_HIP` | `OFF` | the HIP execution space with rocFFT (Linux, ROCm Clang); exclusive with CUDA | | `SOLPROP_NATIVE_CUDA_ARCH` | `HOPPER90` | the Kokkos architecture suffix, `Kokkos_ARCH_` ([CUDA](#cuda)) | @@ -377,14 +374,13 @@ The options are read by `native/CMakeLists.txt` and `native/cmake/HostLibraries. | `SOLPROP_NATIVE_BLAS_ROOT` | empty | a prefix searched first for the BLAS | | `SOLPROP_NATIVE_BLAS_LIBRARIES` | empty | an explicit `;`-separated library list (not together with `SOLPROP_NATIVE_BLAS_VENDOR`) | | `SOLPROP_NATIVE_MKL_ROOT` | `$MKLROOT` | where the LP64 MKL libraries are looked for | -| `SOLPROP_NATIVE_GEMM_PROVIDER` | `HYBRID` | `HYBRID` (vendor GEMM for eligible transforms, custom kernels for the rest), `KOKKOSBLAS_TPL` (requires a BLAS), `KOKKOSBLAS_NATIVE` or `CUSTOM`; the alternatives are for comparison and validation | **FFT** | option | default | values and effect | |---|---|---| | `SOLPROP_NATIVE_FFT_PROVIDER` | `FFTW` | `FFTW`, or `MKL_FFTW` (MKL's FFTW3 interface, Linux). Host builds only | -| `SOLPROP_NATIVE_FFTW_THREADING` | `OPENMP` | `OPENMP` (`libfftw3_omp`) or `PTHREADS` (`libfftw3_threads`); IntelLLVM with FFTW requires `PTHREADS` | +| `SOLPROP_NATIVE_FFTW_THREADING` | `OPENMP` | `OPENMP` (`libfftw3_omp`) or `PTHREADS` (`libfftw3_threads`) | | `SOLPROP_NATIVE_RAMAN_FFT_PROVIDER` | `ACCELERATE` on macOS CPU builds, else `KOKKOSFFT` | construction-time Raman selection: with NEON FFTW, vDSP handles short parallel power-of-two rows (up to 16,384 samples); serial and longer rows use Kokkos-FFT. Without NEON, only short serial rows use Kokkos-FFT. `KOKKOSFFT` forces that provider; `ACCELERATE` is macOS CPU only | | `FFTW_ROOT` | the Homebrew prefix on macOS | a non-default FFTW installation | @@ -396,7 +392,6 @@ The options are read by `native/CMakeLists.txt` and `native/cmake/HostLibraries. | `SOLPROP_NATIVE_WARNINGS_AS_ERRORS` | as `BUILD_TESTING`: `OFF` in pip and uv builds, `ON` in the presets | `-Werror` on the package's own sources. Continuous integration turns it on for its wheels too | | `SOLPROP_NATIVE_ENABLE_CLANG_TIDY` | `OFF` | runs clang-tidy while compiling the package's sources | | `SOLPROP_NATIVE_ENABLE_SANITIZERS` | `OFF` | AddressSanitizer and UBSan on the package's targets; CPU builds only | -| `SOLPROP_NATIVE_BUILD_BENCHMARKS` | `OFF` | builds the `solprop_native_gemm_benchmark` microbenchmark (never installed) | | `SOLPROP_NATIVE_BUNDLED_LICENCES` | the environment variable of that name | a directory of `LICENSE-*.txt` files, installed beside the extension: the licences of the libraries a wheel repair copies in (FFTW, the OpenMP runtime) | The installed module of a local build keeps the directories of a BLAS it linked outside @@ -407,11 +402,12 @@ OpenBLAS is. ### Presets `native/CMakePresets.json` defines development builds, all with `BUILD_TESTING=ON`: -`cpu-debug`, `cpu-tidy` and `cpu-sanitize` (reference profile, Debug), `cpu-benchmark` -(local, Release, IPO and benchmarks), the CUDA builds `cuda` (hybrid GEMM with cuBLAS), -`cuda-tpl`, `cuda-native` and `cuda-custom` (the other GEMM providers, for comparison), -and `hip`. The presets do not choose a compiler: pass `-DCMAKE_CXX_COMPILER=...` (on -macOS, Homebrew's `clang++`) and `-DPython_EXECUTABLE=...`. +`cpu-debug`, `cpu-tidy` and `cpu-sanitize` (reference profile, Debug; `cpu-sanitize` +adds ASan, UBSan that stops at its first report, and Kokkos's view bounds checks), +`cpu-benchmark` (local, Release, IPO), `cuda` (with cuBLAS) and `hip`. Dense transforms +take the vendor GEMM where a transform is eligible and the package's own kernels for the +rest, in every build. The presets do not choose a compiler: pass +`-DCMAKE_CXX_COMPILER=...` (on macOS, Homebrew's `clang++`) and `-DPython_EXECUTABLE=...`. ### Offline builds @@ -461,22 +457,20 @@ pinned whichever is used. | compiler | notes | |---|---| | GCC 14 or newer | the reference route, with GNU OpenMP (`libgomp`). GCC 13 is refused: it compiles the strict floating-point helpers through the stack, and coupled Multistep steps took 1.25 to 1.45 times as long as with GCC 14 | -| IntelLLVM (`icpx`) 2022 or newer | compiled with `-fp-model=precise` (its default is fast floating point) and then `-ffp-contract=fast-honor-pragmas`; requires `SOLPROP_NATIVE_FFTW_THREADING=PTHREADS` with FFTW, because distribution OpenMP FFTW libraries link GNU OpenMP; uses the compiler's own archiver for LTO | | Clang 14 or newer, AMD AOCC | compiled with `-fno-fast-math`, then `-ffp-contract=fast-honor-pragmas` | | provider | how to select it | |---|---| | OpenBLAS | the automatic choice when MKL is not configured; `-DSOLPROP_NATIVE_BLAS_VENDOR=OPENBLAS` forces it, and `-DBLAS_openblas_LIBRARY=/path/to/libopenblas.so` names a specific library | -| MKL BLAS | `source /opt/intel/oneapi/setvars.sh` (sets `MKLROOT`), `-DSOLPROP_NATIVE_MKL_ROOT=...` or `-DSOLPROP_NATIVE_BLAS_VENDOR=MKL`. LP64 interface; GCC links MKL's GNU threading layer, IntelLLVM its Intel layer | +| MKL BLAS | `source /opt/intel/oneapi/setvars.sh` (sets `MKLROOT`), `-DSOLPROP_NATIVE_MKL_ROOT=...` or `-DSOLPROP_NATIVE_BLAS_VENDOR=MKL`. LP64 interface, with MKL's GNU threading layer | | AOCL-BLIS and other BLAS | `-DSOLPROP_NATIVE_BLAS_LIBRARIES="/path/libblis-mt.so;/path/libflame.so"` (BLIS alone lacks complex rotations that Kokkos Kernels' BLAS layer links; AOCL's LP64 libFLAME supplies them and needs `libaoclutils` on the loader path) | | FFTW | the default; `-DFFTW_ROOT=...` for a non-system installation, such as AOCL's FFTW | | MKL FFT | `-DSOLPROP_NATIVE_FFT_PROVIDER=MKL_FFTW`: MKL's FFTW3 interface, independent of the BLAS choice. MKL ignores FFTW's planning flags. Sourcing oneAPI never selects it by itself | **FFTW threading.** `OPENMP` links FFTW's OpenMP library, which must use the same OpenMP runtime as the rest of the build: GNU `libgomp` for distribution packages built with -GCC. `PTHREADS` links FFTW's pthreads library instead; it is required for IntelLLVM and -is the safer choice whenever a second OpenMP runtime could otherwise be loaded. With -pthreads FFTW, start with `OMP_PROC_BIND=false`: a bound OpenMP main thread can confine +GCC. `PTHREADS` links FFTW's pthreads library instead; it is the safer choice whenever +a second OpenMP runtime could otherwise be loaded. With pthreads FFTW, start with `OMP_PROC_BIND=false`: a bound OpenMP main thread can confine the FFTW worker threads it creates to its own core. **OpenBLAS threading.** A pthreads OpenBLAS (the Debian and Ubuntu default) keeps its own @@ -537,8 +531,8 @@ export CMAKE_ARGS="-DSOLPROP_NATIVE_ENABLE_CUDA=ON -DSOLPROP_NATIVE_ENABLE_OPENM uv pip install --python /path/to/env/bin/python . ./native ``` -The `cuda` preset configures the same build for development, with the native tests and -the GEMM benchmark (`cmake --preset cuda -DPython_EXECUTABLE=... -DCUDA_ROOT=...`, from +The `cuda` preset configures the same build for development, with the native tests +(`cmake --preset cuda -DPython_EXECUTABLE=... -DCUDA_ROOT=...`, from `native/`). ### HIP diff --git a/native/.clang-format b/native/.clang-format index acbf22140..15a0947b2 100644 --- a/native/.clang-format +++ b/native/.clang-format @@ -7,3 +7,25 @@ ColumnLimit: 88 DerivePointerAlignment: false PointerAlignment: Left SortIncludes: CaseSensitive +InsertNewlineAtEOF: true +# A NOLINT comment is never reflowed: its reason must stay on the suppressing line. +CommentPragmas: '^ (IWYU pragma:|NOLINT)' +# No RemoveSemicolon: clang-format parses `const auto f = KOKKOS_LAMBDA(...) {...};` as a +# function definition and removes the semicolon the declaration needs. Declaring the +# macro (`Macros:`) fixes that but makes formatting general.cpp minutes slower. +# Include blocks: this package's and the directory's own headers ("..."), then the +# third-party libraries, then the standard library, then platform headers (the operating +# system, the compiler and the device runtimes). An include under #if stays in its own +# block, where it is. +IncludeBlocks: Regroup +IncludeCategories: + - Regex: '^"' + Priority: 1 + - Regex: '^<(Kokkos|impl/Kokkos|nanobind/|fftw3\.h|sleef\.h|gtest/)' + Priority: 2 + CaseSensitive: true + - Regex: '^<[a-z_]+>$' + Priority: 3 + CaseSensitive: true + - Regex: '^<' + Priority: 4 diff --git a/native/.clang-tidy b/native/.clang-tidy index cca785169..6609abb16 100644 --- a/native/.clang-tidy +++ b/native/.clang-tidy @@ -2,43 +2,93 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +# Whole check families. The exclusions, one reason each: +# - bugprone-easily-swappable-parameters: numerical kernels take runs of same-typed +# extents and scalars by design. +# - performance-enum-size: the enums are few and small; a narrower base type gains +# nothing. +# - modernize-use-trailing-return-type: style only; the code uses leading return types. +# - modernize-avoid-c-arrays: small fixed arrays are the idiom in device code. +# - misc-non-private-member-variables-in-classes: plain aggregates (Problem, plans, +# records) are the seam's data. +# - misc-include-cleaner: too noisy across Kokkos's umbrella headers to gate; it runs +# as a weekly report (.github/workflows/native-deep.yml). +# - misc-no-recursion: recursion is a design choice in host code, not a defect. +# - misc-const-correctness: it judges one template instantiation and the active #if +# branch, so it marks const the locals another instantiation's `if constexpr` or a +# device branch assigns (its fixes broke the CPU build in about 30 places). +# - readability-magic-numbers: the numerical constants are cited where they appear. +# - readability-identifier-length: short names follow the mathematics (dz, n, k). +# - readability-math-missing-parentheses: precedence in arithmetic is not ambiguous. +# - readability-uppercase-literal-suffix: style only. +# - readability-trailing-comma: style only, and clang-format owns layout. +# - readability-braces-around-statements: the LLVM convention leaves single +# statements unbraced, and adding braces around bodies with #if is unsafe. +# - readability-function-size, readability-function-cognitive-complexity: the long +# setup functions and hot kernels are split first (a later change), then gated. +# - readability-avoid-nested-conditional-operator: nested conditionals initialise +# const locals, in kernels too; an immediately invoked lambda reads worse. +# - readability-convert-member-functions-to-static: it sees the CPU build only, where +# members such as hybrid_shape_eligible() read no state; their device branches do. +# - readability-use-std-min-max: `if (a > m) m = a` keeps m on a NaN where std::max +# takes the NaN, and std::max is not device code. +# - readability-make-member-function-const: Kokkos views are handles, so a member +# that writes through one would be marked const although it changes the state. +# - clang-analyzer-osx.*: Apple framework checks; the code calls no Objective-C or +# Core Foundation API. +# - clang-analyzer-cplusplus.NewDeleteLeaks: it reports Kokkos's shared allocation +# records (freed through a reference count it cannot follow) as leaks; the code holds +# no raw ownership (native/AGENTS.md), and the ASan builds check for leaks. +# - clang-analyzer-optin.performance.Padding: the cold configuration structs are +# ordered by concern, not by size. Checks: > -*, - clang-analyzer-core.*, - clang-analyzer-deadcode.*, - clang-analyzer-security.*, - bugprone-argument-comment, - bugprone-assert-side-effect, - bugprone-exception-escape, - bugprone-infinite-loop, - bugprone-macro-parentheses, - bugprone-multiple-statement-macro, - bugprone-parent-virtual-call, - bugprone-return-const-ref-from-parameter, - bugprone-sizeof-expression, - bugprone-string-constructor, - bugprone-suspicious-enum-usage, - bugprone-suspicious-memory-comparison, - bugprone-suspicious-realloc-usage, - bugprone-unhandled-exception-at-new, - bugprone-unused-local-non-trivial-variable, - performance-for-range-copy, - performance-implicit-conversion-in-loop, - performance-inefficient-algorithm, - performance-inefficient-string-concatenation, - performance-move-const-arg, - performance-no-automatic-move, - performance-type-promotion-in-math-fn, - performance-unnecessary-copy-initialization, + bugprone-*, + -bugprone-easily-swappable-parameters, + performance-*, + -performance-enum-size, + modernize-*, + -modernize-use-trailing-return-type, + -modernize-avoid-c-arrays, + misc-*, + -misc-non-private-member-variables-in-classes, + -misc-include-cleaner, + -misc-no-recursion, + -misc-const-correctness, + readability-*, + -readability-magic-numbers, + -readability-identifier-length, + -readability-math-missing-parentheses, + -readability-uppercase-literal-suffix, + -readability-trailing-comma, + -readability-braces-around-statements, + -readability-function-size, + -readability-function-cognitive-complexity, + -readability-avoid-nested-conditional-operator, + -readability-make-member-function-const, + -readability-use-std-min-max, + -readability-convert-member-functions-to-static, + cppcoreguidelines-special-member-functions, + clang-analyzer-*, + -clang-analyzer-osx.*, + -clang-analyzer-optin.performance.Padding, + -clang-analyzer-cplusplus.NewDeleteLeaks, portability-restrict-system-includes, - portability-simd-intrinsics, - modernize-use-equals-default, - modernize-use-equals-delete, - modernize-use-nullptr, - modernize-use-override, - readability-avoid-const-params-in-decls, - readability-container-size-empty, - readability-redundant-* + portability-simd-intrinsics WarningsAsErrors: '*' HeaderFilterRegex: '.*/(include|src)/.*' FormatStyle: file +CheckOptions: + # `.value()` is a checked access: it throws std::bad_optional_access. + bugprone-unchecked-optional-access.IgnoreValueCalls: true + # A cast to void is an explicit discard (unique_ptr::release once a capsule owns it). + bugprone-unused-return-value.AllowCastToVoid: true + # A shift or mask by a positive literal (`std::size_t{64} << 20`) is not a sign bug. + bugprone-signed-bitwise.IgnorePositiveIntegerLiterals: true + # De Morgan's rewrite of `!(a <= x && x <= b)` changes what a NaN gives: the guards + # are written to refuse NaN. + readability-simplify-boolean-expr.SimplifyDeMorgan: false + # Reference-counted handles, passed by value by convention: Kokkos views (and the + # aliases named ...View), and nanobind's handles and ndarrays (nb::dict, nb::object, + # nb::handle, nb::ndarray and the bindings' ...Input and ...Array aliases of it). + performance-unnecessary-value-param.AllowedTypes: 'View;nb::;nanobind::;Input$;Array$' diff --git a/native/AGENTS.md b/native/AGENTS.md index 4cad24cba..07cd54577 100644 --- a/native/AGENTS.md +++ b/native/AGENTS.md @@ -77,6 +77,8 @@ The native gate runs from the repository root: ```console uv run poe check-native # native/scripts/check.sh uv run poe check-all # solprop's gate (poe check), then the native gate +uv run poe fmt-native # clang-format every tracked C++ file in place +uv run poe tidy-native # clang-tidy the C++ files changed from main (or those named) ``` `check-native` covers this package's Python format, lint and strict typing, C++ format, @@ -94,8 +96,16 @@ wheel on Linux, macOS and Windows, cached under a hash of `native/`, and solprop default lane runs on it on macOS and Windows with the native backend selected, and in the coverage lane on Linux. The `device-compile` lanes compile the CUDA and HIP targets, tests included, on a change to `native/`, with no device. Running on one stays by hand -(`.github/workflows/cuda.yml` on a self-hosted runner, or `pytest -m gpu` on a machine -with a device). +(`.github/workflows/cuda.yml` on a self-hosted runner, with compute-sanitizer's four +tools, or `pytest -m gpu` on a machine with a device). `.github/workflows/native-deep.yml` +runs weekly and reports without gating: source coverage over the native tests, the +extension under ASan and UBSan through the native pytest suite, and +`misc-include-cleaner`. The tools adopted, scheduled and rejected are in +`docs/dev/tooling.md`, "Native C++ tooling". + +Formatting and clang-tidy are pinned in the `dev` group (LLVM 23.1); `.clang-tidy` +enables whole check families with each exclusion stated. A `NOLINT` names its check +and gives its reason on the same line. New behaviour needs a test; a bug fix needs a regression test. Never disable a test, widen a tolerance after seeing a failure, lower warnings or hide a diagnostic to pass @@ -127,8 +137,8 @@ the gate. contracts (lossless copies, integer accounting, identity metadata, hard-mask zeros, explicit no-ops) stay exact. - Kernels may reorder, vectorise, thread and contract. Contraction is on by default - (`-ffp-contract=fast` for GCC, `-ffp-contract=fast-honor-pragmas` for Clang and - `icpx`: plain `fast` makes Clang ignore the pragmas). A protected block (a stable + (`-ffp-contract=fast` for GCC, `-ffp-contract=fast-honor-pragmas` for Clang: plain + `fast` makes Clang ignore the pragmas). A protected block (a stable increment, a small-argument function, a defect or a compensated sum) stays strict with `SOLPROP_NATIVE_STRICT_FUNCTION` and `SOLPROP_NATIVE_STRICT_BLOCK` (`include/solprop_native/fp.hpp`; GCC ignores `#pragma STDC FP_CONTRACT`) and spells diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 7d5fd206c..8bb38be9b 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -78,8 +78,8 @@ elseif(SOLPROP_NATIVE_ENABLE_CUDA AND "The NVIDIA validation build requires GNU C++; " "found ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}.") elseif(NOT SOLPROP_NATIVE_ENABLE_HIP AND - NOT CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|IntelLLVM|Clang|AMDClang)$") - message(FATAL_ERROR "Linux CPU builds require GNU C++, IntelLLVM, or Clang/AOCC.") + NOT CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AMDClang)$") + message(FATAL_ERROR "Linux CPU builds require GNU C++ or Clang/AOCC.") elseif(SOLPROP_NATIVE_ENABLE_CUDA AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14) # The CPU reference compiler below, for the host pass of nvcc_wrapper too: GCC 13 @@ -115,48 +115,17 @@ if(NOT APPLE AND NOT SOLPROP_NATIVE_ENABLE_CUDA AND add_compile_options(-fno-fast-math) endif() -if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 2022) - message(FATAL_ERROR "Kokkos CPU builds require IntelLLVM 2022 or newer.") - endif() - # icpx defaults to fast floating-point transformations. The adaptive error - # estimate requires precise arithmetic in project and dependency templates. - add_compile_options(-fp-model=precise) - # CMake can find the distribution's older LLVM archiver even when icpx is - # newer. LTO objects require the archive tools shipped with that compiler. - foreach(_tool IN ITEMS ar ranlib) - execute_process(COMMAND "${CMAKE_CXX_COMPILER}" "-print-prog-name=llvm-${_tool}" - OUTPUT_VARIABLE _intel_tool OUTPUT_STRIP_TRAILING_WHITESPACE - COMMAND_ERROR_IS_FATAL ANY) - if(NOT EXISTS "${_intel_tool}") - message(FATAL_ERROR "icpx could not locate its llvm-${_tool}: ${_intel_tool}") - endif() - string(TOUPPER "${_tool}" _tool_upper) - set(CMAKE_CXX_COMPILER_${_tool_upper} "${_intel_tool}" CACHE FILEPATH - "Archive tool matching IntelLLVM bitcode" FORCE) - set(CMAKE_CXX_COMPILER_${_tool_upper} "${_intel_tool}") - endforeach() - # CMake 3.28 expands these paths during project(), before the correction above. - set(CMAKE_CXX_ARCHIVE_CREATE_IPO - "\"${CMAKE_CXX_COMPILER_AR}\" cr ") - set(CMAKE_CXX_ARCHIVE_APPEND_IPO - "\"${CMAKE_CXX_COMPILER_AR}\" r ") - set(CMAKE_CXX_ARCHIVE_FINISH_IPO - "\"${CMAKE_CXX_COMPILER_RANLIB}\" ") -endif() - # solprop SEP 25: floating-point contraction (fused multiply-add) is on by default for # CPU builds, and the few protected blocks opt out (include/solprop_native/fp.hpp). # Clang's plain `-ffp-contract=fast` ignores `#pragma clang fp contract(off)` and # `#pragma STDC FP_CONTRACT OFF`; `fast-honor-pragmas` contracts across statements and # keeps the protected blocks strict. GCC ignores both pragmas: its protected functions -# carry `optimize("fp-contract=off")` instead. icpx is Clang-based and takes the Clang -# form after `-fp-model=precise`, which alone contracts within a statement only. The -# flag is added after `-fno-fast-math` and `-fp-model=precise`, so neither resets it. +# carry `optimize("fp-contract=off")` instead. The flag is added after +# `-fno-fast-math`, so that does not reset it. # Device builds keep their compilers' defaults (nvcc and HIP Clang already contract). set(SOLPROP_NATIVE_FP_CONTRACT_FLAG "") if(NOT SOLPROP_NATIVE_ENABLE_CUDA AND NOT SOLPROP_NATIVE_ENABLE_HIP) - if(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|AppleClang|AMDClang|IntelLLVM)$") + if(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|AppleClang|AMDClang)$") set(SOLPROP_NATIVE_FP_CONTRACT_FLAG "-ffp-contract=fast-honor-pragmas") elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(SOLPROP_NATIVE_FP_CONTRACT_FLAG "-ffp-contract=fast") @@ -164,11 +133,6 @@ if(NOT SOLPROP_NATIVE_ENABLE_CUDA AND NOT SOLPROP_NATIVE_ENABLE_HIP) endif() if(SOLPROP_NATIVE_FP_CONTRACT_FLAG) add_compile_options("${SOLPROP_NATIVE_FP_CONTRACT_FLAG}") - # icpx reports the deliberate override of `-fp-model=precise`'s contraction as a - # warning, which `-Werror` would turn into a failed build. - if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - add_compile_options(-Wno-overriding-option) - endif() endif() if(WIN32) @@ -249,16 +213,6 @@ endif() # SOLPROP_NATIVE_*_THREADS pools, all read once at import; OMP_NUM_THREADS does not # set these counts. option(SOLPROP_NATIVE_ENABLE_OPENMP "Use OpenMP in Kokkos and FFTW" ON) -# Kokkos Threads (C++ threads) is the host space of a build that must add no OpenMP -# runtime to the process: a fallback for Windows builds (solprop SEP 29). FFTW then uses its own -# threads library, Win32 threads on Windows and POSIX threads elsewhere. It replaces -# OpenMP, so it needs SOLPROP_NATIVE_ENABLE_OPENMP=OFF. -option(SOLPROP_NATIVE_ENABLE_THREADS "Use Kokkos Threads and FFTW's threads library" OFF) -if(SOLPROP_NATIVE_ENABLE_THREADS AND SOLPROP_NATIVE_ENABLE_OPENMP) - message(FATAL_ERROR - "SOLPROP_NATIVE_ENABLE_THREADS replaces OpenMP; also set " - "SOLPROP_NATIVE_ENABLE_OPENMP=OFF.") -endif() # Three build profiles. LOCAL (the default) is the fast build for the machine it is # built on: a vendor BLAS, that machine's own architecture and checked IPO. # DISTRIBUTION is the build for wheels that run on other machines: a documented CPU @@ -289,12 +243,6 @@ option(SOLPROP_NATIVE_ENABLE_CLANG_TIDY "Run clang-tidy while compiling project-owned C++ sources" OFF) option(SOLPROP_NATIVE_ENABLE_SANITIZERS "Instrument project targets with AddressSanitizer and UBSan" OFF) -option(SOLPROP_NATIVE_BUILD_BENCHMARKS - "Build local native microbenchmarks (never installed)" OFF) -set(SOLPROP_NATIVE_GEMM_PROVIDER "HYBRID" CACHE STRING - "Dense transform provider: CUSTOM, KOKKOSBLAS_NATIVE, KOKKOSBLAS_TPL, or HYBRID") -set_property(CACHE SOLPROP_NATIVE_GEMM_PROVIDER PROPERTY STRINGS - CUSTOM KOKKOSBLAS_NATIVE KOKKOSBLAS_TPL HYBRID) set(SOLPROP_NATIVE_ENABLE_TPL_BLAS "AUTO" CACHE STRING "Dispatch compatible GEMMs to vendor BLAS: AUTO (the profile's BLAS), ON or OFF") set_property(CACHE SOLPROP_NATIVE_ENABLE_TPL_BLAS PROPERTY STRINGS AUTO ON OFF) @@ -350,25 +298,11 @@ foreach(SOLPROP_NATIVE_DEVICE_SIZE IN ITEMS endif() endforeach() -string(TOUPPER "${SOLPROP_NATIVE_GEMM_PROVIDER}" - SOLPROP_NATIVE_GEMM_PROVIDER) -set(SOLPROP_NATIVE_GEMM_PROVIDERS - CUSTOM KOKKOSBLAS_NATIVE KOKKOSBLAS_TPL HYBRID) -if(NOT SOLPROP_NATIVE_GEMM_PROVIDER IN_LIST SOLPROP_NATIVE_GEMM_PROVIDERS) - message(FATAL_ERROR - "SOLPROP_NATIVE_GEMM_PROVIDER must be CUSTOM, KOKKOSBLAS_NATIVE, " - "KOKKOSBLAS_TPL, or HYBRID; got ${SOLPROP_NATIVE_GEMM_PROVIDER}.") -endif() -if(SOLPROP_NATIVE_GEMM_PROVIDER STREQUAL "KOKKOSBLAS_TPL" AND - NOT SOLPROP_NATIVE_ENABLE_TPL_BLAS) - message(FATAL_ERROR - "KOKKOSBLAS_TPL requires SOLPROP_NATIVE_ENABLE_TPL_BLAS=ON.") -endif() if(SOLPROP_NATIVE_ENABLE_CUDA AND SOLPROP_NATIVE_ENABLE_HIP) message(FATAL_ERROR "CUDA and HIP are separate build variants and cannot both be enabled.") endif() -if((SOLPROP_NATIVE_ENABLE_OPENMP OR SOLPROP_NATIVE_ENABLE_THREADS) AND +if(SOLPROP_NATIVE_ENABLE_OPENMP AND (SOLPROP_NATIVE_ENABLE_CUDA OR SOLPROP_NATIVE_ENABLE_HIP)) message(FATAL_ERROR "Accelerator variants use a Serial host execution space; disable OpenMP.") @@ -400,11 +334,19 @@ if(SOLPROP_NATIVE_CPU_BASELINE_FLAG) string(APPEND SOLPROP_NATIVE_COMPILER_FLAGS " ${SOLPROP_NATIVE_CPU_BASELINE_FLAG}") endif() string(STRIP "${SOLPROP_NATIVE_COMPILER_FLAGS}" SOLPROP_NATIVE_COMPILER_FLAGS) -string(APPEND SOLPROP_NATIVE_COMPILER_FLAGS - " -Wall -Wextra -Wpedantic -Wconversion -Wsign-conversion -Wshadow") -if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - string(APPEND SOLPROP_NATIVE_COMPILER_FLAGS " -fp-model=precise") +# The project's own sources' warnings, defined once: the targets compile with them and the +# build record lists them. MSVC is not a supported compiler (Windows builds use clang++). +set(SOLPROP_NATIVE_WARNING_FLAGS + -Wall -Wextra -Wpedantic -Wconversion -Wsign-conversion -Wshadow -Wundef + -Wnon-virtual-dtor -Woverloaded-virtual) +if(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|AppleClang|AMDClang)$") + list(APPEND SOLPROP_NATIVE_WARNING_FLAGS -Wimplicit-fallthrough) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + list(APPEND SOLPROP_NATIVE_WARNING_FLAGS + -Wduplicated-cond -Wduplicated-branches -Wlogical-op) endif() +list(JOIN SOLPROP_NATIVE_WARNING_FLAGS " " _warning_flags_record) +string(APPEND SOLPROP_NATIVE_COMPILER_FLAGS " ${_warning_flags_record}") if(SOLPROP_NATIVE_FP_CONTRACT_FLAG) string(APPEND SOLPROP_NATIVE_COMPILER_FLAGS " ${SOLPROP_NATIVE_FP_CONTRACT_FLAG}") endif() @@ -422,15 +364,25 @@ if(SOLPROP_NATIVE_ENABLE_COVERAGE) endif() if(SOLPROP_NATIVE_ENABLE_CLANG_TIDY) + # The development group's pinned clang-tidy, in the environment beside the Python + # this build uses, before any machine's LLVM: each LLVM release adds checks. + get_filename_component(_python_bin "${Python_EXECUTABLE}" DIRECTORY) find_program(SOLPROP_NATIVE_CLANG_TIDY clang-tidy - HINTS /opt/homebrew/opt/llvm/bin + HINTS "${_python_bin}" + /opt/homebrew/opt/llvm/bin /usr/local/opt/llvm/bin REQUIRED) + execute_process(COMMAND "${SOLPROP_NATIVE_CLANG_TIDY}" --version + OUTPUT_VARIABLE _clang_tidy_version OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REGEX MATCH "version [0-9.]+" _clang_tidy_version "${_clang_tidy_version}") + message(STATUS "clang-tidy: ${SOLPROP_NATIVE_CLANG_TIDY} (${_clang_tidy_version})") set(SOLPROP_NATIVE_CLANG_TIDY_COMMAND "${SOLPROP_NATIVE_CLANG_TIDY}" "--config-file=${PROJECT_SOURCE_DIR}/.clang-tidy" "--header-filter=^${PROJECT_SOURCE_DIR}/(include|src|tests)/.*") if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # The GCC-only warnings in the compile commands are unknown to clang-tidy's parser. + list(APPEND SOLPROP_NATIVE_CLANG_TIDY_COMMAND "--extra-arg=-Wno-unknown-warning-option") execute_process( COMMAND "${CMAKE_CXX_COMPILER}" -print-file-name=include OUTPUT_VARIABLE SOLPROP_NATIVE_GNU_INCLUDE_DIR @@ -453,6 +405,20 @@ if(SOLPROP_NATIVE_ENABLE_CLANG_TIDY) "--extra-arg=-D__malloc__(...)=__malloc__") endif() endif() + if(APPLE) + # Homebrew's Clang finds the macOS SDK from its own configuration file, which the + # PyPI clang-tidy does not read. + set(_tidy_sysroot "${CMAKE_OSX_SYSROOT}") + if(NOT _tidy_sysroot) + execute_process(COMMAND xcrun --show-sdk-path OUTPUT_VARIABLE _tidy_sysroot + OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY) + endif() + list(APPEND SOLPROP_NATIVE_CLANG_TIDY_COMMAND "--extra-arg=-isysroot${_tidy_sysroot}") + endif() + # The command the build runs, the resolved binary first, one argument per line, for + # native/scripts/tidy.sh (`poe tidy-native`) and the weekly include-cleaner report. + list(JOIN SOLPROP_NATIVE_CLANG_TIDY_COMMAND "\n" _tidy_command) + file(WRITE "${PROJECT_BINARY_DIR}/clang_tidy_command.txt" "${_tidy_command}\n") endif() function(solprop_native_configure_project_target target) @@ -462,13 +428,8 @@ function(solprop_native_configure_project_target target) target_link_options(${target} PRIVATE -fprofile-instr-generate) endif() target_compile_options(${target} PRIVATE - $<$:-Wall> - $<$:-Wextra> - $<$:-Wpedantic> - $<$:-Wconversion> - $<$:-Wsign-conversion> - $<$:-Wshadow> - $<$,$>:-Werror> + "$<$:${SOLPROP_NATIVE_WARNING_FLAGS}>" + $<$,$>:-Werror> ) if(SOLPROP_NATIVE_ENABLE_CLANG_TIDY) set_property(TARGET ${target} PROPERTY CXX_CLANG_TIDY @@ -476,11 +437,13 @@ function(solprop_native_configure_project_target target) endif() if(SOLPROP_NATIVE_ENABLE_SANITIZERS) target_compile_options(${target} PRIVATE - $<$:-fsanitize=address,undefined> - $<$:-fno-omit-frame-pointer> + $<$:-fsanitize=address,undefined> + # A report stops the test there, whatever UBSAN_OPTIONS says. + $<$:-fno-sanitize-recover=all> + $<$:-fno-omit-frame-pointer> ) target_link_options(${target} PRIVATE - $<$:-fsanitize=address,undefined> + $<$:-fsanitize=address,undefined> ) endif() endfunction() @@ -582,9 +545,6 @@ endif() if(SOLPROP_NATIVE_ENABLE_OPENMP) set(Kokkos_ENABLE_SERIAL OFF CACHE BOOL "Disable the Serial backend" FORCE) set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "Enable the OpenMP backend" FORCE) -elseif(SOLPROP_NATIVE_ENABLE_THREADS) - set(Kokkos_ENABLE_SERIAL OFF CACHE BOOL "Disable the Serial backend" FORCE) - set(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "Disable the OpenMP backend" FORCE) else() set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "Enable the Serial backend" FORCE) set(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "Disable the OpenMP backend" FORCE) @@ -594,8 +554,7 @@ set(Kokkos_ARCH_NATIVE ${SOLPROP_NATIVE_HOST_ARCH} CACHE BOOL set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ${SOLPROP_NATIVE_ENABLE_AGGRESSIVE_VECTORIZATION} CACHE BOOL "Enable aggressive vectorization of independent Kokkos loops" FORCE) -set(Kokkos_ENABLE_THREADS ${SOLPROP_NATIVE_ENABLE_THREADS} CACHE BOOL - "Enable the Threads backend" FORCE) +set(Kokkos_ENABLE_THREADS OFF CACHE BOOL "OpenMP is the CPU host space" FORCE) set(Kokkos_ENABLE_CUDA ${SOLPROP_NATIVE_ENABLE_CUDA} CACHE BOOL "Enable the CUDA backend" FORCE) set(Kokkos_ENABLE_HIP ${SOLPROP_NATIVE_ENABLE_HIP} CACHE BOOL @@ -760,7 +719,6 @@ endif() message(STATUS "solprop-native build profile: ${SOLPROP_NATIVE_BUILD_PROFILE}; " "BLAS ${SOLPROP_NATIVE_BLAS_BACKEND} (${SOLPROP_NATIVE_BLAS_SELECTION}); " - "GEMM provider ${SOLPROP_NATIVE_GEMM_PROVIDER}; " "native architecture ${SOLPROP_NATIVE_HOST_ARCH}; " "CPU baseline ${SOLPROP_NATIVE_CPU_BASELINE}; " "IPO ${SOLPROP_NATIVE_IPO_AVAILABLE} (Release builds)") @@ -841,7 +799,7 @@ if(SOLPROP_NATIVE_ENABLE_COVERAGE) target_include_directories(solprop_native_labels SYSTEM PRIVATE "${kokkos_SOURCE_DIR}/core/src") target_compile_options(solprop_native_labels PRIVATE - -Wall -Wextra -Wpedantic -Wconversion -Wsign-conversion -Werror) + ${SOLPROP_NATIVE_WARNING_FLAGS} -Werror) set_target_properties(solprop_native_labels PROPERTIES PREFIX "" CXX_VISIBILITY_PRESET default) install(TARGETS solprop_native_labels LIBRARY DESTINATION solprop_native) @@ -859,7 +817,7 @@ add_library(solprop_native_core STATIC src/core.cpp src/crossing.cpp src/general # judgement round as solprop's Python does, so their # translation units take no contraction on any build (the flag follows the global one, # and the last wins). MSVC's /fp:precise does not contract by default. -if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang|AMDClang|IntelLLVM)$") +if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang|AMDClang)$") set_source_files_properties(src/adams_march.cpp src/watch.cpp src/rk_march.cpp src/session.cpp src/session_statistics.cpp src/session_widths.cpp src/low_storage.cpp src/transverse.cpp src/session_edge.cpp src/dense.cpp @@ -876,7 +834,6 @@ target_link_libraries(solprop_native_core PUBLIC ${SOLPROP_NATIVE_MKL_FFT_LIBRARIES} ) target_compile_definitions(solprop_native_core PUBLIC - SOLPROP_NATIVE_GEMM_PROVIDER_${SOLPROP_NATIVE_GEMM_PROVIDER}=1 SOLPROP_NATIVE_DEVICE_TEAM_SIZE=${SOLPROP_NATIVE_DEVICE_TEAM_SIZE} SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE=${SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE} SOLPROP_NATIVE_BAND_STAGING=$ @@ -952,7 +909,6 @@ target_compile_definitions(_core PRIVATE SOLPROP_NATIVE_KOKKOSFFT_VERSION="2.0.0" SOLPROP_NATIVE_KOKKOSKERNELS_VERSION="5.2.0" SOLPROP_NATIVE_SLEEF_VERSION="${SOLPROP_NATIVE_SLEEF_VERSION}" - SOLPROP_NATIVE_GEMM_PROVIDER="${SOLPROP_NATIVE_GEMM_PROVIDER}" SOLPROP_NATIVE_TPL_BLAS_ENABLED=$ SOLPROP_NATIVE_BLAS_BACKEND="${SOLPROP_NATIVE_BLAS_BACKEND}" SOLPROP_NATIVE_BLAS_THREAD_POLICY="${SOLPROP_NATIVE_BLAS_THREAD_POLICY_ENCODED}" @@ -978,13 +934,6 @@ target_compile_definitions(_core PRIVATE SOLPROP_NATIVE_IPO_ENABLED=$,$> ) -if(SOLPROP_NATIVE_BUILD_BENCHMARKS) - add_executable(solprop_native_gemm_benchmark src/benchmark_gemm.cpp) - target_link_libraries(solprop_native_gemm_benchmark PRIVATE solprop_native_core) - solprop_native_mark_dependency_headers_system(solprop_native_gemm_benchmark) - solprop_native_configure_project_target(solprop_native_gemm_benchmark) -endif() - if(BUILD_TESTING) set(INSTALL_GTEST OFF CACHE BOOL "Do not install the test-only GoogleTest" FORCE) set(gtest_force_shared_crt ON CACHE BOOL @@ -1100,7 +1049,9 @@ if(SKBUILD) "openmp_libraries;${OpenMP_CXX_LIB_NAMES}" "fft_backend;${SOLPROP_NATIVE_HOST_FFT_BACKEND}" "blas_backend;${SOLPROP_NATIVE_BLAS_BACKEND}" - "gemm_provider;${SOLPROP_NATIVE_GEMM_PROVIDER}" + # One GEMM provider remains (the vendor GEMM for an eligible transform, the + # custom kernels for the rest); the field stays for the records that hold it. + "gemm_provider;HYBRID" "raman_fft_provider;${SOLPROP_NATIVE_RAMAN_FFT_PROVIDER}" "vector_math;${SOLPROP_NATIVE_VECTOR_MATH}" "build_profile;${SOLPROP_NATIVE_BUILD_PROFILE}" diff --git a/native/CMakePresets.json b/native/CMakePresets.json index 0bfd70245..c93b5a5c0 100644 --- a/native/CMakePresets.json +++ b/native/CMakePresets.json @@ -35,19 +35,19 @@ "inherits": "cpu-debug", "binaryDir": "${sourceDir}/build/presets/cpu-sanitize", "cacheVariables": { - "SOLPROP_NATIVE_ENABLE_SANITIZERS": "ON" + "SOLPROP_NATIVE_ENABLE_SANITIZERS": "ON", + "Kokkos_ENABLE_DEBUG_BOUNDS_CHECK": "ON" } }, { "name": "cpu-benchmark", - "displayName": "Local (vendor BLAS, machine-native) CPU benchmark build", + "displayName": "Local (vendor BLAS, machine-native) CPU Release build", "inherits": "cpu-debug", "binaryDir": "${sourceDir}/build/presets/cpu-benchmark", "cacheVariables": { "CMAKE_BUILD_TYPE": "Release", "SOLPROP_NATIVE_ENABLE_IPO": "ON", - "SOLPROP_NATIVE_PROFILE": "LOCAL", - "SOLPROP_NATIVE_BUILD_BENCHMARKS": "ON" + "SOLPROP_NATIVE_PROFILE": "LOCAL" } }, { @@ -57,7 +57,6 @@ "cacheVariables": { "BUILD_TESTING": "ON", "CMAKE_BUILD_TYPE": "Release", - "SOLPROP_NATIVE_BUILD_BENCHMARKS": "ON", "SOLPROP_NATIVE_ENABLE_CUDA": "ON", "SOLPROP_NATIVE_ENABLE_HIP": "OFF", "SOLPROP_NATIVE_ENABLE_OPENMP": "OFF", @@ -67,43 +66,12 @@ "SOLPROP_NATIVE_HOST_ARCH": "OFF" } }, - { - "name": "cuda-custom", - "displayName": "NVIDIA CUDA custom-GEMM validation build", - "inherits": "cuda-base", - "binaryDir": "${sourceDir}/build/presets/cuda-custom", - "cacheVariables": { - "SOLPROP_NATIVE_GEMM_PROVIDER": "CUSTOM", - "SOLPROP_NATIVE_ENABLE_TPL_BLAS": "OFF" - } - }, - { - "name": "cuda-native", - "displayName": "NVIDIA CUDA native KokkosBlas validation build", - "inherits": "cuda-base", - "binaryDir": "${sourceDir}/build/presets/cuda-native", - "cacheVariables": { - "SOLPROP_NATIVE_GEMM_PROVIDER": "KOKKOSBLAS_NATIVE", - "SOLPROP_NATIVE_ENABLE_TPL_BLAS": "OFF" - } - }, - { - "name": "cuda-tpl", - "displayName": "NVIDIA CUDA cuBLAS validation build", - "inherits": "cuda-base", - "binaryDir": "${sourceDir}/build/presets/cuda-tpl", - "cacheVariables": { - "SOLPROP_NATIVE_GEMM_PROVIDER": "KOKKOSBLAS_TPL", - "SOLPROP_NATIVE_ENABLE_TPL_BLAS": "ON" - } - }, { "name": "cuda", "displayName": "NVIDIA CUDA hybrid production build", "inherits": "cuda-base", "binaryDir": "${sourceDir}/build/presets/cuda", "cacheVariables": { - "SOLPROP_NATIVE_GEMM_PROVIDER": "HYBRID", "SOLPROP_NATIVE_ENABLE_TPL_BLAS": "ON" } }, @@ -115,12 +83,10 @@ "cacheVariables": { "BUILD_TESTING": "ON", "CMAKE_BUILD_TYPE": "Release", - "SOLPROP_NATIVE_BUILD_BENCHMARKS": "ON", "SOLPROP_NATIVE_ENABLE_CUDA": "OFF", "SOLPROP_NATIVE_ENABLE_HIP": "ON", "SOLPROP_NATIVE_ENABLE_OPENMP": "OFF", "SOLPROP_NATIVE_HIP_ARCH": "AMD_GFX942", - "SOLPROP_NATIVE_GEMM_PROVIDER": "HYBRID", "SOLPROP_NATIVE_ENABLE_TPL_BLAS": "ON", "SOLPROP_NATIVE_ENABLE_IPO": "OFF", "SOLPROP_NATIVE_HOST_ARCH": "OFF" @@ -132,9 +98,6 @@ {"name": "cpu-tidy", "configurePreset": "cpu-tidy"}, {"name": "cpu-sanitize", "configurePreset": "cpu-sanitize"}, {"name": "cpu-benchmark", "configurePreset": "cpu-benchmark"}, - {"name": "cuda-custom", "configurePreset": "cuda-custom"}, - {"name": "cuda-native", "configurePreset": "cuda-native"}, - {"name": "cuda-tpl", "configurePreset": "cuda-tpl"}, {"name": "cuda", "configurePreset": "cuda"}, {"name": "hip", "configurePreset": "hip"} ] diff --git a/native/README.md b/native/README.md index fe9fb3c88..9427f2277 100644 --- a/native/README.md +++ b/native/README.md @@ -91,15 +91,21 @@ and `tests/test_band_stress.py` with `SOLPROP_NATIVE_BAND_TILES=4`; on a GPU, ```sh uv run poe check-native # at the repository root; runs native/scripts/check.sh uv run poe check-all # solprop's gate, then this one +uv run poe fmt-native # format every tracked C++ file in place +uv run poe tidy-native # clang-tidy on the C++ files changed from main, or on those named ``` `poe check-native` runs everything a change to `native/` must pass: Ruff formatting and -lint, strict Pyright, the pinned `clang-format` from the `dev` group, clang-tidy with -strict warnings, the native tests, the native tests under ASan and UBSan, an editable +lint, strict Pyright, the pinned `clang-format` from the `dev` group over every tracked +C++ file, the pinned clang-tidy (whole check families, `.clang-tidy`) with strict +warnings, the native tests, the native tests under ASan and UBSan, an editable local build with the Python tests, a distribution wheel build, and an import smoke test of that wheel in a clean environment (on Linux it checks that the wheel's OpenBLAS loads from the package). It syncs the repository's environment without removing the native -build, honours `CMAKE_ARGS`, and needs Node.js (`npx`) for Pyright. +build, honours `CMAKE_ARGS`, and needs Node.js (`npx`) for Pyright. `SOLPROP_NATIVE_CHECK_STAGES` +picks stages (`lint`, `tidy`, `sanitize`, `pytest`, `wheel`). The slower checks run +weekly in `.github/workflows/native-deep.yml` and report without gating: source +coverage, the extension under ASan and UBSan, and an include-cleaner report. `SOLPROP_NATIVE_QUALIFICATION_PYTHON=/path/to/python` installs the wheel into that environment and runs the Python tests there instead. A change to the backend also passes solprop's own suite with this package installed (`poe check-all`, or `uv run poe test` diff --git a/native/benchmarks/cpu_timer.cpp b/native/benchmarks/cpu_timer.cpp deleted file mode 100644 index 3a459366c..000000000 --- a/native/benchmarks/cpu_timer.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-FileCopyrightText: 2026 John C. Travers and contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -// CPU-only attribution tool for the pinned Kokkos profiling callback ABI. -// OpenMP kernels finish synchronously; disabling tool fences leaves their order intact. -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { -using Clock = std::chrono::steady_clock; -struct Event { - std::string name; - Clock::time_point start; -}; -struct Total { - std::uint64_t calls{}; - double seconds{}; -}; -bool enabled{}; -std::vector events; -std::vector regions; -std::map totals; -void begin(const char* name, std::uint64_t* id) { - *id = 0; - if (enabled) { - events.push_back({name, Clock::now()}); - *id = events.size(); - } -} -void end(std::uint64_t id) { - if (id != 0 && enabled) { - const auto& event = events.at(id - 1); - auto& total = totals[event.name]; - total.seconds += std::chrono::duration(Clock::now() - event.start).count(); - ++total.calls; - } -} -} // namespace -extern "C" { -void kokkosp_request_tool_settings(std::uint32_t, Kokkos_Tools_ToolSettings* settings) { - settings->requires_global_fencing = false; -} -void kokkosp_begin_parallel_for(const char* name, std::uint32_t, std::uint64_t* id) { - begin(name, id); -} -void kokkosp_end_parallel_for(std::uint64_t id) { end(id); } -void kokkosp_begin_parallel_reduce(const char* name, std::uint32_t, std::uint64_t* id) { - begin(name, id); -} -void kokkosp_end_parallel_reduce(std::uint64_t id) { end(id); } -void kokkosp_begin_parallel_scan(const char* name, std::uint32_t, std::uint64_t* id) { - begin(name, id); -} -void kokkosp_end_parallel_scan(std::uint64_t id) { end(id); } -void kokkosp_push_profile_region(const char* name) { - std::uint64_t id{}; - begin(name, &id); - regions.push_back(id); -} -void kokkosp_pop_profile_region() { - end(regions.back()); - regions.pop_back(); -} -void solprop_timer_start() { - events.clear(); - totals.clear(); - enabled = true; -} -void solprop_timer_stop(const char* path) { - enabled = false; - std::ofstream output(path); - output.precision(17); - output << "name\tcalls\tseconds\n"; - for (const auto& [name, total] : totals) - output << name << '\t' << total.calls << '\t' << total.seconds << '\n'; -} -} diff --git a/native/benchmarks/ctc.py b/native/benchmarks/ctc.py deleted file mode 100644 index c173f81ce..000000000 --- a/native/benchmarks/ctc.py +++ /dev/null @@ -1,552 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Measure the reduced and full-shape CtC cases in isolated processes. - -A propagation is the backend's session (solprop SEP 47): its start loads the launch -before the clock starts, and the timed region is the session's advances, each plane's -statistics evaluated there, up to the last plane's observation, before its field is -exported. A trial is the backend's timing probe (``Backend.timing_probe``), which loads -the start once and times single trials from it. ``--planes`` keeps a run's first planes -only, for a short check of the script. The untimed warm-up is the launch plane and one -attempt of a session of the timed run itself, closed there: a backend serves only its -own request's planes, where the step driver's warm-up marched to a short plane of its -own. -""" - -# ruff: noqa: I001 -- Kokkos loads first, before any other OpenMP runtime on macOS. - -from __future__ import annotations - -import argparse -import dataclasses -import hashlib -import json -import math -import os -import statistics -import subprocess -import sys -import threading -import time -import warnings -from typing import Any, TypedDict, cast - -from device_record import DeviceRecord, device_record - - -class Result(DeviceRecord): - """One isolated CtC measurement.""" - - backend: str - device_before: DeviceRecord - case: str - mode: str - seconds: float - construction_seconds: float - planning_seconds: float - raman_planning_seconds: float - trials: int - accepted_steps: int - rejected_steps: int - milliseconds_per_trial: float - milliseconds_per_accepted_step: float | None - modes: int - nodes: int - node_chunk: int - components: int - n_omega: int - n_to: int - raman_terms: int - raman_kernel_transforms: int - raman_memory_ledger: dict[str, int] - field_sha256: str - memory_ledger_bytes: int - kernel_threads: int - native_threads: int - fftw_threads: int - blas_threads: int | str - gemm_provider: str - synthesis_provider: str - projection_provider: str - blas_backend: str - propagator_mode: str - omp_proc_bind: str - omp_places: str - omp_wait_policy: str - load_start: float - load_end: float - - -class Measurement(Result): - """The median run plus spread across isolated processes.""" - - minimum_seconds: float - maximum_seconds: float - median_absolute_deviation: float - - -# The two CtC shapes, defined here rather than read from a stored comparison: hydrogen -# at 40 bar in a 16 um capillary, 400 fs and 10 uJ at 515 nm, no loss, the package's -# own materials. The full shape runs 0.3 m on six modes and a 4 ps, 170 nm-4 um grid; -# the reduced one is the fundamental alone over 0.1 m on a 2 ps, 250 nm-2 um grid, five -# times the pulse, which keeps the launch clear of the temporal collar. -class Shape(TypedDict): - """One CtC shape: its length, modes, band, window and save count.""" - - length: float - modes: int - wavelength_limits: tuple[float, float] - trange: float - n_save: int - - -CASES: dict[str, Shape] = { - "ctc_reduced": { - "length": 0.1, - "modes": 1, - "wavelength_limits": (250e-9, 2e-6), - "trange": 2e-12, - "n_save": 21, - }, - "ctc_full": { - "length": 0.3, - "modes": 6, - "wavelength_limits": (170e-9, 4e-6), - "trange": 4e-12, - "n_save": 201, - }, -} - - -def build_simulation(case_name: str, backend_name: str) -> Any: - from solprop.driver import Simulation - - case = CASES[case_name] - return Simulation.from_capillary( - 16e-6, - case["length"], - "H2", - 40.0, - wavelength=515e-9, - wavelength_limits=case["wavelength_limits"], - trange=case["trange"], - fwhm=400e-15, - energy=10e-6, - modes=case["modes"], - loss=False, - raman=True, - n_save=case["n_save"], - rtol=1e-6, - atol=1e-10, - backend=backend_name, - ) - - -def _metadata_int(metadata: dict[str, object], name: str, default: int) -> int: - value = metadata.get(name, default) - if isinstance(value, bool) or not isinstance(value, int | str): - raise TypeError(f"metadata {name!r} is not integer-valued: {value!r}") - return int(value) - - -def _metadata_float(metadata: dict[str, object], name: str) -> float: - value = metadata.get(name, 0.0) - if isinstance(value, bool) or not isinstance(value, int | float): - raise TypeError(f"metadata {name!r} is not numeric: {value!r}") - return float(value) - - -def _warm(session: Any) -> None: - """Take a started session's launch plane and one attempt beyond it, then close it. - - The untimed warm-up: a backend serves only its own request's planes, so it is - warmed on the timed run's own itinerary, cut short; the timed run loads the - launch again. - """ - try: - for _ in range(2): # the launch plane's batch, then one attempt - if session.finished: - break - batch = session.advance(work_limit=1) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - session.close() - - -def _worker( - backend_name: str, - case_name: str, - mode: str, - trial_count: int, - node_chunk: int | None, - progress_seconds: float, - planes_kept: int | None, -) -> None: - if backend_name == "native": - __import__("solprop_native") - - import numpy as np - - from solprop.backends.node_chunk import plan_node_chunk - from solprop.backends.registry import select - from solprop.backends.events import ObservationReady - from solprop.backends.export import ExportRecipe - from solprop.backends.session import ExecutionOptions, Start - from solprop.backends.sessions import start_session - from solprop.exceptions import MaterialRangeWarning - - with warnings.catch_warnings(): - warnings.simplefilter("ignore", MaterialRangeWarning) - simulation = build_simulation(case_name, backend_name) - problem = simulation.translate() - request = simulation._run_request() # pyright: ignore[reportPrivateUsage] - if planes_kept is not None: - request = request.with_stepper( - dataclasses.replace( - request.stepper, save_planes=request.stepper.save_planes[:planes_kept] - ) - ) - options = None if node_chunk is None else ExecutionOptions(node_chunk=node_chunk) - - constructed = time.perf_counter() - backend: Any = select( - backend_name, - problem, - request, - grid=simulation.grid, - modes=simulation.modes, - profiles=simulation.profiles, - options=options, - ) - node_chunk_used = ( - int(backend.node_chunk) - if backend_name == "native" - else plan_node_chunk(problem, options).chunk - ) - construction_seconds = time.perf_counter() - constructed - dz = float(request.stepper.dz_init) - identity = ExportRecipe(power=False, optics=False) - - def entrance() -> Start: - return Start.entrance(problem, noise=simulation.noise) - - try: - if mode == "trial": - probe = backend.timing_probe(entrance()) - device_before = device_record(backend.metadata()) - probe.trial(0.0, dz) - load_start = os.getloadavg()[0] - started = time.perf_counter() - for _ in range(trial_count): - probe.trial(0.0, dz) - seconds = time.perf_counter() - started - load_end = os.getloadavg()[0] - probe.trial(0.0, dz) - field = probe.field() - accepted = 0 - rejected = 0 - trials = trial_count - else: - device_before = device_record(backend.metadata()) - _warm(start_session(backend, problem, request, entrance(), owns=False)) - session = start_session(backend, problem, request, entrance(), owns=False) - load_start = os.getloadavg()[0] - started = time.perf_counter() - process_started = time.process_time() - attempts = 0 - rejected = 0 - save_planes = [float(plane) for plane in request.stepper.save_planes] - planes = len(save_planes) - progress_state: dict[str, int | float] = { - "completed_planes": 0, - "current_plane": 1, - "target": save_planes[0], - "accepted_steps": 0, - "rejected_steps": 0, - } - progress_stop = threading.Event() - - def emit_progress(kind: str) -> None: - record: dict[str, bool | float | int | str] = { - "progress": True, - "kind": kind, - "planes": planes, - "elapsed_seconds": time.perf_counter() - started, - "process_cpu_seconds": time.process_time() - process_started, - **progress_state, - } - print(json.dumps(record, sort_keys=True), file=sys.stderr, flush=True) - - def progress_heartbeat() -> None: - while not progress_stop.wait(progress_seconds): - emit_progress("heartbeat") - - progress_thread: threading.Thread | None = None - if progress_seconds > 0.0: - progress_thread = threading.Thread( - target=progress_heartbeat, - name="ctc-progress", - daemon=True, - ) - progress_thread.start() - seconds = 0.0 - field = None - try: - while not session.finished: - batch = session.advance() - for event in batch.events: - history = getattr(event, "history", None) - if history is not None: - attempts += history.n_steps - rejected += history.n_rejected - if not isinstance(event, ObservationReady): - continue - if event.plane_index < 0: - continue # the launch plane - plane = event.plane_index + 1 - progress_state["completed_planes"] = plane - progress_state["accepted_steps"] = attempts - rejected - progress_state["rejected_steps"] = rejected - if plane < planes: - progress_state["current_plane"] = plane + 1 - progress_state["target"] = save_planes[plane] - continue - if progress_seconds > 0.0: - emit_progress("complete") - seconds = time.perf_counter() - started - field = np.array(session.export(event.ticket, identity)) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - progress_stop.set() - if progress_thread is not None: - progress_thread.join() - session.close() - load_end = os.getloadavg()[0] - accepted = attempts - rejected - trials = attempts - if field is None: - raise RuntimeError("the CtC session observed no last plane") - if not np.all(np.isfinite(field)): - raise RuntimeError("CtC benchmark produced a non-finite field") - metadata: dict[str, object] = backend.metadata() - device_after = device_record(metadata) - ledger = backend.memory_ledger() - raman_transforms = int(getattr(backend, "_raman_transforms", 0)) - finally: - backend.close() - - threads = _metadata_int(metadata, "thread_count", 0) - fftw_threads = _metadata_int(metadata, "fftw_threads", threads) - blas_backend = str(metadata.get("blas_backend", "not applicable")) - if blas_backend == "Accelerate": - blas_threads: int | str = "system-managed" - elif blas_backend == "OpenBLAS": - blas_threads = int(os.environ.get("OPENBLAS_NUM_THREADS", "1")) - elif blas_backend in {"cuBLAS", "rocBLAS"}: - blas_threads = "device-stream" - elif backend_name == "native": - blas_threads = "disabled" - else: - blas_threads = int(os.environ.get("SOLPROP_BLAS_THREADS", "1")) - result: Result = { - **device_after, - "backend": backend_name, - "device_before": device_before, - "case": case_name, - "mode": mode, - "seconds": seconds, - "construction_seconds": construction_seconds, - "planning_seconds": _metadata_float(metadata, "planning_seconds"), - "raman_planning_seconds": _metadata_float(metadata, "raman_planning_seconds"), - "trials": trials, - "accepted_steps": accepted, - "rejected_steps": rejected, - "milliseconds_per_trial": 1000.0 * seconds / trials, - "milliseconds_per_accepted_step": ( - 1000.0 * seconds / accepted if accepted else None - ), - "modes": problem.shape[0], - "nodes": problem.transform.n_nodes, - "node_chunk": node_chunk_used, - "components": problem.transform.n_pol * problem.shape[1], - "n_omega": problem.shape[-1], - "n_to": problem.grids.n_to, - "raman_terms": ( - 0 - if problem.responses.raman_coefficients is None - else problem.responses.raman_coefficients.n_terms - ), - "raman_kernel_transforms": raman_transforms, - "raman_memory_ledger": { - name: ledger[name] - for name in ( - "node_raman_drive", - "node_raman_spectrum", - "node_raman_convolved", - "raman_undamped", - "raman_kernel_time", - "raman_kernel_spectrum", - "stage_raman_rates", - ) - if name in ledger - }, - "field_sha256": hashlib.sha256(field.tobytes(order="C")).hexdigest(), - "memory_ledger_bytes": sum(ledger.values()), - "kernel_threads": threads, - "native_threads": threads if backend_name == "native" else 0, - "fftw_threads": fftw_threads, - "blas_threads": blas_threads, - "gemm_provider": str(metadata.get("gemm_provider", "not applicable")), - "synthesis_provider": str(metadata.get("synthesis_provider", "not applicable")), - "projection_provider": str( - metadata.get("projection_provider", "not applicable") - ), - "blas_backend": blas_backend, - "propagator_mode": str(metadata.get("propagator_mode", "unknown")), - "omp_proc_bind": os.environ.get("OMP_PROC_BIND", ""), - "omp_places": os.environ.get("OMP_PLACES", ""), - "omp_wait_policy": os.environ.get("OMP_WAIT_POLICY", ""), - "load_start": load_start, - "load_end": load_end, - } - print(json.dumps(result, sort_keys=True)) - - -def _csv(value: str) -> list[str]: - values = [item.strip() for item in value.split(",") if item.strip()] - if not values: - raise argparse.ArgumentTypeError("the list must not be empty") - return values - - -def _run( - backend: str, - threads: int, - blas_threads: int, - case_name: str, - mode: str, - trial_count: int, - node_chunk: int | None, - progress_seconds: float, - planes_kept: int | None, -) -> Result: - environment = os.environ.copy() - environment.update( - { - "OPENBLAS_NUM_THREADS": str(blas_threads), - "SOLPROP_BLAS_THREADS": str(blas_threads), - "SOLPROP_NATIVE_THREADS": str(threads), - } - ) - environment.setdefault("OMP_DYNAMIC", "FALSE") - environment.setdefault("OMP_PROC_BIND", "spread") - environment.setdefault("OMP_PLACES", "cores") - environment.setdefault("OMP_WAIT_POLICY", "active") - command = [ - sys.executable, - __file__, - "--worker", - backend, - "--case", - case_name, - "--mode", - mode, - "--trial-count", - str(trial_count), - ] - if node_chunk is not None: - command.extend(("--node-chunk", str(node_chunk))) - if progress_seconds > 0.0: - command.extend(("--progress-seconds", str(progress_seconds))) - if planes_kept is not None: - command.extend(("--planes", str(planes_kept))) - completed = subprocess.run( - command, - check=True, - stdout=subprocess.PIPE, - text=True, - env=environment, - ) - return cast(Result, json.loads(completed.stdout.splitlines()[-1])) - - -def main() -> None: - """Run isolated CtC workers and emit JSON samples with robust summaries.""" - parser = argparse.ArgumentParser() - parser.add_argument("--worker", choices=("numpy_ref", "native")) - parser.add_argument( - "--case", choices=("ctc_reduced", "ctc_full"), default="ctc_reduced" - ) - parser.add_argument("--mode", choices=("trial", "propagation"), default="trial") - parser.add_argument("--trial-count", type=int, default=5) - parser.add_argument("--node-chunk", type=int) - parser.add_argument("--progress-seconds", type=float, default=0.0) - parser.add_argument("--backends", type=_csv, default=["native"]) - parser.add_argument("--threads", type=_csv, default=["8"]) - parser.add_argument("--blas-threads", type=int, default=1) - parser.add_argument("--repeats", type=int, default=1) - parser.add_argument( - "--planes", - type=int, - help="run the case's first planes only, to check the script", - ) - arguments = parser.parse_args() - if arguments.planes is not None and arguments.planes < 1: - parser.error("--planes must be positive") - if min(arguments.trial_count, arguments.blas_threads, arguments.repeats) < 1: - parser.error("counts must be positive") - if arguments.node_chunk is not None and arguments.node_chunk < 1: - parser.error("--node-chunk must be positive") - if not math.isfinite(arguments.progress_seconds) or arguments.progress_seconds < 0: - parser.error("--progress-seconds must be finite and non-negative") - if arguments.worker is not None: - _worker( - arguments.worker, - arguments.case, - arguments.mode, - arguments.trial_count, - arguments.node_chunk, - arguments.progress_seconds, - arguments.planes, - ) - return - for backend in arguments.backends: - for thread_text in arguments.threads: - threads = int(thread_text) - samples = [ - _run( - backend, - threads, - arguments.blas_threads, - arguments.case, - arguments.mode, - arguments.trial_count, - arguments.node_chunk, - arguments.progress_seconds, - arguments.planes, - ) - for _ in range(arguments.repeats) - ] - median = statistics.median(sample["seconds"] for sample in samples) - chosen = min(samples, key=lambda sample: abs(sample["seconds"] - median)) - measurement: Measurement = { - **chosen, - "minimum_seconds": min(sample["seconds"] for sample in samples), - "maximum_seconds": max(sample["seconds"] for sample in samples), - "median_absolute_deviation": statistics.median( - abs(sample["seconds"] - median) for sample in samples - ), - } - print(json.dumps(measurement, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/native/benchmarks/device_record.py b/native/benchmarks/device_record.py deleted file mode 100644 index 819f1c5eb..000000000 --- a/native/benchmarks/device_record.py +++ /dev/null @@ -1,101 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Small machine record shared by isolated accelerator benchmarks.""" - -from __future__ import annotations - -import csv -import io -import subprocess -from collections.abc import Mapping -from typing import TypedDict - - -class DeviceRecord(TypedDict): - """Execution-space and live accelerator telemetry for one sample.""" - - device_backend: str - device_arch: str - device_name: str - device_uuid: str - device_memory_total_mib: float - device_memory_used_mib: float - device_utilization_percent: float - device_temperature_c: float - device_pstate: str - device_clock_mhz: float - device_power_w: float - device_team_size: int - device_reduction_size: int - - -def _number(value: str) -> float: - """Parse one no-units NVIDIA CSV value, retaining unavailable values as NaN.""" - try: - return float(value) - except ValueError: - return float("nan") - - -def _integer_metadata(metadata: Mapping[str, object], name: str) -> int: - """Read one integer-valued native metadata field without silent coercion.""" - value = metadata.get(name, 0) - if isinstance(value, bool) or not isinstance(value, (int, str)): - raise TypeError(f"metadata field {name!r} is not an integer") - return int(value) - - -def device_record(metadata: Mapping[str, object]) -> DeviceRecord: - """Return build identity and one post-sample NVIDIA telemetry snapshot.""" - backend = str(metadata.get("device_backend", "none")) - record: DeviceRecord = { - "device_backend": backend, - "device_arch": str(metadata.get("device_arch", "host")), - "device_name": "not applicable", - "device_uuid": "not applicable", - "device_memory_total_mib": 0.0, - "device_memory_used_mib": 0.0, - "device_utilization_percent": 0.0, - "device_temperature_c": 0.0, - "device_pstate": "not applicable", - "device_clock_mhz": 0.0, - "device_power_w": 0.0, - "device_team_size": _integer_metadata(metadata, "device_team_size"), - "device_reduction_size": _integer_metadata(metadata, "device_reduction_size"), - } - if backend != "cuda": - return record - query = ( - "name,uuid,memory.total,memory.used,utilization.gpu,temperature.gpu," - "pstate,clocks.sm,power.draw" - ) - completed = subprocess.run( - [ - "nvidia-smi", - f"--query-gpu={query}", - "--format=csv,noheader,nounits", - ], - check=True, - capture_output=True, - text=True, - ) - rows = list(csv.reader(io.StringIO(completed.stdout), skipinitialspace=True)) - if len(rows) != 1 or len(rows[0]) != 9: - raise RuntimeError("CUDA benchmarks require exactly one visible NVIDIA GPU") - values = rows[0] - record.update( - { - "device_name": values[0], - "device_uuid": values[1], - "device_memory_total_mib": _number(values[2]), - "device_memory_used_mib": _number(values[3]), - "device_utilization_percent": _number(values[4]), - "device_temperature_c": _number(values[5]), - "device_pstate": values[6], - "device_clock_mhz": _number(values[7]), - "device_power_w": _number(values[8]), - } - ) - return record diff --git a/native/benchmarks/gemm_shapes.py b/native/benchmarks/gemm_shapes.py deleted file mode 100644 index aaae91a09..000000000 --- a/native/benchmarks/gemm_shapes.py +++ /dev/null @@ -1,219 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Measure a dense transform's cost per source call by shape, for one build. - -The `HYBRID` GEMM policy chooses between the vendor GEMM and the custom kernels by -shape. This times ten fixed IP-DOPRI5 steps (sixty-one source calls) of short solprop -problems whose transforms span the shapes the policy decides on -- a radial field -route from 2 to 256 nodes, and one- to six-mode capillaries as real fields and as -envelopes -- and records each problem's chosen providers beside its median cost per -call over five warmed repetitions. Run it once per build to compare policies (for -example the installed build against one with ``SOLPROP_NATIVE_GEMM_PROVIDER`` forced), -and on a device build to find that device's crossover:: - - OMP_NUM_THREADS=1 SOLPROP_NATIVE_THREADS=1 VECLIB_MAXIMUM_THREADS=1 \\ - python benchmarks/gemm_shapes.py --output gemm-shapes.json - -Each repetition is one run of the backend's session (solprop SEP 47), whose start -loads the launch before the clock starts; the timed region is its advances to the -end. ``--nodes`` and ``--modes`` measure a subset of the shapes. -""" - -# ruff: noqa: I001 -- Kokkos loads first, before any other OpenMP runtime on macOS. - -from __future__ import annotations - -import argparse -import dataclasses -import json -import os -import statistics -import time -import warnings -from pathlib import Path -from typing import Any, TypedDict - -import solprop_native -import numpy as np -from solprop.backends.registry import select -from solprop.backends.session import Start -from solprop.backends.sessions import start_session -from solprop.driver import Simulation, SimulationSettings -from solprop.grid import FieldGrid, RadialGrid -from solprop.nonlinear.responses import KerrResponse -from solprop.physdata import gamma3 -from solprop.pulses import GaussianPulse, SpatialPulse -from solprop.spatial import SpatialMedium -from solprop.stepper import FixedSchedule, PIController - -STEPS = 10 -"""Fixed steps a radial problem takes: six source calls each and one to start.""" - - -class Row(TypedDict): - """One problem's shape, its providers and its cost per source call.""" - - kind: str - size: int - envelope: bool | None - shape: list[int] - n_t: int - synthesis: str - projection: str - calls: int - per_call_ms: float - - -def radial(nodes: int, trange: float) -> Simulation: - """Return a radial field route of `nodes` Hankel nodes on a fixed schedule.""" - length = 1e-3 - grid = FieldGrid.from_limits(length, 800e-9, (500e-9, 1.6e-6), trange) - settings = SimulationSettings( - PIController(rtol=1e-6, atol=0.0), - save_planes=[length], - dz_max=length / STEPS, - dz_init=length / STEPS, - schedule=FixedSchedule(np.full(STEPS, length / STEPS)), - backend="numpy_ref", - ) - pulse = SpatialPulse( - GaussianPulse(wavelength=800e-9, fwhm=30e-15, energy=1e-6), 100e-6 - ) - return Simulation.from_radial( - grid, - RadialGrid(1e-3, nodes), - SpatialMedium("Ar", 1.0), - [KerrResponse(gamma3("Ar", wavelength=800e-9), thg=False)], - pulse, - settings=settings, - ) - - -def capillary(modes: int, trange: float, *, envelope: bool) -> Simulation: - """Return a short argon capillary of `modes` modes, stepped adaptively.""" - length = 1e-3 - return Simulation.from_capillary( - 75e-6, - length, - "Ar", - 1.0, - wavelength=800e-9, - wavelength_limits=(300e-9, 2e-6), - trange=trange, - envelope=envelope, - fwhm=30e-15, - energy=10e-6, - modes=modes, - plasma=False, - raman=False, - rtol=1e-6, - backend="numpy_ref", - save_planes=[length], - ) - - -def measure(simulation: Simulation, kind: str, size: int, envelope: bool | None) -> Row: - """Time one problem's march five times after a warm one; return its row.""" - problem: Any = simulation.translate() - request = simulation._run_request() # pyright: ignore[reportPrivateUsage] - seconds: list[float] = [] - calls = 0 - providers = ("", "") - for repetition in range(6): - backend: Any = select( - "native", - problem, - request, - grid=simulation.grid, - modes=simulation.modes, - profiles=simulation.profiles, - ) - if repetition == 0: - native = backend._native_state - providers = ( - str(native.synthesis_provider), - str(native.projection_provider), - ) - session = start_session( - backend, problem, request, Start.entrance(problem), owns=False - ) - attempts = 0 - try: - clock = time.perf_counter() - while not session.finished: - batch = session.advance() - for event in batch.events: - history = getattr(event, "history", None) - if history is not None: - attempts += int(history.n_steps) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - seconds.append(time.perf_counter() - clock) - finally: - session.close() - backend.close() - calls = 6 * attempts + 1 - return { - "kind": kind, - "size": size, - "envelope": envelope, - "shape": [int(value) for value in problem.shape], - "n_t": int(problem.grids.n_t), - "synthesis": providers[0], - "projection": providers[1], - "calls": calls, - "per_call_ms": 1e3 * statistics.median(seconds[1:]) / calls, - } - - -def _counts(text: str) -> tuple[int, ...]: - """Parse a comma-separated list of positive counts.""" - counts = tuple(int(part) for part in text.split(",") if part.strip()) - if any(count < 1 for count in counts): - raise argparse.ArgumentTypeError("counts must be positive") - return counts - - -def main() -> None: - """Measure every shape for the importable build and write the rows.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument( - "--nodes", - type=_counts, - default=(2, 4, 8, 16, 32, 64, 128, 256), - help="the radial node counts, comma-separated; empty for none", - ) - parser.add_argument( - "--modes", - type=_counts, - default=(1, 2, 3, 6), - help="the capillary mode counts, comma-separated; empty for none", - ) - args = parser.parse_args() - warnings.filterwarnings("ignore") - rows: list[Row] = [] - for nodes in args.nodes: - for trange in (100e-15, 400e-15): - rows.append(measure(radial(nodes, trange), "radial", nodes, None)) - print(rows[-1], flush=True) - for modes in args.modes: - for trange in (200e-15, 800e-15): - for envelope in (False, True): - simulation = capillary(modes, trange, envelope=envelope) - rows.append(measure(simulation, "capillary", modes, envelope)) - print(rows[-1], flush=True) - record = { - "build": dataclasses.asdict(solprop_native.build_info()), - "load": os.getloadavg(), - "rows": rows, - } - args.output.write_text(json.dumps(record, indent=2) + "\n") - - -if __name__ == "__main__": - main() diff --git a/native/benchmarks/m1.py b/native/benchmarks/m1.py deleted file mode 100644 index ef28479da..000000000 --- a/native/benchmarks/m1.py +++ /dev/null @@ -1,408 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Measure the full N=6 soliton run in isolated backend/thread processes. - -A propagation is the backend's session (solprop SEP 47): its start loads the launch -before the clock starts, and the timed region is the session's advances to the one save -plane, stopped when that plane's observation arrives, before its field is exported. A -trial is the backend's timing probe (``Backend.timing_probe``), which loads the start -once and times single trials from it. The untimed warm-up is the launch plane and one -attempt of a session of the timed run itself, closed there: a backend serves only its -own request's planes, where the step driver's warm-up marched to a short plane of its -own. -""" - -# ruff: noqa: I001 -- Kokkos loads first, before any other OpenMP runtime on macOS. - -from __future__ import annotations - -import argparse -import json -import os -import statistics -import subprocess -import sys -import time -from typing import Any, NamedTuple, cast - -from device_record import DeviceRecord, device_record - - -class Result(DeviceRecord): - """One validated M1 measurement.""" - - backend: str - device_before: DeviceRecord - mode: str - trials: int - threads: int - realized_threads: int - fftw_threads: int - seconds: float - accepted_steps: int - rejected_steps: int - milliseconds_per_trial: float - milliseconds_per_accepted_step: float | None - target_fraction: float - omp_proc_bind: str - omp_places: str - omp_wait_policy: str - kmp_blocktime: str - grid_scale: float - n_omega: int - n_to: int - work_elements: int - warmed: bool - load_start: float - load_end: float - - -class Measurement(Result): - """The median run plus the observed three-run interval.""" - - minimum_seconds: float - maximum_seconds: float - - -class Marched(NamedTuple): - """One session's march to its last plane.""" - - seconds: float - accepted: int - rejected: int - field: Any - - -def _warm(session: Any) -> None: - """Take a started session's launch plane and one attempt beyond it, then close it. - - The untimed warm-up: a backend serves only its own request's planes, so it is - warmed on the timed run's own itinerary, cut short; the timed run loads the - launch again. - """ - try: - for _ in range(2): # the launch plane's batch, then one attempt - if session.finished: - break - batch = session.advance(work_limit=1) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - session.close() - - -def _march(session: Any, last: float) -> Marched: - """Advance a started session to its end, timing it up to the plane at `last`. - - The clock stops when that plane's observation arrives, before its field is - exported (the internal convention, as the step driver observed it). - """ - import numpy as np - - from solprop.backends.events import ObservationReady - from solprop.backends.export import ExportRecipe - - accepted = rejected = 0 - seconds = 0.0 - field = None - try: - started = time.perf_counter() - while not session.finished: - batch = session.advance() - for event in batch.events: - history = getattr(event, "history", None) - if history is not None: - taken = int(np.count_nonzero(history.accepted)) - accepted += taken - rejected += history.n_steps - taken - if isinstance(event, ObservationReady) and event.z == last: - seconds = time.perf_counter() - started - field = np.array( - session.export(event.ticket, ExportRecipe(False, False)) - ) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - session.close() - if field is None: - raise RuntimeError(f"the session observed no plane at {last} m") - return Marched(seconds, accepted, rejected, field) - - -def _worker( - backend_name: str, - target_fraction: float, - grid_scale: float, - warmup: bool, - mode: str, - trial_count: int, -) -> None: - if backend_name == "native": - # macOS must load the extension's OpenMP runtime before any other. - __import__("solprop_native") - - import numpy as np - - from solprop.backends.registry import select - from solprop.backends.session import Start - from solprop.backends.sessions import start_session - from solprop.grid import EnvelopeGrid - from solprop.nonlinear.gnlse import problem_from - from solprop.pulses import SechPulse - from solprop.stepper import PIController - from solprop.translate.problem import assemble - - beta2 = -11.83e-27 - gamma = 0.11 - tau0 = 100e-15 - wavelength = 835e-9 - period = np.pi / 2 * tau0**2 / abs(beta2) - target = target_fraction * period - grid = EnvelopeGrid.from_limits(1.0, wavelength, (500e-9, 2e-6), grid_scale * 4e-12) - model = problem_from( - [0.0, 0.0, beta2], - omega_ref=grid.omega0, - gamma=gamma, - shock=False, - ) - peak_power = 36 * abs(beta2) / (gamma * tau0**2) - field = SechPulse(wavelength=wavelength, width=tau0, peak_power=peak_power).build( - grid - ) - problem, request = assemble( - grid, - model.mode, - [model.kerr], - field, - controller=PIController(rtol=1e-10, atol=1e-18), - save_planes=[target], - dz_max=period, - dz_init=1e-4, - density=1.0, - shock=False, - prefactor_index=1.0, - ) - backend: Any = select(backend_name, problem, request, grid=grid) - dz = float(request.stepper.dz_init) - try: - if mode == "trial": - probe = backend.timing_probe(Start.entrance(problem)) - if warmup: - probe.trial(0.0, dz) - device_before = device_record(backend.metadata()) - load_start = os.getloadavg()[0] - started = time.perf_counter() - for _ in range(trial_count): - probe.trial(0.0, dz) - seconds = time.perf_counter() - started - probe.trial(0.0, dz) - field_out = probe.field() - accepted = 0 - rejected = 0 - trials = trial_count - else: - if warmup: - _warm( - start_session( - backend, problem, request, Start.entrance(problem), owns=False - ) - ) - session = start_session( - backend, problem, request, Start.entrance(problem), owns=False - ) - device_before = device_record(backend.metadata()) - load_start = os.getloadavg()[0] - marched = _march(session, target) - seconds = marched.seconds - attempts = marched.accepted + marched.rejected - accepted = marched.accepted - rejected = marched.rejected - # Every trial is an attempt, accepted or rejected. (The step driver's - # summary path added the rejections twice: `n_steps` already held them.) - trials = attempts - field_out = marched.field - load_end = os.getloadavg()[0] - if not np.all(np.isfinite(field_out)): - raise RuntimeError("M1 produced a non-finite field") - metadata = backend.metadata() - device_after = device_record(metadata) - finally: - backend.close() - - recorded_threads = metadata.get("thread_count") - realized = ( - recorded_threads - if isinstance(recorded_threads, int) - else int(os.environ["BENCH_THREADS"]) - ) - recorded_fftw_threads = metadata.get("fftw_threads") - realized_fftw_threads = ( - recorded_fftw_threads if isinstance(recorded_fftw_threads, int) else realized - ) - result: Result = { - **device_after, - "backend": backend_name, - "device_before": device_before, - "mode": mode, - "trials": trials, - "threads": int(os.environ["BENCH_THREADS"]), - "realized_threads": realized, - "fftw_threads": realized_fftw_threads, - "seconds": seconds, - "accepted_steps": accepted, - "rejected_steps": rejected, - "milliseconds_per_trial": 1000.0 * seconds / trials, - "milliseconds_per_accepted_step": ( - 1000.0 * seconds / accepted if accepted else None - ), - "target_fraction": target_fraction, - "omp_proc_bind": os.environ.get("OMP_PROC_BIND", ""), - "omp_places": os.environ.get("OMP_PLACES", ""), - "omp_wait_policy": os.environ.get("OMP_WAIT_POLICY", ""), - "kmp_blocktime": os.environ.get("KMP_BLOCKTIME", ""), - "grid_scale": grid_scale, - "n_omega": grid.n_omega, - "n_to": grid.n_to, - "work_elements": grid.n_to, - "warmed": warmup, - "load_start": load_start, - "load_end": load_end, - } - print(json.dumps(result, sort_keys=True)) - - -def _parse_csv(value: str) -> list[str]: - result = [part.strip() for part in value.split(",") if part.strip()] - if not result: - raise argparse.ArgumentTypeError("the list must not be empty") - return result - - -def _run( - backend: str, - threads: int, - target_fraction: float, - grid_scale: float, - warmup: bool, - mode: str, - trial_count: int, -) -> Result: - environment = os.environ.copy() - environment.update( - { - "BENCH_THREADS": str(threads), - "SOLPROP_NATIVE_THREADS": str(threads), - } - ) - environment.setdefault("OMP_DYNAMIC", "FALSE") - environment.setdefault("OMP_PROC_BIND", "spread") - environment.setdefault("OMP_PLACES", "cores") - command = [ - sys.executable, - __file__, - "--worker", - backend, - "--fraction", - str(target_fraction), - "--grid-scale", - str(grid_scale), - "--mode", - mode, - "--trial-count", - str(trial_count), - ] - if warmup: - command.append("--warmup") - completed = subprocess.run( - command, - check=True, - capture_output=True, - text=True, - env=environment, - ) - return cast(Result, json.loads(completed.stdout.splitlines()[-1])) - - -def main() -> None: - """Run workers and print the median end-to-end M1 table as JSON lines.""" - parser = argparse.ArgumentParser() - parser.add_argument("--worker", choices=("numpy_ref", "native")) - parser.add_argument("--backends", type=_parse_csv, default=["numpy_ref", "native"]) - parser.add_argument("--threads", type=_parse_csv, default=["1", "8"]) - parser.add_argument("--repeats", type=int, default=3) - parser.add_argument( - "--mode", choices=("trial", "propagation"), default="propagation" - ) - parser.add_argument("--trial-count", type=int, default=7) - parser.add_argument( - "--fraction", - type=float, - default=1.0, - help="fraction of the full soliton period to propagate", - ) - parser.add_argument( - "--warmup", - action="store_true", - help="run one untimed adaptive step before the measured run", - ) - parser.add_argument( - "--grid-scale", - type=float, - default=1.0, - help="multiple of the M1 time window, and therefore approximately its FFT size", - ) - arguments = parser.parse_args() - if not 0.0 < arguments.fraction <= 1.0: - parser.error("--fraction must be in (0, 1]") - if arguments.grid_scale <= 0.0: - parser.error("--grid-scale must be positive") - if arguments.worker is not None: - _worker( - arguments.worker, - arguments.fraction, - arguments.grid_scale, - arguments.warmup, - arguments.mode, - arguments.trial_count, - ) - return - if arguments.repeats < 1 or arguments.trial_count < 1: - parser.error("--repeats and --trial-count must be positive") - - for backend in arguments.backends: - if backend not in {"numpy_ref", "native"}: - parser.error(f"unknown backend {backend!r}") - for thread_text in arguments.threads: - threads = int(thread_text) - if threads < 1: - parser.error("thread counts must be positive") - samples = [ - _run( - backend, - threads, - arguments.fraction, - arguments.grid_scale, - arguments.warmup, - arguments.mode, - arguments.trial_count, - ) - for _ in range(arguments.repeats) - ] - middle = statistics.median(sample["seconds"] for sample in samples) - chosen = min(samples, key=lambda sample: abs(sample["seconds"] - middle)) - measurement: Measurement = { - **chosen, - "minimum_seconds": min(sample["seconds"] for sample in samples), - "maximum_seconds": max(sample["seconds"] for sample in samples), - } - print(json.dumps(measurement, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/native/benchmarks/m2.py b/native/benchmarks/m2.py deleted file mode 100644 index f783475d9..000000000 --- a/native/benchmarks/m2.py +++ /dev/null @@ -1,487 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Measure reduced M2 propagation in isolated backend/thread processes. - -A propagation is the backend's session (solprop SEP 47): its start loads the launch -before the clock starts, and the timed region is the session's advances to its last -plane, with every observation's field exported (the internal convention, as the step -driver observed it) and its statistics, which the session evaluates there. A trial is -the backend's timing probe (``Backend.timing_probe``), which loads the start once and -times single trials from it. ``--planes`` keeps a run's first planes only, for a short -check of the script. The untimed warm-up is the launch plane and one attempt of a -session of the timed run itself, closed there: a backend serves only its own request's -planes, where the step driver's warm-up marched to a short plane of its own. -""" - -# ruff: noqa: I001 -- Kokkos loads first, before any other OpenMP runtime on macOS. - -from __future__ import annotations - -import argparse -import dataclasses -import json -import os -import statistics -import subprocess -import sys -import time -from collections.abc import Mapping -from pathlib import Path -from typing import Any, TypedDict, cast - -from device_record import DeviceRecord, device_record - - -class Result(DeviceRecord): - """One reduced-M2 propagation measurement.""" - - backend: str - device_before: DeviceRecord - mode: str - trials: int - case: str - field_family: str - seconds: float - construction_seconds: float - planning_seconds: float - memory_ledger: dict[str, int] - memory_ledger_bytes: int - accepted_steps: int - rejected_steps: int - milliseconds_per_trial: float - milliseconds_per_accepted_step: float | None - modes: int - nodes: int - node_chunk: int - components: int - n_omega: int - n_to: int - warmed: bool - native_threads: int - fftw_threads: int - blas_threads: int | str - blas_thread_policy: str - gemm_provider: str - synthesis_provider: str - projection_provider: str - blas_backend: str - omp_proc_bind: str - omp_places: str - omp_wait_policy: str - load_start: float - load_end: float - - -class Measurement(Result): - """The median run plus its spread across isolated processes.""" - - minimum_seconds: float - maximum_seconds: float - median_absolute_deviation: float - - -class CaseTable(TypedDict): - """Geometry and sampling values of the M2 benchmark cases (``m2_cases.py``).""" - - radius: float - length: float - pressure: float | tuple[float, float] - modes: int - lambda_min: float - lambda_max: float - trange: float - n_save: int - expected_steps: int - - -class FamilyTable(TypedDict): - """Pulse and gas values of the M2 benchmark cases (``m2_cases.py``).""" - - gas: str - wavelength: float - tau_fwhm: float - energy: float - loss: bool - - -def _case_tables(case_name: str) -> tuple[CaseTable, FamilyTable]: - from m2_cases import CASES, FAMILIES - - return cast(CaseTable, CASES[case_name]), cast(FamilyTable, FAMILIES[case_name]) - - -def _metadata_int(metadata: Mapping[str, object], name: str, default: int) -> int: - """Read one integer-valued backend metadata item without a blind cast.""" - value = metadata.get(name, default) - if isinstance(value, bool) or not isinstance(value, int | str): - raise TypeError(f"metadata {name!r} is not integer-valued: {value!r}") - return int(value) - - -def _metadata_float(metadata: Mapping[str, object], name: str) -> float: - """Read one floating-point backend metadata item without a blind cast.""" - value = metadata.get(name, 0.0) - if isinstance(value, bool) or not isinstance(value, int | float): - raise TypeError(f"metadata {name!r} is not numeric: {value!r}") - return float(value) - - -def _warm(session: Any) -> None: - """Take a started session's launch plane and one attempt beyond it, then close it. - - The untimed warm-up: a backend serves only its own request's planes, so it is - warmed on the timed run's own itinerary, cut short; the timed run loads the - launch again. - """ - try: - for _ in range(2): # the launch plane's batch, then one attempt - if session.finished: - break - batch = session.advance(work_limit=1) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - session.close() - - -def _march(session: Any) -> tuple[int, int, Any]: - """Advance a started session to its end, exporting every observation's field. - - Each observation's statistics arrive with it. Returns the attempts, the rejected - attempts among them and the last plane's field, in the internal convention. - """ - import numpy as np - - from solprop.backends.events import ObservationReady - from solprop.backends.export import ExportRecipe - - identity = ExportRecipe(power=False, optics=False) - attempts = rejected = 0 - field = None - try: - while not session.finished: - batch = session.advance() - for event in batch.events: - history = getattr(event, "history", None) - if history is not None: - attempts += history.n_steps - rejected += history.n_rejected - if isinstance(event, ObservationReady): - field = np.array(session.export(event.ticket, identity)) - terminal = batch.terminal - if terminal is not None and terminal.error is not None: - raise terminal.error - session.acknowledge(batch.receipt) - finally: - session.close() - return attempts, rejected, field - - -def _worker( - backend_name: str, - case_name: str, - field_family: str, - save_field: Path | None, - mode: str, - trial_count: int, - planes: int | None, -) -> None: - if backend_name == "native": - # Kokkos loads first, the order documented for local applications. - __import__("solprop_native") - - import numpy as np - - from solprop.backends.node_chunk import plan_node_chunk - from solprop.backends.registry import select - from solprop.backends.session import Start - from solprop.backends.sessions import start_session - from solprop.driver import Simulation - from solprop.exceptions import MaterialRangeWarning - - import warnings - - case, family = _case_tables(case_name) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", MaterialRangeWarning) - simulation = Simulation.from_capillary( - case["radius"], - case["length"], - family["gas"], - case["pressure"], - pressure_floor=0.0, - wavelength=family["wavelength"], - wavelength_limits=(case["lambda_min"], case["lambda_max"]), - trange=case["trange"], - fwhm=family["tau_fwhm"], - energy=family["energy"], - modes=case["modes"], - loss=family["loss"], - envelope=field_family == "envelope", - n_save=case["n_save"], - rtol=1e-6, - atol=1e-10, - backend=backend_name, - ) - problem = simulation.translate() - request = simulation._run_request() # pyright: ignore[reportPrivateUsage] - if planes is not None: - request = request.with_stepper( - dataclasses.replace( - request.stepper, save_planes=request.stepper.save_planes[:planes] - ) - ) - - def entrance() -> Start: - return Start.entrance(problem, noise=simulation.noise) - - construction_started = time.perf_counter() - backend: Any = select( - backend_name, - problem, - request, - grid=simulation.grid, - modes=simulation.modes, - profiles=simulation.profiles, - ) - construction_seconds = time.perf_counter() - construction_started - # The chunk a native response pass holds; the reference's is the host plan's. - node_chunk = ( - int(backend.node_chunk) - if backend_name == "native" - else plan_node_chunk(problem).chunk - ) - dz = float(request.stepper.dz_init) - try: - memory_ledger = backend.memory_ledger() - _warm(start_session(backend, problem, request, entrance(), owns=False)) - if mode == "trial": - probe = backend.timing_probe(entrance()) - device_before = device_record(backend.metadata()) - load_start = os.getloadavg()[0] - started = time.perf_counter() - for _ in range(trial_count): - probe.trial(0.0, dz) - seconds = time.perf_counter() - started - probe.trial(0.0, dz) - field = probe.field() - attempts = 0 - rejected = 0 - trials = trial_count - else: - session = start_session(backend, problem, request, entrance(), owns=False) - device_before = device_record(backend.metadata()) - load_start = os.getloadavg()[0] - started = time.perf_counter() - attempts, rejected, field = _march(session) - seconds = time.perf_counter() - started - trials = attempts - load_end = os.getloadavg()[0] - if not np.all(np.isfinite(field)): - raise RuntimeError("reduced M2 produced a non-finite field") - if save_field is not None: - np.save(save_field, field) - metadata = backend.metadata() - device_after = device_record(metadata) - finally: - backend.close() - - accepted = attempts - rejected - recorded_threads = _metadata_int(metadata, "thread_count", 0) - recorded_fftw_threads = _metadata_int(metadata, "fftw_threads", recorded_threads) - blas_backend = str(metadata.get("blas_backend", "not applicable")) - if blas_backend == "Accelerate": - blas_threads: int | str = "system-managed" - elif blas_backend == "OpenBLAS": - blas_threads = int(os.environ.get("OPENBLAS_NUM_THREADS", "1")) - elif blas_backend in {"cuBLAS", "rocBLAS"}: - blas_threads = "device-stream" - elif backend_name == "native": - blas_threads = "disabled" - else: - blas_threads = _metadata_int( - metadata, - "blas_threads", - int(os.environ.get("SOLPROP_BLAS_THREADS", "1")), - ) - result: Result = { - **device_after, - "backend": backend_name, - "device_before": device_before, - "mode": mode, - "trials": trials, - "case": case_name, - "field_family": field_family, - "seconds": seconds, - "construction_seconds": construction_seconds, - "planning_seconds": _metadata_float(metadata, "planning_seconds"), - "memory_ledger": memory_ledger, - "memory_ledger_bytes": sum(memory_ledger.values()), - "accepted_steps": accepted, - "rejected_steps": rejected, - "milliseconds_per_trial": 1000.0 * seconds / trials, - "milliseconds_per_accepted_step": ( - 1000.0 * seconds / accepted if accepted else None - ), - "modes": problem.shape[0], - "nodes": problem.transform.n_nodes, - "node_chunk": node_chunk, - "components": problem.transform.n_pol * problem.shape[1], - "n_omega": problem.shape[-1], - "n_to": problem.grids.n_to, - "warmed": True, - "native_threads": recorded_threads, - "fftw_threads": recorded_fftw_threads, - "blas_threads": blas_threads, - "blas_thread_policy": str( - metadata.get("blas_thread_policy", "backend-managed") - ), - "gemm_provider": str(metadata.get("gemm_provider", "not applicable")), - "synthesis_provider": str(metadata.get("synthesis_provider", "not applicable")), - "projection_provider": str( - metadata.get("projection_provider", "not applicable") - ), - "blas_backend": blas_backend, - "omp_proc_bind": os.environ.get("OMP_PROC_BIND", ""), - "omp_places": os.environ.get("OMP_PLACES", ""), - "omp_wait_policy": os.environ.get("OMP_WAIT_POLICY", ""), - "load_start": load_start, - "load_end": load_end, - } - print(json.dumps(result, sort_keys=True)) - - -def _parse_csv(value: str) -> list[str]: - result = [part.strip() for part in value.split(",") if part.strip()] - if not result: - raise argparse.ArgumentTypeError("the list must not be empty") - return result - - -def _run( - backend: str, - threads: int, - blas_threads: int, - case_name: str, - field_family: str, - mode: str, - trial_count: int, - planes: int | None, -) -> Result: - environment = os.environ.copy() - environment.update( - { - "OPENBLAS_NUM_THREADS": str(blas_threads), - "SOLPROP_BLAS_THREADS": str(blas_threads), - "SOLPROP_NATIVE_THREADS": str(threads), - } - ) - environment.setdefault("OMP_DYNAMIC", "FALSE") - environment.setdefault("OMP_PROC_BIND", "spread") - environment.setdefault("OMP_PLACES", "cores") - completed = subprocess.run( - [ - sys.executable, - __file__, - "--worker", - backend, - "--case", - case_name, - "--field-family", - field_family, - "--mode", - mode, - "--trial-count", - str(trial_count), - *(() if planes is None else ("--planes", str(planes))), - ], - check=True, - capture_output=True, - text=True, - env=environment, - ) - return cast(Result, json.loads(completed.stdout.splitlines()[-1])) - - -def main() -> None: - """Run isolated workers and emit median reduced-M2 JSON records.""" - parser = argparse.ArgumentParser() - parser.add_argument("--worker", choices=("numpy_ref", "native")) - parser.add_argument( - "--case", choices=("m2_reduced", "m2_full"), default="m2_reduced" - ) - parser.add_argument( - "--field-family", choices=("field_real", "envelope"), default="field_real" - ) - parser.add_argument( - "--save-field", - type=Path, - help="worker-only owning field output, written after the timed interval", - ) - parser.add_argument("--backends", type=_parse_csv, default=["numpy_ref", "native"]) - parser.add_argument("--threads", type=_parse_csv, default=["1", "8"]) - parser.add_argument("--blas-threads", type=int, default=1) - parser.add_argument("--repeats", type=int, default=1) - parser.add_argument( - "--mode", choices=("trial", "propagation"), default="propagation" - ) - parser.add_argument("--trial-count", type=int, default=7) - parser.add_argument( - "--planes", - type=int, - help="run the case's first planes only, to check the script", - ) - arguments = parser.parse_args() - if arguments.planes is not None and arguments.planes < 1: - parser.error("--planes must be positive") - if arguments.worker is not None: - _worker( - arguments.worker, - arguments.case, - arguments.field_family, - arguments.save_field, - arguments.mode, - arguments.trial_count, - arguments.planes, - ) - return - if arguments.save_field is not None: - parser.error("--save-field requires --worker") - if min(arguments.repeats, arguments.blas_threads, arguments.trial_count) < 1: - parser.error("--repeats, --blas-threads and --trial-count must be positive") - for backend in arguments.backends: - for thread_text in arguments.threads: - threads = int(thread_text) - if threads < 1: - parser.error("thread counts must be positive") - samples = [ - _run( - backend, - threads, - arguments.blas_threads, - arguments.case, - arguments.field_family, - arguments.mode, - arguments.trial_count, - arguments.planes, - ) - for _ in range(arguments.repeats) - ] - median = statistics.median(sample["seconds"] for sample in samples) - chosen = min(samples, key=lambda sample: abs(sample["seconds"] - median)) - deviations = [abs(sample["seconds"] - median) for sample in samples] - measurement: Measurement = { - **chosen, - "minimum_seconds": min(sample["seconds"] for sample in samples), - "maximum_seconds": max(sample["seconds"] for sample in samples), - "median_absolute_deviation": statistics.median(deviations), - } - print(json.dumps(measurement, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/native/benchmarks/m2_cases.py b/native/benchmarks/m2_cases.py deleted file mode 100644 index 7ccf4bde2..000000000 --- a/native/benchmarks/m2_cases.py +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""The M2 benchmark cases: vacuum-ultraviolet dispersive-wave emission in helium. - -The full case is the published RDW example (a 7.5 fs, 275 uJ pulse at 800 nm in a -1.5 m, 125 um-radius capillary with a pressure gradient from 0.8 bar to vacuum, four -modes); the reduced case is its short, two-mode companion. ``expected_steps`` records -the step count of each case's independent reference run. -""" - -from __future__ import annotations - -type Value = float | int | str | bool | tuple[float, float] - -RDW: dict[str, Value] = { - "gas": "He", - "wavelength": 800e-9, - "tau_fwhm": 7.5e-15, - "energy": 275e-6, - "loss": True, -} - -CASES: dict[str, dict[str, Value]] = { - "m2_full": { - "radius": 125e-6, - "length": 1.5, - "pressure": (0.8, 0.0), - "modes": 4, - "lambda_min": 90e-9, - "lambda_max": 4e-6, - "trange": 400e-15, - "n_save": 201, - "expected_steps": 4365, - }, - "m2_reduced": { - "radius": 125e-6, - "length": 1.0, - "pressure": (1.2, 0.0), - "modes": 2, - "lambda_min": 100e-9, - "lambda_max": 1.5e-6, - "trange": 200e-15, - "n_save": 51, - "expected_steps": 348, - }, -} - -FAMILIES: dict[str, dict[str, Value]] = dict.fromkeys(CASES, RDW) diff --git a/native/benchmarks/provider_sweep.py b/native/benchmarks/provider_sweep.py deleted file mode 100644 index f70cc13fa..000000000 --- a/native/benchmarks/provider_sweep.py +++ /dev/null @@ -1,195 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Interleave isolated M2 or CtC runs across prebuilt backend variants.""" - -from __future__ import annotations - -import argparse -import json -import os -import statistics -import subprocess -import sys -from pathlib import Path -from typing import TypedDict, cast - - -class Variant(TypedDict): - """One backend label and its import-package root.""" - - label: str - package_root: str - - -class Result(TypedDict): - """One worker result augmented with sweep coordinates.""" - - variant: str - repetition: int - ordinal: int - seconds: float - load_start: float - load_end: float - - -def _variant(value: str) -> Variant: - label, separator, raw_path = value.partition("=") - if not separator or not label or not raw_path: - raise argparse.ArgumentTypeError( - "a Kokkos variant must be LABEL=/path/to/package-root" - ) - path = Path(raw_path).expanduser().resolve() - if not (path / "solprop_native").is_dir(): - raise argparse.ArgumentTypeError( - f"{path} does not contain a solprop_native package directory" - ) - return {"label": label, "package_root": str(path)} - - -def _run( - variant: Variant, - threads: int, - blas_threads: int, - workload: str, - case_name: str, - field_family: str, - mode: str, - trial_count: int, - node_chunk: int | None, -) -> dict[str, object]: - environment = os.environ.copy() - environment.update( - { - "KMP_BLOCKTIME": environment.get("KMP_BLOCKTIME", "50"), - "OPENBLAS_NUM_THREADS": str(blas_threads), - "OMP_DYNAMIC": environment.get("OMP_DYNAMIC", "FALSE"), - "OMP_PLACES": environment.get("OMP_PLACES", "threads"), - "OMP_PROC_BIND": environment.get("OMP_PROC_BIND", "close"), - "SOLPROP_BLAS_THREADS": str(blas_threads), - "SOLPROP_NATIVE_THREADS": str(threads), - } - ) - package_root = variant["package_root"] - previous = environment.get("PYTHONPATH") - environment["PYTHONPATH"] = ( - package_root if not previous else package_root + os.pathsep + previous - ) - command = [sys.executable, str(Path(__file__).with_name(f"{workload}.py"))] - command.extend(("--worker", "native", "--case", case_name)) - if workload == "m2": - command.extend(("--field-family", field_family)) - command.extend(("--mode", mode, "--trial-count", str(trial_count))) - if workload == "ctc": - if node_chunk is not None: - command.extend(("--node-chunk", str(node_chunk))) - completed = subprocess.run( - command, check=True, capture_output=True, text=True, env=environment - ) - return cast(dict[str, object], json.loads(completed.stdout.splitlines()[-1])) - - -def main() -> None: - """Run a round-robin provider sweep and emit samples plus robust summaries.""" - parser = argparse.ArgumentParser() - parser.add_argument( - "--variant", - action="append", - type=_variant, - required=True, - help="LABEL=/path/to/package-root; repeat for each arm", - ) - parser.add_argument("--threads", type=int, default=8) - parser.add_argument("--workload", choices=("m2", "ctc"), default="m2") - parser.add_argument("--case", default="m2_reduced") - parser.add_argument( - "--field-family", choices=("field_real", "envelope"), default="field_real" - ) - parser.add_argument( - "--blas-threads", - type=int, - default=1, - help="process-start BLAS pool, independent of the Kokkos pool", - ) - parser.add_argument("--repeats", type=int, default=7) - parser.add_argument("--mode", choices=("trial", "propagation"), default="trial") - parser.add_argument("--trial-count", type=int, default=7) - parser.add_argument("--node-chunk", type=int) - parser.add_argument( - "--max-load", - type=float, - default=None, - help="fail if either one-minute load sample exceeds this value", - ) - arguments = parser.parse_args() - if ( - arguments.threads < 1 - or arguments.blas_threads < 1 - or arguments.repeats < 1 - or arguments.trial_count < 1 - ): - parser.error("thread counts and repeats must be positive") - if not arguments.case.startswith(f"{arguments.workload}_"): - parser.error("--case must belong to the selected workload") - if arguments.node_chunk is not None and arguments.node_chunk < 1: - parser.error("--node-chunk must be positive") - - samples: dict[str, list[Result]] = { - variant["label"]: [] for variant in arguments.variant - } - ordinal = 0 - for repetition in range(arguments.repeats): - offset = repetition % len(arguments.variant) - ordered = arguments.variant[offset:] + arguments.variant[:offset] - for variant in ordered: - measured = _run( - variant, - arguments.threads, - arguments.blas_threads, - arguments.workload, - arguments.case, - arguments.field_family, - arguments.mode, - arguments.trial_count, - arguments.node_chunk, - ) - sample: Result = { - **cast(Result, measured), - "variant": variant["label"], - "repetition": repetition, - "ordinal": ordinal, - } - ordinal += 1 - if arguments.max_load is not None and max( - sample["load_start"], sample["load_end"] - ) > cast(float, arguments.max_load): - raise RuntimeError( - f"load exceeded {arguments.max_load}: {json.dumps(sample)}" - ) - samples[variant["label"]].append(sample) - print(json.dumps({"kind": "sample", **sample}, sort_keys=True)) - - for label, values in samples.items(): - times = [value["seconds"] for value in values] - median = statistics.median(times) - print( - json.dumps( - { - "kind": "summary", - "variant": label, - "repeats": len(times), - "median_seconds": median, - "median_absolute_deviation": statistics.median( - abs(value - median) for value in times - ), - "minimum_seconds": min(times), - "maximum_seconds": max(times), - }, - sort_keys=True, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/native/ci/build-deps-windows.ps1 b/native/ci/build-deps-windows.ps1 index 1c226f60d..e9cd94cc3 100644 --- a/native/ci/build-deps-windows.ps1 +++ b/native/ci/build-deps-windows.ps1 @@ -88,7 +88,7 @@ $ompLibrary = ($omp -replace '\\', '/') + '/lib/libomp.lib' $openmp = @('-DOpenMP_C_FLAGS=-fopenmp', '-DOpenMP_C_LIB_NAMES=libomp', "-DOpenMP_libomp_LIBRARY=$ompLibrary") -# FFTW, double and single precision, static, with OpenMP. +# FFTW, double precision (the package computes in no other), static, with OpenMP. Get-Source "https://www.fftw.org/fftw-$fftwVersion.tar.gz" $fftwSha256 $fftwSimd = @('-DCMAKE_C_FLAGS=-march=x86-64-v2', '-DENABLE_SSE2=ON', '-DENABLE_AVX=ON') if ($arm) { @@ -105,19 +105,16 @@ if ($arm) { $text = $text.Replace('/* #undef HAVE_NEON */', "#define HAVE_NEON 1`n#define HAVE_ARMV8_CNTVCT_EL0 1") Set-Content $config $text -NoNewline } -foreach ($precision in @('double', 'float')) { - $build = Join-Path $work "fftw-build-$precision" - $flags = @('-S', "$work\fftw-$fftwVersion", '-B', $build, '-G', 'Ninja', - '-DCMAKE_C_COMPILER=clang', '-DCMAKE_BUILD_TYPE=Release', - '-DCMAKE_POLICY_VERSION_MINIMUM=3.5', - '-DBUILD_SHARED_LIBS=OFF', '-DBUILD_TESTS=OFF', '-DENABLE_OPENMP=ON', - '-DENABLE_THREADS=OFF', - "-DCMAKE_INSTALL_PREFIX=$Prefix\fftw") + $fftwSimd + $openmp - if ($precision -eq 'float') { $flags += '-DENABLE_FLOAT=ON' } - cmake @flags; Invoke-Native "FFTW $precision configure" - cmake --build $build --parallel; Invoke-Native "FFTW $precision build" - cmake --install $build; Invoke-Native "FFTW $precision install" -} +$build = Join-Path $work 'fftw-build' +$flags = @('-S', "$work\fftw-$fftwVersion", '-B', $build, '-G', 'Ninja', + '-DCMAKE_C_COMPILER=clang', '-DCMAKE_BUILD_TYPE=Release', + '-DCMAKE_POLICY_VERSION_MINIMUM=3.5', + '-DBUILD_SHARED_LIBS=OFF', '-DBUILD_TESTS=OFF', '-DENABLE_OPENMP=ON', + '-DENABLE_THREADS=OFF', + "-DCMAKE_INSTALL_PREFIX=$Prefix\fftw") + $fftwSimd + $openmp +cmake @flags; Invoke-Native 'FFTW configure' +cmake --build $build --parallel; Invoke-Native 'FFTW build' +cmake --install $build; Invoke-Native 'FFTW install' Copy-Item "$work\fftw-$fftwVersion\COPYING" "$licences\LICENSE-fftw.txt" # OpenBLAS, static, BLAS plus its C LAPACK translation (Kokkos Kernels needs crot_ and @@ -145,11 +142,9 @@ if ($arm) { Build-OpenBlas 'NEHALEM' "$Prefix\openblas-v2" } if ($arm) { - foreach ($library in @('fftw3.lib', 'fftw3f.lib')) { - # The version string names the codelets, as the smoke test reads it elsewhere. - if (-not (Select-String -Path "$Prefix\fftw\lib\$library" -Pattern '-neon' -SimpleMatch -Quiet)) { - throw "$library lacks the NEON codelets" - } + # The version string names the codelets, as the smoke test reads it elsewhere. + if (-not (Select-String -Path "$Prefix\fftw\lib\fftw3.lib" -Pattern '-neon' -SimpleMatch -Quiet)) { + throw 'fftw3.lib lacks the NEON codelets' } } Get-ChildItem "$Prefix\fftw\lib", "$Prefix\openblas\lib", "$omp\lib", "$omp\bin" | diff --git a/native/ci/build-fftw.sh b/native/ci/build-fftw.sh index 03d0a9221..22f5a242b 100755 --- a/native/ci/build-fftw.sh +++ b/native/ci/build-fftw.sh @@ -4,8 +4,8 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -# Build FFTW for a Linux or macOS wheel: double and single precision, shared, with -# its OpenMP and POSIX-threads libraries. +# Build FFTW for a Linux or macOS wheel: double precision (the package computes in no +# other), shared, with its OpenMP and POSIX-threads libraries. # # The wheel repair (auditwheel, delocate) copies these libraries into the package. # The common code is compiled for the wheel's CPU baseline, except on x86-64: there it @@ -14,8 +14,8 @@ # x86-64 FFTW's SSE2 and AVX codelets are chosen at run time (AVX2 and AVX-512 codelets were not a # uniform gain when measured, so they are left out). On Arm the cycle counter is # enabled, so that measured planning (FFTW_MEASURE) works; without one FFTW plans by -# estimate only. Both Arm wheels also enable FFTW's explicit NEON codelets in both -# precisions; compiler auto-vectorisation does not enable these implementations. +# estimate only. Both Arm wheels also enable FFTW's explicit NEON codelets; compiler +# auto-vectorisation does not enable these implementations. # # OpenMP: on Linux the compiler's own runtime (GCC's libgomp in the manylinux image); # on macOS the libomp of LIBOMP_PREFIX (native/ci/build-libomp.sh), the runtime the @@ -68,17 +68,13 @@ esac archive=$(fetch "https://www.fftw.org/fftw-$version.tar.gz" "$sha256" "$work") rm -rf "$work/fftw-$version" tar -xzf "$archive" -C "$work" -for precision in double single; do - build="$work/fftw-build-$precision" - rm -rf "$build" - mkdir -p "$build" - precision_args=(--disable-single) - [[ $precision == single ]] && precision_args=(--enable-single) - (cd "$build" && CFLAGS="$cflags" "$work/fftw-$version/configure" \ - "${configure_args[@]}" "${simd[@]}" "${precision_args[@]}" >configure.log) - make -C "$build" -j"$(jobs)" >"$build/make.log" - make -C "$build" install >"$build/install.log" -done +build="$work/fftw-build" +rm -rf "$build" +mkdir -p "$build" +(cd "$build" && CFLAGS="$cflags" "$work/fftw-$version/configure" \ + "${configure_args[@]}" "${simd[@]}" >configure.log) +make -C "$build" -j"$(jobs)" >"$build/make.log" +make -C "$build" install >"$build/install.log" # The wheel carries FFTW's licence (CMake's SOLPROP_NATIVE_BUNDLED_LICENCES). mkdir -p "$prefix/licences" cp "$work/fftw-$version/COPYING" "$prefix/licences/LICENSE-fftw.txt" diff --git a/native/ci/smoke.py b/native/ci/smoke.py index a9244dcb0..f141c3ab4 100644 --- a/native/ci/smoke.py +++ b/native/ci/smoke.py @@ -12,8 +12,9 @@ It checks that the package imports and passes ``check_build``, that it is the distribution build, that its version is the expected one, and that the libraries it loaded (FFT, BLAS, OpenMP) are the copies inside the installed wheel, not ones from -the build machine or the system. On Arm it checks that both precisions of the bundled -FFTW carry the NEON codelets. It then runs ``python -m solprop_native`` and checks +the build machine or the system, and that the wheel neither carries nor imports a +single-precision FFTW. On Arm it checks that the bundled FFTW carries the NEON codelets. +It then runs ``python -m solprop_native`` and checks that the build record it prints is JSON. """ @@ -42,30 +43,37 @@ def bundled(library: str, package: Path) -> bool: return any(path.is_relative_to(root) for root in roots) -def fftw_versions(fft_library: str) -> dict[str, str]: - """Return the version strings of the bundled FFTW's double and single libraries. +def fftw_version(fft_library: str) -> str: + """Return the bundled FFTW's version string, which names its codelet family. - FFTW names its codelet family in the version (``fftw-3.3.11-neon``). The single - precision library lies beside the double one under the repair's naming. + For example ``fftw-3.3.11-neon``. """ - single = sorted( - { - path.resolve() - for path in Path(fft_library).parent.glob("libfftw3f[-.]*") - if path.suffix not in {".a", ".la"} - } - ) - if len(single) != 1: - sys.exit(f"expected one single-precision FFTW beside {fft_library}: {single}") - versions = {} - for symbol, library in ( - ("fftw_version", fft_library), - ("fftwf_version", str(single[0])), - ): - handle = ctypes.CDLL(library) - address = ctypes.addressof(ctypes.c_char.in_dll(handle, symbol)) - versions[symbol] = ctypes.string_at(address).decode() - return versions + handle = ctypes.CDLL(fft_library) + address = ctypes.addressof(ctypes.c_char.in_dll(handle, "fftw_version")) + return ctypes.string_at(address).decode() + + +def single_precision_fftw(package: Path) -> list[str]: + """Return what in the installed wheel names a single-precision FFTW. + + The package computes in double precision only: the build points Kokkos-FFT's + single-precision library names at the double libraries + (``cmake/HostLibraries.cmake``), so no ``libfftw3f`` is linked or bundled, and the + extension imports no ``fftwf_`` symbol (an import's name is in the extension's + dynamic string table). + """ + found = [ + str(path) + for root in (package, package.parent / "solprop_native.libs") + if root.is_dir() + for path in root.rglob("*fftw3f*") + ] + found += [ + f"fftwf_ in {path.name}" + for path in package.glob("_core*") + if path.suffix in {".so", ".pyd"} and b"fftwf_" in path.read_bytes() + ] + return found def main() -> None: @@ -98,12 +106,14 @@ def main() -> None: and not bundled(library, package) ): sys.exit(f"{name} {library} is not the wheel's own copy (under {package})") + if single := single_precision_fftw(package): + sys.exit(f"the wheel carries a single-precision FFTW: {single}") # Compiling with -mcpu does not enable FFTW's NEON codelets; the recipe must. fftw = {} if platform.machine() in {"arm64", "aarch64"}: - fftw = fftw_versions(info.fft_library) - if scalar := [name for name, version in fftw.items() if "-neon" not in version]: - sys.exit(f"the bundled FFTW lacks NEON codelets: {fftw} ({scalar})") + fftw = {"fftw_version": fftw_version(info.fft_library)} + if "-neon" not in fftw["fftw_version"]: + sys.exit(f"the bundled FFTW lacks NEON codelets: {fftw}") if info.blas_backend == "OpenBLAS" and info.blas_threads is None: sys.exit("the bundled OpenBLAS reports no thread count") # Every library the wheel bundles travels with its licence. diff --git a/native/cmake/HostLibraries.cmake b/native/cmake/HostLibraries.cmake index 784120ac4..5d6d71bc8 100644 --- a/native/cmake/HostLibraries.cmake +++ b/native/cmake/HostLibraries.cmake @@ -219,14 +219,6 @@ if(SOLPROP_NATIVE_ENABLE_CUDA OR SOLPROP_NATIVE_ENABLE_HIP) endif() return() endif() -if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" AND - SOLPROP_NATIVE_ENABLE_OPENMP AND - SOLPROP_NATIVE_FFT_PROVIDER STREQUAL "FFTW" AND - SOLPROP_NATIVE_FFTW_THREADING STREQUAL "OPENMP") - message(FATAL_ERROR - "IntelLLVM/FFTW CPU builds require FFTW_THREADING=PTHREADS. " - "Distribution OpenMP FFTW libraries link GNU OpenMP, not Intel's runtime.") -endif() if(_tpl_request STREQUAL "OFF" AND (NOT SOLPROP_NATIVE_BLAS_VENDOR STREQUAL "AUTO" OR SOLPROP_NATIVE_BLAS_LIBRARIES)) message(FATAL_ERROR @@ -237,11 +229,7 @@ if(NOT SOLPROP_NATIVE_BLAS_VENDOR STREQUAL "AUTO" AND message(FATAL_ERROR "Select BLAS_VENDOR or explicit BLAS_LIBRARIES, not both.") endif() -if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - set(_mkl_thread mkl_intel_thread) -else() - set(_mkl_thread mkl_gnu_thread) -endif() +set(_mkl_thread mkl_gnu_thread) if(NOT SOLPROP_NATIVE_ENABLE_OPENMP) set(_mkl_thread mkl_sequential) endif() @@ -476,8 +464,9 @@ if(SOLPROP_NATIVE_FFT_PROVIDER STREQUAL "MKL_FFTW") HINTS "${SOLPROP_NATIVE_MKL_ROOT}" PATH_SUFFIXES include/fftw REQUIRED) set(FFTW_INCLUDE_DIRS "${SOLPROP_NATIVE_MKL_FFT_INCLUDE}" CACHE PATH "FFTW API headers" FORCE) # Kokkos FFT uses the same FFTW C API for both thread implementations. - # MKL exports both precisions and the planning-thread controls from its LP64 - # interface; its dependency libraries are linked after the FFT/BLAS targets. + # MKL exports the planning-thread controls from its LP64 interface; its + # dependency libraries are linked after the FFT/BLAS targets. The single-precision + # names are the same library, as with FFTW below. foreach(_component IN ITEMS DOUBLE_LIB FLOAT_LIB DOUBLE_OPENMP_LIB FLOAT_OPENMP_LIB) set(FFTW_${_component} "${SOLPROP_NATIVE_mkl_intel_lp64_LIBRARY}" CACHE FILEPATH "MKL FFTW compatibility library" FORCE) @@ -485,19 +474,12 @@ if(SOLPROP_NATIVE_FFT_PROVIDER STREQUAL "MKL_FFTW") set(SOLPROP_NATIVE_MKL_FFT_LIBRARIES "${_mkl_libraries}") set(SOLPROP_NATIVE_HOST_FFT_BACKEND "MKL_FFTW") set(SOLPROP_NATIVE_HOST_FFT_PLANNER "MKL_ignores_FFTW_planning_flags") -elseif(SOLPROP_NATIVE_ENABLE_THREADS) - # Kokkos FFT links FFTW's threads library for the Threads host space. - set(SOLPROP_NATIVE_HOST_FFT_BACKEND "FFTW_THREADS") elseif(SOLPROP_NATIVE_ENABLE_OPENMP) if(SOLPROP_NATIVE_FFTW_THREADING STREQUAL "PTHREADS") find_library(SOLPROP_NATIVE_FFTW_DOUBLE_PTHREADS fftw3_threads HINTS "${FFTW_ROOT}" PATH_SUFFIXES lib lib64 REQUIRED) - find_library(SOLPROP_NATIVE_FFTW_FLOAT_PTHREADS fftw3f_threads - HINTS "${FFTW_ROOT}" PATH_SUFFIXES lib lib64 REQUIRED) set(FFTW_DOUBLE_OPENMP_LIB "${SOLPROP_NATIVE_FFTW_DOUBLE_PTHREADS}" CACHE FILEPATH "FFTW pthread implementation of the thread API" FORCE) - set(FFTW_FLOAT_OPENMP_LIB "${SOLPROP_NATIVE_FFTW_FLOAT_PTHREADS}" - CACHE FILEPATH "FFTW pthread implementation of the thread API" FORCE) set(SOLPROP_NATIVE_HOST_FFT_BACKEND "FFTW_PTHREADS") else() set(SOLPROP_NATIVE_HOST_FFT_BACKEND "FFTW_OPENMP") @@ -505,3 +487,31 @@ elseif(SOLPROP_NATIVE_ENABLE_OPENMP) else() set(SOLPROP_NATIVE_HOST_FFT_BACKEND "FFTW_SERIAL") endif() + +# solprop computes in double precision only. Kokkos-FFT's FindFFTW requires the +# single-precision libraries as well, though each of its fftwf_ calls sits behind an +# `if constexpr` on float that this package never instantiates. Name the double +# libraries for them, as the MKL branch above does, so that a build needs no +# single-precision FFTW and links none. FindFFTW keeps these cached results; a +# non-default FFTW is named with FFTW_ROOT (or FFTWDIR), as for FindFFTW. +if(SOLPROP_NATIVE_FFT_PROVIDER STREQUAL "FFTW") + set(_suffixes "${CMAKE_FIND_LIBRARY_SUFFIXES}") + if(FFTW_USE_STATIC_LIBS) + set(CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_STATIC_LIBRARY_SUFFIX}") + endif() + set(_components DOUBLE_LIB:fftw3) + if(SOLPROP_NATIVE_ENABLE_OPENMP) + list(APPEND _components DOUBLE_OPENMP_LIB:fftw3_omp) + endif() + foreach(_component IN LISTS _components) + string(REPLACE ":" ";" _component "${_component}") + list(GET _component 0 _variable) + list(GET _component 1 _name) + find_library(FFTW_${_variable} NAMES ${_name} HINTS "${FFTW_ROOT}" "$ENV{FFTWDIR}" + PATH_SUFFIXES lib lib64 REQUIRED) + string(REPLACE "DOUBLE" "FLOAT" _float "${_variable}") + set(FFTW_${_float} "${FFTW_${_variable}}" CACHE FILEPATH + "The double-precision FFTW: the build makes no single-precision call" FORCE) + endforeach() + set(CMAKE_FIND_LIBRARY_SUFFIXES "${_suffixes}") +endif() diff --git a/native/cmake/SourceDigest.cmake b/native/cmake/SourceDigest.cmake index 9440f25cf..c94b8cbf4 100644 --- a/native/cmake/SourceDigest.cmake +++ b/native/cmake/SourceDigest.cmake @@ -34,6 +34,7 @@ string(SHA256 digest "${record}") file(CONFIGURE OUTPUT "${OUTPUT}" @ONLY CONTENT "// Generated by cmake/SourceDigest.cmake: the digest of what the extension is compiled from. namespace solprop_native { +// NOLINTNEXTLINE(misc-use-internal-linkage): bindings.cpp declares and reads it const char* source_digest() noexcept { return \"@digest@\"; } } // namespace solprop_native ") diff --git a/native/cmake/VectorMath.cmake b/native/cmake/VectorMath.cmake index 6ef397cb3..b1d83ef37 100644 --- a/native/cmake/VectorMath.cmake +++ b/native/cmake/VectorMath.cmake @@ -85,13 +85,12 @@ function(_solprop_native_add_sleef) FetchContent_MakeAvailable(sleef) include(ExternalProject) # SLEEF requires a C and a C++ compiler of one version: the C driver beside this - # build's C++ compiler (clang beside clang++, gcc beside g++, icx beside icpx), else + # build's C++ compiler (clang beside clang++, gcc beside g++), else # the C compiler this project found. get_filename_component(_cxx_directory "${CMAKE_CXX_COMPILER}" DIRECTORY) get_filename_component(_cxx_name "${CMAKE_CXX_COMPILER}" NAME) string(REGEX REPLACE "clang\\+\\+" "clang" _c_name "${_cxx_name}") string(REGEX REPLACE "g\\+\\+" "gcc" _c_name "${_c_name}") - string(REGEX REPLACE "^icpx" "icx" _c_name "${_c_name}") set(_c_compiler "${_cxx_directory}/${_c_name}") if(_c_name STREQUAL _cxx_name OR NOT EXISTS "${_c_compiler}") set(_c_compiler "${CMAKE_C_COMPILER}") diff --git a/native/include/solprop_native/adams_march.hpp b/native/include/solprop_native/adams_march.hpp index b0f7904dc..f7c586de5 100644 --- a/native/include/solprop_native/adams_march.hpp +++ b/native/include/solprop_native/adams_march.hpp @@ -4,6 +4,8 @@ #pragma once +#include "solprop_native/watch.hpp" + #include #include #include @@ -13,8 +15,6 @@ #include #include -#include "solprop_native/watch.hpp" - /* * The exponential Adams march executed natively (solprop SEP 39 phase 2). * diff --git a/native/include/solprop_native/adams_phi.hpp b/native/include/solprop_native/adams_phi.hpp index 9e9b19e22..d00f7ce07 100644 --- a/native/include/solprop_native/adams_phi.hpp +++ b/native/include/solprop_native/adams_phi.hpp @@ -81,7 +81,7 @@ adams_phi_fixed(const Complex x, const double magnitude, const Complex exp_x, // 1/n for the series' divisors m + order, the doubles the division rounds to. static_assert(adams_series_terms + 2 * adams_max_samples <= 40); constexpr double inverse_divisor[40] = { - 0.0, 1.0 / 1.0, 1.0 / 2.0, 1.0 / 3.0, 1.0 / 4.0, 1.0 / 5.0, + 0.0, 1.0, 1.0 / 2.0, 1.0 / 3.0, 1.0 / 4.0, 1.0 / 5.0, 1.0 / 6.0, 1.0 / 7.0, 1.0 / 8.0, 1.0 / 9.0, 1.0 / 10.0, 1.0 / 11.0, 1.0 / 12.0, 1.0 / 13.0, 1.0 / 14.0, 1.0 / 15.0, 1.0 / 16.0, 1.0 / 17.0, 1.0 / 18.0, 1.0 / 19.0, 1.0 / 20.0, 1.0 / 21.0, 1.0 / 22.0, 1.0 / 23.0, @@ -168,7 +168,7 @@ inline constexpr double adams_lane_reciprocal[adams_max_samples + 2] = { 1.0 / 120.0, 1.0 / 720.0, 1.0 / 5040.0, 1.0 / 40320.0, 1.0 / 362880.0}; inline constexpr double adams_lane_inverse_divisor[adams_series_terms + adams_max_samples + 1] = { - 0.0, 1.0 / 1.0, 1.0 / 2.0, 1.0 / 3.0, 1.0 / 4.0, 1.0 / 5.0, 1.0 / 6.0, + 0.0, 1.0, 1.0 / 2.0, 1.0 / 3.0, 1.0 / 4.0, 1.0 / 5.0, 1.0 / 6.0, 1.0 / 7.0, 1.0 / 8.0, 1.0 / 9.0, 1.0 / 10.0, 1.0 / 11.0, 1.0 / 12.0, 1.0 / 13.0, 1.0 / 14.0, 1.0 / 15.0, 1.0 / 16.0, 1.0 / 17.0, 1.0 / 18.0, 1.0 / 19.0, 1.0 / 20.0, 1.0 / 21.0, 1.0 / 22.0, 1.0 / 23.0, 1.0 / 24.0, 1.0 / 25.0, 1.0 / 26.0, 1.0 / 27.0, @@ -223,6 +223,9 @@ struct AdamsLanes { double dr[B], di[B]; double defect[3][B]; }; +// The lane passes spell their branches as signed 64-bit masks and bitwise selections, +// the form every x86-64 target vectorises. +// NOLINTBEGIN(bugprone-signed-bitwise, readability-implicit-bool-conversion): lane masks // The moments' argument and its branch, halvings and series argument, as // `adams_moment_argument`, `adams_phi` and `adams_phi_argument` form them, with the @@ -257,13 +260,10 @@ inline void adams_lanes_classify(AdamsLanes& lanes, const ExponentialWeights& we const double square = rr + ii; const std::int64_t zero = static_cast(lanes.ar[b] == 0.0) & static_cast(lanes.ai[b] == 0.0); - const std::int64_t large = static_cast(std::isgreater(square, 9.0)); - const std::int64_t above_quarter = - static_cast(std::isgreater(square, 0.25)); - const std::int64_t above_one = - static_cast(std::isgreater(square, 1.0)); - const std::int64_t above_four = - static_cast(std::isgreater(square, 4.0)); + const auto large = static_cast(std::isgreater(square, 9.0)); + const auto above_quarter = static_cast(std::isgreater(square, 0.25)); + const auto above_one = static_cast(std::isgreater(square, 1.0)); + const auto above_four = static_cast(std::isgreater(square, 4.0)); const std::int64_t uncertain = static_cast(!std::islessequal(square, finite)) | static_cast( @@ -279,7 +279,7 @@ inline void adams_lanes_classify(AdamsLanes& lanes, const ExponentialWeights& we // zero 0, large 1, series 2, and 2^-squarings, each factor exact; spelt as // arithmetic and selections of bits, which every x86-64 target vectorises. const std::int64_t branch = (1 - zero) * (2 - large); - const std::int64_t series = static_cast(branch == adams_lane_series); + const auto series = static_cast(branch == adams_lane_series); lanes.branch[b] = branch; lanes.squarings[b] = series * (above_quarter + above_one + above_four); scale[b] = adams_lane_select(above_quarter != 0, 0.5, 1.0) * @@ -329,7 +329,10 @@ inline void adams_lanes_phi(AdamsLanes& lanes, const std::size_t order) { constexpr double stop = quarter_ulp * quarter_ulp; // The series: every lane takes its terms until its own stopping test holds; a lane // that has stopped keeps its sum and carries a zero term. - double term_r[B], term_i[B], sum_r[B], sum_i[B]; + double term_r[B]; + double term_i[B]; + double sum_r[B]; + double sum_i[B]; std::int64_t live[B]; const double first = adams_lane_reciprocal[order]; for (std::size_t b = 0; b < count; ++b) { @@ -412,7 +415,8 @@ inline void adams_lanes_phi(AdamsLanes& lanes, const std::size_t order) { double half = 1.0; // 2^-k, exact for (std::size_t k = 1; k <= order; ++k) { half *= 0.5; - double total_r[B], total_i[B]; + double total_r[B]; + double total_i[B]; for (std::size_t b = 0; b < count; ++b) { const double rr = lanes.qr[0][b] * lanes.qr[k][b]; const double ii = lanes.qi[0][b] * lanes.qi[k][b]; @@ -440,7 +444,10 @@ inline void adams_lanes_phi(AdamsLanes& lanes, const std::size_t order) { // Large arguments: phi_k = (phi_{k-1} - 1/(k-1)!) / x, the inverse by // `strict_divide`'s scaled algorithm. if (lanes.any_large) { - double inverse_r[B], inverse_i[B], last_r[B], last_i[B]; + double inverse_r[B]; + double inverse_i[B]; + double last_r[B]; + double last_i[B]; for (std::size_t b = 0; b < count; ++b) { const double s = Kokkos::fabs(lanes.ar[b]) + Kokkos::fabs(lanes.ai[b]); const double numerator_r = 1.0 / s; @@ -488,6 +495,8 @@ inline void adams_lanes_phi(AdamsLanes& lanes, const std::size_t order) { } } } +// NOLINTEND(bugprone-signed-bitwise, readability-implicit-bool-conversion) + // e^argument for the large branch into `gr`/`gi`: the flow `fr`/`fi`, or under a // carrier frame the flow times exp(-sigma), as `adams_moment_phi` forms it. Strict. SOLPROP_NATIVE_STRICT_FUNCTION diff --git a/native/include/solprop_native/core.hpp b/native/include/solprop_native/core.hpp index ecb44edec..09fa67220 100644 --- a/native/include/solprop_native/core.hpp +++ b/native/include/solprop_native/core.hpp @@ -4,11 +4,11 @@ #pragma once +#include "solprop_native/launch.hpp" + #include #include -#include "solprop_native/launch.hpp" - #include #include #include @@ -138,7 +138,7 @@ inline constexpr std::size_t occupancy_bucket_limit = 128; // checks only the first: a value that fills the block passes its check and the dispatch // then faults on an AMD Instinct MI325X ("Illegal opcode in command stream"). So a // reading there takes two passes, leaving half the block free. -#if defined(KOKKOS_ENABLE_HIP) +#ifdef KOKKOS_ENABLE_HIP inline constexpr std::size_t occupancy_reduction_slots = 64; #else inline constexpr std::size_t occupancy_reduction_slots = occupancy_bucket_limit; @@ -151,7 +151,7 @@ struct OccupancySums { return *this; } }; -#if defined(KOKKOS_ENABLE_HIP) +#ifdef KOKKOS_ENABLE_HIP static_assert((64 + 2) * sizeof(OccupancySums) <= 48 * 1024); #endif diff --git a/native/include/solprop_native/coupled_flow.hpp b/native/include/solprop_native/coupled_flow.hpp index 5aeca605b..8faf107b5 100644 --- a/native/include/solprop_native/coupled_flow.hpp +++ b/native/include/solprop_native/coupled_flow.hpp @@ -46,7 +46,7 @@ class CoupledFlowAction { using IndexView = Kokkos::View; using IndexView2 = Kokkos::View; - CoupledFlowAction(const Problem& problem, const ExecutionSpace& kernels); + CoupledFlowAction(const Problem& problem, ExecutionSpace kernels); ~CoupledFlowAction(); CoupledFlowAction(const CoupledFlowAction&) = delete; CoupledFlowAction& operator=(const CoupledFlowAction&) = delete; diff --git a/native/include/solprop_native/dense_output.hpp b/native/include/solprop_native/dense_output.hpp index 19341d898..41dbc7af8 100644 --- a/native/include/solprop_native/dense_output.hpp +++ b/native/include/solprop_native/dense_output.hpp @@ -4,15 +4,15 @@ #pragma once -#include -#include -#include - #include "solprop_native/adams_march.hpp" #include "solprop_native/core.hpp" #include "solprop_native/dense.hpp" #include "solprop_native/stage_tables.hpp" +#include +#include +#include + /* * The native session's dense output over the resident state (solprop SEP 47 section * 5.3): the session-side half of `session::Dense`, whose field arithmetic is the diff --git a/native/include/solprop_native/dispersive.hpp b/native/include/solprop_native/dispersive.hpp index 614638c8b..a9c7cbedc 100644 --- a/native/include/solprop_native/dispersive.hpp +++ b/native/include/solprop_native/dispersive.hpp @@ -91,7 +91,12 @@ KOKKOS_INLINE_FUNCTION void dispersive_conserving(const Complex x0, const Comple const Complex y0, const Complex y1, const Complex z0, const Complex z1, Complex& out0, Complex& out1) { - Complex a0, a1, b0, b1, c0, c1; + Complex a0; + Complex a1; + Complex b0; + Complex b1; + Complex c0; + Complex c1; dispersive_isotropic(x0, x1, y0, y1, Kokkos::conj(z0), Kokkos::conj(z1), a0, a1); dispersive_isotropic(x0, x1, Kokkos::conj(y0), Kokkos::conj(y1), z0, z1, b0, b1); dispersive_isotropic(Kokkos::conj(x0), Kokkos::conj(x1), y0, y1, z0, z1, c0, c1); @@ -105,11 +110,21 @@ KOKKOS_INLINE_FUNCTION void dispersive_harmonic(const Complex x0, const Complex const Complex z0, const Complex z1, const Complex carrier, Complex& out0, Complex& out1) { - Complex u0, u1, a0, a1, b0, b1, c0, c1; + Complex u0; + Complex u1; + Complex a0; + Complex a1; + Complex b0; + Complex b1; + Complex c0; + Complex c1; dispersive_isotropic(x0, x1, y0, y1, z0, z1, u0, u1); - const Complex x0c = Kokkos::conj(x0), x1c = Kokkos::conj(x1); - const Complex y0c = Kokkos::conj(y0), y1c = Kokkos::conj(y1); - const Complex z0c = Kokkos::conj(z0), z1c = Kokkos::conj(z1); + const Complex x0c = Kokkos::conj(x0); + const Complex x1c = Kokkos::conj(x1); + const Complex y0c = Kokkos::conj(y0); + const Complex y1c = Kokkos::conj(y1); + const Complex z0c = Kokkos::conj(z0); + const Complex z1c = Kokkos::conj(z1); dispersive_isotropic(x0, x1, y0c, y1c, z0c, z1c, a0, a1); dispersive_isotropic(x0c, x1c, y0, y1, z0c, z1c, b0, b1); dispersive_isotropic(x0c, x1c, y0c, y1c, z0, z1, c0, c1); @@ -201,13 +216,17 @@ template KOKKOS_INLINE_FUNCTION void dispersive_conserving_n(const Complex (&x)[Components], const Complex (&y)[Components], const Complex (&z)[Components], Complex (&out)[Components]) { - Complex xc[Components], yc[Components], zc[Components]; + Complex xc[Components]; + Complex yc[Components]; + Complex zc[Components]; for (std::size_t c = 0; c < Components; ++c) { xc[c] = Kokkos::conj(x[c]); yc[c] = Kokkos::conj(y[c]); zc[c] = Kokkos::conj(z[c]); } - Complex a[Components], b[Components], d[Components]; + Complex a[Components]; + Complex b[Components]; + Complex d[Components]; dispersive_isotropic_n(x, y, zc, a); dispersive_isotropic_n(x, yc, z, b); dispersive_isotropic_n(xc, y, z, d); @@ -255,7 +274,7 @@ class DispersiveWeights { using RealView2 = Kokkos::View; using IndexView = Kokkos::View; - DispersiveWeights(const Problem& problem, const ExecutionSpace& kernels); + DispersiveWeights(const Problem& problem, ExecutionSpace kernels); ~DispersiveWeights() = default; DispersiveWeights(const DispersiveWeights&) = delete; DispersiveWeights& operator=(const DispersiveWeights&) = delete; @@ -423,10 +442,10 @@ class DispersiveResponse { bool borrowed_{}, shifted_{}, coarse_pending_{}, closed_{}; ExecutionSpace kernels_, response_, tail_response_, dense_, tail_dense_, matrix_copy_; std::size_t response_chunk_{}; - const ExecutionSpace& response_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& response_for(std::size_t count) const { return count == response_chunk_ ? response_ : tail_response_; } - const ExecutionSpace& dense_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& dense_for(std::size_t count) const { return count == response_chunk_ ? dense_ : tail_dense_; } StageViews stage_; @@ -536,10 +555,10 @@ class SeparableDispersive { std::size_t tail_rows_{}, active_rows_{}; bool active_tail_{}; std::size_t response_chunk_{}; - const ExecutionSpace& response_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& response_for(std::size_t count) const { return count == response_chunk_ ? response_ : tail_response_; } - const ExecutionSpace& dense_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& dense_for(std::size_t count) const { return count == response_chunk_ ? dense_ : tail_dense_; } StageViews stage_; @@ -627,7 +646,7 @@ class GeneralDispersive { std::size_t tail_rows_{}, active_rows_{}, last_chunk_{}; bool active_tail_{}; std::size_t response_chunk_{}; - const ExecutionSpace& response_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& response_for(std::size_t count) const { return count == response_chunk_ ? response_ : tail_response_; } StageViews stage_; diff --git a/native/include/solprop_native/engines.hpp b/native/include/solprop_native/engines.hpp index dff00e333..13f23b52e 100644 --- a/native/include/solprop_native/engines.hpp +++ b/native/include/solprop_native/engines.hpp @@ -4,18 +4,18 @@ #pragma once -#include -#include -#include -#include -#include - #include "solprop_native/adams_march.hpp" #include "solprop_native/core.hpp" #include "solprop_native/rk_march.hpp" #include "solprop_native/session.hpp" #include "solprop_native/stage_tables.hpp" +#include +#include +#include +#include +#include + /* * The engines the native marches hand their field arithmetic to (solprop SEP 39 and * SEP 45): the resident engines over a native state, and the scripted engines of the diff --git a/native/include/solprop_native/envelope_plasma.hpp b/native/include/solprop_native/envelope_plasma.hpp index 5e0e58bb5..eb0ee415f 100644 --- a/native/include/solprop_native/envelope_plasma.hpp +++ b/native/include/solprop_native/envelope_plasma.hpp @@ -27,9 +27,12 @@ KOKKOS_INLINE_FUNCTION EnvelopeAxes envelope_axes( Kokkos::max(Kokkos::abs(second.real()), Kokkos::abs(second.imag()))); if (scale == 0.0) return {}; - const double xr = first.real() / scale, xi = first.imag() / scale; - const double yr = second.real() / scale, yi = second.imag() / scale; - const double xx = xr * xr + yr * yr, yy = xi * xi + yi * yi; + const double xr = first.real() / scale; + const double xi = first.imag() / scale; + const double yr = second.real() / scale; + const double yi = second.imag() / scale; + const double xx = xr * xr + yr * yr; + const double yy = xi * xi + yi * yi; const double xy = xr * xi + yr * yi; // A scaled norm avoids an underflow observed in the device hypot. const double larger = Kokkos::max(Kokkos::abs(xx - yy), Kokkos::abs(2 * xy)); @@ -40,20 +43,22 @@ KOKKOS_INLINE_FUNCTION EnvelopeAxes envelope_axes( const double determinant = xr * yi - xi * yr; const double ratio = Kokkos::min(1.0, Kokkos::abs(determinant) / major_square); const double angle = 0.5 * Kokkos::atan2(2 * xy, xx - yy); - const double cosine = Kokkos::cos(angle), sine = Kokkos::sin(angle); - const double ux = xr * cosine + xi * sine, uy = yr * cosine + yi * sine; + const double cosine = Kokkos::cos(angle); + const double sine = Kokkos::sin(angle); + const double ux = xr * cosine + xi * sine; + const double uy = yr * cosine + yi * sine; // The determinant retains the small axis near a degenerate linear ellipse. const double signed_ratio = determinant / major_square; - return {scale * Kokkos::sqrt(major_square), - ratio, - cosine, - sine, - ux, - uy, - -signed_ratio * uy, - signed_ratio * ux, - scale, - major_square}; + return {.major = scale * Kokkos::sqrt(major_square), + .ratio = ratio, + .cosine = cosine, + .sine = sine, + .ux = ux, + .uy = uy, + .vx = -signed_ratio * uy, + .vy = signed_ratio * ux, + .scale = scale, + .major_square = major_square}; } /** Positive directional cycle moments in s^-1 and a refinement failure flag. */ @@ -78,12 +83,13 @@ envelope_moments(const double major, const double ratio, const double field_min, return {}; if (ratio == 1.0) { const double rate = 0.5 * rate_at(major); - return {rate, rate, false}; + return {.cosine = rate, .sine = rate, .failed = false}; } if (!Kokkos::isfinite(major) || !Kokkos::isfinite(ratio)) - return {0.0, 0.0, true}; + return {.cosine = 0.0, .sine = 0.0, .failed = true}; constexpr double half_pi = std::numbers::pi_v / 2.0; - const double high = field_max / major, low = field_min / major; + const double high = field_max / major; + const double low = field_min / major; const double high_edge = high >= 1.0 ? 0.0 : high <= ratio ? half_pi @@ -94,10 +100,12 @@ envelope_moments(const double major, const double ratio, const double field_min, ? half_pi : Kokkos::acos(Kokkos::sqrt((low * low - ratio * ratio) / (1 - ratio * ratio))); - double previous_c = 0.0, previous_s = 0.0; + double previous_c = 0.0; + double previous_s = 0.0; for (std::size_t level = 0; level < 4; ++level) { const std::size_t order = std::size_t{64} << level; - double mc = 0.0, ms = 0.0; + double mc = 0.0; + double ms = 0.0; for (std::size_t panel = 0; panel < 3; ++panel) { const double lower = panel == 0 ? 0.0 : panel == 1 ? high_edge : low_edge; const double upper = panel == 0 ? high_edge : panel == 1 ? low_edge : half_pi; @@ -106,8 +114,10 @@ envelope_moments(const double major, const double ratio, const double field_min, continue; for (std::size_t index = 0; index < order; ++index) { const double theta = lower + (nodes(level, index) + 1) * (0.5 * width); - const double cosine = Kokkos::cos(theta), sine = Kokkos::sin(theta); - const double c2 = cosine * cosine, s2 = sine * sine; + const double cosine = Kokkos::cos(theta); + const double sine = Kokkos::sin(theta); + const double c2 = cosine * cosine; + const double s2 = sine * sine; const double denominator = c2 + ratio * ratio * s2; const double rate = rate_at(major * Kokkos::sqrt(denominator)) / denominator; const double weight = @@ -119,11 +129,11 @@ envelope_moments(const double major, const double ratio, const double field_min, if (level > 0 && Kokkos::isfinite(mc) && Kokkos::isfinite(ms) && Kokkos::abs(mc - previous_c) <= 2e-6 * Kokkos::max(mc, 1e-30) && Kokkos::abs(ms - previous_s) <= 2e-6 * Kokkos::max(ms, 1e-30)) - return {mc, ms, false}; + return {.cosine = mc, .sine = ms, .failed = false}; previous_c = mc; previous_s = ms; } - return {previous_c, previous_s, true}; + return {.cosine = previous_c, .sine = previous_s, .failed = true}; } /** Reconstruct the two loss-current components in A/m^2 from work in J/m^3. */ @@ -135,7 +145,8 @@ envelope_loss(const EnvelopeAxes& axes, const double mc, const double ms, const Kokkos::complex major_phase(axes.cosine, axes.sine); const Kokkos::complex minor_phase(-axes.sine, axes.cosine); // Dividing moments first avoids an overflowing standalone inverse tiny scale. - const double major_moment = mc / axes.scale, minor_moment = ms / axes.scale; + const double major_moment = mc / axes.scale; + const double minor_moment = ms / axes.scale; const double factor = 2 * work / axes.major_square; return {factor * (major_moment * axes.ux * major_phase + minor_moment * axes.vx * minor_phase), diff --git a/native/include/solprop_native/far_field.hpp b/native/include/solprop_native/far_field.hpp index 0e7b5b20d..061fec5e7 100644 --- a/native/include/solprop_native/far_field.hpp +++ b/native/include/solprop_native/far_field.hpp @@ -147,7 +147,7 @@ KOKKOS_INLINE_FUNCTION std::size_t far_field_band_edge(const std::size_t band, * protected block of plain arithmetic rounds each operation on its own. CUDA's * intrinsics are real. */ -#if defined(__HIPCC__) +#ifdef __HIPCC__ #define SOLPROP_NATIVE_FAR_FIELD_HIP_STRICT _Pragma("clang fp contract(off)") #else #define SOLPROP_NATIVE_FAR_FIELD_HIP_STRICT @@ -172,7 +172,7 @@ KOKKOS_INLINE_FUNCTION double far_field_radius(const double scale, const double const double centre_y) { SOLPROP_NATIVE_STRICT_BLOCK SOLPROP_NATIVE_FAR_FIELD_HIP_STRICT -#if defined(__CUDA_ARCH__) +#ifdef __CUDA_ARCH__ const double across = __dadd_rn(__dmul_rn(scale, k_x), centre_x); const double along = __dadd_rn(__dmul_rn(scale, k_y), centre_y); return __dsqrt_rn(__dadd_rn(__dmul_rn(across, across), __dmul_rn(along, along))); @@ -196,7 +196,7 @@ KOKKOS_INLINE_FUNCTION double far_field_offset(const double scale, const double const double centre) { SOLPROP_NATIVE_STRICT_BLOCK SOLPROP_NATIVE_FAR_FIELD_HIP_STRICT -#if defined(__CUDA_ARCH__) +#ifdef __CUDA_ARCH__ return __dadd_rn(__dmul_rn(scale, k), centre); #else return scale * k + centre; @@ -216,7 +216,7 @@ KOKKOS_INLINE_FUNCTION double far_field_scaled_radius(const double across, const double stretch_y) { SOLPROP_NATIVE_STRICT_BLOCK SOLPROP_NATIVE_FAR_FIELD_HIP_STRICT -#if defined(__CUDA_ARCH__) +#ifdef __CUDA_ARCH__ const double scaled_x = __dmul_rn(stretch_x, across); const double scaled_y = __dmul_rn(stretch_y, along); return __dsqrt_rn( @@ -361,7 +361,7 @@ class FarFieldPass { double evanescent{}; }; - FarFieldPass(const Problem& problem, const ExecutionSpace& execution); + FarFieldPass(const Problem& problem, ExecutionSpace execution); ~FarFieldPass() = default; FarFieldPass(const FarFieldPass&) = delete; FarFieldPass& operator=(const FarFieldPass&) = delete; diff --git a/native/include/solprop_native/fft_plan.hpp b/native/include/solprop_native/fft_plan.hpp index 6d173d3f1..8f9617209 100644 --- a/native/include/solprop_native/fft_plan.hpp +++ b/native/include/solprop_native/fft_plan.hpp @@ -4,19 +4,21 @@ #pragma once +#include "solprop_native/core.hpp" + #include -#include -#include "solprop_native/core.hpp" +#include +#include #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -#include +#include + #include #include #include #include #include -#include #include #include #include @@ -140,14 +142,13 @@ inline bool adds_wisdom(const std::string& planned, const std::string& stored) { found.emplace_back(text.data() + first, last - first); first = last + 1; } - std::sort(found.begin(), found.end()); + std::ranges::sort(found); return found; }; const auto had = lines(stored); - for (const auto line : lines(planned)) - if (!std::binary_search(had.begin(), had.end(), line)) - return true; - return false; + return std::ranges::any_of(lines(planned), [&had](const auto line) { + return !std::ranges::binary_search(had, line); + }); } // Merge `text` beneath the process's wisdom: where both hold an entry, the process's @@ -398,7 +399,7 @@ inline std::vector split_key(const std::string& text, const char by return parts; } -inline double median_execution(const fftw_plan plan) { +inline double median_execution(fftw_plan plan) { using Clock = std::chrono::steady_clock; fftw_execute(plan); const auto first = Clock::now(); @@ -412,7 +413,7 @@ inline double median_execution(const fftw_plan plan) { fftw_execute(plan); batch = std::chrono::duration(Clock::now() - start).count() / repeats; } - std::sort(batches.begin(), batches.end()); + std::ranges::sort(batches); return batches[2]; } } // namespace detail @@ -436,7 +437,8 @@ inline PlanAlone time_fft_plan(const std::string& key, const std::string& wisdom // The step's waiting policy and FFTW team: without the scope LLVM's runtime sleeps // its idle threads at once (`KMP_BLOCKTIME=0`) and each loop pays their wake-up. const CpuTeamScope scope(threads); - const bool r2c = kind == "r2c", c2r = kind == "c2r"; + const bool r2c = kind == "r2c"; + const bool c2r = kind == "c2r"; const bool planes = kind.starts_with("planes-"); const int sign = kind.ends_with("backward") ? FFTW_BACKWARD : FFTW_FORWARD; if (!r2c && !c2r && !planes && !kind.starts_with("c2c-")) @@ -480,7 +482,7 @@ inline PlanAlone time_fft_plan(const std::string& key, const std::string& wisdom if ((flags & FFTW_WISDOM_ONLY) != 0U && fftw_import_wisdom_from_string(wisdom.c_str()) == 0) return std::numeric_limits::quiet_NaN(); - const fftw_plan plan = make(flags); + auto* const plan = make(flags); if (plan == nullptr) { fftw_forget_wisdom(); return std::numeric_limits::quiet_NaN(); @@ -512,7 +514,6 @@ class FftPlanningScope { #include "solprop_native/graph.hpp" #include -#include #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) #include "solprop_native/launch.hpp" diff --git a/native/include/solprop_native/flow_functions.hpp b/native/include/solprop_native/flow_functions.hpp index 4a2aa8c91..d12f768b7 100644 --- a/native/include/solprop_native/flow_functions.hpp +++ b/native/include/solprop_native/flow_functions.hpp @@ -130,7 +130,7 @@ augmented_row(const Complex shift, const double length, const Triangular& triang // block and the next block row's scalars, a later block row the scalars alone. Each // new block is formed from the old ones into the scratch, then copied. for (int term = taylor_order; term > 0; --term) { - const double by = static_cast(term); + const auto by = static_cast(term); for (std::size_t j = 0; j <= count; ++j) { const double previous = j >= 1 ? p[j - 1] : 0.0; for (std::size_t row = 0; row < modes; ++row) diff --git a/native/include/solprop_native/fp.hpp b/native/include/solprop_native/fp.hpp index 1e0f8c3af..73d772512 100644 --- a/native/include/solprop_native/fp.hpp +++ b/native/include/solprop_native/fp.hpp @@ -10,7 +10,7 @@ * Floating-point contraction policy (solprop SEP 25, sections 4.2 and 5). * * CPU builds compile with contraction on across statements (`-ffp-contract=fast` for - * GCC, `-ffp-contract=fast-honor-pragmas` for Clang and icpx; CMakeLists.txt), so the + * GCC, `-ffp-contract=fast-honor-pragmas` for Clang; CMakeLists.txt), so the * compiler may fuse any multiply and add into one rounding. A protected block keeps * the arithmetic as written, each product and sum rounded on its own: the stable phase * increments, the small-argument phi-functions, the ionised fraction (every column's @@ -28,7 +28,7 @@ * function's declaration, after a lambda's parameter list) and starts its body with * `SOLPROP_NATIVE_STRICT_BLOCK`: * - * - Clang and icpx: `#pragma clang fp contract(off)` at the start of the body. The + * - Clang: `#pragma clang fp contract(off)` at the start of the body. The * setting is carried by each instruction, so it survives inlining. * - GCC ignores `#pragma STDC FP_CONTRACT` and Clang's pragma, so the function is * compiled with `optimize("fp-contract=off")`. GCC then does not inline it into a diff --git a/native/include/solprop_native/gemm.hpp b/native/include/solprop_native/gemm.hpp index 6b33b870e..8684fb0ec 100644 --- a/native/include/solprop_native/gemm.hpp +++ b/native/include/solprop_native/gemm.hpp @@ -17,35 +17,6 @@ namespace solprop_native { -enum class GemmProvider { custom, kokkosblas_native, kokkosblas_tpl, hybrid }; - -[[nodiscard]] constexpr GemmProvider compiled_gemm_provider() noexcept { -#if defined(SOLPROP_NATIVE_GEMM_PROVIDER_CUSTOM) - return GemmProvider::custom; -#elif defined(SOLPROP_NATIVE_GEMM_PROVIDER_KOKKOSBLAS_NATIVE) - return GemmProvider::kokkosblas_native; -#elif defined(SOLPROP_NATIVE_GEMM_PROVIDER_KOKKOSBLAS_TPL) - return GemmProvider::kokkosblas_tpl; -#else - return GemmProvider::hybrid; -#endif -} - -[[nodiscard]] constexpr std::string_view -gemm_provider_name(const GemmProvider provider) noexcept { - switch (provider) { - case GemmProvider::custom: - return "custom"; - case GemmProvider::kokkosblas_native: - return "kokkosblas-native"; - case GemmProvider::kokkosblas_tpl: - return "kokkosblas-tpl"; - case GemmProvider::hybrid: - return "hybrid"; - } - return "unknown"; -} - // A host build without a vendor BLAS runs KokkosBlas's native real GEMM arithmetic // through `native_order_gemm`: the same result to // the bit, several times faster on the engine's skinny-by-wide transforms. @@ -205,7 +176,7 @@ void native_order_gemm(const ExecutionSpace& execution_space, const AView& a, // at 316 GFLOP/s, about 54 % of the FP64 peak, at any row pitch. Such a build forms // the engine's real GEMMs with `device_dense_gemm`. inline constexpr bool device_dense_gemm_enabled = -#if defined(KOKKOS_ARCH_AMPERE86) +#ifdef KOKKOS_ARCH_AMPERE86 true; #else false; @@ -360,9 +331,9 @@ void selected_gemm(const ExecutionSpace& execution_space, const AView& a, const BView& b, const CView& c, const typename CView::non_const_value_type beta, const bool tpl_eligible) { - const auto provider = compiled_gemm_provider(); - if (provider == GemmProvider::custom || - (provider == GemmProvider::hybrid && !tpl_eligible)) { + // The hybrid provider: the engine's GEMM for an eligible transform, the custom + // kernel for the rest. + if (!tpl_eligible) { custom_gemm(execution_space, a, b, c, beta); return; } diff --git a/native/include/solprop_native/general_modal.hpp b/native/include/solprop_native/general_modal.hpp index e0ccd8ee6..afa151fcb 100644 --- a/native/include/solprop_native/general_modal.hpp +++ b/native/include/solprop_native/general_modal.hpp @@ -230,10 +230,10 @@ class GeneralModalSource { CpuExecutionPool kernel_pool_, fft_pool_, raman_pool_, raman_tail_pool_; ExecutionSpace kernels_, transforms_, delayed_transforms_; ExecutionSpace response_, tail_response_, dense_, tail_dense_, maps_; - const ExecutionSpace& response_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& response_for(std::size_t count) const { return count == chunk_ ? response_ : tail_response_; } - const ExecutionSpace& dense_for(std::size_t count) const { + [[nodiscard]] const ExecutionSpace& dense_for(std::size_t count) const { return count == chunk_ ? dense_ : tail_dense_; } ComplexView synthesis_, projection_, frequency_factor_, delayed_spectrum_; diff --git a/native/include/solprop_native/graph.hpp b/native/include/solprop_native/graph.hpp index 0b4752696..3e9b1e040 100644 --- a/native/include/solprop_native/graph.hpp +++ b/native/include/solprop_native/graph.hpp @@ -729,8 +729,9 @@ namespace detail { /** Whether a view of `Memory` lives where `Space` runs and not on the host. */ template inline constexpr bool device_resident = - Kokkos::SpaceAccessibility::accessible && - !Kokkos::SpaceAccessibility::accessible; + static_cast(Kokkos::SpaceAccessibility::accessible) && + !static_cast( + Kokkos::SpaceAccessibility::accessible); /** A copy or fill between device buffers, whose kernel (if Kokkos needs one) passes its views as its argument. */ diff --git a/native/include/solprop_native/lagrange_basis.hpp b/native/include/solprop_native/lagrange_basis.hpp index bf11a26b8..67b73bb7e 100644 --- a/native/include/solprop_native/lagrange_basis.hpp +++ b/native/include/solprop_native/lagrange_basis.hpp @@ -164,10 +164,12 @@ inline void trial_tables(const double* nodes, const std::size_t order, for (std::size_t t = 0; t + 1 < width; ++t) spans[t] = 1.0 - nodes[t]; basis_rows(nodes, order, 1.0, true, scratch, carried); + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide basis_rows(through, order + 1, 1.0, true, scratch, endpoint); for (std::size_t index = 0; index < n_orders; ++index) { const std::size_t scored = orders[index]; double* const row = defects + index * width; + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide basis_rows(through, scored + 1, 1.0, true, scratch, row); for (std::size_t t = scored + 1; t < width; ++t) row[t] = 0.0; diff --git a/native/include/solprop_native/launch.hpp b/native/include/solprop_native/launch.hpp index 70f15c7ca..7d2ae3ab4 100644 --- a/native/include/solprop_native/launch.hpp +++ b/native/include/solprop_native/launch.hpp @@ -4,10 +4,10 @@ #pragma once -#include - #include "solprop_native/graph.hpp" +#include + #include #include #include @@ -69,7 +69,7 @@ void parallel_for(const Label& label, const Policy& policy, const Functor& funct left three of the device's compute units to do the work. */ template [[nodiscard]] auto reduce_policy_for(const Policy& policy) { -#if defined(KOKKOS_ENABLE_HIP) +#ifdef KOKKOS_ENABLE_HIP if constexpr (Kokkos::is_execution_policy_v) { using Space = typename Policy::execution_space; if constexpr (std::is_same_v> && diff --git a/native/include/solprop_native/low_storage.hpp b/native/include/solprop_native/low_storage.hpp index c661abe3f..7787dff57 100644 --- a/native/include/solprop_native/low_storage.hpp +++ b/native/include/solprop_native/low_storage.hpp @@ -4,15 +4,15 @@ #pragma once -#include -#include -#include - #include "solprop_native/core.hpp" #include "solprop_native/engines.hpp" #include "solprop_native/rk_march.hpp" #include "solprop_native/stage_tables.hpp" +#include +#include +#include + /* * The low-storage RK4IP replay in the native session (solprop SEP 24, SEP 47 section * 6.3): the engine that takes one in-place step of the march from its stage scalars, diff --git a/native/include/solprop_native/plasma.hpp b/native/include/solprop_native/plasma.hpp index ef707d1aa..6b501f191 100644 --- a/native/include/solprop_native/plasma.hpp +++ b/native/include/solprop_native/plasma.hpp @@ -216,24 +216,42 @@ KOKKOS_INLINE_FUNCTION void band_filter_interior(const double* row, double* out, const int low, const int high, const double* taps, const int half) { switch (half) { - case 0: - return band_filter_interior<0>(row, out, low, high, taps); - case 1: - return band_filter_interior<1>(row, out, low, high, taps); - case 2: - return band_filter_interior<2>(row, out, low, high, taps); - case 3: - return band_filter_interior<3>(row, out, low, high, taps); - case 4: - return band_filter_interior<4>(row, out, low, high, taps); - case 5: - return band_filter_interior<5>(row, out, low, high, taps); - case 6: - return band_filter_interior<6>(row, out, low, high, taps); - case 7: - return band_filter_interior<7>(row, out, low, high, taps); - default: - return band_filter_interior(row, out, low, high, taps); + case 0: { + band_filter_interior<0>(row, out, low, high, taps); + return; + } + case 1: { + band_filter_interior<1>(row, out, low, high, taps); + return; + } + case 2: { + band_filter_interior<2>(row, out, low, high, taps); + return; + } + case 3: { + band_filter_interior<3>(row, out, low, high, taps); + return; + } + case 4: { + band_filter_interior<4>(row, out, low, high, taps); + return; + } + case 5: { + band_filter_interior<5>(row, out, low, high, taps); + return; + } + case 6: { + band_filter_interior<6>(row, out, low, high, taps); + return; + } + case 7: { + band_filter_interior<7>(row, out, low, high, taps); + return; + } + default: { + band_filter_interior(row, out, low, high, taps); + return; + } } } @@ -320,9 +338,9 @@ KOKKOS_INLINE_FUNCTION BandWindow band_support(const int live_first, const int live_last, const int samples, const int half) { if (live_first > live_last) - return {samples, -1}; - return {live_first > half ? live_first - half : 0, - live_last + half < samples - 1 ? live_last + half : samples - 1}; + return {.first = samples, .last = -1}; + return {.first = live_first > half ? live_first - half : 0, + .last = live_last + half < samples - 1 ? live_last + half : samples - 1}; } /** diff --git a/native/include/solprop_native/raman_fft.hpp b/native/include/solprop_native/raman_fft.hpp index 336d7ad0c..515c28468 100644 --- a/native/include/solprop_native/raman_fft.hpp +++ b/native/include/solprop_native/raman_fft.hpp @@ -19,7 +19,7 @@ namespace solprop_native { -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE // The host adapter owns only vendor setup data. All field storage remains in the // workspace's Kokkos Views and is included in its existing allocation ledger. class AccelerateRamanPlan { @@ -30,6 +30,8 @@ class AccelerateRamanPlan { ~AccelerateRamanPlan(); AccelerateRamanPlan(const AccelerateRamanPlan&) = delete; AccelerateRamanPlan& operator=(const AccelerateRamanPlan&) = delete; + AccelerateRamanPlan(AccelerateRamanPlan&&) = delete; + AccelerateRamanPlan& operator=(AccelerateRamanPlan&&) = delete; void execute_task(std::size_t task) const; [[nodiscard]] int tasks() const noexcept; @@ -49,7 +51,7 @@ template class RamanFftPlan { #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) const bool contiguous = (axis == -1 || axis == 1) && input.stride(1) == 1 && output.stride(1) == 1; -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE constexpr bool forward = std::is_same_v; const auto samples = forward ? input.extent(1) : output.extent(1); @@ -91,7 +93,7 @@ template class RamanFftPlan { void execute(const Input& input, const Output& output, const KokkosFFT::Normalization normalization) const { -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE if (accelerate_) { if (normalization != KokkosFFT::Normalization::none) throw std::invalid_argument("Raman vDSP requires explicit normalization"); @@ -114,7 +116,7 @@ template class RamanFftPlan { } [[nodiscard]] const char* provider() const noexcept { -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE if (accelerate_) return "Accelerate_vDSP"; #endif @@ -122,7 +124,7 @@ template class RamanFftPlan { } [[nodiscard]] int tasks(const int fallback_threads) const noexcept { -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE if (accelerate_) return accelerate_->tasks(); #endif @@ -133,7 +135,7 @@ template class RamanFftPlan { using Fallback = KokkosFFT::Plan; Execution execution_; std::unique_ptr fallback_; -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE std::unique_ptr accelerate_; #endif }; diff --git a/native/include/solprop_native/rk_march.hpp b/native/include/solprop_native/rk_march.hpp index 4c35dc2ba..6d8646915 100644 --- a/native/include/solprop_native/rk_march.hpp +++ b/native/include/solprop_native/rk_march.hpp @@ -4,14 +4,14 @@ #pragma once +#include "solprop_native/adams_march.hpp" + #include #include #include #include #include -#include "solprop_native/adams_march.hpp" - /* * The Runge-Kutta march (solprop SEP 45): a trial's stage positions and its scaled * error norm (phase 2), and the adaptive march itself, its controller and its loop diff --git a/native/include/solprop_native/session.hpp b/native/include/solprop_native/session.hpp index babe8fdf4..34632b63a 100644 --- a/native/include/solprop_native/session.hpp +++ b/native/include/solprop_native/session.hpp @@ -4,6 +4,13 @@ #pragma once +#include "solprop_native/adams_march.hpp" +#include "solprop_native/dense.hpp" +#include "solprop_native/rk_march.hpp" +#include "solprop_native/statistics_plan.hpp" +#include "solprop_native/transverse.hpp" +#include "solprop_native/watch.hpp" + #include #include #include @@ -20,13 +27,6 @@ #include #include -#include "solprop_native/adams_march.hpp" -#include "solprop_native/dense.hpp" -#include "solprop_native/rk_march.hpp" -#include "solprop_native/statistics_plan.hpp" -#include "solprop_native/transverse.hpp" -#include "solprop_native/watch.hpp" - /* * The native execution session (solprop SEP 47 sections 5 and 6.1): the one mutable * owner of a trajectory. It holds the march's policy, state and history, the plane diff --git a/native/include/solprop_native/stage_tables.hpp b/native/include/solprop_native/stage_tables.hpp index 263f44a39..4977c8c72 100644 --- a/native/include/solprop_native/stage_tables.hpp +++ b/native/include/solprop_native/stage_tables.hpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -60,8 +61,10 @@ struct Table { " is outside a stage table's span"); } - /** The table's single value at `z`. */ + /** The table's single value at `z`. The scalar tables are checked to hold one value + each when they are loaded (`require_scalar_tables`). */ [[nodiscard]] double value(const double z) const { + assert(n_values <= 1); double out = 0.0; at(z, &out); return out; @@ -159,7 +162,7 @@ struct Sections { private: [[nodiscard]] std::size_t clamped(const double key) const { // `numpy.searchsorted(edges, key, side="right") - 1`, clamped to the sections. - const auto above = std::upper_bound(edges.begin(), edges.end(), key); + const auto above = std::ranges::upper_bound(edges, key); const auto raw = static_cast(above - edges.begin()) - 1; const auto last = static_cast(edges.size()) - 2; return static_cast(std::min(std::max(raw, std::ptrdiff_t{0}), last)); @@ -254,7 +257,7 @@ struct ZSlots { const std::vector& positions) { std::vector wanted; for (const double z : positions) - if (std::find(wanted.begin(), wanted.end(), z) == wanted.end()) + if (std::ranges::find(wanted, z) == wanted.end()) wanted.push_back(z); if (wanted.size() > slots) throw std::invalid_argument("a native call reads the z table at " + @@ -264,7 +267,7 @@ struct ZSlots { std::vector slot_of(wanted.size(), slots); std::vector taken(slots, false); for (std::size_t index = 0; index < wanted.size(); ++index) { - const auto at = std::find(held.begin(), held.end(), wanted[index]); + const auto at = std::ranges::find(held, wanted[index]); if (at != held.end()) { slot_of[index] = static_cast(at - held.begin()); taken[slot_of[index]] = true; @@ -296,8 +299,8 @@ struct ZSlots { std::vector out; out.reserve(positions.size()); for (const double z : positions) - out.push_back(slot_of[static_cast( - std::find(wanted.begin(), wanted.end(), z) - wanted.begin())]); + out.push_back(slot_of[static_cast(std::ranges::find(wanted, z) - + wanted.begin())]); return out; } @@ -314,7 +317,7 @@ struct ZSlots { private: void touch(const std::size_t slot) { - order.erase(std::find(order.begin(), order.end(), slot)); + order.erase(std::ranges::find(order, slot)); order.push_back(slot); } @@ -540,7 +543,7 @@ inline TrialScalars sample(const Tables& tables, Anchor& anchor, if (tables.flow.present()) return trial(tables, row(tables, anchor, end, end)); auto out = step; - std::fill(out.phi_delta.begin(), out.phi_delta.end(), 0.0); + std::ranges::fill(out.phi_delta, 0.0); return out; } @@ -554,6 +557,35 @@ inline TrialScalars sample(const Tables& tables, Anchor& anchor, * real position forms the imaginary part by these operations, so the two agree but for * the sign of a zero (solprop issue #255). */ +/** + * Refuse, by name, a scalar stage table that holds other than one value: the density, + * taper, Kerr density, Raman rates and densities, plasma densities and preionisation, + * dispersive densities and the chord's table, each read with `Table::value`. An absent + * optional table (no values) is not checked. + */ +inline void require_scalar_tables(const Tables& tables) { + const auto scalar = [](const Table& table, const std::string& name) { + if (table.present() && table.n_values != 1) + throw std::invalid_argument("the stage table " + name + " holds " + + std::to_string(table.n_values) + + " values where one is read"); + }; + const auto scalars = [&scalar](const std::vector& list, + const std::string& name) { + for (std::size_t index = 0; index < list.size(); ++index) + scalar(list[index], name + "[" + std::to_string(index) + "]"); + }; + scalar(tables.density, "density"); + scalar(tables.taper, "taper"); + scalar(tables.kerr_density, "kerr_density"); + scalar(tables.chord_table, "chord_table"); + scalars(tables.raman_rates, "raman_rates"); + scalars(tables.plasma_densities, "plasma_densities"); + scalars(tables.plasma_preionisation, "plasma_preionisation"); + scalars(tables.raman_densities, "raman_densities"); + scalars(tables.dispersive_densities, "dispersive_densities"); +} + inline double chord(const Tables& tables, const double z1, const double z2) { if (tables.chord == Chord::constant) return tables.chord_rate * (z2 - z1); @@ -623,7 +655,7 @@ inline ExponentialWeights weights(const double* nodes, const std::size_t n, const std::size_t width = highest + 1; std::array predictor{}; std::array corrector{}; - std::array factored{}; + std::array factored{}; std::array spans{}; std::array carried{}; std::array endpoint{}; diff --git a/native/include/solprop_native/vmath.hpp b/native/include/solprop_native/vmath.hpp index 6d5cc4aea..752774246 100644 --- a/native/include/solprop_native/vmath.hpp +++ b/native/include/solprop_native/vmath.hpp @@ -7,10 +7,10 @@ #include #include -#if defined(SOLPROP_NATIVE_VECTOR_MATH_SLEEF) -#include - +#ifdef SOLPROP_NATIVE_VECTOR_MATH_SLEEF #include + +#include #endif /* @@ -137,7 +137,7 @@ struct Lanes { #error "solprop-native: SLEEF vector maths needs an x86-64 or aarch64 target" #endif -#if defined(SOLPROP_NATIVE_VECTOR_MATH_SLEEF) +#ifdef SOLPROP_NATIVE_VECTOR_MATH_SLEEF inline constexpr bool vectorised = true; inline constexpr std::size_t width = Lanes::width; // The variant, for the build record. diff --git a/native/pyproject.toml b/native/pyproject.toml index a0a634f4c..cc632fa2e 100644 --- a/native/pyproject.toml +++ b/native/pyproject.toml @@ -114,7 +114,7 @@ repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} --requir # copying libomp.dll into the wheel under a mangled name. [tool.cibuildwheel.windows] before-all = "pwsh -NoProfile -ExecutionPolicy Bypass -File {package}\\ci\\build-deps-windows.ps1 -Prefix C:\\solprop-deps" -environment = { CMAKE_GENERATOR = "Ninja", CC = "clang", CXX = "clang++", CMAKE_ARGS = "-DSOLPROP_NATIVE_DISTRIBUTION_BLAS=OPENMP_OPENBLAS -DSOLPROP_NATIVE_OPENBLAS_ROOT=C:/solprop-deps/openblas -DFFTW_ROOT=C:/solprop-deps/fftw -DFFTW_DOUBLE_LIB=C:/solprop-deps/fftw/lib/fftw3.lib -DFFTW_FLOAT_LIB=C:/solprop-deps/fftw/lib/fftw3f.lib -DFFTW_DOUBLE_OPENMP_LIB=C:/solprop-deps/fftw/lib/fftw3_omp.lib -DFFTW_FLOAT_OPENMP_LIB=C:/solprop-deps/fftw/lib/fftw3f_omp.lib -DOpenMP_CXX_FLAGS=-fopenmp -DOpenMP_CXX_LIB_NAMES=libomp -DOpenMP_libomp_LIBRARY=C:/solprop-deps/libomp/lib/libomp.lib -DSOLPROP_NATIVE_BUNDLED_LICENCES=C:/solprop-deps/licences" } +environment = { CMAKE_GENERATOR = "Ninja", CC = "clang", CXX = "clang++", CMAKE_ARGS = "-DSOLPROP_NATIVE_DISTRIBUTION_BLAS=OPENMP_OPENBLAS -DSOLPROP_NATIVE_OPENBLAS_ROOT=C:/solprop-deps/openblas -DFFTW_ROOT=C:/solprop-deps/fftw -DFFTW_DOUBLE_LIB=C:/solprop-deps/fftw/lib/fftw3.lib -DFFTW_DOUBLE_OPENMP_LIB=C:/solprop-deps/fftw/lib/fftw3_omp.lib -DOpenMP_CXX_FLAGS=-fopenmp -DOpenMP_CXX_LIB_NAMES=libomp -DOpenMP_libomp_LIBRARY=C:/solprop-deps/libomp/lib/libomp.lib -DSOLPROP_NATIVE_BUNDLED_LICENCES=C:/solprop-deps/licences" } repair-wheel-command = "delvewheel repair --add-path C:\\solprop-deps\\libomp\\bin -w {dest_dir} -v {wheel}" # The x86-64 wheels hold two builds of the extension: x86-64-v3, and x86-64-v2 for older @@ -160,7 +160,7 @@ select = ["B", "E", "F", "I", "PERF", "PT", "RUF", "UP", "W"] "tests/**" = ["S101"] [tool.pyright] -include = ["src", "tests", "benchmarks"] +include = ["src", "tests"] exclude = ["build", "dist", "dist-*", ".venv"] extraPaths = ["src", "../src"] pythonVersion = "3.14" diff --git a/native/scripts/check.sh b/native/scripts/check.sh index 73da7067a..1361f8975 100755 --- a/native/scripts/check.sh +++ b/native/scripts/check.sh @@ -13,65 +13,16 @@ set -euo pipefail repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) workspace_root=$(cd "$repo_root/.." && pwd) cd "$repo_root" -raman_default=KOKKOSFFT -if [[ $(uname -s) == Darwin ]]; then - raman_default=ACCELERATE -fi -raman_provider=${SOLPROP_NATIVE_CHECK_RAMAN_FFT_PROVIDER:-$raman_default} +# shellcheck source=native/scripts/compiler.sh +source "$repo_root/scripts/compiler.sh" +raman_provider=${SOLPROP_NATIVE_CHECK_RAMAN_FFT_PROVIDER:-$(native_raman_provider)} export CMAKE_ARGS="${CMAKE_ARGS:-} -DSOLPROP_NATIVE_RAMAN_FFT_PROVIDER=$raman_provider" - -if [[ $(uname -s) == "Linux" ]]; then - # CXX when set (Linux CPU builds need GCC 14 or newer; Ubuntu 24.04's g++ is 13). - cxx_compiler=$(command -v "${CXX:-g++}") -elif [[ -x /opt/homebrew/opt/llvm/bin/clang++ ]]; then - cxx_compiler=/opt/homebrew/opt/llvm/bin/clang++ -elif [[ -x /usr/local/opt/llvm/bin/clang++ ]]; then - cxx_compiler=/usr/local/opt/llvm/bin/clang++ -else - cxx_compiler=$(command -v clang++) -fi -cpp_sources=( - benchmarks/cpu_timer.cpp - include/solprop_native/chebyshev.hpp - include/solprop_native/core.hpp - include/solprop_native/coupled_flow.hpp - include/solprop_native/dispersive.hpp - include/solprop_native/envelope_plasma.hpp - include/solprop_native/far_field.hpp - include/solprop_native/fft_plan.hpp - include/solprop_native/fp.hpp - include/solprop_native/general_modal.hpp - include/solprop_native/raman_fft.hpp - include/solprop_native/gemm.hpp - include/solprop_native/lagrange_basis.hpp - include/solprop_native/plasma.hpp - include/solprop_native/slab.hpp - include/solprop_native/z_table.hpp - src/bindings.cpp - src/coverage_labels.cpp - src/core.cpp - src/coupled_flow.cpp - src/dispersive.cpp - src/far_field.cpp - src/general.cpp - src/general_modal.cpp - src/helpers.cpp - src/helpers.hpp - src/raman_fft.cpp - src/benchmark_gemm.cpp - tests/native/bounds.hpp - tests/native/main.cpp - tests/native/test_aperture.cpp - tests/native/test_core.cpp - tests/native/test_coupled_flow.cpp - tests/native/test_dispersive.cpp - tests/native/test_envelope_plasma.cpp - tests/native/test_fft.cpp - tests/native/test_fp.cpp - tests/native/test_general_modal.cpp - tests/native/test_helpers.cpp - tests/native/test_z_table.cpp -) +cxx_compiler=$(native_cxx_compiler) +# Every tracked C++ file, so that none can drop out of the format check. +cpp_sources=() +while IFS= read -r -d '' cpp_source; do + cpp_sources+=("$cpp_source") +done < <(git ls-files -z -- '*.cpp' '*.hpp' '*.h') # The stages to run, all of them by default; CI runs them as parallel jobs. "lint" is # ruff, pyright and clang-format; "tidy" and "sanitize" build and run the native tests diff --git a/native/scripts/compiler.sh b/native/scripts/compiler.sh new file mode 100644 index 000000000..df0c9f3c1 --- /dev/null +++ b/native/scripts/compiler.sh @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +# Sourced by check.sh and tidy.sh: the C++ compiler and Raman FFT provider of the native +# gate's builds, chosen in one place. + +# The compiler: CXX on Linux (CPU builds need GCC 14 or newer; Ubuntu 24.04's g++ is 13), +# Homebrew's Clang on macOS (Apple silicon, then Intel), else the clang++ on PATH. +native_cxx_compiler() { + if [[ $(uname -s) == Linux ]]; then + command -v "${CXX:-g++}" + elif [[ -x /opt/homebrew/opt/llvm/bin/clang++ ]]; then + echo /opt/homebrew/opt/llvm/bin/clang++ + elif [[ -x /usr/local/opt/llvm/bin/clang++ ]]; then + echo /usr/local/opt/llvm/bin/clang++ + else + command -v clang++ + fi +} + +# The Raman FFT provider: Accelerate on macOS, Kokkos-FFT elsewhere. +native_raman_provider() { + if [[ $(uname -s) == Darwin ]]; then + echo ACCELERATE + else + echo KOKKOSFFT + fi +} diff --git a/native/scripts/tidy.sh b/native/scripts/tidy.sh new file mode 100755 index 000000000..ac4b99a92 --- /dev/null +++ b/native/scripts/tidy.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +# Run the pinned clang-tidy over some native C++ files (`poe tidy-native [FILE...]` from +# the repository root), with the compile commands of the cpu-tidy preset, which this +# configures when they are missing or stale, and the clang-tidy command it resolved. With no files it checks the C++ files under native/ +# that differ from main (committed, staged or not) and the untracked ones. Paths are +# relative to the repository root or absolute. The full check, every file as it +# compiles, is the gate's tidy stage. + +set -euo pipefail + +native=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +root=$(cd "$native/.." && pwd) +build="$native/build/presets/cpu-tidy" +cd "$root" + +files=() +if [[ $# -gt 0 ]]; then + files=("$@") +else + base=$(git merge-base "${SOLPROP_TIDY_BASE:-origin/main}" HEAD) + while IFS= read -r -d '' file; do + [[ -f $file ]] && files+=("$file") + done < <({ git diff -z --name-only "$base" -- 'native/*.cpp' 'native/*.hpp' 'native/*.h' + git ls-files -z --others --exclude-standard -- 'native/*.cpp' 'native/*.hpp' 'native/*.h'; } | + sort -zu) +fi +if [[ ${#files[@]} -eq 0 ]]; then + echo "tidy-native: no C++ files to check" + exit 0 +fi + +# shellcheck source=native/scripts/compiler.sh +source "$native/scripts/compiler.sh" +command_file="$build/clang_tidy_command.txt" +# Configure the cpu-tidy preset when its compile commands or clang-tidy command are +# missing or older than the build configuration. +if [[ ! -f "$build/compile_commands.json" || ! -f "$command_file" || + "$native/CMakeLists.txt" -nt "$build/compile_commands.json" || + "$native/CMakeLists.txt" -nt "$command_file" ]]; then + mkdir -p "$native/build" + log="$native/build/tidy-configure.log" + echo "tidy-native: configuring the cpu-tidy preset (log: $log)" + if ! (cd "$native" && cmake --preset cpu-tidy -DCMAKE_CXX_COMPILER="$(native_cxx_compiler)" \ + -DSOLPROP_NATIVE_RAMAN_FFT_PROVIDER="$(native_raman_provider)" \ + -DPython_EXECUTABLE="$root/.venv/bin/python") >"$log" 2>&1; then + tail -n 30 "$log" >&2 + exit 1 + fi +fi + +# SLEEF's header is generated by its own build, which a configure alone does not run. +if [[ -d "$build/_sleef" && ! -f "$build/_sleef/include/sleef.h" ]]; then + cmake --build "$build" --target solprop_native_sleef_build >"$native/build/tidy-sleef.log" 2>&1 +fi + +# The command the build runs: the clang-tidy CMake resolved, its configuration and +# header filter, and the parser arguments beyond the compile commands. +tidy_command=() +while IFS= read -r argument; do + [[ -n $argument ]] && tidy_command+=("$argument") +done < "$command_file" +pinned=$(sed -n 's/.*"clang-tidy==\([0-9.]*\).*/\1/p' "$root/pyproject.toml") +found=$("${tidy_command[0]}" --version | sed -n 's/.*LLVM version \([0-9.]*\).*/\1/p') +if [[ -z $pinned || $found != "$pinned" ]]; then + echo "tidy-native: ${tidy_command[0]} is clang-tidy ${found:-of unknown version}, not the" \ + "pinned ${pinned:-?}; run uv sync, then remove $build to reconfigure" >&2 + exit 1 +fi +# A header is checked with the commands of a source file near it (clang-tidy +# interpolates them), and through the sources that include it. +printf '%s\0' "${files[@]}" | + xargs -0 -n 1 -P "$(getconf _NPROCESSORS_ONLN)" "${tidy_command[@]}" --quiet -p "$build" diff --git a/native/src/adams_march.cpp b/native/src/adams_march.cpp index 2c9b0b7b2..a06152acb 100644 --- a/native/src/adams_march.cpp +++ b/native/src/adams_march.cpp @@ -4,6 +4,8 @@ #include "solprop_native/adams_march.hpp" +#include "solprop_native/core.hpp" + #include #include #include @@ -12,8 +14,6 @@ #include #include -#include "solprop_native/core.hpp" - // GCC does not take a file's `-ffp-contract=off` as an inlining barrier: under // link-time optimisation, with the library's symbols hidden, it would inline a function // of this file into a contracting caller and fuse it (solprop #135). The optimize @@ -48,14 +48,12 @@ double py_max(const double a, const double b) { return b > a ? b : a; } /** `bisect.bisect_left` over an increasing sequence. */ std::size_t bisect_left(const std::vector& values, const double x) { - return static_cast(std::lower_bound(values.begin(), values.end(), x) - - values.begin()); + return static_cast(std::ranges::lower_bound(values, x) - values.begin()); } /** `bisect.bisect_right` over an increasing sequence. */ std::size_t bisect_right(const std::vector& values, const double x) { - return static_cast(std::upper_bound(values.begin(), values.end(), x) - - values.begin()); + return static_cast(std::ranges::upper_bound(values, x) - values.begin()); } void refuse(const char* field, const double value) { @@ -67,8 +65,8 @@ bool increasing(const std::vector& values) { for (std::size_t index = 1; index < values.size(); ++index) if (!(values[index] > values[index - 1])) return false; - return std::all_of(values.begin(), values.end(), - [](const double value) { return std::isfinite(value); }); + return std::ranges::all_of(values, + [](const double value) { return std::isfinite(value); }); } } // namespace @@ -186,7 +184,7 @@ OrderChoice choose_order(const Policy& policy, const Stencil& stencil, const double* norms, const double dz, const bool quiet) { const std::size_t current = stencil.order; if (!policy.variable_order) - return {policy.order, current, norms[0]}; + return {.stored = policy.order, .control = current, .norm = norms[0]}; std::array weights{}; for (std::size_t index = 0; index < stencil.n_orders; ++index) weights[index] = @@ -197,8 +195,10 @@ OrderChoice choose_order(const Policy& policy, const Stencil& stencil, if (weights[index] > weights[best]) best = index; if (stencil.orders[best] != current && weights[best] > policy.hysteresis * weights[0]) - return {stencil.orders[best], stencil.orders[best], norms[best]}; - return {current, current, norms[0]}; + return {.stored = stencil.orders[best], + .control = stencil.orders[best], + .norm = norms[best]}; + return {.stored = current, .control = current, .norm = norms[0]}; } Rejection rejection(const Policy& policy, const double norm, const std::size_t order, @@ -208,7 +208,7 @@ Rejection rejection(const Policy& policy, const double norm, const std::size_t o const double scale = !std::isfinite(norm) ? policy.min_factor : factor(policy, norm, order, policy.rejection_growth); - return {scale, restart ? 0 : count, restart}; + return {.factor = scale, .rejections = restart ? 0 : count, .restart = restart}; } double carrier_capped(const Policy& policy, const double z, const double proposal, @@ -287,15 +287,16 @@ Landing landing(const Policy& policy, const double z, const double dz, return false; return static_cast(candidate.target) > static_cast(held.target); }; - Bound cap{dz, Target::none, 0.0}; - for (const Bound& other : {Bound{policy.dz_max, Target::max_step, 0.0}, - Bound{width, Target::interval_width, 0.0}}) + Bound cap{.dz = dz, .target = Target::none, .at = 0.0}; + for (const Bound& other : + {Bound{.dz = policy.dz_max, .target = Target::max_step, .at = 0.0}, + Bound{.dz = width, .target = Target::interval_width, .at = 0.0}}) if (better(other, cap)) cap = other; bool any = false; Bound land{}; const auto consider = [&](const Target target, const double position) { - const Bound candidate{position - z, target, position}; + const Bound candidate{.dz = position - z, .target = target, .at = position}; if (!any || better(candidate, land)) { land = candidate; any = true; @@ -310,10 +311,10 @@ Landing landing(const Policy& policy, const double z, const double dz, if (!(z_end < from)) consider(Target::end, z_end); if (land.dz <= cap.dz) - return {land.dz, land.target, true, land.at}; + return {.dz = land.dz, .target = land.target, .landed = true, .z_target = land.at}; if (land.dz <= py_min(policy.stretch * cap.dz, policy.dz_max)) - return {land.dz, land.target, true, land.at}; - return {cap.dz, cap.target, false, 0.0}; + return {.dz = land.dz, .target = land.target, .landed = true, .z_target = land.at}; + return {.dz = cap.dz, .target = cap.target, .landed = false, .z_target = 0.0}; } bool next_landing(const Policy& policy, const double z, const double target, @@ -340,7 +341,7 @@ Approach approach(const Policy& policy, const double z, const Landing& stop, if (next_landing(policy, z, target, ahead) && stop.dz < ahead - z && ahead - z < 2.0 * stop.dz) { const double half = 0.5 * (ahead - z); - return {half, z + half, false}; + return {.dz = half, .end = z + half, .kink = false}; } } const double end = stop.landed ? stop.z_target : z + stop.dz; @@ -353,7 +354,8 @@ Approach approach(const Policy& policy, const double z, const Landing& stop, on_kink = true; break; } - return {stop.dz, end, stop.target == Target::hard_kink || on_kink}; + return { + .dz = stop.dz, .end = end, .kink = stop.target == Target::hard_kink || on_kink}; } double proposal(const Policy& policy, const double dz, const double factor) { @@ -361,8 +363,9 @@ double proposal(const Policy& policy, const double dz, const double factor) { } bool replay_kink(const Policy& policy, const double end, const double slack) { - return std::any_of(policy.hard_kinks.begin(), policy.hard_kinks.end(), - [&](const double kink) { return std::fabs(kink - end) <= slack; }); + return std::ranges::any_of(policy.hard_kinks, [&](const double kink) { + return std::fabs(kink - end) <= slack; + }); } bool reached(const double position, const double target, const int ulps) { @@ -400,7 +403,7 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou const double z_end = policy.interpolate ? policy.z_end : target; std::uint64_t accepted = 0; std::array norms{}; - const auto elapsed = [&]() { + const auto elapsed = [&] { return std::chrono::duration(Clock::now() - started).count(); }; const auto record = [&](const double end, const double dz, const double norm, @@ -413,7 +416,7 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou // A replay runs while the schedule has steps and its target is not within the // slack (`HostAdamsMarch._replay_to`); an adaptive march until the target is reached. const std::size_t n_steps = schedule.steps.size(); - const auto running = [&]() { + const auto running = [&] { if (bounds.replay) return state.cursor < n_steps && state.z < target - bounds.slack; return !reached(state.z, target, policy.landing_ulps); @@ -428,8 +431,8 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou const auto take_check = [&]() -> watch::Verdict { if (bounds.owner != nullptr) return watch::owned_check( - *bounds.owner, [&engine]() { return engine.reading(); }, state.z, - bounds.replay, schedule.steps, state.cursor); + *bounds.owner, [&engine] { return engine.reading(); }, state.z, bounds.replay, + schedule.steps, state.cursor); const auto& rules = bounds.watch_policy; if (!watch::due(rules, state.watch, state.z)) { watch::skip(state.watch); @@ -474,7 +477,7 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou fail(result, Stage::startup, error); return result; } - state.history = {Entry{state.z, Identity::accepted, 0}}; + state.history = {Entry{.z = state.z, .identity = Identity::accepted, .row = 0}}; state.order = policy.variable_order ? 1 : policy.order; } Approach step; @@ -554,8 +557,8 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou state.history.end()); std::size_t free = 0; for (;; ++free) - if (std::none_of(kept.begin(), kept.end(), - [free](const Entry& entry) { return entry.row == free; })) + if (std::ranges::none_of( + kept, [free](const Entry& entry) { return entry.row == free; })) break; try { engine.accept(step.dz, free, trial, kept); @@ -563,7 +566,8 @@ Result run(const Policy& policy, State& state, Engine& engine, const Bounds& bou fail(result, Stage::accept, error); return result; } - kept.push_back(Entry{state.z + step.dz, Identity::predicted, free}); + kept.push_back( + Entry{.z = state.z + step.dz, .identity = Identity::predicted, .row = free}); state.history = std::move(kept); ++state.ledger.accepted_steps; state.rejections = 0; diff --git a/native/src/benchmark_gemm.cpp b/native/src/benchmark_gemm.cpp deleted file mode 100644 index c55501540..000000000 --- a/native/src/benchmark_gemm.cpp +++ /dev/null @@ -1,296 +0,0 @@ -// SPDX-FileCopyrightText: 2026 John C. Travers and contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "solprop_native/gemm.hpp" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - -using ExecutionSpace = Kokkos::DefaultExecutionSpace; -using Layout = Kokkos::LayoutRight; -using Range2 = Kokkos::MDRangePolicy< - ExecutionSpace, Kokkos::Rank<2, Kokkos::Iterate::Right, Kokkos::Iterate::Right>>; -using MemorySpace = ExecutionSpace::memory_space; -using Complex = Kokkos::complex; -using RealView = Kokkos::View; -using ComplexView = Kokkos::View; - -struct Shape { - std::size_t output_rows{}; - std::size_t inner{}; - std::size_t samples{}; -}; - -template KOKKOS_INLINE_FUNCTION Value initial_value(const double value) { - if constexpr (std::is_same_v) { - return Complex(value, -0.5 * value); - } else { - return value; - } -} - -template -void initialise(const ExecutionSpace& execution_space, const View& view, - const double seed) { - Kokkos::parallel_for( - "solprop_native.benchmark.initialise", - Range2(execution_space, {0, 0}, {view.extent(0), view.extent(1)}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t column) { - using Value = View::non_const_value_type; - const double value = seed + 0.01 * static_cast(row) + - 0.001 * static_cast(column); - view(row, column) = initial_value(value); - }); -} - -template -double measure(const ExecutionSpace& execution_space, const std::size_t repetitions, - Function&& function) { - for (std::size_t index = 0; index < 3; ++index) - function(); - execution_space.fence("solprop_native GEMM benchmark warmup"); - const auto started = std::chrono::steady_clock::now(); - for (std::size_t index = 0; index < repetitions; ++index) - function(); - execution_space.fence("solprop_native GEMM benchmark boundary"); - return std::chrono::duration(std::chrono::steady_clock::now() - - started) - .count() / - static_cast(repetitions); -} - -template double maximum_difference(const View& left, const View& right) { - double result = 0.0; - Kokkos::parallel_reduce( - "solprop_native.benchmark.compare", - Range2(ExecutionSpace{}, {0, 0}, {left.extent(0), left.extent(1)}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t column, double& update) { - update = - Kokkos::max(update, Kokkos::abs(left(row, column) - right(row, column))); - }, - Kokkos::Max(result)); - return result; -} - -template -void team_gemm(const ExecutionSpace& execution_space, const AView& a, const BView& b, - const CView& c, const typename CView::non_const_value_type beta, - const int team_size) { - using Value = CView::non_const_value_type; - const std::size_t inner = a.extent(1); - const std::size_t samples = c.extent(1); - Kokkos::parallel_for( - "solprop_native.benchmark.team_gemm", - Kokkos::TeamPolicy(execution_space, static_cast(c.extent(0)), - team_size), - KOKKOS_LAMBDA( - const typename Kokkos::TeamPolicy::member_type& team) { - const std::size_t row = static_cast(team.league_rank()); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, samples), - [&](const std::size_t column) { - Value value{}; - for (std::size_t index = 0; index < inner; ++index) - value += a(row, index) * b(index, column); - c(row, column) = value + beta * c(row, column); - }); - }); -} - -double maximum_stacked_difference(const RealView& fine, const RealView& coarse, - const RealView& stacked) { - double result = 0.0; - const auto rows = fine.extent(0); - Kokkos::parallel_reduce( - "solprop_native.benchmark.compare_stacked", - Range2(ExecutionSpace{}, {0, 0}, {2 * rows, fine.extent(1)}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t column, double& update) { - const double expected = - row < rows ? fine(row, column) : coarse(row - rows, column); - update = Kokkos::max(update, Kokkos::abs(expected - stacked(row, column))); - }, - Kokkos::Max(result)); - return result; -} - -std::size_t parsed(const char* value, const std::string_view name) { - const auto result = std::strtoull(value, nullptr, 10); - if (result == 0) { - std::cerr << name << " must be a positive integer\n"; - std::exit(2); - } - return static_cast(result); -} - -void run(const ExecutionSpace& execution_space, const std::string_view operation, - const Shape shape, const std::size_t repetitions, const double beta) { - RealView a("solprop_native.benchmark.a", shape.output_rows, shape.inner); - RealView b("solprop_native.benchmark.b", shape.inner, shape.samples); - RealView custom("solprop_native.benchmark.custom", shape.output_rows, shape.samples); - RealView blas("solprop_native.benchmark.blas", shape.output_rows, shape.samples); - initialise(execution_space, a, 0.25); - initialise(execution_space, b, -0.125); - Kokkos::deep_copy(execution_space, custom, 0.0); - Kokkos::deep_copy(execution_space, blas, 0.0); - solprop_native::custom_gemm(execution_space, a, b, custom, beta); - solprop_native::kokkosblas_gemm(execution_space, a, b, blas, beta); - execution_space.fence("solprop_native GEMM benchmark reference"); - const double difference = maximum_difference(custom, blas); - const double custom_us = measure(execution_space, repetitions, [&] { - solprop_native::custom_gemm(execution_space, a, b, custom, beta); - }); - const double blas_us = measure(execution_space, repetitions, [&] { - solprop_native::kokkosblas_gemm(execution_space, a, b, blas, beta); - }); - std::cout << operation << ",real," << shape.output_rows << ',' << shape.inner << ',' - << shape.samples << ',' << beta << ',' << repetitions << ',' - << std::setprecision(9) << custom_us << ',' << blas_us << ',' - << custom_us / blas_us << ',' << difference << '\n'; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const std::array team_sizes{64, 128, 256, 512}; -#else - const std::array team_sizes{1, 2, 4, 8}; -#endif - for (const int team_size : team_sizes) { - Kokkos::deep_copy(execution_space, custom, 0.0); - Kokkos::deep_copy(execution_space, blas, 0.0); - solprop_native::custom_gemm(execution_space, a, b, custom, beta); - team_gemm(execution_space, a, b, blas, beta, team_size); - execution_space.fence("solprop_native team GEMM benchmark reference"); - const double team_difference = maximum_difference(custom, blas); - const double team_us = measure(execution_space, repetitions, [&] { - team_gemm(execution_space, a, b, blas, beta, team_size); - }); - std::cout << operation << "-team" << team_size << ",real," << shape.output_rows - << ',' << shape.inner << ',' << shape.samples << ',' << beta << ',' - << repetitions << ',' << custom_us << ',' << team_us << ',' - << custom_us / team_us << ',' << team_difference << '\n'; - } - - ComplexView complex_a("solprop_native.benchmark.complex_a", shape.output_rows, - shape.inner); - ComplexView complex_b("solprop_native.benchmark.complex_b", shape.inner, - shape.samples); - ComplexView mixed("solprop_native.benchmark.mixed", shape.output_rows, shape.samples); - ComplexView complex_blas("solprop_native.benchmark.complex_blas", shape.output_rows, - shape.samples); - initialise(execution_space, complex_b, -0.125); - Kokkos::parallel_for( - "solprop_native.benchmark.promote_real_matrix", - Range2(execution_space, {0, 0}, {a.extent(0), a.extent(1)}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t column) { - complex_a(row, column) = Complex(a(row, column), 0.0); - }); - Kokkos::deep_copy(execution_space, mixed, Complex{}); - Kokkos::deep_copy(execution_space, complex_blas, Complex{}); - solprop_native::custom_gemm(execution_space, a, complex_b, mixed, Complex(beta)); - solprop_native::kokkosblas_gemm(execution_space, complex_a, complex_b, complex_blas, - Complex(beta)); - execution_space.fence("solprop_native complex GEMM benchmark reference"); - const double complex_difference = maximum_difference(mixed, complex_blas); - const double mixed_us = measure(execution_space, repetitions, [&] { - solprop_native::custom_gemm(execution_space, a, complex_b, mixed, Complex(beta)); - }); - const double complex_blas_us = measure(execution_space, repetitions, [&] { - solprop_native::kokkosblas_gemm(execution_space, complex_a, complex_b, complex_blas, - Complex(beta)); - }); - std::cout << operation << ",mixed-vs-complex," << shape.output_rows << ',' - << shape.inner << ',' << shape.samples << ',' << beta << ',' << repetitions - << ',' << mixed_us << ',' << complex_blas_us << ',' - << mixed_us / complex_blas_us << ',' << complex_difference << '\n'; -} - -void run_transform_pair(const ExecutionSpace& execution_space, const std::size_t nodes, - const std::size_t modes, const std::size_t samples, - const std::size_t repetitions) { - run(execution_space, "synthesis-overwrite", {nodes, modes, samples}, repetitions, - 0.0); - run(execution_space, "projection-overwrite", {modes, nodes, samples}, repetitions, - 0.0); - run(execution_space, "projection-accumulate", {modes, nodes, samples}, repetitions, - 1.0); - run(execution_space, "fine-coarse-stacked", {2 * modes, nodes, samples}, repetitions, - 0.0); - - RealView fine_matrix("solprop_native.benchmark.fine_matrix", modes, nodes); - RealView coarse_matrix("solprop_native.benchmark.coarse_matrix", modes, nodes); - RealView stacked_matrix("solprop_native.benchmark.stacked_matrix", 2 * modes, nodes); - RealView response("solprop_native.benchmark.response", nodes, samples); - RealView fine("solprop_native.benchmark.fine", modes, samples); - RealView coarse("solprop_native.benchmark.coarse", modes, samples); - RealView stacked("solprop_native.benchmark.stacked", 2 * modes, samples); - initialise(execution_space, fine_matrix, 0.25); - initialise(execution_space, coarse_matrix, -0.5); - initialise(execution_space, response, -0.125); - Kokkos::parallel_for( - "solprop_native.benchmark.stack_projection", - Range2(execution_space, {0, 0}, {2 * modes, nodes}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t column) { - stacked_matrix(row, column) = - row < modes ? fine_matrix(row, column) : coarse_matrix(row - modes, column); - }); - const auto separate = [&] { - solprop_native::kokkosblas_gemm(execution_space, fine_matrix, response, fine, 0.0); - solprop_native::kokkosblas_gemm(execution_space, coarse_matrix, response, coarse, - 0.0); - }; - const auto combined = [&] { - solprop_native::kokkosblas_gemm(execution_space, stacked_matrix, response, stacked, - 0.0); - }; - separate(); - combined(); - execution_space.fence("solprop_native stacked projection reference"); - const double difference = maximum_stacked_difference(fine, coarse, stacked); - const double separate_us = measure(execution_space, repetitions, separate); - const double stacked_us = measure(execution_space, repetitions, combined); - std::cout << "fine-coarse-dispatch,real," << 2 * modes << ',' << nodes << ',' - << samples << ",0," << repetitions << ',' << separate_us << ',' - << stacked_us << ',' << separate_us / stacked_us << ',' << difference - << '\n'; -} - -} // namespace - -int main(int argc, char** argv) { - Kokkos::ScopeGuard guard(argc, argv); - const ExecutionSpace execution_space; - std::cout << "operation,kind,rows,inner,samples,beta,repetitions,baseline_us," - "candidate_us,speedup,max_abs_diff\n"; - if (argc == 5) { - run_transform_pair(execution_space, parsed(argv[1], "nodes"), - parsed(argv[2], "modes"), parsed(argv[3], "samples"), - parsed(argv[4], "repetitions")); - return 0; - } - const std::vector modes{1, 2, 4, 6, 8}; - const std::vector rows{1, 8, 21, 33, 41, 64, 96}; - const std::vector samples{1024, 4096, 8192, 32768, 131072}; - for (const auto mode : modes) { - for (const auto row : rows) { - for (const auto count : samples) { - const std::size_t repetitions = - std::max(7, 1048576 / std::max(count, 1)); - run_transform_pair(execution_space, row, mode, count, repetitions); - } - } - } -} diff --git a/native/src/bindings.cpp b/native/src/bindings.cpp index 88c91bdff..a0c6ac649 100644 --- a/native/src/bindings.cpp +++ b/native/src/bindings.cpp @@ -20,13 +20,14 @@ #include +#include #include // `available_device_memory` calls the vendor runtime directly. Kokkos pulls // these headers in transitively today; declaring them here is what makes the // probe's own dependency explicit rather than a property of a Kokkos version. // The guards resolve only after Kokkos_Core.hpp has defined them. -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA #include #elif defined(KOKKOS_ENABLE_HIP) #include @@ -40,13 +41,12 @@ #include #include -#if defined(_WIN32) +#ifdef _WIN32 #include #else #include #endif -#include #include #include #include @@ -208,8 +208,9 @@ native::CpuExecution cpu_execution_from(const nb::dict& values) { try { for (std::size_t index = 0; index < native::cpu_operation_count; ++index) { const std::string name = cpu_operation_names[index]; - result.grain[index] = {nb::cast(field(name + "_threshold")), - nb::cast(field(name + "_full"))}; + result.grain[index] = {.threshold = + nb::cast(field(name + "_threshold")), + .full = nb::cast(field(name + "_full"))}; } result.response_full_samples = nb::cast(field("response_full_samples")); @@ -721,21 +722,21 @@ native::TrialScalars trial_from(const RealInput phi_delta, const RealInput phi_a const RealInput prefactor, const RealInput density, const RealInput taper, const RealInput preionisation, const RealInput raman_rates) { - return {real_vector(phi_delta), - real_vector(phi_at), - real_vector(prefactor), - real_vector(density), - real_vector(taper), - real_vector(preionisation), - real_vector(raman_rates), - {}, - {}, - {}, - {}, - 0, - {}, - {}, - {}}; + return {.phi_delta = real_vector(phi_delta), + .phi_at = real_vector(phi_at), + .prefactor = real_vector(prefactor), + .density = real_vector(density), + .taper = real_vector(taper), + .preionisation = real_vector(preionisation), + .raman_rates = real_vector(raman_rates), + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = 0, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}}; } // A trial's scalars with its material rows, as the Adams entry points take them. @@ -874,6 +875,7 @@ native::stage::Tables stage_tables_from(const nb::dict values) { (tables.flow.present() && tables.flow.edges.size() < 2) || (tables.transform.present() && tables.transform.edges.size() < 2)) throw std::invalid_argument("the stage tables are inconsistent"); + native::stage::require_scalar_tables(tables); return tables; } @@ -956,8 +958,8 @@ ComplexOutput complex_output(const std::vector& values, nb::capsule owner(pointer, [](void* value) noexcept { const std::unique_ptr held(static_cast(value)); }); - storage.release(); - return ComplexOutput(pointer->data(), shape.size(), shape.data(), owner); + static_cast(storage.release()); // the capsule owns it now + return {pointer->data(), shape.size(), shape.data(), owner}; } /** A complex array owning `storage`. */ @@ -967,8 +969,8 @@ ComplexOutput owned_complex(std::unique_ptr storage, nb::capsule owner(pointer, [](void* value) noexcept { const std::unique_ptr held(static_cast(value)); }); - storage.release(); - return ComplexOutput(pointer->data(), shape.size(), shape.data(), owner); + static_cast(storage.release()); // the capsule owns it now + return {pointer->data(), shape.size(), shape.data(), owner}; } /** A complex array owning `storage`, uninitialised doubles a native copy filled. */ @@ -978,8 +980,8 @@ ComplexOutput complex_buffer_output(native::crossing::HostBuffer storage, nb::capsule owner(pointer, [](void* value) noexcept { const native::crossing::HostBuffer held(static_cast(value)); }); - return ComplexOutput(reinterpret_cast*>(pointer), shape.size(), - shape.data(), owner); + return {reinterpret_cast*>(pointer), shape.size(), shape.data(), + owner}; } RealOutput real_output(std::vector values, const std::size_t rows) { @@ -990,9 +992,9 @@ RealOutput real_output(std::vector values, const std::size_t rows) { nb::capsule owner(pointer, [](void* value) noexcept { const std::unique_ptr held(static_cast(value)); }); - storage.release(); + static_cast(storage.release()); // the capsule owns it now const std::array shape{rows, pointer->size() / rows}; - return RealOutput(pointer->data(), shape.size(), shape.data(), owner); + return {pointer->data(), shape.size(), shape.data(), owner}; } void initialize_kokkos(const int threads, const int kernels, const int fft, @@ -1012,8 +1014,11 @@ void initialize_kokkos(const int threads, const int kernels, const int fft, throw std::runtime_error( "Kokkos was already initialized with another thread count"); } - native::configure_thread_policy( - {kernels, fft, raman_fft, raman_tail_fft, band_tiles}); + native::configure_thread_policy({.kernels = kernels, + .fft = fft, + .raman_fft = raman_fft, + .raman_tail_fft = raman_tail_fft, + .band_tiles = band_tiles}); } // ------------------------------------------------------------- the native Adams march @@ -1400,10 +1405,11 @@ march::State march_state_from(const nb::dict values) { state.cursor = nb::cast(values["cursor"]); for (const auto entry : nb::cast(values["history"])) { const auto parts = nb::cast(entry); - state.history.push_back({nb::cast(parts[0]), - nb::cast(parts[1]) ? march::Identity::predicted - : march::Identity::accepted, - nb::cast(parts[2])}); + state.history.push_back({.z = nb::cast(parts[0]), + .identity = nb::cast(parts[1]) + ? march::Identity::predicted + : march::Identity::accepted, + .row = nb::cast(parts[2])}); } const auto ledger = nb::cast(values["ledger"]); state.ledger.startup = nb::cast(ledger["startup"]); @@ -1530,7 +1536,8 @@ class ScriptedMarch { ScriptedMarch(const nb::dict policy, const RealInput norms, const double carrier_rate, const RealInput steps, const RealInput edges) : policy_(march_policy_from(policy)), norms_(real_vector(norms)), - carrier_rate_(carrier_rate), schedule_{real_vector(steps), real_vector(edges)} { + carrier_rate_(carrier_rate), + schedule_{.steps = real_vector(steps), .edges = real_vector(edges)} { if (norms_.size() % 3 != 0) throw std::invalid_argument("a script of norms is (attempts, 3)"); } @@ -1546,7 +1553,7 @@ class ScriptedMarch { } // What the march asked, one dict an entry: a startup's position, or a trial's // position, step, nodes, rows and orders with the row its accept freed. - nb::list log() const { + [[nodiscard]] nb::list log() const { nb::list out; for (const auto& entry : log_) { nb::dict event; @@ -1582,7 +1589,7 @@ class ScriptedRkMarch { ScriptedRkMarch(const nb::dict policy, const RealInput norms, const double h0, const RealInput steps, const RealInput edges) : policy_(rk_policy_from(policy)), norms_(real_vector(norms)), h0_(h0), - schedule_{real_vector(steps), real_vector(edges)} {} + schedule_{.steps = real_vector(steps), .edges = real_vector(edges)} {} nb::dict run(const nb::dict state_values, const nb::dict bounds_values) { auto state = rk_state_from(state_values); const auto bounds = march_bounds_from(bounds_values); @@ -1591,7 +1598,7 @@ class ScriptedRkMarch { return rk_result_to(result, state); } // Each trial's position and step, one dict a trial. - nb::list log() const { + [[nodiscard]] nb::list log() const { nb::list out; for (const auto& entry : log_) { nb::dict event; @@ -1626,13 +1633,15 @@ class BoundState { ~BoundState() { close(); const std::scoped_lock lock(runtime_state.mutex); - const auto found = - std::find(runtime_state.states.begin(), runtime_state.states.end(), this); + const auto found = std::ranges::find(runtime_state.states, this); if (found != runtime_state.states.end()) runtime_state.states.erase(found); } BoundState(const BoundState&) = delete; BoundState& operator=(const BoundState&) = delete; + // The runtime's list of live states holds this address. + BoundState(BoundState&&) = delete; + BoundState& operator=(BoundState&&) = delete; void set_initial(const std::span field, const std::span noise) { @@ -1680,8 +1689,11 @@ class BoundState { const std::size_t sections = row_sum.size(); if (sections == 0 || condition_bound.size() % sections != 0) throw std::invalid_argument("the flow guard's certificate is one row a section"); - flow_guard_ = FlowGuard{std::move(row_sum), std::move(condition_bound), - taylor_scale, max_squarings, allowance}; + flow_guard_ = FlowGuard{.row_sum = std::move(row_sum), + .condition_bound = std::move(condition_bound), + .taylor_scale = taylor_scale, + .max_squarings = max_squarings, + .allowance = allowance}; flow_key_.reset(); } void guarded_flow_actions(const std::size_t section, const double dz, @@ -1756,7 +1768,7 @@ class BoundState { rk_nodes_ = std::move(nodes); rk_rtol_ = rtol; rk_atol_ = atol; - rk_schedule_ = {std::move(steps), std::move(edges)}; + rk_schedule_ = {.steps = std::move(steps), .edges = std::move(edges)}; } nb::dict rk_march_run(const nb::dict state_values, const nb::dict bounds_values) { require_unleased(); @@ -1774,7 +1786,7 @@ class BoundState { native::rk::Result result; { const nb::gil_scoped_release release; - result = native::rk::run(*rk_policy_, state, engine, bounds, rk_schedule_); + result = native::rk::run(rk_policy_.value(), state, engine, bounds, rk_schedule_); } return rk_result_to(result, state); } @@ -1823,7 +1835,8 @@ class BoundState { open_state(); if (flows > 2) throw std::invalid_argument("the low-storage march holds at most two flows"); - low_storage_ = native::rk::low_storage::Config{flows, separable, origin}; + low_storage_ = native::rk::low_storage::Config{ + .flows = flows, .separable = separable, .origin = origin}; } // solprop SEP 45 phase 4: a weighted controller's weights, and its norm from a // trial's partials, for both Runge-Kutta loops. @@ -1916,7 +1929,7 @@ class BoundState { general_state_->field_to_host(destination); } /** The state's element count, `columns * n_omega`. */ - std::size_t elements() const noexcept { return elements_; } + [[nodiscard]] std::size_t elements() const noexcept { return elements_; } std::vector adams_source(const std::vector& input, const native::TrialScalars& scalars) { require_open(); @@ -1973,9 +1986,9 @@ class BoundState { const auto row = native::stage::row(*stage_tables_, stage_anchor_, z0, z1); auto step = native::stage::trial(*stage_tables_, row); auto sample = native::stage::sample(*stage_tables_, stage_anchor_, step, z1); - return {std::move(step), std::move(sample)}; + return {.step = std::move(step), .sample = std::move(sample)}; } - std::vector stage_rate(const double z) const { + [[nodiscard]] std::vector stage_rate(const double z) const { require_open(); if (!stage_tables_) throw std::logic_error("no stage tables were uploaded"); @@ -2002,13 +2015,14 @@ class BoundState { auto state = march_state_from(state_values); const auto bounds = march_bounds_from(bounds_values); auto& general = open_state(); - native::engines::ResidentEngine engine(general, *stage_tables_, stage_anchor_, - march_fused_, march_compensated_, - march_interpolate_, march_fault_); + native::engines::ResidentEngine engine( + general, stage_tables_.value(), stage_anchor_, march_fused_, march_compensated_, + march_interpolate_, march_fault_); march::Result result; { const nb::gil_scoped_release release; - result = march::run(*march_policy_, state, engine, bounds, march_schedule_); + result = + march::run(march_policy_.value(), state, engine, bounds, march_schedule_); } return march_result_to(result, state); } @@ -2050,7 +2064,7 @@ class BoundState { return general_state_->projected_to_host(coarse); } // Not `require_open`: a closed state, whose pointer is null, reads as having none. - std::map dispersive_path() const { + [[nodiscard]] std::map dispersive_path() const { return general_state_ ? general_state_->dispersive_path() : std::map{}; } @@ -2094,51 +2108,51 @@ class BoundState { require_built(); return general_state_->memory_ledger(); } - native::CpuSizing execution_sizing() const { + [[nodiscard]] native::CpuSizing execution_sizing() const { require_built(); return general_state_->execution_sizing(); } - nb::dict cpu_execution() const { + [[nodiscard]] nb::dict cpu_execution() const { require_built(); return cpu_execution_dict(general_state_->cpu_execution()); } - nb::dict fft_planning() const { + [[nodiscard]] nb::dict fft_planning() const { require_built(); return fft_planning_dict(general_state_->fft_planning()); } - double planning_seconds() const { + [[nodiscard]] double planning_seconds() const { require_built(); return general_state_->planning_seconds(); } - double raman_planning_seconds() const { + [[nodiscard]] double raman_planning_seconds() const { require_built(); return general_state_->raman_planning_seconds(); } - std::string raman_fft_provider() const { + [[nodiscard]] std::string raman_fft_provider() const { require_built(); return general_state_->raman_fft_provider(); } - std::map raman_fft_tasks() const { + [[nodiscard]] std::map raman_fft_tasks() const { require_built(); return general_state_->raman_fft_tasks(); } - std::size_t raman_kernel_transforms() const { + [[nodiscard]] std::size_t raman_kernel_transforms() const { require_built(); return general_state_->raman_kernel_transforms(); } - bool fsal_ready() const { + [[nodiscard]] bool fsal_ready() const { require_built(); return general_state_->fsal_ready(); } - std::size_t rk_source_evaluations() const { + [[nodiscard]] std::size_t rk_source_evaluations() const { require_built(); return general_state_->rk_source_evaluations(); } - std::size_t axis_scans() const { + [[nodiscard]] std::size_t axis_scans() const { require_built(); return general_state_->axis_scans(); } - double quadrature_error() const { + [[nodiscard]] double quadrature_error() const { require_open(); return general_state_->quadrature_error(); } @@ -2146,27 +2160,29 @@ class BoundState { require_open(); return general_state_->quadrature_peak(); } - std::string synthesis_provider() const { + [[nodiscard]] std::string synthesis_provider() const { require_built(); return general_state_->synthesis_provider(); } - std::string projection_provider() const { + [[nodiscard]] std::string projection_provider() const { require_built(); return general_state_->projection_provider(); } - std::string gemm_shape_policy() const { + [[nodiscard]] std::string gemm_shape_policy() const { require_built(); return general_state_->gemm_shape_policy(); } - std::size_t response_team_size() const { + [[nodiscard]] std::size_t response_team_size() const { require_built(); return general_state_->response_team_size(); } - std::size_t columns() const noexcept { return columns_; } - std::size_t stages() const noexcept { return stages_; } - std::size_t raman_terms() const noexcept { return raman_terms_; } - bool raman() const noexcept { return raman_; } - bool closed() const noexcept { return !general_state_ || general_state_->closed(); } + [[nodiscard]] std::size_t columns() const noexcept { return columns_; } + [[nodiscard]] std::size_t stages() const noexcept { return stages_; } + [[nodiscard]] std::size_t raman_terms() const noexcept { return raman_terms_; } + [[nodiscard]] bool raman() const noexcept { return raman_; } + [[nodiscard]] bool closed() const noexcept { + return !general_state_ || general_state_->closed(); + } // Closes a session that leases this state first (below, after its class). void close() noexcept; @@ -2245,7 +2261,7 @@ class BoundState { return method; } throw std::invalid_argument( - "a session's march is \"adams\", \"runge_kutta\" or \"low_storage\""); + R"(a session's march is "adams", "runge_kutta" or "low_storage")"); } // The state's field and statistics, as a session reads them. std::unique_ptr session_resident() { @@ -2292,7 +2308,7 @@ class BoundState { if (closed()) throw std::logic_error("native state is closed"); } - native::GeneralState& built_state() const { + [[nodiscard]] native::GeneralState& built_state() const { require_built(); return *general_state_; } @@ -2350,7 +2366,7 @@ moved_array(std::vector&& values, const std::size_t rows, nb::capsule owner(pointer, [](void* value) noexcept { const std::unique_ptr> held(static_cast*>(value)); }); - storage.release(); + static_cast(storage.release()); // the capsule owns it now const std::array shape{rows, columns}; return {pointer->data(), shape.size(), shape.data(), owner}; } @@ -2390,6 +2406,8 @@ class BoundSession { ~BoundSession() { close(); } BoundSession(const BoundSession&) = delete; BoundSession& operator=(const BoundSession&) = delete; + BoundSession(BoundSession&&) = delete; + BoundSession& operator=(BoundSession&&) = delete; // The advance runs without the interpreter lock; its batch becomes a record with it. BatchRecord advance(const std::size_t work_limit, const double deadline) { @@ -2426,8 +2444,10 @@ class BoundSession { const auto receipt = session_->request_snapshot(); return nb::make_tuple(receipt.request, receipt.coalesced); } - native::session::Status status() const noexcept { return session_->status(); } - std::size_t elements() const { return session_->elements(); } + [[nodiscard]] native::session::Status status() const noexcept { + return session_->status(); + } + [[nodiscard]] std::size_t elements() const { return session_->elements(); } // Continue a resumed run's monitor record; takes the session's lock. void continue_monitor(const nb::dict record, const bool plane, const nb::handle unwatched, const nb::dict edge_record) { @@ -2823,7 +2843,7 @@ std::string decode_record_text(std::string value) { return value; } -#if defined(_WIN32) +#ifdef _WIN32 // Windows looks a symbol up in one module only, so the search walks the modules this // extension imports (its FFT, BLAS and threading libraries, under whatever names the // wheel repair gave them), as dlsym on the extension's own handle does on POSIX. @@ -2966,8 +2986,8 @@ nb::dict build_record() { result["kokkos_fft_version"] = SOLPROP_NATIVE_KOKKOSFFT_VERSION; result["kokkos_kernels_version"] = SOLPROP_NATIVE_KOKKOSKERNELS_VERSION; result["nanobind_version"] = SOLPROP_NATIVE_NANOBIND_VERSION; - result["gemm_provider"] = SOLPROP_NATIVE_GEMM_PROVIDER; - result["tpl_blas_enabled"] = static_cast(SOLPROP_NATIVE_TPL_BLAS_ENABLED); + result["gemm_provider"] = "HYBRID"; + result["tpl_blas_enabled"] = (SOLPROP_NATIVE_TPL_BLAS_ENABLED != 0); result["blas_backend"] = SOLPROP_NATIVE_BLAS_BACKEND; result["blas_thread_policy"] = decode_record_text(SOLPROP_NATIVE_BLAS_THREAD_POLICY); result["blas_selection"] = decode_record_text(SOLPROP_NATIVE_BLAS_SELECTION); @@ -3000,7 +3020,7 @@ nb::dict build_record() { result["blas_threads"] = blas_threads; result["blas_threading"] = blas_threading; result["execution_space"] = native::ExecutionSpace::name(); -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA result["fft_backend"] = "cuFFT"; result["fft_library"] = loaded_library("cufftExecZ2Z"); result["blas_library"] = loaded_library("cublasDgemm_v2"); @@ -3030,7 +3050,7 @@ nb::dict build_record() { result["device_count"] = 0; #endif result["raman_fft_option"] = SOLPROP_NATIVE_RAMAN_FFT_PROVIDER; -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE result["raman_fft_library"] = loaded_library("vDSP_fftm_zropD"); #else result["raman_fft_library"] = result["fft_library"]; @@ -3038,7 +3058,7 @@ nb::dict build_record() { result["openmp_library"] = loaded_library("omp_get_num_threads"); result["device_team_size"] = SOLPROP_NATIVE_DEVICE_TEAM_SIZE_RECORD; result["device_reduction_size"] = SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE_RECORD; -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP result["openmp_enabled"] = true; #else result["openmp_enabled"] = false; @@ -3065,12 +3085,11 @@ nb::dict build_record() { result["compiler"] = __VERSION__; result["compiler_flags"] = decode_record_text(SOLPROP_NATIVE_COMPILER_FLAGS); result["build_type"] = SOLPROP_NATIVE_BUILD_TYPE; - result["coverage_instrumented"] = - static_cast(SOLPROP_NATIVE_COVERAGE_INSTRUMENTED); + result["coverage_instrumented"] = (SOLPROP_NATIVE_COVERAGE_INSTRUMENTED != 0); result["coverage_source_hash"] = SOLPROP_NATIVE_COVERAGE_SOURCE_HASH; result["precision"] = "complex128"; result["fft_normalization"] = "none (explicit inverse scaling)"; -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA result["fft_planner"] = "Kokkos-FFT persistent Plan / cuFFT"; #elif defined(KOKKOS_ENABLE_HIP) result["fft_planner"] = "Kokkos-FFT persistent Plan / rocFFT"; @@ -3082,10 +3101,10 @@ nb::dict build_record() { "SOLPROP_FFTW_PLAN overrides"; #endif result["layout"] = "LayoutRight"; - result["native_arch_enabled"] = static_cast(SOLPROP_NATIVE_HOST_ARCH_ENABLED); + result["native_arch_enabled"] = (SOLPROP_NATIVE_HOST_ARCH_ENABLED != 0); result["aggressive_vectorization_enabled"] = - static_cast(SOLPROP_NATIVE_AGGRESSIVE_VECTORIZATION_ENABLED); - result["ipo_enabled"] = static_cast(SOLPROP_NATIVE_IPO_ENABLED); + (SOLPROP_NATIVE_AGGRESSIVE_VECTORIZATION_ENABLED != 0); + result["ipo_enabled"] = (SOLPROP_NATIVE_IPO_ENABLED != 0); const std::string cpu_baseline = SOLPROP_NATIVE_CPU_BASELINE; const std::string architecture_policy = SOLPROP_NATIVE_HOST_ARCH_ENABLED ? "native architecture" @@ -3106,15 +3125,13 @@ nb::dict build_record() { "; no fast-math; " + contraction_policy + "; " + architecture_policy + "; " + ipo_policy + "; " + vectorization_policy; -#if defined(__FAST_MATH__) +#ifdef __FAST_MATH__ result["fast_math_enabled"] = true; #else result["fast_math_enabled"] = false; #endif -#if defined(__apple_build_version__) +#ifdef __apple_build_version__ result["compiler_family"] = "AppleClang"; -#elif defined(__INTEL_LLVM_COMPILER) - result["compiler_family"] = "IntelLLVM"; #elif defined(__clang__) result["compiler_family"] = "Clang"; #elif defined(__GNUC__) @@ -3129,8 +3146,11 @@ nb::dict build_record() { } // namespace NB_MODULE(_core, module_) { - nb::exception(module_, "EnvelopeConvergenceError"); - nb::exception(module_, "BackendRefusal"); + // Each registration makes the Python exception type; the handle need not outlive it. + const nb::exception envelope_convergence_error( + module_, "EnvelopeConvergenceError"); + const nb::exception backend_refusal(module_, + "BackendRefusal"); module_.doc() = "Private nanobind seam for solprop-native"; module_.def("initialize_runtime", &initialize_kokkos, "threads"_a, "kernels"_a = 0, "fft"_a = 0, "raman_fft"_a = 0, "raman_tail_fft"_a = 0, @@ -3253,7 +3273,7 @@ NB_MODULE(_core, module_) { }, "policy"_a, "occupancy"_a); module_.def("available_device_memory", []() -> nb::object { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA std::size_t free_bytes = 0; std::size_t total_bytes = 0; if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess) @@ -3530,7 +3550,7 @@ NB_MODULE(_core, module_) { const std::unique_ptr held( static_cast(value)); }); - storage.release(); + static_cast(storage.release()); // the capsule owns it now const std::array shape{extents[0], extents[1], extents[2]}; return ComplexOutput(pointer->data(), shape.size(), shape.data(), owner); }, @@ -3743,8 +3763,9 @@ NB_MODULE(_core, module_) { [](BoundState& self, const nb::dict policy, const bool fused, const bool compensated, const bool interpolate, const RealInput steps, const RealInput edges) { - self.march_configure(march_policy_from(policy), fused, compensated, - interpolate, {real_vector(steps), real_vector(edges)}); + self.march_configure( + march_policy_from(policy), fused, compensated, interpolate, + {.steps = real_vector(steps), .edges = real_vector(edges)}); }, "policy"_a, "fused"_a, "compensated"_a, "interpolate"_a, "steps"_a.noconvert(), "edges"_a.noconvert()) @@ -4280,8 +4301,8 @@ NB_MODULE(_core, module_) { .def_prop_ro("closed", &BoundState::closed) .def("close", &BoundState::close, nb::call_guard()); // solprop SEP 47: the native execution session and its compact event records. - nb::exception(module_, "SessionLifecycleError", - PyExc_RuntimeError); + const nb::exception lifecycle_error( + module_, "SessionLifecycleError", PyExc_RuntimeError); nb::class_(module_, "SessionEvent") .def_prop_ro("kind", [](const native::session::Event& event) { diff --git a/native/src/bindings_transverse.cpp b/native/src/bindings_transverse.cpp index 7d01aa2fb..52d66167b 100644 --- a/native/src/bindings_transverse.cpp +++ b/native/src/bindings_transverse.cpp @@ -248,10 +248,10 @@ void bind_transverse(nb::module_& module_) { "launch", [](transverse::Edge& self, const nb::dict reading, const double z, const std::optional> sums, - const std::optional noise) { + const std::optional& noise) { std::optional held; if (sums) - held = transverse::LaunchSums{sums->first, sums->second}; + held = transverse::LaunchSums{.total = sums->first, .ramp = sums->second}; std::optional carried; if (noise) carried = transverse_noise_from(*noise); @@ -266,7 +266,7 @@ void bind_transverse(nb::module_& module_) { const std::optional> sums) { std::optional held; if (sums) - held = transverse::ReadSums{sums->first, sums->second}; + held = transverse::ReadSums{.raw = sums->first, .noise = sums->second}; return transverse_notice_to( self.read(reading_from(reading), z, held ? &*held : nullptr)); }, diff --git a/native/src/core.cpp b/native/src/core.cpp index 37a685063..29ff8696a 100644 --- a/native/src/core.cpp +++ b/native/src/core.cpp @@ -14,7 +14,7 @@ #include #include -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP #include #endif @@ -30,7 +30,11 @@ ThreadPolicy thread_policy() { if (configured_threads.kernels != 0) return configured_threads; const int count = Kokkos::DefaultHostExecutionSpace().concurrency(); - return {count, count, count, count, -1}; + return {.kernels = count, + .fft = count, + .raman_fft = count, + .raman_tail_fft = count, + .band_tiles = -1}; } void configure_thread_policy(ThreadPolicy requested) { @@ -62,7 +66,7 @@ ExecutionSpace execution_instance(const int threads) { #else static_cast(threads); #endif - return ExecutionSpace(); + return {}; } void require_cpu_execution(const CpuExecution& execution) { @@ -198,7 +202,7 @@ namespace { // variable in a device build under -Werror). std::atomic openmp_blas{false}; thread_local int blas_holds = 0; -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP thread_local int blas_saved_default = 0; // The calling thread's scopes, and its OpenMP default before the outermost of them. thread_local int scope_depth = 0; @@ -218,10 +222,10 @@ thread_local int scope_saved_blocktime = -1; // OpenMP between that width and Kokkos' at every pair: 9 to 19 times slower on two // Intel Xeon Gold 6240 (solprop issue #130). void enter_runtime_default() noexcept { -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP if (scope_depth++ != 0) return; -#if defined(KMP_VERSION_MAJOR) +#ifdef KMP_VERSION_MAJOR // LLVM's runtime: idle threads spin for a short while inside native work and sleep // at once outside it (solprop ADR-071). scope_saved_blocktime = -1; @@ -244,12 +248,12 @@ void enter_runtime_default() noexcept { #endif } void leave_runtime_default() noexcept { -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP if (--scope_depth != 0) return; if (scope_saved_default != 0) omp_set_num_threads(scope_saved_default); -#if defined(KMP_VERSION_MAJOR) +#ifdef KMP_VERSION_MAJOR if (scope_saved_blocktime >= 0) kmp_set_blocktime(scope_saved_blocktime); #endif @@ -263,13 +267,13 @@ void leave_runtime_default() noexcept { // scope names its width (Kokkos's instances, `fftw_team_loop`), so the default reaches // only OpenBLAS. void hold_blas() noexcept { -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP blas_saved_default = omp_get_max_threads(); omp_set_num_threads(1); #endif } void release_blas() noexcept { -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP omp_set_num_threads(blas_saved_default); #endif } @@ -310,7 +314,7 @@ int scoped_block_time() noexcept { int scoped_openmp_default() noexcept { const CpuTeamScope scope(0); -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP return omp_get_max_threads(); #else return 1; diff --git a/native/src/coupled_flow.cpp b/native/src/coupled_flow.cpp index 71796578c..1ee2c368c 100644 --- a/native/src/coupled_flow.cpp +++ b/native/src/coupled_flow.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/coupled_flow.hpp" + #include "solprop_native/launch.hpp" #include @@ -11,6 +12,7 @@ #include #include #include +#include #include namespace solprop_native { @@ -31,7 +33,7 @@ void require_size(const std::size_t actual, const std::size_t expected, } bool finite(const std::vector& values) { - return std::all_of(values.begin(), values.end(), [](const Complex& value) { + return std::ranges::all_of(values, [](const Complex& value) { return std::isfinite(value.real()) && std::isfinite(value.imag()); }); } @@ -353,12 +355,11 @@ void require_coupled_flow(const Problem& problem) { throw std::invalid_argument("coupled flow residual data must be finite"); } -CoupledFlowAction::CoupledFlowAction(const Problem& problem, - const ExecutionSpace& kernels) +CoupledFlowAction::CoupledFlowAction(const Problem& problem, ExecutionSpace kernels) : n_modes_(problem.n_modes), n_omega_(problem.n_omega), n_stages_(problem.n_stages), active_(problem.flow_active), sections_(problem.flow_sections), terms_(problem.flow_terms), identity_(problem.n_stages + 1, 0U), - listed_counts_(problem.n_stages, 0U), kernels_(kernels) { + listed_counts_(problem.n_stages, 0U), kernels_(std::move(kernels)) { require_coupled_flow(problem); if (!problem.coupled_flow) throw std::invalid_argument("the problem does not declare a coupled flow"); @@ -439,7 +440,7 @@ CoupledFlowAction::form_actions(const std::size_t section, const double dz, throw std::invalid_argument("flow_section names no section of the coupled flow"); require_size(fractions.size(), n_stages_ + 1, "stage fractions"); if (!std::isfinite(dz) || dz <= 0.0 || - !std::all_of(fractions.begin(), fractions.end(), [](const double value) { + !std::ranges::all_of(fractions, [](const double value) { return std::isfinite(value) && value >= 0.0 && value <= 1.0; })) throw std::invalid_argument("coupled flow step and fractions must be finite, the " @@ -558,8 +559,8 @@ void CoupledFlowAction::set_trial(const std::size_t section, return; } require_size(weights.size(), n_stages_ * terms_, "flow_weights"); - if (!std::all_of(weights.begin(), weights.end(), - [](const double value) { return std::isfinite(value); })) + if (!std::ranges::all_of(weights, + [](const double value) { return std::isfinite(value); })) throw std::invalid_argument("flow_weights must be finite"); upload(kernels_, weights_, weights); const auto& listed = host_active_terms_; diff --git a/native/src/crossing.cpp b/native/src/crossing.cpp index e84f9b5cc..c4fc436f4 100644 --- a/native/src/crossing.cpp +++ b/native/src/crossing.cpp @@ -13,7 +13,7 @@ #include #include -#if defined(__linux__) +#ifdef __linux__ #include #include #endif @@ -30,7 +30,7 @@ constexpr bool device_build = false; // Kokkos holds its copies to one thread at a time per instance, so the workers call the // vendor's synchronous copy, which is safe from many threads: it is what was measured. void device_read(void* destination, const void* source, const std::size_t bytes) { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA if (cudaMemcpy(destination, source, bytes, cudaMemcpyDeviceToHost) != cudaSuccess) throw std::runtime_error("a state's copy to the host failed"); #elif defined(KOKKOS_ENABLE_HIP) @@ -43,7 +43,7 @@ void device_read(void* destination, const void* source, const std::size_t bytes) } void device_write(void* destination, const void* source, const std::size_t bytes) { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA if (cudaMemcpy(destination, source, bytes, cudaMemcpyHostToDevice) != cudaSuccess) throw std::runtime_error("a state's copy to the device failed"); #elif defined(KOKKOS_ENABLE_HIP) @@ -57,7 +57,7 @@ void device_write(void* destination, const void* source, const std::size_t bytes /** The device a new thread must select: the choice is per thread on both vendors. */ int current_device() { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA return Kokkos::Cuda{}.cuda_device(); #elif defined(KOKKOS_ENABLE_HIP) return Kokkos::HIP{}.hip_device(); @@ -67,7 +67,7 @@ int current_device() { } void select_device([[maybe_unused]] const int device) { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA if (cudaSetDevice(device) != cudaSuccess) throw std::runtime_error("a crossing's worker could not select its device"); #elif defined(KOKKOS_ENABLE_HIP) @@ -78,7 +78,7 @@ void select_device([[maybe_unused]] const int device) { /** The host threads this process may run on, at most `worker_limit`. */ std::size_t worker_count() { -#if defined(__linux__) +#ifdef __linux__ cpu_set_t allowed; CPU_ZERO(&allowed); if (sched_getaffinity(0, sizeof(allowed), &allowed) == 0) @@ -98,7 +98,7 @@ void BufferDeleter::operator()(double* pointer) const noexcept { std::free(point HostBuffer host_buffer(const std::size_t count) { const std::size_t bytes = count * sizeof(double); -#if defined(__linux__) +#ifdef __linux__ if (takes_ring(bytes)) { // The size huge pages come in, which a large buffer is aligned to. constexpr std::size_t huge_page = std::size_t{2} << 20; diff --git a/native/src/dense.cpp b/native/src/dense.cpp index 91db7bb82..fa2013310 100644 --- a/native/src/dense.cpp +++ b/native/src/dense.cpp @@ -3,14 +3,13 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/dense_output.hpp" +#include "solprop_native/engines.hpp" +#include "solprop_native/lagrange_basis.hpp" #include #include #include -#include "solprop_native/engines.hpp" -#include "solprop_native/lagrange_basis.hpp" - // As in session.cpp (solprop #135): the weights an observation reads round as // solprop's Python forms them, so this file takes no contraction (CMake) and the // optimize attribute this pragma gives every function below keeps GCC from inlining @@ -149,6 +148,7 @@ void RungeKuttaDense::stepped(const march::Result& /*result*/) {} bool RungeKuttaDense::crosses(const double position, const double target, const int ulps) const { + // NOLINTNEXTLINE(readability-suspicious-call-argument): solprop's reached(target, z) return !march::reached(target, position, ulps); } diff --git a/native/src/dispersive.cpp b/native/src/dispersive.cpp index 1588b4041..dfdc2f8c2 100644 --- a/native/src/dispersive.cpp +++ b/native/src/dispersive.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/dispersive.hpp" + #include "solprop_native/gemm.hpp" #include "solprop_native/launch.hpp" #include "solprop_native/slab.hpp" @@ -98,10 +99,12 @@ template struct SelfP x0 = x0 - offset[base]; x1 = x1 - offset[base + 1]; } - Complex p0, p1; + Complex p0; + Complex p1; dispersive_conserving(x0, x1, x0, x1, x0, x1, p0, p1); if constexpr (Form == DispersiveForm::harmonic) { - Complex h0, h1; + Complex h0; + Complex h1; dispersive_harmonic(x0, x1, x0, x1, x0, x1, carrier[time], h0, h1); p0 = p0 + h0; p1 = p1 + h1; @@ -155,10 +158,18 @@ struct PairProducts { if constexpr (Form == DispersiveForm::real) { if constexpr (Components == 2) { const std::size_t at1 = (base + 1) * n_to + time; - const double x0 = field[at0], x1 = field[at1]; - const double f0 = first[at0], f1 = first[at1]; - const double h0 = second[at0], h1 = second[at1]; - double p0 = 0.0, p1 = 0.0, a0 = 0.0, a1 = 0.0, b0 = 0.0, b1 = 0.0; + const double x0 = field[at0]; + const double x1 = field[at1]; + const double f0 = first[at0]; + const double f1 = first[at1]; + const double h0 = second[at0]; + const double h1 = second[at1]; + double p0 = 0.0; + double p1 = 0.0; + double a0 = 0.0; + double a1 = 0.0; + double b0 = 0.0; + double b1 = 0.0; dispersive_isotropic(x0, x1, x0, x1, f0, f1, p0, p1); dispersive_isotropic(x0, x1, x0, x1, h0, h1, a0, a1); dispersive_isotropic(x0, x1, f0, f1, f0, f1, b0, b1); @@ -167,7 +178,9 @@ struct PairProducts { second[at0] = (c0 * a0 + c1 * b0) * collar; second[at1] = (c0 * a1 + c1 * b1) * collar; } else { - const double x = field[at0], f = first[at0], h = second[at0]; + const double x = field[at0]; + const double f = first[at0]; + const double h = second[at0]; first[at0] = dispersive_real_pair(x, f) * collar; second[at0] = (c0 * dispersive_real_pair(x, h) + c1 * dispersive_real_cross(x, f)) * @@ -176,20 +189,29 @@ struct PairProducts { } else { if constexpr (Components == 2) { const std::size_t at1 = (base + 1) * n_to + time; - Complex x0 = field[at0], x1 = field[at1]; + Complex x0 = field[at0]; + Complex x1 = field[at1]; if (shifted) { x0 = x0 - offset[base]; x1 = x1 - offset[base + 1]; } - const Complex f0 = first[at0], f1 = first[at1]; - const Complex h0 = second[at0], h1 = second[at1]; - Complex p0, p1, a0, a1, b0, b1; + const Complex f0 = first[at0]; + const Complex f1 = first[at1]; + const Complex h0 = second[at0]; + const Complex h1 = second[at1]; + Complex p0; + Complex p1; + Complex a0; + Complex a1; + Complex b0; + Complex b1; dispersive_conserving(x0, x1, x0, x1, f0, f1, p0, p1); dispersive_conserving(x0, x1, x0, x1, h0, h1, a0, a1); dispersive_conserving(x0, x1, f0, f1, f0, f1, b0, b1); if constexpr (Form == DispersiveForm::harmonic) { const Complex c = carrier[time]; - Complex g0, g1; + Complex g0; + Complex g1; dispersive_harmonic(x0, x1, x0, x1, f0, f1, c, g0, g1); p0 = p0 + g0; p1 = p1 + g1; @@ -505,7 +527,9 @@ template struct GeneralPairs { } return; } - Value x[Components], f[Components], h[Components]; + Value x[Components]; + Value f[Components]; + Value h[Components]; for (std::size_t c = 0; c < Components; ++c) { const std::size_t at = (base + c) * n_to + time; if constexpr (Real) { @@ -520,7 +544,9 @@ template struct GeneralPairs { h[c] = second_field[at]; } } - Value p[Components], a[Components], b[Components]; + Value p[Components]; + Value a[Components]; + Value b[Components]; if constexpr (Real) { dispersive_isotropic_n(x, x, f, p); dispersive_isotropic_n(x, x, h, a); @@ -624,7 +650,7 @@ void require_dispersive(const Problem& problem) { std::vector distinct; for (std::size_t group = 0; group < groups; ++group) { const auto key = keys.empty() ? 0 : keys[group]; - if (std::find(distinct.begin(), distinct.end(), key) == distinct.end()) + if (std::ranges::find(distinct, key) == distinct.end()) distinct.push_back(key); } const std::size_t fields = problem.dispersive_symmetric ? groups : 2 * groups; @@ -691,9 +717,8 @@ void dispersive_mixture_rows( // --- the layout and the stage's weights ---------------------------------------------- -DispersiveWeights::DispersiveWeights(const Problem& problem, - const ExecutionSpace& kernels) - : kernels_(kernels) { +DispersiveWeights::DispersiveWeights(const Problem& problem, ExecutionSpace kernels) + : kernels_(std::move(kernels)) { require_dispersive(problem); if (!problem.dispersive) throw std::invalid_argument("the problem does not carry the dispersive response"); @@ -713,7 +738,7 @@ DispersiveWeights::DispersiveWeights(const Problem& problem, // `dispersive_terms_layout`). cube_of_.assign(groups_, 0); for (std::size_t group = 0; group < groups_; ++group) { - const auto at = std::find(cube_keys_.begin(), cube_keys_.end(), keys_[group]); + const auto at = std::ranges::find(cube_keys_, keys_[group]); cube_of_[group] = static_cast(at - cube_keys_.begin()); if (at == cube_keys_.end()) cube_keys_.push_back(keys_[group]); @@ -770,7 +795,7 @@ void DispersiveWeights::require_densities(const std::vector& densities) void DispersiveWeights::set_densities(const std::vector& densities) { require_densities(densities); - std::copy(densities.begin(), densities.end(), densities_.begin()); + std::ranges::copy(densities, densities_.begin()); } void DispersiveWeights::update(const std::size_t stage) { diff --git a/native/src/engines.cpp b/native/src/engines.cpp index 3d1c25616..bdd50ba7f 100644 --- a/native/src/engines.cpp +++ b/native/src/engines.cpp @@ -318,7 +318,7 @@ ScriptedRungeKutta::ScriptedRungeKutta(const std::vector& norms, double ScriptedRungeKutta::trial(const double z, const double dz) { if (next_ >= norms_.size()) throw std::out_of_range("the script ran out of norms"); - log_.push_back(ScriptedTrial{z, dz}); + log_.push_back(ScriptedTrial{.z = z, .dz = dz}); const double noise = norms_[next_++]; return h0_ > 0.0 ? noise * std::pow(dz / h0_, power_) : noise; } diff --git a/native/src/far_field.cpp b/native/src/far_field.cpp index 3e2666b30..693ba2626 100644 --- a/native/src/far_field.cpp +++ b/native/src/far_field.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/far_field.hpp" + #include "solprop_native/launch.hpp" #include @@ -13,6 +14,7 @@ #include #include #include +#include #include namespace solprop_native { @@ -71,8 +73,8 @@ std::size_t far_field_band_count(const std::size_t n_x, const std::size_t n_y, return std::max(1, std::min(units, admitted)); } -FarFieldPass::FarFieldPass(const Problem& problem, const ExecutionSpace& execution) - : execution_(execution), n_x_(problem.n_x), n_y_(problem.n_y), +FarFieldPass::FarFieldPass(const Problem& problem, ExecutionSpace execution) + : execution_(std::move(execution)), n_x_(problem.n_x), n_y_(problem.n_y), n_omega_(problem.n_omega), n_pol_(problem.n_pol_state), n_t_(problem.n_t), dt_(problem.dt), envelope_(problem.family == FieldFamily::envelope) { if (!problem.free_geometry() || problem.spatial_k_x.size() != n_x_ || @@ -224,7 +226,8 @@ std::vector FarFieldPass::run(const std::uint64_t flags, if (table_size != 0) Kokkos::deep_copy(execution_, tables_, host_tables_); accumulate(layout, n_bands, n_apertures, n_regions, exponent, - Sharing{shared, one_distance, circles}, field, inputs); + Sharing{.centre = shared, .distance = one_distance, .circles = circles}, + field, inputs); finish(layout, n_bands, n_apertures, n_regions, intensity_factor, packed_size); Kokkos::deep_copy(execution_, host_packed_, packed_); execution_.fence("solprop_native far-field statistics boundary"); @@ -243,7 +246,8 @@ void FarFieldPass::axis_spectrum(const bool closed, const ComplexView2& spectrum launch::parallel_for( "solprop_native.far_field.axis_spectrum", Range2(execution_, {0, 0}, {n_pol_, n}), KOKKOS_LAMBDA(const std::size_t pol, const std::size_t bin) { - double real = 0.0, imag = 0.0; + double real = 0.0; + double imag = 0.0; for (std::size_t band = 0; band < n_bands; ++band) { real += partials((band * slots + first + 2 * pol) * n + bin); imag += partials((band * slots + first + 2 * pol + 1) * n + bin); @@ -408,12 +412,14 @@ void FarFieldPass::accumulate(const FarFieldLayout& layout, const std::size_t n_ transmission = far_field_shaped_weight( far_field_offset(reach, kx, centre_x), far_field_offset(reach, ky, centre_y), - FarFieldOutline{tables(constants + far_field_half_width), - tables(constants + far_field_half_height), - tables(constants + far_field_stretch_x), - tables(constants + far_field_stretch_y), - tables(constants + far_field_inverse_width), - tables(constants + far_field_edge)}, + FarFieldOutline{ + .half_width = tables(constants + far_field_half_width), + .half_height = tables(constants + far_field_half_height), + .stretch_x = tables(constants + far_field_stretch_x), + .stretch_y = tables(constants + far_field_stretch_y), + .inverse_width = + tables(constants + far_field_inverse_width), + .edge = tables(constants + far_field_edge)}, shape, code); } if (transmission == 0.0) @@ -462,7 +468,7 @@ void FarFieldPass::finish(const FarFieldLayout& layout, const std::size_t n_band const double density = envelope ? dt_ * dt_ / (2.0 * std::numbers::pi) : 2.0 * (dt_ * dt_) / std::numbers::pi; const double energy = dt_ / static_cast(n_t_); - const std::size_t leading = static_cast(layout.beam >= 0); + const auto leading = static_cast(layout.beam >= 0); const std::size_t absorbed = layout.absorbed >= 0 ? far_field_absorbed_channels : 0; const std::size_t spectral = (packed_size - absorbed) / n; const std::size_t regions = layout.regions >= 0 ? n_regions : 0; @@ -482,6 +488,8 @@ void FarFieldPass::finish(const FarFieldLayout& layout, const std::size_t n_band std::size_t rest = row; long slot = -1; bool amplitude = false; + // Each test takes the previous range off the row before comparing. + // NOLINTBEGIN(bugprone-assignment-in-if-condition): a packed-row cascade if (rest < leading) { slot = layout.beam; } else if ((rest -= leading) < regions) { @@ -497,6 +505,7 @@ void FarFieldPass::finish(const FarFieldLayout& layout, const std::size_t n_band slot = layout.axis; amplitude = true; } + // NOLINTEND(bugprone-assignment-in-if-condition) const auto first = static_cast(slot); double value = 0.0; if (amplitude) { diff --git a/native/src/general.cpp b/native/src/general.cpp index ad47f5cfb..3e70c97bd 100644 --- a/native/src/general.cpp +++ b/native/src/general.cpp @@ -15,16 +15,16 @@ #include "solprop_native/gemm.hpp" #include "solprop_native/general_modal.hpp" #include "solprop_native/plasma.hpp" - #include "solprop_native/raman_fft.hpp" #include "solprop_native/slab.hpp" #include "solprop_native/vmath.hpp" #include "solprop_native/z_table.hpp" + #include #include #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) #include -#if defined(__SSE2__) +#ifdef __SSE2__ #include #define SOLPROP_NATIVE_STREAMING_STORES 1 #endif @@ -43,15 +43,17 @@ #include #include #include +#include #include #include -#if defined(__APPLE__) +#ifdef __APPLE__ #include #elif defined(__GLIBC__) #include #endif namespace solprop_native { +namespace { /** The three squared norms an accepted step reports, as one reduction value. */ struct NormSums { double old_sum = 0.0; @@ -64,6 +66,7 @@ struct NormSums { return *this; } }; +} // namespace } // namespace solprop_native template <> struct Kokkos::reduction_identity { @@ -71,6 +74,7 @@ template <> struct Kokkos::reduction_identity { }; namespace solprop_native { +namespace { /** A plane's scaled fluence, one reduction value: its sum and its index moments. */ struct FluenceMoments { double total = 0.0; @@ -83,6 +87,7 @@ struct FluenceMoments { return *this; } }; +} // namespace } // namespace solprop_native template <> struct Kokkos::reduction_identity { @@ -178,12 +183,13 @@ void apply_generator(const ExecutionSpace& space, const std::size_t n, #endif } -template KOKKOS_INLINE_FUNCTION void capture_anchor(const Ts&...) {} +template +KOKKOS_INLINE_FUNCTION void capture_anchor(const Ts&... /*unused*/) {} // Net process allocator growth includes opaque vendor FFT plan/workspace // allocations. It is measured separately from Kokkos Views and is not a peak bound. std::size_t heap_bytes() { -#if defined(__APPLE__) +#ifdef __APPLE__ malloc_statistics_t statistics{}; malloc_zone_statistics(nullptr, &statistics); return statistics.size_in_use; @@ -208,7 +214,7 @@ std::size_t heap_bytes() { #endif } std::size_t device_free_bytes() { -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA std::size_t available{}, total{}; if (cudaMemGetInfo(&available, &total) != cudaSuccess) throw std::runtime_error("cudaMemGetInfo failed during FFT planning"); @@ -284,7 +290,7 @@ using StripPlan = KokkosFFT::Plan struct EnvelopeStream { ComplexView2 node, projected, coarse, stacked, spectrum, coarse_spectrum; std::unique_ptr forward, coarse_forward; - std::size_t bytes() const { + [[nodiscard]] std::size_t bytes() const { return sizeof(Complex) * (node.size() + projected.size() + coarse.size() + stacked.size() + spectrum.size() + coarse_spectrum.size()); @@ -296,7 +302,7 @@ struct FieldCurrentStream { RealView2 node, projected, coarse, stacked; ComplexView2 spectrum, coarse_spectrum; std::unique_ptr forward, coarse_forward; - std::size_t bytes() const { + [[nodiscard]] std::size_t bytes() const { return sizeof(double) * (node.size() + projected.size() + coarse.size() + stacked.size()) + sizeof(Complex) * (spectrum.size() + coarse_spectrum.size()); @@ -316,6 +322,7 @@ void cartesian_for(const char* label, const ExecutionSpace& execution, label, Kokkos::RangePolicy(execution, 0, rows), KOKKOS_LAMBDA(const std::size_t row) { for (std::size_t inner = 0; inner < width; ++inner) + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide element(row, inner); }); #endif @@ -337,7 +344,7 @@ void cartesian_for(const char* label, const ExecutionSpace& execution, // builds compile out the host traversals that use these, hence `maybe_unused`. [[maybe_unused]] KOKKOS_INLINE_FUNCTION void stream_store(Complex& target, const Complex value) { -#if defined(SOLPROP_NATIVE_STREAMING_STORES) +#ifdef SOLPROP_NATIVE_STREAMING_STORES _mm_stream_pd(reinterpret_cast(&target), _mm_set_pd(value.imag(), value.real())); #else @@ -346,7 +353,7 @@ void cartesian_for(const char* label, const ExecutionSpace& execution, } [[maybe_unused]] KOKKOS_INLINE_FUNCTION void stream_fence() { -#if defined(SOLPROP_NATIVE_STREAMING_STORES) +#ifdef SOLPROP_NATIVE_STREAMING_STORES _mm_sfence(); #endif } @@ -357,11 +364,12 @@ void cartesian_for(const char* label, const ExecutionSpace& execution, struct CrossingTile { std::size_t first_row, last_row, first_bin, last_bin; - std::size_t size() const noexcept { + [[nodiscard]] std::size_t size() const noexcept { return (last_row - first_row) * (last_bin - first_bin); } // An element's place in a tile buffer, rows outermost. - std::size_t at(const std::size_t row, const std::size_t bin) const noexcept { + [[nodiscard]] std::size_t at(const std::size_t row, + const std::size_t bin) const noexcept { return (row - first_row) * (last_bin - first_bin) + (bin - first_bin); } // Every element in the traversal order `crossing_for` names. @@ -393,8 +401,10 @@ void crossing_tiles(const char* label, const ExecutionSpace& execution, const std::size_t first_row = (index / bin_tiles) * tile_rows; const std::size_t first_bin = (index % bin_tiles) * tile_bins; visit(CrossingTile{ - first_row, rows - first_row < tile_rows ? rows : first_row + tile_rows, - first_bin, width - first_bin < tile_bins ? width : first_bin + tile_bins}); + .first_row = first_row, + .last_row = rows - first_row < tile_rows ? rows : first_row + tile_rows, + .first_bin = first_bin, + .last_bin = width - first_bin < tile_bins ? width : first_bin + tile_bins}); // Each thread orders its own streaming stores before the region ends. if constexpr (Streaming) stream_fence(); @@ -615,7 +625,7 @@ inline void seed_fftw_wisdom(const ExecutionSpace& space, Complex* input, fftw_plan_with_nthreads(space.concurrency()); auto* in = reinterpret_cast(input); auto* out = reinterpret_cast(output); - const fftw_plan seeded = fftw_plan_many_dft( + auto* const seeded = fftw_plan_many_dft( static_cast(extents.size()), extents.data(), howmany, in, nullptr, 1, input_distance, out, nullptr, 1, output_distance, sign, *chosen); // The policy has always tolerated a plane it cannot seed; a profile records it. @@ -814,7 +824,8 @@ struct SpatialCoefficients { const Complex prefactor = propagating ? numerator(omega) / root.real() : clamped(omega); #endif - return {linear - 0.5 * spectral_rate(omega), active ? prefactor : Complex{}}; + return {.linear = linear - 0.5 * spectral_rate(omega), + .prefactor = active ? prefactor : Complex{}}; } }; @@ -961,6 +972,7 @@ ionisation_log_rate(const double log_field, const View& coefficients, long raw_index = static_cast((log_field - log_field_min) / d_log_field); if (raw_index < 0) raw_index = 0; + // NOLINTNEXTLINE(modernize-use-integer-sign-comparison): device code; std::cmp_* is host-only if (raw_index >= static_cast(intervals)) raw_index = static_cast(intervals - 1); const auto index = static_cast(raw_index); @@ -1284,7 +1296,7 @@ void require_general_problem(const Problem& problem) { if (!std::isfinite(value)) throw std::invalid_argument("time_window_derivative is not finite"); for (std::size_t index = 0; index < problem.n_omega; ++index) - if (problem.active_mask[index] && + if (problem.active_mask[index] != 0 && (!(problem.omega[index] > 0.0) || !std::isfinite(problem.omega[index]))) throw std::invalid_argument( "held plasma current requires positive active omega"); @@ -1348,21 +1360,21 @@ void require_general_problem(const Problem& problem) { require_extent(problem.channel_parameters.size(), checked_product(problem.n_channels, 9, "channel parameters"), "channel_parameters"); - if (!std::all_of(problem.channel_parameters.begin(), - problem.channel_parameters.end(), - [](double value) { return std::isfinite(value); }) || - !std::all_of(problem.channel_coefficients.begin(), - problem.channel_coefficients.end(), - [](double value) { return std::isfinite(value); })) + if (!std::ranges::all_of(problem.channel_parameters, + [](double value) { return std::isfinite(value); }) || + !std::ranges::all_of(problem.channel_coefficients, + [](double value) { return std::isfinite(value); })) throw std::invalid_argument("channel parameters and coefficients must be finite"); std::size_t expected_offset = 0; + // Each interval holds four coefficients. + const std::size_t total_intervals = problem.channel_coefficients.size() / 4; for (std::size_t channel = 0; channel < problem.n_channels; ++channel) { const double* row = problem.channel_parameters.data() + 9 * channel; if (!(row[1] > 0.0 && row[2] > 0.0 && row[3] > row[2] && row[4] >= 0.0 && row[5] >= 1.0 && row[5] == std::floor(row[5]) && row[6] == static_cast(expected_offset) && row[7] > 0.0 && row[8] > 0.0) || - row[5] > static_cast(problem.channel_coefficients.size() / 4)) + row[5] > static_cast(total_intervals)) throw std::invalid_argument( "channel constants and coefficient ranges must be positive and consistent"); expected_offset = checked_sum( @@ -1405,8 +1417,8 @@ void require_general_problem(const Problem& problem) { // solprop SEP 18's running-integral rule. Translation certified the taps; only their // extent, finiteness, the trapezoid's identity tap and the route are checked here. if (problem.plasma_taps.empty() || problem.plasma_taps.size() > plasma_max_taps || - !std::all_of(problem.plasma_taps.begin(), problem.plasma_taps.end(), - [](double value) { return std::isfinite(value); })) + !std::ranges::all_of(problem.plasma_taps, + [](double value) { return std::isfinite(value); })) throw std::invalid_argument("plasma_taps must hold one to nine finite taps"); if (!problem.plasma_band && (problem.plasma_taps.size() != 1 || problem.plasma_taps[0] != 1.0)) @@ -1426,7 +1438,7 @@ void require_general_problem(const Problem& problem) { throw std::invalid_argument("envelope plasma requires separate envelope currents " "without THG or field stashing"); const auto rule_size = - checked_product(problem.n_channels, 4 * 512, "envelope rules"); + checked_product(problem.n_channels, std::size_t{4} * 512, "envelope rules"); require_extent(problem.envelope_rule_nodes.size(), rule_size, "envelope_rule_nodes"); require_extent(problem.envelope_rule_weights.size(), rule_size, @@ -1434,7 +1446,8 @@ void require_general_problem(const Problem& problem) { for (std::size_t channel = 0; channel < problem.n_channels; ++channel) for (std::size_t level = 0; level < 4; ++level) { const auto order = std::size_t{64} << level; - double sum = 0.0, previous = -1.0; + double sum = 0.0; + double previous = -1.0; for (std::size_t index = 0; index < 512; ++index) { const auto offset = (channel * 4 + level) * 512 + index; const auto node = problem.envelope_rule_nodes[offset]; @@ -1499,8 +1512,8 @@ void require_general_problem(const Problem& problem) { !(problem.raman_epsilon0 > 0.0) || !std::isfinite(problem.raman_epsilon0)) throw std::invalid_argument( "Raman fraction, dt_fine and epsilon0 must be positive and finite"); - if (!std::all_of(problem.raman_undamped.begin(), problem.raman_undamped.end(), - [](const double value) { return std::isfinite(value); })) + if (!std::ranges::all_of(problem.raman_undamped, + [](const double value) { return std::isfinite(value); })) throw std::invalid_argument("raman_undamped must contain only finite values"); } else if (problem.raman_thg || problem.n_raman_terms != 0 || !problem.raman_undamped.empty() || !problem.raman_tensor_weights.empty() || @@ -1625,7 +1638,7 @@ void accept_update(const ExecutionSpace& execution, const std::size_t count, update.error_sum += squared_abs(one_residual); }, Kokkos::Sum(sums)); - Kokkos::single(Kokkos::PerTeam(team), [&]() { + Kokkos::single(Kokkos::PerTeam(team), [&] { values.old_partial[slot] = sums.old_sum; values.new_partial[slot] = sums.new_sum; values.error_partial[slot] = sums.error_sum; @@ -1665,9 +1678,13 @@ struct EntryBlock { double magnitude[stage_block], inverse[stage_block]; double noise_cosine[stage_block], noise_sine[stage_block]; - EntryFactors at(const std::size_t bin) const { - return {cosine[bin], sine[bin], magnitude[bin], - inverse[bin], noise_cosine[bin], noise_sine[bin]}; + [[nodiscard]] EntryFactors at(const std::size_t bin) const { + return {.cosine = cosine[bin], + .sine = sine[bin], + .magnitude = magnitude[bin], + .inverse = inverse[bin], + .noise_cosine = noise_cosine[bin], + .noise_sine = noise_sine[bin]}; } }; @@ -1967,8 +1984,10 @@ struct StageTerms { const double error_weight, const std::size_t column, const std::size_t first, const std::size_t count) const { - double real[stage_block], imag[stage_block]; - double source_real[stage_block], source_imag[stage_block]; + double real[stage_block]; + double imag[stage_block]; + double source_real[stage_block]; + double source_imag[stage_block]; for (std::size_t bin = 0; bin < count; ++bin) { real[bin] = 0.0; imag[bin] = 0.0; @@ -2044,8 +2063,10 @@ struct StageTerms { combine_block(const std::size_t stage, const double dz, const std::size_t column, const std::size_t first, const std::size_t count, const EntryBlock& factors, const bool store_whole) const { - double real[stage_block], imag[stage_block]; - double sum_real[stage_block], sum_imag[stage_block]; + double real[stage_block]; + double imag[stage_block]; + double sum_real[stage_block]; + double sum_imag[stage_block]; Complex* inverse = &stage_inverse(column, first); for (std::size_t bin = 0; bin < count; ++bin) { if constexpr (Mode == EntryPhase::repeated) { @@ -2336,8 +2357,9 @@ class GeneralWorkspace { require_open(); if (!initialized_) throw std::logic_error("no initial field has been loaded"); - if constexpr (!Kokkos::SpaceAccessibility::accessible) + if constexpr (!static_cast( + Kokkos::SpaceAccessibility::accessible)) return nullptr; execution_space_.fence("solprop_native host_field"); return current_.data(); @@ -2382,12 +2404,15 @@ class GeneralWorkspace { } std::map raman_fft_tasks() const { #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const int full = 0, tail = 0, kernel = 0, products = 0; + const int full = 0; + const int tail = 0; + const int kernel = 0; + const int products = 0; #else - const int full = raman_execution_space_.concurrency(), - tail = raman_tail_execution_space_.concurrency(), - kernel = raman_kernel_execution_space_.concurrency(), - products = raman_product_execution_space_.concurrency(); + const int full = raman_execution_space_.concurrency(); + const int tail = raman_tail_execution_space_.concurrency(); + const int kernel = raman_kernel_execution_space_.concurrency(); + const int products = raman_product_execution_space_.concurrency(); #endif // `products` is nonzero when the modal products replace the node drives. return { @@ -2409,7 +2434,7 @@ class GeneralWorkspace { double quadrature_peak(); std::string synthesis_provider() const; std::string projection_provider() const; - static std::string gemm_shape_policy(); + static std::string_view gemm_shape_policy(); std::size_t response_team_size() const noexcept; bool fsal_ready() const noexcept { return has_recycled_; } std::size_t rk_source_evaluations() const noexcept { return rk_source_evaluations_; } @@ -2482,8 +2507,12 @@ class GeneralWorkspace { // Free space and the hybrid planar kind (`Problem::separable_geometry`). TransverseLayout transverse_layout() const noexcept { const std::size_t plane = transverse_.extent(1) * transverse_.extent(2); - return problem_.slab_geometry() ? TransverseLayout{problem_.n_p, true, plane} - : TransverseLayout{problem_.n_x, false, plane}; + return problem_.slab_geometry() ? TransverseLayout{.inner = problem_.n_p, + .channel_fastest = true, + .plane = plane} + : TransverseLayout{.inner = problem_.n_x, + .channel_fastest = false, + .plane = plane}; } RegisterIndex register_index() const noexcept { #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) @@ -2491,8 +2520,11 @@ class GeneralWorkspace { #else constexpr bool plane = true; #endif - return {transverse_layout(), problem_.n_pol_state, problem_.n_omega, plane, - problem_.n_pol_state == 1}; + return {.layout = transverse_layout(), + .n_pol = problem_.n_pol_state, + .n = problem_.n_omega, + .plane = plane, + .scalar = problem_.n_pol_state == 1}; } // A stage's scratch positions: the volume route's own order on a device, or the // scratch's (`register_index`). Every other pass over the scratch forms its own input @@ -2667,7 +2699,7 @@ class GeneralWorkspace { static bool host_forward_pack() noexcept { static const bool enabled = [] { const char* text = std::getenv("SOLPROP_NATIVE_FORWARD_PACK"); - return text == nullptr || std::string(text) != "0"; + return text == nullptr || std::string_view(text) != "0"; }(); return enabled; } @@ -2712,7 +2744,7 @@ class GeneralWorkspace { const RealView2& output, bool diagnose = false, double preionisation = 0.0); void form_absorber(); - void split_step(double dz, const ComplexView2& values); + void split_step(double dz, const ComplexView2& advanced); void form_split_factor(double dz); void split_columns(double dz, std::size_t active_rows = 0, double parseval = 0.0, std::size_t first_column = 0); @@ -2722,7 +2754,7 @@ class GeneralWorkspace { double power_scale); void separable_to_time(const TransverseChunk& chunk); void separable_to_frequency(const TransverseChunk& chunk); - void separable_split_step(double dz, const ComplexView2& values); + void separable_split_step(double dz, const ComplexView2& advanced); void mask(const ComplexView2& values); bool use_packed_radial_blas() const noexcept; // A square radial analytic map synthesises the retained spectrum, before padding, @@ -3238,13 +3270,11 @@ bool GeneralWorkspace::use_packed_radial_blas() const noexcept { #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) return false; #else - const auto provider = compiled_gemm_provider(); // Two packed work arrays add two complex-equivalent time columns to the // shared conservative twenty-column inventory. Decide before allocation. - constexpr std::size_t max_samples = 64 * 1024 * 1024 / (22 * sizeof(Complex)); - return (provider == GemmProvider::hybrid || - provider == GemmProvider::kokkosblas_tpl) && - !problem_.separable_geometry() && !problem_.spatial_spectral_weights.empty() && + constexpr std::size_t max_samples = + std::size_t{64} * 1024 * 1024 / (22 * sizeof(Complex)); + return !problem_.separable_geometry() && !problem_.spatial_spectral_weights.empty() && problem_.n_modes == problem_.n_nodes && problem_.node_chunk == problem_.n_nodes && problem_.n_pol_state == 1 && problem_.n_pol_transform == 1 && problem_.family == FieldFamily::envelope && @@ -3263,8 +3293,7 @@ bool GeneralWorkspace::use_packed_radial_blas() const noexcept { // modal time field, and a dispersive response its own synthesis, so they keep the // time-domain product; skinny capillary maps would multiply the transforms. bool GeneralWorkspace::retained_synthesis() const noexcept { - return compiled_gemm_provider() != GemmProvider::custom && - problem_.family == FieldFamily::field_analytic && + return problem_.family == FieldFamily::field_analytic && !problem_.spatial_spectral_weights.empty() && !problem_.separable_geometry() && !problem_.general_modal && !problem_.dispersive && !problem_.in_stage_absorber && !problem_.raman && @@ -3383,23 +3412,18 @@ bool GeneralWorkspace::use_blas_for_synthesis() const noexcept { // The hybrid planar kind contracts its small x rule per y row natively. if (problem_.separable_geometry()) return false; - const auto provider = compiled_gemm_provider(); - if (provider == GemmProvider::custom) - return false; if (problem_.n_pol_state != 1) return false; - if (problem_.family == FieldFamily::envelope) { -#if defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) || \ - defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || \ - defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) - if (provider != GemmProvider::kokkosblas_tpl && provider != GemmProvider::hybrid) - return false; -#else +#if !defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) + // Without a vendor BLAS an envelope keeps the custom kernels. + if (problem_.family == FieldFamily::envelope) return false; #endif - } else if (problem_.family != FieldFamily::field_real) { + if (problem_.family != FieldFamily::envelope && + problem_.family != FieldFamily::field_real) return false; - } #if defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) // Spatial envelope matrices can be large even with a short temporal grid. if (!problem_.spatial_spectral_weights.empty() && @@ -3409,7 +3433,7 @@ bool GeneralWorkspace::use_blas_for_synthesis() const noexcept { problem_.n_to >= 256) return true; #endif - return provider != GemmProvider::hybrid || hybrid_shape_eligible(); + return hybrid_shape_eligible(); } // An analytic field's or an envelope's synthesis is a real matrix times complex @@ -3424,20 +3448,14 @@ bool GeneralWorkspace::use_interleaved_synthesis() const noexcept { if (problem_.family == FieldFamily::field_real || problem_.separable_geometry() || problem_.n_pol_state != 1 || problem_.n_pol_transform != 1) return false; - const auto provider = compiled_gemm_provider(); - if (provider == GemmProvider::custom) - return false; - return provider != GemmProvider::hybrid || hybrid_shape_eligible(); + return hybrid_shape_eligible(); } bool GeneralWorkspace::use_interleaved_projection() const noexcept { if (problem_.family != FieldFamily::envelope || problem_.separable_geometry() || problem_.n_pol_state != 1 || problem_.n_pol_transform != 1) return false; - const auto provider = compiled_gemm_provider(); - if (provider == GemmProvider::custom) - return false; - return provider != GemmProvider::hybrid || hybrid_shape_eligible(); + return hybrid_shape_eligible(); } bool GeneralWorkspace::use_blas_for_projection() const noexcept { @@ -3446,21 +3464,15 @@ bool GeneralWorkspace::use_blas_for_projection() const noexcept { // The hybrid planar kind contracts its small x rule per y row natively. if (problem_.separable_geometry()) return false; - const auto provider = compiled_gemm_provider(); - if (provider == GemmProvider::custom) - return false; if (problem_.n_pol_state != 1) return false; - if (problem_.family == FieldFamily::envelope) { -#if defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) || \ - defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || \ - defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) - if (provider != GemmProvider::kokkosblas_tpl && provider != GemmProvider::hybrid) - return false; -#else +#if !defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) + // Without a vendor BLAS an envelope keeps the custom kernels. + if (problem_.family == FieldFamily::envelope) return false; #endif - } #if defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) // Spatial envelope matrices can be large even with a short temporal grid. if (!problem_.spatial_spectral_weights.empty() && @@ -3470,7 +3482,7 @@ bool GeneralWorkspace::use_blas_for_projection() const noexcept { problem_.n_to >= 256) return true; #endif - return provider != GemmProvider::hybrid || hybrid_shape_eligible(); + return hybrid_shape_eligible(); } bool GeneralWorkspace::use_chunked_blas_for_synthesis() const noexcept { @@ -3528,7 +3540,7 @@ std::string GeneralWorkspace::projection_provider() const { return std::string(use_blas_for_projection() ? provider : std::string_view("custom")); } -std::string GeneralWorkspace::gemm_shape_policy() { +std::string_view GeneralWorkspace::gemm_shape_policy() { return "host radial scalar envelope uses packed real BLAS for whole transforms " "with n_nodes/n_to >= 256 and a 22-column inventory within 64 MiB; " "device radial scalar envelope uses vendor complex BLAS with " @@ -3549,7 +3561,7 @@ std::string GeneralWorkspace::gemm_shape_policy() { std::size_t GeneralWorkspace::response_team_size() const noexcept { constexpr auto configured = static_cast(SOLPROP_NATIVE_DEVICE_TEAM_SIZE); -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA // Measured on a Hopper GPU, the crossover is sharp: 4,096-sample fibre problems // favour teams of 256, while 8,192 samples and more favour 512. Keep an explicitly // configured smaller team intact. @@ -4399,17 +4411,32 @@ void GeneralWorkspace::allocate() { const bool slab = problem_.slab_geometry(); separable_dispersive_ = std::make_unique( problem_, kernel_pool_, - SeparableDispersive::StageViews{ - fine_spectrum_, projected_spectrum_, node_real_, polarisation_real_, - node_complex_, polarisation_complex_, slab ? synthesis_ : RealView2{}, - slab ? projection_ : RealView2{}, time_window_, carrier_}); + SeparableDispersive::StageViews{.fine_spectrum = fine_spectrum_, + .projected_spectrum = projected_spectrum_, + .node_real = node_real_, + .polarisation_real = polarisation_real_, + .node_complex = node_complex_, + .polarisation_complex = polarisation_complex_, + .synthesis = slab ? synthesis_ : RealView2{}, + .projection = slab ? projection_ : RealView2{}, + .time_window = time_window_, + .carrier = carrier_}); } else if (problem_.dispersive && !problem_.general_modal) { dispersive_ = std::make_unique( problem_, kernel_pool_, DispersiveResponse::StageViews{ - fine_spectrum_, projected_spectrum_, projected_coarse_spectrum_, node_real_, - polarisation_real_, node_complex_, polarisation_complex_, scaled_synthesis_, - scaled_projection_, scaled_projection_coarse_, time_window_, carrier_}); + .fine_spectrum = fine_spectrum_, + .projected_spectrum = projected_spectrum_, + .projected_coarse_spectrum = projected_coarse_spectrum_, + .node_real = node_real_, + .polarisation_real = polarisation_real_, + .node_complex = node_complex_, + .polarisation_complex = polarisation_complex_, + .scaled_synthesis = scaled_synthesis_, + .scaled_projection = scaled_projection_, + .scaled_projection_coarse = scaled_projection_coarse_, + .time_window = time_window_, + .carrier = carrier_}); } if (problem_.dispersive) { // The response holds execution-space copies of its layout (the general route's @@ -4491,7 +4518,7 @@ void GeneralWorkspace::upload() { copy_vector(execution_space_, held_derivative_, problem_.time_window_derivative); std::vector inverse(problem_.n_omega, 0.0); for (std::size_t index = 0; index < problem_.n_omega; ++index) - if (problem_.active_mask[index]) + if (problem_.active_mask[index] != 0) inverse[index] = 1.0 / problem_.omega[index]; copy_vector(execution_space_, held_inverse_, inverse); } @@ -4500,9 +4527,8 @@ void GeneralWorkspace::upload() { lossless_phase_ = false; if (!problem_.phi_table) { copy_vector(execution_space_, phi_terms_, problem_.phi_terms); - lossless_phase_ = - std::all_of(problem_.phi_terms.begin(), problem_.phi_terms.end(), - [](const Complex& term) { return term.real() == 0.0; }); + lossless_phase_ = std::ranges::all_of( + problem_.phi_terms, [](const Complex& term) { return term.real() == 0.0; }); } if (!problem_.prefactor_table) copy_vector(execution_space_, prefactor_terms_, problem_.prefactor_terms); @@ -4516,9 +4542,8 @@ void GeneralWorkspace::upload() { copy_vector(execution_space_, projection_coarse_, problem_.projection_coarse_matrix); copy_vector(execution_space_, absorber_fine_, problem_.absorber_fine); - spatial_only_absorber_ = - std::all_of(problem_.absorber_fine.begin(), problem_.absorber_fine.end(), - [](const double rate) { return rate == 0.0; }); + spatial_only_absorber_ = std::ranges::all_of( + problem_.absorber_fine, [](const double rate) { return rate == 0.0; }); copy_vector(execution_space_, absorber_coarse_, problem_.absorber_coarse); if (problem_.plasma) copy_vector(execution_space_, rate_coefficients_, problem_.rate_coefficients); @@ -4537,6 +4562,7 @@ void GeneralWorkspace::upload() { // transform/propagation rounding budgets rather than bitwise agreement. const ExecutionSpace& GeneralWorkspace::fft_space_for(const std::size_t samples, const std::size_t length) { + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide return fft_pool_.get(CpuOperation::fft, samples, length); } @@ -5096,9 +5122,9 @@ void GeneralWorkspace::require_trial_scalars(const TrialScalars& scalars) const if (problem_.raman) { require_extent(scalars.raman_rates.size(), problem_.n_stages * problem_.n_raman_terms, "raman_rates"); - if (!std::all_of( - scalars.raman_rates.begin(), scalars.raman_rates.end(), - [](const double value) { return std::isfinite(value) && value >= 0.0; })) + if (!std::ranges::all_of(scalars.raman_rates, [](const double value) { + return std::isfinite(value) && value >= 0.0; + })) throw std::invalid_argument("Raman decay rates must be finite and non-negative"); } } @@ -5184,7 +5210,7 @@ void GeneralWorkspace::upload_trial(const TrialScalars& scalars, const std::vector& source) { if (source.size() != stage_offsets_[region + 1] - stage_offsets_[region]) throw std::invalid_argument("fixed problem array has the wrong extent"); - std::copy(source.begin(), source.end(), base + stage_offsets_[region]); + std::ranges::copy(source, base + stage_offsets_[region]); }; const auto fill = [&](const std::size_t region, const double value) { std::fill(base + stage_offsets_[region], base + stage_offsets_[region + 1], @@ -5231,7 +5257,7 @@ void GeneralWorkspace::upload_trial(const TrialScalars& scalars, fill_slot(1, *sample); if (rate != nullptr) { require_extent(rate->size(), problem_.n_phi_terms, "Adams rate weights"); - std::copy(rate->begin(), rate->end(), staging.data() + stage_offsets_[13]); + std::ranges::copy(*rate, staging.data() + stage_offsets_[13]); } graph::deep_copy(execution_space_, stage_scalars_, staging); bind_stage_scalars(0); @@ -5250,7 +5276,7 @@ void GeneralWorkspace::prepare_raman(const std::vector& rates, if (!problem_.raman) throw std::logic_error("this state has no Raman workspace"); require_extent(rates.size(), problem_.n_raman_terms, "Raman decay rates"); - if (!std::all_of(rates.begin(), rates.end(), [](const double value) { + if (!std::ranges::all_of(rates, [](const double value) { return std::isfinite(value) && value >= 0.0; })) throw std::invalid_argument("Raman decay rates must be finite and non-negative"); @@ -5261,8 +5287,9 @@ void GeneralWorkspace::prepare_raman(const std::vector& rates, if (problem_.n_raman_components > 1) { require_extent(densities.size(), problem_.n_raman_components, "Raman amplitude densities"); - if (!std::all_of(densities.begin(), densities.end(), - [](double value) { return std::isfinite(value) && value >= 0.0; })) + if (!std::ranges::all_of(densities, [](double value) { + return std::isfinite(value) && value >= 0.0; + })) throw std::invalid_argument( "Raman amplitude densities must be finite and non-negative"); auto density_host = Kokkos::create_mirror_view(stage_raman_densities_); @@ -5429,17 +5456,18 @@ std::vector GeneralWorkspace::trial_step(const double dz, execution_space_, count, n, dz, KOKKOS_LAMBDA(const std::size_t column) { const auto slot = column * slots; - return UpdateColumn{current.data() + column * current.stride(0), - solution.data() + column * solution.stride(0), - error.data() + column * error.stride(0), - whole.data() + column * whole.stride(0), - advanced.data() + column * advanced.stride(0), - residual.data() + column * residual.stride(0), - in_norm.data(), - partials.data() + slot, - partials.data() + partials.stride(0) + slot, - partials.data() + 2 * partials.stride(0) + slot, - carried.data() + column * carried.stride(0)}; + return UpdateColumn{.current = current.data() + column * current.stride(0), + .solution = solution.data() + column * solution.stride(0), + .error = error.data() + column * error.stride(0), + .whole = whole.data() + column * whole.stride(0), + .advanced = advanced.data() + column * advanced.stride(0), + .residual = residual.data() + column * residual.stride(0), + .in_norm = in_norm.data(), + .old_partial = partials.data() + slot, + .new_partial = partials.data() + partials.stride(0) + slot, + .error_partial = + partials.data() + 2 * partials.stride(0) + slot, + .carried = carried.data() + column * carried.stride(0)}; }, "solprop_native.step.accept_error_partials", slots); return collect_trial(); @@ -5466,12 +5494,14 @@ std::vector GeneralWorkspace::adams_source(const std::vector& require_extent(input.size(), problem_.n_columns() * problem_.n_omega, "Adams input"); // The flow is set aside for the evaluation and put back whatever happens. auto held_flow = std::exchange(flow_, nullptr); + // An aggregate, so it declares no copy or move; it is never copied or moved. + // NOLINTNEXTLINE(cppcoreguidelines-special-member-functions): a local scope guard struct RestoreFlow { std::unique_ptr& slot; std::unique_ptr& held; // NOLINTNEXTLINE(modernize-use-equals-default): restores the slot (see FlowAside). ~RestoreFlow() { slot = std::move(held); } - } restore{flow_, held_flow}; + } restore{.slot = flow_, .held = held_flow}; require_trial_scalars(scalars); upload_trial(scalars); if (general_) @@ -5757,7 +5787,8 @@ void GeneralWorkspace::low_storage_flows_formed(const bool rebuild_table) { const std::size_t width = columns - first < stage_block ? columns - first : stage_block; const bool in_band = active(omega) != 0U; - double real[stage_block], imag[stage_block]; + double real[stage_block]; + double imag[stage_block]; for (std::size_t offset = 0; offset < width; ++offset) { const Complex x = delta(2) * @@ -5783,16 +5814,17 @@ void GeneralWorkspace::low_storage_flows_formed(const bool rebuild_table) { const std::size_t column = item / blocks; const std::size_t first = (item % blocks) * stage_block; const std::size_t bins = n - first < stage_block ? n - first : stage_block; - double real[stage_block], imag[stage_block]; + double real[stage_block]; + double imag[stage_block]; const auto form = [&](const std::size_t flow, const std::size_t offset) { terms.phase_block(offset, column, first, bins, real, imag); vmath::exp_complex(real, imag, real, imag, bins); for (std::size_t bin = 0; bin < bins; ++bin) flows(flow, column, first + bin) = Complex(real[bin], imag[bin]); }; - if (count == 1) + if (count == 1) { form(0, 2 * n_phi); - else { + } else { form(0, 0); form(1, n_phi); } @@ -5860,6 +5892,7 @@ void GeneralWorkspace::low_storage_separable_pass(const std::size_t pass, } } +namespace { // Each column's masked squared norm of `state` into `partials(0, column)`, in a team // per column: a device's final fold traverses like the others and reduces here, over // one volume, instead of reducing inside the fold's crossing (SEP 28). @@ -5885,6 +5918,7 @@ void column_norms(const char* label, const ExecutionSpace& execution, Kokkos::single(Kokkos::PerTeam(team), [&] { partials(0, column) = total; }); }); } +} // namespace void GeneralWorkspace::low_storage_separable_input(const std::size_t pass) { const auto state = current_; @@ -6355,7 +6389,7 @@ SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex adams_phase_sum( real = real + weight * fixed.real(); imag = imag + weight * fixed.imag(); } - return Complex(real, imag); + return {real, imag}; } } // namespace @@ -6631,9 +6665,15 @@ GeneralWorkspace::adams_fused_trial(const double dz, const AdamsWeights& weights advanced(column, index) = propagator(column, index) * (current(column, index) + dz * total); }); - const AdamsDefectNorms reducer{weights.n_rows, store, source, - propagator, norm_mask_, adams_divisor_, - weights, dz, n}; + const AdamsDefectNorms reducer{.value_count = weights.n_rows, + .store = store, + .source = source, + .propagator = propagator, + .in_norm = norm_mask_, + .divisor = adams_divisor_, + .weights = weights, + .dz = dz, + .n = n}; return adams_norms( "solprop_native.adams.defect_norms", Kokkos::RangePolicy(execution_space_, 0, columns * n), reducer, @@ -7350,6 +7390,9 @@ void GeneralWorkspace::adams_compensation_start(const std::size_t slot, // `strict_scale(1, x)` is x, so its exponential is the same bits. A host block with // vector maths forms the same values on its lanes // (`adams_lanes_predicted_compensation`). +// The strict compensation helpers keep their linkage: changing it could change how GCC +// inlines strict code (fp.hpp), which needs its own A/B. +// NOLINTBEGIN(misc-use-internal-linkage): strict helpers, linkage unchanged template SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex adams_predicted_compensation(const ExponentialWeights& weights, const Complex x, @@ -7422,6 +7465,7 @@ adams_observed_compensation(const ExponentialWeights& weights, const std::size_t return total; } +// NOLINTEND(misc-use-internal-linkage) namespace { // The elements a host task of the elementwise trial's passes takes in turn. Each pass @@ -7460,7 +7504,7 @@ adams_phase_block(const Terms& phi_terms, const Weights& phi_delta, } } -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES // The history rows a pass reads, rotated by a carrier frame's factors // (`adams_history`). Strict. @@ -7546,7 +7590,8 @@ adams_lanes_predicted_compensation(AdamsLanes& lanes, const ExponentialWeights& lanes.ci[t][b] = ri + ir; } } - double total_r[AdamsLanes::B], total_i[AdamsLanes::B]; + double total_r[AdamsLanes::B]; + double total_i[AdamsLanes::B]; for (std::size_t b = 0; b < count; ++b) { total_r[b] = 0.0; total_i[b] = 0.0; @@ -7609,7 +7654,8 @@ inline void adams_lanes_corrected_compensation(AdamsLanes& lanes, SOLPROP_NATIVE_STRICT_BLOCK constexpr std::size_t B = AdamsLanes::B; const std::size_t count = lanes.count; - double total_r[B], total_i[B]; + double total_r[B]; + double total_i[B]; const double weight0 = weights.comp_corrector[0]; for (std::size_t b = 0; b < count; ++b) { const double scaled_r = inverse_dz * lanes.xr[b]; @@ -7653,7 +7699,10 @@ inline void adams_lanes_defects(AdamsLanes& lanes, const ExponentialWeights& wei constexpr std::size_t B = AdamsLanes::B; const std::size_t count = lanes.count; for (std::size_t row = 0; row < scored; ++row) { - double difference_r[B], difference_i[B], weight_r[B], weight_i[B]; + double difference_r[B]; + double difference_i[B]; + double weight_r[B]; + double weight_i[B]; const double weight0 = weights.rows[row][0]; for (std::size_t b = 0; b < count; ++b) { difference_r[b] = weight0 * lanes.wr[b]; @@ -7679,7 +7728,8 @@ inline void adams_lanes_defects(AdamsLanes& lanes, const ExponentialWeights& wei weight_i[b] = weight_i[b] + imag; } } - double defect_r[B], defect_i[B]; + double defect_r[B]; + double defect_i[B]; for (std::size_t b = 0; b < count; ++b) { const double rr = difference_r[b] * weight_r[b]; const double ii = difference_i[b] * weight_i[b]; @@ -7691,7 +7741,8 @@ inline void adams_lanes_defects(AdamsLanes& lanes, const ExponentialWeights& wei defect_i[b] = dz * product_i; } if (weights.compensated) { - double total_r[B], total_i[B]; + double total_r[B]; + double total_i[B]; const double first = weights.comp_rows[row][0]; for (std::size_t b = 0; b < count; ++b) { total_r[b] = first * lanes.nr[b]; @@ -7724,7 +7775,7 @@ inline void adams_lanes_defects(AdamsLanes& lanes, const ExponentialWeights& wei } #endif -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES // The whole step's phase increment `sum_t dPhi_t Phi_t` of every lane into `xr`/`xi`, // as `adams_phase_sum` forms it, and with `Rate` a varying operator's endpoint rate // `sum_t w_t Phi_t` into `dr`/`di`, as `adams_corrected_compensation` forms it, in the @@ -7773,7 +7824,7 @@ adams_lanes_phase(AdamsLanes& lanes, const Terms& phi_terms, const Weights& phi_ } #endif -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES /** * The predict pass on a host block's lanes, the elements [first, last) of a column * whose exponents are formed: the flow, the moments, the predictor's carried samples @@ -7858,7 +7909,8 @@ inline double adams_lanes_predict( adams_lanes_history(lanes, weights, store, weights.order, column, first); // The prediction exp(x) u + dz sum_k M_k sum_j c_jk F_j. const Complex* accepted = ¤t(column, first); - double total_r[B], total_i[B]; + double total_r[B]; + double total_i[B]; for (std::size_t b = 0; b < n; ++b) { lanes.ur[b] = accepted[b].real(); lanes.ui[b] = accepted[b].imag(); @@ -7866,7 +7918,8 @@ inline double adams_lanes_predict( total_i[b] = 0.0; } for (std::size_t k = 0; k < weights.order; ++k) { - double combined_r[B], combined_i[B]; + double combined_r[B]; + double combined_i[B]; for (std::size_t b = 0; b < n; ++b) { combined_r[b] = 0.0; combined_i[b] = 0.0; @@ -7987,7 +8040,7 @@ struct ExponentialCorrection { // The host lanes' predict pass formed the endpoint rate (off a separable route). bool rate_formed; -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES // The pass on a host block's lanes (`AdamsLanes`), as the element-by-element form // below writes it; the norms' sums are taken in element order. void lanes_block(const std::size_t column, const std::size_t first, @@ -8012,13 +8065,15 @@ struct ExponentialCorrection { lanes.mi[k][b] = moment[b].imag(); } } - double total_r[B], total_i[B]; + double total_r[B]; + double total_i[B]; for (std::size_t b = 0; b < count; ++b) { total_r[b] = 0.0; total_i[b] = 0.0; } for (std::size_t k = 0; k < weights.count; ++k) { - double combined_r[B], combined_i[B]; + double combined_r[B]; + double combined_i[B]; const double weight0 = weights.corrector[0][k]; for (std::size_t b = 0; b < count; ++b) { combined_r[b] = weight0 * lanes.wr[b]; @@ -8181,7 +8236,7 @@ struct ExponentialCorrection { const auto first = (task % blocks) * adams_block; const auto last = first + adams_block < n ? first + adams_block : n; const double magnitude = device_scale ? Kokkos::sqrt(magnitude_sum()) : 0.0; -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES if constexpr (vmath::vectorised) { lanes_block(column, first, last, magnitude, update); return; @@ -8283,7 +8338,7 @@ std::vector GeneralWorkspace::adams_elementwise_trial( // weights cross, with the step's and the sample's scalars, in the trial's one copy. upload_trial(step, &sample, compensated ? &rate : nullptr); const std::size_t blocks = (n + adams_block - 1) / adams_block; -#if defined(KOKKOS_ENABLE_CUDA) +#ifdef KOKKOS_ENABLE_CUDA // A device thread takes one element of a long FP64 chain, and Kokkos's CUDA range // reduction gives each block 256 of them: a four-mode capillary's 10496 elements // were 41 blocks on an NVIDIA A40's 84 multiprocessors. Blocks of 128 take them @@ -8316,7 +8371,7 @@ std::vector GeneralWorkspace::adams_elementwise_trial( const std::size_t count = weights.count; const std::size_t formed = count > weights.order ? count : weights.order; const double inverse_dz = 1.0 / dz; -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES // A host block's predict pass forms the corrector's endpoint rate beside the phase, // from the weights the trial uploaded with its scalars. const auto endpoint_rates = stage_rate_weights_; @@ -8353,7 +8408,7 @@ std::vector GeneralWorkspace::adams_elementwise_trial( #endif flow(column, index) = exponential(x); } -#if defined(SOLPROP_NATIVE_ADAMS_LANES) +#ifdef SOLPROP_NATIVE_ADAMS_LANES if constexpr (passes) { magnitude += separable ? adams_lanes_predict( @@ -8425,35 +8480,35 @@ std::vector GeneralWorkspace::adams_elementwise_trial( // On a device the source's launches replay as one graph while they repeat. adams_source_graph_.run(execution_space_, [&] { adams_evaluate_source(sample); }); std::swap(current_, adams_saved_); - const ExponentialCorrection reducer{weights.n_rows, - store, - moments, - stages_, - carried, - phi_terms, - adams_source_, - flow, - current_, - advanced_, - exponent, - lab, - adams_endpoint_, - adams_endpoint_rate_, - stage_rate_weights_, - in_norm, - adams_divisor_, - adams_scale_weights_, - adams_magnitude_sum_, - weights, - dz, - inverse_dz, - adams_rtol_, - adams_atol_, - n, - blocks, - n_phi, - device_scale, - !separable}; + const ExponentialCorrection reducer{.value_count = weights.n_rows, + .store = store, + .moments = moments, + .stages = stages_, + .carried = carried, + .phi_terms = phi_terms, + .source = adams_source_, + .flow = flow, + .current = current_, + .advanced = advanced_, + .exponent = exponent, + .lab = lab, + .endpoint = adams_endpoint_, + .endpoint_rate = adams_endpoint_rate_, + .rate_weights = stage_rate_weights_, + .in_norm = in_norm, + .divisor = adams_divisor_, + .scale_weights = adams_scale_weights_, + .magnitude_sum = adams_magnitude_sum_, + .weights = weights, + .dz = dz, + .inverse_dz = inverse_dz, + .rtol = adams_rtol_, + .atol = adams_atol_, + .n = n, + .blocks = blocks, + .n_phi = n_phi, + .device_scale = device_scale, + .rate_formed = !separable}; auto norms = adams_norms("solprop_native.adams.exponential_correct", tasks, reducer, weights.n_rows); adams_trial_compensated_ = compensated; @@ -9242,10 +9297,20 @@ std::vector GeneralWorkspace::adams_exponential_trial( } }); #endif - const CoupledDefectNorms reducer{weights.n_rows, store, moments, source, - carried, endpoint, block, slots, - norm_mask_, adams_divisor_, weights, dz, - modes, adams_channels_}; + const CoupledDefectNorms reducer{.value_count = weights.n_rows, + .store = store, + .moments = moments, + .source = source, + .carried = carried, + .endpoint = endpoint, + .block = block, + .slots = slots, + .in_norm = norm_mask_, + .divisor = adams_divisor_, + .weights = weights, + .dz = dz, + .modes = modes, + .channels = adams_channels_}; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) // A device item is a (bin, channel) pair, a host item a bin. const auto items = n * modes; @@ -9519,14 +9584,14 @@ void GeneralWorkspace::rk_dense_hold(const double dz) { weights.dense[stage] = rk_dense_weights_[stage]; } // The trial's own scalars are bound: their last row is the whole step's. - const StepExponent step{problem_.separable_geometry(), - spatial_, - active_mask_, - stage_phi_delta_, - phi_terms_, - problem_.n_pol_state, - problem_.n_stages, - problem_.n_phi_terms}; + const StepExponent step{.separable = problem_.separable_geometry(), + .spatial = spatial_, + .active = active_mask_, + .delta = stage_phi_delta_, + .phi_terms = phi_terms_, + .n_pol = problem_.n_pol_state, + .n_stages = problem_.n_stages, + .n_phi = problem_.n_phi_terms}; const auto current = current_; const auto stages = stages_; const auto coefficients = rk_dense_coefficients_; @@ -9569,14 +9634,14 @@ void GeneralWorkspace::rk_dense_observe(const double theta, const TrialScalars& require_trial_scalars(carry); graph::deep_copy(execution_space_, rk_dense_held_, current_); upload_trial(carry); - const StepExponent flow{problem_.separable_geometry(), - spatial_, - active_mask_, - stage_phi_delta_, - phi_terms_, - problem_.n_pol_state, - problem_.n_stages, - problem_.n_phi_terms}; + const StepExponent flow{.separable = problem_.separable_geometry(), + .spatial = spatial_, + .active = active_mask_, + .delta = stage_phi_delta_, + .phi_terms = phi_terms_, + .n_pol = problem_.n_pol_state, + .n_stages = problem_.n_stages, + .n_phi = problem_.n_phi_terms}; const double rest = 1.0 - theta; const auto coefficients = rk_dense_coefficients_; const auto stiff = rk_dense_stiff_; @@ -9625,7 +9690,8 @@ void GeneralWorkspace::rk_dense_release() noexcept { rk_dense_coefficients_ = {}; rk_dense_stiff_ = {}; rk_dense_held_ = {}; - rk_dense_weights_ = {}; + // A moved-in empty vector releases the storage and cannot throw (`= {}` may). + rk_dense_weights_ = std::vector(); rk_dense_ready_ = false; } @@ -9671,9 +9737,12 @@ GeneralWorkspace::transverse_chunk(const std::size_t first_unit) const noexcept const auto units = std::min(transverse_units_per_chunk(), transverse_units() - first_unit); if (problem_.slab_geometry()) - return {first_unit * problem_.n_p, units * problem_.n_p, first_unit * problem_.n_x, - units * problem_.n_x}; - return {first_unit, units, first_unit, units}; + return {.first_row = first_unit * problem_.n_p, + .rows = units * problem_.n_p, + .first_node = first_unit * problem_.n_x, + .nodes = units * problem_.n_x}; + return { + .first_row = first_unit, .rows = units, .first_node = first_unit, .nodes = units}; } void GeneralWorkspace::transverse_forward() { @@ -9744,7 +9813,8 @@ void GeneralWorkspace::separable_stage_input(const std::size_t stage, const doub crossing_tiles( "solprop_native.free.stage_input", execution_space_, problem_.n_columns(), problem_.n_omega, [=](const CrossingTile& tile) { - PropagatorTile step, finish; + PropagatorTile step; + PropagatorTile finish; Complex linear[PropagatorTile::capacity]; tile.visit([&](const std::size_t column, const std::size_t omega) { const auto k = tile.at(column, omega); @@ -10677,8 +10747,9 @@ void GeneralWorkspace::general_core(const std::size_t stage) { if (general_->dispersive() != nullptr) general_->dispersive()->begin(stage); general_->evaluate(); - } else + } else { graph::deep_copy(execution_space_, general_->source(), Complex{}); + } } // The transforms scaled by stage `stage`'s taper, whose value `taper` the host holds. @@ -11096,11 +11167,11 @@ void GeneralWorkspace::response_impl(const std::size_t stage, // Cache contiguous pointwise addresses to avoid repeated View stride loads. // Plasma retains its measured recurrence-loop layout; applying // the same change there regresses HE plasma by 8-9% (kokkos-reopt report). - const auto real_input = input_real.data(); - const auto complex_input = input_complex.data(); - const auto real_target = target_real.data(); - const auto complex_target = target_complex.data(); - const auto window_data = window.data(); + auto* const real_input = input_real.data(); + auto* const complex_input = input_complex.data(); + auto* const real_target = target_real.data(); + auto* const complex_target = target_complex.data(); + auto* const window_data = window.data(); const double pointwise_factor = Plasma ? 0.0 : kerr_coefficient * kerr_density(stage); for (std::size_t time = 0; time < n_to; ++time) { @@ -11606,7 +11677,7 @@ KOKKOS_INLINE_FUNCTION BandWindow band_window_of(const BandLive& live, if constexpr (band_windows) return band_support(live.min_val, live.max_val, samples, half); else - return {0, samples - 1}; + return {.first = 0, .last = samples - 1}; } /** Count `index` into `live` when `value` is not exactly zero. */ @@ -12565,7 +12636,7 @@ void GeneralWorkspace::team_band_statistics(const std::size_t first_node, band_rates_block(magnitudes, &rates(slot, start), count, lower, upper, highest, within); }); - } else + } else { #endif Kokkos::parallel_for( Kokkos::TeamThreadRange(team, samples), [&](const int sample) { @@ -12586,6 +12657,9 @@ void GeneralWorkspace::team_band_statistics(const std::size_t first_node, d_log_field, field_min, field_max, rate_max, intervals); }); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + } +#endif team.team_barrier(); team_band_filter(team, tile, rates, slot, filtered, slot, samples, taps, half); @@ -12660,7 +12734,7 @@ void GeneralWorkspace::team_band_statistics(const std::size_t first_node, exposed[k] = fraction[k]; } }); - } else + } else { #endif Kokkos::parallel_for( Kokkos::TeamThreadRange(team, samples), [&](const int sample) { @@ -12669,6 +12743,9 @@ void GeneralWorkspace::team_band_statistics(const std::size_t first_node, filtered(slot, time) = rates(slot, time) * (1.0 - fraction); exposures(slot, time) = fraction; }); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + } +#endif team.team_barrier(); double paid = 0.0; Kokkos::parallel_reduce( @@ -12774,8 +12851,8 @@ void GeneralWorkspace::finish_raman(const std::size_t stage, const double w = drives == 3 ? convolved(row + 2, time) : 0.0; const double factor = epsilon0 * density(stage); if (family == FieldFamily::envelope) { - const Complex x = node_complex(base, time), - y = node_complex(base + 1, time); + const Complex x = node_complex(base, time); + const Complex y = node_complex(base + 1, time); out_complex(base, time) = (out_complex(base, time) + factor * ((u + v) * x + w * y)) * window(time); @@ -13699,14 +13776,14 @@ void GeneralWorkspace::form_absorber() { "solprop_native.absorber.nonlinear_term", Range2(temporal_execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_to}), KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { - if (family == FieldFamily::envelope) + if (family == FieldFamily::envelope) { absorber_complex(column, time) = (spatial ? absorber_complex(column, time) : Complex{}) - 0.5 * rate(time) * modal_complex(column, time); - else if (family == FieldFamily::field_real) + } else if (family == FieldFamily::field_real) { absorber_real(column, time) = (spatial ? absorber_real(column, time) : 0.0) - 0.5 * rate(time) * modal_real(column, time); - else { + } else { double transverse = 0.0; if (spatial) for (std::size_t source = 0; source < modes; ++source) @@ -13969,7 +14046,9 @@ void GeneralWorkspace::scan_plasma_statistics(const std::size_t first_node, node_count), KOKKOS_LAMBDA(const std::size_t node) { for (std::size_t channel = 0; channel < channels; ++channel) { - double exposure = 0.0, previous_rate = 0.0, largest_rate = 0.0; + double exposure = 0.0; + double previous_rate = 0.0; + double largest_rate = 0.0; for (std::size_t time = 0; time < n_to; ++time) { const double first = (real_family ? node_real(node * n_pol, time) @@ -14094,8 +14173,8 @@ std::vector GeneralWorkspace::statistics( require_open(); const auto& selected_weight = power_weight.empty() ? problem_.power_weight : power_weight; - statistics_unit_weight_ = std::all_of(selected_weight.begin(), selected_weight.end(), - [](double value) { return value == 1.0; }); + statistics_unit_weight_ = + std::ranges::all_of(selected_weight, [](double value) { return value == 1.0; }); if (!selected_weight.empty()) { require_extent(selected_weight.size(), statistics_power_weight_.size(), "statistics power weight"); @@ -14481,7 +14560,8 @@ std::vector GeneralWorkspace::statistics( double power = 0.0; for (std::size_t row = 0; row < rows; ++row) { for (std::size_t pol = 0; pol < n_pol; ++pol) { - Complex value{}, weighted_value{}; + Complex value{}; + Complex weighted_value{}; for (std::size_t mode = 0; mode < modes; ++mode) { value += matrix(row * modes + mode) * field(mode * n_pol + pol, time); if (!unit_weight) @@ -15006,13 +15086,13 @@ void GeneralWorkspace::separable_fluence(const double intensity_factor) { const auto scratch = transverse_; const auto closure_scratch = flux_transverse_; const auto partials = spatial_statistics_partials_; - const PhysicalSpectrum spectrum{transverse_layout(), - synthesis_, - problem_.n_x, - problem_.n_p, - static_cast(problem_.n_t) / - static_cast(problem_.n_to), - problem_.slab_geometry()}; + const PhysicalSpectrum spectrum{.layout = transverse_layout(), + .synthesis = synthesis_, + .n_x = problem_.n_x, + .n_p = problem_.n_p, + .crop_scale = static_cast(problem_.n_t) / + static_cast(problem_.n_to), + .slab = problem_.slab_geometry()}; const auto n_nodes = problem_.n_nodes; const auto n_pol = problem_.n_pol_state; const auto n = problem_.n_omega; @@ -15053,7 +15133,9 @@ void GeneralWorkspace::separable_fluence(const double intensity_factor) { KOKKOS_LAMBDA(const std::size_t item) { const std::size_t first = item * run; const std::size_t count = n_nodes - first < run ? n_nodes - first : run; - double sum[run] = {}, electric_size[run] = {}, magnetic_size[run] = {}; + double sum[run] = {}; + double electric_size[run] = {}; + double magnetic_size[run] = {}; for (std::size_t index = 0; index < n; ++index) { const double weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; for (std::size_t pol = 0; pol < n_pol; ++pol) { @@ -15136,8 +15218,9 @@ void GeneralWorkspace::plane_fluence_moments(double (&moments)[plane_moment_slot const double sample = Kokkos::max(partials(node, 0), 0.0) / maximum; partials(node, 0) = sample; update.total += sample; + const std::size_t row = node / n_x; update.x += sample * static_cast(node % n_x); - update.y += sample * static_cast(node / n_x); + update.y += sample * static_cast(row); }, Kokkos::Sum(sums)); moments[0] = worst.val; @@ -15194,12 +15277,13 @@ std::vector GeneralWorkspace::plane_fluence_trace( Kokkos::RangePolicy(execution_space_, 0, problem_.n_nodes), KOKKOS_LAMBDA(const std::size_t node, double& update) { const double x = (static_cast(node % n_x) * dx - center_x) / extent; - const double y = (static_cast(node / n_x) * dy - center_y) / extent; + const std::size_t row = node / n_x; + const double y = (static_cast(row) * dy - center_y) / extent; update += samples(node, 0) * (x * x + y * y); }, Kokkos::Sum(second)); - const double last_x = static_cast(n_x - 1); - const double last_y = static_cast(n_y - 1); + const auto last_x = static_cast(n_x - 1); + const auto last_y = static_cast(n_y - 1); // One thread per ring: a ring's samples are few, and a thread's sum needs no // reduction. launch::parallel_for( @@ -15487,18 +15571,18 @@ std::vector GeneralWorkspace::aperture_statistics( } FarFieldPass::Inputs GeneralWorkspace::far_field_inputs() const { - return FarFieldPass::Inputs{spatial_spectral_weights_, - spatial_on_axis_, - spatial_.square, - spatial_.square_low, - spatial_.transverse, - spatial_.transverse_low, - spatial_.spectral_rate, - spatial_.user_rate, - spatial_.transverse_rate, - omega_, - active_mask_, - spatial_.evanescent}; + return FarFieldPass::Inputs{.weights = spatial_spectral_weights_, + .on_axis = spatial_on_axis_, + .square = spatial_.square, + .square_low = spatial_.square_low, + .transverse = spatial_.transverse, + .transverse_low = spatial_.transverse_low, + .spectral_rate = spatial_.spectral_rate, + .user_rate = spatial_.user_rate, + .transverse_rate = spatial_.transverse_rate, + .omega = omega_, + .active = active_mask_, + .evanescent = spatial_.evanescent}; } std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t flags, @@ -15602,7 +15686,8 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla const bool live = active(index) != 0U; double power = 0.0; for (std::size_t pol = 0; pol < n_pol; ++pol) { - Complex electric{}, magnetic{}; + Complex electric{}; + Complex magnetic{}; for (std::size_t mode = 0; mode < n_modes; ++mode) { const Complex value = axis_weights(mode) * current(mode * n_pol + pol, index); @@ -15655,7 +15740,8 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla KOKKOS_LAMBDA(const std::size_t time) { double power = 0.0; for (std::size_t pol = 0; pol < n_pol; ++pol) { - Complex electric{}, magnetic{}; + Complex electric{}; + Complex magnetic{}; for (std::size_t mode = 0; mode < n_modes; ++mode) { electric += axis_weights(mode) * field(mode * n_pol + pol, time); magnetic += axis_weights(mode) * closure(mode * n_pol + pol, time); @@ -15715,7 +15801,8 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla "solprop_native.spatial.physical_intensity", Range2(execution_space_, {0, 0}, {n_nodes, n_t}), KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { - double total = 0.0, size = 0.0; + double total = 0.0; + double size = 0.0; for (std::size_t pol = 0; pol < n_pol; ++pol) { const std::size_t at = 2 * (pol * n_t + time); const Complex electric = @@ -15735,7 +15822,8 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla Kokkos::RangePolicy(execution_space_, 0, n_nodes), KOKKOS_LAMBDA(const std::size_t node) { double peak = lowest; - double sum = 0.0, size = 0.0; + double sum = 0.0; + double size = 0.0; for (std::size_t time = 0; time < n_t; ++time) { peak = Kokkos::max(peak, power(node, time)); sum += power(node, time); @@ -16273,6 +16361,7 @@ MonitorSums monitor_sums(const char* label, const ExecutionSpace& execution, double total = 0.0; double regions[4] = {0.0, 0.0, 0.0, 0.0}; for (std::size_t index = first; index < last; ++index) { + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide const double value = power(row, index); total += value; for (std::size_t region = 0; region < 4; ++region) @@ -16616,8 +16705,8 @@ EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, return real * real + imaginary * imaginary; }; const auto sums = [&](const char* label, const RealView2& weights) { - return monitor_sums(label, execution_space_, n, rows, EdgeWeights{weights, first}, - power); + return monitor_sums(label, execution_space_, n, rows, + EdgeWeights{.weights = weights, .first = first}, power); }; take(sums("solprop_native.edge.faces", edge_faces_)); if (whole) @@ -16640,7 +16729,7 @@ EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, }; const auto sums = [&](const char* label, const RealView2& weights) { return monitor_sums(label, execution_space_, n_pol * n, nodes - first, - EdgeWeights{weights, first}, power); + EdgeWeights{.weights = weights, .first = first}, power); }; take(sums("solprop_native.edge.faces", edge_faces_)); if (whole) @@ -16687,16 +16776,30 @@ EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, if (strip || want_current) { for (const auto& line : edge_lines_) { if (!separable) { - take_line(line, EdgeRadialValue{edge_synthesis_, current, edge_scale_, n_pol, n, - problem_.n_modes}); + take_line(line, EdgeRadialValue{.synthesis = edge_synthesis_, + .current = current, + .scale = edge_scale_, + .n_pol = n_pol, + .n = n, + .modes = problem_.n_modes}); } else if (problem_.slab_geometry()) { - take_line(line, EdgeScratchValue{transverse_, transverse_units(), 1, 0, 1}); + take_line(line, EdgeScratchValue{.scratch = transverse_, + .units = transverse_units(), + .across = 1, + .row_stride = 0, + .node_stride = 1}); } else if (line.axis == 1) { - take_line(line, EdgeScratchValue{transverse_, transverse_units(), problem_.n_y, - problem_.n_x, 1}); + take_line(line, EdgeScratchValue{.scratch = transverse_, + .units = transverse_units(), + .across = problem_.n_y, + .row_stride = problem_.n_x, + .node_stride = 1}); } else { - take_line(line, EdgeScratchValue{transverse_, transverse_units(), problem_.n_x, - 1, problem_.n_x}); + take_line(line, EdgeScratchValue{.scratch = transverse_, + .units = transverse_units(), + .across = problem_.n_x, + .row_stride = 1, + .node_stride = problem_.n_x}); } } } @@ -18053,7 +18156,7 @@ std::string GeneralState::projection_provider() const { } std::string GeneralState::gemm_shape_policy() const { const CpuTeamScope team(workspace_->cpu_team()); - return GeneralWorkspace::gemm_shape_policy(); + return std::string(GeneralWorkspace::gemm_shape_policy()); } std::size_t GeneralState::response_team_size() const noexcept { return workspace_ ? workspace_->response_team_size() : 0; diff --git a/native/src/general_modal.cpp b/native/src/general_modal.cpp index e9342573f..8e1d8fad0 100644 --- a/native/src/general_modal.cpp +++ b/native/src/general_modal.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/general_modal.hpp" + #include "solprop_native/fft_plan.hpp" #include "solprop_native/launch.hpp" @@ -30,7 +31,7 @@ constexpr std::size_t max_components = 3; // before its vectoriser runs, and its early complete unrolling leaves such a nest alone // unless asked: the loop then ran a sample at a time, packing pairs through the stack // (a failed store-to-load forward per sample), at about 60 % of a general modal run. -// Clang and icpx unroll the nest themselves and device builds keep their own unrolling, +// Clang unrolls the nest itself and device builds keep their own unrolling, // so the macro is empty there. Unrolling moves no rounding. #if defined(__CUDACC__) || defined(__HIPCC__) || defined(__clang__) #define SOLPROP_NATIVE_UNROLL @@ -285,9 +286,8 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) // A gas region's dispersive channels respond where every regional tensor may be // zero; with no regional tensor and no delayed response the contraction is // identically zero, and it and its transform are skipped. - const bool tensor = - std::any_of(problem.general_tensor.begin(), problem.general_tensor.end(), - [](const double value) { return value != 0.0; }); + const bool tensor = std::ranges::any_of( + problem.general_tensor, [](const double value) { return value != 0.0; }); regional_ = tensor || terms_ != 0; responds_ = tensor || problem.dispersive; const auto chunk_rows = chunk_ * components_; @@ -358,7 +358,8 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) if (problem.dispersive) dispersive_ = std::make_unique( problem, kernel_pool_, - GeneralDispersive::StageViews{node_spectrum_, node_field_, node_source_}); + GeneralDispersive::StageViews{ + .spectrum = node_spectrum_, .field = node_field_, .source = node_source_}); kernels_.fence("solprop_native general modal upload"); } @@ -500,8 +501,9 @@ void GeneralModalSource::evaluate() { else graph::execute(*forward_plan_, node_polarisation_, node_source_, KokkosFFT::Normalization::none); - } else if (!live) + } else if (!live) { graph::deep_copy(kernels_, node_source_, Complex{}); + } // The products' spectra join the regional polarisation's, or are the chunk's. if (live) dispersive_->combine(!regional_); @@ -743,12 +745,13 @@ void GeneralModalSource::respond(const std::size_t first, const std::size_t coun respond_components<2, true>(first, count); else respond_components<1, true>(first, count); - } else if (components_ == 3) + } else if (components_ == 3) { respond_components<3, false>(first, count); - else if (components_ == 2) + } else if (components_ == 2) { respond_components<2, false>(first, count); - else + } else { respond_components<1, false>(first, count); + } } template @@ -834,7 +837,7 @@ void GeneralModalSource::respond_components(const std::size_t first, for (std::size_t i = 0; i < components; ++i) polarisation(row + i, time) = epsilon0 * values[i]; }); - } else + } else { for (std::size_t i = 0; i < components; ++i) { Kokkos::parallel_for( Kokkos::ThreadVectorRange(team, samples), [=](const int sample) { @@ -860,6 +863,7 @@ void GeneralModalSource::respond_components(const std::size_t first, polarisation(row + i, time) = (epsilon0 * value).real(); }); } + } if (!delayed) return; // The delayed response's pair drive: the real field's `E_k E_l`, or the diff --git a/native/src/helpers.cpp b/native/src/helpers.cpp index b181d624c..8b80c1ee3 100644 --- a/native/src/helpers.cpp +++ b/native/src/helpers.cpp @@ -91,10 +91,9 @@ void trial_tables(const Input& nodes, const std::size_t order, const Output& endpoint, const Output& defects) { const std::size_t n = nodes.shape(0); const std::size_t n_orders = orders.size(); - const std::size_t highest = - orders.empty() ? 0 : *std::max_element(orders.begin(), orders.end()); - const bool scored = std::all_of(orders.begin(), orders.end(), - [](const std::size_t value) { return value > 0; }); + const std::size_t highest = orders.empty() ? 0 : *std::ranges::max_element(orders); + const bool scored = + std::ranges::all_of(orders, [](const std::size_t value) { return value > 0; }); if (n_orders == 0 || n_orders > 3 || !scored || order == 0 || order > highest || highest > n || highest + 1 > lagrange::max_nodes) throw std::invalid_argument("trial_tables: order " + std::to_string(order) + diff --git a/native/src/low_storage.cpp b/native/src/low_storage.cpp index e9998449e..fad1ea646 100644 --- a/native/src/low_storage.cpp +++ b/native/src/low_storage.cpp @@ -4,6 +4,8 @@ #include "solprop_native/low_storage.hpp" +#include "solprop_native/adams_march.hpp" + #include #include #include @@ -11,8 +13,6 @@ #include #include -#include "solprop_native/adams_march.hpp" - // As in adams_march.cpp (solprop #135): the step's positions and the table keys round // as solprop's Python does, so this file takes no contraction (CMake) and the optimize // attribute this pragma gives every function below keeps GCC from inlining one into a @@ -60,8 +60,8 @@ void put(std::vector& phi_delta, const std::size_t row, const std::size_ const std::vector& weights) { if (weights.size() != n_phi) throw std::logic_error("an increment's weights are not the stage rows' width"); - std::copy(weights.begin(), weights.end(), - phi_delta.begin() + static_cast(row * n_phi)); + std::ranges::copy(weights, + phi_delta.begin() + static_cast(row * n_phi)); } } // namespace @@ -103,7 +103,7 @@ double schedule_key(const std::vector& keys, const std::vector& // `searchsorted(edges, z + 0.5 * dz) - 1`, clamped to the steps. const double half = 0.5 * dz; const double midpoint = z + half; - const auto found = std::lower_bound(edges.begin(), edges.end(), midpoint); + const auto found = std::ranges::lower_bound(edges, midpoint); const auto index = static_cast(found - edges.begin()) - 1; const auto last = static_cast(keys.size()) - 1; return keys[static_cast( diff --git a/native/src/raman_fft.cpp b/native/src/raman_fft.cpp index bf15c01a8..f062d8c84 100644 --- a/native/src/raman_fft.cpp +++ b/native/src/raman_fft.cpp @@ -4,8 +4,6 @@ #include "solprop_native/raman_fft.hpp" -#include - #include #include #include @@ -15,6 +13,8 @@ #include #include +#include + namespace solprop_native { struct AccelerateRamanPlan::Implementation { @@ -52,8 +52,8 @@ struct AccelerateRamanPlan::Implementation { const auto last = plan.rows * (task + 1) / plan.task_count; auto* input = plan.source + static_cast(first) * plan.source_stride; auto* output = plan.target + static_cast(first) * plan.target_stride; - DSPDoubleSplitComplex source_split{input, input + 1}; - DSPDoubleSplitComplex target_split{output, output + 1}; + DSPDoubleSplitComplex source_split{.realp = input, .imagp = input + 1}; + DSPDoubleSplitComplex target_split{.realp = output, .imagp = output + 1}; const auto log_length = static_cast(std::countr_zero(plan.samples)); if (plan.forward) { vDSP_fftm_zropD(plan.setups[task].get(), &source_split, 2, plan.source_stride, diff --git a/native/src/rk_march.cpp b/native/src/rk_march.cpp index 301bdfa47..359593807 100644 --- a/native/src/rk_march.cpp +++ b/native/src/rk_march.cpp @@ -4,6 +4,8 @@ #include "solprop_native/rk_march.hpp" +#include "solprop_native/core.hpp" + #include #include #include @@ -11,8 +13,6 @@ #include #include -#include "solprop_native/core.hpp" - // As in adams_march.cpp (solprop #135): the optimize attribute this pragma gives every // function below keeps GCC from inlining it into a contracting caller under link-time // optimisation. Clang carries the file's setting on each instruction. @@ -170,7 +170,7 @@ Result run(const Policy& policy, State& state, Engine& engine, const double target = bounds.target; const int ulps = policy.landings.landing_ulps; std::uint64_t accepted = 0; - const auto elapsed = [&]() { + const auto elapsed = [&] { return std::chrono::duration(Clock::now() - started).count(); }; const auto record = [&](const double end, const double dz, const double norm, @@ -192,7 +192,7 @@ Result run(const Policy& policy, State& state, Engine& engine, }; // The step's boundary check, as the Adams march takes it (SEP 45 4.8); true when the // call returns. - const auto watched = [&]() { + const auto watched = [&] { if (!bounds.watch) return false; const auto& rules = bounds.watch_policy; @@ -201,7 +201,7 @@ Result run(const Policy& policy, State& state, Engine& engine, // A session's owning monitor checks the step itself (SEP 47 section 6.4). try { verdict = watch::owned_check( - *bounds.owner, [&engine]() { return engine.reading(); }, state.z, + *bounds.owner, [&engine] { return engine.reading(); }, state.z, bounds.replay, schedule.steps, state.cursor); } catch (const std::exception& error) { fail(result, march::Stage::accept, error); @@ -266,7 +266,7 @@ Result run(const Policy& policy, State& state, Engine& engine, }; // A stop, after the attempt's commit and accounting (SEP 47 section 5.5), the // call's own bound, and the readings' buffer; true when the call returns. - const auto bounded = [&]() { + const auto bounded = [&] { if (march::stop_requested(bounds)) { result.reason = march::Reason::stopped; return true; @@ -291,7 +291,7 @@ Result run(const Policy& policy, State& state, Engine& engine, const std::size_t n_steps = schedule.steps.size(); if (schedule.edges.size() != n_steps + 1) throw std::invalid_argument("a schedule has one edge more than its steps"); - const auto running = [&]() { + const auto running = [&] { return state.cursor < n_steps && state.z < target - bounds.slack; }; // A stop already requested ends the call before its first attempt. @@ -381,6 +381,7 @@ Result run(const Policy& policy, State& state, Engine& engine, } state.proposal = next; } else { + // NOLINTNEXTLINE(readability-suspicious-call-argument): solprop's reached(target, z) if (dense && !march::reached(target, end, ulps)) { if (!bounds.extend) { // solprop records the step's extension from the state before it, then diff --git a/native/src/session.cpp b/native/src/session.cpp index b5045c6c6..a1bb88870 100644 --- a/native/src/session.cpp +++ b/native/src/session.cpp @@ -4,6 +4,8 @@ #include "solprop_native/session.hpp" +#include "solprop_native/core.hpp" + #include #include #include @@ -13,8 +15,6 @@ #include #include -#include "solprop_native/core.hpp" - // As in adams_march.cpp (solprop #135): the scheduler's comparisons and the export's // recipe round as solprop's Python does, so this file takes no contraction (CMake) and // the optimize attribute this pragma gives every function below keeps GCC from @@ -206,7 +206,7 @@ Session::Session(std::unique_ptr resident, Method method, Plan plan, "a session interpolates its planes with the march's dense output"); dense_ = runge_kutta.dense.get(); } - if (!std::is_sorted(plan_.planes.begin(), plan_.planes.end())) + if (!std::ranges::is_sorted(plan_.planes)) throw std::invalid_argument("a session's save planes increase"); if (plan_.next_plane > plan_.planes.size()) throw std::invalid_argument("a session's first plane is past its planes"); @@ -445,7 +445,7 @@ void Session::progress(Batch& batch) { event.kind = EventKind::progress; event.z = here(); drain(batch); - batch.event = event; + batch.event = std::move(event); } void Session::snapshot(Batch& batch) { @@ -459,7 +459,7 @@ void Session::snapshot(Batch& batch) { event.ticket = true; event.generation = generation_; drain(batch); - batch.event = event; + batch.event = std::move(event); } void Session::terminal(Batch& batch, const Outcome outcome, Failure failure) { @@ -473,7 +473,7 @@ void Session::terminal(Batch& batch, const Outcome outcome, Failure failure) { event.ticket = outcome != Outcome::completed && !poisoned(); event.generation = generation_; drain(batch); - batch.event = event; + batch.event = std::move(event); } Batch Session::issue(Batch batch, const Outcome outcome) { @@ -515,7 +515,7 @@ void Session::warning(Batch& batch, const Notice& notice) { event.generation = generation_; event.notice = notice; drain(batch); - batch.event = event; + batch.event = std::move(event); } void Session::monitor_stop(Batch& batch, const Notice& notice) { @@ -581,7 +581,7 @@ bool Session::attach(Batch& batch) { // `DriverSession._attach_monitor` after the launch plane's commit: restore a resumed // record, then record and judge the launch (`attach_monitor`). attached_ = true; - const auto& plan = *plan_.monitor; + const auto& plan = plan_.monitor.value(); if (plan.resume) { // `BoundaryMonitor.resume` continues the edge's record first. if (edge_) @@ -691,15 +691,15 @@ Batch Session::advance(const std::size_t work_limit, const double deadline) { if (state_.load() == State::finished) throw LifecycleError("the session has ended and cannot advance"); const auto started = Clock::now(); - const auto elapsed = [&started]() { + const auto elapsed = [&started] { return std::chrono::duration(Clock::now() - started).count(); }; const std::size_t budget = std::max(work_limit, 1); const std::uint64_t before = attempts_; Batch batch; - const auto made = [&]() { return static_cast(attempts_ - before); }; + const auto made = [&] { return static_cast(attempts_ - before); }; // Issue a filled batch, terminal or not. - const auto ready = [&]() { + const auto ready = [&] { const Outcome outcome = batch.event.kind == EventKind::terminal ? batch.event.outcome : Outcome::none; return issue(std::move(batch), outcome); @@ -807,7 +807,7 @@ Batch Session::advance(const std::size_t work_limit, const double deadline) { target, landing_ulps_) : 0.0; const std::size_t n_steps = schedule().steps.size(); - const auto running = [&]() { + const auto running = [&] { // `replay_to`'s and `march_to`'s loops: to the edge within the slack, or // reached. const double z = position(); @@ -1057,11 +1057,11 @@ SnapshotReceipt Session::request_snapshot() { std::uint64_t value = latch_.load(std::memory_order_acquire); while (true) { if ((value & 1U) != 0) - return {value >> 1U, true}; + return {.request = value >> 1U, .coalesced = true}; const std::uint64_t next = (((value >> 1U) + 1) << 1U) | 1U; if (latch_.compare_exchange_weak(value, next, std::memory_order_acq_rel)) { halt_.store(true, std::memory_order_release); - return {next >> 1U, false}; + return {.request = next >> 1U, .coalesced = false}; } } } @@ -1119,10 +1119,12 @@ bool Session::close() noexcept { runge_kutta->dense.reset(); } resident_.reset(); - history_ = {}; - row_z_ = {}; - row_step_ = {}; - row_values_ = {}; + // Move-assigning an empty vector releases the storage and cannot throw; assigning + // `{}` goes through the initializer-list assignment, which may. + history_ = std::vector(); + row_z_ = std::vector(); + row_step_ = std::vector(); + row_values_ = std::vector(); return true; } diff --git a/native/src/session_edge.cpp b/native/src/session_edge.cpp index 2cb9fd8d8..563f67797 100644 --- a/native/src/session_edge.cpp +++ b/native/src/session_edge.cpp @@ -70,7 +70,7 @@ bool Session::edge_launch(Batch& batch) { const transverse::Reading reading = edge_reading(true, !resumed, edge_->every_step() && resumed); // The input noise's readings, on a resumed launch too: the reads judge above them. - const auto& noise = plan_.monitor->edge_noise; + const auto& noise = plan_.monitor.value().edge_noise; return edge_judged( batch, edge_->launch(reading, position(), nullptr, noise ? &*noise : nullptr), Continuation::edge_launch); diff --git a/native/src/session_statistics.cpp b/native/src/session_statistics.cpp index d739bb788..9f11ff108 100644 --- a/native/src/session_statistics.cpp +++ b/native/src/session_statistics.cpp @@ -2,11 +2,10 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/statistics_plan.hpp" - #include "solprop_native/chebyshev.hpp" #include "solprop_native/core.hpp" #include "solprop_native/far_field.hpp" +#include "solprop_native/statistics_plan.hpp" #include #include @@ -82,9 +81,9 @@ class Wanted { bool operator()(const Stat stat) const { return set_[static_cast(stat)]; } - bool any(std::initializer_list stats) const { - return std::any_of(stats.begin(), stats.end(), - [this](const Stat stat) { return (*this)(stat); }); + [[nodiscard]] bool any(std::initializer_list stats) const { + return std::ranges::any_of(stats, + [this](const Stat stat) { return (*this)(stat); }); } private: @@ -199,14 +198,13 @@ void species(const Plan& plan, const double z, const double taper, if (weights.size() != n_nodes) throw std::invalid_argument("species statistics require one weight per node"); const auto valid = [](const std::vector& values) { - return std::all_of(values.begin(), values.end(), [](const double value) { + return std::ranges::all_of(values, [](const double value) { return std::isfinite(value) && value >= 0.0; }); }; if (fractions.empty() || !valid(densities) || !valid(fractions) || !valid(peaks) || !valid(potentials) || !valid(weights) || - std::any_of(fractions.begin(), fractions.end(), - [](const double value) { return value > 1.0; })) + std::ranges::any_of(fractions, [](const double value) { return value > 1.0; })) throw std::invalid_argument("species statistics require finite non-negative " "populations and fractions in [0, 1]"); std::vector electrons(n_species * n_nodes); @@ -637,7 +635,7 @@ template void Table::validate(const char* name) const { check(n_terms >= 1 && n_values >= 1, std::string("table has no terms: ") + name); check(coefficients.size() == (edges.size() - 1) * n_terms * n_values, std::string("table's coefficients have the wrong extent: ") + name); - check(std::is_sorted(edges.begin(), edges.end()), + check(std::ranges::is_sorted(edges), std::string("table's edges do not increase: ") + name); check(lower <= edges.front() && upper >= edges.back(), std::string("table's span is not inside its slack: ") + name); @@ -729,17 +727,16 @@ void Plan::validate() const { "lacks the on-axis synthesis its axis row reads"); check(channel_preionisation.size() == n_channels, "lacks the preionisation its axis row reads"); - check(axis_node < static_cast(n_nodes), - "has an axis node past the nodes"); + check(std::cmp_less(axis_node, n_nodes), "has an axis node past the nodes"); check(std::isfinite(pad_scale) && pad_scale > 0.0, "has a pad scale that is not positive"); } if (wanted(Stat::zdw)) - check(zdw.size() == n_modes && std::all_of(zdw.begin(), zdw.end(), - [](const RealTable& table) { - return table.present() && - table.n_values == 1; - }), + check(zdw.size() == n_modes && std::ranges::all_of(zdw, + [](const RealTable& table) { + return table.present() && + table.n_values == 1; + }), "lacks a mode's zero-dispersion wavelength table"); if (wanted.any({Stat::beam_radius, Stat::fwhm_r})) { check(route == Route::spatial && widths != Widths::none, diff --git a/native/src/session_widths.cpp b/native/src/session_widths.cpp index 642c94619..968f99991 100644 --- a/native/src/session_widths.cpp +++ b/native/src/session_widths.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -84,7 +85,7 @@ double interp(const double point, const std::vector& x, if (point > x.back()) return samples.back(); // The last `j` with `x[j] <= point`. - const auto above = std::upper_bound(x.begin(), x.end(), point); + const auto above = std::ranges::upper_bound(x, point); const auto j = static_cast(above - x.begin()) - 1; if (j == n - 1 || x[j] == point) return samples[j]; @@ -119,10 +120,9 @@ std::array line_widths(const std::vector& x, const std::vector& values, const double e2_fraction) { const auto finite = [](const double value) { return std::isfinite(value); }; - if (x.size() < 2 || values.size() != x.size() || - !std::all_of(x.begin(), x.end(), finite) || - !std::all_of(values.begin(), values.end(), finite) || - std::any_of(values.begin(), values.end(), [](const double v) { return v < 0.0; })) + if (x.size() < 2 || values.size() != x.size() || !std::ranges::all_of(x, finite) || + !std::ranges::all_of(values, finite) || + std::ranges::any_of(values, [](const double v) { return v < 0.0; })) throw std::invalid_argument("Cartesian fluence widths require finite axes " "and matching nonnegative samples"); require_uniform(x); @@ -215,7 +215,7 @@ std::array radial(const double* radius, const double* fluence, throw std::invalid_argument("radial fluence moments are not representable"); if (energy <= 0.0) return undefined(); - const double second = std::sqrt(2.0) * std::sqrt(moment / energy); + const double second = std::numbers::sqrt2 * std::sqrt(moment / energy); double largest = fluence[0]; for (std::size_t index = 1; index < n; ++index) largest = std::max(largest, fluence[index]); diff --git a/native/src/solprop_native/__init__.py b/native/src/solprop_native/__init__.py index 5ec1553af..cff779b31 100644 --- a/native/src/solprop_native/__init__.py +++ b/native/src/solprop_native/__init__.py @@ -591,7 +591,9 @@ class BuildInfo: ``"scalar libm"``, the system's, on Arm and in device builds (``SOLPROP_NATIVE_VECTOR_MATH``). gemm_provider, tpl_blas_enabled, blas_backend, blas_thread_policy - Dense-transform implementation and external-library threading policy. + Dense-transform implementation (always ``"HYBRID"``: the vendor GEMM for an + eligible transform, the custom kernels for the rest) and external-library + threading policy. blas_selection How configuration chose the BLAS: ``automatic: ...``, ``explicit: ...`` or ``distribution: ...`` or ``disabled: ...``, including the native-order GEMM @@ -957,9 +959,9 @@ def check_build() -> None: "the build enables OpenMP but its Kokkos execution space is " f"{info.execution_space!r}" ) - # A CPU build runs OpenMP, or Kokkos Threads where it must add no OpenMP runtime. + # A CPU build runs OpenMP. expected_execution = { - "none": {"OpenMP", "Threads"}, + "none": {"OpenMP"}, "cuda": {"Cuda"}, "hip": {"HIP"}, } @@ -972,11 +974,8 @@ def check_build() -> None: raise BackendError( f"the {info.device_backend} build has no visible accelerator device" ) - providers = {"CUSTOM", "KOKKOSBLAS_NATIVE", "KOKKOSBLAS_TPL", "HYBRID"} - if info.gemm_provider not in providers: + if info.gemm_provider != "HYBRID": raise BackendError(f"unknown compiled GEMM provider {info.gemm_provider!r}") - if info.gemm_provider == "KOKKOSBLAS_TPL" and not info.tpl_blas_enabled: - raise BackendError("the TPL GEMM provider was built without a BLAS TPL") if info.tpl_blas_enabled and info.blas_backend == "none": raise BackendError("the build enables a BLAS TPL but identifies no provider") if info.build_profile not in {"local", "distribution", "reference"}: diff --git a/native/src/transverse.cpp b/native/src/transverse.cpp index ddb347542..18617139e 100644 --- a/native/src/transverse.cpp +++ b/native/src/transverse.cpp @@ -210,11 +210,11 @@ void Edge::resume(const Resumption& record) { return; resumed_ = true; launch_ = launch; - baseline_ = *record.edge_baseline; + baseline_ = record.edge_baseline.value(); if (fits(record.edge_max)) - peak_ = *record.edge_max; + peak_ = record.edge_max.value(); if (fits(record.edge_max_raw)) - peak_raw_ = *record.edge_max_raw; + peak_raw_ = record.edge_max_raw.value(); allowance_ = record.noise_allowance.value_or(0.0); ramp_raw_ = record.raw_ramp_removed.value_or(0.0); ramp_noise_ = record.noise_ramp_removed.value_or(0.0); @@ -340,8 +340,8 @@ Notice Edge::run_launch() { case Stage::launch_clip: while (face_ < n_faces_) { const std::size_t face = face_++; - baseline_[face] = (*held_.faces)[face] / launch_; - const double clipped = (*held_.clip)[face] / launch_; + baseline_[face] = held_.faces.value()[face] / launch_; + const double clipped = held_.clip.value()[face] / launch_; if (clipped > config_.warn) { Notice notice = notice_of(Notice::Kind::clip, clipped); notice.face = face; @@ -463,7 +463,7 @@ Notice Edge::judge_faces(const Reading& reading, const double z) { // `max` of `(judged, name)` tuples, ties broken on the name. std::array judged{}; for (std::size_t face = 0; face < n_faces_; ++face) { - const double share = (*reading.faces)[face] / launch_; + const double share = reading.faces.value()[face] / launch_; peak_raw_[face] = py_max(peak_raw_[face], share); judged[face] = share - baseline_[face] - allowance_; peak_[face] = py_max(peak_[face], judged[face]); diff --git a/native/tests/native/execution.hpp b/native/tests/native/execution.hpp index e61a02244..cc95a94c9 100644 --- a/native/tests/native/execution.hpp +++ b/native/tests/native/execution.hpp @@ -15,7 +15,7 @@ namespace solprop_native::test { inline CpuExecution make_table() { CpuExecution table; -#if defined(__APPLE__) +#ifdef __APPLE__ constexpr std::size_t stage = 8192; constexpr std::size_t fft = 32768; constexpr std::size_t fft_thread = 32768; @@ -30,7 +30,8 @@ inline CpuExecution make_table() { #endif const auto set = [&](CpuOperation operation, std::size_t threshold, std::size_t full) { - table.grain[static_cast(operation)] = {threshold, full}; + table.grain[static_cast(operation)] = {.threshold = threshold, + .full = full}; }; set(CpuOperation::stage, stage, stage); set(CpuOperation::response, 32768, 196608); diff --git a/native/tests/native/main.cpp b/native/tests/native/main.cpp index 7d9a924f1..8dab7d460 100644 --- a/native/tests/native/main.cpp +++ b/native/tests/native/main.cpp @@ -3,7 +3,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include - #include #include diff --git a/native/tests/native/test_adams_march.cpp b/native/tests/native/test_adams_march.cpp index 08603955d..e588e8e5b 100644 --- a/native/tests/native/test_adams_march.cpp +++ b/native/tests/native/test_adams_march.cpp @@ -10,6 +10,8 @@ #include "solprop_native/adams_march.hpp" +#include + #include #include #include @@ -17,8 +19,6 @@ #include #include -#include - namespace { namespace march = solprop_native::march; diff --git a/native/tests/native/test_adams_phi.cpp b/native/tests/native/test_adams_phi.cpp index 11de12f3a..faf33b3ac 100644 --- a/native/tests/native/test_adams_phi.cpp +++ b/native/tests/native/test_adams_phi.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -92,7 +93,7 @@ void add_threshold(std::vector& out, const double t) { for (const double r : {std::nextafter(t, 0.0), t, std::nextafter(t, infinity)}) { out.emplace_back(r, 0.0); out.emplace_back(0.0, -r); - const double d = r / std::sqrt(2.0); + const double d = r / std::numbers::sqrt2; for (const double e : {std::nextafter(d, 0.0), d, std::nextafter(d, infinity)}) out.emplace_back(-e, e); } @@ -143,6 +144,7 @@ TEST(AdamsPhiLanes, EdgeArgumentsAreTheScalarFormsBits) { TEST(AdamsPhiLanes, EachLaneIsItsOwnWhateverItsNeighbours) { // Mixed blocks of every length: small, series, near-threshold and large arguments // side by side, so that lanes stop their series, square and branch apart. + // NOLINTNEXTLINE(bugprone-random-generator-seed): a fixed seed keeps the test reproducible std::mt19937_64 generator(20261029); std::uniform_real_distribution phase(-3.2, 3.2); std::uniform_real_distribution exponent(-12.0, 1.2); diff --git a/native/tests/native/test_aperture.cpp b/native/tests/native/test_aperture.cpp index a518b1842..325511dc8 100644 --- a/native/tests/native/test_aperture.cpp +++ b/native/tests/native/test_aperture.cpp @@ -15,7 +15,6 @@ #include "solprop_native/far_field.hpp" #include - #include #include @@ -81,6 +80,7 @@ Case make_case(const std::size_t n_x, const std::size_t n_y, const std::size_t n problem.spatial_k_x = fft_axis(n_x, n_y > 1 ? 1.5e6 : 1.0e5); problem.spatial_k_y = n_y > 1 ? fft_axis(n_y, 1.2e6) : std::vector{0.0}; const std::size_t columns = n_x * n_y; + // NOLINTNEXTLINE(bugprone-random-generator-seed): a fixed seed keeps the test reproducible std::mt19937_64 random(7); std::normal_distribution normal; data.field.resize(columns * n_pol * n); @@ -175,18 +175,18 @@ std::vector run(Case& data, const std::uint64_t flags, const int profile Kokkos::deep_copy(field, host); native::FarFieldPass pass(data.problem, native::ExecutionSpace{}); const native::FarFieldPass::Inputs inputs{ - device("test.weights", data.weights), - device("test.on_axis", data.on_axis), - device("test.square", data.square), - device("test.square_low", data.square_low), - device("test.transverse", data.transverse), - device("test.transverse_low", data.transverse_low), - device("test.spectral_rate", data.spectral_rate), - device("test.user_rate", data.user_rate), - device("test.transverse_rate", data.transverse_rate), - device("test.omega", data.omega), - device("test.active", data.active), - data.evanescent}; + .weights = device("test.weights", data.weights), + .on_axis = device("test.on_axis", data.on_axis), + .square = device("test.square", data.square), + .square_low = device("test.square_low", data.square_low), + .transverse = device("test.transverse", data.transverse), + .transverse_low = device("test.transverse_low", data.transverse_low), + .spectral_rate = device("test.spectral_rate", data.spectral_rate), + .user_rate = device("test.user_rate", data.user_rate), + .transverse_rate = device("test.transverse_rate", data.transverse_rate), + .omega = device("test.omega", data.omega), + .active = device("test.active", data.active), + .evanescent = data.evanescent}; auto values = pass.run(flags, apertures, exponent, data.regions, 0.5, field, inputs); pass.close(); return values; @@ -219,12 +219,13 @@ double host_weight(const Case& data, const std::size_t index, const double kx, return native::far_field_shaped_weight( native::far_field_offset(scale, kx, centre_x), native::far_field_offset(scale, ky, centre_y), - native::FarFieldOutline{constants[native::far_field_half_width], - constants[native::far_field_half_height], - constants[native::far_field_stretch_x], - constants[native::far_field_stretch_y], - constants[native::far_field_inverse_width], - constants[native::far_field_edge]}, + native::FarFieldOutline{.half_width = constants[native::far_field_half_width], + .half_height = constants[native::far_field_half_height], + .stretch_x = constants[native::far_field_stretch_x], + .stretch_y = constants[native::far_field_stretch_y], + .inverse_width = + constants[native::far_field_inverse_width], + .edge = constants[native::far_field_edge]}, shape, code); } @@ -244,14 +245,18 @@ std::vector reference(const Case& data, const int profile, const int exp const std::size_t row = native::far_field_aperture_constants + n; const std::size_t n_apertures = data.apertures.size() / row; const std::size_t n_regions = data.regions.size() / 4; - std::vector beam(n), absorbed(native::far_field_absorbed_channels), axis(n); + std::vector beam(n); + std::vector absorbed(native::far_field_absorbed_channels); + std::vector axis(n); std::vector> regions(n_regions, std::vector(n)); std::vector> collected(n_apertures, std::vector(n)); std::vector> through(n_apertures, std::vector(n)); for (std::size_t bin = 0; bin < n; ++bin) { // Each amplitude and its magnetic closure, the same sum with conj(eta) applied. - std::vector> amplitude(n_pol), axial(n_apertures * n_pol); - std::vector> closed(n_pol), closed_axial(n_apertures * n_pol); + std::vector> amplitude(n_pol); + std::vector> axial(n_apertures * n_pol); + std::vector> closed(n_pol); + std::vector> closed_axial(n_apertures * n_pol); for (std::size_t column = 0; column < columns; ++column) { const double kx = problem.spatial_k_x[column % problem.n_x]; const double ky = problem.spatial_k_y[column / problem.n_x]; @@ -327,6 +332,7 @@ std::vector reference(const Case& data, const int profile, const int exp .real(); } std::vector packed; + packed.reserve(beam.size()); for (const double value : beam) packed.push_back(0.5 * density * value); for (const double value : absorbed) @@ -515,9 +521,12 @@ TEST(FarFieldPass, TheShapesAreTheirFormulas) { const auto outline = [](const double half_width, const double half_height, const double stretch_x, const double stretch_y, const double width) { - return native::FarFieldOutline{ - half_width, half_height, stretch_x, stretch_y, width > 0.0 ? 1.0 / width : 0.0, - width}; + return native::FarFieldOutline{.half_width = half_width, + .half_height = half_height, + .stretch_x = stretch_x, + .stretch_y = stretch_y, + .inverse_width = width > 0.0 ? 1.0 / width : 0.0, + .edge = width}; }; // The offset is X - x, so x >= X reads across <= 0, whichever side is asked for. EXPECT_EQ(native::far_field_offset(2.0, -3.0, 1.0), -5.0); diff --git a/native/tests/native/test_core.cpp b/native/tests/native/test_core.cpp index 2e2446c37..235f173d8 100644 --- a/native/tests/native/test_core.cpp +++ b/native/tests/native/test_core.cpp @@ -2,15 +2,13 @@ // // SPDX-License-Identifier: GPL-3.0-or-later +#include "bounds.hpp" +#include "execution.hpp" #include "solprop_native/core.hpp" #include "solprop_native/gemm.hpp" #include "solprop_native/plasma.hpp" -#include "bounds.hpp" -#include "execution.hpp" - #include - #include #include @@ -26,7 +24,7 @@ #include #include -#if defined(KOKKOS_ENABLE_OPENMP) +#ifdef KOKKOS_ENABLE_OPENMP #include #endif @@ -95,7 +93,7 @@ std::vector real_time(const std::vector& spectrum, native::Problem base_problem(const native::FieldFamily family) { native::Problem problem; problem.cpu_execution = native::test::table(); - problem.n_t = family == native::FieldFamily::envelope ? 4 : 4; + problem.n_t = 4; problem.n_to = family == native::FieldFamily::envelope ? 4 : 8; problem.n_omega = family == native::FieldFamily::envelope ? 4 : 3; problem.n_stages = 1; @@ -132,21 +130,23 @@ native::Problem base_problem(const native::FieldFamily family) { native::TrialScalars trial(const native::Problem& problem, const double taper = 1.0) { return { - std::vector((problem.n_stages + 1) * problem.n_phi_terms, 0.0), - std::vector(problem.n_stages * problem.n_phi_terms, 0.0), - std::vector(problem.n_stages * problem.n_prefactor_terms, 1.0), - std::vector(problem.n_stages, 1.0), - std::vector(problem.n_stages, taper), - {}, - std::vector(problem.n_stages * problem.n_raman_terms, 0.0), - {}, - {}, - {}, - {}, - 0, - {}, - {}, - {}, + .phi_delta = + std::vector((problem.n_stages + 1) * problem.n_phi_terms, 0.0), + .phi_at = std::vector(problem.n_stages * problem.n_phi_terms, 0.0), + .prefactor = + std::vector(problem.n_stages * problem.n_prefactor_terms, 1.0), + .density = std::vector(problem.n_stages, 1.0), + .taper = std::vector(problem.n_stages, taper), + .preionisation = {}, + .raman_rates = std::vector(problem.n_stages * problem.n_raman_terms, 0.0), + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = 0, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}, }; } @@ -944,14 +944,15 @@ TEST(GeneralState, HeldCurrentMatchesTheDiscreteSinusoidalDrift) { problem.field_min = 1e6; problem.field_max = 1e7; problem.log_field_min = std::log(problem.field_min); - problem.d_log_field = std::log(10.0); + problem.d_log_field = std::numbers::ln10; constexpr double population = 0.25; // The unit mode at bin one produces E_n = 2*synthesis/N*cos(n theta). // The exact finite trapezoid sum of cos is dt*cot(theta/2)*sin(n theta)/2. const double amplitude = 2.0 * problem.synthesis / static_cast(problem.n_to); const double current_scale = population * problem.charge_mass_ratio * amplitude * problem.dt_fine / (2.0 * std::tan(theta / 2.0)); - std::vector product(problem.n_to), edge(problem.n_to); + std::vector product(problem.n_to); + std::vector edge(problem.n_to); double operand = 0.0; for (std::size_t time = 0; time < problem.n_to; ++time) { const double angle = theta * static_cast(time); @@ -969,7 +970,8 @@ TEST(GeneralState, HeldCurrentMatchesTheDiscreteSinusoidalDrift) { const double tolerance = 512.0 * static_cast(problem.n_to) * kEpsilon * operand; ASSERT_GT(tolerance, 0.0); - const auto x = dft(product), z = dft(edge); + const auto x = dft(product); + const auto z = dft(edge); native::GeneralState state(problem); auto scalars = trial(problem); scalars.preionisation = {population}; @@ -1374,7 +1376,6 @@ TEST(Gemm, EngineSelectsTheNativeOrderKernelWithoutAVendorBlas) { TEST(GeneralState, ProviderPolicyAndLifecycleAreIntrospectable) { native::GeneralState state(multimode_problem()); - const auto compiled = native::compiled_gemm_provider(); // A host build's hybrid policy sends every eligible transform to the vendor GEMM; a // device build keeps its shape guard, which this two-node problem is below. #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) @@ -1382,11 +1383,9 @@ TEST(GeneralState, ProviderPolicyAndLifecycleAreIntrospectable) { #else constexpr bool hybrid_vendor = true; #endif - const auto expected = - compiled == native::GemmProvider::custom || - (compiled == native::GemmProvider::hybrid && !hybrid_vendor) - ? std::string("custom") - : std::string(native::selected_blas_provider_name()); + const auto expected = hybrid_vendor + ? std::string(native::selected_blas_provider_name()) + : std::string("custom"); EXPECT_EQ(state.synthesis_provider(), expected); EXPECT_EQ(state.projection_provider(), expected); EXPECT_NE(state.gemm_shape_policy().find("every whole transform on the host"), @@ -1410,6 +1409,10 @@ struct SynthesisCapture { [](const char* name) { synthesis_regions.emplace_back(name); }); } ~SynthesisCapture() { Kokkos::Tools::Experimental::set_callbacks(previous); } + SynthesisCapture(const SynthesisCapture&) = delete; + SynthesisCapture& operator=(const SynthesisCapture&) = delete; + SynthesisCapture(SynthesisCapture&&) = delete; + SynthesisCapture& operator=(SynthesisCapture&&) = delete; }; // Five nodes and modes in chunks of three, the tail two. The square map takes the @@ -1464,12 +1467,10 @@ void check_radial_analytic_chunks(const bool absorber, const char* expected_regi std::count(synthesis_regions.begin(), synthesis_regions.end(), expected_region); const auto custom = std::count(synthesis_regions.begin(), synthesis_regions.end(), "solprop_native.transform.synthesis.custom"); - const bool uses_blas = - native::compiled_gemm_provider() != native::GemmProvider::custom; - EXPECT_EQ(selected, uses_blas ? 2 : 0); - EXPECT_EQ(custom, uses_blas ? 0 : 2); + EXPECT_EQ(selected, 2); + EXPECT_EQ(custom, 0); // Only the absorber-free map keeps the retained spectra. - EXPECT_EQ(ledger.at("retained_synthesis") > 0, uses_blas && !absorber); + EXPECT_EQ(ledger.at("retained_synthesis") > 0, !absorber); chunked.finish(0.01); const auto actual = chunked.field_to_host(); // Both computations have one inverse/forward FFT and two length-five products; @@ -1507,38 +1508,33 @@ TEST(GeneralState, ChunkedBlasBuffersHaveExactPersistentByteCost) { native::GeneralState state(problem); const auto ledger = state.memory_ledger(); - const auto provider = native::compiled_gemm_provider(); - const auto expected = provider == native::GemmProvider::custom - ? 0U - : (3U + 2U) * problem.n_modes * 3U * sizeof(double); - EXPECT_EQ(ledger.at("chunk_transform_buffers"), expected); - - if (provider != native::GemmProvider::custom) { - auto whole_problem = problem; - whole_problem.node_chunk = whole_problem.n_nodes; - const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, - {0.5, 0.0}, {-0.2, 0.1}, {0.05, -0.025}}; - native::GeneralState whole(whole_problem); - whole.set_initial(initial, {}); - static_cast(whole.trial_step(0.01, trial(whole_problem))); - whole.finish(0.01); - state.set_initial(initial, {}); - static_cast(state.trial_step(0.01, trial(problem))); - state.finish(0.01); - const auto reference = whole.field_to_host(); - const auto chunked = state.field_to_host(); - const auto dot_length = static_cast(problem.n_nodes); - const double dot_gamma = dot_length * kEpsilon / (1.0 - dot_length * kEpsilon); - const double tolerance = - 512.0 * kEpsilon * std::ceil(std::log2(static_cast(problem.n_to))) + - 4.0 * dot_gamma; - for (std::size_t index = 0; index < chunked.size(); ++index) - expect_complex_near(chunked[index], reference[index], - tolerance * std::max(1.0, Kokkos::abs(reference[index]))); - } + EXPECT_EQ(ledger.at("chunk_transform_buffers"), + (3U + 2U) * problem.n_modes * 3U * sizeof(double)); + + auto whole_problem = problem; + whole_problem.node_chunk = whole_problem.n_nodes; + const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, + {0.5, 0.0}, {-0.2, 0.1}, {0.05, -0.025}}; + native::GeneralState whole(whole_problem); + whole.set_initial(initial, {}); + static_cast(whole.trial_step(0.01, trial(whole_problem))); + whole.finish(0.01); + state.set_initial(initial, {}); + static_cast(state.trial_step(0.01, trial(problem))); + state.finish(0.01); + const auto reference = whole.field_to_host(); + const auto chunked = state.field_to_host(); + const auto dot_length = static_cast(problem.n_nodes); + const double dot_gamma = dot_length * kEpsilon / (1.0 - dot_length * kEpsilon); + const double tolerance = + 512.0 * kEpsilon * std::ceil(std::log2(static_cast(problem.n_to))) + + 4.0 * dot_gamma; + for (std::size_t index = 0; index < chunked.size(); ++index) + expect_complex_near(chunked[index], reference[index], + tolerance * std::max(1.0, Kokkos::abs(reference[index]))); } -#if defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) +#ifdef SOLPROP_NATIVE_TPL_BLAS_OPENBLAS TEST(GeneralState, OpenBlasEnvelopePolicyAllocatesAndExecutesComplexTransforms) { auto problem = multimode_problem(8); problem.family = native::FieldFamily::envelope; @@ -1649,10 +1645,11 @@ PlanarElement planar_element(const native::Problem& problem, const std::size_t c const double attenuation = propagating ? root.imag() : std::min(std::abs(root.imag()), problem.spatial_evanescent_rate); - return {Complex(-attenuation, -(root.real() - problem.spatial_frame[omega])) - + return {.linear = + Complex(-attenuation, -(root.real() - problem.spatial_frame[omega])) - 0.5 * problem.spatial_spectral_rate[omega], - propagating ? problem.spatial_numerator[omega] / root.real() - : problem.spatial_clamped[omega]}; + .prefactor = propagating ? problem.spatial_numerator[omega] / root.real() + : problem.spatial_clamped[omega]}; } // The stage straight from the definitions: the inverse DFT in y, each row's real time @@ -1660,7 +1657,9 @@ PlanarElement planar_element(const native::Problem& problem, const std::size_t c // and each column's prefactor. std::vector planar_reference(const native::Problem& problem, const std::vector& columns) { - const auto n_p = problem.n_p, n_x = problem.n_x, n_y = problem.n_y; + const auto n_p = problem.n_p; + const auto n_x = problem.n_x; + const auto n_y = problem.n_y; const auto n = problem.n_omega; std::vector> rows(n_p * n_y); for (std::size_t iy = 0; iy < n_y; ++iy) @@ -1848,7 +1847,7 @@ std::vector filter_tolerance(const std::vector& row, std::abs(row[static_cast(2 * last - index)]); return std::abs(row[static_cast(index)]); }; - const double operations = static_cast(2 * half + 3); + const auto operations = static_cast(2 * half + 3); const double gamma = operations * kEpsilon / (1.0 - operations * kEpsilon); std::vector tolerance(row.size()); for (std::ptrdiff_t index = 0; index < samples; ++index) { @@ -1890,7 +1889,8 @@ team_band_scan(const std::vector& values, const std::size_t rows, Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, filtered); const auto host_prefixes = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, prefixes); - std::vector filtered_values(rows * samples), prefix_values(rows * samples); + std::vector filtered_values(rows * samples); + std::vector prefix_values(rows * samples); for (std::size_t row = 0; row < rows; ++row) for (std::size_t sample = 0; sample < samples; ++sample) { filtered_values[row * samples + sample] = host_filtered(row, sample); @@ -2311,8 +2311,11 @@ WindowedScan windowed_band_scan(const std::vector& row, Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, windowed); const auto host_totals = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, totals); - WindowedScan result{std::vector(samples), std::vector(samples), - std::vector(samples), window, host_totals(0)}; + WindowedScan result{.filtered = std::vector(samples), + .whole = std::vector(samples), + .windowed = std::vector(samples), + .window = window, + .total = host_totals(0)}; for (std::size_t i = 0; i < samples; ++i) { result.filtered[i] = host_filtered(0, i); result.whole[i] = host_whole(0, i); @@ -2554,7 +2557,8 @@ std::vector low_storage_reference(const native::Problem& problem, result[element] = base[element] + weight * slope[element]; return result; }; - std::vector picture(size), g1(size); + std::vector picture(size); + std::vector g1(size); const auto f1 = multimode_reference(problem, state); for (std::size_t element = 0; element < size; ++element) { picture[element] = flow[element] * state[element]; @@ -2757,7 +2761,8 @@ TEST(CpuThreadPolicy, AutomaticBoundariesAreMonotoneAndCapped) { native::test::table()[native::CpuOperation::modal].threshold, native::test::table()[native::CpuOperation::adams].full}; for (const int cap : {1, 2, 3, 8, 72}) { - native::ThreadPolicy policy{cap, cap, cap, cap}; + native::ThreadPolicy policy{ + .kernels = cap, .fft = cap, .raman_fft = cap, .raman_tail_fft = cap}; for (std::size_t index = 0; index < operations.size(); ++index) { const auto width = [&](const std::size_t elements) { return native::operation_threads(policy, native::test::table(), @@ -2786,7 +2791,8 @@ TEST(CpuThreadPolicy, AutomaticBoundariesAreMonotoneAndCapped) { } TEST(CpuThreadPolicy, ShortCopiesDoNotInheritTheStageWidth) { - native::ThreadPolicy policy{18, 18, 18, 18}; + native::ThreadPolicy policy{ + .kernels = 18, .fft = 18, .raman_fft = 18, .raman_tail_fft = 18}; EXPECT_EQ(native::operation_threads(policy, native::test::table(), native::CpuPool::kernels, native::CpuOperation::copy, 8192), @@ -2807,10 +2813,11 @@ TEST(CpuThreadPolicy, SmoothGridSizesKeepThePowerOfTwoWidths) { // stage of 8004 elements and coarse FFT batch of 32000 samples take the widths of // 8196 and 32768, rather than falling to one thread just under a power-of-two // boundary. macOS keeps the earlier boundaries, where the serial ones are faster. -#if defined(__APPLE__) +#ifdef __APPLE__ GTEST_SKIP() << "macOS keeps the 8192/32768 boundaries"; #endif - native::ThreadPolicy policy{18, 18, 18, 18}; + native::ThreadPolicy policy{ + .kernels = 18, .fft = 18, .raman_fft = 18, .raman_tail_fft = 18}; const auto width = [&](const native::CpuOperation operation, const std::size_t n) { return native::operation_threads(policy, native::test::table(), native::CpuPool::kernels, operation, n); @@ -2823,11 +2830,15 @@ TEST(CpuThreadPolicy, SmoothGridSizesKeepThePowerOfTwoWidths) { } TEST(CpuThreadPolicy, CoupledMatrixWindowRetainsFullWidth) { - native::ThreadPolicy policy{72, 72, 72, 72, -1}; + native::ThreadPolicy policy{.kernels = 72, + .fft = 72, + .raman_fft = 72, + .raman_tail_fft = 72, + .band_tiles = -1}; // Active bins * modes^3 in the measured three-mode wall, independent of runtime. - EXPECT_EQ(native::operation_threads(policy, native::test::table(), - native::CpuPool::kernels, - native::CpuOperation::flow, 1433 * 3 * 3 * 3), + EXPECT_EQ(native::operation_threads( + policy, native::test::table(), native::CpuPool::kernels, + native::CpuOperation::flow, std::size_t{1433} * 3 * 3 * 3), 72); } @@ -2835,7 +2846,8 @@ TEST(CpuThreadPolicy, ResponseShapeThresholdsAreMonotoneAndCapped) { using native::CpuOperation; using native::CpuPool; for (const int cap : {1, 3, 18, 72}) { - native::ThreadPolicy policy{cap, cap, cap, cap}; + native::ThreadPolicy policy{ + .kernels = cap, .fft = cap, .raman_fft = cap, .raman_tail_fft = cap}; const auto width = [&](const std::size_t rows, const std::size_t samples) { return native::operation_threads(policy, native::test::table(), CpuPool::kernels, CpuOperation::response, rows * samples, samples); @@ -2871,7 +2883,8 @@ TEST(CpuThreadPolicy, ResponseShapeThresholdsAreMonotoneAndCapped) { TEST(CpuThreadPolicy, ExplicitPoolsBypassSizingIndependently) { using native::CpuOperation; using native::CpuPool; - native::ThreadPolicy policy{7, 6, 5, 4}; + native::ThreadPolicy policy{ + .kernels = 7, .fft = 6, .raman_fft = 5, .raman_tail_fft = 4}; const std::array pools{CpuPool::kernels, CpuPool::fft, CpuPool::raman_fft, CpuPool::raman_tail_fft}; for (unsigned requested = 0; requested < pools.size(); ++requested) { @@ -2899,7 +2912,8 @@ TEST(CpuThreadPolicy, ExplicitPoolsBypassSizingIndependently) { } TEST(CpuThreadPolicy, GeneralModalTransformsKeepTheirMeasuredWidth) { - native::ThreadPolicy policy{18, 18, 18, 18}; + native::ThreadPolicy policy{ + .kernels = 18, .fft = 18, .raman_fft = 18, .raman_tail_fft = 18}; for (const std::size_t elements : {98496U, 209304U}) { EXPECT_EQ(native::operation_threads(policy, native::test::table(), native::CpuPool::kernels, @@ -2930,7 +2944,7 @@ TEST(CpuThreadPolicy, MetadataNamesEveryOperationClass) { } TEST(CpuThreadPolicy, ACompactTeamScopeHoldsAnOpenMpBlasAtOneThread) { -#if !defined(KOKKOS_ENABLE_OPENMP) +#ifndef KOKKOS_ENABLE_OPENMP GTEST_SKIP() << "the hold is an OpenMP build's"; #else const int before = omp_get_max_threads(); @@ -3004,12 +3018,13 @@ TEST(CpuThreadPolicy, TransformBatchesThreadByRowsAndRowLength) { if (!native::cpu_size_policy_enabled) GTEST_SKIP() << "needs automatic per-operation sizing"; native::CpuExecution execution = native::test::table(); - execution.grain[static_cast(native::CpuOperation::fft)] = {8192, - 1U << 22}; + execution.grain[static_cast(native::CpuOperation::fft)] = { + .threshold = 8192, .full = 1U << 22}; execution.fft_thread_elements = 4096; execution.fft_long_row = 16384; execution.medium_threads = 4; - const native::ThreadPolicy policy{18, 18, 18, 18}; + const native::ThreadPolicy policy{ + .kernels = 18, .fft = 18, .raman_fft = 18, .raman_tail_fft = 18}; const auto threads = [&](const std::size_t rows, const std::size_t length) { return native::operation_threads(policy, execution, native::CpuPool::fft, native::CpuOperation::fft, rows * length, length); diff --git a/native/tests/native/test_coupled_flow.cpp b/native/tests/native/test_coupled_flow.cpp index c7f2c401f..112b7a9b3 100644 --- a/native/tests/native/test_coupled_flow.cpp +++ b/native/tests/native/test_coupled_flow.cpp @@ -2,12 +2,11 @@ // // SPDX-License-Identifier: GPL-3.0-or-later +#include "execution.hpp" #include "solprop_native/core.hpp" #include "solprop_native/coupled_flow.hpp" #include "solprop_native/flow_functions.hpp" -#include "execution.hpp" - #include #include @@ -44,7 +43,7 @@ double fill(const std::size_t index, const double scale) { } // Bins 1 and 3 carry the block; 0 and 2 have none, and the action is the identity. -const std::vector kSlot{0, 1, 0, 2}; +constexpr std::array kSlot{0, 1, 0, 2}; constexpr std::size_t kActive = 2; std::size_t block(const std::size_t count) { return count * kActive * kModes * kModes; } @@ -84,7 +83,7 @@ native::Problem flow_problem(const bool residual) { problem.coupled_flow = true; problem.flow_active = kActive; problem.flow_sections = kSections; - problem.flow_slot = kSlot; + problem.flow_slot.assign(kSlot.begin(), kSlot.end()); if (residual) { problem.flow_terms = kTerms; for (std::size_t index = 0; index < block(kSections); ++index) @@ -114,21 +113,21 @@ Actions actions() { native::TrialScalars scalars(const bool residual, const std::size_t section) { native::TrialScalars result{ - std::vector((kStages + 1), kStep), - std::vector(kStages, kStep), - std::vector(kStages, 1.0), - std::vector(kStages, 1.0), - std::vector(kStages, 1.0), - {}, - {}, - {}, - {}, - {}, - {}, - section, - {}, - {}, - {}, + .phi_delta = std::vector((kStages + 1), kStep), + .phi_at = std::vector(kStages, kStep), + .prefactor = std::vector(kStages, 1.0), + .density = std::vector(kStages, 1.0), + .taper = std::vector(kStages, 1.0), + .preionisation = {}, + .raman_rates = {}, + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = section, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}, }; if (residual) for (std::size_t index = 0; index < kStages * kTerms; ++index) @@ -138,6 +137,7 @@ native::TrialScalars scalars(const bool residual, const std::size_t section) { std::vector initial() { std::vector field; + field.reserve(kModes * kOmega); for (std::size_t index = 0; index < kModes * kOmega; ++index) field.emplace_back(fill(index + 41, 1.0), fill(index + 43, 1.0)); return field; @@ -578,7 +578,7 @@ native::Problem forming_problem(const bool fallback) { } // Distinct lengths, a zero one and a repeated whole one. -const std::vector kFractions{0.0, 0.5, 1.0, 1.0}; +std::vector fractions() { return {0.0, 0.5, 1.0, 1.0}; } constexpr double kLength = 0.02; // |l lambda| up to 1.9: the squarings are exercised struct Formed { @@ -589,7 +589,7 @@ struct Formed { Formed form(const native::Problem& problem, const std::size_t section) { native::GeneralState state(problem); Formed result; - result.measured = state.form_flow_actions(section, kLength, kFractions); + result.measured = state.form_flow_actions(section, kLength, fractions()); result.forward = state.flow_actions_to_host(false); result.inverse = state.flow_actions_to_host(true); return result; @@ -620,7 +620,7 @@ TEST(CoupledFlowFormation, BothRoutesFormTheSameActions) { for (const std::size_t section : {std::size_t{0}, std::size_t{1}}) { const auto eigen = form(forming_problem(false), section); const auto squared = form(forming_problem(true), section); - for (std::size_t slot = 1; slot < kFractions.size(); ++slot) + for (std::size_t slot = 1; slot < fractions().size(); ++slot) for (std::size_t bin = 0; bin < kActive; ++bin) { const auto forward = block_at(eigen.forward, slot, bin); const auto inverse = block_at(eigen.inverse, slot, bin); @@ -673,7 +673,7 @@ TEST(CoupledFlowFormation, FormedActionsCarryATrialAsLoadedOnesDo) { const auto trial = scalars(false, 1); native::GeneralState state(problem); state.set_initial(initial(), {}); - static_cast(state.form_flow_actions(1, kLength, kFractions)); + static_cast(state.form_flow_actions(1, kLength, fractions())); static_cast(state.trial_step(kStep, trial)); state.finish(kStep); // The same trial on the same actions, formed here and loaded there: rounding only. @@ -685,16 +685,16 @@ TEST(CoupledFlowFormation, FormedActionsCarryATrialAsLoadedOnesDo) { TEST(CoupledFlowFormation, FormationAndItsDataAreChecked) { native::GeneralState plain(flow_problem(false)); - EXPECT_THROW(static_cast(plain.form_flow_actions(0, kLength, kFractions)), + EXPECT_THROW(static_cast(plain.form_flow_actions(0, kLength, fractions())), std::logic_error); EXPECT_THROW(static_cast(plain.flow_actions_to_host(false)), std::logic_error); native::GeneralState state(forming_problem(false)); EXPECT_THROW( - static_cast(state.form_flow_actions(kSections, kLength, kFractions)), + static_cast(state.form_flow_actions(kSections, kLength, fractions())), std::invalid_argument); EXPECT_THROW(static_cast(state.form_flow_actions(0, kLength, {0.0, 1.0})), std::invalid_argument); - EXPECT_THROW(static_cast(state.form_flow_actions(0, -kLength, kFractions)), + EXPECT_THROW(static_cast(state.form_flow_actions(0, -kLength, fractions())), std::invalid_argument); EXPECT_THROW( static_cast(state.form_flow_actions(0, kLength, {0.0, 0.5, 2.0, 1.0})), diff --git a/native/tests/native/test_crossing.cpp b/native/tests/native/test_crossing.cpp index 46e92eae5..ee9b3f998 100644 --- a/native/tests/native/test_crossing.cpp +++ b/native/tests/native/test_crossing.cpp @@ -31,7 +31,8 @@ class RingSizes : public ::testing::TestWithParam {}; TEST_P(RingSizes, CrossesEveryByteBothWays) { const auto source = counting(GetParam()); - std::vector down(source.size(), 0), up(source.size(), 0); + std::vector down(source.size(), 0); + std::vector up(source.size(), 0); Ring ring; ring.to_host(down.data(), source.data(), source.size()); ring.from_host(up.data(), down.data(), down.size()); diff --git a/native/tests/native/test_dispersive.cpp b/native/tests/native/test_dispersive.cpp index d024df28a..6b07a3f3a 100644 --- a/native/tests/native/test_dispersive.cpp +++ b/native/tests/native/test_dispersive.cpp @@ -2,13 +2,11 @@ // // SPDX-License-Identifier: GPL-3.0-or-later +#include "execution.hpp" #include "solprop_native/core.hpp" #include "solprop_native/dispersive.hpp" -#include "execution.hpp" - #include - #include #include @@ -53,7 +51,8 @@ TEST(DispersiveBodies, TheContractionIsTheIsotropicTensorSum) { const Complex z[2] = {{0.9, 0.1}, {-0.2, 0.8}}; Complex expected[2]; tensor_sum(x, y, z, expected); - Complex out0, out1; + Complex out0; + Complex out1; native::dispersive_isotropic(x[0], x[1], y[0], y[1], z[0], z[1], out0, out1); EXPECT_LT(distance(out0, expected[0]), 16 * kEpsilon); EXPECT_LT(distance(out1, expected[1]), 16 * kEpsilon); @@ -71,7 +70,9 @@ TEST(DispersiveBodies, TheScalarClosedFormsAreTheGeneralGroups) { const Complex f{-1.2, 0.35}; const Complex carrier{std::cos(0.7), std::sin(0.7)}; const Complex zero{}; - Complex general, unused, harmonic; + Complex general; + Complex unused; + Complex harmonic; native::dispersive_conserving(u, zero, u, zero, u, zero, general, unused); EXPECT_LT(distance(native::dispersive_conserving_self(u), general), 16 * kEpsilon); native::dispersive_conserving(u, zero, u, zero, f, zero, general, unused); @@ -98,8 +99,10 @@ TEST(DispersiveBodies, TheScalarClosedFormsAreTheGeneralGroups) { TEST(DispersiveBodies, TheConservingGroupIsTheQuartetsWithTwoPositiveLegs) { // `|u|^2 u` contracted isotropically: `(u . u*) u` twice and `(u . u) u*` once, the // number-conserving group of `(u, u, u)` (SEP 6 Phase 0). - const Complex u0{0.4, 0.9}, u1{-0.7, 0.2}; - Complex out0, out1; + const Complex u0{0.4, 0.9}; + const Complex u1{-0.7, 0.2}; + Complex out0; + Complex out1; native::dispersive_conserving(u0, u1, u0, u1, u0, u1, out0, out1); const Complex square = u0 * u0 + u1 * u1; const double modulus = @@ -140,10 +143,13 @@ TEST(DispersiveBodies, TheThreeComponentContractionIsTheTensorSum) { native::dispersive_isotropic_n(x, y, z, out); for (int c = 0; c < 3; ++c) EXPECT_LT(distance(out[c], expected[c]), 32 * kEpsilon) << c; - const Complex x2[2] = {x[0], x[1]}, y2[2] = {y[0], y[1]}, z2[2] = {z[0], z[1]}; + const Complex x2[2] = {x[0], x[1]}; + const Complex y2[2] = {y[0], y[1]}; + const Complex z2[2] = {z[0], z[1]}; Complex two[2]; native::dispersive_isotropic_n(x2, y2, z2, two); - Complex out0, out1; + Complex out0; + Complex out1; native::dispersive_isotropic(x[0], x[1], y[0], y[1], z[0], z[1], out0, out1); // Each body is within 16 eps of the tensor sum (the test above), so 32 eps of the // other. @@ -171,13 +177,18 @@ TEST(DispersiveBodies, TheDistinctLegGroupsAreTheirTensorSums) { const Complex x[3] = {{0.3, -1.1}, {0.7, 0.2}, {-0.5, 0.4}}; const Complex y[3] = {{-0.4, 0.5}, {1.3, -0.6}, {0.2, 0.9}}; const Complex z[3] = {{0.9, 0.1}, {-0.2, 0.8}, {0.6, -0.3}}; - Complex xc[3], yc[3], zc[3]; + Complex xc[3]; + Complex yc[3]; + Complex zc[3]; for (int c = 0; c < 3; ++c) { xc[c] = Kokkos::conj(x[c]); yc[c] = Kokkos::conj(y[c]); zc[c] = Kokkos::conj(z[c]); } - Complex a[3], b[3], d[3], out[3]; + Complex a[3]; + Complex b[3]; + Complex d[3]; + Complex out[3]; tensor_sum3(x, y, zc, a); tensor_sum3(x, yc, z, b); tensor_sum3(xc, y, z, d); @@ -185,10 +196,18 @@ TEST(DispersiveBodies, TheDistinctLegGroupsAreTheirTensorSums) { for (int c = 0; c < 3; ++c) EXPECT_LT(distance(out[c], a[c] + b[c] + d[c]), 3 * 32 * kEpsilon) << c; - const Complex x2[2] = {x[0], x[1]}, y2[2] = {y[0], y[1]}, z2[2] = {z[0], z[1]}; - const Complex xc2[2] = {xc[0], xc[1]}, yc2[2] = {yc[0], yc[1]}, - zc2[2] = {zc[0], zc[1]}; - Complex p[2], q[2], r[2], u[2], out0, out1; + const Complex x2[2] = {x[0], x[1]}; + const Complex y2[2] = {y[0], y[1]}; + const Complex z2[2] = {z[0], z[1]}; + const Complex xc2[2] = {xc[0], xc[1]}; + const Complex yc2[2] = {yc[0], yc[1]}; + const Complex zc2[2] = {zc[0], zc[1]}; + Complex p[2]; + Complex q[2]; + Complex r[2]; + Complex u[2]; + Complex out0; + Complex out1; tensor_sum(x2, y2, zc2, p); tensor_sum(x2, yc2, z2, q); tensor_sum(xc2, y2, z2, r); diff --git a/native/tests/native/test_envelope_plasma.cpp b/native/tests/native/test_envelope_plasma.cpp index e6aee32e6..ceebd3acd 100644 --- a/native/tests/native/test_envelope_plasma.cpp +++ b/native/tests/native/test_envelope_plasma.cpp @@ -26,8 +26,8 @@ struct PolynomialRate { }; std::array polynomial_rules(const bool inconsistent = false) { - RuleView nodes("envelope_test.nodes", 4, 512), - weights("envelope_test.weights", 4, 512); + RuleView nodes("envelope_test.nodes", 4, 512); + RuleView weights("envelope_test.weights", 4, 512); auto host_nodes = Kokkos::create_mirror_view(nodes); auto host_weights = Kokkos::create_mirror_view(weights); for (std::size_t level = 0; level < 4; ++level) { @@ -81,8 +81,10 @@ void polynomial_moments_and_directional_work() { // Two ordered sums of at most 128 terms, plus bounded elementary arithmetic. constexpr double tolerance = 1024 * std::numeric_limits::epsilon(); for (std::size_t index = 0; index < ratios.size(); ++index) { - const double ratio = ratios[index], square = ratio * ratio; - const double mc = 2.0 * (3.0 + square), ms = 2.0 * (1.0 + 3.0 * square); + const double ratio = ratios[index]; + const double square = ratio * ratio; + const double mc = 2.0 * (3.0 + square); + const double ms = 2.0 * (1.0 + 3.0 * square); EXPECT_NEAR(actual(index, 0), 2.0, tolerance); EXPECT_NEAR(actual(index, 1), ratio, tolerance); EXPECT_NEAR(actual(index, 2), mc, tolerance * mc); @@ -97,7 +99,8 @@ void polynomial_moments_and_directional_work() { } void phase_and_real_rotation() { - constexpr double phase = 0.37, angle = 0.61; + constexpr double phase = 0.37; + constexpr double angle = 0.61; const Complex common(std::cos(phase), std::sin(phase)); const Complex first = common * Complex(2.0 * std::cos(angle), -0.7 * std::sin(angle)); const Complex second = common * Complex(2.0 * std::sin(angle), 0.7 * std::cos(angle)); @@ -180,7 +183,8 @@ void scaled_magnitude_preserves_tiny_and_huge_components() { "envelope_test.scaled", Kokkos::RangePolicy(native::ExecutionSpace{}, 0, count), KOKKOS_LAMBDA(const std::size_t index) { - const double first = pairs(index, 0), second = pairs(index, 1); + const double first = pairs(index, 0); + const double second = pairs(index, 1); const auto one = native::envelope_axes(Complex(first, second), Complex{}); const auto two = native::envelope_axes(Complex(first, 0.0), Complex(second, 0.0)); diff --git a/native/tests/native/test_fft.cpp b/native/tests/native/test_fft.cpp index 67d59ff5d..a78924958 100644 --- a/native/tests/native/test_fft.cpp +++ b/native/tests/native/test_fft.cpp @@ -104,7 +104,10 @@ TEST(FftProvider, RealAndComplexBatchesPreserveSignAndNormalization) { } TEST(FftProvider, StackedTailPlansLeaveEveryBlocksPaddingUntouched) { - constexpr std::size_t length = 32, rows = 7, active = 3, blocks = 3; + constexpr std::size_t length = 32; + constexpr std::size_t rows = 7; + constexpr std::size_t active = 3; + constexpr std::size_t blocks = 3; const RealRows input("fft_contract.stacked_input", blocks * rows, length); const ComplexRows output("fft_contract.stacked_output", blocks * rows, length / 2 + 1); @@ -421,7 +424,7 @@ TEST(RamanFftProvider, PreservesRawNormalizationInputAndPartialRows) { row < tail ? fft_bound(length, length) : 0.0); } } -#if defined(SOLPROP_NATIVE_RAMAN_ACCELERATE) +#ifdef SOLPROP_NATIVE_RAMAN_ACCELERATE const bool neon = std::string_view(fftw_version).find("-neon") != std::string_view::npos; const bool accelerated = length != 15 && (neon ? threads > 1 && length <= 16384 diff --git a/native/tests/native/test_general_modal.cpp b/native/tests/native/test_general_modal.cpp index 6515fc636..6bb2d7f0f 100644 --- a/native/tests/native/test_general_modal.cpp +++ b/native/tests/native/test_general_modal.cpp @@ -2,13 +2,11 @@ // // SPDX-License-Identifier: GPL-3.0-or-later +#include "execution.hpp" #include "solprop_native/core.hpp" #include "solprop_native/general_modal.hpp" -#include "execution.hpp" - #include - #include #include @@ -123,6 +121,7 @@ native::Problem modal_problem(const bool real_form, const std::size_t chunk, std::vector scaled_state(const native::Problem& problem) { std::vector state; + state.reserve(kModes * problem.n_omega); for (std::size_t index = 0; index < kModes * problem.n_omega; ++index) state.emplace_back(fill(index + 17, 1.0), fill(index + 23, 1.0)); return state; @@ -494,8 +493,21 @@ TEST(GeneralModal, TheStateCarriesTheSourceAndItsLedger) { for (std::size_t index = 0; index < field.size(); ++index) field[index] = Complex(fill(index, 1.0), fill(index + 1, 1.0)); state.set_initial(field, {}); - const native::TrialScalars scalars{{0.0, 0.0}, {0.0}, {1.0}, {1.0}, {1.0}, {}, {}, {}, - {}, {}, {}, 0, {}, {}, {}}; + const native::TrialScalars scalars{.phi_delta = {0.0, 0.0}, + .phi_at = {0.0}, + .prefactor = {1.0}, + .density = {1.0}, + .taper = {1.0}, + .preionisation = {}, + .raman_rates = {}, + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = 0, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}}; const auto first = state.trial_step(0.01, scalars); const auto stage = state.stage_to_host(0); const double magnitude = std::accumulate( @@ -667,12 +679,25 @@ TEST(GeneralModal, TheAdamsRowsNeedTheHistory) { TEST(GeneralModal, ALinearGuideLeavesTheStageZero) { auto problem = modal_problem(true, 2, false); - std::fill(problem.general_tensor.begin(), problem.general_tensor.end(), 0.0); + std::ranges::fill(problem.general_tensor, 0.0); native::GeneralState state(problem); std::vector field(kModes * problem.n_omega, Complex(1.0, 0.5)); state.set_initial(field, {}); - const native::TrialScalars scalars{{0.0, 0.0}, {0.0}, {1.0}, {1.0}, {1.0}, {}, {}, {}, - {}, {}, {}, 0, {}, {}, {}}; + const native::TrialScalars scalars{.phi_delta = {0.0, 0.0}, + .phi_at = {0.0}, + .prefactor = {1.0}, + .density = {1.0}, + .taper = {1.0}, + .preionisation = {}, + .raman_rates = {}, + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = 0, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}}; static_cast(state.trial_step(0.01, scalars)); // Exact: a zero tensor contracts to exact zeros. for (const auto value : state.stage_to_host(0)) { @@ -914,8 +939,21 @@ TEST(GeneralModal, TheStateCombinesAMovingGuideSMapsPerTrial) { for (std::size_t index = 0; index < field.size(); ++index) field[index] = Complex(fill(index, 1.0), fill(index + 1, 1.0)); state.set_initial(field, {}); - native::TrialScalars scalars{{0.0, 0.0}, {0.0}, {1.0}, {1.0}, {1.0}, {}, {}, {}, - {}, {}, {}, 0, {}, {}, {}}; + native::TrialScalars scalars{.phi_delta = {0.0, 0.0}, + .phi_at = {0.0}, + .prefactor = {1.0}, + .density = {1.0}, + .taper = {1.0}, + .preionisation = {}, + .raman_rates = {}, + .kerr_density = {}, + .plasma_densities = {}, + .plasma_preionisation = {}, + .raman_densities = {}, + .flow_section = 0, + .flow_weights = {}, + .transform_weights = {}, + .dispersive_densities = {}}; EXPECT_THROW(static_cast(state.trial_step(0.01, scalars)), std::invalid_argument); scalars.transform_weights.assign(kMapTerms, 0.0); diff --git a/native/tests/native/test_helpers.cpp b/native/tests/native/test_helpers.cpp index 6ca2b0787..f9a012cee 100644 --- a/native/tests/native/test_helpers.cpp +++ b/native/tests/native/test_helpers.cpp @@ -159,7 +159,7 @@ TEST(LagrangeBasis, TrialTablesOnEqualSteps) { EXPECT_EQ(corrector[1], 0.5); EXPECT_EQ(corrector[2], 0.5); for (std::size_t column = 0; column < 4; ++column) { - EXPECT_EQ(corrector[3 * 4 + column], 0.0); // past its nodes + EXPECT_EQ(corrector[std::size_t{3} * 4 + column], 0.0); // past its nodes EXPECT_EQ(corrector[column * 4 + 3], 0.0); } const std::array first_order{1.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0}; diff --git a/native/tests/native/test_low_storage.cpp b/native/tests/native/test_low_storage.cpp index 5a91718d9..302386208 100644 --- a/native/tests/native/test_low_storage.cpp +++ b/native/tests/native/test_low_storage.cpp @@ -10,6 +10,9 @@ // flight, by a failure or a divergence. #include "solprop_native/low_storage.hpp" +#include "solprop_native/session.hpp" + +#include #include #include @@ -19,10 +22,6 @@ #include #include -#include - -#include "solprop_native/session.hpp" - namespace { namespace low_storage = solprop_native::rk::low_storage; namespace march = solprop_native::march; @@ -134,7 +133,7 @@ TEST(LowStorage, EachPositionIsItsOwnAnchorAndTheEndPadsTheStages) { const double dz = 0.0625; const double midpoint = z + 0.5 * dz; const double end = z + dz; - const low_storage::Config recompute{0, false, 0.0}; + const low_storage::Config recompute{.flows = 0, .separable = false, .origin = 0.0}; const auto scalars = low_storage::scalars(tables, anchor, recompute, z, dz, false, std::nan("")); EXPECT_EQ(scalars.phi_at, (std::vector{z, midpoint, end, end})); @@ -154,7 +153,7 @@ TEST(LowStorage, AConstantOperatorsTableIsBuiltFromItsOrigin) { const double z = 0.375; const double dz = 0.1; const double key = 0.09999999999999998; - const low_storage::Config table{1, false, 0.25}; + const low_storage::Config table{.flows = 1, .separable = false, .origin = 0.25}; const auto built = low_storage::scalars(tables, anchor, table, z, dz, true, key); // fl(origin + key / 2) - origin, not key / 2: solprop's engines build it so. const double middle = 0.25 + 0.5 * key; @@ -162,7 +161,7 @@ TEST(LowStorage, AConstantOperatorsTableIsBuiltFromItsOrigin) { const auto reused = low_storage::scalars(tables, anchor, table, z, dz, false, key); EXPECT_EQ(reused.phi_delta, (std::vector(5, 0.0))); // A separable operator's increments are the distances themselves. - const low_storage::Config separable{1, true, 0.25}; + const low_storage::Config separable{.flows = 1, .separable = true, .origin = 0.25}; const auto spatial = low_storage::scalars(tables, anchor, separable, z, dz, true, key); EXPECT_EQ(spatial.phi_delta, @@ -188,7 +187,9 @@ TEST(LowStorage, AnExpansionsIncrementsAreItsWeightsDifferences) { const double dz = 0.2; const double midpoint = z + 0.5 * dz; const auto scalars = low_storage::scalars( - tables, anchor, low_storage::Config{2, false, 0.0}, z, dz, false, std::nan("")); + tables, anchor, + low_storage::Config{.flows = 2, .separable = false, .origin = 0.0}, z, dz, false, + std::nan("")); stage::Anchor fresh; const auto first = stage::row(tables, fresh, z, midpoint).phi_delta; const auto second = stage::row(tables, fresh, midpoint, z + dz).phi_delta; diff --git a/native/tests/native/test_rk_march.cpp b/native/tests/native/test_rk_march.cpp index 937b2070f..e5ce98eea 100644 --- a/native/tests/native/test_rk_march.cpp +++ b/native/tests/native/test_rk_march.cpp @@ -8,12 +8,12 @@ #include "solprop_native/rk_march.hpp" +#include + #include #include #include -#include - namespace { namespace rk = solprop_native::rk; namespace march = solprop_native::march; diff --git a/native/tests/native/test_session.cpp b/native/tests/native/test_session.cpp index 4d89b162a..66de519ba 100644 --- a/native/tests/native/test_session.cpp +++ b/native/tests/native/test_session.cpp @@ -11,6 +11,8 @@ #include "solprop_native/session.hpp" +#include + #include #include #include @@ -23,8 +25,6 @@ #include #include -#include - namespace { namespace session = solprop_native::session; namespace march = solprop_native::march; @@ -78,6 +78,7 @@ class ToyDense final : public session::Dense { void stepped(const march::Result& /*result*/) override {} [[nodiscard]] bool crosses(const double position, const double target, const int ulps) const override { + // NOLINTNEXTLINE(readability-suspicious-call-argument): solprop's reached(target, z) return !march::reached(target, position, ulps); } void observe(const double position) override { @@ -258,17 +259,17 @@ std::vector drive(session::Session& run, const std::size_t work_limit, run.acknowledge(batch.sequence); continue; } - Seen entry{event.kind, - event.observation, - event.z, - event.plane_index, - std::move(held.attempts), - std::move(held.row_z), - std::move(held.row_step), - std::move(held.rows), - event.has_stats ? batch.plane_stats : std::vector{}, - {}, - event.outcome}; + Seen entry{.kind = event.kind, + .observation = event.observation, + .z = event.z, + .plane = event.plane_index, + .attempts = std::move(held.attempts), + .row_z = std::move(held.row_z), + .row_step = std::move(held.row_step), + .rows = std::move(held.rows), + .stats = event.has_stats ? batch.plane_stats : std::vector{}, + .field = {}, + .outcome = event.outcome}; held = Held{}; if (event.ticket) { entry.field.resize(2 * run.elements()); @@ -491,7 +492,7 @@ TEST(Session, StatusAndControlsNeedNoLock) { std::atomic gate{false}; built.engine->hold(&entered, &gate); session::Batch batch; - std::thread advancing([&]() { batch = run.advance(256, 10.0); }); + std::thread advancing([&] { batch = run.advance(256, 10.0); }); while (!entered.load()) std::this_thread::yield(); const auto status = run.status(); diff --git a/native/tests/native/test_session_monitor.cpp b/native/tests/native/test_session_monitor.cpp index a5479f5c7..6f222d670 100644 --- a/native/tests/native/test_session_monitor.cpp +++ b/native/tests/native/test_session_monitor.cpp @@ -13,14 +13,14 @@ #include "solprop_native/session.hpp" +#include + #include #include #include #include #include -#include - namespace { namespace session = solprop_native::session; namespace rk = solprop_native::rk; @@ -149,6 +149,7 @@ session::Batch next(session::Session& run, const session::EventKind kind, // The rows' accepted-step values a batch carries. std::vector rows_of(const session::Batch& batch) { std::vector steps; + steps.reserve(batch.row_z.size()); for (std::size_t row = 0; row < batch.row_z.size(); ++row) steps.push_back(batch.row_values[2 * row + 1]); return steps; @@ -193,8 +194,8 @@ TEST(SessionMonitor, AWarningIsABarrierBeforeItsStepsRow) { ASSERT_FALSE(rows.empty()); for (std::size_t index = 0; index < rows.size(); ++index) EXPECT_EQ(rows[index], 3.0 + static_cast(index)); - EXPECT_EQ(rows.back(), - static_cast(3 + plane.attempts.size() / session::attempt_columns)); + const std::size_t attempts = plane.attempts.size() / session::attempt_columns; + EXPECT_EQ(rows.back(), static_cast(3 + attempts)); next(run, session::EventKind::observation); next(run, session::EventKind::terminal); const auto record = run.monitor_record(); diff --git a/native/tests/native/test_session_transverse.cpp b/native/tests/native/test_session_transverse.cpp index 47d466f54..4d41f40b0 100644 --- a/native/tests/native/test_session_transverse.cpp +++ b/native/tests/native/test_session_transverse.cpp @@ -14,6 +14,8 @@ #include "solprop_native/session.hpp" +#include + #include #include #include @@ -21,8 +23,6 @@ #include #include -#include - namespace { namespace session = solprop_native::session; namespace rk = solprop_native::rk; diff --git a/native/tests/native/test_transverse.cpp b/native/tests/native/test_transverse.cpp index 578c56b2b..5b5afd167 100644 --- a/native/tests/native/test_transverse.cpp +++ b/native/tests/native/test_transverse.cpp @@ -11,14 +11,14 @@ #include "solprop_native/transverse.hpp" +#include + #include #include #include #include #include -#include - namespace { namespace transverse = solprop_native::transverse; using Kind = transverse::Notice::Kind; diff --git a/native/tests/native/test_vmath.cpp b/native/tests/native/test_vmath.cpp index 4e12931ef..c428d79c1 100644 --- a/native/tests/native/test_vmath.cpp +++ b/native/tests/native/test_vmath.cpp @@ -61,7 +61,12 @@ TEST(VectorMath, EveryElementOfABlockIsItsOwnValue) { for (const std::size_t n : lengths()) { const auto x = uniform(n, -30.0, 5.0, 11U + static_cast(n)); const auto angle = uniform(n, -400.0, 400.0, 29U + static_cast(n)); - std::vector e(n), l(n), s(n), c(n), re(n), im(n); + std::vector e(n); + std::vector l(n); + std::vector s(n); + std::vector c(n); + std::vector re(n); + std::vector im(n); std::vector positive(n); for (std::size_t i = 0; i < n; ++i) positive[i] = std::exp(x[i]); @@ -70,8 +75,12 @@ TEST(VectorMath, EveryElementOfABlockIsItsOwnValue) { vmath::sincos(angle.data(), s.data(), c.data(), n); vmath::exp_complex(x.data(), angle.data(), re.data(), im.data(), n); for (std::size_t i = 0; i < n; ++i) { - double one_e = 0.0, one_l = 0.0, one_s = 0.0, one_c = 0.0, one_re = 0.0, - one_im = 0.0; + double one_e = 0.0; + double one_l = 0.0; + double one_s = 0.0; + double one_c = 0.0; + double one_re = 0.0; + double one_im = 0.0; vmath::exp(&x[i], &one_e, 1); vmath::log(&positive[i], &one_l, 1); vmath::sincos(&angle[i], &one_s, &one_c, 1); @@ -94,7 +103,8 @@ TEST(VectorMath, InPlaceGivesTheSameValues) { auto in_place = x; vmath::exp(in_place.data(), in_place.data(), n); EXPECT_EQ(in_place, out); - std::vector s(n), c(n); + std::vector s(n); + std::vector c(n); vmath::sincos(x.data(), s.data(), c.data(), n); auto sine = x; vmath::sincos(sine.data(), sine.data(), c.data(), n); @@ -112,7 +122,12 @@ TEST(VectorMath, AgreesWithTheSystemLibm) { const auto magnitude_exponent = uniform(n, -300.0, 300.0, 13U); const auto phase = uniform(n, -1.0e4, 1.0e4, 17U); const auto huge_phase = uniform(n, -1.0e15, 1.0e15, 19U); - std::vector out(n), s(n), c(n), re(n), im(n), magnitude(n); + std::vector out(n); + std::vector s(n); + std::vector c(n); + std::vector re(n); + std::vector im(n); + std::vector magnitude(n); for (const auto* values : {&exponent, &small}) { vmath::exp(values->data(), out.data(), n); for (std::size_t i = 0; i < n; ++i) @@ -179,7 +194,8 @@ TEST(VectorMath, SpecialValuesFollowC99) { expect_within_two_ulps(l[6], std::log(std::numeric_limits::denorm_min())); const std::vector angle = {0.0, -0.0, infinity, -infinity, quiet_nan}; - std::vector s(angle.size()), c(angle.size()); + std::vector s(angle.size()); + std::vector c(angle.size()); vmath::sincos(angle.data(), s.data(), c.data(), angle.size()); EXPECT_EQ(s[0], 0.0); EXPECT_FALSE(std::signbit(s[0])); @@ -196,10 +212,12 @@ TEST(VectorMath, SpecialValuesFollowC99) { // NaN in either part makes both parts NaN. const std::vector a = {0.0, -0.0, quiet_nan, 0.0}; const std::vector b = {0.5, -0.5, 0.5, quiet_nan}; - std::vector re(a.size()), im(a.size()); + std::vector re(a.size()); + std::vector im(a.size()); vmath::exp_complex(a.data(), b.data(), re.data(), im.data(), a.size()); for (std::size_t i = 0; i < 2; ++i) { - double sine = 0.0, cosine = 0.0; + double sine = 0.0; + double cosine = 0.0; vmath::sincos(&b[i], &sine, &cosine, 1); EXPECT_EQ(re[i], cosine); EXPECT_EQ(im[i], sine); @@ -211,7 +229,7 @@ TEST(VectorMath, SpecialValuesFollowC99) { } TEST(VectorMath, RouteMatchesTheBuild) { -#if defined(SOLPROP_NATIVE_VECTOR_MATH_SLEEF) +#ifdef SOLPROP_NATIVE_VECTOR_MATH_SLEEF EXPECT_TRUE(vmath::vectorised); EXPECT_GE(vmath::width, 2U); #else diff --git a/native/tests/native/test_watch.cpp b/native/tests/native/test_watch.cpp index 92bb6916f..5bffa8e72 100644 --- a/native/tests/native/test_watch.cpp +++ b/native/tests/native/test_watch.cpp @@ -9,14 +9,14 @@ #include "solprop_native/watch.hpp" +#include + #include #include #include #include #include -#include - namespace { namespace watch = solprop_native::watch; diff --git a/native/tests/native/test_z_table.cpp b/native/tests/native/test_z_table.cpp index 324e8bb5d..7bd5ad8d2 100644 --- a/native/tests/native/test_z_table.cpp +++ b/native/tests/native/test_z_table.cpp @@ -4,6 +4,7 @@ #include "solprop_native/chebyshev.hpp" #include "solprop_native/core.hpp" +#include "solprop_native/stage_tables.hpp" #include "solprop_native/z_table.hpp" #include @@ -15,6 +16,7 @@ #include #include #include +#include #include namespace { @@ -199,4 +201,31 @@ TEST(ZTable, BlockedEvaluationIsThePerValueRecursionExactly) { } } +// A scalar stage table holds one value: the loader refuses one of two by name, before +// any read could write past the single value it returns. +TEST(StageTable, LoaderRefusesAMultiValuedScalarTableByName) { + native::stage::Table table; + table.edges = {0.0, 1.0}; + table.lower = 0.0; + table.upper = 1.0; + table.n_terms = 1; + table.n_values = 1; + table.coefficients = {2.5}; + native::stage::Tables tables; + tables.density = table; + EXPECT_NO_THROW(native::stage::require_scalar_tables(tables)); + EXPECT_EQ(tables.density.value(0.5), 2.5); + auto wide = table; + wide.n_values = 2; + wide.coefficients = {2.5, -1.0}; + tables.plasma_densities = {table, wide}; + try { + native::stage::require_scalar_tables(tables); + FAIL() << "a two-value plasma density table was accepted"; + } catch (const std::invalid_argument& error) { + EXPECT_NE(std::string(error.what()).find("plasma_densities[1]"), std::string::npos) + << error.what(); + } +} + } // namespace diff --git a/native/tests/test_api_contract.py b/native/tests/test_api_contract.py index 9f81821a8..3e4d3753a 100644 --- a/native/tests/test_api_contract.py +++ b/native/tests/test_api_contract.py @@ -120,12 +120,7 @@ def test_build_info_is_frozen_and_complete() -> None: assert info.kokkos_version == "5.2.1" assert info.kokkos_fft_version == "2.0.0" assert info.kokkos_kernels_version == "5.2.0" - assert info.gemm_provider in { - "CUSTOM", - "KOKKOSBLAS_NATIVE", - "KOKKOSBLAS_TPL", - "HYBRID", - } + assert info.gemm_provider == "HYBRID" if info.tpl_blas_enabled: assert info.blas_backend != "none" else: @@ -186,13 +181,9 @@ def test_build_info_is_frozen_and_complete() -> None: assert info.cpu_baseline in info.compiler_flags assert info.cpu_baseline in info.optimization_policy if info.device_backend == "none": - if info.openmp_enabled: - assert info.execution_space == "OpenMP" - assert info.fft_backend in {"FFTW_OPENMP", "FFTW_PTHREADS", "MKL_FFTW"} - else: - # The Kokkos Threads build adds no OpenMP runtime (solprop SEP 29). - assert info.execution_space == "Threads" - assert info.fft_backend == "FFTW_THREADS" + assert info.openmp_enabled + assert info.execution_space == "OpenMP" + assert info.fft_backend in {"FFTW_OPENMP", "FFTW_PTHREADS", "MKL_FFTW"} for count in ( info.thread_count, info.fftw_threads, @@ -745,15 +736,6 @@ def run() -> ComplexArray: def run_ignored(_: int) -> ComplexArray: return run() - if native.build_info().execution_space == "Threads": - # Kokkos Threads dispatches only from the thread that started it: another - # Python thread gets a RuntimeError, never a crash (solprop SEP 29 phase 0). - with ( - ThreadPoolExecutor(max_workers=1) as executor, - pytest.raises(RuntimeError, match="master"), - ): - executor.submit(run).result() - return with ThreadPoolExecutor(max_workers=2) as executor: results = list(executor.map(run_ignored, range(2))) bound = step_bound(grid.n_to, 4) diff --git a/native/tests/test_backend.py b/native/tests/test_backend.py index 1c611a166..5ca13935e 100644 --- a/native/tests/test_backend.py +++ b/native/tests/test_backend.py @@ -784,21 +784,10 @@ def test_metadata_and_memory_ledger_are_self_consistent( ).endswith(("thread", "threads")) assert cast(float, metadata["planning_seconds"]) >= 0.0 # A one-mode envelope at one node. A host build folds its one-element transforms - # into the response column (solprop SEP 25), whatever the GEMM policy. A device - # build's forced vendor policy multiplies its real transform matrices into the - # samples' interleaved real view; its hybrid policy keeps the custom kernel under - # its shape guard (one mode). + # into the response column (solprop SEP 25). A device build's hybrid policy keeps + # the custom kernel under its shape guard (one mode). build = solprop_native.build_info() - if build.device_backend == "none": - expected_provider = "folded" - elif build.gemm_provider == "KOKKOSBLAS_TPL": - expected_provider = ( - str(metadata["synthesis_provider"]).removesuffix("-interleaved-real") - + "-interleaved-real" - ) - assert str(metadata["synthesis_provider"]).endswith("-interleaved-real") - else: - expected_provider = "custom" + expected_provider = "folded" if build.device_backend == "none" else "custom" assert metadata["synthesis_provider"] == expected_provider assert metadata["projection_provider"] == expected_provider assert "every whole transform on the host" in cast( @@ -1002,12 +991,9 @@ def test_packed_real_blas_radial_envelope_matches_the_reference_when_eligible() try: state = load_native(backend, problem.initial.field0, clear_fsal=False) info = build_info() - # The native rule: a HYBRID or KOKKOSBLAS_TPL build on a host execution - # space, at this shape; the dev build is HYBRID without TPL BLAS and packs too. - eligible = ( - info.gemm_provider in {"HYBRID", "KOKKOSBLAS_TPL"} - and info.device_backend == "none" - ) + # The native rule: a host execution space, at this shape; the dev build + # without TPL BLAS packs too. + eligible = info.device_backend == "none" provider = str(backend.metadata()["synthesis_provider"]) ledger = backend.memory_ledger() if eligible: diff --git a/native/tests/test_envelope_plasma.py b/native/tests/test_envelope_plasma.py index 80a9fb5c4..1e73c1a85 100644 --- a/native/tests/test_envelope_plasma.py +++ b/native/tests/test_envelope_plasma.py @@ -196,10 +196,7 @@ def test_envelope_current_streams_use_large_radial_projection() -> None: native_trial(actual, 0.0, 0.0) observed = np.asarray(state.stage_to_host(0)).reshape(problem.shape) info = build_info() - if info.device_backend == "none" and info.gemm_provider in { - "HYBRID", - "KOKKOSBLAS_TPL", - }: + if info.device_backend == "none": assert str(actual.metadata()["projection_provider"]).endswith( "-packed-real" ) diff --git a/native/tests/test_radial.py b/native/tests/test_radial.py index 3b6378b02..d4f8314ec 100644 --- a/native/tests/test_radial.py +++ b/native/tests/test_radial.py @@ -36,7 +36,7 @@ def test_radial_short_envelope_uses_device_blas( n_r: int, n_pol: int, chunk: int, eligible: bool ) -> None: info = build_info() - if info.device_backend == "none" or info.gemm_provider != "HYBRID": + if info.device_backend == "none": pytest.skip("default device HYBRID dispatch qualification") if not info.tpl_blas_enabled: pytest.skip("device vendor BLAS is disabled") @@ -123,7 +123,7 @@ def test_radial_partial_chunks_preserve_the_reference_source( # An analytic field synthesises the retained spectrum, on a host and on a # device; the others keep their time-domain products. info = build_info() - retained = family == "analytic" and info.gemm_provider != "CUSTOM" + retained = family == "analytic" assert info.device_backend in {"none", "cuda", "hip"} assert ( str(backend.metadata()["synthesis_provider"]).endswith( diff --git a/pyproject.toml b/pyproject.toml index b1b82c9e7..d0cacbc6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,11 +81,15 @@ dev = [ "drawsvg>=2.4", "ruff>=0.16.6", "psutil>=7.1.0", - # The native gate (native/scripts/check.sh). The C++ formatter is pinned so the check - # does not depend on the machine's LLVM (Ubuntu's 18 and Homebrew's 23 format - # native/src/general.cpp differently); the build tools let it build the editable - # package and the distribution wheel without isolation. + # The native gate (native/scripts/check.sh). The C++ formatter and clang-tidy are + # pinned so the check does not depend on the machine's LLVM (Ubuntu's 18 and + # Homebrew's 23 format native/src/general.cpp differently, and each LLVM release + # adds checks); clang-tidy's PyPI wheels start at 23.1.3, the nearest to the + # formatter's 23.1.0, and exist for Linux and macOS (the gate is bash) but not + # Windows on Arm, where a sync would build LLVM. The build tools let it build the + # editable package and the distribution wheel without isolation. "clang-format==23.1.0", + "clang-tidy==23.1.4; sys_platform == 'linux' or sys_platform == 'darwin'", "nanobind>=2.9", "scikit-build-core>=0.11", "setuptools-scm>=9", @@ -459,6 +463,19 @@ help = "The definition-of-done gate, in order." cmd = "bash native/scripts/check.sh" help = "The native package's gate (C++ and its Python adapter)." +# The pinned clang-format over every tracked C++ file under native/ (the lint stage of +# check-native checks the same list). +[tool.poe.tasks.fmt-native] +shell = "git ls-files -z -- 'native/*.cpp' 'native/*.hpp' 'native/*.h' | xargs -0 clang-format -i" +interpreter = "bash" +help = "Format every tracked native C++ file with the pinned clang-format." + +# The pinned clang-tidy over the given native C++ files, or those that differ from main, +# with the cpu-tidy preset's compile commands (native/scripts/tidy.sh). +[tool.poe.tasks.tidy-native] +cmd = "bash native/scripts/tidy.sh" +help = "clang-tidy on the given native C++ files, or on those changed from main." + [tool.poe.tasks.fetch-wheel] cmd = "python native/scripts/fetch_wheel.py" help = "Install the native wheel CI built for this checkout; needs `gh`, and no compiler." diff --git a/tests/backends/test_spatial_execution.py b/tests/backends/test_spatial_execution.py index 938757879..1874c7bb3 100644 --- a/tests/backends/test_spatial_execution.py +++ b/tests/backends/test_spatial_execution.py @@ -76,11 +76,10 @@ def test_radial_real_blas_preserves_transform_and_memory_contract( try: metadata, ledger = backend.metadata(), backend.memory_ledger() if backend.name == "native": - # Portable/custom and device builds keep their existing provider. + # Device builds keep their existing provider. packed = str(metadata["synthesis_provider"]).endswith("-packed-real") eligible = ( metadata["device_backend"] == "none" - and metadata["gemm_provider"] in {"HYBRID", "KOKKOSBLAS_TPL"} and n_pol == 1 and chunk == n_r and n_r >= 256 diff --git a/uv.lock b/uv.lock index 28b7d8a1a..aa7f09e75 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.14" [manifest] @@ -228,6 +228,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/e0/d44445801720bb21d9d8fb6a2c16f78065dc1f0d7c9379c5214dd07a2db8/clang_format-23.1.0-py2.py3-none-win_arm64.whl", hash = "sha256:6b565baa8d478eaac44a8a68802cce1e6e998e9c80680d222862761d1afe62e1", size = 1410985, upload-time = "2026-08-26T19:54:09.659Z" }, ] +[[package]] +name = "clang-tidy" +version = "23.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/f1/ce2407dc5ca76d5ada9449fa4203ac563f4f563224f29fa06bcd88a3959a/clang_tidy-23.1.4.tar.gz", hash = "sha256:71ec19e17c485cdb1d80f9c631f525b0b0e5f8a0429f164df75b3869ae61ab44", size = 12404, upload-time = "2026-10-26T08:37:15.954Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/18/581961a934c9df0861fa91dcf2e6c553e1d4e194de10fd80c1c633b731be/clang_tidy-23.1.4-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:16be65b3396d096826cda8b94e7734913b4e2ffe0053fac6b8ccb2efc8d68cb1", size = 33090266, upload-time = "2026-10-26T08:36:39.686Z" }, + { url = "https://files.pythonhosted.org/packages/0c/59/637ce45a6cfa15ddb926e1feeee8077b9ebeb2956548375f24e16d51a908/clang_tidy-23.1.4-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:08e977575e9f800c1c806f2001a3cdfd01323491aec05e86dd24be9a5c2585f4", size = 32338052, upload-time = "2026-10-26T08:36:43.098Z" }, + { url = "https://files.pythonhosted.org/packages/7d/95/ba9463f94319dddd0c3a9d3f65d432ddf71fe9b203dcd7ea63bcff8999e2/clang_tidy-23.1.4-py2.py3-none-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87596cd77dedfdea23ced1d13e57e869dcaee99745bfe942796196da3c197671", size = 43724944, upload-time = "2026-10-26T08:36:46.323Z" }, + { url = "https://files.pythonhosted.org/packages/80/85/9b483f300124a9245ec6da8975612f1841e26da13b27ad8d17f2006e13bd/clang_tidy-23.1.4-py2.py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bb31e4f319ecbe14acb75f23c3050d0f1d4b1d92ea1c66091bc76e4eac2c1ac", size = 45608750, upload-time = "2026-10-26T08:36:49.483Z" }, + { url = "https://files.pythonhosted.org/packages/af/32/4856b0bb395031bb1b8ef6866747ced3428006f05e3b26db0a4eadb57405/clang_tidy-23.1.4-py2.py3-none-manylinux_2_28_i686.whl", hash = "sha256:84b9021905a851948c7f0735baea4d00a7fffd87437bdd106a339e76f6cd7b13", size = 51180158, upload-time = "2026-10-26T08:36:53.432Z" }, + { url = "https://files.pythonhosted.org/packages/3c/c0/28da9c922d84e968e67fb826c4c3a6e3a631ef667474df858c5ca921a420/clang_tidy-23.1.4-py2.py3-none-manylinux_2_31_armv7l.whl", hash = "sha256:8d20ac0cf4dbed1526493e02316796bffc075e05e393519e953261909ba2c8e6", size = 41813066, upload-time = "2026-10-26T08:36:56.54Z" }, + { url = "https://files.pythonhosted.org/packages/32/46/c8487562afaabba832be98bc4eeb39a13b5f5f5a270b4e2547043ada4ea5/clang_tidy-23.1.4-py2.py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:900276dcb577ed8cb411958efc64feb1ba7334e3c9e70ea5d2e6b5c29aa5335f", size = 42757090, upload-time = "2026-10-26T08:37:00.151Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2b/1103bf0e0ae41f3682a4825ce6c7b6290852c4677b2aa4e632b2c43ac60c/clang_tidy-23.1.4-py2.py3-none-musllinux_1_2_i686.whl", hash = "sha256:f428092411271394559c70f7bed218e20409bbd2ee16d0c4cc93f7c6361a595a", size = 53434463, upload-time = "2026-10-26T08:37:03.631Z" }, + { url = "https://files.pythonhosted.org/packages/0b/89/f8a6c51ce1caf68ad4d9b1abf2f8b3b81490f8c9d959c6b3fee1c490d64b/clang_tidy-23.1.4-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7c34a8ec573e747412a15a4fea3cd65221b03b2da9a6462f17cbc8bfd47dc7fb", size = 48308787, upload-time = "2026-10-26T08:37:07.276Z" }, +] + [[package]] name = "click" version = "8.5.0" @@ -1581,6 +1598,7 @@ dev = [ { name = "basedpyright" }, { name = "beartype" }, { name = "clang-format" }, + { name = "clang-tidy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "codespell" }, { name = "drawsvg" }, { name = "hypothesis" }, @@ -1631,6 +1649,7 @@ dev = [ { name = "basedpyright", specifier = "==1.39.10" }, { name = "beartype" }, { name = "clang-format", specifier = "==23.1.0" }, + { name = "clang-tidy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'", specifier = "==23.1.4" }, { name = "codespell" }, { name = "drawsvg", specifier = ">=2.4" }, { name = "hypothesis" },